diff options
| author | Luke "Jared" Bennett <lbennett@gitlab.com> | 2017-04-04 14:56:27 +0100 |
|---|---|---|
| committer | Luke "Jared" Bennett <lbennett@gitlab.com> | 2017-04-04 14:56:27 +0100 |
| commit | ccca73d779ae0e22a86c9586fd52b15a8600b9f3 (patch) | |
| tree | 7385d9a2019bfa98affe9bc83c0afd7da2fd730d /lib | |
| parent | c252c03401881fd7dbf7fab984285c402eb31d5f (diff) | |
| parent | 5f7ebfb9aa023849b8b296bf9cf0f472b886d072 (diff) | |
| download | gitlab-ce-ccca73d779ae0e22a86c9586fd52b15a8600b9f3.tar.gz | |
Merge branch 'master' into add-sentry-js-again-with-vue
Diffstat (limited to 'lib')
395 files changed, 11142 insertions, 2973 deletions
diff --git a/lib/additional_email_headers_interceptor.rb b/lib/additional_email_headers_interceptor.rb new file mode 100644 index 00000000000..2358fa6bbfd --- /dev/null +++ b/lib/additional_email_headers_interceptor.rb @@ -0,0 +1,8 @@ +class AdditionalEmailHeadersInterceptor + def self.delivering_email(message) + message.headers( + 'Auto-Submitted' => 'auto-generated', + 'X-Auto-Response-Suppress' => 'All' + ) + end +end diff --git a/lib/api/access_requests.rb b/lib/api/access_requests.rb index 789f45489eb..a5c9f0b509c 100644 --- a/lib/api/access_requests.rb +++ b/lib/api/access_requests.rb @@ -10,7 +10,7 @@ module API params do requires :id, type: String, desc: "The #{source_type} ID" end - resource source_type.pluralize do + resource source_type.pluralize, requirements: { id: %r{[^/]+} } do desc "Gets a list of access requests for a #{source_type}." do detail 'This feature was introduced in GitLab 8.11.' success Entities::AccessRequester diff --git a/lib/api/api.rb b/lib/api/api.rb index 6cf6b501021..1bf20f76ad6 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -1,7 +1,47 @@ module API class API < Grape::API include APIGuard - version 'v3', using: :path + + version %w(v3 v4), using: :path + + version 'v3', using: :path do + helpers ::API::V3::Helpers + + mount ::API::V3::AwardEmoji + mount ::API::V3::Boards + mount ::API::V3::Branches + mount ::API::V3::BroadcastMessages + mount ::API::V3::Builds + mount ::API::V3::Commits + mount ::API::V3::DeployKeys + mount ::API::V3::Environments + mount ::API::V3::Files + mount ::API::V3::Groups + mount ::API::V3::Issues + mount ::API::V3::Labels + mount ::API::V3::Members + mount ::API::V3::MergeRequestDiffs + mount ::API::V3::MergeRequests + mount ::API::V3::Notes + mount ::API::V3::Pipelines + mount ::API::V3::ProjectHooks + mount ::API::V3::Milestones + mount ::API::V3::Projects + mount ::API::V3::ProjectSnippets + mount ::API::V3::Repositories + mount ::API::V3::Runners + mount ::API::V3::Services + mount ::API::V3::Settings + mount ::API::V3::Snippets + mount ::API::V3::Subscriptions + mount ::API::V3::SystemHooks + mount ::API::V3::Tags + mount ::API::V3::Templates + mount ::API::V3::Todos + mount ::API::V3::Triggers + mount ::API::V3::Users + mount ::API::V3::Variables + end before { allow_access_with_scope :api } @@ -23,6 +63,10 @@ module API error! e.message, e.status, e.headers end + rescue_from Gitlab::Auth::TooManyIps do |e| + rack_response({ 'message' => '403 Forbidden' }.to_json, 403) + end + rescue_from :all do |exception| handle_api_exception(exception) end @@ -40,7 +84,6 @@ module API mount ::API::Boards mount ::API::Branches mount ::API::BroadcastMessages - mount ::API::Builds mount ::API::Commits mount ::API::CommitStatuses mount ::API::DeployKeys @@ -50,6 +93,7 @@ module API mount ::API::Groups mount ::API::Internal mount ::API::Issues + mount ::API::Jobs mount ::API::Keys mount ::API::Labels mount ::API::Lint @@ -65,6 +109,7 @@ module API mount ::API::Projects mount ::API::ProjectSnippets mount ::API::Repositories + mount ::API::Runner mount ::API::Runners mount ::API::Services mount ::API::Session diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index df6db140d0e..9fcf04efa38 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -6,7 +6,7 @@ module API module APIGuard extend ActiveSupport::Concern - PRIVATE_TOKEN_HEADER = "HTTP_PRIVATE_TOKEN" + PRIVATE_TOKEN_HEADER = "HTTP_PRIVATE_TOKEN".freeze PRIVATE_TOKEN_PARAM = :private_token included do |base| @@ -114,14 +114,14 @@ module API private def install_error_responders(base) - error_classes = [ MissingTokenError, TokenNotFoundError, - ExpiredError, RevokedError, InsufficientScopeError] + error_classes = [MissingTokenError, TokenNotFoundError, + ExpiredError, RevokedError, InsufficientScopeError] base.send :rescue_from, *error_classes, oauth2_bearer_token_error_handler end def oauth2_bearer_token_error_handler - Proc.new do |e| + proc do |e| response = case e when MissingTokenError @@ -160,13 +160,10 @@ module API # Exceptions # - class MissingTokenError < StandardError; end - - class TokenNotFoundError < StandardError; end - - class ExpiredError < StandardError; end - - class RevokedError < StandardError; end + MissingTokenError = Class.new(StandardError) + TokenNotFoundError = Class.new(StandardError) + ExpiredError = Class.new(StandardError) + RevokedError = Class.new(StandardError) class InsufficientScopeError < StandardError attr_reader :scopes diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb index 58a4df54bea..56f19f89642 100644 --- a/lib/api/award_emoji.rb +++ b/lib/api/award_emoji.rb @@ -3,19 +3,26 @@ module API include PaginationParams before { authenticate! } - AWARDABLES = %w[issue merge_request snippet] - - resource :projects do - AWARDABLES.each do |awardable_type| - awardable_string = awardable_type.pluralize - awardable_id_string = "#{awardable_type}_id" + AWARDABLES = [ + { type: 'issue', find_by: :iid }, + { type: 'merge_request', find_by: :iid }, + { type: 'snippet', find_by: :id } + ].freeze + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: { id: %r{[^/]+} } do + AWARDABLES.each do |awardable_params| + awardable_string = awardable_params[:type].pluralize + awardable_id_string = "#{awardable_params[:type]}_#{awardable_params[:find_by]}" params do - requires :id, type: String, desc: 'The ID of a project' requires :"#{awardable_id_string}", type: Integer, desc: "The ID of an Issue, Merge Request or Snippet" end - [ ":id/#{awardable_string}/:#{awardable_id_string}/award_emoji", + [ + ":id/#{awardable_string}/:#{awardable_id_string}/award_emoji", ":id/#{awardable_string}/:#{awardable_id_string}/notes/:note_id/award_emoji" ].each do |endpoint| @@ -28,8 +35,8 @@ module API end get endpoint do if can_read_awardable? - awards = paginate(awardable.award_emoji) - present awards, with: Entities::AwardEmoji + awards = awardable.award_emoji + present paginate(awards), with: Entities::AwardEmoji else not_found!("Award Emoji") end @@ -82,7 +89,6 @@ module API unauthorized! unless award.user == current_user || current_user.admin? award.destroy - present award, with: Entities::AwardEmoji end end end @@ -104,10 +110,10 @@ module API note_id = params.delete(:note_id) awardable.notes.find(note_id) - elsif params.include?(:issue_id) - user_project.issues.find(params[:issue_id]) - elsif params.include?(:merge_request_id) - user_project.merge_requests.find(params[:merge_request_id]) + elsif params.include?(:issue_iid) + user_project.issues.find_by!(iid: params[:issue_iid]) + elsif params.include?(:merge_request_iid) + user_project.merge_requests.find_by!(iid: params[:merge_request_iid]) else user_project.snippets.find(params[:snippet_id]) end diff --git a/lib/api/boards.rb b/lib/api/boards.rb index 4ac491edc1b..5a2d7a681e3 100644 --- a/lib/api/boards.rb +++ b/lib/api/boards.rb @@ -1,19 +1,23 @@ module API - # Boards API class Boards < Grape::API + include PaginationParams + before { authenticate! } params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects do + resource :projects, requirements: { id: %r{[^/]+} } do desc 'Get all project boards' do detail 'This feature was introduced in 8.13' success Entities::Board end + params do + use :pagination + end get ':id/boards' do authorize!(:read_board, user_project) - present user_project.boards, with: Entities::Board + present paginate(user_project.boards), with: Entities::Board end params do @@ -37,12 +41,15 @@ module API end desc 'Get the lists of a project board' do - detail 'Does not include `backlog` and `done` lists. This feature was introduced in 8.13' + detail 'Does not include `done` list. This feature was introduced in 8.13' success Entities::List end + params do + use :pagination + end get '/lists' do authorize!(:read_board, user_project) - present board_lists, with: Entities::List + present paginate(board_lists), with: Entities::List end desc 'Get a list of a project board' do @@ -120,9 +127,7 @@ module API service = ::Boards::Lists::DestroyService.new(user_project, current_user) - if service.execute(list) - present list, with: Entities::List - else + unless service.execute(list) render_api_error!({ error: 'List could not be deleted!' }, 400) end end diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 0950c3d2e88..f35084a582a 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -1,22 +1,25 @@ require 'mime/types' module API - # Projects API class Branches < Grape::API - before { authenticate! } + include PaginationParams + before { authorize! :download_code, user_project } params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects do + resource :projects, requirements: { id: %r{[^/]+} } do desc 'Get a project repository branches' do success Entities::RepoBranch end + params do + use :pagination + end get ":id/repository/branches" do - branches = user_project.repository.branches.sort_by(&:name) + branches = ::Kaminari.paginate_array(user_project.repository.branches.sort_by(&:name)) - present branches, with: Entities::RepoBranch, project: user_project + present paginate(branches), with: Entities::RepoBranch, project: user_project end desc 'Get a single branch' do @@ -84,7 +87,7 @@ module API branch = user_project.repository.find_branch(params[:branch]) not_found!("Branch") unless branch protected_branch = user_project.protected_branches.find_by(name: branch.name) - protected_branch.destroy if protected_branch + protected_branch&.destroy present branch, with: Entities::RepoBranch, project: user_project end @@ -93,13 +96,14 @@ module API success Entities::RepoBranch end params do - requires :branch_name, type: String, desc: 'The name of the branch' + requires :branch, type: String, desc: 'The name of the branch' requires :ref, type: String, desc: 'Create branch from commit sha or existing branch' end post ":id/repository/branches" do authorize_push_project + result = CreateBranchService.new(user_project, current_user). - execute(params[:branch_name], params[:ref]) + execute(params[:branch], params[:ref]) if result[:status] == :success present result[:branch], @@ -120,25 +124,16 @@ module API result = DeleteBranchService.new(user_project, current_user). execute(params[:branch]) - if result[:status] == :success - { - branch_name: params[:branch] - } - else + if result[:status] != :success render_api_error!(result[:message], result[:return_code]) end end - # Delete all merged branches - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # DELETE /projects/:id/repository/branches/delete_merged + desc 'Delete all merged branches' delete ":id/repository/merged_branches" do DeleteMergedBranchesService.new(user_project, current_user).async_execute - status(200) + accepted! end end end diff --git a/lib/api/broadcast_messages.rb b/lib/api/broadcast_messages.rb index 1217002bf8e..395c401203c 100644 --- a/lib/api/broadcast_messages.rb +++ b/lib/api/broadcast_messages.rb @@ -91,7 +91,7 @@ module API delete ':id' do message = find_message - present message.destroy, with: Entities::BroadcastMessage + message.destroy end end end diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index b6e6820c3f4..827a38d33da 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -2,7 +2,10 @@ require 'mime/types' module API class CommitStatuses < Grape::API - resource :projects do + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: { id: %r{[^/]+} } do include PaginationParams before { authenticate! } @@ -11,7 +14,6 @@ module API success Entities::CommitStatus end params do - requires :id, type: String, desc: 'The ID of a project' requires :sha, type: String, desc: 'The commit hash' optional :ref, type: String, desc: 'The ref' optional :stage, type: String, desc: 'The stage' @@ -37,15 +39,15 @@ module API success Entities::CommitStatus end params do - requires :id, type: String, desc: 'The ID of a project' requires :sha, type: String, desc: 'The commit hash' requires :state, type: String, desc: 'The state of the status', - values: ['pending', 'running', 'success', 'failed', 'canceled'] + values: %w(pending running success failed canceled) optional :ref, type: String, desc: 'The ref' optional :target_url, type: String, desc: 'The target URL to associate with this status' optional :description, type: String, desc: 'A short description of the status' optional :name, type: String, desc: 'A string label to differentiate this status from the status of other systems. Default: "default"' optional :context, type: String, desc: 'A string label to differentiate this status from the status of other systems. Default: "default"' + optional :coverage, type: Float, desc: 'The total code coverage' end post ':id/statuses/:sha' do authorize! :create_commit_status, user_project @@ -71,13 +73,15 @@ module API status = GenericCommitStatus.running_or_pending.find_or_initialize_by( project: @project, pipeline: pipeline, - user: current_user, name: name, ref: ref, - target_url: params[:target_url], - description: params[:description] + user: current_user ) + optional_attributes = + attributes_for_keys(%w[target_url description coverage]) + + status.update(optional_attributes) if optional_attributes.any? render_validation_error!(status) if status.invalid? begin diff --git a/lib/api/commits.rb b/lib/api/commits.rb index e6d707f3c3d..66b37fd2bcc 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -10,33 +10,42 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects do + resource :projects, requirements: { id: %r{[^/]+} } do desc 'Get a project repository commits' do success Entities::RepoCommit end params do optional :ref_name, type: String, desc: 'The name of a repository branch or tag, if not given the default branch is used' - optional :since, type: String, desc: 'Only commits after or in this date will be returned' - optional :until, type: String, desc: 'Only commits before or in this date will be returned' - optional :page, type: Integer, default: 0, desc: 'The page for pagination' - optional :per_page, type: Integer, default: 20, desc: 'The number of results per page' + optional :since, type: DateTime, desc: 'Only commits after or on this date will be returned' + optional :until, type: DateTime, desc: 'Only commits before or on this date will be returned' optional :path, type: String, desc: 'The file path' + use :pagination end get ":id/repository/commits" do - # TODO remove the next line for 9.0, use DateTime type in the params block - datetime_attributes! :since, :until - - ref = params[:ref_name] || user_project.try(:default_branch) || 'master' - offset = params[:page] * params[:per_page] + path = params[:path] + before = params[:until] + after = params[:since] + ref = params[:ref_name] || user_project.try(:default_branch) || 'master' + offset = (params[:page] - 1) * params[:per_page] commits = user_project.repository.commits(ref, - path: params[:path], + path: path, limit: params[:per_page], offset: offset, - after: params[:since], - before: params[:until]) + before: before, + after: after) + + commit_count = + if path || before || after + user_project.repository.count_commits(ref: ref, path: path, before: before, after: after) + else + # Cacheable commit count. + user_project.repository.commit_count_for_ref(ref) + end - present commits, with: Entities::RepoCommit + paginated_commits = Kaminari.paginate_array(commits, total_count: commit_count) + + present paginate(paginated_commits), with: Entities::RepoCommit end desc 'Commit multiple file changes as one commit' do @@ -44,7 +53,7 @@ module API detail 'This feature was introduced in GitLab 8.13' end params do - requires :branch_name, type: String, desc: 'The name of branch' + requires :branch, type: String, desc: 'The name of branch' requires :commit_message, type: String, desc: 'Commit message' requires :actions, type: Array[Hash], desc: 'Actions to perform in commit' optional :author_email, type: String, desc: 'Author email for commit' @@ -53,15 +62,7 @@ module API post ":id/repository/commits" do authorize! :push_code, user_project - attrs = declared_params - attrs[:source_branch] = attrs[:branch_name] - attrs[:target_branch] = attrs[:branch_name] - attrs[:actions].map! do |action| - action[:action] = action[:action].to_sym - action[:file_path].slice!(0) if action[:file_path] && action[:file_path].start_with?('/') - action[:previous_path].slice!(0) if action[:previous_path] && action[:previous_path].start_with?('/') - action - end + attrs = declared_params.merge(start_branch: declared_params[:branch], target_branch: declared_params[:branch]) result = ::Files::MultiService.new(user_project, current_user, attrs).execute @@ -114,7 +115,7 @@ module API commit = user_project.commit(params[:sha]) not_found! 'Commit' unless commit - notes = Note.where(commit_id: commit.id).order(:created_at) + notes = user_project.notes.where(commit_id: commit.id).order(:created_at) present paginate(notes), with: Entities::CommitNote end @@ -138,9 +139,7 @@ module API commit_params = { commit: commit, - create_merge_request: false, - source_project: user_project, - source_branch: commit.cherry_pick_branch_name, + start_branch: params[:branch], target_branch: params[:branch] } @@ -163,7 +162,7 @@ module API optional :path, type: String, desc: 'The file path' given :path do requires :line, type: Integer, desc: 'The line number' - requires :line_type, type: String, values: ['new', 'old'], default: 'new', desc: 'The type of the line' + requires :line_type, type: String, values: %w(new old), default: 'new', desc: 'The type of the line' end end post ':id/repository/commits/:sha/comments' do diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb index 85360730841..b888ede6fe8 100644 --- a/lib/api/deploy_keys.rb +++ b/lib/api/deploy_keys.rb @@ -1,120 +1,109 @@ module API - # Projects API class DeployKeys < Grape::API + include PaginationParams + before { authenticate! } + desc 'Return all deploy keys' + params do + use :pagination + end get "deploy_keys" do authenticated_as_admin! - keys = DeployKey.all - present keys, with: Entities::SSHKey + present paginate(DeployKey.all), with: Entities::SSHKey end params do requires :id, type: String, desc: 'The ID of the project' end - resource :projects do + resource :projects, requirements: { id: %r{[^/]+} } do before { authorize_admin_project } - # Routing "projects/:id/keys/..." is DEPRECATED and WILL BE REMOVED in version 9.0 - # Use "projects/:id/deploy_keys/..." instead. - # - %w(keys deploy_keys).each do |path| - desc "Get a specific project's deploy keys" do - success Entities::SSHKey - end - get ":id/#{path}" do - present user_project.deploy_keys, with: Entities::SSHKey - end - - desc 'Get single deploy key' do - success Entities::SSHKey - end - params do - requires :key_id, type: Integer, desc: 'The ID of the deploy key' - end - get ":id/#{path}/:key_id" do - key = user_project.deploy_keys.find params[:key_id] - present key, with: Entities::SSHKey - end - - # TODO: for 9.0 we should check if params are there with the params block - # grape provides, at this point we'd change behaviour so we can't - # Behaviour now if you don't provide all required params: it renders a - # validation error or two. - desc 'Add new deploy key to currently authenticated user' do - success Entities::SSHKey - end - post ":id/#{path}" do - attrs = attributes_for_keys [:title, :key] - attrs[:key].strip! if attrs[:key] + desc "Get a specific project's deploy keys" do + success Entities::SSHKey + end + params do + use :pagination + end + get ":id/deploy_keys" do + present paginate(user_project.deploy_keys), with: Entities::SSHKey + end - # Check for an existing key joined to this project - key = user_project.deploy_keys.find_by(key: attrs[:key]) - if key - present key, with: Entities::SSHKey - break - end + desc 'Get single deploy key' do + success Entities::SSHKey + end + params do + requires :key_id, type: Integer, desc: 'The ID of the deploy key' + end + get ":id/deploy_keys/:key_id" do + key = user_project.deploy_keys.find params[:key_id] + present key, with: Entities::SSHKey + end - # Check for available deploy keys in other projects - key = current_user.accessible_deploy_keys.find_by(key: attrs[:key]) - if key - user_project.deploy_keys << key - present key, with: Entities::SSHKey - break - end + desc 'Add new deploy key to currently authenticated user' do + success Entities::SSHKey + end + params do + requires :key, type: String, desc: 'The new deploy key' + requires :title, type: String, desc: 'The name of the deploy key' + end + post ":id/deploy_keys" do + params[:key].strip! - # Create a new deploy key - key = DeployKey.new attrs - if key.valid? && user_project.deploy_keys << key - present key, with: Entities::SSHKey - else - render_validation_error!(key) - end + # Check for an existing key joined to this project + key = user_project.deploy_keys.find_by(key: params[:key]) + if key + present key, with: Entities::SSHKey + break end - desc 'Enable a deploy key for a project' do - detail 'This feature was added in GitLab 8.11' - success Entities::SSHKey - end - params do - requires :key_id, type: Integer, desc: 'The ID of the deploy key' + # Check for available deploy keys in other projects + key = current_user.accessible_deploy_keys.find_by(key: params[:key]) + if key + user_project.deploy_keys << key + present key, with: Entities::SSHKey + break end - post ":id/#{path}/:key_id/enable" do - key = ::Projects::EnableDeployKeyService.new(user_project, - current_user, declared_params).execute - if key - present key, with: Entities::SSHKey - else - not_found!('Deploy Key') - end + # Create a new deploy key + key = DeployKey.new(declared_params(include_missing: false)) + if key.valid? && user_project.deploy_keys << key + present key, with: Entities::SSHKey + else + render_validation_error!(key) end + end - desc 'Disable a deploy key for a project' do - detail 'This feature was added in GitLab 8.11' - success Entities::SSHKey - end - params do - requires :key_id, type: Integer, desc: 'The ID of the deploy key' - end - delete ":id/#{path}/:key_id/disable" do - key = user_project.deploy_keys_projects.find_by(deploy_key_id: params[:key_id]) - key.destroy + desc 'Enable a deploy key for a project' do + detail 'This feature was added in GitLab 8.11' + success Entities::SSHKey + end + params do + requires :key_id, type: Integer, desc: 'The ID of the deploy key' + end + post ":id/deploy_keys/:key_id/enable" do + key = ::Projects::EnableDeployKeyService.new(user_project, + current_user, declared_params).execute - present key.deploy_key, with: Entities::SSHKey + if key + present key, with: Entities::SSHKey + else + not_found!('Deploy Key') end + end - desc 'Delete existing deploy key of currently authenticated user' do - success Key - end - params do - requires :key_id, type: Integer, desc: 'The ID of the deploy key' - end - delete ":id/#{path}/:key_id" do - key = user_project.deploy_keys.find(params[:key_id]) - key.destroy - end + desc 'Delete deploy key for a project' do + success Key + end + params do + requires :key_id, type: Integer, desc: 'The ID of the deploy key' + end + delete ":id/deploy_keys/:key_id" do + key = user_project.deploy_keys_projects.find_by(deploy_key_id: params[:key_id]) + not_found!('Deploy Key') unless key + + key.destroy end end end diff --git a/lib/api/deployments.rb b/lib/api/deployments.rb index c5feb49b22f..46b936897f6 100644 --- a/lib/api/deployments.rb +++ b/lib/api/deployments.rb @@ -1,5 +1,5 @@ module API - # Deployments RESTfull API endpoints + # Deployments RESTful API endpoints class Deployments < Grape::API include PaginationParams @@ -8,7 +8,7 @@ module API params do requires :id, type: String, desc: 'The project ID' end - resource :projects do + resource :projects, requirements: { id: %r{[^/]+} } do desc 'Get all deployments of the project' do detail 'This feature was introduced in GitLab 8.11.' success Entities::Deployment diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 9f59939e9ae..00d44821e3f 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -26,7 +26,7 @@ module API expose :last_sign_in_at expose :confirmed_at expose :email - expose :theme_id, :color_scheme_id, :projects_limit, :current_sign_in_at + expose :color_scheme_id, :projects_limit, :current_sign_in_at expose :identities, using: Entities::Identity expose :can_create_group?, as: :can_create_group expose :can_create_project?, as: :can_create_project @@ -49,7 +49,8 @@ module API class ProjectHook < Hook expose :project_id, :issues_events, :merge_requests_events - expose :note_events, :build_events, :pipeline_events, :wiki_page_events + expose :note_events, :pipeline_events, :wiki_page_events + expose :build_events, as: :job_events end class BasicProjectDetails < Grape::Entity @@ -69,9 +70,8 @@ module API class Project < Grape::Entity expose :id, :description, :default_branch, :tag_list - expose :public?, as: :public expose :archived?, as: :archived - expose :visibility_level, :ssh_url_to_repo, :http_url_to_repo, :web_url + expose :visibility, :ssh_url_to_repo, :http_url_to_repo, :web_url expose :owner, using: Entities::UserBasic, unless: ->(project, options) { project.group } expose :name, :name_with_namespace expose :path, :path_with_namespace @@ -81,7 +81,7 @@ module API expose(:issues_enabled) { |project, options| project.feature_available?(:issues, options[:current_user]) } expose(:merge_requests_enabled) { |project, options| project.feature_available?(:merge_requests, options[:current_user]) } expose(:wiki_enabled) { |project, options| project.feature_available?(:wiki, options[:current_user]) } - expose(:builds_enabled) { |project, options| project.feature_available?(:builds, options[:current_user]) } + expose(:jobs_enabled) { |project, options| project.feature_available?(:builds, options[:current_user]) } expose(:snippets_enabled) { |project, options| project.feature_available?(:snippets, options[:current_user]) } expose :created_at, :last_activity_at @@ -94,11 +94,11 @@ module API expose :star_count, :forks_count expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:current_user]) && project.default_issues_tracker? } expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] } - expose :public_builds + expose :public_builds, as: :public_jobs expose :shared_with_groups do |project, options| SharedGroup.represent(project.project_group_links.all, options) end - expose :only_allow_merge_if_build_succeeds + expose :only_allow_merge_if_pipeline_succeeds expose :request_access_enabled expose :only_allow_merge_if_all_discussions_are_resolved @@ -110,7 +110,7 @@ module API expose :storage_size expose :repository_size expose :lfs_objects_size - expose :build_artifacts_size + expose :build_artifacts_size, as: :job_artifacts_size end class Member < UserBasic @@ -132,18 +132,20 @@ module API end class Group < Grape::Entity - expose :id, :name, :path, :description, :visibility_level + expose :id, :name, :path, :description, :visibility expose :lfs_enabled?, as: :lfs_enabled expose :avatar_url expose :web_url expose :request_access_enabled + expose :full_name, :full_path + expose :parent_id expose :statistics, if: :statistics do with_options format_with: -> (value) { value.to_i } do expose :storage_size expose :repository_size expose :lfs_objects_size - expose :build_artifacts_size + expose :build_artifacts_size, as: :job_artifacts_size end end end @@ -153,10 +155,27 @@ module API expose :shared_projects, using: Entities::Project end + class RepoCommit < Grape::Entity + expose :id, :short_id, :title, :created_at + expose :parent_ids + expose :safe_message, as: :message + expose :author_name, :author_email, :authored_date + expose :committer_name, :committer_email, :committed_date + end + + class RepoCommitStats < Grape::Entity + expose :additions, :deletions, :total + end + + class RepoCommitDetail < RepoCommit + expose :stats, using: Entities::RepoCommitStats + expose :status + end + class RepoBranch < Grape::Entity expose :name - expose :commit do |repo_branch, options| + expose :commit, using: Entities::RepoCommit do |repo_branch, options| options[:project].repository.commit(repo_branch.dereferenced_target) end @@ -185,36 +204,17 @@ module API expose :id, :name, :type, :path expose :mode do |obj, options| - filemode = obj.mode.to_s(8) + filemode = obj.mode filemode = "0" + filemode if filemode.length < 6 filemode end end - class RepoCommit < Grape::Entity - expose :id, :short_id, :title, :author_name, :author_email, :created_at - expose :committer_name, :committer_email - expose :safe_message, as: :message - end - - class RepoCommitStats < Grape::Entity - expose :additions, :deletions, :total - end - - class RepoCommitDetail < RepoCommit - expose :parent_ids, :committed_date, :authored_date - expose :stats, using: Entities::RepoCommitStats - expose :status - end - class ProjectSnippet < Grape::Entity expose :id, :title, :file_name expose :author, using: Entities::UserBasic expose :updated_at, :created_at - # TODO (rspeicher): Deprecated; remove in 9.0 - expose(:expires_at) { |snippet| nil } - expose :web_url do |snippet, options| Gitlab::UrlBuilder.build(snippet) end @@ -250,14 +250,11 @@ module API expose :start_date end - class Issue < ProjectEntity + class IssueBasic < ProjectEntity expose :label_names, as: :labels expose :milestone, using: Entities::Milestone expose :assignee, :author, using: Entities::UserBasic - expose :subscribed do |issue, options| - issue.subscribed?(options[:current_user], options[:project] || issue.project) - end expose :user_notes_count expose :upvotes, :downvotes expose :due_date @@ -268,6 +265,12 @@ module API end end + class Issue < IssueBasic + expose :subscribed do |issue, options| + issue.subscribed?(options[:current_user], options[:project] || issue.project) + end + end + class IssuableTimeStats < Grape::Entity expose :time_estimate expose :total_time_spent @@ -280,7 +283,7 @@ module API expose :id end - class MergeRequest < ProjectEntity + class MergeRequestBasic < ProjectEntity expose :target_branch, :source_branch expose :upvotes, :downvotes expose :author, :assignee, using: Entities::UserBasic @@ -288,13 +291,10 @@ module API expose :label_names, as: :labels expose :work_in_progress?, as: :work_in_progress expose :milestone, using: Entities::Milestone - expose :merge_when_build_succeeds + expose :merge_when_pipeline_succeeds expose :merge_status expose :diff_head_sha, as: :sha expose :merge_commit_sha - expose :subscribed do |merge_request, options| - merge_request.subscribed?(options[:current_user], options[:project]) - end expose :user_notes_count expose :should_remove_source_branch?, as: :should_remove_source_branch expose :force_remove_source_branch?, as: :force_remove_source_branch @@ -304,6 +304,12 @@ module API end end + class MergeRequest < MergeRequestBasic + expose :subscribed do |merge_request, options| + merge_request.subscribed?(options[:current_user], options[:project]) + end + end + class MergeRequestChanges < MergeRequest expose :diffs, as: :changes, using: Entities::RepoDiff do |compare, _| compare.raw_diffs(all_diffs: true).to_a @@ -339,9 +345,6 @@ module API expose :created_at, :updated_at expose :system?, as: :system expose :noteable_id, :noteable_type - # upvote? and downvote? are deprecated, always return false - expose(:upvote?) { |note| false } - expose(:downvote?) { |note| false } end class AwardEmoji < Grape::Entity @@ -368,7 +371,7 @@ module API class CommitStatus < Grape::Entity expose :id, :sha, :ref, :status, :name, :target_url, :description, - :created_at, :started_at, :finished_at, :allow_failure + :created_at, :started_at, :finished_at, :allow_failure, :coverage expose :author, using: Entities::UserBasic end @@ -381,9 +384,7 @@ module API expose :author, using: Entities::UserBasic, if: ->(event, options) { event.author } expose :author_username do |event, options| - if event.author - event.author.username - end + event.author&.username end end @@ -399,7 +400,8 @@ module API expose :target_type expose :target do |todo, options| - Entities.const_get(todo.target_type).represent(todo.target, options) + target = todo.target_type == 'Commit' ? 'RepoCommit' : todo.target_type + Entities.const_get(target).represent(todo.target, options) end expose :target_url do |todo, options| @@ -417,7 +419,7 @@ module API end class Namespace < Grape::Entity - expose :id, :name, :path, :kind + expose :id, :name, :path, :kind, :full_path end class MemberAccess < Grape::Entity @@ -453,7 +455,8 @@ module API class ProjectService < Grape::Entity expose :id, :title, :created_at, :updated_at, :active expose :push_events, :issues_events, :merge_requests_events - expose :tag_push_events, :note_events, :build_events, :pipeline_events + expose :tag_push_events, :note_events, :pipeline_events + expose :build_events, as: :job_events # Expose serialized properties expose :properties do |service, options| field_names = service.fields. @@ -556,12 +559,15 @@ module API expose :updated_at expose :home_page_url expose :default_branch_protection - expose :restricted_visibility_levels + expose(:restricted_visibility_levels) do |setting, _options| + setting.restricted_visibility_levels.map { |level| Gitlab::VisibilityLevel.string_level(level) } + end expose :max_attachment_size expose :session_expire_delay - expose :default_project_visibility - expose :default_snippet_visibility - expose :default_group_visibility + expose(:default_project_visibility) { |setting, _options| Gitlab::VisibilityLevel.string_level(setting.default_project_visibility) } + expose(:default_snippet_visibility) { |setting, _options| Gitlab::VisibilityLevel.string_level(setting.default_snippet_visibility) } + expose(:default_group_visibility) { |setting, _options| Gitlab::VisibilityLevel.string_level(setting.default_group_visibility) } + expose :default_artifacts_expire_in expose :domain_whitelist expose :domain_blacklist_enabled expose :domain_blacklist @@ -574,6 +580,8 @@ module API expose :koding_url expose :plantuml_enabled expose :plantuml_url + expose :terminal_max_session_time + expose :polling_interval_multiplier end class Release < Grape::Entity @@ -593,10 +601,6 @@ module API end end - class TriggerRequest < Grape::Entity - expose :id, :variables - end - class Runner < Grape::Entity expose :id expose :description @@ -621,7 +625,11 @@ module API end end - class BuildArtifactFile < Grape::Entity + class RunnerRegistrationDetails < Grape::Entity + expose :id, :token + end + + class JobArtifactFile < Grape::Entity expose :filename, :size end @@ -629,18 +637,21 @@ module API expose :id, :sha, :ref, :status end - class Build < Grape::Entity + class Job < Grape::Entity expose :id, :status, :stage, :name, :ref, :tag, :coverage expose :created_at, :started_at, :finished_at expose :user, with: User - expose :artifacts_file, using: BuildArtifactFile, if: -> (build, opts) { build.artifacts? } + expose :artifacts_file, using: JobArtifactFile, if: -> (job, opts) { job.artifacts? } expose :commit, with: RepoCommit expose :runner, with: Runner expose :pipeline, with: PipelineBasic end class Trigger < Grape::Entity - expose :token, :created_at, :updated_at, :deleted_at, :last_used + expose :id + expose :token, :description + expose :created_at, :updated_at, :deleted_at, :last_used + expose :owner, using: Entities::UserBasic end class Variable < Grape::Entity @@ -661,14 +672,14 @@ module API end class Environment < EnvironmentBasic - expose :project, using: Entities::Project + expose :project, using: Entities::BasicProjectDetails end class Deployment < Grape::Entity expose :id, :iid, :ref, :sha, :created_at expose :user, using: Entities::UserBasic expose :environment, using: Entities::EnvironmentBasic - expose :deployable, using: Entities::Build + expose :deployable, using: Entities::Job end class RepoLicense < Grape::Entity @@ -695,5 +706,99 @@ module API expose :id, :message, :starts_at, :ends_at, :color, :font expose :active?, as: :active end + + class PersonalAccessToken < Grape::Entity + expose :id, :name, :revoked, :created_at, :scopes + expose :active?, as: :active + expose :expires_at do |personal_access_token| + personal_access_token.expires_at ? personal_access_token.expires_at.strftime("%Y-%m-%d") : nil + end + end + + class PersonalAccessTokenWithToken < PersonalAccessToken + expose :token + end + + class ImpersonationToken < PersonalAccessTokenWithToken + expose :impersonation + end + + module JobRequest + class JobInfo < Grape::Entity + expose :name, :stage + expose :project_id, :project_name + end + + class GitInfo < Grape::Entity + expose :repo_url, :ref, :sha, :before_sha + expose :ref_type do |model| + if model.tag + 'tag' + else + 'branch' + end + end + end + + class RunnerInfo < Grape::Entity + expose :timeout + end + + class Step < Grape::Entity + expose :name, :script, :timeout, :when, :allow_failure + end + + class Image < Grape::Entity + expose :name + end + + class Artifacts < Grape::Entity + expose :name, :untracked, :paths, :when, :expire_in + end + + class Cache < Grape::Entity + expose :key, :untracked, :paths + end + + class Credentials < Grape::Entity + expose :type, :url, :username, :password + end + + class ArtifactFile < Grape::Entity + expose :filename, :size + end + + class Dependency < Grape::Entity + expose :id, :name, :token + expose :artifacts_file, using: ArtifactFile, if: ->(job, _) { job.artifacts? } + end + + class Response < Grape::Entity + expose :id + expose :token + expose :allow_git_fetch + + expose :job_info, using: JobInfo do |model| + model + end + + expose :git_info, using: GitInfo do |model| + model + end + + expose :runner_info, using: RunnerInfo do |model| + model + end + + expose :variables + expose :steps, using: Step + expose :image, using: Image + expose :services, using: Image + expose :artifacts, using: Artifacts + expose :cache, using: Cache + expose :credentials, using: Credentials + expose :dependencies, using: Dependency + end + end end end diff --git a/lib/api/environments.rb b/lib/api/environments.rb index 1a7e68f0528..945771d46f3 100644 --- a/lib/api/environments.rb +++ b/lib/api/environments.rb @@ -9,7 +9,7 @@ module API params do requires :id, type: String, desc: 'The project ID' end - resource :projects do + resource :projects, requirements: { id: %r{[^/]+} } do desc 'Get all environments of the project' do detail 'This feature was introduced in GitLab 8.11.' success Entities::Environment @@ -79,7 +79,24 @@ module API environment = user_project.environments.find(params[:environment_id]) - present environment.destroy, with: Entities::Environment + environment.destroy + end + + desc 'Stops an existing environment' do + success Entities::Environment + end + params do + requires :environment_id, type: Integer, desc: 'The environment ID' + end + post ':id/environments/:environment_id/stop' do + authorize! :create_deployment, user_project + + environment = user_project.environments.find(params[:environment_id]) + + environment.stop_with_action!(current_user) + + status 200 + present environment, with: Entities::Environment end end end diff --git a/lib/api/files.rb b/lib/api/files.rb index 2e79e22e649..33fc970dc09 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -1,12 +1,11 @@ module API - # Projects API class Files < Grape::API helpers do def commit_params(attrs) { file_path: attrs[:file_path], - source_branch: attrs[:branch_name], - target_branch: attrs[:branch_name], + start_branch: attrs[:branch], + target_branch: attrs[:branch], commit_message: attrs[:commit_message], file_content: attrs[:content], file_content_encoding: attrs[:encoding], @@ -15,16 +14,29 @@ module API } end + def assign_file_vars! + authorize! :download_code, user_project + + @commit = user_project.commit(params[:ref]) + not_found!('Commit') unless @commit + + @repo = user_project.repository + @blob = @repo.blob_at(@commit.sha, params[:file_path]) + + not_found!('File') unless @blob + @blob.load_all_data!(@repo) + end + def commit_response(attrs) { file_path: attrs[:file_path], - branch_name: attrs[:branch_name] + branch: attrs[:branch] } end params :simple_file_params do - requires :file_path, type: String, desc: 'The path to new file. Ex. lib/class.rb' - requires :branch_name, type: String, desc: 'The name of branch' + requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' + requires :branch, type: String, desc: 'The name of branch' requires :commit_message, type: String, desc: 'Commit Message' optional :author_email, type: String, desc: 'The email of the author' optional :author_name, type: String, desc: 'The name of the author' @@ -40,35 +52,36 @@ module API params do requires :id, type: String, desc: 'The project ID' end - resource :projects do - desc 'Get a file from repository' + resource :projects, requirements: { id: %r{[^/]+} } do + desc 'Get raw file contents from the repository' params do - requires :file_path, type: String, desc: 'The path to the file. Ex. lib/class.rb' - requires :ref, type: String, desc: 'The name of branch, tag, or commit' + requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' + requires :ref, type: String, desc: 'The name of branch, tag commit' end - get ":id/repository/files" do - authorize! :download_code, user_project - - commit = user_project.commit(params[:ref]) - not_found!('Commit') unless commit + get ":id/repository/files/:file_path/raw" do + assign_file_vars! - repo = user_project.repository - blob = repo.blob_at(commit.sha, params[:file_path]) - not_found!('File') unless blob + send_git_blob @repo, @blob + end - blob.load_all_data!(repo) - status(200) + desc 'Get a file from the repository' + params do + requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' + requires :ref, type: String, desc: 'The name of branch, tag or commit' + end + get ":id/repository/files/:file_path", requirements: { file_path: /.+/ } do + assign_file_vars! { - file_name: blob.name, - file_path: blob.path, - size: blob.size, + file_name: @blob.name, + file_path: @blob.path, + size: @blob.size, encoding: "base64", - content: Base64.strict_encode64(blob.data), + content: Base64.strict_encode64(@blob.data), ref: params[:ref], - blob_id: blob.id, - commit_id: commit.id, - last_commit_id: repo.last_commit_id_for_path(commit.sha, params[:file_path]) + blob_id: @blob.id, + commit_id: @commit.id, + last_commit_id: @repo.last_commit_id_for_path(@commit.sha, params[:file_path]) } end @@ -76,7 +89,7 @@ module API params do use :extended_file_params end - post ":id/repository/files" do + post ":id/repository/files/:file_path", requirements: { file_path: /.+/ } do authorize! :push_code, user_project file_params = declared_params(include_missing: false) @@ -94,7 +107,7 @@ module API params do use :extended_file_params end - put ":id/repository/files" do + put ":id/repository/files/:file_path", requirements: { file_path: /.+/ } do authorize! :push_code, user_project file_params = declared_params(include_missing: false) @@ -113,16 +126,13 @@ module API params do use :simple_file_params end - delete ":id/repository/files" do + delete ":id/repository/files/:file_path", requirements: { file_path: /.+/ } do authorize! :push_code, user_project file_params = declared_params(include_missing: false) - result = ::Files::DeleteService.new(user_project, current_user, commit_params(file_params)).execute + result = ::Files::DestroyService.new(user_project, current_user, commit_params(file_params)).execute - if result[:status] == :success - status(200) - commit_response(file_params) - else + if result[:status] != :success render_api_error!(result[:message], 400) end end diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 7682d286866..8f3799417e3 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -7,7 +7,7 @@ module API helpers do params :optional_params do optional :description, type: String, desc: 'The description of the group' - optional :visibility_level, type: Integer, desc: 'The visibility level of the group' + optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the group' optional :lfs_enabled, type: Boolean, desc: 'Enable/disable LFS for the projects in this group' optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access' end @@ -36,12 +36,15 @@ module API optional :skip_groups, type: Array[Integer], desc: 'Array of group ids to exclude from list' optional :all_available, type: Boolean, desc: 'Show all group that you have access to' optional :search, type: String, desc: 'Search for a specific group' + optional :owned, type: Boolean, default: false, desc: 'Limit by owned by authenticated user' optional :order_by, type: String, values: %w[name path], default: 'name', desc: 'Order by name or path' optional :sort, type: String, values: %w[asc desc], default: 'asc', desc: 'Sort by asc (ascending) or desc (descending)' use :pagination end get do - groups = if current_user.admin + groups = if params[:owned] + current_user.owned_groups + elsif current_user.admin Group.all elsif params[:all_available] GroupsFinder.new.execute(current_user) @@ -56,23 +59,13 @@ module API present_groups groups, statistics: params[:statistics] && current_user.is_admin? end - desc 'Get list of owned groups for authenticated user' do - success Entities::Group - end - params do - use :pagination - use :statistics_params - end - get '/owned' do - present_groups current_user.owned_groups, statistics: params[:statistics] - end - desc 'Create a group. Available only for users who can create groups.' do success Entities::Group end params do requires :name, type: String, desc: 'The name of the group' requires :path, type: String, desc: 'The path of the group' + optional :parent_id, type: Integer, desc: 'The parent group id for creating nested group' use :optional_params end post do @@ -91,7 +84,7 @@ module API params do requires :id, type: String, desc: 'The ID of a group' end - resource :groups do + resource :groups, requirements: { id: %r{[^/]+} } do desc 'Update a group. Available only for users who can administrate groups.' do success Entities::Group end @@ -99,7 +92,7 @@ module API optional :name, type: String, desc: 'The name of the group' optional :path, type: String, desc: 'The path of the group' use :optional_params - at_least_one_of :name, :path, :description, :visibility_level, + at_least_one_of :name, :path, :description, :visibility, :lfs_enabled, :request_access_enabled end put ':id' do @@ -125,7 +118,7 @@ module API delete ":id" do group = find_group!(params[:id]) authorize! :admin_group, group - DestroyGroupService.new(group, current_user).execute + ::Groups::DestroyService.new(group, current_user).execute end desc 'Get a list of projects in this group.' do @@ -133,7 +126,7 @@ module API end params do optional :archived, type: Boolean, default: false, desc: 'Limit by archived status' - optional :visibility, type: String, values: %w[public internal private], + optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'Limit by visibility' optional :search, type: String, desc: 'Return list of authorized projects matching the search criteria' optional :order_by, type: String, values: %w[id name path created_at updated_at last_activity_at], @@ -142,6 +135,9 @@ module API desc: 'Return projects sorted in ascending and descending order' optional :simple, type: Boolean, default: false, desc: 'Return only the ID, URL, name, and path of each project' + optional :owned, type: Boolean, default: false, desc: 'Limit by owned by authenticated user' + optional :starred, type: Boolean, default: false, desc: 'Limit by starred status' + use :pagination end get ":id/projects" do @@ -158,7 +154,7 @@ module API params do requires :project_id, type: String, desc: 'The ID or path of the project' end - post ":id/projects/:project_id" do + post ":id/projects/:project_id", requirements: { project_id: /.+/ } do authenticated_as_admin! group = find_group!(params[:id]) project = find_project!(params[:project_id]) diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 49c5f0652ab..61527c1e20b 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -3,7 +3,7 @@ module API include Gitlab::Utils include Helpers::Pagination - SUDO_HEADER = "HTTP_SUDO" + SUDO_HEADER = "HTTP_SUDO".freeze SUDO_PARAM = :sudo def declared_params(options = {}) @@ -45,7 +45,7 @@ module API if id =~ /^\d+$/ Project.find_by(id: id) else - Project.find_with_namespace(id) + Project.find_by_full_path(id) end end @@ -82,16 +82,27 @@ module API label || not_found!('Label') end - def find_project_issue(id) - IssuesFinder.new(current_user, project_id: user_project.id).find(id) + def find_project_issue(iid) + IssuesFinder.new(current_user, project_id: user_project.id).find_by!(iid: iid) end - def find_project_merge_request(id) - MergeRequestsFinder.new(current_user, project_id: user_project.id).find(id) + def find_project_merge_request(iid) + MergeRequestsFinder.new(current_user, project_id: user_project.id).find_by!(iid: iid) + end + + def find_project_snippet(id) + finder_params = { filter: :by_project, project: user_project } + SnippetsFinder.new.execute(current_user, finder_params).find(id) + end + + def find_merge_request_with_access(iid, access_level = :read_merge_request) + merge_request = user_project.merge_requests.find_by!(iid: iid) + authorize! access_level, merge_request + merge_request end def authenticate! - unauthorized! unless current_user + unauthorized! unless current_user && can?(current_user, :access_api) end def authenticate_non_get! @@ -110,7 +121,7 @@ module API forbidden! unless current_user.is_admin? end - def authorize!(action, subject = nil) + def authorize!(action, subject = :global) forbidden! unless can?(current_user, action, subject) end @@ -128,7 +139,7 @@ module API end end - def can?(object, action, subject) + def can?(object, action, subject = :global) Ability.allowed?(object, action, subject) end @@ -147,33 +158,21 @@ module API params_hash = custom_params || params attrs = {} keys.each do |key| - if params_hash[key].present? or (params_hash.has_key?(key) and params_hash[key] == false) + if params_hash[key].present? || (params_hash.has_key?(key) && params_hash[key] == false) attrs[key] = params_hash[key] end end ActionController::Parameters.new(attrs).permit! end - # Checks the occurrences of datetime attributes, each attribute if present in the params hash must be in ISO 8601 - # format (YYYY-MM-DDTHH:MM:SSZ) or a Bad Request error is invoked. - # - # Parameters: - # keys (required) - An array consisting of elements that must be parseable as dates from the params hash - def datetime_attributes!(*keys) - keys.each do |key| - begin - params[key] = Time.xmlschema(params[key]) if params[key].present? - rescue ArgumentError - message = "\"" + key.to_s + "\" must be a timestamp in ISO 8601 format: YYYY-MM-DDTHH:MM:SSZ" - render_api_error!(message, 400) - end - end - end - def filter_by_iid(items, iid) items.where(iid: iid) end + def filter_by_search(items, text) + items.search(text) + end + # error helpers def forbidden!(reason = nil) @@ -219,12 +218,20 @@ module API render_api_error!('204 No Content', 204) end + def accepted! + render_api_error!('202 Accepted', 202) + end + def render_validation_error!(model) if model.errors.any? render_api_error!(model.errors.messages || '400 Bad Request', 400) end end + def render_spam_error! + render_api_error!({ error: 'Spam detected' }, 400) + end + def render_api_error!(message, status) error!({ 'message' => message }, status, header) end @@ -250,6 +257,18 @@ module API # project helpers def filter_projects(projects) + if params[:membership] + projects = projects.merge(current_user.authorized_projects) + end + + if params[:owned] + projects = projects.merge(current_user.owned_projects) + end + + if params[:starred] + projects = projects.merge(current_user.starred_projects) + end + if params[:search].present? projects = projects.search(params[:search]) end @@ -298,7 +317,7 @@ module API header['X-Sendfile'] = path body else - path + file path end end @@ -322,16 +341,17 @@ module API def initial_current_user return @initial_current_user if defined?(@initial_current_user) + Gitlab::Auth::UniqueIpsLimiter.limit_user! do + @initial_current_user ||= find_user_by_private_token(scopes: @scopes) + @initial_current_user ||= doorkeeper_guard(scopes: @scopes) + @initial_current_user ||= find_user_from_warden - @initial_current_user ||= find_user_by_private_token(scopes: @scopes) - @initial_current_user ||= doorkeeper_guard(scopes: @scopes) - @initial_current_user ||= find_user_from_warden + unless @initial_current_user && Gitlab::UserAccess.new(@initial_current_user).allowed? + @initial_current_user = nil + end - unless @initial_current_user && Gitlab::UserAccess.new(@initial_current_user).allowed? - @initial_current_user = nil + @initial_current_user end - - @initial_current_user end def sudo! @@ -374,14 +394,6 @@ module API header(*Gitlab::Workhorse.send_git_archive(repository, ref: ref, format: format)) end - def issue_entity(project) - if project.has_external_issue_tracker? - Entities::ExternalIssue - else - Entities::Issue - end - end - # The Grape Error Middleware only has access to env but no params. We workaround this by # defining a method that returns the right value. def define_params_for_grape_middleware diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index e8975eb57e0..2135a787b11 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -9,11 +9,11 @@ module API # In addition, they may have a '.git' extension and multiple namespaces # # Transform all these cases to 'namespace/project' - def clean_project_path(project_path, storage_paths = Repository.storages.values) + def clean_project_path(project_path, storages = Gitlab.config.repositories.storages.values) project_path = project_path.sub(/\.git\z/, '') - storage_paths.each do |storage_path| - storage_path = File.expand_path(storage_path) + storages.each do |storage| + storage_path = File.expand_path(storage['path']) if project_path.start_with?(storage_path) project_path = project_path.sub(storage_path, '') @@ -30,7 +30,7 @@ module API def wiki? @wiki ||= project_path.end_with?('.wiki') && - !Project.find_with_namespace(project_path) + !Project.find_by_full_path(project_path) end def project @@ -41,7 +41,7 @@ module API # the wiki repository as well. project_path.chomp!('.wiki') if wiki? - Project.find_with_namespace(project_path) + Project.find_by_full_path(project_path) end end diff --git a/lib/api/helpers/pagination.rb b/lib/api/helpers/pagination.rb index 2199eea7e5f..0764b58fb4c 100644 --- a/lib/api/helpers/pagination.rb +++ b/lib/api/helpers/pagination.rb @@ -2,7 +2,7 @@ module API module Helpers module Pagination def paginate(relation) - relation.page(params[:page]).per(params[:per_page].to_i).tap do |data| + relation.page(params[:page]).per(params[:per_page]).tap do |data| add_pagination_headers(data) end end diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb new file mode 100644 index 00000000000..74848a6e144 --- /dev/null +++ b/lib/api/helpers/runner.rb @@ -0,0 +1,69 @@ +module API + module Helpers + module Runner + JOB_TOKEN_HEADER = 'HTTP_JOB_TOKEN'.freeze + JOB_TOKEN_PARAM = :token + UPDATE_RUNNER_EVERY = 10 * 60 + + def runner_registration_token_valid? + ActiveSupport::SecurityUtils.variable_size_secure_compare(params[:token], + current_application_settings.runners_registration_token) + end + + def get_runner_version_from_params + return unless params['info'].present? + attributes_for_keys(%w(name version revision platform architecture), params['info']) + end + + def authenticate_runner! + forbidden! unless current_runner + end + + def current_runner + @runner ||= ::Ci::Runner.find_by_token(params[:token].to_s) + end + + def update_runner_info + return unless update_runner? + + current_runner.contacted_at = Time.now + current_runner.assign_attributes(get_runner_version_from_params) + current_runner.save if current_runner.changed? + end + + def update_runner? + # Use a random threshold to prevent beating DB updates. + # It generates a distribution between [40m, 80m]. + # + contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY) + + current_runner.contacted_at.nil? || + (Time.now - current_runner.contacted_at) >= contacted_at_max_age + end + + def validate_job!(job) + not_found! unless job + + yield if block_given? + + forbidden!('Project has been deleted!') unless job.project + forbidden!('Job has been erased!') if job.erased? + end + + def authenticate_job!(job) + validate_job!(job) do + forbidden! unless job_token_valid?(job) + end + end + + def job_token_valid?(job) + token = (params[JOB_TOKEN_PARAM] || env[JOB_TOKEN_HEADER]).to_s + token && job.valid_token?(token) + end + + def max_artifacts_size + current_application_settings.max_artifacts_size.megabytes.to_i + end + end + end +end diff --git a/lib/api/internal.rb b/lib/api/internal.rb index d235977fbd8..523f38d129e 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -132,6 +132,18 @@ module API { success: true, recovery_codes: codes } end + + post "/notify_post_receive" do + status 200 + + return unless Gitlab::GitalyClient.enabled? + + begin + Gitlab::GitalyClient::Notifications.new(params[:repo_path]).post_receive + rescue GRPC::Unavailable => e + render_api_error(e, 500) + end + end end end end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index fe016c1ec0a..4dce5dd130a 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -10,19 +10,9 @@ module API args.delete(:id) args[:milestone_title] = args.delete(:milestone) + args[:label_name] = args.delete(:labels) - match_all_labels = args.delete(:match_all_labels) - labels = args.delete(:labels) - args[:label_name] = labels if match_all_labels - - args[:search] = "#{Issue.reference_prefix}#{args.delete(:iid)}" if args.key?(:iid) - - issues = IssuesFinder.new(current_user, args).execute.inc_notes_with_associations - - # TODO: Remove in 9.0 pass `label_name: args.delete(:labels)` to IssuesFinder - if !match_all_labels && labels.present? - issues = issues.includes(:labels).where('labels.title' => labels.split(',')) - end + issues = IssuesFinder.new(current_user, args).execute issues.reorder(args[:order_by] => args[:sort]) end @@ -35,6 +25,7 @@ module API optional :sort, type: String, values: %w[asc desc], default: 'desc', desc: 'Return issues sorted in `asc` or `desc` order.' optional :milestone, type: String, desc: 'Return issues for a specific milestone' + optional :iids, type: Array[Integer], desc: 'The IID array of issues' use :pagination end @@ -50,7 +41,7 @@ module API resource :issues do desc "Get currently authenticated user's issues" do - success Entities::Issue + success Entities::IssueBasic end params do optional :state, type: String, values: %w[opened closed all], default: 'all', @@ -60,62 +51,61 @@ module API get do issues = find_issues(scope: 'authored') - present paginate(issues), with: Entities::Issue, current_user: current_user + present paginate(issues), with: Entities::IssueBasic, current_user: current_user end end params do requires :id, type: String, desc: 'The ID of a group' end - resource :groups do + resource :groups, requirements: { id: %r{[^/]+} } do desc 'Get a list of group issues' do - success Entities::Issue + success Entities::IssueBasic end params do - optional :state, type: String, values: %w[opened closed all], default: 'opened', + optional :state, type: String, values: %w[opened closed all], default: 'all', desc: 'Return opened, closed, or all issues' use :issues_params end get ":id/issues" do group = find_group!(params[:id]) - issues = find_issues(group_id: group.id, state: params[:state] || 'opened', match_all_labels: true) + issues = find_issues(group_id: group.id) - present paginate(issues), with: Entities::Issue, current_user: current_user + present paginate(issues), with: Entities::IssueBasic, current_user: current_user end end params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects do + resource :projects, requirements: { id: %r{[^/]+} } do include TimeTrackingEndpoints desc 'Get a list of project issues' do - success Entities::Issue + success Entities::IssueBasic end params do optional :state, type: String, values: %w[opened closed all], default: 'all', desc: 'Return opened, closed, or all issues' - optional :iid, type: Integer, desc: 'Return the issue having the given `iid`' use :issues_params end get ":id/issues" do - project = find_project(params[:id]) + project = find_project!(params[:id]) issues = find_issues(project_id: project.id) - present paginate(issues), with: Entities::Issue, current_user: current_user, project: user_project + present paginate(issues), with: Entities::IssueBasic, current_user: current_user, project: user_project end desc 'Get a single project issue' do success Entities::Issue end params do - requires :issue_id, type: Integer, desc: 'The ID of a project issue' + requires :issue_iid, type: Integer, desc: 'The internal ID of a project issue' end - get ":id/issues/:issue_id" do - issue = find_project_issue(params[:issue_id]) + get ":id/issues/:issue_iid" do + issue = find_project_issue(params[:issue_iid]) present issue, with: Entities::Issue, current_user: current_user, project: user_project end @@ -126,8 +116,10 @@ module API requires :title, type: String, desc: 'The title of an issue' optional :created_at, type: DateTime, desc: 'Date time when the issue was created. Available only for admins and project owners.' - optional :merge_request_for_resolving_discussions, type: Integer, + optional :merge_request_to_resolve_discussions_of, type: Integer, desc: 'The IID of a merge request for which to resolve discussions' + optional :discussion_to_resolve, type: String, + desc: 'The ID of a discussion to resolve, also pass `merge_request_to_resolve_discussions_of`' use :issue_params end post ':id/issues' do @@ -138,12 +130,6 @@ module API issue_params = declared_params(include_missing: false) - if merge_request_iid = params[:merge_request_for_resolving_discussions] - issue_params[:merge_request_for_resolving_discussions] = MergeRequestsFinder.new(current_user, project_id: user_project.id). - execute. - find_by(iid: merge_request_iid) - end - issue = ::Issues::CreateService.new(user_project, current_user, issue_params.merge(request: request, api: true)).execute @@ -162,7 +148,7 @@ module API success Entities::Issue end params do - requires :issue_id, type: Integer, desc: 'The ID of a project issue' + requires :issue_iid, type: Integer, desc: 'The internal ID of a project issue' optional :title, type: String, desc: 'The title of an issue' optional :updated_at, type: DateTime, desc: 'Date time when the issue was updated. Available only for admins and project owners.' @@ -171,8 +157,8 @@ module API at_least_one_of :title, :description, :assignee_id, :milestone_id, :labels, :created_at, :due_date, :confidential, :state_event end - put ':id/issues/:issue_id' do - issue = user_project.issues.find(params.delete(:issue_id)) + put ':id/issues/:issue_iid' do + issue = user_project.issues.find_by!(iid: params.delete(:issue_iid)) authorize! :update_issue, issue # Setting created_at time only allowed for admins and project owners @@ -180,9 +166,13 @@ module API params.delete(:updated_at) end + update_params = declared_params(include_missing: false).merge(request: request, api: true) + issue = ::Issues::UpdateService.new(user_project, current_user, - declared_params(include_missing: false)).execute(issue) + update_params).execute(issue) + + render_spam_error! if issue.spam? if issue.valid? present issue, with: Entities::Issue, current_user: current_user, project: user_project @@ -195,11 +185,11 @@ module API success Entities::Issue end params do - requires :issue_id, type: Integer, desc: 'The ID of a project issue' + requires :issue_iid, type: Integer, desc: 'The internal ID of a project issue' requires :to_project_id, type: Integer, desc: 'The ID of the new project' end - post ':id/issues/:issue_id/move' do - issue = user_project.issues.find_by(id: params[:issue_id]) + post ':id/issues/:issue_iid/move' do + issue = user_project.issues.find_by(iid: params[:issue_iid]) not_found!('Issue') unless issue new_project = Project.find_by(id: params[:to_project_id]) @@ -215,10 +205,10 @@ module API desc 'Delete a project issue' params do - requires :issue_id, type: Integer, desc: 'The ID of a project issue' + requires :issue_iid, type: Integer, desc: 'The internal ID of a project issue' end - delete ":id/issues/:issue_id" do - issue = user_project.issues.find_by(id: params[:issue_id]) + delete ":id/issues/:issue_iid" do + issue = user_project.issues.find_by(iid: params[:issue_iid]) not_found!('Issue') unless issue authorize!(:destroy_issue, issue) diff --git a/lib/api/builds.rb b/lib/api/jobs.rb index af61be343be..ffab0aafe59 100644 --- a/lib/api/builds.rb +++ b/lib/api/jobs.rb @@ -1,5 +1,5 @@ module API - class Builds < Grape::API + class Jobs < Grape::API include PaginationParams before { authenticate! } @@ -7,16 +7,19 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects do + resource :projects, requirements: { id: %r{[^/]+} } do helpers do params :optional_scope do optional :scope, types: [String, Array[String]], desc: 'The scope of builds to show', - values: ['pending', 'running', 'failed', 'success', 'canceled'], + values: ::CommitStatus::AVAILABLE_STATUSES, coerce_with: ->(scope) { - if scope.is_a?(String) + case scope + when String [scope] - elsif scope.is_a?(Hashie::Mash) + when Hashie::Mash scope.values + when Hashie::Array + scope else ['unknown'] end @@ -24,79 +27,72 @@ module API end end - desc 'Get a project builds' do - success Entities::Build + desc 'Get a projects jobs' do + success Entities::Job end params do use :optional_scope use :pagination end - get ':id/builds' do + get ':id/jobs' do builds = user_project.builds.order('id DESC') builds = filter_builds(builds, params[:scope]) - present paginate(builds), with: Entities::Build, - user_can_download_artifacts: can?(current_user, :read_build, user_project) + present paginate(builds), with: Entities::Job end - desc 'Get builds for a specific commit of a project' do - success Entities::Build + desc 'Get pipeline jobs' do + success Entities::Job end params do - requires :sha, type: String, desc: 'The SHA id of a commit' + requires :pipeline_id, type: Integer, desc: 'The pipeline ID' use :optional_scope use :pagination end - get ':id/repository/commits/:sha/builds' do - authorize_read_builds! - - return not_found! unless user_project.commit(params[:sha]) - - pipelines = user_project.pipelines.where(sha: params[:sha]) - builds = user_project.builds.where(pipeline: pipelines).order('id DESC') + get ':id/pipelines/:pipeline_id/jobs' do + pipeline = user_project.pipelines.find(params[:pipeline_id]) + builds = pipeline.builds builds = filter_builds(builds, params[:scope]) - present paginate(builds), with: Entities::Build, - user_can_download_artifacts: can?(current_user, :read_build, user_project) + present paginate(builds), with: Entities::Job end - desc 'Get a specific build of a project' do - success Entities::Build + desc 'Get a specific job of a project' do + success Entities::Job end params do - requires :build_id, type: Integer, desc: 'The ID of a build' + requires :job_id, type: Integer, desc: 'The ID of a job' end - get ':id/builds/:build_id' do + get ':id/jobs/:job_id' do authorize_read_builds! - build = get_build!(params[:build_id]) + build = get_build!(params[:job_id]) - present build, with: Entities::Build, - user_can_download_artifacts: can?(current_user, :read_build, user_project) + present build, with: Entities::Job end - desc 'Download the artifacts file from build' do + desc 'Download the artifacts file from a job' do detail 'This feature was introduced in GitLab 8.5' end params do - requires :build_id, type: Integer, desc: 'The ID of a build' + requires :job_id, type: Integer, desc: 'The ID of a job' end - get ':id/builds/:build_id/artifacts' do + get ':id/jobs/:job_id/artifacts' do authorize_read_builds! - build = get_build!(params[:build_id]) + build = get_build!(params[:job_id]) present_artifacts!(build.artifacts_file) end - desc 'Download the artifacts file from build' do + desc 'Download the artifacts file from a job' do detail 'This feature was introduced in GitLab 8.10' end params do requires :ref_name, type: String, desc: 'The ref from repository' - requires :job, type: String, desc: 'The name for the build' + requires :job, type: String, desc: 'The name for the job' end - get ':id/builds/artifacts/:ref_name/download', + get ':id/jobs/artifacts/:ref_name/download', requirements: { ref_name: /.+/ } do authorize_read_builds! @@ -109,14 +105,14 @@ module API # TODO: We should use `present_file!` and leave this implementation for backward compatibility (when build trace # is saved in the DB instead of file). But before that, we need to consider how to replace the value of # `runners_token` with some mask (like `xxxxxx`) when sending trace file directly by workhorse. - desc 'Get a trace of a specific build of a project' + desc 'Get a trace of a specific job of a project' params do - requires :build_id, type: Integer, desc: 'The ID of a build' + requires :job_id, type: Integer, desc: 'The ID of a job' end - get ':id/builds/:build_id/trace' do + get ':id/jobs/:job_id/trace' do authorize_read_builds! - build = get_build!(params[:build_id]) + build = get_build!(params[:job_id]) header 'Content-Disposition', "infile; filename=\"#{build.id}.log\"" content_type 'text/plain' @@ -126,96 +122,91 @@ module API body trace end - desc 'Cancel a specific build of a project' do - success Entities::Build + desc 'Cancel a specific job of a project' do + success Entities::Job end params do - requires :build_id, type: Integer, desc: 'The ID of a build' + requires :job_id, type: Integer, desc: 'The ID of a job' end - post ':id/builds/:build_id/cancel' do + post ':id/jobs/:job_id/cancel' do authorize_update_builds! - build = get_build!(params[:build_id]) + build = get_build!(params[:job_id]) build.cancel - present build, with: Entities::Build, - user_can_download_artifacts: can?(current_user, :read_build, user_project) + present build, with: Entities::Job end desc 'Retry a specific build of a project' do - success Entities::Build + success Entities::Job end params do - requires :build_id, type: Integer, desc: 'The ID of a build' + requires :job_id, type: Integer, desc: 'The ID of a build' end - post ':id/builds/:build_id/retry' do + post ':id/jobs/:job_id/retry' do authorize_update_builds! - build = get_build!(params[:build_id]) - return forbidden!('Build is not retryable') unless build.retryable? + build = get_build!(params[:job_id]) + return forbidden!('Job is not retryable') unless build.retryable? build = Ci::Build.retry(build, current_user) - present build, with: Entities::Build, - user_can_download_artifacts: can?(current_user, :read_build, user_project) + present build, with: Entities::Job end - desc 'Erase build (remove artifacts and build trace)' do - success Entities::Build + desc 'Erase job (remove artifacts and the trace)' do + success Entities::Job end params do - requires :build_id, type: Integer, desc: 'The ID of a build' + requires :job_id, type: Integer, desc: 'The ID of a build' end - post ':id/builds/:build_id/erase' do + post ':id/jobs/:job_id/erase' do authorize_update_builds! - build = get_build!(params[:build_id]) - return forbidden!('Build is not erasable!') unless build.erasable? + build = get_build!(params[:job_id]) + return forbidden!('Job is not erasable!') unless build.erasable? build.erase(erased_by: current_user) - present build, with: Entities::Build, - user_can_download_artifacts: can?(current_user, :download_build_artifacts, user_project) + present build, with: Entities::Job end desc 'Keep the artifacts to prevent them from being deleted' do - success Entities::Build + success Entities::Job end params do - requires :build_id, type: Integer, desc: 'The ID of a build' + requires :job_id, type: Integer, desc: 'The ID of a job' end - post ':id/builds/:build_id/artifacts/keep' do + post ':id/jobs/:job_id/artifacts/keep' do authorize_update_builds! - build = get_build!(params[:build_id]) + build = get_build!(params[:job_id]) return not_found!(build) unless build.artifacts? build.keep_artifacts! status 200 - present build, with: Entities::Build, - user_can_download_artifacts: can?(current_user, :read_build, user_project) + present build, with: Entities::Job end - desc 'Trigger a manual build' do - success Entities::Build + desc 'Trigger a manual job' do + success Entities::Job detail 'This feature was added in GitLab 8.11' end params do - requires :build_id, type: Integer, desc: 'The ID of a Build' + requires :job_id, type: Integer, desc: 'The ID of a Job' end - post ":id/builds/:build_id/play" do + post ":id/jobs/:job_id/play" do authorize_read_builds! - build = get_build!(params[:build_id]) + build = get_build!(params[:job_id]) - bad_request!("Unplayable Build") unless build.playable? + bad_request!("Unplayable Job") unless build.playable? build.play(current_user) status 200 - present build, with: Entities::Build, - user_can_download_artifacts: can?(current_user, :read_build, user_project) + present build, with: Entities::Job end end diff --git a/lib/api/labels.rb b/lib/api/labels.rb index 652786d4e3e..20b25529d0c 100644 --- a/lib/api/labels.rb +++ b/lib/api/labels.rb @@ -1,17 +1,21 @@ module API - # Labels API class Labels < Grape::API + include PaginationParams + before { authenticate! } params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects do + resource :projects, requirements: { id: %r{[^/]+} } do desc 'Get all labels of the project' do success Entities::Label end + params do + use :pagination + end get ':id/labels' do - present available_labels, with: Entities::Label, current_user: current_user, project: user_project + present paginate(available_labels), with: Entities::Label, current_user: current_user, project: user_project end desc 'Create a new label' do @@ -19,7 +23,7 @@ module API end params do requires :name, type: String, desc: 'The name of the label to be created' - requires :color, type: String, desc: "The color of the label given in 6-digit hex notation with leading '#' sign (e.g. #FFAABB)" + requires :color, type: String, desc: "The color of the label given in 6-digit hex notation with leading '#' sign (e.g. #FFAABB) or one of the allowed CSS color names" optional :description, type: String, desc: 'The description of label to be created' optional :priority, type: Integer, desc: 'The priority of the label', allow_blank: true end @@ -30,7 +34,7 @@ module API conflict!('Label already exists') if label priority = params.delete(:priority) - label = user_project.labels.create(declared_params(include_missing: false)) + label = ::Labels::CreateService.new(declared_params(include_missing: false)).execute(project: user_project) if label.valid? label.prioritize!(user_project, priority) if priority @@ -52,7 +56,7 @@ module API label = user_project.labels.find_by(title: params[:name]) not_found!('Label') unless label - present label.destroy, with: Entities::Label, current_user: current_user, project: user_project + label.destroy end desc 'Update an existing label. At least one optional parameter is required.' do @@ -61,7 +65,7 @@ module API params do requires :name, type: String, desc: 'The name of the label to be updated' optional :new_name, type: String, desc: 'The new name of the label' - optional :color, type: String, desc: "The new color of the label given in 6-digit hex notation with leading '#' sign (e.g. #FFAABB)" + optional :color, type: String, desc: "The new color of the label given in 6-digit hex notation with leading '#' sign (e.g. #FFAABB) or one of the allowed CSS color names" optional :description, type: String, desc: 'The new description of label' optional :priority, type: Integer, desc: 'The priority of the label', allow_blank: true at_least_one_of :new_name, :color, :description, :priority @@ -78,7 +82,8 @@ module API # Rename new name to the actual label attribute name label_params[:name] = label_params.delete(:new_name) if label_params.key?(:new_name) - render_validation_error!(label) unless label.update(label_params) + label = ::Labels::UpdateService.new(label_params).execute(label) + render_validation_error!(label) unless label.valid? if update_priority if priority.nil? diff --git a/lib/api/members.rb b/lib/api/members.rb index d85f1f78cd6..c200e46a328 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -10,7 +10,7 @@ module API params do requires :id, type: String, desc: "The #{source_type} ID" end - resource source_type.pluralize do + resource source_type.pluralize, requirements: { id: %r{[^/]+} } do desc 'Gets a list of group or project members viewable by the authenticated user.' do success Entities::Member end @@ -55,24 +55,13 @@ module API authorize_admin_source!(source_type, source) member = source.members.find_by(user_id: params[:user_id]) + conflict!('Member already exists') if member - # We need this explicit check because `source.add_user` doesn't - # currently return the member created so it would return 201 even if - # the member already existed... - # The `source_type == 'group'` check is to ensure back-compatibility - # but 409 behavior should be used for both project and group members in 9.0! - conflict!('Member already exists') if source_type == 'group' && member - - unless member - member = source.add_user(params[:user_id], params[:access_level], current_user: current_user, expires_at: params[:expires_at]) - end + member = source.add_user(params[:user_id], params[:access_level], current_user: current_user, expires_at: params[:expires_at]) if member.persisted? && member.valid? present member.user, with: Entities::Member, member: member else - # This is to ensure back-compatibility but 400 behavior should be used - # for all validation errors in 9.0! - render_api_error!('Access level is not known', 422) if member.errors.key?(:access_level) render_validation_error!(member) end end @@ -86,18 +75,14 @@ module API optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY' end put ":id/members/:user_id" do - source = find_source(source_type, params[:id]) + source = find_source(source_type, params.delete(:id)) authorize_admin_source!(source_type, source) - member = source.members.find_by!(user_id: params[:user_id]) - attrs = attributes_for_keys [:access_level, :expires_at] + member = source.members.find_by!(user_id: params.delete(:user_id)) - if member.update_attributes(attrs) + if member.update_attributes(declared_params(include_missing: false)) present member.user, with: Entities::Member, member: member else - # This is to ensure back-compatibility but 400 behavior should be used - # for all validation errors in 9.0! - render_api_error!('Access level is not known', 422) if member.errors.key?(:access_level) render_validation_error!(member) end end @@ -108,24 +93,10 @@ module API end delete ":id/members/:user_id" do source = find_source(source_type, params[:id]) + # Ensure that memeber exists + source.members.find_by!(user_id: params[:user_id]) - # This is to ensure back-compatibility but find_by! should be used - # in that casse in 9.0! - member = source.members.find_by(user_id: params[:user_id]) - - # This is to ensure back-compatibility but this should be removed in - # favor of find_by! in 9.0! - not_found!("Member: user_id:#{params[:user_id]}") if source_type == 'group' && member.nil? - - # This is to ensure back-compatibility but 204 behavior should be used - # for all DELETE endpoints in 9.0! - if member.nil? - { message: "Access revoked", id: params[:user_id].to_i } - else - ::Members::DestroyService.new(source, current_user, declared_params).execute - - present member.user, with: Entities::Member, member: member - end + ::Members::DestroyService.new(source, current_user, declared_params).execute end end end diff --git a/lib/api/merge_request_diffs.rb b/lib/api/merge_request_diffs.rb index 07435d78468..4b79eac2b8b 100644 --- a/lib/api/merge_request_diffs.rb +++ b/lib/api/merge_request_diffs.rb @@ -1,25 +1,27 @@ module API # MergeRequestDiff API class MergeRequestDiffs < Grape::API + include PaginationParams + before { authenticate! } - resource :projects do + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: { id: %r{[^/]+} } do desc 'Get a list of merge request diff versions' do detail 'This feature was introduced in GitLab 8.12.' success Entities::MergeRequestDiff end params do - requires :id, type: String, desc: 'The ID of a project' - requires :merge_request_id, type: Integer, desc: 'The ID of a merge request' + requires :merge_request_iid, type: Integer, desc: 'The IID of a merge request' + use :pagination end + get ":id/merge_requests/:merge_request_iid/versions" do + merge_request = find_merge_request_with_access(params[:merge_request_iid]) - get ":id/merge_requests/:merge_request_id/versions" do - merge_request = user_project.merge_requests. - find(params[:merge_request_id]) - - authorize! :read_merge_request, merge_request - present merge_request.merge_request_diffs, with: Entities::MergeRequestDiff + present paginate(merge_request.merge_request_diffs), with: Entities::MergeRequestDiff end desc 'Get a single merge request diff version' do @@ -28,16 +30,13 @@ module API end params do - requires :id, type: String, desc: 'The ID of a project' - requires :merge_request_id, type: Integer, desc: 'The ID of a merge request' + requires :merge_request_iid, type: Integer, desc: 'The IID of a merge request' requires :version_id, type: Integer, desc: 'The ID of a merge request diff version' end - get ":id/merge_requests/:merge_request_id/versions/:version_id" do - merge_request = user_project.merge_requests. - find(params[:merge_request_id]) + get ":id/merge_requests/:merge_request_iid/versions/:version_id" do + merge_request = find_merge_request_with_access(params[:merge_request_iid]) - authorize! :read_merge_request, merge_request present merge_request.merge_request_diffs.find(params[:version_id]), with: Entities::MergeRequestDiffFull end end diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index e77af4b7a0d..c8033664133 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -2,14 +2,12 @@ module API class MergeRequests < Grape::API include PaginationParams - DEPRECATION_MESSAGE = 'This endpoint is deprecated and will be removed in GitLab 9.0.'.freeze - before { authenticate! } params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects do + resource :projects, requirements: { id: %r{[^/]+} } do include TimeTrackingEndpoints helpers do @@ -27,6 +25,14 @@ module API render_api_error!(errors, 400) end + def issue_entity(project) + if project.has_external_issue_tracker? + Entities::ExternalIssue + else + Entities::IssueBasic + end + end + params :optional_params do optional :description, type: String, desc: 'The description of the merge request' optional :assignee_id, type: Integer, desc: 'The ID of a user to assign the merge request' @@ -37,7 +43,7 @@ module API end desc 'List merge requests' do - success Entities::MergeRequest + success Entities::MergeRequestBasic end params do optional :state, type: String, values: %w[opened closed merged all], default: 'all', @@ -46,14 +52,14 @@ module API desc: 'Return merge requests ordered by `created_at` or `updated_at` fields.' optional :sort, type: String, values: %w[asc desc], default: 'desc', desc: 'Return merge requests sorted in `asc` or `desc` order.' - optional :iid, type: Array[Integer], desc: 'The IID of the merge requests' + optional :iids, type: Array[Integer], desc: 'The IID array of merge requests' use :pagination end get ":id/merge_requests" do authorize! :read_merge_request, user_project merge_requests = user_project.merge_requests.inc_notes_with_associations - merge_requests = filter_by_iid(merge_requests, params[:iid]) if params[:iid].present? + merge_requests = filter_by_iid(merge_requests, params[:iids]) if params[:iids].present? merge_requests = case params[:state] @@ -64,7 +70,7 @@ module API end merge_requests = merge_requests.reorder(params[:order_by] => params[:sort]) - present paginate(merge_requests), with: Entities::MergeRequest, current_user: current_user, project: user_project + present paginate(merge_requests), with: Entities::MergeRequestBasic, current_user: current_user, project: user_project end desc 'Create a merge request' do @@ -95,190 +101,141 @@ module API desc 'Delete a merge request' params do - requires :merge_request_id, type: Integer, desc: 'The ID of a merge request' + requires :merge_request_iid, type: Integer, desc: 'The IID of a merge request' end - delete ":id/merge_requests/:merge_request_id" do - merge_request = find_project_merge_request(params[:merge_request_id]) + delete ":id/merge_requests/:merge_request_iid" do + merge_request = find_project_merge_request(params[:merge_request_iid]) authorize!(:destroy_merge_request, merge_request) merge_request.destroy end - # Routing "merge_request/:merge_request_id/..." is DEPRECATED and WILL BE REMOVED in version 9.0 - # Use "merge_requests/:merge_request_id/..." instead. - # params do - requires :merge_request_id, type: Integer, desc: 'The ID of a merge request' + requires :merge_request_iid, type: Integer, desc: 'The IID of a merge request' end - { ":id/merge_request/:merge_request_id" => :deprecated, ":id/merge_requests/:merge_request_id" => :ok }.each do |path, status| - desc 'Get a single merge request' do - if status == :deprecated - detail DEPRECATION_MESSAGE - end - success Entities::MergeRequest - end - get path do - merge_request = find_project_merge_request(params[:merge_request_id]) - authorize! :read_merge_request, merge_request - present merge_request, with: Entities::MergeRequest, current_user: current_user, project: user_project - end - - desc 'Get the commits of a merge request' do - success Entities::RepoCommit - end - get "#{path}/commits" do - merge_request = find_project_merge_request(params[:merge_request_id]) - authorize! :read_merge_request, merge_request - present merge_request.commits, with: Entities::RepoCommit - end - - desc 'Show the merge request changes' do - success Entities::MergeRequestChanges - end - get "#{path}/changes" do - merge_request = find_project_merge_request(params[:merge_request_id]) - authorize! :read_merge_request, merge_request - present merge_request, with: Entities::MergeRequestChanges, current_user: current_user - end - - desc 'Update a merge request' do - success Entities::MergeRequest - end - params do - optional :title, type: String, allow_blank: false, desc: 'The title of the merge request' - optional :target_branch, type: String, allow_blank: false, desc: 'The target branch' - optional :state_event, type: String, values: %w[close reopen merge], - desc: 'Status of the merge request' - use :optional_params - at_least_one_of :title, :target_branch, :description, :assignee_id, - :milestone_id, :labels, :state_event, - :remove_source_branch - end - put path do - merge_request = find_project_merge_request(params.delete(:merge_request_id)) - authorize! :update_merge_request, merge_request - - mr_params = declared_params(include_missing: false) - mr_params[:force_remove_source_branch] = mr_params.delete(:remove_source_branch) if mr_params[:remove_source_branch].present? - - merge_request = ::MergeRequests::UpdateService.new(user_project, current_user, mr_params).execute(merge_request) + desc 'Get a single merge request' do + success Entities::MergeRequest + end + get ':id/merge_requests/:merge_request_iid' do + merge_request = find_merge_request_with_access(params[:merge_request_iid]) - if merge_request.valid? - present merge_request, with: Entities::MergeRequest, current_user: current_user, project: user_project - else - handle_merge_request_errors! merge_request.errors - end - end + present merge_request, with: Entities::MergeRequest, current_user: current_user, project: user_project + end - desc 'Merge a merge request' do - success Entities::MergeRequest - end - params do - optional :merge_commit_message, type: String, desc: 'Custom merge commit message' - optional :should_remove_source_branch, type: Boolean, - desc: 'When true, the source branch will be deleted if possible' - optional :merge_when_build_succeeds, type: Boolean, - desc: 'When true, this merge request will be merged when the pipeline succeeds' - optional :sha, type: String, desc: 'When present, must have the HEAD SHA of the source branch' - end - put "#{path}/merge" do - merge_request = find_project_merge_request(params[:merge_request_id]) + desc 'Get the commits of a merge request' do + success Entities::RepoCommit + end + get ':id/merge_requests/:merge_request_iid/commits' do + merge_request = find_merge_request_with_access(params[:merge_request_iid]) + commits = ::Kaminari.paginate_array(merge_request.commits) - # Merge request can not be merged - # because user dont have permissions to push into target branch - unauthorized! unless merge_request.can_be_merged_by?(current_user) + present paginate(commits), with: Entities::RepoCommit + end - not_allowed! unless merge_request.mergeable_state? + desc 'Show the merge request changes' do + success Entities::MergeRequestChanges + end + get ':id/merge_requests/:merge_request_iid/changes' do + merge_request = find_merge_request_with_access(params[:merge_request_iid]) - render_api_error!('Branch cannot be merged', 406) unless merge_request.mergeable? + present merge_request, with: Entities::MergeRequestChanges, current_user: current_user + end - if params[:sha] && merge_request.diff_head_sha != params[:sha] - render_api_error!("SHA does not match HEAD of source branch: #{merge_request.diff_head_sha}", 409) - end + desc 'Update a merge request' do + success Entities::MergeRequest + end + params do + optional :title, type: String, allow_blank: false, desc: 'The title of the merge request' + optional :target_branch, type: String, allow_blank: false, desc: 'The target branch' + optional :state_event, type: String, values: %w[close reopen], + desc: 'Status of the merge request' + use :optional_params + at_least_one_of :title, :target_branch, :description, :assignee_id, + :milestone_id, :labels, :state_event, + :remove_source_branch + end + put ':id/merge_requests/:merge_request_iid' do + merge_request = find_merge_request_with_access(params.delete(:merge_request_iid), :update_merge_request) - merge_params = { - commit_message: params[:merge_commit_message], - should_remove_source_branch: params[:should_remove_source_branch] - } + mr_params = declared_params(include_missing: false) + mr_params[:force_remove_source_branch] = mr_params.delete(:remove_source_branch) if mr_params[:remove_source_branch].present? - if params[:merge_when_build_succeeds] && merge_request.head_pipeline && merge_request.head_pipeline.active? - ::MergeRequests::MergeWhenPipelineSucceedsService - .new(merge_request.target_project, current_user, merge_params) - .execute(merge_request) - else - ::MergeRequests::MergeService - .new(merge_request.target_project, current_user, merge_params) - .execute(merge_request) - end + merge_request = ::MergeRequests::UpdateService.new(user_project, current_user, mr_params).execute(merge_request) + if merge_request.valid? present merge_request, with: Entities::MergeRequest, current_user: current_user, project: user_project + else + handle_merge_request_errors! merge_request.errors end + end - desc 'Cancel merge if "Merge When Pipeline Succeeds" is enabled' do - success Entities::MergeRequest - end - post "#{path}/cancel_merge_when_build_succeeds" do - merge_request = find_project_merge_request(params[:merge_request_id]) + desc 'Merge a merge request' do + success Entities::MergeRequest + end + params do + optional :merge_commit_message, type: String, desc: 'Custom merge commit message' + optional :should_remove_source_branch, type: Boolean, + desc: 'When true, the source branch will be deleted if possible' + optional :merge_when_pipeline_succeeds, type: Boolean, + desc: 'When true, this merge request will be merged when the pipeline succeeds' + optional :sha, type: String, desc: 'When present, must have the HEAD SHA of the source branch' + end + put ':id/merge_requests/:merge_request_iid/merge' do + merge_request = find_project_merge_request(params[:merge_request_iid]) - unauthorized! unless merge_request.can_cancel_merge_when_build_succeeds?(current_user) + # Merge request can not be merged + # because user dont have permissions to push into target branch + unauthorized! unless merge_request.can_be_merged_by?(current_user) - ::MergeRequest::MergeWhenPipelineSucceedsService - .new(merge_request.target_project, current_user) - .cancel(merge_request) - end + not_allowed! unless merge_request.mergeable_state? - desc 'Get the comments of a merge request' do - detail 'Duplicate. DEPRECATED and WILL BE REMOVED in 9.0' - success Entities::MRNote - end - params do - use :pagination + render_api_error!('Branch cannot be merged', 406) unless merge_request.mergeable? + + if params[:sha] && merge_request.diff_head_sha != params[:sha] + render_api_error!("SHA does not match HEAD of source branch: #{merge_request.diff_head_sha}", 409) end - get "#{path}/comments" do - merge_request = find_project_merge_request(params[:merge_request_id]) - authorize! :read_merge_request, merge_request + merge_params = { + commit_message: params[:merge_commit_message], + should_remove_source_branch: params[:should_remove_source_branch] + } - present paginate(merge_request.notes.fresh), with: Entities::MRNote + if params[:merge_when_pipeline_succeeds] && merge_request.head_pipeline && merge_request.head_pipeline.active? + ::MergeRequests::MergeWhenPipelineSucceedsService + .new(merge_request.target_project, current_user, merge_params) + .execute(merge_request) + else + ::MergeRequests::MergeService + .new(merge_request.target_project, current_user, merge_params) + .execute(merge_request) end - desc 'Post a comment to a merge request' do - detail 'Duplicate. DEPRECATED and WILL BE REMOVED in 9.0' - success Entities::MRNote - end - params do - requires :note, type: String, desc: 'The text of the comment' - end - post "#{path}/comments" do - merge_request = find_project_merge_request(params[:merge_request_id]) - authorize! :create_note, merge_request + present merge_request, with: Entities::MergeRequest, current_user: current_user, project: user_project + end - opts = { - note: params[:note], - noteable_type: 'MergeRequest', - noteable_id: merge_request.id - } + desc 'Cancel merge if "Merge When Pipeline Succeeds" is enabled' do + success Entities::MergeRequest + end + post ':id/merge_requests/:merge_request_iid/cancel_merge_when_pipeline_succeeds' do + merge_request = find_project_merge_request(params[:merge_request_iid]) - note = ::Notes::CreateService.new(user_project, current_user, opts).execute + unauthorized! unless merge_request.can_cancel_merge_when_pipeline_succeeds?(current_user) - if note.save - present note, with: Entities::MRNote - else - render_api_error!("Failed to save note #{note.errors.messages}", 400) - end - end + ::MergeRequest::MergeWhenPipelineSucceedsService + .new(merge_request.target_project, current_user) + .cancel(merge_request) + end - desc 'List issues that will be closed on merge' do - success Entities::MRNote - end - params do - use :pagination - end - get "#{path}/closes_issues" do - merge_request = find_project_merge_request(params[:merge_request_id]) - issues = ::Kaminari.paginate_array(merge_request.closes_issues(current_user)) - present paginate(issues), with: issue_entity(user_project), current_user: current_user - end + desc 'List issues that will be closed on merge' do + success Entities::MRNote + end + params do + use :pagination + end + get ':id/merge_requests/:merge_request_iid/closes_issues' do + merge_request = find_merge_request_with_access(params[:merge_request_iid]) + issues = ::Kaminari.paginate_array(merge_request.closes_issues(current_user)) + present paginate(issues), with: issue_entity(user_project), current_user: current_user end end end diff --git a/lib/api/milestones.rb b/lib/api/milestones.rb index 3c373a84ec5..e7ab82f08db 100644 --- a/lib/api/milestones.rb +++ b/lib/api/milestones.rb @@ -23,14 +23,15 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects do + resource :projects, requirements: { id: %r{[^/]+} } do desc 'Get a list of project milestones' do success Entities::Milestone end params do optional :state, type: String, values: %w[active closed all], default: 'all', desc: 'Return "active", "closed", or "all" milestones' - optional :iid, type: Array[Integer], desc: 'The IID of the milestone' + optional :iids, type: Array[Integer], desc: 'The IIDs of the milestones' + optional :search, type: String, desc: 'The search criteria for the title or description of the milestone' use :pagination end get ":id/milestones" do @@ -38,7 +39,8 @@ module API milestones = user_project.milestones milestones = filter_milestones_state(milestones, params[:state]) - milestones = filter_by_iid(milestones, params[:iid]) if params[:iid].present? + milestones = filter_by_iid(milestones, params[:iids]) if params[:iids].present? + milestones = filter_by_search(milestones, params[:search]) if params[:search] present paginate(milestones), with: Entities::Milestone end @@ -101,7 +103,7 @@ module API end desc 'Get all issues for a single project milestone' do - success Entities::Issue + success Entities::IssueBasic end params do requires :milestone_id, type: Integer, desc: 'The ID of a project milestone' @@ -114,11 +116,38 @@ module API finder_params = { project_id: user_project.id, - milestone_title: milestone.title + milestone_title: milestone.title, + sort: 'position_asc' } issues = IssuesFinder.new(current_user, finder_params).execute - present paginate(issues), with: Entities::Issue, current_user: current_user, project: user_project + present paginate(issues), with: Entities::IssueBasic, current_user: current_user, project: user_project + end + + desc 'Get all merge requests for a single project milestone' do + detail 'This feature was introduced in GitLab 9.' + success Entities::MergeRequestBasic + end + params do + requires :milestone_id, type: Integer, desc: 'The ID of a project milestone' + use :pagination + end + get ':id/milestones/:milestone_id/merge_requests' do + authorize! :read_milestone, user_project + + milestone = user_project.milestones.find(params[:milestone_id]) + + finder_params = { + project_id: user_project.id, + milestone_id: milestone.id, + sort: 'position_asc' + } + + merge_requests = MergeRequestsFinder.new(current_user, finder_params).execute + present paginate(merge_requests), + with: Entities::MergeRequestBasic, + current_user: current_user, + project: user_project end end end diff --git a/lib/api/notes.rb b/lib/api/notes.rb index 284e4cf549a..de39e579ac3 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -4,12 +4,12 @@ module API before { authenticate! } - NOTEABLE_TYPES = [Issue, MergeRequest, Snippet] + NOTEABLE_TYPES = [Issue, MergeRequest, Snippet].freeze params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects do + resource :projects, requirements: { id: %r{[^/]+} } do NOTEABLE_TYPES.each do |noteable_type| noteables_str = noteable_type.to_s.underscore.pluralize @@ -21,7 +21,7 @@ module API use :pagination end get ":id/#{noteables_str}/:noteable_id/notes" do - noteable = user_project.send(noteables_str.to_sym).find(params[:noteable_id]) + noteable = find_project_noteable(noteables_str, params[:noteable_id]) if can?(current_user, noteable_read_ability_name(noteable), noteable) # We exclude notes that are cross-references and that cannot be viewed @@ -49,7 +49,7 @@ module API requires :noteable_id, type: Integer, desc: 'The ID of the noteable' end get ":id/#{noteables_str}/:noteable_id/notes/:note_id" do - noteable = user_project.send(noteables_str.to_sym).find(params[:noteable_id]) + noteable = find_project_noteable(noteables_str, params[:noteable_id]) note = noteable.notes.find(params[:note_id]) can_read_note = can?(current_user, noteable_read_ability_name(noteable), noteable) && !note.cross_reference_not_visible_for?(current_user) @@ -69,22 +69,28 @@ module API optional :created_at, type: String, desc: 'The creation date of the note' end post ":id/#{noteables_str}/:noteable_id/notes" do + noteable = find_project_noteable(noteables_str, params[:noteable_id]) + opts = { - note: params[:body], - noteable_type: noteables_str.classify, - noteable_id: params[:noteable_id] + note: params[:body], + noteable_type: noteables_str.classify, + noteable_id: noteable.id } - if params[:created_at] && (current_user.is_admin? || user_project.owner == current_user) - opts[:created_at] = params[:created_at] - end + if can?(current_user, noteable_read_ability_name(noteable), noteable) + if params[:created_at] && (current_user.is_admin? || user_project.owner == current_user) + opts[:created_at] = params[:created_at] + end - note = ::Notes::CreateService.new(user_project, current_user, opts).execute + note = ::Notes::CreateService.new(user_project, current_user, opts).execute - if note.valid? - present note, with: Entities::const_get(note.class.name) + if note.valid? + present note, with: Entities.const_get(note.class.name) + else + not_found!("Note #{note.errors.messages}") + end else - not_found!("Note #{note.errors.messages}") + not_found!("Note") end end @@ -125,14 +131,16 @@ module API note = user_project.notes.find(params[:note_id]) authorize! :admin_note, note - ::Notes::DeleteService.new(user_project, current_user).execute(note) - - present note, with: Entities::Note + ::Notes::DestroyService.new(user_project, current_user).execute(note) end end end helpers do + def find_project_noteable(noteables_str, noteable_id) + public_send("find_project_#{noteables_str.singularize}", noteable_id) + end + def noteable_read_ability_name(noteable) "read_#{noteable.class.to_s.underscore}".to_sym end diff --git a/lib/api/notification_settings.rb b/lib/api/notification_settings.rb index c5e9b3ad69b..992ea5dc24d 100644 --- a/lib/api/notification_settings.rb +++ b/lib/api/notification_settings.rb @@ -48,14 +48,14 @@ module API end %w[group project].each do |source_type| - resource source_type.pluralize do + params do + requires :id, type: String, desc: "The #{source_type} ID" + end + resource source_type.pluralize, requirements: { id: %r{[^/]+} } do desc "Get #{source_type} level notification level settings, defaults to Global" do detail 'This feature was introduced in GitLab 8.12' success Entities::NotificationSetting end - params do - requires :id, type: String, desc: 'The group ID or project ID or project NAMESPACE/PROJECT_NAME' - end get ":id/notification_settings" do source = find_source(source_type, params[:id]) @@ -69,7 +69,6 @@ module API success Entities::NotificationSetting end params do - requires :id, type: String, desc: 'The group ID or project ID or project NAMESPACE/PROJECT_NAME' optional :level, type: String, desc: "The #{source_type} notification level" NotificationSetting::EMAIL_EVENTS.each do |event| optional event, type: Boolean, desc: 'Enable/disable this notification' diff --git a/lib/api/pagination_params.rb b/lib/api/pagination_params.rb index 8c1e4381a74..f566eb3ed2b 100644 --- a/lib/api/pagination_params.rb +++ b/lib/api/pagination_params.rb @@ -15,8 +15,8 @@ module API included do helpers do params :pagination do - optional :page, type: Integer, desc: 'Current page number' - optional :per_page, type: Integer, desc: 'Number of items per page' + optional :page, type: Integer, default: 1, desc: 'Current page number' + optional :per_page, type: Integer, default: 20, desc: 'Number of items per page' end end end diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb index b634b1d0222..754c3d85a04 100644 --- a/lib/api/pipelines.rb +++ b/lib/api/pipelines.rb @@ -7,23 +7,23 @@ module API params do requires :id, type: String, desc: 'The project ID' end - resource :projects do + resource :projects, requirements: { id: %r{[^/]+} } do desc 'Get all Pipelines of the project' do detail 'This feature was introduced in GitLab 8.11.' - success Entities::Pipeline + success Entities::PipelineBasic end params do use :pagination - optional :scope, type: String, values: ['running', 'branches', 'tags'], + optional :scope, type: String, values: %w(running branches tags), desc: 'Either running, branches, or tags' end get ':id/pipelines' do authorize! :read_pipeline, user_project pipelines = PipelinesFinder.new(user_project).execute(scope: params[:scope]) - present paginate(pipelines), with: Entities::Pipeline + present paginate(pipelines), with: Entities::PipelineBasic end - + desc 'Create a new pipeline' do detail 'This feature was introduced in GitLab 8.14' success Entities::Pipeline @@ -58,7 +58,7 @@ module API present pipeline, with: Entities::Pipeline end - desc 'Retry failed builds in the pipeline' do + desc 'Retry builds in the pipeline' do detail 'This feature was introduced in GitLab 8.11.' success Entities::Pipeline end diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb index cb679e6658a..53791166c33 100644 --- a/lib/api/project_hooks.rb +++ b/lib/api/project_hooks.rb @@ -24,7 +24,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects do + resource :projects, requirements: { id: %r{[^/]+} } do desc 'Get project hooks' do success Entities::ProjectHook end @@ -32,9 +32,7 @@ module API use :pagination end get ":id/hooks" do - hooks = paginate user_project.hooks - - present hooks, with: Entities::ProjectHook + present paginate(user_project.hooks), with: Entities::ProjectHook end desc 'Get a project hook' do @@ -92,12 +90,9 @@ module API requires :hook_id, type: Integer, desc: 'The ID of the hook to delete' end delete ":id/hooks/:hook_id" do - begin - present user_project.hooks.destroy(params[:hook_id]), with: Entities::ProjectHook - rescue - # ProjectHook can raise Error if hook_id not found - not_found!("Error deleting hook #{params[:hook_id]}") - end + hook = user_project.hooks.find(params.delete(:hook_id)) + + hook.destroy end end end diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb index 9d8c5b63685..cfee38a9baf 100644 --- a/lib/api/project_snippets.rb +++ b/lib/api/project_snippets.rb @@ -7,7 +7,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects do + resource :projects, requirements: { id: %r{[^/]+} } do helpers do def handle_project_member_errors(errors) if errors[:project_access].any? @@ -50,19 +50,19 @@ module API requires :title, type: String, desc: 'The title of the snippet' requires :file_name, type: String, desc: 'The file name of the snippet' requires :code, type: String, desc: 'The content of the snippet' - requires :visibility_level, type: Integer, - values: [Gitlab::VisibilityLevel::PRIVATE, - Gitlab::VisibilityLevel::INTERNAL, - Gitlab::VisibilityLevel::PUBLIC], - desc: 'The visibility level of the snippet' + requires :visibility, type: String, + values: Gitlab::VisibilityLevel.string_values, + desc: 'The visibility of the snippet' end post ":id/snippets" do authorize! :create_project_snippet, user_project - snippet_params = declared_params + snippet_params = declared_params.merge(request: request, api: true) snippet_params[:content] = snippet_params.delete(:code) snippet = CreateSnippetService.new(user_project, current_user, snippet_params).execute + render_spam_error! if snippet.spam? + if snippet.persisted? present snippet, with: Entities::ProjectSnippet else @@ -78,11 +78,9 @@ module API optional :title, type: String, desc: 'The title of the snippet' optional :file_name, type: String, desc: 'The file name of the snippet' optional :code, type: String, desc: 'The content of the snippet' - optional :visibility_level, type: Integer, - values: [Gitlab::VisibilityLevel::PRIVATE, - Gitlab::VisibilityLevel::INTERNAL, - Gitlab::VisibilityLevel::PUBLIC], - desc: 'The visibility level of the snippet' + optional :visibility, type: String, + values: Gitlab::VisibilityLevel.string_values, + desc: 'The visibility of the snippet' at_least_one_of :title, :file_name, :code, :visibility_level end put ":id/snippets/:snippet_id" do @@ -92,12 +90,16 @@ module API authorize! :update_project_snippet, snippet snippet_params = declared_params(include_missing: false) + .merge(request: request, api: true) + snippet_params[:content] = snippet_params.delete(:code) if snippet_params[:code].present? UpdateSnippetService.new(user_project, current_user, snippet, snippet_params).execute - if snippet.persisted? + render_spam_error! if snippet.spam? + + if snippet.valid? present snippet, with: Entities::ProjectSnippet else render_validation_error!(snippet) diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 941f47114a4..0fbe1669d45 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -16,26 +16,12 @@ module API optional :shared_runners_enabled, type: Boolean, desc: 'Flag indication if shared runners are enabled for that project' optional :container_registry_enabled, type: Boolean, desc: 'Flag indication if the container registry is enabled for that project' optional :lfs_enabled, type: Boolean, desc: 'Flag indication if Git LFS is enabled for that project' - optional :public, type: Boolean, desc: 'Create a public project. The same as visibility_level = 20.' - optional :visibility_level, type: Integer, values: [ - Gitlab::VisibilityLevel::PRIVATE, - Gitlab::VisibilityLevel::INTERNAL, - Gitlab::VisibilityLevel::PUBLIC ], desc: 'Create a public project. The same as visibility_level = 20.' + optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The visibility of the project.' optional :public_builds, type: Boolean, desc: 'Perform public builds' optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access' - optional :only_allow_merge_if_build_succeeds, type: Boolean, desc: 'Only allow to merge if builds succeed' + optional :only_allow_merge_if_pipeline_succeeds, type: Boolean, desc: 'Only allow to merge if builds succeed' optional :only_allow_merge_if_all_discussions_are_resolved, type: Boolean, desc: 'Only allow to merge if all discussions are resolved' end - - def map_public_to_visibility_level(attrs) - publik = attrs.delete(:public) - if !publik.nil? && !attrs[:visibility_level].present? - # Since setting the public attribute to private could mean either - # private or internal, use the more conservative option, private. - attrs[:visibility_level] = (publik == true) ? Gitlab::VisibilityLevel::PUBLIC : Gitlab::VisibilityLevel::PRIVATE - end - attrs - end end resource :projects do @@ -58,9 +44,12 @@ module API params :filter_params do optional :archived, type: Boolean, default: false, desc: 'Limit by archived status' - optional :visibility, type: String, values: %w[public internal private], + optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'Limit by visibility' - optional :search, type: String, desc: 'Return list of authorized projects matching the search criteria' + optional :search, type: String, desc: 'Return list of projects matching the search criteria' + optional :owned, type: Boolean, default: false, desc: 'Limit by owned by authenticated user' + optional :starred, type: Boolean, default: false, desc: 'Limit by starred status' + optional :membership, type: Boolean, default: false, desc: 'Limit by projects that the current user is a member of' end params :statistics_params do @@ -93,91 +82,23 @@ module API params do use :collection_params end - get '/visible' do - entity = current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails - present_projects ProjectsFinder.new.execute(current_user), with: entity - end - - desc 'Get a projects list for authenticated user' do - success Entities::BasicProjectDetails - end - params do - use :collection_params - end get do - authenticate! - - present_projects current_user.authorized_projects, - with: Entities::ProjectWithAccess - end - - desc 'Get an owned projects list for authenticated user' do - success Entities::BasicProjectDetails - end - params do - use :collection_params - use :statistics_params - end - get '/owned' do - authenticate! - - present_projects current_user.owned_projects, - with: Entities::ProjectWithAccess, - statistics: params[:statistics] - end - - desc 'Gets starred project for the authenticated user' do - success Entities::BasicProjectDetails - end - params do - use :collection_params - end - get '/starred' do - authenticate! - - present_projects current_user.viewable_starred_projects - end - - desc 'Get all projects for admin user' do - success Entities::BasicProjectDetails - end - params do - use :collection_params - use :statistics_params - end - get '/all' do - authenticated_as_admin! - - present_projects Project.all, with: Entities::ProjectWithAccess, statistics: params[:statistics] - end - - desc 'Search for projects the current user has access to' do - success Entities::Project - end - params do - requires :query, type: String, desc: 'The project name to be searched' - use :sort_params - use :pagination - end - get "/search/:query", requirements: { query: /[^\/]+/ } do - search_service = Search::GlobalService.new(current_user, search: params[:query]).execute - projects = search_service.objects('projects', params[:page]) - projects = projects.reorder(params[:order_by] => params[:sort]) - - present paginate(projects), with: Entities::Project + entity = current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails + present_projects ProjectsFinder.new.execute(current_user), with: entity, statistics: params[:statistics] end desc 'Create new project' do success Entities::Project end params do - requires :name, type: String, desc: 'The name of the project' + optional :name, type: String, desc: 'The name of the project' optional :path, type: String, desc: 'The path of the repository' + at_least_one_of :name, :path use :optional_params use :create_params end post do - attrs = map_public_to_visibility_level(declared_params(include_missing: false)) + attrs = declared_params(include_missing: false) project = ::Projects::CreateService.new(current_user, attrs).execute if project.saved? @@ -206,7 +127,7 @@ module API user = User.find_by(id: params.delete(:user_id)) not_found!('User') unless user - attrs = map_public_to_visibility_level(declared_params(include_missing: false)) + attrs = declared_params(include_missing: false) project = ::Projects::CreateService.new(user, attrs).execute if project.saved? @@ -221,7 +142,7 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects, requirements: { id: /[^\/]+/ } do + resource :projects, requirements: { id: %r{[^/]+} } do desc 'Get a single project' do success Entities::ProjectWithAccess end @@ -247,7 +168,7 @@ module API params do optional :namespace, type: String, desc: 'The ID or name of the namespace that the project will be forked into' end - post 'fork/:id' do + post ':id/fork' do fork_params = declared_params(include_missing: false) namespace_id = fork_params[:namespace] @@ -284,16 +205,16 @@ module API at_least_one_of :name, :description, :issues_enabled, :merge_requests_enabled, :wiki_enabled, :builds_enabled, :snippets_enabled, :shared_runners_enabled, :container_registry_enabled, - :lfs_enabled, :public, :visibility_level, :public_builds, - :request_access_enabled, :only_allow_merge_if_build_succeeds, + :lfs_enabled, :visibility, :public_builds, + :request_access_enabled, :only_allow_merge_if_pipeline_succeeds, :only_allow_merge_if_all_discussions_are_resolved, :path, :default_branch end put ':id' do authorize_admin_project - attrs = map_public_to_visibility_level(declared_params(include_missing: false)) + attrs = declared_params(include_missing: false) authorize! :rename_project, user_project if attrs[:name].present? - authorize! :change_visibility_level, user_project if attrs[:visibility_level].present? + authorize! :change_visibility_level, user_project if attrs[:visibility].present? result = ::Projects::UpdateService.new(user_project, current_user, attrs).execute @@ -344,7 +265,7 @@ module API desc 'Unstar a project' do success Entities::Project end - delete ':id/star' do + post ':id/unstar' do if current_user.starred?(user_project) current_user.toggle_star(user_project) user_project.reload @@ -359,6 +280,8 @@ module API delete ":id" do authorize! :remove_project, user_project ::Projects::DestroyService.new(user_project, current_user, {}).async_execute + + accepted! end desc 'Mark this project as forked from another' @@ -428,7 +351,6 @@ module API not_found!('Group Link') unless link link.destroy - no_content! end desc 'Upload a file' @@ -452,6 +374,19 @@ module API present paginate(users), with: Entities::UserBasic end + + desc 'Start the housekeeping task for a project' do + detail 'This feature was introduced in GitLab 9.0.' + end + post ':id/housekeeping' do + authorize_admin_project + + begin + ::Projects::HousekeepingService.new(user_project).execute + rescue ::Projects::HousekeepingService::LeaseTaken => error + conflict!(error.message) + end + end end end end diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index 4ca6646a6f1..8f16e532ecb 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -2,12 +2,14 @@ require 'mime/types' module API class Repositories < Grape::API + include PaginationParams + before { authorize! :download_code, user_project } params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects do + resource :projects, requirements: { id: %r{[^/]+} } do helpers do def handle_project_member_errors(errors) if errors[:project_access].any? @@ -15,61 +17,67 @@ module API end not_found! end + + def assign_blob_vars! + authorize! :download_code, user_project + + @repo = user_project.repository + + begin + @blob = Gitlab::Git::Blob.raw(@repo, params[:sha]) + @blob.load_all_data!(@repo) + rescue + not_found! 'Blob' + end + + not_found! 'Blob' unless @blob + end end desc 'Get a project repository tree' do success Entities::RepoTreeObject end params do - optional :ref_name, type: String, desc: 'The name of a repository branch or tag, if not given the default branch is used' + optional :ref, type: String, desc: 'The name of a repository branch or tag, if not given the default branch is used' optional :path, type: String, desc: 'The path of the tree' optional :recursive, type: Boolean, default: false, desc: 'Used to get a recursive tree' + use :pagination end get ':id/repository/tree' do - ref = params[:ref_name] || user_project.try(:default_branch) || 'master' + ref = params[:ref] || user_project.try(:default_branch) || 'master' path = params[:path] || nil commit = user_project.commit(ref) not_found!('Tree') unless commit tree = user_project.repository.tree(commit.id, path, recursive: params[:recursive]) - - present tree.sorted_entries, with: Entities::RepoTreeObject + entries = ::Kaminari.paginate_array(tree.sorted_entries) + present paginate(entries), with: Entities::RepoTreeObject end - desc 'Get a raw file contents' + desc 'Get raw blob contents from the repository' params do requires :sha, type: String, desc: 'The commit, branch name, or tag name' - requires :filepath, type: String, desc: 'The path to the file to display' end - get [ ":id/repository/blobs/:sha", ":id/repository/commits/:sha/blob" ] do - repo = user_project.repository - - commit = repo.commit(params[:sha]) - not_found! "Commit" unless commit + get ':id/repository/blobs/:sha/raw' do + assign_blob_vars! - blob = Gitlab::Git::Blob.find(repo, commit.id, params[:filepath]) - not_found! "File" unless blob - - send_git_blob repo, blob + send_git_blob @repo, @blob end - desc 'Get a raw blob contents by blob sha' + desc 'Get a blob from the repository' params do requires :sha, type: String, desc: 'The commit, branch name, or tag name' end - get ':id/repository/raw_blobs/:sha' do - repo = user_project.repository - - begin - blob = Gitlab::Git::Blob.raw(repo, params[:sha]) - rescue - not_found! 'Blob' - end - - not_found! 'Blob' unless blob - - send_git_blob repo, blob + get ':id/repository/blobs/:sha' do + assign_blob_vars! + + { + size: @blob.size, + encoding: "base64", + content: Base64.strict_encode64(@blob.data), + sha: @blob.id + } end desc 'Get an archive of the repository' @@ -100,10 +108,13 @@ module API desc 'Get repository contributors' do success Entities::Contributor end + params do + use :pagination + end get ':id/repository/contributors' do begin - present user_project.repository.contributors, - with: Entities::Contributor + contributors = ::Kaminari.paginate_array(user_project.repository.contributors) + present paginate(contributors), with: Entities::Contributor rescue not_found! end diff --git a/lib/api/runner.rb b/lib/api/runner.rb new file mode 100644 index 00000000000..4c9db2c8716 --- /dev/null +++ b/lib/api/runner.rb @@ -0,0 +1,264 @@ +module API + class Runner < Grape::API + helpers ::API::Helpers::Runner + + resource :runners do + desc 'Registers a new Runner' do + success Entities::RunnerRegistrationDetails + http_codes [[201, 'Runner was created'], [403, 'Forbidden']] + end + params do + requires :token, type: String, desc: 'Registration token' + optional :description, type: String, desc: %q(Runner's description) + optional :info, type: Hash, desc: %q(Runner's metadata) + optional :locked, type: Boolean, desc: 'Should Runner be locked for current project' + optional :run_untagged, type: Boolean, desc: 'Should Runner handle untagged jobs' + optional :tag_list, type: Array[String], desc: %q(List of Runner's tags) + end + post '/' do + attributes = attributes_for_keys [:description, :locked, :run_untagged, :tag_list] + + runner = + if runner_registration_token_valid? + # Create shared runner. Requires admin access + Ci::Runner.create(attributes.merge(is_shared: true)) + elsif project = Project.find_by(runners_token: params[:token]) + # Create a specific runner for project. + project.runners.create(attributes) + end + + return forbidden! unless runner + + if runner.id + runner.update(get_runner_version_from_params) + present runner, with: Entities::RunnerRegistrationDetails + else + not_found! + end + end + + desc 'Deletes a registered Runner' do + http_codes [[204, 'Runner was deleted'], [403, 'Forbidden']] + end + params do + requires :token, type: String, desc: %q(Runner's authentication token) + end + delete '/' do + authenticate_runner! + Ci::Runner.find_by_token(params[:token]).destroy + end + + desc 'Validates authentication credentials' do + http_codes [[200, 'Credentials are valid'], [403, 'Forbidden']] + end + params do + requires :token, type: String, desc: %q(Runner's authentication token) + end + post '/verify' do + authenticate_runner! + status 200 + end + end + + resource :jobs do + desc 'Request a job' do + success Entities::JobRequest::Response + http_codes [[201, 'Job was scheduled'], + [204, 'No job for Runner'], + [403, 'Forbidden']] + end + params do + requires :token, type: String, desc: %q(Runner's authentication token) + optional :last_update, type: String, desc: %q(Runner's queue last_update token) + optional :info, type: Hash, desc: %q(Runner's metadata) + end + post '/request' do + authenticate_runner! + no_content! unless current_runner.active? + update_runner_info + + if current_runner.is_runner_queue_value_latest?(params[:last_update]) + header 'X-GitLab-Last-Update', params[:last_update] + Gitlab::Metrics.add_event(:build_not_found_cached) + return no_content! + end + + new_update = current_runner.ensure_runner_queue_value + result = ::Ci::RegisterJobService.new(current_runner).execute + + if result.valid? + if result.build + Gitlab::Metrics.add_event(:build_found, + project: result.build.project.path_with_namespace) + present result.build, with: Entities::JobRequest::Response + else + Gitlab::Metrics.add_event(:build_not_found) + header 'X-GitLab-Last-Update', new_update + no_content! + end + else + # We received build that is invalid due to concurrency conflict + Gitlab::Metrics.add_event(:build_invalid) + conflict! + end + end + + desc 'Updates a job' do + http_codes [[200, 'Job was updated'], [403, 'Forbidden']] + end + params do + requires :token, type: String, desc: %q(Runners's authentication token) + requires :id, type: Integer, desc: %q(Job's ID) + optional :trace, type: String, desc: %q(Job's full trace) + optional :state, type: String, desc: %q(Job's status: success, failed) + end + put '/:id' do + job = Ci::Build.find_by_id(params[:id]) + authenticate_job!(job) + + job.update_attributes(trace: params[:trace]) if params[:trace] + + Gitlab::Metrics.add_event(:update_build, + project: job.project.path_with_namespace) + + case params[:state].to_s + when 'success' + job.success + when 'failed' + job.drop + end + end + + desc 'Appends a patch to the job trace' do + http_codes [[202, 'Trace was patched'], + [400, 'Missing Content-Range header'], + [403, 'Forbidden'], + [416, 'Range not satisfiable']] + end + params do + requires :id, type: Integer, desc: %q(Job's ID) + optional :token, type: String, desc: %q(Job's authentication token) + end + patch '/:id/trace' do + job = Ci::Build.find_by_id(params[:id]) + authenticate_job!(job) + + error!('400 Missing header Content-Range', 400) unless request.headers.has_key?('Content-Range') + content_range = request.headers['Content-Range'] + content_range = content_range.split('-') + + current_length = job.trace_length + unless current_length == content_range[0].to_i + return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{current_length}" }) + end + + job.append_trace(request.body.read, content_range[0].to_i) + + status 202 + header 'Job-Status', job.status + header 'Range', "0-#{job.trace_length}" + end + + desc 'Authorize artifacts uploading for job' do + http_codes [[200, 'Upload allowed'], + [403, 'Forbidden'], + [405, 'Artifacts support not enabled'], + [413, 'File too large']] + end + params do + requires :id, type: Integer, desc: %q(Job's ID) + optional :token, type: String, desc: %q(Job's authentication token) + optional :filesize, type: Integer, desc: %q(Artifacts filesize) + end + post '/:id/artifacts/authorize' do + not_allowed! unless Gitlab.config.artifacts.enabled + require_gitlab_workhorse! + Gitlab::Workhorse.verify_api_request!(headers) + + job = Ci::Build.find_by_id(params[:id]) + authenticate_job!(job) + forbidden!('Job is not running') unless job.running? + + if params[:filesize] + file_size = params[:filesize].to_i + file_to_large! unless file_size < max_artifacts_size + end + + status 200 + content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE + Gitlab::Workhorse.artifact_upload_ok + end + + desc 'Upload artifacts for job' do + success Entities::JobRequest::Response + http_codes [[201, 'Artifact uploaded'], + [400, 'Bad request'], + [403, 'Forbidden'], + [405, 'Artifacts support not enabled'], + [413, 'File too large']] + end + params do + requires :id, type: Integer, desc: %q(Job's ID) + optional :token, type: String, desc: %q(Job's authentication token) + optional :expire_in, type: String, desc: %q(Specify when artifacts should expire) + optional :file, type: File, desc: %q(Artifact's file) + optional 'file.path', type: String, desc: %q(path to locally stored body (generated by Workhorse)) + optional 'file.name', type: String, desc: %q(real filename as send in Content-Disposition (generated by Workhorse)) + optional 'file.type', type: String, desc: %q(real content type as send in Content-Type (generated by Workhorse)) + optional 'metadata.path', type: String, desc: %q(path to locally stored body (generated by Workhorse)) + optional 'metadata.name', type: String, desc: %q(filename (generated by Workhorse)) + end + post '/:id/artifacts' do + not_allowed! unless Gitlab.config.artifacts.enabled + require_gitlab_workhorse! + + job = Ci::Build.find_by_id(params[:id]) + authenticate_job!(job) + forbidden!('Job is not running!') unless job.running? + + artifacts_upload_path = ArtifactUploader.artifacts_upload_path + artifacts = uploaded_file(:file, artifacts_upload_path) + metadata = uploaded_file(:metadata, artifacts_upload_path) + + bad_request!('Missing artifacts file!') unless artifacts + file_to_large! unless artifacts.size < max_artifacts_size + + job.artifacts_file = artifacts + job.artifacts_metadata = metadata + job.artifacts_expire_in = params['expire_in'] || + Gitlab::CurrentSettings.current_application_settings.default_artifacts_expire_in + + if job.save + present job, with: Entities::JobRequest::Response + else + render_validation_error!(job) + end + end + + desc 'Download the artifacts file for job' do + http_codes [[200, 'Upload allowed'], + [403, 'Forbidden'], + [404, 'Artifact not found']] + end + params do + requires :id, type: Integer, desc: %q(Job's ID) + optional :token, type: String, desc: %q(Job's authentication token) + end + get '/:id/artifacts' do + job = Ci::Build.find_by_id(params[:id]) + authenticate_job!(job) + + artifacts_file = job.artifacts_file + unless artifacts_file.file_storage? + return redirect_to job.artifacts_file.url + end + + unless artifacts_file.exists? + not_found! + end + + present_file!(artifacts_file.path, artifacts_file.filename) + end + end + end +end diff --git a/lib/api/runners.rb b/lib/api/runners.rb index 4816b5ed1b7..a77c876a749 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -14,7 +14,7 @@ module API use :pagination end get do - runners = filter_runners(current_user.ci_authorized_runners, params[:scope], without: ['specific', 'shared']) + runners = filter_runners(current_user.ci_authorized_runners, params[:scope], without: %w(specific shared)) present paginate(runners), with: Entities::Runner end @@ -60,8 +60,9 @@ module API put ':id' do runner = get_runner(params.delete(:id)) authenticate_update_runner!(runner) + update_service = Ci::UpdateRunnerService.new(runner) - if runner.update(declared_params(include_missing: false)) + if update_service.update(declared_params(include_missing: false)) present runner, with: Entities::RunnerDetails, current_user: current_user else render_validation_error!(runner) @@ -77,16 +78,15 @@ module API delete ':id' do runner = get_runner(params[:id]) authenticate_delete_runner!(runner) - runner.destroy! - present runner, with: Entities::Runner + runner.destroy! end end params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects do + resource :projects, requirements: { id: %r{[^/]+} } do before { authorize_admin_project } desc 'Get runners available for project' do @@ -135,8 +135,6 @@ module API forbidden!("Only one project associated with the runner. Please remove the runner instead") if runner.projects.count == 1 runner_project.destroy - - present runner, with: Entities::Runner end end diff --git a/lib/api/services.rb b/lib/api/services.rb index 3a9dfbb237c..4e0c9cb1f63 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -107,26 +107,6 @@ module API desc: 'Enable SSL verification for communication' } ], - 'builds-email' => [ - { - required: true, - name: :recipients, - type: String, - desc: 'Comma-separated list of recipient email addresses' - }, - { - required: false, - name: :add_pusher, - type: Boolean, - desc: 'Add pusher to recipients list' - }, - { - required: false, - name: :notify_only_broken_builds, - type: Boolean, - desc: 'Notify only broken builds' - } - ], 'campfire' => [ { required: true, @@ -145,7 +125,7 @@ module API name: :room, type: String, desc: 'Campfire room' - }, + } ], 'custom-issue-tracker' => [ { @@ -403,9 +383,9 @@ module API }, { required: false, - name: :notify_only_broken_builds, + name: :notify_only_broken_pipelines, type: Boolean, - desc: 'Notify only broken builds' + desc: 'Notify only broken pipelines' } ], 'pivotaltracker' => [ @@ -422,6 +402,14 @@ module API desc: 'Comma-separated list of branches which will be automatically inspected. Leave blank to include all branches.' } ], + 'prometheus' => [ + { + required: true, + name: :api_url, + type: String, + desc: 'Prometheus API Base URL, like http://prometheus.example.com/' + } + ], 'pushover' => [ { required: true, @@ -534,7 +522,49 @@ module API desc: 'The password of the user' } ] - }.freeze + } + + service_classes = [ + AsanaService, + AssemblaService, + BambooService, + BugzillaService, + BuildkiteService, + CampfireService, + CustomIssueTrackerService, + DroneCiService, + EmailsOnPushService, + ExternalWikiService, + FlowdockService, + GemnasiumService, + HipchatService, + IrkerService, + JiraService, + KubernetesService, + MattermostSlashCommandsService, + SlackSlashCommandsService, + PipelinesEmailService, + PivotaltrackerService, + PrometheusService, + PushoverService, + RedmineService, + SlackService, + MattermostService, + TeamcityService, + ] + + if Rails.env.development? + services['mock-ci'] = [ + { + required: true, + name: :mock_service_url, + type: String, + desc: 'URL to the mock service' + } + ] + + service_classes << MockCiService + end trigger_services = { 'mattermost-slash-commands' => [ @@ -553,7 +583,10 @@ module API ] }.freeze - resource :projects do + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: { id: %r{[^/]+} } do before { authenticate! } before { authorize_admin_project } @@ -568,6 +601,19 @@ module API services.each do |service_slug, settings| desc "Set #{service_slug} service for project" params do + service_classes.each do |service| + event_names = service.try(:event_names) || next + event_names.each do |event_name| + services[service.to_param.tr("_", "-")] << { + required: false, + name: event_name.to_sym, + type: String, + desc: ServicesHelper.service_event_description(event_name) + } + end + end + services.freeze + settings.each do |setting| if setting[:required] requires setting[:name], type: setting[:type], desc: setting[:desc] @@ -581,7 +627,7 @@ module API service_params = declared_params(include_missing: false).merge(active: true) if service.update_attributes(service_params) - true + present service, with: Entities::ProjectService, include_passwords: current_user.is_admin? else render_api_error!('400 Bad Request', 400) end @@ -599,9 +645,7 @@ module API hash.merge!(key => nil) end - if service.update_attributes(attrs.merge(active: false)) - true - else + unless service.update_attributes(attrs.merge(active: false)) render_api_error!('400 Bad Request', 400) end end @@ -619,10 +663,18 @@ module API end trigger_services.each do |service_slug, settings| + helpers do + def chat_command_service(project, service_slug, params) + project.services.active.where(template: false).find do |service| + service.try(:token) == params[:token] && service.to_param == service_slug.underscore + end + end + end + params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects do + resource :projects, requirements: { id: %r{[^/]+} } do desc "Trigger a slash command for #{service_slug}" do detail 'Added in GitLab 8.13' end @@ -637,9 +689,8 @@ module API # This is not accurate, but done to prevent leakage of the project names not_found!('Service') unless project - service = project.find_or_initialize_service(service_slug.underscore) - - result = service.try(:active?) && service.try(:trigger, params) + service = chat_command_service(project, service_slug, params) + result = service.try(:trigger, params) if result status result[:status] || 200 diff --git a/lib/api/settings.rb b/lib/api/settings.rb index c5eff16a5de..c7f97ad2aab 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -21,9 +21,9 @@ module API end params do optional :default_branch_protection, type: Integer, values: [0, 1, 2], desc: 'Determine if developers can push to master' - optional :default_project_visibility, type: Integer, values: Gitlab::VisibilityLevel.values, desc: 'The default project visibility' - optional :default_snippet_visibility, type: Integer, values: Gitlab::VisibilityLevel.values, desc: 'The default snippet visibility' - optional :default_group_visibility, type: Integer, values: Gitlab::VisibilityLevel.values, desc: 'The default group visibility' + optional :default_project_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default project visibility' + optional :default_snippet_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default snippet visibility' + optional :default_group_visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'The default group visibility' optional :restricted_visibility_levels, type: Array[String], desc: 'Selected levels cannot be used by non-admin users for projects or snippets. If the public level is restricted, user profiles are only visible to logged in users.' optional :import_sources, type: Array[String], values: %w[github bitbucket gitlab google_code fogbugz git gitlab_project], desc: 'Enabled sources for code import during project creation. OmniAuth must be configured for GitHub, Bitbucket, and GitLab.com' @@ -56,7 +56,9 @@ module API given shared_runners_enabled: ->(val) { val } do requires :shared_runners_text, type: String, desc: 'Shared runners text ' end - optional :max_artifacts_size, type: Integer, desc: "Set the maximum file size each build's artifacts can have" + optional :max_artifacts_size, type: Integer, desc: "Set the maximum file size for each job's artifacts" + optional :default_artifacts_expire_in, type: String, desc: "Set the default expiration time for each job's artifacts" + optional :max_pages_size, type: Integer, desc: 'Maximum size of pages in MB' optional :container_registry_token_expire_delay, type: Integer, desc: 'Authorization token duration (minutes)' optional :metrics_enabled, type: Boolean, desc: 'Enable the InfluxDB metrics' given metrics_enabled: ->(val) { val } do @@ -107,6 +109,8 @@ module API requires :housekeeping_full_repack_period, type: Integer, desc: "Number of Git pushes after which a full 'git repack' is run." requires :housekeeping_gc_period, type: Integer, desc: "Number of Git pushes after which 'git gc' is run." end + optional :terminal_max_session_time, type: Integer, desc: 'Maximum time for web terminal websocket connection (in seconds). Set to 0 for unlimited time.' + optional :polling_interval_multiplier, type: BigDecimal, desc: 'Interval multiplier used by endpoints that perform polling. Set to 0 to disable polling.' at_least_one_of :default_branch_protection, :default_project_visibility, :default_snippet_visibility, :default_group_visibility, :restricted_visibility_levels, :import_sources, :enabled_git_access_protocol, :gravatar_enabled, :default_projects_limit, @@ -115,15 +119,19 @@ module API :send_user_confirmation_email, :domain_whitelist, :domain_blacklist_enabled, :after_sign_up_text, :signin_enabled, :require_two_factor_authentication, :home_page_url, :after_sign_out_path, :sign_in_text, :help_page_text, - :shared_runners_enabled, :max_artifacts_size, :container_registry_token_expire_delay, + :shared_runners_enabled, :max_artifacts_size, + :default_artifacts_expire_in, :max_pages_size, + :container_registry_token_expire_delay, :metrics_enabled, :sidekiq_throttling_enabled, :recaptcha_enabled, :akismet_enabled, :admin_notification_email, :sentry_enabled, :repository_storage, :repository_checks_enabled, :koding_enabled, :plantuml_enabled, :version_check_enabled, :email_author_in_body, :html_emails_enabled, - :housekeeping_enabled + :housekeeping_enabled, :terminal_max_session_time, :polling_interval_multiplier end put "application/settings" do - if current_settings.update_attributes(declared_params(include_missing: false)) + attrs = declared_params(include_missing: false) + + if current_settings.update_attributes(attrs) present current_settings, with: Entities::ApplicationSetting else render_validation_error!(current_settings) diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb index e096e636806..b93fdc62808 100644 --- a/lib/api/snippets.rb +++ b/lib/api/snippets.rb @@ -58,15 +58,17 @@ module API requires :title, type: String, desc: 'The title of a snippet' requires :file_name, type: String, desc: 'The name of a snippet file' requires :content, type: String, desc: 'The content of a snippet' - optional :visibility_level, type: Integer, - values: Gitlab::VisibilityLevel.values, - default: Gitlab::VisibilityLevel::INTERNAL, - desc: 'The visibility level of the snippet' + optional :visibility, type: String, + values: Gitlab::VisibilityLevel.string_values, + default: 'internal', + desc: 'The visibility of the snippet' end post do - attrs = declared_params(include_missing: false) + attrs = declared_params(include_missing: false).merge(request: request, api: true) snippet = CreateSnippetService.new(nil, current_user, attrs).execute + render_spam_error! if snippet.spam? + if snippet.persisted? present snippet, with: Entities::PersonalSnippet else @@ -83,19 +85,22 @@ module API optional :title, type: String, desc: 'The title of a snippet' optional :file_name, type: String, desc: 'The name of a snippet file' optional :content, type: String, desc: 'The content of a snippet' - optional :visibility_level, type: Integer, - values: Gitlab::VisibilityLevel.values, - desc: 'The visibility level of the snippet' - at_least_one_of :title, :file_name, :content, :visibility_level + optional :visibility, type: String, + values: Gitlab::VisibilityLevel.string_values, + desc: 'The visibility of the snippet' + at_least_one_of :title, :file_name, :content, :visibility end put ':id' do snippet = snippets_for_current_user.find_by(id: params.delete(:id)) return not_found!('Snippet') unless snippet authorize! :update_personal_snippet, snippet - attrs = declared_params(include_missing: false) + attrs = declared_params(include_missing: false).merge(request: request, api: true) UpdateSnippetService.new(nil, current_user, snippet, attrs).execute + + render_spam_error! if snippet.spam? + if snippet.persisted? present snippet, with: Entities::PersonalSnippet else @@ -113,9 +118,10 @@ module API delete ':id' do snippet = snippets_for_current_user.find_by(id: params.delete(:id)) return not_found!('Snippet') unless snippet + authorize! :destroy_personal_snippet, snippet + snippet.destroy - no_content! end desc 'Get a raw snippet' do diff --git a/lib/api/subscriptions.rb b/lib/api/subscriptions.rb index 10749b34004..dbe54d3cd31 100644 --- a/lib/api/subscriptions.rb +++ b/lib/api/subscriptions.rb @@ -3,8 +3,7 @@ module API before { authenticate! } subscribable_types = { - 'merge_request' => proc { |id| user_project.merge_requests.find(id) }, - 'merge_requests' => proc { |id| user_project.merge_requests.find(id) }, + 'merge_requests' => proc { |id| find_merge_request_with_access(id, :update_merge_request) }, 'issues' => proc { |id| find_project_issue(id) }, 'labels' => proc { |id| find_project_label(id) }, } @@ -13,7 +12,7 @@ module API requires :id, type: String, desc: 'The ID of a project' requires :subscribable_id, type: String, desc: 'The ID of a resource' end - resource :projects do + resource :projects, requirements: { id: %r{[^/]+} } do subscribable_types.each do |type, finder| type_singularized = type.singularize entity_class = Entities.const_get(type_singularized.camelcase) @@ -21,7 +20,7 @@ module API desc 'Subscribe to a resource' do success entity_class end - post ":id/#{type}/:subscribable_id/subscription" do + post ":id/#{type}/:subscribable_id/subscribe" do resource = instance_exec(params[:subscribable_id], &finder) if resource.subscribed?(current_user, user_project) @@ -35,7 +34,7 @@ module API desc 'Unsubscribe from a resource' do success entity_class end - delete ":id/#{type}/:subscribable_id/subscription" do + post ":id/#{type}/:subscribable_id/unsubscribe" do resource = instance_exec(params[:subscribable_id], &finder) if !resource.subscribed?(current_user, user_project) diff --git a/lib/api/system_hooks.rb b/lib/api/system_hooks.rb index 708ec8cfe70..ed7b23b474a 100644 --- a/lib/api/system_hooks.rb +++ b/lib/api/system_hooks.rb @@ -1,6 +1,7 @@ module API - # Hooks API class SystemHooks < Grape::API + include PaginationParams + before do authenticate! authenticated_as_admin! @@ -10,10 +11,11 @@ module API desc 'Get the list of system hooks' do success Entities::Hook end + params do + use :pagination + end get do - hooks = SystemHook.all - - present hooks, with: Entities::Hook + present paginate(SystemHook.all), with: Entities::Hook end desc 'Create a new system hook' do @@ -64,7 +66,7 @@ module API hook = SystemHook.find_by(id: params[:id]) not_found!('System hook') unless hook - present hook.destroy, with: Entities::Hook + hook.destroy end end end diff --git a/lib/api/tags.rb b/lib/api/tags.rb index 5b345db3a41..c7b1efe0bfa 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -1,18 +1,22 @@ module API - # Git Tags API class Tags < Grape::API + include PaginationParams + before { authorize! :download_code, user_project } params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects do + resource :projects, requirements: { id: %r{[^/]+} } do desc 'Get a project repository tags' do success Entities::RepoTag end + params do + use :pagination + end get ":id/repository/tags" do - present user_project.repository.tags.sort_by(&:name).reverse, - with: Entities::RepoTag, project: user_project + tags = ::Kaminari.paginate_array(user_project.repository.tags.sort_by(&:name).reverse) + present paginate(tags), with: Entities::RepoTag, project: user_project end desc 'Get a single repository tag' do @@ -40,7 +44,7 @@ module API post ':id/repository/tags' do authorize_push_project - result = CreateTagService.new(user_project, current_user). + result = ::Tags::CreateService.new(user_project, current_user). execute(params[:tag_name], params[:ref], params[:message], params[:release_description]) if result[:status] == :success @@ -59,14 +63,10 @@ module API delete ":id/repository/tags/:tag_name", requirements: { tag_name: /.+/ } do authorize_push_project - result = DeleteTagService.new(user_project, current_user). + result = ::Tags::DestroyService.new(user_project, current_user). execute(params[:tag_name]) - if result[:status] == :success - { - tag_name: params[:tag_name] - } - else + if result[:status] != :success render_api_error!(result[:message], result[:return_code]) end end diff --git a/lib/api/templates.rb b/lib/api/templates.rb index e23f99256a5..0fc13b35d5b 100644 --- a/lib/api/templates.rb +++ b/lib/api/templates.rb @@ -1,5 +1,7 @@ module API class Templates < Grape::API + include PaginationParams + GLOBAL_TEMPLATE_TYPES = { gitignores: { klass: Gitlab::Template::GitignoreTemplate, @@ -24,7 +26,6 @@ module API /[\<\{\[] (fullname|name\sof\s(author|copyright\sowner)) [\>\}\]]/xi.freeze - DEPRECATION_MESSAGE = ' This endpoint is deprecated and will be removed in GitLab 9.0.'.freeze helpers do def parsed_license_template @@ -46,74 +47,64 @@ module API end end - { "licenses" => :deprecated, "templates/licenses" => :ok }.each do |route, status| - desc 'Get the list of the available license template' do - detailed_desc = 'This feature was introduced in GitLab 8.7.' - detailed_desc << DEPRECATION_MESSAGE unless status == :ok - detail detailed_desc - success Entities::RepoLicense - end - params do - optional :popular, type: Boolean, desc: 'If passed, returns only popular licenses' - end - get route do - options = { - featured: declared(params).popular.present? ? true : nil - } - present Licensee::License.all(options), with: Entities::RepoLicense - end + desc 'Get the list of the available license template' do + detail 'This feature was introduced in GitLab 8.7.' + success ::API::Entities::RepoLicense + end + params do + optional :popular, type: Boolean, desc: 'If passed, returns only popular licenses' + use :pagination + end + get "templates/licenses" do + options = { + featured: declared(params).popular.present? ? true : nil + } + licences = ::Kaminari.paginate_array(Licensee::License.all(options)) + present paginate(licences), with: Entities::RepoLicense end - { "licenses/:name" => :deprecated, "templates/licenses/:name" => :ok }.each do |route, status| - desc 'Get the text for a specific license' do - detailed_desc = 'This feature was introduced in GitLab 8.7.' - detailed_desc << DEPRECATION_MESSAGE unless status == :ok - detail detailed_desc - success Entities::RepoLicense - end - params do - requires :name, type: String, desc: 'The name of the template' - end - get route, requirements: { name: /[\w\.-]+/ } do - not_found!('License') unless Licensee::License.find(declared(params).name) + desc 'Get the text for a specific license' do + detail 'This feature was introduced in GitLab 8.7.' + success ::API::Entities::RepoLicense + end + params do + requires :name, type: String, desc: 'The name of the template' + end + get "templates/licenses/:name", requirements: { name: /[\w\.-]+/ } do + not_found!('License') unless Licensee::License.find(declared(params).name) - template = parsed_license_template + template = parsed_license_template - present template, with: Entities::RepoLicense - end + present template, with: ::API::Entities::RepoLicense end GLOBAL_TEMPLATE_TYPES.each do |template_type, properties| klass = properties[:klass] gitlab_version = properties[:gitlab_version] - { template_type => :deprecated, "templates/#{template_type}" => :ok }.each do |route, status| - desc 'Get the list of the available template' do - detailed_desc = "This feature was introduced in GitLab #{gitlab_version}." - detailed_desc << DEPRECATION_MESSAGE unless status == :ok - detail detailed_desc - success Entities::TemplatesList - end - get route do - present klass.all, with: Entities::TemplatesList - end + desc 'Get the list of the available template' do + detail "This feature was introduced in GitLab #{gitlab_version}." + success Entities::TemplatesList + end + params do + use :pagination + end + get "templates/#{template_type}" do + templates = ::Kaminari.paginate_array(klass.all) + present paginate(templates), with: Entities::TemplatesList end - { "#{template_type}/:name" => :deprecated, "templates/#{template_type}/:name" => :ok }.each do |route, status| - desc 'Get the text for a specific template present in local filesystem' do - detailed_desc = "This feature was introduced in GitLab #{gitlab_version}." - detailed_desc << DEPRECATION_MESSAGE unless status == :ok - detail detailed_desc - success Entities::Template - end - params do - requires :name, type: String, desc: 'The name of the template' - end - get route do - new_template = klass.find(declared(params).name) + desc 'Get the text for a specific template present in local filesystem' do + detail "This feature was introduced in GitLab #{gitlab_version}." + success Entities::Template + end + params do + requires :name, type: String, desc: 'The name of the template' + end + get "templates/#{template_type}/:name" do + new_template = klass.find(declared(params).name) - render_response(template_type, new_template) - end + render_response(template_type, new_template) end end end diff --git a/lib/api/time_tracking_endpoints.rb b/lib/api/time_tracking_endpoints.rb index 85b5f7d98b8..05b4b490e27 100644 --- a/lib/api/time_tracking_endpoints.rb +++ b/lib/api/time_tracking_endpoints.rb @@ -5,11 +5,11 @@ module API included do helpers do def issuable_name - declared_params.has_key?(:issue_id) ? 'issue' : 'merge_request' + declared_params.has_key?(:issue_iid) ? 'issue' : 'merge_request' end def issuable_key - "#{issuable_name}_id".to_sym + "#{issuable_name}_iid".to_sym end def update_issuable_key @@ -50,7 +50,7 @@ module API issuable_name = name.end_with?('Issues') ? 'issue' : 'merge_request' issuable_collection_name = issuable_name.pluralize - issuable_key = "#{issuable_name}_id".to_sym + issuable_key = "#{issuable_name}_iid".to_sym desc "Set a time estimate for a project #{issuable_name}" params do diff --git a/lib/api/todos.rb b/lib/api/todos.rb index ed8f48aa1e3..d1f7e364029 100644 --- a/lib/api/todos.rb +++ b/lib/api/todos.rb @@ -5,22 +5,22 @@ module API before { authenticate! } ISSUABLE_TYPES = { - 'merge_requests' => ->(id) { user_project.merge_requests.find(id) }, - 'issues' => ->(id) { find_project_issue(id) } - } + 'merge_requests' => ->(iid) { find_merge_request_with_access(iid) }, + 'issues' => ->(iid) { find_project_issue(iid) } + }.freeze params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects do + resource :projects, requirements: { id: %r{[^/]+} } do ISSUABLE_TYPES.each do |type, finder| - type_id_str = "#{type.singularize}_id".to_sym + type_id_str = "#{type.singularize}_iid".to_sym desc 'Create a todo on an issuable' do success Entities::Todo end params do - requires type_id_str, type: Integer, desc: 'The ID of an issuable' + requires type_id_str, type: Integer, desc: 'The IID of an issuable' end post ":id/#{type}/:#{type_id_str}/todo" do issuable = instance_exec(params[type_id_str], &finder) @@ -58,7 +58,7 @@ module API params do requires :id, type: Integer, desc: 'The ID of the todo being marked as done' end - delete ':id' do + post ':id/mark_as_done' do todo = current_user.todos.find(params[:id]) TodoService.new.mark_todos_as_done([todo], current_user) @@ -66,9 +66,11 @@ module API end desc 'Mark all todos as done' - delete do + post '/mark_as_done' do todos = find_todos TodoService.new.mark_todos_as_done(todos, current_user) + + no_content! end end end diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb index 87a717ba751..a9f2ca2608e 100644 --- a/lib/api/triggers.rb +++ b/lib/api/triggers.rb @@ -5,38 +5,33 @@ module API params do requires :id, type: String, desc: 'The ID of a project' end - resource :projects do - desc 'Trigger a GitLab project build' do - success Entities::TriggerRequest + resource :projects, requirements: { id: %r{[^/]+} } do + desc 'Trigger a GitLab project pipeline' do + success Entities::Pipeline end params do requires :ref, type: String, desc: 'The commit sha or name of a branch or tag' requires :token, type: String, desc: 'The unique token of trigger' optional :variables, type: Hash, desc: 'The list of variables to be injected into build' end - post ":id/(ref/:ref/)trigger/builds" do + post ":id/(ref/:ref/)trigger/pipeline", requirements: { ref: /.+/ } do project = find_project(params[:id]) trigger = Ci::Trigger.find_by_token(params[:token].to_s) not_found! unless project && trigger unauthorized! unless trigger.project == project # validate variables - variables = params[:variables] - if variables - unless variables.all? { |key, value| key.is_a?(String) && value.is_a?(String) } - render_api_error!('variables needs to be a map of key-valued strings', 400) - end - - # convert variables from Mash to Hash - variables = variables.to_h + variables = params[:variables].to_h + unless variables.all? { |key, value| key.is_a?(String) && value.is_a?(String) } + render_api_error!('variables needs to be a map of key-valued strings', 400) end # create request and trigger builds trigger_request = Ci::CreateTriggerRequestService.new.execute(project, trigger, params[:ref].to_s, variables) if trigger_request - present trigger_request, with: Entities::TriggerRequest + present trigger_request.pipeline, with: Entities::Pipeline else - errors = 'No builds created' + errors = 'No pipeline created' render_api_error!(errors, 400) end end @@ -60,13 +55,13 @@ module API success Entities::Trigger end params do - requires :token, type: String, desc: 'The unique token of trigger' + requires :trigger_id, type: Integer, desc: 'The trigger ID' end - get ':id/triggers/:token' do + get ':id/triggers/:trigger_id' do authenticate! authorize! :admin_build, user_project - trigger = user_project.triggers.find_by(token: params[:token].to_s) + trigger = user_project.triggers.find(params.delete(:trigger_id)) return not_found!('Trigger') unless trigger present trigger, with: Entities::Trigger @@ -75,31 +70,79 @@ module API desc 'Create a trigger' do success Entities::Trigger end + params do + requires :description, type: String, desc: 'The trigger description' + end post ':id/triggers' do authenticate! authorize! :admin_build, user_project - trigger = user_project.triggers.create + trigger = user_project.triggers.create( + declared_params(include_missing: false).merge(owner: current_user)) - present trigger, with: Entities::Trigger + if trigger.valid? + present trigger, with: Entities::Trigger + else + render_validation_error!(trigger) + end + end + + desc 'Update a trigger' do + success Entities::Trigger + end + params do + requires :trigger_id, type: Integer, desc: 'The trigger ID' + optional :description, type: String, desc: 'The trigger description' + end + put ':id/triggers/:trigger_id' do + authenticate! + authorize! :admin_build, user_project + + trigger = user_project.triggers.find(params.delete(:trigger_id)) + return not_found!('Trigger') unless trigger + + if trigger.update(declared_params(include_missing: false)) + present trigger, with: Entities::Trigger + else + render_validation_error!(trigger) + end + end + + desc 'Take ownership of trigger' do + success Entities::Trigger + end + params do + requires :trigger_id, type: Integer, desc: 'The trigger ID' + end + post ':id/triggers/:trigger_id/take_ownership' do + authenticate! + authorize! :admin_build, user_project + + trigger = user_project.triggers.find(params.delete(:trigger_id)) + return not_found!('Trigger') unless trigger + + if trigger.update(owner: current_user) + status :ok + present trigger, with: Entities::Trigger + else + render_validation_error!(trigger) + end end desc 'Delete a trigger' do success Entities::Trigger end params do - requires :token, type: String, desc: 'The unique token of trigger' + requires :trigger_id, type: Integer, desc: 'The trigger ID' end - delete ':id/triggers/:token' do + delete ':id/triggers/:trigger_id' do authenticate! authorize! :admin_build, user_project - trigger = user_project.triggers.find_by(token: params[:token].to_s) + trigger = user_project.triggers.find(params.delete(:trigger_id)) return not_found!('Trigger') unless trigger trigger.destroy - - present trigger, with: Entities::Trigger end end end diff --git a/lib/api/users.rb b/lib/api/users.rb index 11a7368b4c0..530ca0b5235 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -9,6 +9,11 @@ module API resource :users, requirements: { uid: /[0-9]*/, id: /[0-9]*/ } do helpers do + def find_user(params) + id = params[:user_id] || params[:id] + User.find_by(id: id) || not_found!('User') + end + params :optional_attributes do optional :skype, type: String, desc: 'The Skype username' optional :linkedin, type: String, desc: 'The LinkedIn username' @@ -22,7 +27,7 @@ module API optional :location, type: String, desc: 'The location of the user' optional :admin, type: Boolean, desc: 'Flag indicating the user is an administrator' optional :can_create_group, type: Boolean, desc: 'Flag indicating the user can create groups' - optional :confirm, type: Boolean, desc: 'Flag indicating the account needs to be confirmed' + optional :skip_confirmation, type: Boolean, default: false, desc: 'Flag indicating the account is confirmed' optional :external, type: Boolean, desc: 'Flag indicating the user is an external user' all_or_none_of :extern_uid, :provider end @@ -40,7 +45,7 @@ module API use :pagination end get do - unless can?(current_user, :read_users_list, nil) + unless can?(current_user, :read_users_list) render_api_error!("Not authorized.", 403) end @@ -82,7 +87,9 @@ module API end params do requires :email, type: String, desc: 'The email of the user' - requires :password, type: String, desc: 'The password of the new user' + optional :password, type: String, desc: 'The password of the new user' + optional :reset_password, type: Boolean, desc: 'Flag indicating the user will be sent a password reset token' + at_least_one_of :password, :reset_password requires :name, type: String, desc: 'The name of the user' requires :username, type: String, desc: 'The username of the user' use :optional_attributes @@ -90,19 +97,10 @@ module API post do authenticated_as_admin! - # Filter out params which are used later - user_params = declared_params(include_missing: false) - identity_attrs = user_params.slice(:provider, :extern_uid) - confirm = user_params.delete(:confirm) - - user = User.new(user_params.except(:extern_uid, :provider)) - user.skip_confirmation! unless confirm + params = declared_params(include_missing: false) + user = ::Users::CreateService.new(current_user, params).execute - if identity_attrs.any? - user.identities.build(identity_attrs) - end - - if user.save + if user.persisted? present user, with: Entities::UserPublic else conflict!('Email has already been taken') if User. @@ -160,6 +158,8 @@ module API end end + user_params[:password_expires_at] = Time.now if user_params[:password].present? + if user.update_attributes(user_params.except(:extern_uid, :provider)) present user, with: Entities::UserPublic else @@ -195,6 +195,7 @@ module API end params do requires :id, type: Integer, desc: 'The ID of the user' + use :pagination end get ':id/keys' do authenticated_as_admin! @@ -202,7 +203,7 @@ module API user = User.find_by(id: params[:id]) not_found!('User') unless user - present user.keys, with: Entities::SSHKey + present paginate(user.keys), with: Entities::SSHKey end desc 'Delete an existing SSH key from a specified user. Available only for admins.' do @@ -221,7 +222,7 @@ module API key = user.keys.find_by(id: params[:key_id]) not_found!('Key') unless key - present key.destroy, with: Entities::SSHKey + key.destroy end desc 'Add an email address to a specified user. Available only for admins.' do @@ -252,13 +253,14 @@ module API end params do requires :id, type: Integer, desc: 'The ID of the user' + use :pagination end get ':id/emails' do authenticated_as_admin! user = User.find_by(id: params[:id]) not_found!('User') unless user - present user.emails, with: Entities::Email + present paginate(user.emails), with: Entities::Email end desc 'Delete an email address of a specified user. Available only for admins.' do @@ -291,14 +293,14 @@ module API user = User.find_by(id: params[:id]) not_found!('User') unless user - DeleteUserService.new(current_user).execute(user) + DeleteUserWorker.perform_async(current_user.id, user.id) end desc 'Block a user. Available only for admins.' params do requires :id, type: Integer, desc: 'The ID of the user' end - put ':id/block' do + post ':id/block' do authenticated_as_admin! user = User.find_by(id: params[:id]) not_found!('User') unless user @@ -314,7 +316,7 @@ module API params do requires :id, type: Integer, desc: 'The ID of the user' end - put ':id/unblock' do + post ':id/unblock' do authenticated_as_admin! user = User.find_by(id: params[:id]) not_found!('User') unless user @@ -346,6 +348,76 @@ module API present paginate(events), with: Entities::Event end + + params do + requires :user_id, type: Integer, desc: 'The ID of the user' + end + segment ':user_id' do + resource :impersonation_tokens do + helpers do + def finder(options = {}) + user = find_user(params) + PersonalAccessTokensFinder.new({ user: user, impersonation: true }.merge(options)) + end + + def find_impersonation_token + finder.find_by(id: declared_params[:impersonation_token_id]) || not_found!('Impersonation Token') + end + end + + before { authenticated_as_admin! } + + desc 'Retrieve impersonation tokens. Available only for admins.' do + detail 'This feature was introduced in GitLab 9.0' + success Entities::ImpersonationToken + end + params do + use :pagination + optional :state, type: String, default: 'all', values: %w[all active inactive], desc: 'Filters (all|active|inactive) impersonation_tokens' + end + get { present paginate(finder(declared_params(include_missing: false)).execute), with: Entities::ImpersonationToken } + + desc 'Create a impersonation token. Available only for admins.' do + detail 'This feature was introduced in GitLab 9.0' + success Entities::ImpersonationToken + end + params do + requires :name, type: String, desc: 'The name of the impersonation token' + optional :expires_at, type: Date, desc: 'The expiration date in the format YEAR-MONTH-DAY of the impersonation token' + optional :scopes, type: Array, desc: 'The array of scopes of the impersonation token' + end + post do + impersonation_token = finder.build(declared_params(include_missing: false)) + + if impersonation_token.save + present impersonation_token, with: Entities::ImpersonationToken + else + render_validation_error!(impersonation_token) + end + end + + desc 'Retrieve impersonation token. Available only for admins.' do + detail 'This feature was introduced in GitLab 9.0' + success Entities::ImpersonationToken + end + params do + requires :impersonation_token_id, type: Integer, desc: 'The ID of the impersonation token' + end + get ':impersonation_token_id' do + present find_impersonation_token, with: Entities::ImpersonationToken + end + + desc 'Revoke a impersonation token. Available only for admins.' do + detail 'This feature was introduced in GitLab 9.0' + end + params do + requires :impersonation_token_id, type: Integer, desc: 'The ID of the impersonation token' + end + delete ':impersonation_token_id' do + find_impersonation_token.revoke! + end + end + end end resource :user do @@ -359,8 +431,11 @@ module API desc "Get the currently authenticated user's SSH keys" do success Entities::SSHKey end + params do + use :pagination + end get "keys" do - present current_user.keys, with: Entities::SSHKey + present paginate(current_user.keys), with: Entities::SSHKey end desc 'Get a single key owned by currently authenticated user' do @@ -403,14 +478,17 @@ module API key = current_user.keys.find_by(id: params[:key_id]) not_found!('Key') unless key - present key.destroy, with: Entities::SSHKey + key.destroy end desc "Get the currently authenticated user's email addresses" do success Entities::Email end + params do + use :pagination + end get "emails" do - present current_user.emails, with: Entities::Email + present paginate(current_user.emails), with: Entities::Email end desc 'Get a single email address owned by the currently authenticated user' do diff --git a/lib/api/v3/award_emoji.rb b/lib/api/v3/award_emoji.rb new file mode 100644 index 00000000000..b96b2d70b12 --- /dev/null +++ b/lib/api/v3/award_emoji.rb @@ -0,0 +1,130 @@ +module API + module V3 + class AwardEmoji < Grape::API + include PaginationParams + + before { authenticate! } + AWARDABLES = %w[issue merge_request snippet].freeze + + resource :projects, requirements: { id: %r{[^/]+} } do + AWARDABLES.each do |awardable_type| + awardable_string = awardable_type.pluralize + awardable_id_string = "#{awardable_type}_id" + + params do + requires :id, type: String, desc: 'The ID of a project' + requires :"#{awardable_id_string}", type: Integer, desc: "The ID of an Issue, Merge Request or Snippet" + end + + [ + ":id/#{awardable_string}/:#{awardable_id_string}/award_emoji", + ":id/#{awardable_string}/:#{awardable_id_string}/notes/:note_id/award_emoji" + ].each do |endpoint| + + desc 'Get a list of project +awardable+ award emoji' do + detail 'This feature was introduced in 8.9' + success Entities::AwardEmoji + end + params do + use :pagination + end + get endpoint do + if can_read_awardable? + awards = awardable.award_emoji + present paginate(awards), with: Entities::AwardEmoji + else + not_found!("Award Emoji") + end + end + + desc 'Get a specific award emoji' do + detail 'This feature was introduced in 8.9' + success Entities::AwardEmoji + end + params do + requires :award_id, type: Integer, desc: 'The ID of the award' + end + get "#{endpoint}/:award_id" do + if can_read_awardable? + present awardable.award_emoji.find(params[:award_id]), with: Entities::AwardEmoji + else + not_found!("Award Emoji") + end + end + + desc 'Award a new Emoji' do + detail 'This feature was introduced in 8.9' + success Entities::AwardEmoji + end + params do + requires :name, type: String, desc: 'The name of a award_emoji (without colons)' + end + post endpoint do + not_found!('Award Emoji') unless can_read_awardable? && can_award_awardable? + + award = awardable.create_award_emoji(params[:name], current_user) + + if award.persisted? + present award, with: Entities::AwardEmoji + else + not_found!("Award Emoji #{award.errors.messages}") + end + end + + desc 'Delete a +awardables+ award emoji' do + detail 'This feature was introduced in 8.9' + success Entities::AwardEmoji + end + params do + requires :award_id, type: Integer, desc: 'The ID of an award emoji' + end + delete "#{endpoint}/:award_id" do + award = awardable.award_emoji.find(params[:award_id]) + + unauthorized! unless award.user == current_user || current_user.admin? + + award.destroy + present award, with: Entities::AwardEmoji + end + end + end + end + + helpers do + def can_read_awardable? + can?(current_user, read_ability(awardable), awardable) + end + + def can_award_awardable? + awardable.user_can_award?(current_user, params[:name]) + end + + def awardable + @awardable ||= + begin + if params.include?(:note_id) + note_id = params.delete(:note_id) + + awardable.notes.find(note_id) + elsif params.include?(:issue_id) + user_project.issues.find(params[:issue_id]) + elsif params.include?(:merge_request_id) + user_project.merge_requests.find(params[:merge_request_id]) + else + user_project.snippets.find(params[:snippet_id]) + end + end + end + + def read_ability(awardable) + case awardable + when Note + read_ability(awardable.noteable) + else + :"read_#{awardable.class.to_s.underscore}" + end + end + end + end + end +end diff --git a/lib/api/v3/boards.rb b/lib/api/v3/boards.rb new file mode 100644 index 00000000000..94acc67171e --- /dev/null +++ b/lib/api/v3/boards.rb @@ -0,0 +1,72 @@ +module API + module V3 + class Boards < Grape::API + before { authenticate! } + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: { id: %r{[^/]+} } do + desc 'Get all project boards' do + detail 'This feature was introduced in 8.13' + success ::API::Entities::Board + end + get ':id/boards' do + authorize!(:read_board, user_project) + present user_project.boards, with: ::API::Entities::Board + end + + params do + requires :board_id, type: Integer, desc: 'The ID of a board' + end + segment ':id/boards/:board_id' do + helpers do + def project_board + board = user_project.boards.first + + if params[:board_id] == board.id + board + else + not_found!('Board') + end + end + + def board_lists + project_board.lists.destroyable + end + end + + desc 'Get the lists of a project board' do + detail 'Does not include `done` list. This feature was introduced in 8.13' + success ::API::Entities::List + end + get '/lists' do + authorize!(:read_board, user_project) + present board_lists, with: ::API::Entities::List + end + + desc 'Delete a board list' do + detail 'This feature was introduced in 8.13' + success ::API::Entities::List + end + params do + requires :list_id, type: Integer, desc: 'The ID of a board list' + end + delete "/lists/:list_id" do + authorize!(:admin_list, user_project) + + list = board_lists.find(params[:list_id]) + + service = ::Boards::Lists::DestroyService.new(user_project, current_user) + + if service.execute(list) + present list, with: ::API::Entities::List + else + render_api_error!({ error: 'List could not be deleted!' }, 400) + end + end + end + end + end + end +end diff --git a/lib/api/v3/branches.rb b/lib/api/v3/branches.rb new file mode 100644 index 00000000000..0a877b960f6 --- /dev/null +++ b/lib/api/v3/branches.rb @@ -0,0 +1,72 @@ +require 'mime/types' + +module API + module V3 + class Branches < Grape::API + before { authenticate! } + before { authorize! :download_code, user_project } + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: { id: %r{[^/]+} } do + desc 'Get a project repository branches' do + success ::API::Entities::RepoBranch + end + get ":id/repository/branches" do + branches = user_project.repository.branches.sort_by(&:name) + + present branches, with: ::API::Entities::RepoBranch, project: user_project + end + + desc 'Delete a branch' + params do + requires :branch, type: String, desc: 'The name of the branch' + end + delete ":id/repository/branches/:branch", requirements: { branch: /.+/ } do + authorize_push_project + + result = DeleteBranchService.new(user_project, current_user). + execute(params[:branch]) + + if result[:status] == :success + status(200) + { + branch_name: params[:branch] + } + else + render_api_error!(result[:message], result[:return_code]) + end + end + + desc 'Delete all merged branches' + delete ":id/repository/merged_branches" do + DeleteMergedBranchesService.new(user_project, current_user).async_execute + + status(200) + end + + desc 'Create branch' do + success ::API::Entities::RepoBranch + end + params do + requires :branch_name, type: String, desc: 'The name of the branch' + requires :ref, type: String, desc: 'Create branch from commit sha or existing branch' + end + post ":id/repository/branches" do + authorize_push_project + result = CreateBranchService.new(user_project, current_user). + execute(params[:branch_name], params[:ref]) + + if result[:status] == :success + present result[:branch], + with: ::API::Entities::RepoBranch, + project: user_project + else + render_api_error!(result[:message], 400) + end + end + end + end + end +end diff --git a/lib/api/v3/broadcast_messages.rb b/lib/api/v3/broadcast_messages.rb new file mode 100644 index 00000000000..417e4ad0b26 --- /dev/null +++ b/lib/api/v3/broadcast_messages.rb @@ -0,0 +1,31 @@ +module API + module V3 + class BroadcastMessages < Grape::API + include PaginationParams + + before { authenticate! } + before { authenticated_as_admin! } + + resource :broadcast_messages do + helpers do + def find_message + BroadcastMessage.find(params[:id]) + end + end + + desc 'Delete a broadcast message' do + detail 'This feature was introduced in GitLab 8.12.' + success ::API::Entities::BroadcastMessage + end + params do + requires :id, type: Integer, desc: 'Broadcast message ID' + end + delete ':id' do + message = find_message + + present message.destroy, with: ::API::Entities::BroadcastMessage + end + end + end + end +end diff --git a/lib/api/v3/builds.rb b/lib/api/v3/builds.rb new file mode 100644 index 00000000000..6f97102c6ef --- /dev/null +++ b/lib/api/v3/builds.rb @@ -0,0 +1,255 @@ +module API + module V3 + class Builds < Grape::API + include PaginationParams + + before { authenticate! } + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects do + helpers do + params :optional_scope do + optional :scope, types: [String, Array[String]], desc: 'The scope of builds to show', + values: %w(pending running failed success canceled skipped), + coerce_with: ->(scope) { + if scope.is_a?(String) + [scope] + elsif scope.is_a?(Hashie::Mash) + scope.values + else + ['unknown'] + end + } + end + end + + desc 'Get a project builds' do + success ::API::V3::Entities::Build + end + params do + use :optional_scope + use :pagination + end + get ':id/builds' do + builds = user_project.builds.order('id DESC') + builds = filter_builds(builds, params[:scope]) + + present paginate(builds), with: ::API::V3::Entities::Build + end + + desc 'Get builds for a specific commit of a project' do + success ::API::V3::Entities::Build + end + params do + requires :sha, type: String, desc: 'The SHA id of a commit' + use :optional_scope + use :pagination + end + get ':id/repository/commits/:sha/builds' do + authorize_read_builds! + + return not_found! unless user_project.commit(params[:sha]) + + pipelines = user_project.pipelines.where(sha: params[:sha]) + builds = user_project.builds.where(pipeline: pipelines).order('id DESC') + builds = filter_builds(builds, params[:scope]) + + present paginate(builds), with: ::API::V3::Entities::Build + end + + desc 'Get a specific build of a project' do + success ::API::V3::Entities::Build + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end + get ':id/builds/:build_id' do + authorize_read_builds! + + build = get_build!(params[:build_id]) + + present build, with: ::API::V3::Entities::Build + end + + desc 'Download the artifacts file from build' do + detail 'This feature was introduced in GitLab 8.5' + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end + get ':id/builds/:build_id/artifacts' do + authorize_read_builds! + + build = get_build!(params[:build_id]) + + present_artifacts!(build.artifacts_file) + end + + desc 'Download the artifacts file from build' do + detail 'This feature was introduced in GitLab 8.10' + end + params do + requires :ref_name, type: String, desc: 'The ref from repository' + requires :job, type: String, desc: 'The name for the build' + end + get ':id/builds/artifacts/:ref_name/download', + requirements: { ref_name: /.+/ } do + authorize_read_builds! + + builds = user_project.latest_successful_builds_for(params[:ref_name]) + latest_build = builds.find_by!(name: params[:job]) + + present_artifacts!(latest_build.artifacts_file) + end + + # TODO: We should use `present_file!` and leave this implementation for backward compatibility (when build trace + # is saved in the DB instead of file). But before that, we need to consider how to replace the value of + # `runners_token` with some mask (like `xxxxxx`) when sending trace file directly by workhorse. + desc 'Get a trace of a specific build of a project' + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end + get ':id/builds/:build_id/trace' do + authorize_read_builds! + + build = get_build!(params[:build_id]) + + header 'Content-Disposition', "infile; filename=\"#{build.id}.log\"" + content_type 'text/plain' + env['api.format'] = :binary + + trace = build.trace + body trace + end + + desc 'Cancel a specific build of a project' do + success ::API::V3::Entities::Build + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end + post ':id/builds/:build_id/cancel' do + authorize_update_builds! + + build = get_build!(params[:build_id]) + + build.cancel + + present build, with: ::API::V3::Entities::Build + end + + desc 'Retry a specific build of a project' do + success ::API::V3::Entities::Build + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end + post ':id/builds/:build_id/retry' do + authorize_update_builds! + + build = get_build!(params[:build_id]) + return forbidden!('Build is not retryable') unless build.retryable? + + build = Ci::Build.retry(build, current_user) + + present build, with: ::API::V3::Entities::Build + end + + desc 'Erase build (remove artifacts and build trace)' do + success ::API::V3::Entities::Build + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end + post ':id/builds/:build_id/erase' do + authorize_update_builds! + + build = get_build!(params[:build_id]) + return forbidden!('Build is not erasable!') unless build.erasable? + + build.erase(erased_by: current_user) + present build, with: ::API::V3::Entities::Build + end + + desc 'Keep the artifacts to prevent them from being deleted' do + success ::API::V3::Entities::Build + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end + post ':id/builds/:build_id/artifacts/keep' do + authorize_update_builds! + + build = get_build!(params[:build_id]) + return not_found!(build) unless build.artifacts? + + build.keep_artifacts! + + status 200 + present build, with: ::API::V3::Entities::Build + end + + desc 'Trigger a manual build' do + success ::API::V3::Entities::Build + detail 'This feature was added in GitLab 8.11' + end + params do + requires :build_id, type: Integer, desc: 'The ID of a Build' + end + post ":id/builds/:build_id/play" do + authorize_read_builds! + + build = get_build!(params[:build_id]) + + bad_request!("Unplayable Job") unless build.playable? + + build.play(current_user) + + status 200 + present build, with: ::API::V3::Entities::Build + end + end + + helpers do + def get_build(id) + user_project.builds.find_by(id: id.to_i) + end + + def get_build!(id) + get_build(id) || not_found! + end + + def present_artifacts!(artifacts_file) + if !artifacts_file.file_storage? + redirect_to(build.artifacts_file.url) + elsif artifacts_file.exists? + present_file!(artifacts_file.path, artifacts_file.filename) + else + not_found! + end + end + + def filter_builds(builds, scope) + return builds if scope.nil? || scope.empty? + + available_statuses = ::CommitStatus::AVAILABLE_STATUSES + + unknown = scope - available_statuses + render_api_error!('Scope contains invalid value(s)', 400) unless unknown.empty? + + builds.where(status: available_statuses && scope) + end + + def authorize_read_builds! + authorize! :read_build, user_project + end + + def authorize_update_builds! + authorize! :update_build, user_project + end + end + end + end +end diff --git a/lib/api/v3/commits.rb b/lib/api/v3/commits.rb new file mode 100644 index 00000000000..3414a2883e5 --- /dev/null +++ b/lib/api/v3/commits.rb @@ -0,0 +1,196 @@ +require 'mime/types' + +module API + module V3 + class Commits < Grape::API + include PaginationParams + + before { authenticate! } + before { authorize! :download_code, user_project } + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: { id: %r{[^/]+} } do + desc 'Get a project repository commits' do + success ::API::Entities::RepoCommit + end + params do + optional :ref_name, type: String, desc: 'The name of a repository branch or tag, if not given the default branch is used' + optional :since, type: DateTime, desc: 'Only commits after or in this date will be returned' + optional :until, type: DateTime, desc: 'Only commits before or in this date will be returned' + optional :page, type: Integer, default: 0, desc: 'The page for pagination' + optional :per_page, type: Integer, default: 20, desc: 'The number of results per page' + optional :path, type: String, desc: 'The file path' + end + get ":id/repository/commits" do + ref = params[:ref_name] || user_project.try(:default_branch) || 'master' + offset = params[:page] * params[:per_page] + + commits = user_project.repository.commits(ref, + path: params[:path], + limit: params[:per_page], + offset: offset, + after: params[:since], + before: params[:until]) + + present commits, with: ::API::Entities::RepoCommit + end + + desc 'Commit multiple file changes as one commit' do + success ::API::Entities::RepoCommitDetail + detail 'This feature was introduced in GitLab 8.13' + end + params do + requires :branch_name, type: String, desc: 'The name of branch' + requires :commit_message, type: String, desc: 'Commit message' + requires :actions, type: Array[Hash], desc: 'Actions to perform in commit' + optional :author_email, type: String, desc: 'Author email for commit' + optional :author_name, type: String, desc: 'Author name for commit' + end + post ":id/repository/commits" do + authorize! :push_code, user_project + + attrs = declared_params.dup + branch = attrs.delete(:branch_name) + attrs.merge!(branch: branch, start_branch: branch, target_branch: branch) + + result = ::Files::MultiService.new(user_project, current_user, attrs).execute + + if result[:status] == :success + commit_detail = user_project.repository.commits(result[:result], limit: 1).first + present commit_detail, with: ::API::Entities::RepoCommitDetail + else + render_api_error!(result[:message], 400) + end + end + + desc 'Get a specific commit of a project' do + success ::API::Entities::RepoCommitDetail + failure [[404, 'Not Found']] + end + params do + requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' + end + get ":id/repository/commits/:sha" do + commit = user_project.commit(params[:sha]) + + not_found! "Commit" unless commit + + present commit, with: ::API::Entities::RepoCommitDetail + end + + desc 'Get the diff for a specific commit of a project' do + failure [[404, 'Not Found']] + end + params do + requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' + end + get ":id/repository/commits/:sha/diff" do + commit = user_project.commit(params[:sha]) + + not_found! "Commit" unless commit + + commit.raw_diffs.to_a + end + + desc "Get a commit's comments" do + success ::API::Entities::CommitNote + failure [[404, 'Not Found']] + end + params do + use :pagination + requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' + end + get ':id/repository/commits/:sha/comments' do + commit = user_project.commit(params[:sha]) + + not_found! 'Commit' unless commit + notes = Note.where(commit_id: commit.id).order(:created_at) + + present paginate(notes), with: ::API::Entities::CommitNote + end + + desc 'Cherry pick commit into a branch' do + detail 'This feature was introduced in GitLab 8.15' + success ::API::Entities::RepoCommit + end + params do + requires :sha, type: String, desc: 'A commit sha to be cherry picked' + requires :branch, type: String, desc: 'The name of the branch' + end + post ':id/repository/commits/:sha/cherry_pick' do + authorize! :push_code, user_project + + commit = user_project.commit(params[:sha]) + not_found!('Commit') unless commit + + branch = user_project.repository.find_branch(params[:branch]) + not_found!('Branch') unless branch + + commit_params = { + commit: commit, + start_branch: params[:branch], + target_branch: params[:branch] + } + + result = ::Commits::CherryPickService.new(user_project, current_user, commit_params).execute + + if result[:status] == :success + branch = user_project.repository.find_branch(params[:branch]) + present user_project.repository.commit(branch.dereferenced_target), with: ::API::Entities::RepoCommit + else + render_api_error!(result[:message], 400) + end + end + + desc 'Post comment to commit' do + success ::API::Entities::CommitNote + end + params do + requires :sha, type: String, regexp: /\A\h{6,40}\z/, desc: "The commit's SHA" + requires :note, type: String, desc: 'The text of the comment' + optional :path, type: String, desc: 'The file path' + given :path do + requires :line, type: Integer, desc: 'The line number' + requires :line_type, type: String, values: %w(new old), default: 'new', desc: 'The type of the line' + end + end + post ':id/repository/commits/:sha/comments' do + commit = user_project.commit(params[:sha]) + not_found! 'Commit' unless commit + + opts = { + note: params[:note], + noteable_type: 'Commit', + commit_id: commit.id + } + + if params[:path] + commit.raw_diffs(all_diffs: true).each do |diff| + next unless diff.new_path == params[:path] + lines = Gitlab::Diff::Parser.new.parse(diff.diff.each_line) + + lines.each do |line| + next unless line.new_pos == params[:line] && line.type == params[:line_type] + break opts[:line_code] = Gitlab::Diff::LineCode.generate(diff.new_path, line.new_pos, line.old_pos) + end + + break if opts[:line_code] + end + + opts[:type] = LegacyDiffNote.name if opts[:line_code] + end + + note = ::Notes::CreateService.new(user_project, current_user, opts).execute + + if note.save + present note, with: ::API::Entities::CommitNote + else + render_api_error!("Failed to save note #{note.errors.messages}", 400) + end + end + end + end + end +end diff --git a/lib/api/v3/deploy_keys.rb b/lib/api/v3/deploy_keys.rb new file mode 100644 index 00000000000..bbb174b6003 --- /dev/null +++ b/lib/api/v3/deploy_keys.rb @@ -0,0 +1,122 @@ +module API + module V3 + class DeployKeys < Grape::API + before { authenticate! } + + get "deploy_keys" do + authenticated_as_admin! + + keys = DeployKey.all + present keys, with: ::API::Entities::SSHKey + end + + params do + requires :id, type: String, desc: 'The ID of the project' + end + resource :projects, requirements: { id: %r{[^/]+} } do + before { authorize_admin_project } + + %w(keys deploy_keys).each do |path| + desc "Get a specific project's deploy keys" do + success ::API::Entities::SSHKey + end + get ":id/#{path}" do + present user_project.deploy_keys, with: ::API::Entities::SSHKey + end + + desc 'Get single deploy key' do + success ::API::Entities::SSHKey + end + params do + requires :key_id, type: Integer, desc: 'The ID of the deploy key' + end + get ":id/#{path}/:key_id" do + key = user_project.deploy_keys.find params[:key_id] + present key, with: ::API::Entities::SSHKey + end + + desc 'Add new deploy key to currently authenticated user' do + success ::API::Entities::SSHKey + end + params do + requires :key, type: String, desc: 'The new deploy key' + requires :title, type: String, desc: 'The name of the deploy key' + end + post ":id/#{path}" do + params[:key].strip! + + # Check for an existing key joined to this project + key = user_project.deploy_keys.find_by(key: params[:key]) + if key + present key, with: ::API::Entities::SSHKey + break + end + + # Check for available deploy keys in other projects + key = current_user.accessible_deploy_keys.find_by(key: params[:key]) + if key + user_project.deploy_keys << key + present key, with: ::API::Entities::SSHKey + break + end + + # Create a new deploy key + key = DeployKey.new(declared_params(include_missing: false)) + if key.valid? && user_project.deploy_keys << key + present key, with: ::API::Entities::SSHKey + else + render_validation_error!(key) + end + end + + desc 'Enable a deploy key for a project' do + detail 'This feature was added in GitLab 8.11' + success ::API::Entities::SSHKey + end + params do + requires :key_id, type: Integer, desc: 'The ID of the deploy key' + end + post ":id/#{path}/:key_id/enable" do + key = ::Projects::EnableDeployKeyService.new(user_project, + current_user, declared_params).execute + + if key + present key, with: ::API::Entities::SSHKey + else + not_found!('Deploy Key') + end + end + + desc 'Disable a deploy key for a project' do + detail 'This feature was added in GitLab 8.11' + success ::API::Entities::SSHKey + end + params do + requires :key_id, type: Integer, desc: 'The ID of the deploy key' + end + delete ":id/#{path}/:key_id/disable" do + key = user_project.deploy_keys_projects.find_by(deploy_key_id: params[:key_id]) + key.destroy + + present key.deploy_key, with: ::API::Entities::SSHKey + end + + desc 'Delete deploy key for a project' do + success Key + end + params do + requires :key_id, type: Integer, desc: 'The ID of the deploy key' + end + delete ":id/#{path}/:key_id" do + key = user_project.deploy_keys_projects.find_by(deploy_key_id: params[:key_id]) + if key + key.destroy + else + not_found!('Deploy Key') + end + end + end + end + end + end +end diff --git a/lib/api/v3/deployments.rb b/lib/api/v3/deployments.rb new file mode 100644 index 00000000000..1d4972eda26 --- /dev/null +++ b/lib/api/v3/deployments.rb @@ -0,0 +1,43 @@ +module API + module V3 + # Deployments RESTful API endpoints + class Deployments < Grape::API + include PaginationParams + + before { authenticate! } + + params do + requires :id, type: String, desc: 'The project ID' + end + resource :projects, requirements: { id: %r{[^/]+} } do + desc 'Get all deployments of the project' do + detail 'This feature was introduced in GitLab 8.11.' + success ::API::V3::Deployments + end + params do + use :pagination + end + get ':id/deployments' do + authorize! :read_deployment, user_project + + present paginate(user_project.deployments), with: ::API::V3::Deployments + end + + desc 'Gets a specific deployment' do + detail 'This feature was introduced in GitLab 8.11.' + success ::API::V3::Deployments + end + params do + requires :deployment_id, type: Integer, desc: 'The deployment ID' + end + get ':id/deployments/:deployment_id' do + authorize! :read_deployment, user_project + + deployment = user_project.deployments.find(params[:deployment_id]) + + present deployment, with: ::API::V3::Deployments + end + end + end + end +end diff --git a/lib/api/v3/entities.rb b/lib/api/v3/entities.rb new file mode 100644 index 00000000000..832b4bdeb4f --- /dev/null +++ b/lib/api/v3/entities.rb @@ -0,0 +1,253 @@ +module API + module V3 + module Entities + class ProjectSnippet < Grape::Entity + expose :id, :title, :file_name + expose :author, using: ::API::Entities::UserBasic + expose :updated_at, :created_at + expose(:expires_at) { |snippet| nil } + + expose :web_url do |snippet, options| + Gitlab::UrlBuilder.build(snippet) + end + end + + class Note < Grape::Entity + expose :id + expose :note, as: :body + expose :attachment_identifier, as: :attachment + expose :author, using: ::API::Entities::UserBasic + expose :created_at, :updated_at + expose :system?, as: :system + expose :noteable_id, :noteable_type + # upvote? and downvote? are deprecated, always return false + expose(:upvote?) { |note| false } + expose(:downvote?) { |note| false } + end + + class Event < Grape::Entity + expose :title, :project_id, :action_name + expose :target_id, :target_type, :author_id + expose :data, :target_title + expose :created_at + expose :note, using: Entities::Note, if: ->(event, options) { event.note? } + expose :author, using: ::API::Entities::UserBasic, if: ->(event, options) { event.author } + + expose :author_username do |event, options| + event.author&.username + end + end + + class AwardEmoji < Grape::Entity + expose :id + expose :name + expose :user, using: ::API::Entities::UserBasic + expose :created_at, :updated_at + expose :awardable_id, :awardable_type + end + + class Project < Grape::Entity + expose :id, :description, :default_branch, :tag_list + expose :public?, as: :public + expose :archived?, as: :archived + expose :visibility_level, :ssh_url_to_repo, :http_url_to_repo, :web_url + expose :owner, using: ::API::Entities::UserBasic, unless: ->(project, options) { project.group } + expose :name, :name_with_namespace + expose :path, :path_with_namespace + expose :container_registry_enabled + + # Expose old field names with the new permissions methods to keep API compatible + expose(:issues_enabled) { |project, options| project.feature_available?(:issues, options[:current_user]) } + expose(:merge_requests_enabled) { |project, options| project.feature_available?(:merge_requests, options[:current_user]) } + expose(:wiki_enabled) { |project, options| project.feature_available?(:wiki, options[:current_user]) } + expose(:builds_enabled) { |project, options| project.feature_available?(:builds, options[:current_user]) } + expose(:snippets_enabled) { |project, options| project.feature_available?(:snippets, options[:current_user]) } + + expose :created_at, :last_activity_at + expose :shared_runners_enabled + expose :lfs_enabled?, as: :lfs_enabled + expose :creator_id + expose :namespace, using: 'API::Entities::Namespace' + expose :forked_from_project, using: ::API::Entities::BasicProjectDetails, if: lambda{ |project, options| project.forked? } + expose :avatar_url + expose :star_count, :forks_count + expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:current_user]) && project.default_issues_tracker? } + expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] } + expose :public_builds + expose :shared_with_groups do |project, options| + ::API::Entities::SharedGroup.represent(project.project_group_links.all, options) + end + expose :only_allow_merge_if_pipeline_succeeds, as: :only_allow_merge_if_build_succeeds + expose :request_access_enabled + expose :only_allow_merge_if_all_discussions_are_resolved + + expose :statistics, using: '::API::V3::Entities::ProjectStatistics', if: :statistics + end + + class ProjectWithAccess < Project + expose :permissions do + expose :project_access, using: ::API::Entities::ProjectAccess do |project, options| + project.project_members.find_by(user_id: options[:current_user].id) + end + + expose :group_access, using: ::API::Entities::GroupAccess do |project, options| + if project.group + project.group.group_members.find_by(user_id: options[:current_user].id) + end + end + end + end + + class MergeRequest < Grape::Entity + expose :id, :iid + expose(:project_id) { |entity| entity.project.id } + expose :title, :description + expose :state, :created_at, :updated_at + expose :target_branch, :source_branch + expose :upvotes, :downvotes + expose :author, :assignee, using: ::API::Entities::UserBasic + expose :source_project_id, :target_project_id + expose :label_names, as: :labels + expose :work_in_progress?, as: :work_in_progress + expose :milestone, using: ::API::Entities::Milestone + expose :merge_when_pipeline_succeeds, as: :merge_when_build_succeeds + expose :merge_status + expose :diff_head_sha, as: :sha + expose :merge_commit_sha + expose :subscribed do |merge_request, options| + merge_request.subscribed?(options[:current_user], options[:project]) + end + expose :user_notes_count + expose :should_remove_source_branch?, as: :should_remove_source_branch + expose :force_remove_source_branch?, as: :force_remove_source_branch + + expose :web_url do |merge_request, options| + Gitlab::UrlBuilder.build(merge_request) + end + end + + class Group < Grape::Entity + expose :id, :name, :path, :description, :visibility_level + expose :lfs_enabled?, as: :lfs_enabled + expose :avatar_url + expose :web_url + expose :request_access_enabled + expose :full_name, :full_path + expose :parent_id + + expose :statistics, if: :statistics do + with_options format_with: -> (value) { value.to_i } do + expose :storage_size + expose :repository_size + expose :lfs_objects_size + expose :build_artifacts_size + end + end + end + + class GroupDetail < Group + expose :projects, using: Entities::Project + expose :shared_projects, using: Entities::Project + end + + class ApplicationSetting < Grape::Entity + expose :id + expose :default_projects_limit + expose :signup_enabled + expose :signin_enabled + expose :gravatar_enabled + expose :sign_in_text + expose :after_sign_up_text + expose :created_at + expose :updated_at + expose :home_page_url + expose :default_branch_protection + expose :restricted_visibility_levels + expose :max_attachment_size + expose :session_expire_delay + expose :default_project_visibility + expose :default_snippet_visibility + expose :default_group_visibility + expose :domain_whitelist + expose :domain_blacklist_enabled + expose :domain_blacklist + expose :user_oauth_applications + expose :after_sign_out_path + expose :container_registry_token_expire_delay + expose :repository_storage + expose :repository_storages + expose :koding_enabled + expose :koding_url + expose :plantuml_enabled + expose :plantuml_url + expose :terminal_max_session_time + end + + class Environment < ::API::Entities::EnvironmentBasic + expose :project, using: Entities::Project + end + + class Trigger < Grape::Entity + expose :token, :created_at, :updated_at, :deleted_at, :last_used + expose :owner, using: ::API::Entities::UserBasic + end + + class TriggerRequest < Grape::Entity + expose :id, :variables + end + + class Build < Grape::Entity + expose :id, :status, :stage, :name, :ref, :tag, :coverage + expose :created_at, :started_at, :finished_at + expose :user, with: ::API::Entities::User + expose :artifacts_file, using: ::API::Entities::JobArtifactFile, if: -> (build, opts) { build.artifacts? } + expose :commit, with: ::API::Entities::RepoCommit + expose :runner, with: ::API::Entities::Runner + expose :pipeline, with: ::API::Entities::PipelineBasic + end + + class BuildArtifactFile < Grape::Entity + expose :filename, :size + end + + class Deployment < Grape::Entity + expose :id, :iid, :ref, :sha, :created_at + expose :user, using: ::API::Entities::UserBasic + expose :environment, using: ::API::Entities::EnvironmentBasic + expose :deployable, using: Entities::Build + end + + class MergeRequestChanges < MergeRequest + expose :diffs, as: :changes, using: ::API::Entities::RepoDiff do |compare, _| + compare.raw_diffs(all_diffs: true).to_a + end + end + + class ProjectStatistics < Grape::Entity + expose :commit_count + expose :storage_size + expose :repository_size + expose :lfs_objects_size + expose :build_artifacts_size + end + + class ProjectService < Grape::Entity + expose :id, :title, :created_at, :updated_at, :active + expose :push_events, :issues_events, :merge_requests_events + expose :tag_push_events, :note_events, :build_events, :pipeline_events + # Expose serialized properties + expose :properties do |service, options| + field_names = service.fields. + select { |field| options[:include_passwords] || field[:type] != 'password' }. + map { |field| field[:name] } + service.properties.slice(*field_names) + end + end + + class ProjectHook < ::API::Entities::Hook + expose :project_id, :issues_events, :merge_requests_events + expose :note_events, :build_events, :pipeline_events, :wiki_page_events + end + end + end +end diff --git a/lib/api/v3/environments.rb b/lib/api/v3/environments.rb new file mode 100644 index 00000000000..6bb4e016a01 --- /dev/null +++ b/lib/api/v3/environments.rb @@ -0,0 +1,87 @@ +module API + module V3 + class Environments < Grape::API + include ::API::Helpers::CustomValidators + include PaginationParams + + before { authenticate! } + + params do + requires :id, type: String, desc: 'The project ID' + end + resource :projects, requirements: { id: %r{[^/]+} } do + desc 'Get all environments of the project' do + detail 'This feature was introduced in GitLab 8.11.' + success Entities::Environment + end + params do + use :pagination + end + get ':id/environments' do + authorize! :read_environment, user_project + + present paginate(user_project.environments), with: Entities::Environment + end + + desc 'Creates a new environment' do + detail 'This feature was introduced in GitLab 8.11.' + success Entities::Environment + end + params do + requires :name, type: String, desc: 'The name of the environment to be created' + optional :external_url, type: String, desc: 'URL on which this deployment is viewable' + optional :slug, absence: { message: "is automatically generated and cannot be changed" } + end + post ':id/environments' do + authorize! :create_environment, user_project + + environment = user_project.environments.create(declared_params) + + if environment.persisted? + present environment, with: Entities::Environment + else + render_validation_error!(environment) + end + end + + desc 'Updates an existing environment' do + detail 'This feature was introduced in GitLab 8.11.' + success Entities::Environment + end + params do + requires :environment_id, type: Integer, desc: 'The environment ID' + optional :name, type: String, desc: 'The new environment name' + optional :external_url, type: String, desc: 'The new URL on which this deployment is viewable' + optional :slug, absence: { message: "is automatically generated and cannot be changed" } + end + put ':id/environments/:environment_id' do + authorize! :update_environment, user_project + + environment = user_project.environments.find(params[:environment_id]) + + update_params = declared_params(include_missing: false).extract!(:name, :external_url) + if environment.update(update_params) + present environment, with: Entities::Environment + else + render_validation_error!(environment) + end + end + + desc 'Deletes an existing environment' do + detail 'This feature was introduced in GitLab 8.11.' + success Entities::Environment + end + params do + requires :environment_id, type: Integer, desc: 'The environment ID' + end + delete ':id/environments/:environment_id' do + authorize! :update_environment, user_project + + environment = user_project.environments.find(params[:environment_id]) + + present environment.destroy, with: Entities::Environment + end + end + end + end +end diff --git a/lib/api/v3/files.rb b/lib/api/v3/files.rb new file mode 100644 index 00000000000..13542b0c71c --- /dev/null +++ b/lib/api/v3/files.rb @@ -0,0 +1,138 @@ +module API + module V3 + class Files < Grape::API + helpers do + def commit_params(attrs) + { + file_path: attrs[:file_path], + start_branch: attrs[:branch], + target_branch: attrs[:branch], + commit_message: attrs[:commit_message], + file_content: attrs[:content], + file_content_encoding: attrs[:encoding], + author_email: attrs[:author_email], + author_name: attrs[:author_name] + } + end + + def commit_response(attrs) + { + file_path: attrs[:file_path], + branch: attrs[:branch] + } + end + + params :simple_file_params do + requires :file_path, type: String, desc: 'The path to new file. Ex. lib/class.rb' + requires :branch_name, type: String, desc: 'The name of branch' + requires :commit_message, type: String, desc: 'Commit Message' + optional :author_email, type: String, desc: 'The email of the author' + optional :author_name, type: String, desc: 'The name of the author' + end + + params :extended_file_params do + use :simple_file_params + requires :content, type: String, desc: 'File content' + optional :encoding, type: String, values: %w[base64], desc: 'File encoding' + end + end + + params do + requires :id, type: String, desc: 'The project ID' + end + resource :projects, requirements: { id: %r{[^/]+} } do + desc 'Get a file from repository' + params do + requires :file_path, type: String, desc: 'The path to the file. Ex. lib/class.rb' + requires :ref, type: String, desc: 'The name of branch, tag, or commit' + end + get ":id/repository/files" do + authorize! :download_code, user_project + + commit = user_project.commit(params[:ref]) + not_found!('Commit') unless commit + + repo = user_project.repository + blob = repo.blob_at(commit.sha, params[:file_path]) + not_found!('File') unless blob + + blob.load_all_data!(repo) + status(200) + + { + file_name: blob.name, + file_path: blob.path, + size: blob.size, + encoding: "base64", + content: Base64.strict_encode64(blob.data), + ref: params[:ref], + blob_id: blob.id, + commit_id: commit.id, + last_commit_id: repo.last_commit_id_for_path(commit.sha, params[:file_path]) + } + end + + desc 'Create new file in repository' + params do + use :extended_file_params + end + post ":id/repository/files" do + authorize! :push_code, user_project + + file_params = declared_params(include_missing: false) + file_params[:branch] = file_params.delete(:branch_name) + + result = ::Files::CreateService.new(user_project, current_user, commit_params(file_params)).execute + + if result[:status] == :success + status(201) + commit_response(file_params) + else + render_api_error!(result[:message], 400) + end + end + + desc 'Update existing file in repository' + params do + use :extended_file_params + end + put ":id/repository/files" do + authorize! :push_code, user_project + + file_params = declared_params(include_missing: false) + file_params[:branch] = file_params.delete(:branch_name) + + result = ::Files::UpdateService.new(user_project, current_user, commit_params(file_params)).execute + + if result[:status] == :success + status(200) + commit_response(file_params) + else + http_status = result[:http_status] || 400 + render_api_error!(result[:message], http_status) + end + end + + desc 'Delete an existing file in repository' + params do + use :simple_file_params + end + delete ":id/repository/files" do + authorize! :push_code, user_project + + file_params = declared_params(include_missing: false) + file_params[:branch] = file_params.delete(:branch_name) + + result = ::Files::DestroyService.new(user_project, current_user, commit_params(file_params)).execute + + if result[:status] == :success + status(200) + commit_response(file_params) + else + render_api_error!(result[:message], 400) + end + end + end + end + end +end diff --git a/lib/api/v3/groups.rb b/lib/api/v3/groups.rb new file mode 100644 index 00000000000..c5b37622d79 --- /dev/null +++ b/lib/api/v3/groups.rb @@ -0,0 +1,181 @@ +module API + module V3 + class Groups < Grape::API + include PaginationParams + + before { authenticate! } + + helpers do + params :optional_params do + optional :description, type: String, desc: 'The description of the group' + optional :visibility_level, type: Integer, desc: 'The visibility level of the group' + optional :lfs_enabled, type: Boolean, desc: 'Enable/disable LFS for the projects in this group' + optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access' + end + + params :statistics_params do + optional :statistics, type: Boolean, default: false, desc: 'Include project statistics' + end + + def present_groups(groups, options = {}) + options = options.reverse_merge( + with: Entities::Group, + current_user: current_user, + ) + + groups = groups.with_statistics if options[:statistics] + present paginate(groups), options + end + end + + resource :groups do + desc 'Get a groups list' do + success Entities::Group + end + params do + use :statistics_params + optional :skip_groups, type: Array[Integer], desc: 'Array of group ids to exclude from list' + optional :all_available, type: Boolean, desc: 'Show all group that you have access to' + optional :search, type: String, desc: 'Search for a specific group' + optional :order_by, type: String, values: %w[name path], default: 'name', desc: 'Order by name or path' + optional :sort, type: String, values: %w[asc desc], default: 'asc', desc: 'Sort by asc (ascending) or desc (descending)' + use :pagination + end + get do + groups = if current_user.admin + Group.all + elsif params[:all_available] + GroupsFinder.new.execute(current_user) + else + current_user.groups + end + + groups = groups.search(params[:search]) if params[:search].present? + groups = groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present? + groups = groups.reorder(params[:order_by] => params[:sort]) + + present_groups groups, statistics: params[:statistics] && current_user.is_admin? + end + + desc 'Get list of owned groups for authenticated user' do + success Entities::Group + end + params do + use :pagination + use :statistics_params + end + get '/owned' do + present_groups current_user.owned_groups, statistics: params[:statistics] + end + + desc 'Create a group. Available only for users who can create groups.' do + success Entities::Group + end + params do + requires :name, type: String, desc: 'The name of the group' + requires :path, type: String, desc: 'The path of the group' + optional :parent_id, type: Integer, desc: 'The parent group id for creating nested group' + use :optional_params + end + post do + authorize! :create_group + + group = ::Groups::CreateService.new(current_user, declared_params(include_missing: false)).execute + + if group.persisted? + present group, with: Entities::Group, current_user: current_user + else + render_api_error!("Failed to save group #{group.errors.messages}", 400) + end + end + end + + params do + requires :id, type: String, desc: 'The ID of a group' + end + resource :groups, requirements: { id: %r{[^/]+} } do + desc 'Update a group. Available only for users who can administrate groups.' do + success Entities::Group + end + params do + optional :name, type: String, desc: 'The name of the group' + optional :path, type: String, desc: 'The path of the group' + use :optional_params + at_least_one_of :name, :path, :description, :visibility_level, + :lfs_enabled, :request_access_enabled + end + put ':id' do + group = find_group!(params[:id]) + authorize! :admin_group, group + + if ::Groups::UpdateService.new(group, current_user, declared_params(include_missing: false)).execute + present group, with: Entities::GroupDetail, current_user: current_user + else + render_validation_error!(group) + end + end + + desc 'Get a single group, with containing projects.' do + success Entities::GroupDetail + end + get ":id" do + group = find_group!(params[:id]) + present group, with: Entities::GroupDetail, current_user: current_user + end + + desc 'Remove a group.' + delete ":id" do + group = find_group!(params[:id]) + authorize! :admin_group, group + present ::Groups::DestroyService.new(group, current_user).execute, with: Entities::GroupDetail, current_user: current_user + end + + desc 'Get a list of projects in this group.' do + success Entities::Project + end + params do + optional :archived, type: Boolean, default: false, desc: 'Limit by archived status' + optional :visibility, type: String, values: %w[public internal private], + desc: 'Limit by visibility' + optional :search, type: String, desc: 'Return list of authorized projects matching the search criteria' + optional :order_by, type: String, values: %w[id name path created_at updated_at last_activity_at], + default: 'created_at', desc: 'Return projects ordered by field' + optional :sort, type: String, values: %w[asc desc], default: 'desc', + desc: 'Return projects sorted in ascending and descending order' + optional :simple, type: Boolean, default: false, + desc: 'Return only the ID, URL, name, and path of each project' + optional :owned, type: Boolean, default: false, desc: 'Limit by owned by authenticated user' + optional :starred, type: Boolean, default: false, desc: 'Limit by starred status' + + use :pagination + end + get ":id/projects" do + group = find_group!(params[:id]) + projects = GroupProjectsFinder.new(group).execute(current_user) + projects = filter_projects(projects) + entity = params[:simple] ? ::API::Entities::BasicProjectDetails : Entities::Project + present paginate(projects), with: entity, current_user: current_user + end + + desc 'Transfer a project to the group namespace. Available only for admin.' do + success Entities::GroupDetail + end + params do + requires :project_id, type: String, desc: 'The ID or path of the project' + end + post ":id/projects/:project_id", requirements: { project_id: /.+/ } do + authenticated_as_admin! + group = find_group!(params[:id]) + project = find_project!(params[:project_id]) + result = ::Projects::TransferService.new(project, current_user).execute(group) + + if result + present group, with: Entities::GroupDetail, current_user: current_user + else + render_api_error!("Failed to transfer project #{project.errors.messages}", 400) + end + end + end + end + end +end diff --git a/lib/api/v3/helpers.rb b/lib/api/v3/helpers.rb new file mode 100644 index 00000000000..0f234d4cdad --- /dev/null +++ b/lib/api/v3/helpers.rb @@ -0,0 +1,19 @@ +module API + module V3 + module Helpers + def find_project_issue(id) + IssuesFinder.new(current_user, project_id: user_project.id).find(id) + end + + def find_project_merge_request(id) + MergeRequestsFinder.new(current_user, project_id: user_project.id).find(id) + end + + def find_merge_request_with_access(id, access_level = :read_merge_request) + merge_request = user_project.merge_requests.find(id) + authorize! access_level, merge_request + merge_request + end + end + end +end diff --git a/lib/api/v3/issues.rb b/lib/api/v3/issues.rb new file mode 100644 index 00000000000..715083fc4f8 --- /dev/null +++ b/lib/api/v3/issues.rb @@ -0,0 +1,231 @@ +module API + module V3 + class Issues < Grape::API + include PaginationParams + + before { authenticate! } + + helpers do + def find_issues(args = {}) + args = params.merge(args) + + args.delete(:id) + args[:milestone_title] = args.delete(:milestone) + + match_all_labels = args.delete(:match_all_labels) + labels = args.delete(:labels) + args[:label_name] = labels if match_all_labels + + # IssuesFinder expects iids + args[:iids] = args.delete(:iid) if args.key?(:iid) + + issues = IssuesFinder.new(current_user, args).execute.inc_notes_with_associations + + if !match_all_labels && labels.present? + issues = issues.includes(:labels).where('labels.title' => labels.split(',')) + end + + issues.reorder(args[:order_by] => args[:sort]) + end + + params :issues_params do + optional :labels, type: String, desc: 'Comma-separated list of label names' + optional :milestone, type: String, desc: 'Milestone title' + optional :order_by, type: String, values: %w[created_at updated_at], default: 'created_at', + desc: 'Return issues ordered by `created_at` or `updated_at` fields.' + optional :sort, type: String, values: %w[asc desc], default: 'desc', + desc: 'Return issues sorted in `asc` or `desc` order.' + optional :milestone, type: String, desc: 'Return issues for a specific milestone' + use :pagination + end + + params :issue_params do + optional :description, type: String, desc: 'The description of an issue' + optional :assignee_id, type: Integer, desc: 'The ID of a user to assign issue' + optional :milestone_id, type: Integer, desc: 'The ID of a milestone to assign issue' + optional :labels, type: String, desc: 'Comma-separated list of label names' + optional :due_date, type: String, desc: 'Date time string in the format YEAR-MONTH-DAY' + optional :confidential, type: Boolean, desc: 'Boolean parameter if the issue should be confidential' + end + end + + resource :issues do + desc "Get currently authenticated user's issues" do + success ::API::Entities::Issue + end + params do + optional :state, type: String, values: %w[opened closed all], default: 'all', + desc: 'Return opened, closed, or all issues' + use :issues_params + end + get do + issues = find_issues(scope: 'authored') + + present paginate(issues), with: ::API::Entities::Issue, current_user: current_user + end + end + + params do + requires :id, type: String, desc: 'The ID of a group' + end + resource :groups, requirements: { id: %r{[^/]+} } do + desc 'Get a list of group issues' do + success ::API::Entities::Issue + end + params do + optional :state, type: String, values: %w[opened closed all], default: 'all', + desc: 'Return opened, closed, or all issues' + use :issues_params + end + get ":id/issues" do + group = find_group!(params[:id]) + + issues = find_issues(group_id: group.id, match_all_labels: true) + + present paginate(issues), with: ::API::Entities::Issue, current_user: current_user + end + end + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: { id: %r{[^/]+} } do + include TimeTrackingEndpoints + + desc 'Get a list of project issues' do + detail 'iid filter is deprecated have been removed on V4' + success ::API::Entities::Issue + end + params do + optional :state, type: String, values: %w[opened closed all], default: 'all', + desc: 'Return opened, closed, or all issues' + optional :iid, type: Integer, desc: 'Return the issue having the given `iid`' + use :issues_params + end + get ":id/issues" do + project = find_project!(params[:id]) + + issues = find_issues(project_id: project.id) + + present paginate(issues), with: ::API::Entities::Issue, current_user: current_user, project: user_project + end + + desc 'Get a single project issue' do + success ::API::Entities::Issue + end + params do + requires :issue_id, type: Integer, desc: 'The ID of a project issue' + end + get ":id/issues/:issue_id" do + issue = find_project_issue(params[:issue_id]) + present issue, with: ::API::Entities::Issue, current_user: current_user, project: user_project + end + + desc 'Create a new project issue' do + success ::API::Entities::Issue + end + params do + requires :title, type: String, desc: 'The title of an issue' + optional :created_at, type: DateTime, + desc: 'Date time when the issue was created. Available only for admins and project owners.' + optional :merge_request_for_resolving_discussions, type: Integer, + desc: 'The IID of a merge request for which to resolve discussions' + use :issue_params + end + post ':id/issues' do + # Setting created_at time only allowed for admins and project owners + unless current_user.admin? || user_project.owner == current_user + params.delete(:created_at) + end + + issue_params = declared_params(include_missing: false) + issue_params = issue_params.merge(merge_request_to_resolve_discussions_of: issue_params.delete(:merge_request_for_resolving_discussions)) + + issue = ::Issues::CreateService.new(user_project, + current_user, + issue_params.merge(request: request, api: true)).execute + render_spam_error! if issue.spam? + + if issue.valid? + present issue, with: ::API::Entities::Issue, current_user: current_user, project: user_project + else + render_validation_error!(issue) + end + end + + desc 'Update an existing issue' do + success ::API::Entities::Issue + end + params do + requires :issue_id, type: Integer, desc: 'The ID of a project issue' + optional :title, type: String, desc: 'The title of an issue' + optional :updated_at, type: DateTime, + desc: 'Date time when the issue was updated. Available only for admins and project owners.' + optional :state_event, type: String, values: %w[reopen close], desc: 'State of the issue' + use :issue_params + at_least_one_of :title, :description, :assignee_id, :milestone_id, + :labels, :created_at, :due_date, :confidential, :state_event + end + put ':id/issues/:issue_id' do + issue = user_project.issues.find(params.delete(:issue_id)) + authorize! :update_issue, issue + + # Setting created_at time only allowed for admins and project owners + unless current_user.admin? || user_project.owner == current_user + params.delete(:updated_at) + end + + update_params = declared_params(include_missing: false).merge(request: request, api: true) + + issue = ::Issues::UpdateService.new(user_project, + current_user, + update_params).execute(issue) + + render_spam_error! if issue.spam? + + if issue.valid? + present issue, with: ::API::Entities::Issue, current_user: current_user, project: user_project + else + render_validation_error!(issue) + end + end + + desc 'Move an existing issue' do + success ::API::Entities::Issue + end + params do + requires :issue_id, type: Integer, desc: 'The ID of a project issue' + requires :to_project_id, type: Integer, desc: 'The ID of the new project' + end + post ':id/issues/:issue_id/move' do + issue = user_project.issues.find_by(id: params[:issue_id]) + not_found!('Issue') unless issue + + new_project = Project.find_by(id: params[:to_project_id]) + not_found!('Project') unless new_project + + begin + issue = ::Issues::MoveService.new(user_project, current_user).execute(issue, new_project) + present issue, with: ::API::Entities::Issue, current_user: current_user, project: user_project + rescue ::Issues::MoveService::MoveError => error + render_api_error!(error.message, 400) + end + end + + desc 'Delete a project issue' + params do + requires :issue_id, type: Integer, desc: 'The ID of a project issue' + end + delete ":id/issues/:issue_id" do + issue = user_project.issues.find_by(id: params[:issue_id]) + not_found!('Issue') unless issue + + authorize!(:destroy_issue, issue) + + status(200) + issue.destroy + end + end + end + end +end diff --git a/lib/api/v3/labels.rb b/lib/api/v3/labels.rb new file mode 100644 index 00000000000..bd5eb2175e8 --- /dev/null +++ b/lib/api/v3/labels.rb @@ -0,0 +1,34 @@ +module API + module V3 + class Labels < Grape::API + before { authenticate! } + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: { id: %r{[^/]+} } do + desc 'Get all labels of the project' do + success ::API::Entities::Label + end + get ':id/labels' do + present available_labels, with: ::API::Entities::Label, current_user: current_user, project: user_project + end + + desc 'Delete an existing label' do + success ::API::Entities::Label + end + params do + requires :name, type: String, desc: 'The name of the label to be deleted' + end + delete ':id/labels' do + authorize! :admin_label, user_project + + label = user_project.labels.find_by(title: params[:name]) + not_found!('Label') unless label + + present label.destroy, with: ::API::Entities::Label, current_user: current_user, project: user_project + end + end + end + end +end diff --git a/lib/api/v3/members.rb b/lib/api/v3/members.rb new file mode 100644 index 00000000000..684860b553e --- /dev/null +++ b/lib/api/v3/members.rb @@ -0,0 +1,134 @@ +module API + module V3 + class Members < Grape::API + include PaginationParams + + before { authenticate! } + + helpers ::API::Helpers::MembersHelpers + + %w[group project].each do |source_type| + params do + requires :id, type: String, desc: "The #{source_type} ID" + end + resource source_type.pluralize, requirements: { id: %r{[^/]+} } do + desc 'Gets a list of group or project members viewable by the authenticated user.' do + success ::API::Entities::Member + end + params do + optional :query, type: String, desc: 'A query string to search for members' + use :pagination + end + get ":id/members" do + source = find_source(source_type, params[:id]) + + users = source.users + users = users.merge(User.search(params[:query])) if params[:query] + + present paginate(users), with: ::API::Entities::Member, source: source + end + + desc 'Gets a member of a group or project.' do + success ::API::Entities::Member + end + params do + requires :user_id, type: Integer, desc: 'The user ID of the member' + end + get ":id/members/:user_id" do + source = find_source(source_type, params[:id]) + + members = source.members + member = members.find_by!(user_id: params[:user_id]) + + present member.user, with: ::API::Entities::Member, member: member + end + + desc 'Adds a member to a group or project.' do + success ::API::Entities::Member + end + params do + requires :user_id, type: Integer, desc: 'The user ID of the new member' + requires :access_level, type: Integer, desc: 'A valid access level (defaults: `30`, developer access level)' + optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY' + end + post ":id/members" do + source = find_source(source_type, params[:id]) + authorize_admin_source!(source_type, source) + + member = source.members.find_by(user_id: params[:user_id]) + + # We need this explicit check because `source.add_user` doesn't + # currently return the member created so it would return 201 even if + # the member already existed... + # The `source_type == 'group'` check is to ensure back-compatibility + # but 409 behavior should be used for both project and group members in 9.0! + conflict!('Member already exists') if source_type == 'group' && member + + unless member + member = source.add_user(params[:user_id], params[:access_level], current_user: current_user, expires_at: params[:expires_at]) + end + if member.persisted? && member.valid? + present member.user, with: ::API::Entities::Member, member: member + else + # This is to ensure back-compatibility but 400 behavior should be used + # for all validation errors in 9.0! + render_api_error!('Access level is not known', 422) if member.errors.key?(:access_level) + render_validation_error!(member) + end + end + + desc 'Updates a member of a group or project.' do + success ::API::Entities::Member + end + params do + requires :user_id, type: Integer, desc: 'The user ID of the new member' + requires :access_level, type: Integer, desc: 'A valid access level' + optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY' + end + put ":id/members/:user_id" do + source = find_source(source_type, params.delete(:id)) + authorize_admin_source!(source_type, source) + + member = source.members.find_by!(user_id: params.delete(:user_id)) + + if member.update_attributes(declared_params(include_missing: false)) + present member.user, with: ::API::Entities::Member, member: member + else + # This is to ensure back-compatibility but 400 behavior should be used + # for all validation errors in 9.0! + render_api_error!('Access level is not known', 422) if member.errors.key?(:access_level) + render_validation_error!(member) + end + end + + desc 'Removes a user from a group or project.' + params do + requires :user_id, type: Integer, desc: 'The user ID of the member' + end + delete ":id/members/:user_id" do + source = find_source(source_type, params[:id]) + + # This is to ensure back-compatibility but find_by! should be used + # in that casse in 9.0! + member = source.members.find_by(user_id: params[:user_id]) + + # This is to ensure back-compatibility but this should be removed in + # favor of find_by! in 9.0! + not_found!("Member: user_id:#{params[:user_id]}") if source_type == 'group' && member.nil? + + # This is to ensure back-compatibility but 204 behavior should be used + # for all DELETE endpoints in 9.0! + if member.nil? + status(200 ) + { message: "Access revoked", id: params[:user_id].to_i } + else + ::Members::DestroyService.new(source, current_user, declared_params).execute + + present member.user, with: ::API::Entities::Member, member: member + end + end + end + end + end + end +end diff --git a/lib/api/v3/merge_request_diffs.rb b/lib/api/v3/merge_request_diffs.rb new file mode 100644 index 00000000000..35f462e907b --- /dev/null +++ b/lib/api/v3/merge_request_diffs.rb @@ -0,0 +1,44 @@ +module API + module V3 + # MergeRequestDiff API + class MergeRequestDiffs < Grape::API + before { authenticate! } + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: { id: %r{[^/]+} } do + desc 'Get a list of merge request diff versions' do + detail 'This feature was introduced in GitLab 8.12.' + success ::API::Entities::MergeRequestDiff + end + + params do + requires :merge_request_id, type: Integer, desc: 'The ID of a merge request' + end + + get ":id/merge_requests/:merge_request_id/versions" do + merge_request = find_merge_request_with_access(params[:merge_request_id]) + + present merge_request.merge_request_diffs, with: ::API::Entities::MergeRequestDiff + end + + desc 'Get a single merge request diff version' do + detail 'This feature was introduced in GitLab 8.12.' + success ::API::Entities::MergeRequestDiffFull + end + + params do + requires :merge_request_id, type: Integer, desc: 'The ID of a merge request' + requires :version_id, type: Integer, desc: 'The ID of a merge request diff version' + end + + get ":id/merge_requests/:merge_request_id/versions/:version_id" do + merge_request = find_merge_request_with_access(params[:merge_request_id]) + + present merge_request.merge_request_diffs.find(params[:version_id]), with: ::API::Entities::MergeRequestDiffFull + end + end + end + end +end diff --git a/lib/api/v3/merge_requests.rb b/lib/api/v3/merge_requests.rb new file mode 100644 index 00000000000..3077240e650 --- /dev/null +++ b/lib/api/v3/merge_requests.rb @@ -0,0 +1,290 @@ +module API + module V3 + class MergeRequests < Grape::API + include PaginationParams + + DEPRECATION_MESSAGE = 'This endpoint is deprecated and has been removed on V4'.freeze + + before { authenticate! } + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: { id: %r{[^/]+} } do + include TimeTrackingEndpoints + + helpers do + def handle_merge_request_errors!(errors) + if errors[:project_access].any? + error!(errors[:project_access], 422) + elsif errors[:branch_conflict].any? + error!(errors[:branch_conflict], 422) + elsif errors[:validate_fork].any? + error!(errors[:validate_fork], 422) + elsif errors[:validate_branches].any? + conflict!(errors[:validate_branches]) + end + + render_api_error!(errors, 400) + end + + def issue_entity(project) + if project.has_external_issue_tracker? + ::API::Entities::ExternalIssue + else + ::API::Entities::Issue + end + end + + params :optional_params do + optional :description, type: String, desc: 'The description of the merge request' + optional :assignee_id, type: Integer, desc: 'The ID of a user to assign the merge request' + optional :milestone_id, type: Integer, desc: 'The ID of a milestone to assign the merge request' + optional :labels, type: String, desc: 'Comma-separated list of label names' + optional :remove_source_branch, type: Boolean, desc: 'Remove source branch when merging' + end + end + + desc 'List merge requests' do + detail 'iid filter is deprecated have been removed on V4' + success ::API::V3::Entities::MergeRequest + end + params do + optional :state, type: String, values: %w[opened closed merged all], default: 'all', + desc: 'Return opened, closed, merged, or all merge requests' + optional :order_by, type: String, values: %w[created_at updated_at], default: 'created_at', + desc: 'Return merge requests ordered by `created_at` or `updated_at` fields.' + optional :sort, type: String, values: %w[asc desc], default: 'desc', + desc: 'Return merge requests sorted in `asc` or `desc` order.' + optional :iid, type: Array[Integer], desc: 'The IID of the merge requests' + use :pagination + end + get ":id/merge_requests" do + authorize! :read_merge_request, user_project + + merge_requests = user_project.merge_requests.inc_notes_with_associations + merge_requests = filter_by_iid(merge_requests, params[:iid]) if params[:iid].present? + + merge_requests = + case params[:state] + when 'opened' then merge_requests.opened + when 'closed' then merge_requests.closed + when 'merged' then merge_requests.merged + else merge_requests + end + + merge_requests = merge_requests.reorder(params[:order_by] => params[:sort]) + present paginate(merge_requests), with: ::API::V3::Entities::MergeRequest, current_user: current_user, project: user_project + end + + desc 'Create a merge request' do + success ::API::V3::Entities::MergeRequest + end + params do + requires :title, type: String, desc: 'The title of the merge request' + requires :source_branch, type: String, desc: 'The source branch' + requires :target_branch, type: String, desc: 'The target branch' + optional :target_project_id, type: Integer, + desc: 'The target project of the merge request defaults to the :id of the project' + use :optional_params + end + post ":id/merge_requests" do + authorize! :create_merge_request, user_project + + mr_params = declared_params(include_missing: false) + mr_params[:force_remove_source_branch] = mr_params.delete(:remove_source_branch) if mr_params[:remove_source_branch].present? + + merge_request = ::MergeRequests::CreateService.new(user_project, current_user, mr_params).execute + + if merge_request.valid? + present merge_request, with: ::API::V3::Entities::MergeRequest, current_user: current_user, project: user_project + else + handle_merge_request_errors! merge_request.errors + end + end + + desc 'Delete a merge request' + params do + requires :merge_request_id, type: Integer, desc: 'The ID of a merge request' + end + delete ":id/merge_requests/:merge_request_id" do + merge_request = find_project_merge_request(params[:merge_request_id]) + + authorize!(:destroy_merge_request, merge_request) + + status(200) + merge_request.destroy + end + + params do + requires :merge_request_id, type: Integer, desc: 'The ID of a merge request' + end + { ":id/merge_request/:merge_request_id" => :deprecated, ":id/merge_requests/:merge_request_id" => :ok }.each do |path, status| + desc 'Get a single merge request' do + if status == :deprecated + detail DEPRECATION_MESSAGE + end + success ::API::V3::Entities::MergeRequest + end + get path do + merge_request = find_merge_request_with_access(params[:merge_request_id]) + + present merge_request, with: ::API::V3::Entities::MergeRequest, current_user: current_user, project: user_project + end + + desc 'Get the commits of a merge request' do + success ::API::Entities::RepoCommit + end + get "#{path}/commits" do + merge_request = find_merge_request_with_access(params[:merge_request_id]) + + present merge_request.commits, with: ::API::Entities::RepoCommit + end + + desc 'Show the merge request changes' do + success ::API::Entities::MergeRequestChanges + end + get "#{path}/changes" do + merge_request = find_merge_request_with_access(params[:merge_request_id]) + + present merge_request, with: ::API::Entities::MergeRequestChanges, current_user: current_user + end + + desc 'Update a merge request' do + success ::API::V3::Entities::MergeRequest + end + params do + optional :title, type: String, allow_blank: false, desc: 'The title of the merge request' + optional :target_branch, type: String, allow_blank: false, desc: 'The target branch' + optional :state_event, type: String, values: %w[close reopen merge], + desc: 'Status of the merge request' + use :optional_params + at_least_one_of :title, :target_branch, :description, :assignee_id, + :milestone_id, :labels, :state_event, + :remove_source_branch + end + put path do + merge_request = find_merge_request_with_access(params.delete(:merge_request_id), :update_merge_request) + + mr_params = declared_params(include_missing: false) + mr_params[:force_remove_source_branch] = mr_params.delete(:remove_source_branch) if mr_params[:remove_source_branch].present? + + merge_request = ::MergeRequests::UpdateService.new(user_project, current_user, mr_params).execute(merge_request) + + if merge_request.valid? + present merge_request, with: ::API::V3::Entities::MergeRequest, current_user: current_user, project: user_project + else + handle_merge_request_errors! merge_request.errors + end + end + + desc 'Merge a merge request' do + success ::API::V3::Entities::MergeRequest + end + params do + optional :merge_commit_message, type: String, desc: 'Custom merge commit message' + optional :should_remove_source_branch, type: Boolean, + desc: 'When true, the source branch will be deleted if possible' + optional :merge_when_build_succeeds, type: Boolean, + desc: 'When true, this merge request will be merged when the build succeeds' + optional :sha, type: String, desc: 'When present, must have the HEAD SHA of the source branch' + end + put "#{path}/merge" do + merge_request = find_project_merge_request(params[:merge_request_id]) + + # Merge request can not be merged + # because user dont have permissions to push into target branch + unauthorized! unless merge_request.can_be_merged_by?(current_user) + + not_allowed! unless merge_request.mergeable_state? + + render_api_error!('Branch cannot be merged', 406) unless merge_request.mergeable? + + if params[:sha] && merge_request.diff_head_sha != params[:sha] + render_api_error!("SHA does not match HEAD of source branch: #{merge_request.diff_head_sha}", 409) + end + + merge_params = { + commit_message: params[:merge_commit_message], + should_remove_source_branch: params[:should_remove_source_branch] + } + + if params[:merge_when_build_succeeds] && merge_request.head_pipeline && merge_request.head_pipeline.active? + ::MergeRequests::MergeWhenPipelineSucceedsService + .new(merge_request.target_project, current_user, merge_params) + .execute(merge_request) + else + ::MergeRequests::MergeService + .new(merge_request.target_project, current_user, merge_params) + .execute(merge_request) + end + + present merge_request, with: ::API::V3::Entities::MergeRequest, current_user: current_user, project: user_project + end + + desc 'Cancel merge if "Merge When Build succeeds" is enabled' do + success ::API::V3::Entities::MergeRequest + end + post "#{path}/cancel_merge_when_build_succeeds" do + merge_request = find_project_merge_request(params[:merge_request_id]) + + unauthorized! unless merge_request.can_cancel_merge_when_pipeline_succeeds?(current_user) + + ::MergeRequest::MergeWhenPipelineSucceedsService + .new(merge_request.target_project, current_user) + .cancel(merge_request) + end + + desc 'Get the comments of a merge request' do + detail 'Duplicate. DEPRECATED and HAS BEEN REMOVED in V4' + success ::API::Entities::MRNote + end + params do + use :pagination + end + get "#{path}/comments" do + merge_request = find_merge_request_with_access(params[:merge_request_id]) + present paginate(merge_request.notes.fresh), with: ::API::Entities::MRNote + end + + desc 'Post a comment to a merge request' do + detail 'Duplicate. DEPRECATED and HAS BEEN REMOVED in V4' + success ::API::Entities::MRNote + end + params do + requires :note, type: String, desc: 'The text of the comment' + end + post "#{path}/comments" do + merge_request = find_merge_request_with_access(params[:merge_request_id], :create_note) + + opts = { + note: params[:note], + noteable_type: 'MergeRequest', + noteable_id: merge_request.id + } + + note = ::Notes::CreateService.new(user_project, current_user, opts).execute + + if note.save + present note, with: ::API::Entities::MRNote + else + render_api_error!("Failed to save note #{note.errors.messages}", 400) + end + end + + desc 'List issues that will be closed on merge' do + success ::API::Entities::MRNote + end + params do + use :pagination + end + get "#{path}/closes_issues" do + merge_request = find_merge_request_with_access(params[:merge_request_id]) + issues = ::Kaminari.paginate_array(merge_request.closes_issues(current_user)) + present paginate(issues), with: issue_entity(user_project), current_user: current_user + end + end + end + end + end +end diff --git a/lib/api/v3/milestones.rb b/lib/api/v3/milestones.rb new file mode 100644 index 00000000000..be90cec4afc --- /dev/null +++ b/lib/api/v3/milestones.rb @@ -0,0 +1,64 @@ +module API + module V3 + class Milestones < Grape::API + include PaginationParams + + before { authenticate! } + + helpers do + def filter_milestones_state(milestones, state) + case state + when 'active' then milestones.active + when 'closed' then milestones.closed + else milestones + end + end + end + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: { id: %r{[^/]+} } do + desc 'Get a list of project milestones' do + success ::API::Entities::Milestone + end + params do + optional :state, type: String, values: %w[active closed all], default: 'all', + desc: 'Return "active", "closed", or "all" milestones' + optional :iid, type: Array[Integer], desc: 'The IID of the milestone' + use :pagination + end + get ":id/milestones" do + authorize! :read_milestone, user_project + + milestones = user_project.milestones + milestones = filter_milestones_state(milestones, params[:state]) + milestones = filter_by_iid(milestones, params[:iid]) if params[:iid].present? + + present paginate(milestones), with: ::API::Entities::Milestone + end + + desc 'Get all issues for a single project milestone' do + success ::API::Entities::Issue + end + params do + requires :milestone_id, type: Integer, desc: 'The ID of a project milestone' + use :pagination + end + get ':id/milestones/:milestone_id/issues' do + authorize! :read_milestone, user_project + + milestone = user_project.milestones.find(params[:milestone_id]) + + finder_params = { + project_id: user_project.id, + milestone_title: milestone.title + } + + issues = IssuesFinder.new(current_user, finder_params).execute + present paginate(issues), with: ::API::Entities::Issue, current_user: current_user, project: user_project + end + end + end + end +end diff --git a/lib/api/v3/notes.rb b/lib/api/v3/notes.rb new file mode 100644 index 00000000000..4f8e0eff4ff --- /dev/null +++ b/lib/api/v3/notes.rb @@ -0,0 +1,148 @@ +module API + module V3 + class Notes < Grape::API + include PaginationParams + + before { authenticate! } + + NOTEABLE_TYPES = [Issue, MergeRequest, Snippet].freeze + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: { id: %r{[^/]+} } do + NOTEABLE_TYPES.each do |noteable_type| + noteables_str = noteable_type.to_s.underscore.pluralize + + desc 'Get a list of project +noteable+ notes' do + success ::API::V3::Entities::Note + end + params do + requires :noteable_id, type: Integer, desc: 'The ID of the noteable' + use :pagination + end + get ":id/#{noteables_str}/:noteable_id/notes" do + noteable = user_project.send(noteables_str.to_sym).find(params[:noteable_id]) + + if can?(current_user, noteable_read_ability_name(noteable), noteable) + # We exclude notes that are cross-references and that cannot be viewed + # by the current user. By doing this exclusion at this level and not + # at the DB query level (which we cannot in that case), the current + # page can have less elements than :per_page even if + # there's more than one page. + notes = + # paginate() only works with a relation. This could lead to a + # mismatch between the pagination headers info and the actual notes + # array returned, but this is really a edge-case. + paginate(noteable.notes). + reject { |n| n.cross_reference_not_visible_for?(current_user) } + present notes, with: ::API::V3::Entities::Note + else + not_found!("Notes") + end + end + + desc 'Get a single +noteable+ note' do + success ::API::V3::Entities::Note + end + params do + requires :note_id, type: Integer, desc: 'The ID of a note' + requires :noteable_id, type: Integer, desc: 'The ID of the noteable' + end + get ":id/#{noteables_str}/:noteable_id/notes/:note_id" do + noteable = user_project.send(noteables_str.to_sym).find(params[:noteable_id]) + note = noteable.notes.find(params[:note_id]) + can_read_note = can?(current_user, noteable_read_ability_name(noteable), noteable) && !note.cross_reference_not_visible_for?(current_user) + + if can_read_note + present note, with: ::API::V3::Entities::Note + else + not_found!("Note") + end + end + + desc 'Create a new +noteable+ note' do + success ::API::V3::Entities::Note + end + params do + requires :noteable_id, type: Integer, desc: 'The ID of the noteable' + requires :body, type: String, desc: 'The content of a note' + optional :created_at, type: String, desc: 'The creation date of the note' + end + post ":id/#{noteables_str}/:noteable_id/notes" do + opts = { + note: params[:body], + noteable_type: noteables_str.classify, + noteable_id: params[:noteable_id] + } + + noteable = user_project.send(noteables_str.to_sym).find(params[:noteable_id]) + + if can?(current_user, noteable_read_ability_name(noteable), noteable) + if params[:created_at] && (current_user.is_admin? || user_project.owner == current_user) + opts[:created_at] = params[:created_at] + end + + note = ::Notes::CreateService.new(user_project, current_user, opts).execute + if note.valid? + present note, with: ::API::V3::Entities.const_get(note.class.name) + else + not_found!("Note #{note.errors.messages}") + end + else + not_found!("Note") + end + end + + desc 'Update an existing +noteable+ note' do + success ::API::V3::Entities::Note + end + params do + requires :noteable_id, type: Integer, desc: 'The ID of the noteable' + requires :note_id, type: Integer, desc: 'The ID of a note' + requires :body, type: String, desc: 'The content of a note' + end + put ":id/#{noteables_str}/:noteable_id/notes/:note_id" do + note = user_project.notes.find(params[:note_id]) + + authorize! :admin_note, note + + opts = { + note: params[:body] + } + + note = ::Notes::UpdateService.new(user_project, current_user, opts).execute(note) + + if note.valid? + present note, with: ::API::V3::Entities::Note + else + render_api_error!("Failed to save note #{note.errors.messages}", 400) + end + end + + desc 'Delete a +noteable+ note' do + success ::API::V3::Entities::Note + end + params do + requires :noteable_id, type: Integer, desc: 'The ID of the noteable' + requires :note_id, type: Integer, desc: 'The ID of a note' + end + delete ":id/#{noteables_str}/:noteable_id/notes/:note_id" do + note = user_project.notes.find(params[:note_id]) + authorize! :admin_note, note + + ::Notes::DestroyService.new(user_project, current_user).execute(note) + + present note, with: ::API::V3::Entities::Note + end + end + end + + helpers do + def noteable_read_ability_name(noteable) + "read_#{noteable.class.to_s.underscore}".to_sym + end + end + end + end +end diff --git a/lib/api/v3/pipelines.rb b/lib/api/v3/pipelines.rb new file mode 100644 index 00000000000..82827249244 --- /dev/null +++ b/lib/api/v3/pipelines.rb @@ -0,0 +1,36 @@ +module API + module V3 + class Pipelines < Grape::API + include PaginationParams + + before { authenticate! } + + params do + requires :id, type: String, desc: 'The project ID' + end + resource :projects, requirements: { id: %r{[^/]+} } do + desc 'Get all Pipelines of the project' do + detail 'This feature was introduced in GitLab 8.11.' + success ::API::Entities::Pipeline + end + params do + use :pagination + optional :scope, type: String, values: %w(running branches tags), + desc: 'Either running, branches, or tags' + end + get ':id/pipelines' do + authorize! :read_pipeline, user_project + + pipelines = PipelinesFinder.new(user_project).execute(scope: params[:scope]) + present paginate(pipelines), with: ::API::Entities::Pipeline + end + end + + helpers do + def pipeline + @pipeline ||= user_project.pipelines.find(params[:pipeline_id]) + end + end + end + end +end diff --git a/lib/api/v3/project_hooks.rb b/lib/api/v3/project_hooks.rb new file mode 100644 index 00000000000..94614bfc8b6 --- /dev/null +++ b/lib/api/v3/project_hooks.rb @@ -0,0 +1,106 @@ +module API + module V3 + class ProjectHooks < Grape::API + include PaginationParams + + before { authenticate! } + before { authorize_admin_project } + + helpers do + params :project_hook_properties do + requires :url, type: String, desc: "The URL to send the request to" + optional :push_events, type: Boolean, desc: "Trigger hook on push events" + optional :issues_events, type: Boolean, desc: "Trigger hook on issues events" + optional :merge_requests_events, type: Boolean, desc: "Trigger hook on merge request events" + optional :tag_push_events, type: Boolean, desc: "Trigger hook on tag push events" + optional :note_events, type: Boolean, desc: "Trigger hook on note(comment) events" + optional :build_events, type: Boolean, desc: "Trigger hook on build events" + optional :pipeline_events, type: Boolean, desc: "Trigger hook on pipeline events" + optional :wiki_page_events, type: Boolean, desc: "Trigger hook on wiki events" + optional :enable_ssl_verification, type: Boolean, desc: "Do SSL verification when triggering the hook" + optional :token, type: String, desc: "Secret token to validate received payloads; this will not be returned in the response" + end + end + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: { id: %r{[^/]+} } do + desc 'Get project hooks' do + success ::API::V3::Entities::ProjectHook + end + params do + use :pagination + end + get ":id/hooks" do + hooks = paginate user_project.hooks + + present hooks, with: ::API::V3::Entities::ProjectHook + end + + desc 'Get a project hook' do + success ::API::V3::Entities::ProjectHook + end + params do + requires :hook_id, type: Integer, desc: 'The ID of a project hook' + end + get ":id/hooks/:hook_id" do + hook = user_project.hooks.find(params[:hook_id]) + present hook, with: ::API::V3::Entities::ProjectHook + end + + desc 'Add hook to project' do + success ::API::V3::Entities::ProjectHook + end + params do + use :project_hook_properties + end + post ":id/hooks" do + hook = user_project.hooks.new(declared_params(include_missing: false)) + + if hook.save + present hook, with: ::API::V3::Entities::ProjectHook + else + error!("Invalid url given", 422) if hook.errors[:url].present? + + not_found!("Project hook #{hook.errors.messages}") + end + end + + desc 'Update an existing project hook' do + success ::API::V3::Entities::ProjectHook + end + params do + requires :hook_id, type: Integer, desc: "The ID of the hook to update" + use :project_hook_properties + end + put ":id/hooks/:hook_id" do + hook = user_project.hooks.find(params.delete(:hook_id)) + + if hook.update_attributes(declared_params(include_missing: false)) + present hook, with: ::API::V3::Entities::ProjectHook + else + error!("Invalid url given", 422) if hook.errors[:url].present? + + not_found!("Project hook #{hook.errors.messages}") + end + end + + desc 'Deletes project hook' do + success ::API::V3::Entities::ProjectHook + end + params do + requires :hook_id, type: Integer, desc: 'The ID of the hook to delete' + end + delete ":id/hooks/:hook_id" do + begin + present user_project.hooks.destroy(params[:hook_id]), with: ::API::V3::Entities::ProjectHook + rescue + # ProjectHook can raise Error if hook_id not found + not_found!("Error deleting hook #{params[:hook_id]}") + end + end + end + end + end +end diff --git a/lib/api/v3/project_snippets.rb b/lib/api/v3/project_snippets.rb new file mode 100644 index 00000000000..fc065a22d74 --- /dev/null +++ b/lib/api/v3/project_snippets.rb @@ -0,0 +1,143 @@ +module API + module V3 + class ProjectSnippets < Grape::API + include PaginationParams + + before { authenticate! } + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: { id: %r{[^/]+} } do + helpers do + def handle_project_member_errors(errors) + if errors[:project_access].any? + error!(errors[:project_access], 422) + end + not_found! + end + + def snippets_for_current_user + finder_params = { filter: :by_project, project: user_project } + SnippetsFinder.new.execute(current_user, finder_params) + end + end + + desc 'Get all project snippets' do + success ::API::V3::Entities::ProjectSnippet + end + params do + use :pagination + end + get ":id/snippets" do + present paginate(snippets_for_current_user), with: ::API::V3::Entities::ProjectSnippet + end + + desc 'Get a single project snippet' do + success ::API::V3::Entities::ProjectSnippet + end + params do + requires :snippet_id, type: Integer, desc: 'The ID of a project snippet' + end + get ":id/snippets/:snippet_id" do + snippet = snippets_for_current_user.find(params[:snippet_id]) + present snippet, with: ::API::V3::Entities::ProjectSnippet + end + + desc 'Create a new project snippet' do + success ::API::V3::Entities::ProjectSnippet + end + params do + requires :title, type: String, desc: 'The title of the snippet' + requires :file_name, type: String, desc: 'The file name of the snippet' + requires :code, type: String, desc: 'The content of the snippet' + requires :visibility_level, type: Integer, + values: [Gitlab::VisibilityLevel::PRIVATE, + Gitlab::VisibilityLevel::INTERNAL, + Gitlab::VisibilityLevel::PUBLIC], + desc: 'The visibility level of the snippet' + end + post ":id/snippets" do + authorize! :create_project_snippet, user_project + snippet_params = declared_params.merge(request: request, api: true) + snippet_params[:content] = snippet_params.delete(:code) + + snippet = CreateSnippetService.new(user_project, current_user, snippet_params).execute + + render_spam_error! if snippet.spam? + + if snippet.persisted? + present snippet, with: ::API::V3::Entities::ProjectSnippet + else + render_validation_error!(snippet) + end + end + + desc 'Update an existing project snippet' do + success ::API::V3::Entities::ProjectSnippet + end + params do + requires :snippet_id, type: Integer, desc: 'The ID of a project snippet' + optional :title, type: String, desc: 'The title of the snippet' + optional :file_name, type: String, desc: 'The file name of the snippet' + optional :code, type: String, desc: 'The content of the snippet' + optional :visibility_level, type: Integer, + values: [Gitlab::VisibilityLevel::PRIVATE, + Gitlab::VisibilityLevel::INTERNAL, + Gitlab::VisibilityLevel::PUBLIC], + desc: 'The visibility level of the snippet' + at_least_one_of :title, :file_name, :code, :visibility_level + end + put ":id/snippets/:snippet_id" do + snippet = snippets_for_current_user.find_by(id: params.delete(:snippet_id)) + not_found!('Snippet') unless snippet + + authorize! :update_project_snippet, snippet + + snippet_params = declared_params(include_missing: false) + .merge(request: request, api: true) + + snippet_params[:content] = snippet_params.delete(:code) if snippet_params[:code].present? + + UpdateSnippetService.new(user_project, current_user, snippet, + snippet_params).execute + + render_spam_error! if snippet.spam? + + if snippet.valid? + present snippet, with: ::API::V3::Entities::ProjectSnippet + else + render_validation_error!(snippet) + end + end + + desc 'Delete a project snippet' + params do + requires :snippet_id, type: Integer, desc: 'The ID of a project snippet' + end + delete ":id/snippets/:snippet_id" do + snippet = snippets_for_current_user.find_by(id: params[:snippet_id]) + not_found!('Snippet') unless snippet + + authorize! :admin_project_snippet, snippet + snippet.destroy + + status(200) + end + + desc 'Get a raw project snippet' + params do + requires :snippet_id, type: Integer, desc: 'The ID of a project snippet' + end + get ":id/snippets/:snippet_id/raw" do + snippet = snippets_for_current_user.find_by(id: params[:snippet_id]) + not_found!('Snippet') unless snippet + + env['api.format'] = :txt + content_type 'text/plain' + present snippet.content + end + end + end + end +end diff --git a/lib/api/v3/projects.rb b/lib/api/v3/projects.rb new file mode 100644 index 00000000000..b753dbab381 --- /dev/null +++ b/lib/api/v3/projects.rb @@ -0,0 +1,474 @@ +module API + module V3 + class Projects < Grape::API + include PaginationParams + + before { authenticate_non_get! } + + after_validation do + set_only_allow_merge_if_pipeline_succeeds! + end + + helpers do + params :optional_params do + optional :description, type: String, desc: 'The description of the project' + optional :issues_enabled, type: Boolean, desc: 'Flag indication if the issue tracker is enabled' + optional :merge_requests_enabled, type: Boolean, desc: 'Flag indication if merge requests are enabled' + optional :wiki_enabled, type: Boolean, desc: 'Flag indication if the wiki is enabled' + optional :builds_enabled, type: Boolean, desc: 'Flag indication if builds are enabled' + optional :snippets_enabled, type: Boolean, desc: 'Flag indication if snippets are enabled' + optional :shared_runners_enabled, type: Boolean, desc: 'Flag indication if shared runners are enabled for that project' + optional :container_registry_enabled, type: Boolean, desc: 'Flag indication if the container registry is enabled for that project' + optional :lfs_enabled, type: Boolean, desc: 'Flag indication if Git LFS is enabled for that project' + optional :public, type: Boolean, desc: 'Create a public project. The same as visibility_level = 20.' + optional :visibility_level, type: Integer, values: [ + Gitlab::VisibilityLevel::PRIVATE, + Gitlab::VisibilityLevel::INTERNAL, + Gitlab::VisibilityLevel::PUBLIC + ], desc: 'Create a public project. The same as visibility_level = 20.' + optional :public_builds, type: Boolean, desc: 'Perform public builds' + optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access' + optional :only_allow_merge_if_build_succeeds, type: Boolean, desc: 'Only allow to merge if builds succeed' + optional :only_allow_merge_if_pipeline_succeeds, type: Boolean, desc: 'Only allow to merge if builds succeed' + optional :only_allow_merge_if_all_discussions_are_resolved, type: Boolean, desc: 'Only allow to merge if all discussions are resolved' + end + + def map_public_to_visibility_level(attrs) + publik = attrs.delete(:public) + if !publik.nil? && !attrs[:visibility_level].present? + # Since setting the public attribute to private could mean either + # private or internal, use the more conservative option, private. + attrs[:visibility_level] = (publik == true) ? Gitlab::VisibilityLevel::PUBLIC : Gitlab::VisibilityLevel::PRIVATE + end + attrs + end + + def set_only_allow_merge_if_pipeline_succeeds! + if params.has_key?(:only_allow_merge_if_build_succeeds) + params[:only_allow_merge_if_pipeline_succeeds] = params.delete(:only_allow_merge_if_build_succeeds) + end + end + end + + resource :projects do + helpers do + params :collection_params do + use :sort_params + use :filter_params + use :pagination + + optional :simple, type: Boolean, default: false, + desc: 'Return only the ID, URL, name, and path of each project' + end + + params :sort_params do + optional :order_by, type: String, values: %w[id name path created_at updated_at last_activity_at], + default: 'created_at', desc: 'Return projects ordered by field' + optional :sort, type: String, values: %w[asc desc], default: 'desc', + desc: 'Return projects sorted in ascending and descending order' + end + + params :filter_params do + optional :archived, type: Boolean, default: false, desc: 'Limit by archived status' + optional :visibility, type: String, values: %w[public internal private], + desc: 'Limit by visibility' + optional :search, type: String, desc: 'Return list of authorized projects matching the search criteria' + end + + params :statistics_params do + optional :statistics, type: Boolean, default: false, desc: 'Include project statistics' + end + + params :create_params do + optional :namespace_id, type: Integer, desc: 'Namespace ID for the new project. Default to the user namespace.' + optional :import_url, type: String, desc: 'URL from which the project is imported' + end + + def present_projects(projects, options = {}) + options = options.reverse_merge( + with: ::API::V3::Entities::Project, + current_user: current_user, + simple: params[:simple], + ) + + projects = filter_projects(projects) + projects = projects.with_statistics if options[:statistics] + options[:with] = ::API::Entities::BasicProjectDetails if options[:simple] + + present paginate(projects), options + end + end + + desc 'Get a list of visible projects for authenticated user' do + success ::API::Entities::BasicProjectDetails + end + params do + use :collection_params + end + get '/visible' do + entity = current_user ? ::API::V3::Entities::ProjectWithAccess : ::API::Entities::BasicProjectDetails + present_projects ProjectsFinder.new.execute(current_user), with: entity + end + + desc 'Get a projects list for authenticated user' do + success ::API::Entities::BasicProjectDetails + end + params do + use :collection_params + end + get do + authenticate! + + present_projects current_user.authorized_projects, + with: ::API::V3::Entities::ProjectWithAccess + end + + desc 'Get an owned projects list for authenticated user' do + success ::API::Entities::BasicProjectDetails + end + params do + use :collection_params + use :statistics_params + end + get '/owned' do + authenticate! + + present_projects current_user.owned_projects, + with: ::API::V3::Entities::ProjectWithAccess, + statistics: params[:statistics] + end + + desc 'Gets starred project for the authenticated user' do + success ::API::Entities::BasicProjectDetails + end + params do + use :collection_params + end + get '/starred' do + authenticate! + + present_projects current_user.viewable_starred_projects + end + + desc 'Get all projects for admin user' do + success ::API::Entities::BasicProjectDetails + end + params do + use :collection_params + use :statistics_params + end + get '/all' do + authenticated_as_admin! + + present_projects Project.all, with: ::API::V3::Entities::ProjectWithAccess, statistics: params[:statistics] + end + + desc 'Search for projects the current user has access to' do + success ::API::V3::Entities::Project + end + params do + requires :query, type: String, desc: 'The project name to be searched' + use :sort_params + use :pagination + end + get "/search/:query", requirements: { query: /[^\/]+/ } do + search_service = Search::GlobalService.new(current_user, search: params[:query]).execute + projects = search_service.objects('projects', params[:page]) + projects = projects.reorder(params[:order_by] => params[:sort]) + + present paginate(projects), with: ::API::V3::Entities::Project + end + + desc 'Create new project' do + success ::API::V3::Entities::Project + end + params do + optional :name, type: String, desc: 'The name of the project' + optional :path, type: String, desc: 'The path of the repository' + at_least_one_of :name, :path + use :optional_params + use :create_params + end + post do + attrs = map_public_to_visibility_level(declared_params(include_missing: false)) + project = ::Projects::CreateService.new(current_user, attrs).execute + + if project.saved? + present project, with: ::API::V3::Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, project) + else + if project.errors[:limit_reached].present? + error!(project.errors[:limit_reached], 403) + end + render_validation_error!(project) + end + end + + desc 'Create new project for a specified user. Only available to admin users.' do + success ::API::V3::Entities::Project + end + params do + requires :name, type: String, desc: 'The name of the project' + requires :user_id, type: Integer, desc: 'The ID of a user' + optional :default_branch, type: String, desc: 'The default branch of the project' + use :optional_params + use :create_params + end + post "user/:user_id" do + authenticated_as_admin! + user = User.find_by(id: params.delete(:user_id)) + not_found!('User') unless user + + attrs = map_public_to_visibility_level(declared_params(include_missing: false)) + project = ::Projects::CreateService.new(user, attrs).execute + + if project.saved? + present project, with: ::API::V3::Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, project) + else + render_validation_error!(project) + end + end + end + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: { id: %r{[^/]+} } do + desc 'Get a single project' do + success ::API::V3::Entities::ProjectWithAccess + end + get ":id" do + entity = current_user ? ::API::V3::Entities::ProjectWithAccess : ::API::Entities::BasicProjectDetails + present user_project, with: entity, current_user: current_user, + user_can_admin_project: can?(current_user, :admin_project, user_project) + end + + desc 'Get events for a single project' do + success ::API::V3::Entities::Event + end + params do + use :pagination + end + get ":id/events" do + present paginate(user_project.events.recent), with: ::API::V3::Entities::Event + end + + desc 'Fork new project for the current user or provided namespace.' do + success ::API::V3::Entities::Project + end + params do + optional :namespace, type: String, desc: 'The ID or name of the namespace that the project will be forked into' + end + post 'fork/:id' do + fork_params = declared_params(include_missing: false) + namespace_id = fork_params[:namespace] + + if namespace_id.present? + fork_params[:namespace] = if namespace_id =~ /^\d+$/ + Namespace.find_by(id: namespace_id) + else + Namespace.find_by_path_or_name(namespace_id) + end + + unless fork_params[:namespace] && can?(current_user, :create_projects, fork_params[:namespace]) + not_found!('Target Namespace') + end + end + + forked_project = ::Projects::ForkService.new(user_project, current_user, fork_params).execute + + if forked_project.errors.any? + conflict!(forked_project.errors.messages) + else + present forked_project, with: ::API::V3::Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, forked_project) + end + end + + desc 'Update an existing project' do + success ::API::V3::Entities::Project + end + params do + optional :name, type: String, desc: 'The name of the project' + optional :default_branch, type: String, desc: 'The default branch of the project' + optional :path, type: String, desc: 'The path of the repository' + use :optional_params + at_least_one_of :name, :description, :issues_enabled, :merge_requests_enabled, + :wiki_enabled, :builds_enabled, :snippets_enabled, + :shared_runners_enabled, :container_registry_enabled, + :lfs_enabled, :public, :visibility_level, :public_builds, + :request_access_enabled, :only_allow_merge_if_build_succeeds, + :only_allow_merge_if_all_discussions_are_resolved, :path, + :default_branch + end + put ':id' do + authorize_admin_project + attrs = map_public_to_visibility_level(declared_params(include_missing: false)) + authorize! :rename_project, user_project if attrs[:name].present? + authorize! :change_visibility_level, user_project if attrs[:visibility_level].present? + + result = ::Projects::UpdateService.new(user_project, current_user, attrs).execute + + if result[:status] == :success + present user_project, with: ::API::V3::Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, user_project) + else + render_validation_error!(user_project) + end + end + + desc 'Archive a project' do + success ::API::V3::Entities::Project + end + post ':id/archive' do + authorize!(:archive_project, user_project) + + user_project.archive! + + present user_project, with: ::API::V3::Entities::Project + end + + desc 'Unarchive a project' do + success ::API::V3::Entities::Project + end + post ':id/unarchive' do + authorize!(:archive_project, user_project) + + user_project.unarchive! + + present user_project, with: ::API::V3::Entities::Project + end + + desc 'Star a project' do + success ::API::V3::Entities::Project + end + post ':id/star' do + if current_user.starred?(user_project) + not_modified! + else + current_user.toggle_star(user_project) + user_project.reload + + present user_project, with: ::API::V3::Entities::Project + end + end + + desc 'Unstar a project' do + success ::API::V3::Entities::Project + end + delete ':id/star' do + if current_user.starred?(user_project) + current_user.toggle_star(user_project) + user_project.reload + + present user_project, with: ::API::V3::Entities::Project + else + not_modified! + end + end + + desc 'Remove a project' + delete ":id" do + authorize! :remove_project, user_project + + status(200) + ::Projects::DestroyService.new(user_project, current_user, {}).async_execute + end + + desc 'Mark this project as forked from another' + params do + requires :forked_from_id, type: String, desc: 'The ID of the project it was forked from' + end + post ":id/fork/:forked_from_id" do + authenticated_as_admin! + + forked_from_project = find_project!(params[:forked_from_id]) + not_found!("Source Project") unless forked_from_project + + if user_project.forked_from_project.nil? + user_project.create_forked_project_link(forked_to_project_id: user_project.id, forked_from_project_id: forked_from_project.id) + else + render_api_error!("Project already forked", 409) + end + end + + desc 'Remove a forked_from relationship' + delete ":id/fork" do + authorize! :remove_fork_project, user_project + + if user_project.forked? + status(200) + user_project.forked_project_link.destroy + else + not_modified! + end + end + + desc 'Share the project with a group' do + success ::API::Entities::ProjectGroupLink + end + params do + requires :group_id, type: Integer, desc: 'The ID of a group' + requires :group_access, type: Integer, values: Gitlab::Access.values, desc: 'The group access level' + optional :expires_at, type: Date, desc: 'Share expiration date' + end + post ":id/share" do + authorize! :admin_project, user_project + group = Group.find_by_id(params[:group_id]) + + unless group && can?(current_user, :read_group, group) + not_found!('Group') + end + + unless user_project.allowed_to_share_with_group? + return render_api_error!("The project sharing with group is disabled", 400) + end + + link = user_project.project_group_links.new(declared_params(include_missing: false)) + + if link.save + present link, with: ::API::Entities::ProjectGroupLink + else + render_api_error!(link.errors.full_messages.first, 409) + end + end + + params do + requires :group_id, type: Integer, desc: 'The ID of the group' + end + delete ":id/share/:group_id" do + authorize! :admin_project, user_project + + link = user_project.project_group_links.find_by(group_id: params[:group_id]) + not_found!('Group Link') unless link + + link.destroy + no_content! + end + + desc 'Upload a file' + params do + requires :file, type: File, desc: 'The file to be uploaded' + end + post ":id/uploads" do + ::Projects::UploadService.new(user_project, params[:file]).execute + end + + desc 'Get the users list of a project' do + success ::API::Entities::UserBasic + end + params do + optional :search, type: String, desc: 'Return list of users matching the search criteria' + use :pagination + end + get ':id/users' do + users = user_project.team.users + users = users.search(params[:search]) if params[:search].present? + + present paginate(users), with: ::API::Entities::UserBasic + end + end + end + end +end diff --git a/lib/api/v3/repositories.rb b/lib/api/v3/repositories.rb new file mode 100644 index 00000000000..e4d14bc8168 --- /dev/null +++ b/lib/api/v3/repositories.rb @@ -0,0 +1,109 @@ +require 'mime/types' + +module API + module V3 + class Repositories < Grape::API + before { authorize! :download_code, user_project } + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: { id: %r{[^/]+} } do + helpers do + def handle_project_member_errors(errors) + if errors[:project_access].any? + error!(errors[:project_access], 422) + end + not_found! + end + end + + desc 'Get a project repository tree' do + success ::API::Entities::RepoTreeObject + end + params do + optional :ref_name, type: String, desc: 'The name of a repository branch or tag, if not given the default branch is used' + optional :path, type: String, desc: 'The path of the tree' + optional :recursive, type: Boolean, default: false, desc: 'Used to get a recursive tree' + end + get ':id/repository/tree' do + ref = params[:ref_name] || user_project.try(:default_branch) || 'master' + path = params[:path] || nil + + commit = user_project.commit(ref) + not_found!('Tree') unless commit + + tree = user_project.repository.tree(commit.id, path, recursive: params[:recursive]) + + present tree.sorted_entries, with: ::API::Entities::RepoTreeObject + end + + desc 'Get a raw file contents' + params do + requires :sha, type: String, desc: 'The commit, branch name, or tag name' + requires :filepath, type: String, desc: 'The path to the file to display' + end + get [":id/repository/blobs/:sha", ":id/repository/commits/:sha/blob"] do + repo = user_project.repository + commit = repo.commit(params[:sha]) + not_found! "Commit" unless commit + blob = Gitlab::Git::Blob.find(repo, commit.id, params[:filepath]) + not_found! "File" unless blob + send_git_blob repo, blob + end + + desc 'Get a raw blob contents by blob sha' + params do + requires :sha, type: String, desc: 'The commit, branch name, or tag name' + end + get ':id/repository/raw_blobs/:sha' do + repo = user_project.repository + begin + blob = Gitlab::Git::Blob.raw(repo, params[:sha]) + rescue + not_found! 'Blob' + end + not_found! 'Blob' unless blob + send_git_blob repo, blob + end + + desc 'Get an archive of the repository' + params do + optional :sha, type: String, desc: 'The commit sha of the archive to be downloaded' + optional :format, type: String, desc: 'The archive format' + end + get ':id/repository/archive', requirements: { format: Gitlab::Regex.archive_formats_regex } do + begin + send_git_archive user_project.repository, ref: params[:sha], format: params[:format] + rescue + not_found!('File') + end + end + + desc 'Compare two branches, tags, or commits' do + success ::API::Entities::Compare + end + params do + requires :from, type: String, desc: 'The commit, branch name, or tag name to start comparison' + requires :to, type: String, desc: 'The commit, branch name, or tag name to stop comparison' + end + get ':id/repository/compare' do + compare = Gitlab::Git::Compare.new(user_project.repository.raw_repository, params[:from], params[:to]) + present compare, with: ::API::Entities::Compare + end + + desc 'Get repository contributors' do + success ::API::Entities::Contributor + end + get ':id/repository/contributors' do + begin + present user_project.repository.contributors, + with: ::API::Entities::Contributor + rescue + not_found! + end + end + end + end + end +end diff --git a/lib/api/v3/runners.rb b/lib/api/v3/runners.rb new file mode 100644 index 00000000000..1934d6e578c --- /dev/null +++ b/lib/api/v3/runners.rb @@ -0,0 +1,65 @@ +module API + module V3 + class Runners < Grape::API + include PaginationParams + + before { authenticate! } + + resource :runners do + desc 'Remove a runner' do + success ::API::Entities::Runner + end + params do + requires :id, type: Integer, desc: 'The ID of the runner' + end + delete ':id' do + runner = Ci::Runner.find(params[:id]) + not_found!('Runner') unless runner + + authenticate_delete_runner!(runner) + + status(200) + runner.destroy + end + end + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: { id: %r{[^/]+} } do + before { authorize_admin_project } + + desc "Disable project's runner" do + success ::API::Entities::Runner + end + params do + requires :runner_id, type: Integer, desc: 'The ID of the runner' + end + delete ':id/runners/:runner_id' do + runner_project = user_project.runner_projects.find_by(runner_id: params[:runner_id]) + not_found!('Runner') unless runner_project + + runner = runner_project.runner + forbidden!("Only one project associated with the runner. Please remove the runner instead") if runner.projects.count == 1 + + runner_project.destroy + + present runner, with: ::API::Entities::Runner + end + end + + helpers do + def authenticate_delete_runner!(runner) + return if current_user.is_admin? + forbidden!("Runner is shared") if runner.is_shared? + forbidden!("Runner associated with more than one project") if runner.projects.count > 1 + forbidden!("No access granted") unless user_can_access_runner?(runner) + end + + def user_can_access_runner?(runner) + current_user.ci_authorized_runners.exists?(runner.id) + end + end + end + end +end diff --git a/lib/api/v3/services.rb b/lib/api/v3/services.rb new file mode 100644 index 00000000000..3bacaeee032 --- /dev/null +++ b/lib/api/v3/services.rb @@ -0,0 +1,644 @@ +module API + module V3 + class Services < Grape::API + services = { + 'asana' => [ + { + required: true, + name: :api_key, + type: String, + desc: 'User API token' + }, + { + required: false, + name: :restrict_to_branch, + type: String, + desc: 'Comma-separated list of branches which will be automatically inspected. Leave blank to include all branches' + } + ], + 'assembla' => [ + { + required: true, + name: :token, + type: String, + desc: 'The authentication token' + }, + { + required: false, + name: :subdomain, + type: String, + desc: 'Subdomain setting' + } + ], + 'bamboo' => [ + { + required: true, + name: :bamboo_url, + type: String, + desc: 'Bamboo root URL like https://bamboo.example.com' + }, + { + required: true, + name: :build_key, + type: String, + desc: 'Bamboo build plan key like' + }, + { + required: true, + name: :username, + type: String, + desc: 'A user with API access, if applicable' + }, + { + required: true, + name: :password, + type: String, + desc: 'Passord of the user' + } + ], + 'bugzilla' => [ + { + required: true, + name: :new_issue_url, + type: String, + desc: 'New issue URL' + }, + { + required: true, + name: :issues_url, + type: String, + desc: 'Issues URL' + }, + { + required: true, + name: :project_url, + type: String, + desc: 'Project URL' + }, + { + required: false, + name: :description, + type: String, + desc: 'Description' + }, + { + required: false, + name: :title, + type: String, + desc: 'Title' + } + ], + 'buildkite' => [ + { + required: true, + name: :token, + type: String, + desc: 'Buildkite project GitLab token' + }, + { + required: true, + name: :project_url, + type: String, + desc: 'The buildkite project URL' + }, + { + required: false, + name: :enable_ssl_verification, + type: Boolean, + desc: 'Enable SSL verification for communication' + } + ], + 'builds-email' => [ + { + required: true, + name: :recipients, + type: String, + desc: 'Comma-separated list of recipient email addresses' + }, + { + required: false, + name: :add_pusher, + type: Boolean, + desc: 'Add pusher to recipients list' + }, + { + required: false, + name: :notify_only_broken_builds, + type: Boolean, + desc: 'Notify only broken builds' + } + ], + 'campfire' => [ + { + required: true, + name: :token, + type: String, + desc: 'Campfire token' + }, + { + required: false, + name: :subdomain, + type: String, + desc: 'Campfire subdomain' + }, + { + required: false, + name: :room, + type: String, + desc: 'Campfire room' + } + ], + 'custom-issue-tracker' => [ + { + required: true, + name: :new_issue_url, + type: String, + desc: 'New issue URL' + }, + { + required: true, + name: :issues_url, + type: String, + desc: 'Issues URL' + }, + { + required: true, + name: :project_url, + type: String, + desc: 'Project URL' + }, + { + required: false, + name: :description, + type: String, + desc: 'Description' + }, + { + required: false, + name: :title, + type: String, + desc: 'Title' + } + ], + 'drone-ci' => [ + { + required: true, + name: :token, + type: String, + desc: 'Drone CI token' + }, + { + required: true, + name: :drone_url, + type: String, + desc: 'Drone CI URL' + }, + { + required: false, + name: :enable_ssl_verification, + type: Boolean, + desc: 'Enable SSL verification for communication' + } + ], + 'emails-on-push' => [ + { + required: true, + name: :recipients, + type: String, + desc: 'Comma-separated list of recipient email addresses' + }, + { + required: false, + name: :disable_diffs, + type: Boolean, + desc: 'Disable code diffs' + }, + { + required: false, + name: :send_from_committer_email, + type: Boolean, + desc: 'Send from committer' + } + ], + 'external-wiki' => [ + { + required: true, + name: :external_wiki_url, + type: String, + desc: 'The URL of the external Wiki' + } + ], + 'flowdock' => [ + { + required: true, + name: :token, + type: String, + desc: 'Flowdock token' + } + ], + 'gemnasium' => [ + { + required: true, + name: :api_key, + type: String, + desc: 'Your personal API key on gemnasium.com' + }, + { + required: true, + name: :token, + type: String, + desc: "The project's slug on gemnasium.com" + } + ], + 'hipchat' => [ + { + required: true, + name: :token, + type: String, + desc: 'The room token' + }, + { + required: false, + name: :room, + type: String, + desc: 'The room name or ID' + }, + { + required: false, + name: :color, + type: String, + desc: 'The room color' + }, + { + required: false, + name: :notify, + type: Boolean, + desc: 'Enable notifications' + }, + { + required: false, + name: :api_version, + type: String, + desc: 'Leave blank for default (v2)' + }, + { + required: false, + name: :server, + type: String, + desc: 'Leave blank for default. https://hipchat.example.com' + } + ], + 'irker' => [ + { + required: true, + name: :recipients, + type: String, + desc: 'Recipients/channels separated by whitespaces' + }, + { + required: false, + name: :default_irc_uri, + type: String, + desc: 'Default: irc://irc.network.net:6697' + }, + { + required: false, + name: :server_host, + type: String, + desc: 'Server host. Default localhost' + }, + { + required: false, + name: :server_port, + type: Integer, + desc: 'Server port. Default 6659' + }, + { + required: false, + name: :colorize_messages, + type: Boolean, + desc: 'Colorize messages' + } + ], + 'jira' => [ + { + required: true, + name: :url, + type: String, + desc: 'The URL to the JIRA project which is being linked to this GitLab project, e.g., https://jira.example.com' + }, + { + required: true, + name: :project_key, + type: String, + desc: 'The short identifier for your JIRA project, all uppercase, e.g., PROJ' + }, + { + required: false, + name: :username, + type: String, + desc: 'The username of the user created to be used with GitLab/JIRA' + }, + { + required: false, + name: :password, + type: String, + desc: 'The password of the user created to be used with GitLab/JIRA' + }, + { + required: false, + name: :jira_issue_transition_id, + type: Integer, + desc: 'The ID of a transition that moves issues to a closed state. You can find this number under the JIRA workflow administration (**Administration > Issues > Workflows**) by selecting **View** under **Operations** of the desired workflow of your project. The ID of each state can be found inside the parenthesis of each transition name under the **Transitions (id)** column ([see screenshot][trans]). By default, this ID is set to `2`' + } + ], + + 'kubernetes' => [ + { + required: true, + name: :namespace, + type: String, + desc: 'The Kubernetes namespace to use' + }, + { + required: true, + name: :api_url, + type: String, + desc: 'The URL to the Kubernetes cluster API, e.g., https://kubernetes.example.com' + }, + { + required: true, + name: :token, + type: String, + desc: 'The service token to authenticate against the Kubernetes cluster with' + }, + { + required: false, + name: :ca_pem, + type: String, + desc: 'A custom certificate authority bundle to verify the Kubernetes cluster with (PEM format)' + }, + ], + 'mattermost-slash-commands' => [ + { + required: true, + name: :token, + type: String, + desc: 'The Mattermost token' + } + ], + 'slack-slash-commands' => [ + { + required: true, + name: :token, + type: String, + desc: 'The Slack token' + } + ], + 'pipelines-email' => [ + { + required: true, + name: :recipients, + type: String, + desc: 'Comma-separated list of recipient email addresses' + }, + { + required: false, + name: :notify_only_broken_builds, + type: Boolean, + desc: 'Notify only broken builds' + } + ], + 'pivotaltracker' => [ + { + required: true, + name: :token, + type: String, + desc: 'The Pivotaltracker token' + }, + { + required: false, + name: :restrict_to_branch, + type: String, + desc: 'Comma-separated list of branches which will be automatically inspected. Leave blank to include all branches.' + } + ], + 'pushover' => [ + { + required: true, + name: :api_key, + type: String, + desc: 'The application key' + }, + { + required: true, + name: :user_key, + type: String, + desc: 'The user key' + }, + { + required: true, + name: :priority, + type: String, + desc: 'The priority' + }, + { + required: true, + name: :device, + type: String, + desc: 'Leave blank for all active devices' + }, + { + required: true, + name: :sound, + type: String, + desc: 'The sound of the notification' + } + ], + 'redmine' => [ + { + required: true, + name: :new_issue_url, + type: String, + desc: 'The new issue URL' + }, + { + required: true, + name: :project_url, + type: String, + desc: 'The project URL' + }, + { + required: true, + name: :issues_url, + type: String, + desc: 'The issues URL' + }, + { + required: false, + name: :description, + type: String, + desc: 'The description of the tracker' + } + ], + 'slack' => [ + { + required: true, + name: :webhook, + type: String, + desc: 'The Slack webhook. e.g. https://hooks.slack.com/services/...' + }, + { + required: false, + name: :new_issue_url, + type: String, + desc: 'The user name' + }, + { + required: false, + name: :channel, + type: String, + desc: 'The channel name' + } + ], + 'mattermost' => [ + { + required: true, + name: :webhook, + type: String, + desc: 'The Mattermost webhook. e.g. http://mattermost_host/hooks/...' + } + ], + 'teamcity' => [ + { + required: true, + name: :teamcity_url, + type: String, + desc: 'TeamCity root URL like https://teamcity.example.com' + }, + { + required: true, + name: :build_type, + type: String, + desc: 'Build configuration ID' + }, + { + required: true, + name: :username, + type: String, + desc: 'A user with permissions to trigger a manual build' + }, + { + required: true, + name: :password, + type: String, + desc: 'The password of the user' + } + ] + } + + trigger_services = { + 'mattermost-slash-commands' => [ + { + name: :token, + type: String, + desc: 'The Mattermost token' + } + ], + 'slack-slash-commands' => [ + { + name: :token, + type: String, + desc: 'The Slack token' + } + ] + }.freeze + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: { id: %r{[^/]+} } do + before { authenticate! } + before { authorize_admin_project } + + helpers do + def service_attributes(service) + service.fields.inject([]) do |arr, hash| + arr << hash[:name].to_sym + end + end + end + + desc "Delete a service for project" + params do + requires :service_slug, type: String, values: services.keys, desc: 'The name of the service' + end + delete ":id/services/:service_slug" do + service = user_project.find_or_initialize_service(params[:service_slug].underscore) + + attrs = service_attributes(service).inject({}) do |hash, key| + hash.merge!(key => nil) + end + + if service.update_attributes(attrs.merge(active: false)) + status(200) + true + else + render_api_error!('400 Bad Request', 400) + end + end + + desc 'Get the service settings for project' do + success Entities::ProjectService + end + params do + requires :service_slug, type: String, values: services.keys, desc: 'The name of the service' + end + get ":id/services/:service_slug" do + service = user_project.find_or_initialize_service(params[:service_slug].underscore) + present service, with: Entities::ProjectService, include_passwords: current_user.is_admin? + end + end + + trigger_services.each do |service_slug, settings| + helpers do + def chat_command_service(project, service_slug, params) + project.services.active.where(template: false).find do |service| + service.try(:token) == params[:token] && service.to_param == service_slug.underscore + end + end + end + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: { id: %r{[^/]+} } do + desc "Trigger a slash command for #{service_slug}" do + detail 'Added in GitLab 8.13' + end + params do + settings.each do |setting| + requires setting[:name], type: setting[:type], desc: setting[:desc] + end + end + post ":id/services/#{service_slug.underscore}/trigger" do + project = find_project(params[:id]) + + # This is not accurate, but done to prevent leakage of the project names + not_found!('Service') unless project + + service = chat_command_service(project, service_slug, params) + result = service.try(:trigger, params) + + if result + status result[:status] || 200 + present result + else + not_found!('Service') + end + end + end + end + end + end +end diff --git a/lib/api/v3/settings.rb b/lib/api/v3/settings.rb new file mode 100644 index 00000000000..748d6b97d4f --- /dev/null +++ b/lib/api/v3/settings.rb @@ -0,0 +1,137 @@ +module API + module V3 + class Settings < Grape::API + before { authenticated_as_admin! } + + helpers do + def current_settings + @current_setting ||= + (ApplicationSetting.current || ApplicationSetting.create_from_defaults) + end + end + + desc 'Get the current application settings' do + success Entities::ApplicationSetting + end + get "application/settings" do + present current_settings, with: Entities::ApplicationSetting + end + + desc 'Modify application settings' do + success Entities::ApplicationSetting + end + params do + optional :default_branch_protection, type: Integer, values: [0, 1, 2], desc: 'Determine if developers can push to master' + optional :default_project_visibility, type: Integer, values: Gitlab::VisibilityLevel.values, desc: 'The default project visibility' + optional :default_snippet_visibility, type: Integer, values: Gitlab::VisibilityLevel.values, desc: 'The default snippet visibility' + optional :default_group_visibility, type: Integer, values: Gitlab::VisibilityLevel.values, desc: 'The default group visibility' + optional :restricted_visibility_levels, type: Array[String], desc: 'Selected levels cannot be used by non-admin users for projects or snippets. If the public level is restricted, user profiles are only visible to logged in users.' + optional :import_sources, type: Array[String], values: %w[github bitbucket gitlab google_code fogbugz git gitlab_project], + desc: 'Enabled sources for code import during project creation. OmniAuth must be configured for GitHub, Bitbucket, and GitLab.com' + optional :disabled_oauth_sign_in_sources, type: Array[String], desc: 'Disable certain OAuth sign-in sources' + optional :enabled_git_access_protocol, type: String, values: %w[ssh http nil], desc: 'Allow only the selected protocols to be used for Git access.' + optional :gravatar_enabled, type: Boolean, desc: 'Flag indicating if the Gravatar service is enabled' + optional :default_projects_limit, type: Integer, desc: 'The maximum number of personal projects' + optional :max_attachment_size, type: Integer, desc: 'Maximum attachment size in MB' + optional :session_expire_delay, type: Integer, desc: 'Session duration in minutes. GitLab restart is required to apply changes.' + optional :user_oauth_applications, type: Boolean, desc: 'Allow users to register any application to use GitLab as an OAuth provider' + optional :user_default_external, type: Boolean, desc: 'Newly registered users will by default be external' + optional :signup_enabled, type: Boolean, desc: 'Flag indicating if sign up is enabled' + optional :send_user_confirmation_email, type: Boolean, desc: 'Send confirmation email on sign-up' + optional :domain_whitelist, type: String, desc: 'ONLY users with e-mail addresses that match these domain(s) will be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com' + optional :domain_blacklist_enabled, type: Boolean, desc: 'Enable domain blacklist for sign ups' + given domain_blacklist_enabled: ->(val) { val } do + requires :domain_blacklist, type: String, desc: 'Users with e-mail addresses that match these domain(s) will NOT be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com' + end + optional :after_sign_up_text, type: String, desc: 'Text shown after sign up' + optional :signin_enabled, type: Boolean, desc: 'Flag indicating if sign in is enabled' + optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users to setup Two-factor authentication' + given require_two_factor_authentication: ->(val) { val } do + requires :two_factor_grace_period, type: Integer, desc: 'Amount of time (in hours) that users are allowed to skip forced configuration of two-factor authentication' + end + optional :home_page_url, type: String, desc: 'We will redirect non-logged in users to this page' + optional :after_sign_out_path, type: String, desc: 'We will redirect users to this page after they sign out' + optional :sign_in_text, type: String, desc: 'The sign in text of the GitLab application' + optional :help_page_text, type: String, desc: 'Custom text displayed on the help page' + optional :shared_runners_enabled, type: Boolean, desc: 'Enable shared runners for new projects' + given shared_runners_enabled: ->(val) { val } do + requires :shared_runners_text, type: String, desc: 'Shared runners text ' + end + optional :max_artifacts_size, type: Integer, desc: "Set the maximum file size each build's artifacts can have" + optional :max_pages_size, type: Integer, desc: 'Maximum size of pages in MB' + optional :container_registry_token_expire_delay, type: Integer, desc: 'Authorization token duration (minutes)' + optional :metrics_enabled, type: Boolean, desc: 'Enable the InfluxDB metrics' + given metrics_enabled: ->(val) { val } do + requires :metrics_host, type: String, desc: 'The InfluxDB host' + requires :metrics_port, type: Integer, desc: 'The UDP port to use for connecting to InfluxDB' + requires :metrics_pool_size, type: Integer, desc: 'The amount of InfluxDB connections to open' + requires :metrics_timeout, type: Integer, desc: 'The amount of seconds after which an InfluxDB connection will time out' + requires :metrics_method_call_threshold, type: Integer, desc: 'A method call is only tracked when it takes longer to complete than the given amount of milliseconds.' + requires :metrics_sample_interval, type: Integer, desc: 'The sampling interval in seconds' + requires :metrics_packet_size, type: Integer, desc: 'The amount of points to store in a single UDP packet' + end + optional :sidekiq_throttling_enabled, type: Boolean, desc: 'Enable Sidekiq Job Throttling' + given sidekiq_throttling_enabled: ->(val) { val } do + requires :sidekiq_throttling_queus, type: Array[String], desc: 'Choose which queues you wish to throttle' + requires :sidekiq_throttling_factor, type: Float, desc: 'The factor by which the queues should be throttled. A value between 0.0 and 1.0, exclusive.' + end + optional :recaptcha_enabled, type: Boolean, desc: 'Helps prevent bots from creating accounts' + given recaptcha_enabled: ->(val) { val } do + requires :recaptcha_site_key, type: String, desc: 'Generate site key at http://www.google.com/recaptcha' + requires :recaptcha_private_key, type: String, desc: 'Generate private key at http://www.google.com/recaptcha' + end + optional :akismet_enabled, type: Boolean, desc: 'Helps prevent bots from creating issues' + given akismet_enabled: ->(val) { val } do + requires :akismet_api_key, type: String, desc: 'Generate API key at http://www.akismet.com' + end + optional :admin_notification_email, type: String, desc: 'Abuse reports will be sent to this address if it is set. Abuse reports are always available in the admin area.' + optional :sentry_enabled, type: Boolean, desc: 'Sentry is an error reporting and logging tool which is currently not shipped with GitLab, get it here: https://getsentry.com' + given sentry_enabled: ->(val) { val } do + requires :sentry_dsn, type: String, desc: 'Sentry Data Source Name' + end + optional :repository_storage, type: String, desc: 'Storage paths for new projects' + optional :repository_checks_enabled, type: Boolean, desc: "GitLab will periodically run 'git fsck' in all project and wiki repositories to look for silent disk corruption issues." + optional :koding_enabled, type: Boolean, desc: 'Enable Koding' + given koding_enabled: ->(val) { val } do + requires :koding_url, type: String, desc: 'The Koding team URL' + end + optional :plantuml_enabled, type: Boolean, desc: 'Enable PlantUML' + given plantuml_enabled: ->(val) { val } do + requires :plantuml_url, type: String, desc: 'The PlantUML server URL' + end + optional :version_check_enabled, type: Boolean, desc: 'Let GitLab inform you when an update is available.' + optional :email_author_in_body, type: Boolean, desc: 'Some email servers do not support overriding the email sender name. Enable this option to include the name of the author of the issue, merge request or comment in the email body instead.' + optional :html_emails_enabled, type: Boolean, desc: 'By default GitLab sends emails in HTML and plain text formats so mail clients can choose what format to use. Disable this option if you only want to send emails in plain text format.' + optional :housekeeping_enabled, type: Boolean, desc: 'Enable automatic repository housekeeping (git repack, git gc)' + given housekeeping_enabled: ->(val) { val } do + requires :housekeeping_bitmaps_enabled, type: Boolean, desc: "Creating pack file bitmaps makes housekeeping take a little longer but bitmaps should accelerate 'git clone' performance." + requires :housekeeping_incremental_repack_period, type: Integer, desc: "Number of Git pushes after which an incremental 'git repack' is run." + requires :housekeeping_full_repack_period, type: Integer, desc: "Number of Git pushes after which a full 'git repack' is run." + requires :housekeeping_gc_period, type: Integer, desc: "Number of Git pushes after which 'git gc' is run." + end + optional :terminal_max_session_time, type: Integer, desc: 'Maximum time for web terminal websocket connection (in seconds). Set to 0 for unlimited time.' + at_least_one_of :default_branch_protection, :default_project_visibility, :default_snippet_visibility, + :default_group_visibility, :restricted_visibility_levels, :import_sources, + :enabled_git_access_protocol, :gravatar_enabled, :default_projects_limit, + :max_attachment_size, :session_expire_delay, :disabled_oauth_sign_in_sources, + :user_oauth_applications, :user_default_external, :signup_enabled, + :send_user_confirmation_email, :domain_whitelist, :domain_blacklist_enabled, + :after_sign_up_text, :signin_enabled, :require_two_factor_authentication, + :home_page_url, :after_sign_out_path, :sign_in_text, :help_page_text, + :shared_runners_enabled, :max_artifacts_size, :max_pages_size, :container_registry_token_expire_delay, + :metrics_enabled, :sidekiq_throttling_enabled, :recaptcha_enabled, + :akismet_enabled, :admin_notification_email, :sentry_enabled, + :repository_storage, :repository_checks_enabled, :koding_enabled, :plantuml_enabled, + :version_check_enabled, :email_author_in_body, :html_emails_enabled, + :housekeeping_enabled, :terminal_max_session_time + end + put "application/settings" do + if current_settings.update_attributes(declared_params(include_missing: false)) + present current_settings, with: Entities::ApplicationSetting + else + render_validation_error!(current_settings) + end + end + end + end +end diff --git a/lib/api/v3/snippets.rb b/lib/api/v3/snippets.rb new file mode 100644 index 00000000000..07dac7e9904 --- /dev/null +++ b/lib/api/v3/snippets.rb @@ -0,0 +1,138 @@ +module API + module V3 + class Snippets < Grape::API + include PaginationParams + + before { authenticate! } + + resource :snippets do + helpers do + def snippets_for_current_user + SnippetsFinder.new.execute(current_user, filter: :by_user, user: current_user) + end + + def public_snippets + SnippetsFinder.new.execute(current_user, filter: :public) + end + end + + desc 'Get a snippets list for authenticated user' do + detail 'This feature was introduced in GitLab 8.15.' + success ::API::Entities::PersonalSnippet + end + params do + use :pagination + end + get do + present paginate(snippets_for_current_user), with: ::API::Entities::PersonalSnippet + end + + desc 'List all public snippets current_user has access to' do + detail 'This feature was introduced in GitLab 8.15.' + success ::API::Entities::PersonalSnippet + end + params do + use :pagination + end + get 'public' do + present paginate(public_snippets), with: ::API::Entities::PersonalSnippet + end + + desc 'Get a single snippet' do + detail 'This feature was introduced in GitLab 8.15.' + success ::API::Entities::PersonalSnippet + end + params do + requires :id, type: Integer, desc: 'The ID of a snippet' + end + get ':id' do + snippet = snippets_for_current_user.find(params[:id]) + present snippet, with: ::API::Entities::PersonalSnippet + end + + desc 'Create new snippet' do + detail 'This feature was introduced in GitLab 8.15.' + success ::API::Entities::PersonalSnippet + end + params do + requires :title, type: String, desc: 'The title of a snippet' + requires :file_name, type: String, desc: 'The name of a snippet file' + requires :content, type: String, desc: 'The content of a snippet' + optional :visibility_level, type: Integer, + values: Gitlab::VisibilityLevel.values, + default: Gitlab::VisibilityLevel::INTERNAL, + desc: 'The visibility level of the snippet' + end + post do + attrs = declared_params(include_missing: false).merge(request: request, api: true) + snippet = CreateSnippetService.new(nil, current_user, attrs).execute + + if snippet.persisted? + present snippet, with: ::API::Entities::PersonalSnippet + else + render_validation_error!(snippet) + end + end + + desc 'Update an existing snippet' do + detail 'This feature was introduced in GitLab 8.15.' + success ::API::Entities::PersonalSnippet + end + params do + requires :id, type: Integer, desc: 'The ID of a snippet' + optional :title, type: String, desc: 'The title of a snippet' + optional :file_name, type: String, desc: 'The name of a snippet file' + optional :content, type: String, desc: 'The content of a snippet' + optional :visibility_level, type: Integer, + values: Gitlab::VisibilityLevel.values, + desc: 'The visibility level of the snippet' + at_least_one_of :title, :file_name, :content, :visibility_level + end + put ':id' do + snippet = snippets_for_current_user.find_by(id: params.delete(:id)) + return not_found!('Snippet') unless snippet + authorize! :update_personal_snippet, snippet + + attrs = declared_params(include_missing: false) + + UpdateSnippetService.new(nil, current_user, snippet, attrs).execute + if snippet.persisted? + present snippet, with: ::API::Entities::PersonalSnippet + else + render_validation_error!(snippet) + end + end + + desc 'Remove snippet' do + detail 'This feature was introduced in GitLab 8.15.' + success ::API::Entities::PersonalSnippet + end + params do + requires :id, type: Integer, desc: 'The ID of a snippet' + end + delete ':id' do + snippet = snippets_for_current_user.find_by(id: params.delete(:id)) + return not_found!('Snippet') unless snippet + authorize! :destroy_personal_snippet, snippet + snippet.destroy + no_content! + end + + desc 'Get a raw snippet' do + detail 'This feature was introduced in GitLab 8.15.' + end + params do + requires :id, type: Integer, desc: 'The ID of a snippet' + end + get ":id/raw" do + snippet = snippets_for_current_user.find_by(id: params.delete(:id)) + return not_found!('Snippet') unless snippet + + env['api.format'] = :txt + content_type 'text/plain' + present snippet.content + end + end + end + end +end diff --git a/lib/api/v3/subscriptions.rb b/lib/api/v3/subscriptions.rb new file mode 100644 index 00000000000..068750ec077 --- /dev/null +++ b/lib/api/v3/subscriptions.rb @@ -0,0 +1,53 @@ +module API + module V3 + class Subscriptions < Grape::API + before { authenticate! } + + subscribable_types = { + 'merge_request' => proc { |id| find_merge_request_with_access(id, :update_merge_request) }, + 'merge_requests' => proc { |id| find_merge_request_with_access(id, :update_merge_request) }, + 'issues' => proc { |id| find_project_issue(id) }, + 'labels' => proc { |id| find_project_label(id) }, + } + + params do + requires :id, type: String, desc: 'The ID of a project' + requires :subscribable_id, type: String, desc: 'The ID of a resource' + end + resource :projects, requirements: { id: %r{[^/]+} } do + subscribable_types.each do |type, finder| + type_singularized = type.singularize + entity_class = ::API::Entities.const_get(type_singularized.camelcase) + + desc 'Subscribe to a resource' do + success entity_class + end + post ":id/#{type}/:subscribable_id/subscription" do + resource = instance_exec(params[:subscribable_id], &finder) + + if resource.subscribed?(current_user, user_project) + not_modified! + else + resource.subscribe(current_user, user_project) + present resource, with: entity_class, current_user: current_user, project: user_project + end + end + + desc 'Unsubscribe from a resource' do + success entity_class + end + delete ":id/#{type}/:subscribable_id/subscription" do + resource = instance_exec(params[:subscribable_id], &finder) + + if !resource.subscribed?(current_user, user_project) + not_modified! + else + resource.unsubscribe(current_user, user_project) + present resource, with: entity_class, current_user: current_user, project: user_project + end + end + end + end + end + end +end diff --git a/lib/api/v3/system_hooks.rb b/lib/api/v3/system_hooks.rb new file mode 100644 index 00000000000..5787c06fc12 --- /dev/null +++ b/lib/api/v3/system_hooks.rb @@ -0,0 +1,32 @@ +module API + module V3 + class SystemHooks < Grape::API + before do + authenticate! + authenticated_as_admin! + end + + resource :hooks do + desc 'Get the list of system hooks' do + success ::API::Entities::Hook + end + get do + present SystemHook.all, with: ::API::Entities::Hook + end + + desc 'Delete a hook' do + success ::API::Entities::Hook + end + params do + requires :id, type: Integer, desc: 'The ID of the system hook' + end + delete ":id" do + hook = SystemHook.find_by(id: params[:id]) + not_found!('System hook') unless hook + + present hook.destroy, with: ::API::Entities::Hook + end + end + end + end +end diff --git a/lib/api/v3/tags.rb b/lib/api/v3/tags.rb new file mode 100644 index 00000000000..c2541de2f50 --- /dev/null +++ b/lib/api/v3/tags.rb @@ -0,0 +1,40 @@ +module API + module V3 + class Tags < Grape::API + before { authorize! :download_code, user_project } + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: { id: %r{[^/]+} } do + desc 'Get a project repository tags' do + success ::API::Entities::RepoTag + end + get ":id/repository/tags" do + tags = user_project.repository.tags.sort_by(&:name).reverse + present tags, with: ::API::Entities::RepoTag, project: user_project + end + + desc 'Delete a repository tag' + params do + requires :tag_name, type: String, desc: 'The name of the tag' + end + delete ":id/repository/tags/:tag_name", requirements: { tag_name: /.+/ } do + authorize_push_project + + result = ::Tags::DestroyService.new(user_project, current_user). + execute(params[:tag_name]) + + if result[:status] == :success + status(200) + { + tag_name: params[:tag_name] + } + else + render_api_error!(result[:message], result[:return_code]) + end + end + end + end + end +end diff --git a/lib/api/v3/templates.rb b/lib/api/v3/templates.rb new file mode 100644 index 00000000000..4c577a8d2b7 --- /dev/null +++ b/lib/api/v3/templates.rb @@ -0,0 +1,122 @@ +module API + module V3 + class Templates < Grape::API + GLOBAL_TEMPLATE_TYPES = { + gitignores: { + klass: Gitlab::Template::GitignoreTemplate, + gitlab_version: 8.8 + }, + gitlab_ci_ymls: { + klass: Gitlab::Template::GitlabCiYmlTemplate, + gitlab_version: 8.9 + }, + dockerfiles: { + klass: Gitlab::Template::DockerfileTemplate, + gitlab_version: 8.15 + } + }.freeze + PROJECT_TEMPLATE_REGEX = + /[\<\{\[] + (project|description| + one\sline\s.+\swhat\sit\sdoes\.) # matching the start and end is enough here + [\>\}\]]/xi.freeze + YEAR_TEMPLATE_REGEX = /[<{\[](year|yyyy)[>}\]]/i.freeze + FULLNAME_TEMPLATE_REGEX = + /[\<\{\[] + (fullname|name\sof\s(author|copyright\sowner)) + [\>\}\]]/xi.freeze + DEPRECATION_MESSAGE = ' This endpoint is deprecated and has been removed in V4.'.freeze + + helpers do + def parsed_license_template + # We create a fresh Licensee::License object since we'll modify its + # content in place below. + template = Licensee::License.new(params[:name]) + + template.content.gsub!(YEAR_TEMPLATE_REGEX, Time.now.year.to_s) + template.content.gsub!(PROJECT_TEMPLATE_REGEX, params[:project]) if params[:project].present? + + fullname = params[:fullname].presence || current_user.try(:name) + template.content.gsub!(FULLNAME_TEMPLATE_REGEX, fullname) if fullname + template + end + + def render_response(template_type, template) + not_found!(template_type.to_s.singularize) unless template + present template, with: ::API::Entities::Template + end + end + + { "licenses" => :deprecated, "templates/licenses" => :ok }.each do |route, status| + desc 'Get the list of the available license template' do + detailed_desc = 'This feature was introduced in GitLab 8.7.' + detailed_desc << DEPRECATION_MESSAGE unless status == :ok + detail detailed_desc + success ::API::Entities::RepoLicense + end + params do + optional :popular, type: Boolean, desc: 'If passed, returns only popular licenses' + end + get route do + options = { + featured: declared(params).popular.present? ? true : nil + } + present Licensee::License.all(options), with: ::API::Entities::RepoLicense + end + end + + { "licenses/:name" => :deprecated, "templates/licenses/:name" => :ok }.each do |route, status| + desc 'Get the text for a specific license' do + detailed_desc = 'This feature was introduced in GitLab 8.7.' + detailed_desc << DEPRECATION_MESSAGE unless status == :ok + detail detailed_desc + success ::API::Entities::RepoLicense + end + params do + requires :name, type: String, desc: 'The name of the template' + end + get route, requirements: { name: /[\w\.-]+/ } do + not_found!('License') unless Licensee::License.find(declared(params).name) + + template = parsed_license_template + + present template, with: ::API::Entities::RepoLicense + end + end + + GLOBAL_TEMPLATE_TYPES.each do |template_type, properties| + klass = properties[:klass] + gitlab_version = properties[:gitlab_version] + + { template_type => :deprecated, "templates/#{template_type}" => :ok }.each do |route, status| + desc 'Get the list of the available template' do + detailed_desc = "This feature was introduced in GitLab #{gitlab_version}." + detailed_desc << DEPRECATION_MESSAGE unless status == :ok + detail detailed_desc + success ::API::Entities::TemplatesList + end + get route do + present klass.all, with: ::API::Entities::TemplatesList + end + end + + { "#{template_type}/:name" => :deprecated, "templates/#{template_type}/:name" => :ok }.each do |route, status| + desc 'Get the text for a specific template present in local filesystem' do + detailed_desc = "This feature was introduced in GitLab #{gitlab_version}." + detailed_desc << DEPRECATION_MESSAGE unless status == :ok + detail detailed_desc + success ::API::Entities::Template + end + params do + requires :name, type: String, desc: 'The name of the template' + end + get route do + new_template = klass.find(declared(params).name) + + render_response(template_type, new_template) + end + end + end + end + end +end diff --git a/lib/api/v3/time_tracking_endpoints.rb b/lib/api/v3/time_tracking_endpoints.rb new file mode 100644 index 00000000000..81ae4e8137d --- /dev/null +++ b/lib/api/v3/time_tracking_endpoints.rb @@ -0,0 +1,116 @@ +module API + module V3 + module TimeTrackingEndpoints + extend ActiveSupport::Concern + + included do + helpers do + def issuable_name + declared_params.has_key?(:issue_id) ? 'issue' : 'merge_request' + end + + def issuable_key + "#{issuable_name}_id".to_sym + end + + def update_issuable_key + "update_#{issuable_name}".to_sym + end + + def read_issuable_key + "read_#{issuable_name}".to_sym + end + + def load_issuable + @issuable ||= begin + case issuable_name + when 'issue' + find_project_issue(params.delete(issuable_key)) + when 'merge_request' + find_project_merge_request(params.delete(issuable_key)) + end + end + end + + def update_issuable(attrs) + custom_params = declared_params(include_missing: false) + custom_params.merge!(attrs) + + issuable = update_service.new(user_project, current_user, custom_params).execute(load_issuable) + if issuable.valid? + present issuable, with: ::API::Entities::IssuableTimeStats + else + render_validation_error!(issuable) + end + end + + def update_service + issuable_name == 'issue' ? ::Issues::UpdateService : ::MergeRequests::UpdateService + end + end + + issuable_name = name.end_with?('Issues') ? 'issue' : 'merge_request' + issuable_collection_name = issuable_name.pluralize + issuable_key = "#{issuable_name}_id".to_sym + + desc "Set a time estimate for a project #{issuable_name}" + params do + requires issuable_key, type: Integer, desc: "The ID of a project #{issuable_name}" + requires :duration, type: String, desc: 'The duration to be parsed' + end + post ":id/#{issuable_collection_name}/:#{issuable_key}/time_estimate" do + authorize! update_issuable_key, load_issuable + + status :ok + update_issuable(time_estimate: Gitlab::TimeTrackingFormatter.parse(params.delete(:duration))) + end + + desc "Reset the time estimate for a project #{issuable_name}" + params do + requires issuable_key, type: Integer, desc: "The ID of a project #{issuable_name}" + end + post ":id/#{issuable_collection_name}/:#{issuable_key}/reset_time_estimate" do + authorize! update_issuable_key, load_issuable + + status :ok + update_issuable(time_estimate: 0) + end + + desc "Add spent time for a project #{issuable_name}" + params do + requires issuable_key, type: Integer, desc: "The ID of a project #{issuable_name}" + requires :duration, type: String, desc: 'The duration to be parsed' + end + post ":id/#{issuable_collection_name}/:#{issuable_key}/add_spent_time" do + authorize! update_issuable_key, load_issuable + + update_issuable(spend_time: { + duration: Gitlab::TimeTrackingFormatter.parse(params.delete(:duration)), + user: current_user + }) + end + + desc "Reset spent time for a project #{issuable_name}" + params do + requires issuable_key, type: Integer, desc: "The ID of a project #{issuable_name}" + end + post ":id/#{issuable_collection_name}/:#{issuable_key}/reset_spent_time" do + authorize! update_issuable_key, load_issuable + + status :ok + update_issuable(spend_time: { duration: :reset, user: current_user }) + end + + desc "Show time stats for a project #{issuable_name}" + params do + requires issuable_key, type: Integer, desc: "The ID of a project #{issuable_name}" + end + get ":id/#{issuable_collection_name}/:#{issuable_key}/time_stats" do + authorize! read_issuable_key, load_issuable + + present load_issuable, with: ::API::Entities::IssuableTimeStats + end + end + end + end +end diff --git a/lib/api/v3/todos.rb b/lib/api/v3/todos.rb new file mode 100644 index 00000000000..e3b311d61cd --- /dev/null +++ b/lib/api/v3/todos.rb @@ -0,0 +1,30 @@ +module API + module V3 + class Todos < Grape::API + before { authenticate! } + + resource :todos do + desc 'Mark a todo as done' do + success ::API::Entities::Todo + end + params do + requires :id, type: Integer, desc: 'The ID of the todo being marked as done' + end + delete ':id' do + todo = current_user.todos.find(params[:id]) + TodoService.new.mark_todos_as_done([todo], current_user) + + present todo.reload, with: ::API::Entities::Todo, current_user: current_user + end + + desc 'Mark all todos as done' + delete do + status(200) + + todos = TodosFinder.new(current_user, params).execute + TodoService.new.mark_todos_as_done(todos, current_user).size + end + end + end + end +end diff --git a/lib/api/v3/triggers.rb b/lib/api/v3/triggers.rb new file mode 100644 index 00000000000..a23d6b6b48c --- /dev/null +++ b/lib/api/v3/triggers.rb @@ -0,0 +1,103 @@ +module API + module V3 + class Triggers < Grape::API + include PaginationParams + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: { id: %r{[^/]+} } do + desc 'Trigger a GitLab project build' do + success ::API::V3::Entities::TriggerRequest + end + params do + requires :ref, type: String, desc: 'The commit sha or name of a branch or tag' + requires :token, type: String, desc: 'The unique token of trigger' + optional :variables, type: Hash, desc: 'The list of variables to be injected into build' + end + post ":id/(ref/:ref/)trigger/builds", requirements: { ref: /.+/ } do + project = find_project(params[:id]) + trigger = Ci::Trigger.find_by_token(params[:token].to_s) + not_found! unless project && trigger + unauthorized! unless trigger.project == project + + # validate variables + variables = params[:variables].to_h + unless variables.all? { |key, value| key.is_a?(String) && value.is_a?(String) } + render_api_error!('variables needs to be a map of key-valued strings', 400) + end + + # create request and trigger builds + trigger_request = Ci::CreateTriggerRequestService.new.execute(project, trigger, params[:ref].to_s, variables) + if trigger_request + present trigger_request, with: ::API::V3::Entities::TriggerRequest + else + errors = 'No builds created' + render_api_error!(errors, 400) + end + end + + desc 'Get triggers list' do + success ::API::V3::Entities::Trigger + end + params do + use :pagination + end + get ':id/triggers' do + authenticate! + authorize! :admin_build, user_project + + triggers = user_project.triggers.includes(:trigger_requests) + + present paginate(triggers), with: ::API::V3::Entities::Trigger + end + + desc 'Get specific trigger of a project' do + success ::API::V3::Entities::Trigger + end + params do + requires :token, type: String, desc: 'The unique token of trigger' + end + get ':id/triggers/:token' do + authenticate! + authorize! :admin_build, user_project + + trigger = user_project.triggers.find_by(token: params[:token].to_s) + return not_found!('Trigger') unless trigger + + present trigger, with: ::API::V3::Entities::Trigger + end + + desc 'Create a trigger' do + success ::API::V3::Entities::Trigger + end + post ':id/triggers' do + authenticate! + authorize! :admin_build, user_project + + trigger = user_project.triggers.create + + present trigger, with: ::API::V3::Entities::Trigger + end + + desc 'Delete a trigger' do + success ::API::V3::Entities::Trigger + end + params do + requires :token, type: String, desc: 'The unique token of trigger' + end + delete ':id/triggers/:token' do + authenticate! + authorize! :admin_build, user_project + + trigger = user_project.triggers.find_by(token: params[:token].to_s) + return not_found!('Trigger') unless trigger + + trigger.destroy + + present trigger, with: ::API::V3::Entities::Trigger + end + end + end + end +end diff --git a/lib/api/v3/users.rb b/lib/api/v3/users.rb new file mode 100644 index 00000000000..5e18cecc431 --- /dev/null +++ b/lib/api/v3/users.rb @@ -0,0 +1,202 @@ +module API + module V3 + class Users < Grape::API + include PaginationParams + + before do + allow_access_with_scope :read_user if request.get? + authenticate! + end + + resource :users, requirements: { uid: /[0-9]*/, id: /[0-9]*/ } do + helpers do + params :optional_attributes do + optional :skype, type: String, desc: 'The Skype username' + optional :linkedin, type: String, desc: 'The LinkedIn username' + optional :twitter, type: String, desc: 'The Twitter username' + optional :website_url, type: String, desc: 'The website of the user' + optional :organization, type: String, desc: 'The organization of the user' + optional :projects_limit, type: Integer, desc: 'The number of projects a user can create' + optional :extern_uid, type: String, desc: 'The external authentication provider UID' + optional :provider, type: String, desc: 'The external provider' + optional :bio, type: String, desc: 'The biography of the user' + optional :location, type: String, desc: 'The location of the user' + optional :admin, type: Boolean, desc: 'Flag indicating the user is an administrator' + optional :can_create_group, type: Boolean, desc: 'Flag indicating the user can create groups' + optional :confirm, type: Boolean, default: true, desc: 'Flag indicating the account needs to be confirmed' + optional :external, type: Boolean, desc: 'Flag indicating the user is an external user' + all_or_none_of :extern_uid, :provider + end + end + + desc 'Create a user. Available only for admins.' do + success ::API::Entities::UserPublic + end + params do + requires :email, type: String, desc: 'The email of the user' + optional :password, type: String, desc: 'The password of the new user' + optional :reset_password, type: Boolean, desc: 'Flag indicating the user will be sent a password reset token' + at_least_one_of :password, :reset_password + requires :name, type: String, desc: 'The name of the user' + requires :username, type: String, desc: 'The username of the user' + use :optional_attributes + end + post do + authenticated_as_admin! + + params = declared_params(include_missing: false) + user = ::Users::CreateService.new(current_user, params.merge!(skip_confirmation: !params[:confirm])).execute + + if user.persisted? + present user, with: ::API::Entities::UserPublic + else + conflict!('Email has already been taken') if User. + where(email: user.email). + count > 0 + + conflict!('Username has already been taken') if User. + where(username: user.username). + count > 0 + + render_validation_error!(user) + end + end + + desc 'Get the SSH keys of a specified user. Available only for admins.' do + success ::API::Entities::SSHKey + end + params do + requires :id, type: Integer, desc: 'The ID of the user' + use :pagination + end + get ':id/keys' do + authenticated_as_admin! + + user = User.find_by(id: params[:id]) + not_found!('User') unless user + + present paginate(user.keys), with: ::API::Entities::SSHKey + end + + desc 'Get the emails addresses of a specified user. Available only for admins.' do + success ::API::Entities::Email + end + params do + requires :id, type: Integer, desc: 'The ID of the user' + use :pagination + end + get ':id/emails' do + authenticated_as_admin! + user = User.find_by(id: params[:id]) + not_found!('User') unless user + + present user.emails, with: ::API::Entities::Email + end + + desc 'Block a user. Available only for admins.' + params do + requires :id, type: Integer, desc: 'The ID of the user' + end + put ':id/block' do + authenticated_as_admin! + user = User.find_by(id: params[:id]) + not_found!('User') unless user + + if !user.ldap_blocked? + user.block + else + forbidden!('LDAP blocked users cannot be modified by the API') + end + end + + desc 'Unblock a user. Available only for admins.' + params do + requires :id, type: Integer, desc: 'The ID of the user' + end + put ':id/unblock' do + authenticated_as_admin! + user = User.find_by(id: params[:id]) + not_found!('User') unless user + + if user.ldap_blocked? + forbidden!('LDAP blocked users cannot be unblocked by the API') + else + user.activate + end + end + + desc 'Get the contribution events of a specified user' do + detail 'This feature was introduced in GitLab 8.13.' + success ::API::V3::Entities::Event + end + params do + requires :id, type: Integer, desc: 'The ID of the user' + use :pagination + end + get ':id/events' do + user = User.find_by(id: params[:id]) + not_found!('User') unless user + + events = user.events. + merge(ProjectsFinder.new.execute(current_user)). + references(:project). + with_associations. + recent + + present paginate(events), with: ::API::V3::Entities::Event + end + + desc 'Delete an existing SSH key from a specified user. Available only for admins.' do + success ::API::Entities::SSHKey + end + params do + requires :id, type: Integer, desc: 'The ID of the user' + requires :key_id, type: Integer, desc: 'The ID of the SSH key' + end + delete ':id/keys/:key_id' do + authenticated_as_admin! + + user = User.find_by(id: params[:id]) + not_found!('User') unless user + + key = user.keys.find_by(id: params[:key_id]) + not_found!('Key') unless key + + present key.destroy, with: ::API::Entities::SSHKey + end + end + + resource :user do + desc "Get the currently authenticated user's SSH keys" do + success ::API::Entities::SSHKey + end + params do + use :pagination + end + get "keys" do + present current_user.keys, with: ::API::Entities::SSHKey + end + + desc "Get the currently authenticated user's email addresses" do + success ::API::Entities::Email + end + get "emails" do + present current_user.emails, with: ::API::Entities::Email + end + + desc 'Delete an SSH key from the currently authenticated user' do + success ::API::Entities::SSHKey + end + params do + requires :key_id, type: Integer, desc: 'The ID of the SSH key' + end + delete "keys/:key_id" do + key = current_user.keys.find_by(id: params[:key_id]) + not_found!('Key') unless key + + present key.destroy, with: ::API::Entities::SSHKey + end + end + end + end +end diff --git a/lib/api/v3/variables.rb b/lib/api/v3/variables.rb new file mode 100644 index 00000000000..83972b1e7ce --- /dev/null +++ b/lib/api/v3/variables.rb @@ -0,0 +1,29 @@ +module API + module V3 + class Variables < Grape::API + include PaginationParams + + before { authenticate! } + before { authorize! :admin_build, user_project } + + params do + requires :id, type: String, desc: 'The ID of a project' + end + + resource :projects, requirements: { id: %r{[^/]+} } do + desc 'Delete an existing variable from a project' do + success ::API::Entities::Variable + end + params do + requires :key, type: String, desc: 'The key of the variable' + end + delete ':id/variables/:key' do + variable = user_project.variables.find_by(key: params[:key]) + not_found!('Variable') unless variable + + present variable.destroy, with: ::API::Entities::Variable + end + end + end + end +end diff --git a/lib/api/variables.rb b/lib/api/variables.rb index f623b1dfe9f..5acde41551b 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -1,5 +1,4 @@ module API - # Projects variables API class Variables < Grape::API include PaginationParams @@ -10,7 +9,7 @@ module API requires :id, type: String, desc: 'The ID of a project' end - resource :projects do + resource :projects, requirements: { id: %r{[^/]+} } do desc 'Get project variables' do success Entities::Variable end @@ -81,10 +80,9 @@ module API end delete ':id/variables/:key' do variable = user_project.variables.find_by(key: params[:key]) + not_found!('Variable') unless variable - return not_found!('Variable') unless variable - - present variable.destroy, with: Entities::Variable + variable.destroy end end end diff --git a/lib/backup/database.rb b/lib/backup/database.rb index 22319ec6623..4016ac76348 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -5,7 +5,7 @@ module Backup attr_reader :config, :db_file_name def initialize - @config = YAML.load_file(File.join(Rails.root,'config','database.yml'))[Rails.env] + @config = YAML.load_file(File.join(Rails.root, 'config', 'database.yml'))[Rails.env] @db_file_name = File.join(Gitlab.config.backup.path, 'db', 'database.sql.gz') end @@ -13,28 +13,32 @@ module Backup FileUtils.mkdir_p(File.dirname(db_file_name)) FileUtils.rm_f(db_file_name) compress_rd, compress_wr = IO.pipe - compress_pid = spawn(*%W(gzip -1 -c), in: compress_rd, out: [db_file_name, 'w', 0600]) + compress_pid = spawn(*%w(gzip -1 -c), in: compress_rd, out: [db_file_name, 'w', 0600]) compress_rd.close - dump_pid = case config["adapter"] - when /^mysql/ then - $progress.print "Dumping MySQL database #{config['database']} ... " - # Workaround warnings from MySQL 5.6 about passwords on cmd line - ENV['MYSQL_PWD'] = config["password"].to_s if config["password"] - spawn('mysqldump', *mysql_args, config['database'], out: compress_wr) - when "postgresql" then - $progress.print "Dumping PostgreSQL database #{config['database']} ... " - pg_env - pgsql_args = ["--clean"] # Pass '--clean' to include 'DROP TABLE' statements in the DB dump. - if Gitlab.config.backup.pg_schema - pgsql_args << "-n" - pgsql_args << Gitlab.config.backup.pg_schema + dump_pid = + case config["adapter"] + when /^mysql/ then + $progress.print "Dumping MySQL database #{config['database']} ... " + # Workaround warnings from MySQL 5.6 about passwords on cmd line + ENV['MYSQL_PWD'] = config["password"].to_s if config["password"] + spawn('mysqldump', *mysql_args, config['database'], out: compress_wr) + when "postgresql" then + $progress.print "Dumping PostgreSQL database #{config['database']} ... " + pg_env + pgsql_args = ["--clean"] # Pass '--clean' to include 'DROP TABLE' statements in the DB dump. + if Gitlab.config.backup.pg_schema + pgsql_args << "-n" + pgsql_args << Gitlab.config.backup.pg_schema + end + spawn('pg_dump', *pgsql_args, config['database'], out: compress_wr) end - spawn('pg_dump', *pgsql_args, config['database'], out: compress_wr) - end compress_wr.close - success = [compress_pid, dump_pid].all? { |pid| Process.waitpid(pid); $?.success? } + success = [compress_pid, dump_pid].all? do |pid| + Process.waitpid(pid) + $?.success? + end report_success(success) abort 'Backup failed' unless success @@ -42,23 +46,27 @@ module Backup def restore decompress_rd, decompress_wr = IO.pipe - decompress_pid = spawn(*%W(gzip -cd), out: decompress_wr, in: db_file_name) + decompress_pid = spawn(*%w(gzip -cd), out: decompress_wr, in: db_file_name) decompress_wr.close - restore_pid = case config["adapter"] - when /^mysql/ then - $progress.print "Restoring MySQL database #{config['database']} ... " - # Workaround warnings from MySQL 5.6 about passwords on cmd line - ENV['MYSQL_PWD'] = config["password"].to_s if config["password"] - spawn('mysql', *mysql_args, config['database'], in: decompress_rd) - when "postgresql" then - $progress.print "Restoring PostgreSQL database #{config['database']} ... " - pg_env - spawn('psql', config['database'], in: decompress_rd) - end + restore_pid = + case config["adapter"] + when /^mysql/ then + $progress.print "Restoring MySQL database #{config['database']} ... " + # Workaround warnings from MySQL 5.6 about passwords on cmd line + ENV['MYSQL_PWD'] = config["password"].to_s if config["password"] + spawn('mysql', *mysql_args, config['database'], in: decompress_rd) + when "postgresql" then + $progress.print "Restoring PostgreSQL database #{config['database']} ... " + pg_env + spawn('psql', config['database'], in: decompress_rd) + end decompress_rd.close - success = [decompress_pid, restore_pid].all? { |pid| Process.waitpid(pid); $?.success? } + success = [decompress_pid, restore_pid].all? do |pid| + Process.waitpid(pid) + $?.success? + end report_success(success) abort 'Restore failed' unless success diff --git a/lib/backup/files.rb b/lib/backup/files.rb index cedbb289f6a..30a91647b77 100644 --- a/lib/backup/files.rb +++ b/lib/backup/files.rb @@ -8,6 +8,7 @@ module Backup @name = name @app_files_dir = File.realpath(app_files_dir) @files_parent_dir = File.realpath(File.join(@app_files_dir, '..')) + @backup_files_dir = File.join(Gitlab.config.backup.path, File.basename(@app_files_dir) ) @backup_tarball = File.join(Gitlab.config.backup.path, name + '.tar.gz') end @@ -15,14 +16,28 @@ module Backup def dump FileUtils.mkdir_p(Gitlab.config.backup.path) FileUtils.rm_f(backup_tarball) - run_pipeline!([%W(tar -C #{app_files_dir} -cf - .), %W(gzip -c -1)], out: [backup_tarball, 'w', 0600]) + + if ENV['STRATEGY'] == 'copy' + cmd = %W(cp -a #{app_files_dir} #{Gitlab.config.backup.path}) + output, status = Gitlab::Popen.popen(cmd) + + unless status.zero? + puts output + abort 'Backup failed' + end + + run_pipeline!([%W(tar -C #{@backup_files_dir} -cf - .), %w(gzip -c -1)], out: [backup_tarball, 'w', 0600]) + FileUtils.rm_rf(@backup_files_dir) + else + run_pipeline!([%W(tar -C #{app_files_dir} -cf - .), %w(gzip -c -1)], out: [backup_tarball, 'w', 0600]) + end end def restore backup_existing_files_dir create_files_dir - run_pipeline!([%W(gzip -cd), %W(tar -C #{app_files_dir} -xf -)], in: backup_tarball) + run_pipeline!([%w(gzip -cd), %W(tar -C #{app_files_dir} -xf -)], in: backup_tarball) end def backup_existing_files_dir @@ -32,7 +47,7 @@ module Backup end end - def run_pipeline!(cmd_list, options={}) + def run_pipeline!(cmd_list, options = {}) status_list = Open3.pipeline(*cmd_list, options) abort 'Backup failed' unless status_list.compact.all?(&:success?) end diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index cefbfdce3bb..7b4476fa4db 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -1,8 +1,8 @@ module Backup class Manager - ARCHIVES_TO_BACKUP = %w[uploads builds artifacts lfs registry] - FOLDERS_TO_BACKUP = %w[repositories db] - FILE_NAME_SUFFIX = '_gitlab_backup.tar' + ARCHIVES_TO_BACKUP = %w[uploads builds artifacts pages lfs registry].freeze + FOLDERS_TO_BACKUP = %w[repositories db].freeze + FILE_NAME_SUFFIX = '_gitlab_backup.tar'.freeze def pack # Make sure there is a connection @@ -20,13 +20,13 @@ module Backup Dir.chdir(Gitlab.config.backup.path) do File.open("#{Gitlab.config.backup.path}/backup_information.yml", "w+") do |file| - file << s.to_yaml.gsub(/^---\n/,'') + file << s.to_yaml.gsub(/^---\n/, '') end # create archive $progress.print "Creating backup archive: #{tar_file} ... " # Set file permissions on open to prevent chmod races. - tar_system_options = {out: [tar_file, 'w', Gitlab.config.backup.archive_permissions]} + tar_system_options = { out: [tar_file, 'w', Gitlab.config.backup.archive_permissions] } if Kernel.system('tar', '-cf', '-', *backup_contents, tar_system_options) $progress.puts "done".color(:green) else @@ -50,8 +50,9 @@ module Backup directory = connect_to_remote_directory(connection_settings) if directory.files.create(key: tar_file, body: File.open(tar_file), public: false, - multipart_chunk_size: Gitlab.config.backup.upload.multipart_chunk_size, - encryption: Gitlab.config.backup.upload.encryption) + multipart_chunk_size: Gitlab.config.backup.upload.multipart_chunk_size, + encryption: Gitlab.config.backup.upload.encryption, + storage_class: Gitlab.config.backup.upload.storage_class) $progress.puts "done".color(:green) else puts "uploading backup to #{remote_directory} failed".color(:red) @@ -123,11 +124,11 @@ module Backup exit 1 end - if ENV['BACKUP'].present? - tar_file = "#{ENV['BACKUP']}#{FILE_NAME_SUFFIX}" - else - tar_file = file_list.first - end + tar_file = if ENV['BACKUP'].present? + "#{ENV['BACKUP']}#{FILE_NAME_SUFFIX}" + else + file_list.first + end unless File.exist?(tar_file) $progress.puts "The backup file #{tar_file} does not exist!" @@ -158,7 +159,7 @@ module Backup end def tar_version - tar_version, _ = Gitlab::Popen.popen(%W(tar --version)) + tar_version, _ = Gitlab::Popen.popen(%w(tar --version)) tar_version.force_encoding('locale').split("\n").first end diff --git a/lib/backup/pages.rb b/lib/backup/pages.rb new file mode 100644 index 00000000000..215ded93bfe --- /dev/null +++ b/lib/backup/pages.rb @@ -0,0 +1,13 @@ +require 'backup/files' + +module Backup + class Pages < Files + def initialize + super('pages', Gitlab.config.pages.path) + end + + def create_files_dir + Dir.mkdir(app_files_dir, 0700) + end + end +end diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index d746070913d..6b29600a751 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -2,7 +2,7 @@ require 'yaml' module Backup class Repository - + # rubocop:disable Metrics/AbcSize def dump prepare @@ -12,7 +12,7 @@ module Backup path_to_project_bundle = path_to_bundle(project) # Create namespace dir if missing - FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.path)) if project.namespace + FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.full_path)) if project.namespace if project.empty_repo? $progress.puts "[SKIPPED]".color(:cyan) @@ -68,7 +68,8 @@ module Backup end def restore - Gitlab.config.repositories.storages.each do |name, path| + Gitlab.config.repositories.storages.each do |name, repository_storage| + path = repository_storage['path'] next unless File.exist?(path) # Move repos dir to 'repositories.old' dir @@ -85,11 +86,11 @@ module Backup project.ensure_dir_exist - if File.exists?(path_to_project_bundle) - cmd = %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_project_bundle} #{path_to_project_repo}) - else - cmd = %W(#{Gitlab.config.git.bin_path} init --bare #{path_to_project_repo}) - end + cmd = if File.exist?(path_to_project_bundle) + %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_project_bundle} #{path_to_project_repo}) + else + %W(#{Gitlab.config.git.bin_path} init --bare #{path_to_project_repo}) + end output, status = Gitlab::Popen.popen(cmd) if status.zero? @@ -150,6 +151,7 @@ module Backup puts output end end + # rubocop:enable Metrics/AbcSize protected @@ -179,8 +181,9 @@ module Backup return unless Dir.exist?(path) dir_entries = Dir.entries(path) - %w[annex custom_hooks].each do |entry| - yield(entry) if dir_entries.include?(entry) + + if dir_entries.include?('custom_hooks') || dir_entries.include?('custom_hooks.tar') + yield('custom_hooks') end end @@ -193,13 +196,13 @@ module Backup end def silent - {err: '/dev/null', out: '/dev/null'} + { err: '/dev/null', out: '/dev/null' } end private def repository_storage_paths_args - Gitlab.config.repositories.storages.values + Gitlab.config.repositories.storages.values.map { |rs| rs['path'] } end end end diff --git a/lib/backup/uploads.rb b/lib/backup/uploads.rb index 9261f77f3c9..35118375499 100644 --- a/lib/backup/uploads.rb +++ b/lib/backup/uploads.rb @@ -2,7 +2,6 @@ require 'backup/files' module Backup class Uploads < Files - def initialize super('uploads', Rails.root.join('public/uploads')) end diff --git a/lib/banzai/cross_project_reference.rb b/lib/banzai/cross_project_reference.rb index 0257848b6bc..e2b57adf611 100644 --- a/lib/banzai/cross_project_reference.rb +++ b/lib/banzai/cross_project_reference.rb @@ -14,7 +14,7 @@ module Banzai def project_from_ref(ref) return context[:project] unless ref - Project.find_with_namespace(ref) + Project.find_by_full_path(ref) end end end diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 6d04f68c8f9..8bc2dd18bda 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -33,7 +33,12 @@ module Banzai # Returns a String replaced with the return of the block. def self.references_in(text, pattern = object_class.reference_pattern) text.gsub(pattern) do |match| - yield match, $~[object_sym].to_i, $~[:project], $~[:namespace], $~ + symbol = $~[object_sym] + if object_class.reference_valid?(symbol) + yield match, symbol.to_i, $~[:project], $~[:namespace], $~ + else + match + end end end @@ -153,13 +158,14 @@ module Banzai title = object_link_title(object) klass = reference_class(object_sym) - data = data_attributes_for(link_content || match, project, object) + data = data_attributes_for(link_content || match, project, object, link: !!link_content) - if matches.names.include?("url") && matches[:url] - url = matches[:url] - else - url = url_for_object_cached(object, project) - end + url = + if matches.names.include?("url") && matches[:url] + matches[:url] + else + url_for_object_cached(object, project) + end content = link_content || object_link_text(object, matches) @@ -172,9 +178,10 @@ module Banzai end end - def data_attributes_for(text, project, object) + def data_attributes_for(text, project, object, link: false) data_attribute( original: text, + link: link, project: project.id, object_sym => object.id ) @@ -232,18 +239,13 @@ module Banzai # path. def projects_per_reference @projects_per_reference ||= begin - hash = {} refs = Set.new references_per_project.each do |project_ref, _| refs << project_ref end - find_projects_for_paths(refs.to_a).each do |project| - hash[project.path_with_namespace] = project - end - - hash + find_projects_for_paths(refs.to_a).index_by(&:full_path) end end @@ -284,7 +286,7 @@ module Banzai end def current_project_namespace_path - @current_project_namespace_path ||= project.namespace.path + @current_project_namespace_path ||= project.namespace.full_path end private diff --git a/lib/banzai/filter/autolink_filter.rb b/lib/banzai/filter/autolink_filter.rb index 80c844baecd..b8d2673c1a6 100644 --- a/lib/banzai/filter/autolink_filter.rb +++ b/lib/banzai/filter/autolink_filter.rb @@ -37,7 +37,7 @@ module Banzai and contains(., '://') and not(starts-with(., 'http')) and not(starts-with(., 'ftp')) - ]) + ]).freeze def call return doc if context[:autolink] == false diff --git a/lib/banzai/filter/emoji_filter.rb b/lib/banzai/filter/emoji_filter.rb index a8c1ca0c60a..d6138816e70 100644 --- a/lib/banzai/filter/emoji_filter.rb +++ b/lib/banzai/filter/emoji_filter.rb @@ -17,8 +17,8 @@ module Banzai next unless content.include?(':') || node.text.match(emoji_unicode_pattern) - html = emoji_name_image_filter(content) - html = emoji_unicode_image_filter(html) + html = emoji_unicode_element_unicode_filter(content) + html = emoji_name_element_unicode_filter(html) next if html == content @@ -27,33 +27,30 @@ module Banzai doc end - # Replace :emoji: with corresponding images. + # Replace :emoji: with corresponding gl-emoji unicode. # # text - String text to replace :emoji: in. # - # Returns a String with :emoji: replaced with images. - def emoji_name_image_filter(text) + # Returns a String with :emoji: replaced with gl-emoji unicode. + def emoji_name_element_unicode_filter(text) text.gsub(emoji_pattern) do |match| name = $1 - emoji_image_tag(name, emoji_url(name)) + Gitlab::Emoji.gl_emoji_tag(name) end end - # Replace unicode emoji with corresponding images if they exist. + # Replace unicode emoji with corresponding gl-emoji unicode. # # text - String text to replace unicode emoji in. # - # Returns a String with unicode emoji replaced with images. - def emoji_unicode_image_filter(text) + # Returns a String with unicode emoji replaced with gl-emoji unicode. + def emoji_unicode_element_unicode_filter(text) text.gsub(emoji_unicode_pattern) do |moji| - emoji_image_tag(Gitlab::Emoji.emojis_by_moji[moji]['name'], emoji_unicode_url(moji)) + emoji_info = Gitlab::Emoji.emojis_by_moji[moji] + Gitlab::Emoji.gl_emoji_tag(emoji_info['name']) end end - def emoji_image_tag(emoji_name, emoji_url) - "<img class='emoji' title=':#{emoji_name}:' alt=':#{emoji_name}:' src='#{emoji_url}' height='20' width='20' align='absmiddle' />" - end - # Build a regexp that matches all valid :emoji: names. def self.emoji_pattern @emoji_pattern ||= /:(#{Gitlab::Emoji.emojis_names.map { |name| Regexp.escape(name) }.join('|')}):/ @@ -66,52 +63,13 @@ module Banzai private - def emoji_url(name) - emoji_path = emoji_filename(name) - - if context[:asset_host] - # Asset host is specified. - url_to_image(emoji_path) - elsif context[:asset_root] - # Gitlab url is specified - File.join(context[:asset_root], url_to_image(emoji_path)) - else - # All other cases - url_to_image(emoji_path) - end - end - - def emoji_unicode_url(moji) - emoji_unicode_path = emoji_unicode_filename(moji) - - if context[:asset_host] - url_to_image(emoji_unicode_path) - elsif context[:asset_root] - File.join(context[:asset_root], url_to_image(emoji_unicode_path)) - else - url_to_image(emoji_unicode_path) - end - end - - def url_to_image(image) - ActionController::Base.helpers.url_to_image(image) - end - def emoji_pattern self.class.emoji_pattern end - def emoji_filename(name) - "#{Gitlab::Emoji.emoji_filename(name)}.png" - end - def emoji_unicode_pattern self.class.emoji_unicode_pattern end - - def emoji_unicode_filename(name) - "#{Gitlab::Emoji.emoji_unicode_filename(name)}.png" - end end end end diff --git a/lib/banzai/filter/gollum_tags_filter.rb b/lib/banzai/filter/gollum_tags_filter.rb index d08267a9d6c..0ea4eeaed5b 100644 --- a/lib/banzai/filter/gollum_tags_filter.rb +++ b/lib/banzai/filter/gollum_tags_filter.rb @@ -149,11 +149,12 @@ module Banzai name, reference = *parts.compact.map(&:strip) end - if url?(reference) - href = reference - else - href = ::File.join(project_wiki_base_path, reference) - end + href = + if url?(reference) + reference + else + ::File.join(project_wiki_base_path, reference) + end content_tag(:a, name || reference, href: href, class: 'gfm') end diff --git a/lib/banzai/filter/image_link_filter.rb b/lib/banzai/filter/image_link_filter.rb index f0fb6084a35..123c92fd250 100644 --- a/lib/banzai/filter/image_link_filter.rb +++ b/lib/banzai/filter/image_link_filter.rb @@ -2,29 +2,22 @@ module Banzai module Filter # HTML filter that wraps links around inline images. class ImageLinkFilter < HTML::Pipeline::Filter - # Find every image that isn't already wrapped in an `a` tag, create # a new node (a link to the image source), copy the image as a child # of the anchor, and then replace the img with the link-wrapped version. def call doc.xpath('descendant-or-self::img[not(ancestor::a)]').each do |img| - div = doc.document.create_element( - 'div', - class: 'image-container' - ) - link = doc.document.create_element( 'a', class: 'no-attachment-icon', href: img['src'], - target: '_blank' + target: '_blank', + rel: 'noopener noreferrer' ) link.children = img.clone - div.children = link - - img.replace(div) + img.replace(link) end doc diff --git a/lib/banzai/filter/issue_reference_filter.rb b/lib/banzai/filter/issue_reference_filter.rb index 4d1bc687696..044d18ff824 100644 --- a/lib/banzai/filter/issue_reference_filter.rb +++ b/lib/banzai/filter/issue_reference_filter.rb @@ -39,11 +39,12 @@ module Banzai projects_per_reference.each do |path, project| issue_ids = references_per_project[path] - if project.default_issues_tracker? - issues = project.issues.where(iid: issue_ids.to_a) - else - issues = issue_ids.map { |id| ExternalIssue.new(id, project) } - end + issues = + if project.default_issues_tracker? + project.issues.where(iid: issue_ids.to_a) + else + issue_ids.map { |id| ExternalIssue.new(id, project) } + end issues.each do |issue| hash[project][issue.iid.to_i] = issue @@ -62,7 +63,7 @@ module Banzai end end - def data_attributes_for(text, project, object) + def data_attributes_for(text, project, object, link: false) if object.is_a?(ExternalIssue) data_attribute( project: project.id, diff --git a/lib/banzai/filter/merge_request_reference_filter.rb b/lib/banzai/filter/merge_request_reference_filter.rb index ac5216d9cfb..3888acf935e 100644 --- a/lib/banzai/filter/merge_request_reference_filter.rb +++ b/lib/banzai/filter/merge_request_reference_filter.rb @@ -11,8 +11,8 @@ module Banzai MergeRequest end - def find_object(project, id) - project.merge_requests.find_by(iid: id) + def find_object(project, iid) + merge_requests_per_project[project][iid] end def url_for_object(mr, project) @@ -21,6 +21,31 @@ module Banzai only_path: context[:only_path]) end + def project_from_ref(ref) + projects_per_reference[ref || current_project_path] + end + + # Returns a Hash containing the merge_requests per Project instance. + def merge_requests_per_project + @merge_requests_per_project ||= begin + hash = Hash.new { |h, k| h[k] = {} } + + projects_per_reference.each do |path, project| + merge_request_ids = references_per_project[path] + + merge_requests = project.merge_requests + .where(iid: merge_request_ids.to_a) + .includes(target_project: :namespace) + + merge_requests.each do |merge_request| + hash[project][merge_request.iid.to_i] = merge_request + end + end + + hash + end + end + def object_link_text_extras(object, matches) extras = super diff --git a/lib/banzai/filter/plantuml_filter.rb b/lib/banzai/filter/plantuml_filter.rb new file mode 100644 index 00000000000..b2537117558 --- /dev/null +++ b/lib/banzai/filter/plantuml_filter.rb @@ -0,0 +1,39 @@ +require "nokogiri" +require "asciidoctor-plantuml/plantuml" + +module Banzai + module Filter + # HTML that replaces all `code plantuml` tags with PlantUML img tags. + # + class PlantumlFilter < HTML::Pipeline::Filter + def call + return doc unless doc.at('pre.plantuml') && settings.plantuml_enabled + + plantuml_setup + + doc.css('pre.plantuml').each do |el| + img_tag = Nokogiri::HTML::DocumentFragment.parse( + Asciidoctor::PlantUml::Processor.plantuml_content(el.content, {})) + el.replace img_tag + end + + doc + end + + private + + def settings + ApplicationSetting.current || ApplicationSetting.create_from_defaults + end + + def plantuml_setup + Asciidoctor::PlantUml.configure do |conf| + conf.url = settings.plantuml_url + conf.png_enable = settings.plantuml_enabled + conf.svg_enable = false + conf.txt_enable = false + end + end + end + end +end diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index ab7af1cad21..6640168bfa2 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -53,6 +53,10 @@ module Banzai context[:project] end + def skip_project_check? + context[:skip_project_check] + end + def reference_class(type) "gfm gfm-#{type} has-tooltip" end diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index af1e575fc89..d5f9e252f62 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -35,6 +35,10 @@ module Banzai # Allow span elements whitelist[:elements].push('span') + # Allow html5 details/summary elements + whitelist[:elements].push('details') + whitelist[:elements].push('summary') + # Allow abbr elements with title attribute whitelist[:elements].push('abbr') whitelist[:attributes]['abbr'] = %w(title) diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index 026b81ac175..9f09ca90697 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -5,8 +5,6 @@ module Banzai # HTML Filter to highlight fenced code blocks # class SyntaxHighlightFilter < HTML::Pipeline::Filter - include Rouge::Plugins::Redcarpet - def call doc.search('pre > code').each do |node| highlight_node(node) @@ -20,17 +18,19 @@ module Banzai code = node.text css_classes = "code highlight" lexer = lexer_for(language) + lang = lexer.tag begin - code = format(lex(lexer, code)) + code = Rouge::Formatters::HTMLGitlab.format(lex(lexer, code), tag: lang) - css_classes << " js-syntax-highlight #{lexer.tag}" + css_classes << " js-syntax-highlight #{lang}" rescue + lang = nil # Gracefully handle syntax highlighter bugs/errors to ensure # users can still access an issue/comment/etc. end - highlighted = %(<pre class="#{css_classes}" v-pre="true"><code>#{code}</code></pre>) + highlighted = %(<pre class="#{css_classes}" lang="#{lang}" v-pre="true"><code>#{code}</code></pre>) # Extracted to a method to measure it replace_parent_pre_element(node, highlighted) @@ -43,10 +43,6 @@ module Banzai lexer.lex(code) end - def format(tokens) - rouge_formatter.format(tokens) - end - def lexer_for(language) (Rouge::Lexer.find(language) || Rouge::Lexers::PlainText).new end @@ -55,11 +51,6 @@ module Banzai # Replace the parent `pre` element with the entire highlighted block node.parent.replace(highlighted) end - - # Override Rouge::Plugins::Redcarpet#rouge_formatter - def rouge_formatter(lexer = nil) - @rouge_formatter ||= Rouge::Formatters::HTML.new - end end end end diff --git a/lib/banzai/filter/user_reference_filter.rb b/lib/banzai/filter/user_reference_filter.rb index f842b1fb779..a798927823f 100644 --- a/lib/banzai/filter/user_reference_filter.rb +++ b/lib/banzai/filter/user_reference_filter.rb @@ -24,7 +24,7 @@ module Banzai end def call - return doc if project.nil? + return doc if project.nil? && !skip_project_check? ref_pattern = User.reference_pattern ref_pattern_start = /\A#{ref_pattern}\z/ @@ -58,9 +58,9 @@ module Banzai # have `gfm` and `gfm-project_member` class names attached for styling. def user_link_filter(text, link_content: nil) self.class.references_in(text) do |match, username| - if username == 'all' + if username == 'all' && !skip_project_check? link_to_all(link_content: link_content) - elsif namespace = namespaces[username] + elsif namespace = namespaces[username.downcase] link_to_namespace(namespace, link_content: link_content) || match else match @@ -74,10 +74,7 @@ module Banzai # The keys of this Hash are the namespace paths, the values the # corresponding Namespace objects. def namespaces - @namespaces ||= - Namespace.where(path: usernames).each_with_object({}) do |row, hash| - hash[row.path] = row - end + @namespaces ||= Namespace.where_full_path_in(usernames).index_by(&:full_path).transform_keys(&:downcase) end # Returns all usernames referenced in the current document. @@ -122,7 +119,7 @@ module Banzai def link_to_namespace(namespace, link_content: nil) if namespace.is_a?(Group) - link_to_group(namespace.path, namespace, link_content: link_content) + link_to_group(namespace.full_path, namespace, link_content: link_content) else link_to_user(namespace.path, namespace, link_content: link_content) end @@ -133,7 +130,7 @@ module Banzai data = data_attribute(group: namespace.id) content = link_content || Group.reference_prefix + group - link_tag(url, data, content, namespace.name) + link_tag(url, data, content, namespace.full_name) end def link_to_user(user, namespace, link_content: nil) diff --git a/lib/banzai/filter/video_link_filter.rb b/lib/banzai/filter/video_link_filter.rb index ac7bbcb0d10..35cb10eae5d 100644 --- a/lib/banzai/filter/video_link_filter.rb +++ b/lib/banzai/filter/video_link_filter.rb @@ -35,13 +35,15 @@ module Banzai src: element['src'], width: '400', controls: true, - 'data-setup' => '{}') + 'data-setup' => '{}', + 'data-title' => element['title'] || element['alt']) link = doc.document.create_element( 'a', element['title'] || element['alt'], href: element['src'], target: '_blank', + rel: 'noopener noreferrer', title: "Download '#{element['title'] || element['alt']}'") download_paragraph = doc.document.create_element('p') download_paragraph.children = link diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index 5a1f873496c..fd4a6a107c2 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -1,9 +1,16 @@ module Banzai module Pipeline class GfmPipeline < BasePipeline + # These filters convert GitLab Flavored Markdown (GFM) to HTML. + # The handlers defined in app/assets/javascripts/copy_as_gfm.js + # consequently convert that same HTML to GFM to be copied to the clipboard. + # Every filter that generates HTML from GFM should have a handler in + # app/assets/javascripts/copy_as_gfm.js, in reverse order. + # The GFM-to-HTML-to-GFM cycle is tested in spec/features/copy_as_gfm_spec.rb. def self.filters @filters ||= FilterArray[ Filter::SyntaxHighlightFilter, + Filter::PlantumlFilter, Filter::SanitizationFilter, Filter::MathFilter, diff --git a/lib/banzai/querying.rb b/lib/banzai/querying.rb index 1e1b51e683e..fb2faae02bc 100644 --- a/lib/banzai/querying.rb +++ b/lib/banzai/querying.rb @@ -1,18 +1,64 @@ module Banzai module Querying + module_function + # Searches a Nokogiri document using a CSS query, optionally optimizing it # whenever possible. # - # document - A document/element to search. - # query - The CSS query to use. + # document - A document/element to search. + # query - The CSS query to use. + # reference_options - A hash with nodes filter options # - # Returns a Nokogiri::XML::NodeSet. - def self.css(document, query) + # Returns an array of Nokogiri::XML::Element objects if location is specified + # in reference_options. Otherwise it would a Nokogiri::XML::NodeSet. + def css(document, query, reference_options = {}) # When using "a.foo" Nokogiri compiles this to "//a[...]" but # "descendant::a[...]" is quite a bit faster and achieves the same result. xpath = Nokogiri::CSS.xpath_for(query)[0].gsub(%r{^//}, 'descendant::') + xpath = restrict_to_p_nodes_at_root(xpath) if filter_nodes_at_beginning?(reference_options) + nodes = document.xpath(xpath) + + filter_nodes(nodes, reference_options) + end + + def restrict_to_p_nodes_at_root(xpath) + xpath.gsub('descendant::', './p/') + end + + def filter_nodes(nodes, reference_options) + if filter_nodes_at_beginning?(reference_options) + filter_nodes_at_beginning(nodes) + else + nodes + end + end + + def filter_nodes_at_beginning?(reference_options) + reference_options && reference_options[:location] == :beginning + end + + # Selects child nodes if they are present in the beginning among other siblings. + # + # nodes - A Nokogiri::XML::NodeSet. + # + # Returns an array of Nokogiri::XML::Element objects. + def filter_nodes_at_beginning(nodes) + parents_and_nodes = nodes.group_by(&:parent) + filtered_nodes = [] + + parents_and_nodes.each do |parent, nodes| + children = parent.children + nodes = nodes.to_a + + children.each do |child| + next if child.text.blank? + node = nodes.shift + break unless node == child + filtered_nodes << node + end + end - document.xpath(xpath) + filtered_nodes end end end diff --git a/lib/banzai/reference_extractor.rb b/lib/banzai/reference_extractor.rb index b26a41a1f3b..8e3b0c4db79 100644 --- a/lib/banzai/reference_extractor.rb +++ b/lib/banzai/reference_extractor.rb @@ -16,6 +16,11 @@ module Banzai processor.process(html_documents) end + def reset_memoized_values + @html_documents = nil + @texts_and_contexts = [] + end + private def html_documents diff --git a/lib/banzai/reference_parser/base_parser.rb b/lib/banzai/reference_parser/base_parser.rb index d8a855ec1fe..52fdb9a2140 100644 --- a/lib/banzai/reference_parser/base_parser.rb +++ b/lib/banzai/reference_parser/base_parser.rb @@ -33,7 +33,7 @@ module Banzai # they have access to. class BaseParser class << self - attr_accessor :reference_type + attr_accessor :reference_type, :reference_options end # Returns the attribute name containing the value for every object to be @@ -134,9 +134,7 @@ module Banzai ids = unique_attribute_values(nodes, attribute) rows = collection_objects_for_ids(collection, ids) - rows.each_with_object({}) do |row, hash| - hash[row.id] = row - end + rows.index_by(&:id) end # Returns an Array containing all unique values of an attribute of the @@ -182,9 +180,10 @@ module Banzai # the references. def process(documents) type = self.class.reference_type + reference_options = self.class.reference_options nodes = documents.flat_map do |document| - Querying.css(document, "a[data-reference-type='#{type}'].gfm").to_a + Querying.css(document, "a[data-reference-type='#{type}'].gfm", reference_options).to_a end gather_references(nodes) @@ -209,7 +208,7 @@ module Banzai grouped_objects_for_nodes(nodes, Project, 'data-project') end - def can?(user, permission, subject) + def can?(user, permission, subject = :global) Ability.allowed?(user, permission, subject) end diff --git a/lib/banzai/reference_parser/directly_addressed_user_parser.rb b/lib/banzai/reference_parser/directly_addressed_user_parser.rb new file mode 100644 index 00000000000..77df9bbd024 --- /dev/null +++ b/lib/banzai/reference_parser/directly_addressed_user_parser.rb @@ -0,0 +1,8 @@ +module Banzai + module ReferenceParser + class DirectlyAddressedUserParser < UserParser + self.reference_type = :user + self.reference_options = { location: :beginning } + end + end +end diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index f31fb6c3f71..74663556cbb 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -52,9 +52,9 @@ module Banzai end # Same as +render_field+, but without consulting or updating the cache field - def cacheless_render_field(object, field) + def cacheless_render_field(object, field, options = {}) text = object.__send__(field) - context = object.banzai_render_context(field) + context = object.banzai_render_context(field).merge(options) cacheless_render(text, context) end diff --git a/lib/bitbucket/connection.rb b/lib/bitbucket/connection.rb index 7e55cf4deab..b9279c33f5b 100644 --- a/lib/bitbucket/connection.rb +++ b/lib/bitbucket/connection.rb @@ -1,8 +1,8 @@ module Bitbucket class Connection - DEFAULT_API_VERSION = '2.0' - DEFAULT_BASE_URI = 'https://api.bitbucket.org/' - DEFAULT_QUERY = {} + DEFAULT_API_VERSION = '2.0'.freeze + DEFAULT_BASE_URI = 'https://api.bitbucket.org/'.freeze + DEFAULT_QUERY = {}.freeze attr_reader :expires_at, :expires_in, :refresh_token, :token @@ -24,9 +24,7 @@ module Bitbucket response.parsed end - def expired? - connection.expired? - end + delegate :expired?, to: :connection def refresh! response = connection.refresh! diff --git a/lib/bitbucket/error/unauthorized.rb b/lib/bitbucket/error/unauthorized.rb index 5e2eb57bb0e..efe10542f19 100644 --- a/lib/bitbucket/error/unauthorized.rb +++ b/lib/bitbucket/error/unauthorized.rb @@ -1,6 +1,5 @@ module Bitbucket module Error - class Unauthorized < StandardError - end + Unauthorized = Class.new(StandardError) end end diff --git a/lib/bitbucket/representation/repo.rb b/lib/bitbucket/representation/repo.rb index 423eff8f2a5..59b0fda8e14 100644 --- a/lib/bitbucket/representation/repo.rb +++ b/lib/bitbucket/representation/repo.rb @@ -23,7 +23,7 @@ module Bitbucket url = raw['links']['clone'].find { |link| link['name'] == 'https' }.fetch('href') if token.present? - clone_url = URI::parse(url) + clone_url = URI.parse(url) clone_url.user = "x-token-auth:#{token}" clone_url.to_s else diff --git a/lib/ci/ansi2html.rb b/lib/ci/ansi2html.rb index c10d3616f31..b3ccad7b28d 100644 --- a/lib/ci/ansi2html.rb +++ b/lib/ci/ansi2html.rb @@ -13,7 +13,7 @@ module Ci 5 => 'magenta', 6 => 'cyan', 7 => 'white', # not that this is gray in the dark (aka default) color table - } + }.freeze STYLE_SWITCHES = { bold: 0x01, @@ -21,7 +21,7 @@ module Ci underline: 0x04, conceal: 0x08, cross: 0x10, - } + }.freeze def self.convert(ansi, state = nil) Converter.new.convert(ansi, state) @@ -29,64 +29,108 @@ module Ci class Converter def on_0(s) reset() end + def on_1(s) enable(STYLE_SWITCHES[:bold]) end + def on_3(s) enable(STYLE_SWITCHES[:italic]) end + def on_4(s) enable(STYLE_SWITCHES[:underline]) end + def on_8(s) enable(STYLE_SWITCHES[:conceal]) end + def on_9(s) enable(STYLE_SWITCHES[:cross]) end def on_21(s) disable(STYLE_SWITCHES[:bold]) end + def on_22(s) disable(STYLE_SWITCHES[:bold]) end + def on_23(s) disable(STYLE_SWITCHES[:italic]) end + def on_24(s) disable(STYLE_SWITCHES[:underline]) end + def on_28(s) disable(STYLE_SWITCHES[:conceal]) end + def on_29(s) disable(STYLE_SWITCHES[:cross]) end def on_30(s) set_fg_color(0) end + def on_31(s) set_fg_color(1) end + def on_32(s) set_fg_color(2) end + def on_33(s) set_fg_color(3) end + def on_34(s) set_fg_color(4) end + def on_35(s) set_fg_color(5) end + def on_36(s) set_fg_color(6) end + def on_37(s) set_fg_color(7) end + def on_38(s) set_fg_color_256(s) end + def on_39(s) set_fg_color(9) end def on_40(s) set_bg_color(0) end + def on_41(s) set_bg_color(1) end + def on_42(s) set_bg_color(2) end + def on_43(s) set_bg_color(3) end + def on_44(s) set_bg_color(4) end + def on_45(s) set_bg_color(5) end + def on_46(s) set_bg_color(6) end + def on_47(s) set_bg_color(7) end + def on_48(s) set_bg_color_256(s) end + def on_49(s) set_bg_color(9) end def on_90(s) set_fg_color(0, 'l') end + def on_91(s) set_fg_color(1, 'l') end + def on_92(s) set_fg_color(2, 'l') end + def on_93(s) set_fg_color(3, 'l') end + def on_94(s) set_fg_color(4, 'l') end + def on_95(s) set_fg_color(5, 'l') end + def on_96(s) set_fg_color(6, 'l') end + def on_97(s) set_fg_color(7, 'l') end + def on_99(s) set_fg_color(9, 'l') end def on_100(s) set_bg_color(0, 'l') end + def on_101(s) set_bg_color(1, 'l') end + def on_102(s) set_bg_color(2, 'l') end + def on_103(s) set_bg_color(3, 'l') end + def on_104(s) set_bg_color(4, 'l') end + def on_105(s) set_bg_color(5, 'l') end + def on_106(s) set_bg_color(6, 'l') end + def on_107(s) set_bg_color(7, 'l') end + def on_109(s) set_bg_color(9, 'l') end attr_accessor :offset, :n_open_tags, :fg_color, :bg_color, :style_mask - STATE_PARAMS = [:offset, :n_open_tags, :fg_color, :bg_color, :style_mask] + STATE_PARAMS = [:offset, :n_open_tags, :fg_color, :bg_color, :style_mask].freeze def convert(raw, new_state) reset_state @@ -126,7 +170,7 @@ module Ci # We are only interested in color and text style changes - triggered by # sequences starting with '\e[' and ending with 'm'. Any other control # sequence gets stripped (including stuff like "delete last line") - return unless indicator == '[' and terminator == 'm' + return unless indicator == '[' && terminator == 'm' close_open_tags() diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index c4bdef781f7..746e76a1b1f 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -18,24 +18,31 @@ module Ci if current_runner.is_runner_queue_value_latest?(params[:last_update]) header 'X-GitLab-Last-Update', params[:last_update] + Gitlab::Metrics.add_event(:build_not_found_cached) return build_not_found! end new_update = current_runner.ensure_runner_queue_value - build = Ci::RegisterBuildService.new.execute(current_runner) + result = Ci::RegisterJobService.new(current_runner).execute - if build - Gitlab::Metrics.add_event(:build_found, - project: build.project.path_with_namespace) + if result.valid? + if result.build + Gitlab::Metrics.add_event(:build_found, + project: result.build.project.path_with_namespace) - present build, with: Entities::BuildDetails - else - Gitlab::Metrics.add_event(:build_not_found) + present result.build, with: Entities::BuildDetails + else + Gitlab::Metrics.add_event(:build_not_found) - header 'X-GitLab-Last-Update', new_update + header 'X-GitLab-Last-Update', new_update - build_not_found! + build_not_found! + end + else + # We received build that is invalid due to concurrency conflict + Gitlab::Metrics.add_event(:build_invalid) + conflict! end end @@ -160,7 +167,10 @@ module Ci build.artifacts_file = artifacts build.artifacts_metadata = metadata - build.artifacts_expire_in = params['expire_in'] + build.artifacts_expire_in = + params['expire_in'] || + Gitlab::CurrentSettings.current_application_settings + .default_artifacts_expire_in if build.save present(build, with: Entities::BuildDetails) @@ -207,6 +217,7 @@ module Ci build = Ci::Build.find_by_id(params[:id]) authenticate_build!(build) + status(200) build.erase_artifacts! end end diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb index 5ff25a3a9b2..996990b464f 100644 --- a/lib/ci/api/helpers.rb +++ b/lib/ci/api/helpers.rb @@ -1,7 +1,7 @@ module Ci module API module Helpers - BUILD_TOKEN_HEADER = "HTTP_BUILD_TOKEN" + BUILD_TOKEN_HEADER = "HTTP_BUILD_TOKEN".freeze BUILD_TOKEN_PARAM = :token UPDATE_RUNNER_EVERY = 10 * 60 @@ -60,7 +60,7 @@ module Ci end def build_not_found! - if headers['User-Agent'].to_s.match(/gitlab-ci-multi-runner \d+\.\d+\.\d+(~beta\.\d+\.g[0-9a-f]+)? /) + if headers['User-Agent'].to_s =~ /gitlab-ci-multi-runner \d+\.\d+\.\d+(~beta\.\d+\.g[0-9a-f]+)? / no_content! else not_found! @@ -73,7 +73,7 @@ module Ci def get_runner_version_from_params return unless params["info"].present? - attributes_for_keys(["name", "version", "revision", "platform", "architecture"], params["info"]) + attributes_for_keys(%w(name version revision platform architecture), params["info"]) end def max_artifacts_size diff --git a/lib/ci/api/runners.rb b/lib/ci/api/runners.rb index bcc82969eb3..45aa2adccf5 100644 --- a/lib/ci/api/runners.rb +++ b/lib/ci/api/runners.rb @@ -1,44 +1,38 @@ module Ci module API - # Runners API class Runners < Grape::API resource :runners do - # Delete runner - # Parameters: - # token (required) - The unique token of runner - # - # Example Request: - # GET /runners/delete + desc 'Delete a runner' + params do + requires :token, type: String, desc: 'The unique token of the runner' + end delete "delete" do - required_attributes! [:token] authenticate_runner! + + status(200) Ci::Runner.find_by_token(params[:token]).destroy end - # Register a new runner - # - # Note: This is an "internal" API called when setting up - # runners, so it is authenticated differently. - # - # Parameters: - # token (required) - The unique token of runner - # - # Example Request: - # POST /runners/register + desc 'Register a new runner' do + success Entities::Runner + end + params do + requires :token, type: String, desc: 'The unique token of the runner' + optional :description, type: String, desc: 'The description of the runner' + optional :tag_list, type: Array[String], desc: 'A list of tags the runner should run for' + optional :run_untagged, type: Boolean, desc: 'Flag if the runner should execute untagged jobs' + optional :locked, type: Boolean, desc: 'Lock this runner for this specific project' + end post "register" do - required_attributes! [:token] - - attributes = attributes_for_keys( - [:description, :tag_list, :run_untagged, :locked] - ) + runner_params = declared(params, include_missing: false).except(:token) runner = if runner_registration_token_valid? # Create shared runner. Requires admin access - Ci::Runner.create(attributes.merge(is_shared: true)) + Ci::Runner.create(runner_params.merge(is_shared: true)) elsif project = Project.find_by(runners_token: params[:token]) # Create a specific runner for project. - project.runners.create(attributes) + project.runners.create(runner_params) end return forbidden! unless runner diff --git a/lib/ci/api/triggers.rb b/lib/ci/api/triggers.rb index 63b42113513..6e622601680 100644 --- a/lib/ci/api/triggers.rb +++ b/lib/ci/api/triggers.rb @@ -1,41 +1,30 @@ module Ci module API - # Build Trigger API class Triggers < Grape::API resource :projects do - # Trigger a GitLab CI project build - # - # Parameters: - # id (required) - The ID of a CI project - # ref (required) - The name of project's branch or tag - # token (required) - The uniq token of trigger - # Example Request: - # POST /projects/:id/ref/:ref/trigger + desc 'Trigger a GitLab CI project build' do + success Entities::TriggerRequest + end + params do + requires :id, type: Integer, desc: 'The ID of a CI project' + requires :ref, type: String, desc: "The name of project's branch or tag" + requires :token, type: String, desc: 'The unique token of the trigger' + optional :variables, type: Hash, desc: 'Optional build variables' + end post ":id/refs/:ref/trigger" do - required_attributes! [:token] - - project = Project.find_by(ci_id: params[:id].to_i) - trigger = Ci::Trigger.find_by_token(params[:token].to_s) + project = Project.find_by(ci_id: params[:id]) + trigger = Ci::Trigger.find_by_token(params[:token]) not_found! unless project && trigger unauthorized! unless trigger.project == project - # validate variables - variables = params[:variables] - if variables - unless variables.is_a?(Hash) - render_api_error!('variables needs to be a hash', 400) - end - - unless variables.all? { |key, value| key.is_a?(String) && value.is_a?(String) } - render_api_error!('variables needs to be a map of key-valued strings', 400) - end - - # convert variables from Mash to Hash - variables = variables.to_h + # Validate variables + variables = params[:variables].to_h + unless variables.all? { |key, value| key.is_a?(String) && value.is_a?(String) } + render_api_error!('variables needs to be a map of key-valued strings', 400) end # create request and trigger builds - trigger_request = Ci::CreateTriggerRequestService.new.execute(project, trigger, params[:ref].to_s, variables) + trigger_request = Ci::CreateTriggerRequestService.new.execute(project, trigger, params[:ref], variables) if trigger_request present trigger_request, with: Entities::TriggerRequest else diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 7463bd719d5..15a461a16dd 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -1,6 +1,6 @@ module Ci class GitlabCiYamlProcessor - class ValidationError < StandardError; end + ValidationError = Class.new(StandardError) include Gitlab::Ci::Config::Entry::LegacyValidationHelpers @@ -58,9 +58,10 @@ module Ci commands: job[:commands], tag_list: job[:tags] || [], name: job[:name].to_s, - allow_failure: job[:allow_failure] || false, + allow_failure: job[:ignore], when: job[:when] || 'on_success', environment: job[:environment_name], + coverage_regex: job[:coverage], yaml_variables: yaml_variables(name), options: { image: job[:image], diff --git a/lib/constraints/project_url_constrainer.rb b/lib/constraints/project_url_constrainer.rb index 730b05bed97..a10b4657d7d 100644 --- a/lib/constraints/project_url_constrainer.rb +++ b/lib/constraints/project_url_constrainer.rb @@ -8,6 +8,6 @@ class ProjectUrlConstrainer return false end - Project.find_with_namespace(full_path).present? + Project.find_by_full_path(full_path).present? end end diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb index 2edddb84fc3..7f5f6d9ddb6 100644 --- a/lib/container_registry/client.rb +++ b/lib/container_registry/client.rb @@ -5,7 +5,7 @@ module ContainerRegistry class Client attr_accessor :uri - MANIFEST_VERSION = 'application/vnd.docker.distribution.manifest.v2+json' + MANIFEST_VERSION = 'application/vnd.docker.distribution.manifest.v2+json'.freeze # Taken from: FaradayMiddleware::FollowRedirects REDIRECT_CODES = Set.new [301, 302, 303, 307] diff --git a/lib/extracts_path.rb b/lib/extracts_path.rb index 82551f1f222..dd864eea3fa 100644 --- a/lib/extracts_path.rb +++ b/lib/extracts_path.rb @@ -2,7 +2,7 @@ # file path string when combined in a request parameter module ExtractsPath # Raised when given an invalid file path - class InvalidPathError < StandardError; end + InvalidPathError = Class.new(StandardError) # Given a string containing both a Git tree-ish, such as a branch or tag, and # a filesystem path joined by forward slashes, attempts to separate the two. @@ -42,7 +42,7 @@ module ExtractsPath return pair unless @project - if id.match(/^([[:alnum:]]{40})(.+)/) + if id =~ /^(\h{40})(.+)/ # If the ref appears to be a SHA, we're done, just split the string pair = $~.captures else diff --git a/lib/file_size_validator.rb b/lib/file_size_validator.rb index 440dd44ece7..eb19ab45ac3 100644 --- a/lib/file_size_validator.rb +++ b/lib/file_size_validator.rb @@ -32,9 +32,9 @@ class FileSizeValidator < ActiveModel::EachValidator end def validate_each(record, attribute, value) - raise(ArgumentError, "A CarrierWave::Uploader::Base object was expected") unless value.kind_of? CarrierWave::Uploader::Base + raise(ArgumentError, "A CarrierWave::Uploader::Base object was expected") unless value.is_a? CarrierWave::Uploader::Base - value = (options[:tokenizer] || DEFAULT_TOKENIZER).call(value) if value.kind_of?(String) + value = (options[:tokenizer] || DEFAULT_TOKENIZER).call(value) if value.is_a?(String) CHECKS.each do |key, validity_check| next unless check_value = options[key] diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb index 9b484a2ecfd..8c28009b9c6 100644 --- a/lib/gitlab/access.rb +++ b/lib/gitlab/access.rb @@ -5,7 +5,7 @@ # module Gitlab module Access - class AccessDeniedError < StandardError; end + AccessDeniedError = Class.new(StandardError) NO_ACCESS = 0 GUEST = 10 @@ -21,9 +21,7 @@ module Gitlab PROTECTION_DEV_CAN_MERGE = 3 class << self - def values - options.values - end + delegate :values, to: :options def all_values options_with_owner.values diff --git a/lib/gitlab/allowable.rb b/lib/gitlab/allowable.rb index f48abcc86d5..e4f7cad2b79 100644 --- a/lib/gitlab/allowable.rb +++ b/lib/gitlab/allowable.rb @@ -1,6 +1,6 @@ module Gitlab module Allowable - def can?(user, action, subject) + def can?(user, action, subject = :global) Ability.allowed?(user, action, subject) end end diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb index 0618107e2c3..d575367d81a 100644 --- a/lib/gitlab/asciidoc.rb +++ b/lib/gitlab/asciidoc.rb @@ -36,6 +36,9 @@ module Gitlab html = Banzai.post_process(html, context) + filter = Banzai::Filter::SanitizationFilter.new(html) + html = filter.call.to_s + html.html_safe end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 8dda65c71ef..eee5601b0ed 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -1,41 +1,55 @@ module Gitlab module Auth - class MissingPersonalTokenError < StandardError; end + MissingPersonalTokenError = Class.new(StandardError) - SCOPES = [:api, :read_user] - DEFAULT_SCOPES = [:api] - OPTIONAL_SCOPES = SCOPES - DEFAULT_SCOPES + # Scopes used for GitLab API access + API_SCOPES = [:api, :read_user].freeze + + # Scopes used for OpenID Connect + OPENID_SCOPES = [:openid].freeze + + # Default scopes for OAuth applications that don't define their own + DEFAULT_SCOPES = [:api].freeze + + # Other available scopes + OPTIONAL_SCOPES = (API_SCOPES + OPENID_SCOPES - DEFAULT_SCOPES).freeze class << self def find_for_git_client(login, password, project:, ip:) raise "Must provide an IP for rate limiting" if ip.nil? + # `user_with_password_for_git` should be the last check + # because it's the most expensive, especially when LDAP + # is enabled. result = service_request_check(login, password, project) || build_access_token_check(login, password) || - user_with_password_for_git(login, password) || - oauth_access_token_check(login, password) || lfs_token_check(login, password) || - personal_access_token_check(login, password) || + oauth_access_token_check(login, password) || + user_with_password_for_git(login, password) || + personal_access_token_check(password) || Gitlab::Auth::Result.new rate_limit!(ip, success: result.success?, login: login) + Gitlab::Auth::UniqueIpsLimiter.limit_user!(result.actor) result end def find_with_user_password(login, password) - user = User.by_login(login) + Gitlab::Auth::UniqueIpsLimiter.limit_user! do + user = User.by_login(login) - # If no user is found, or it's an LDAP server, try LDAP. - # LDAP users are only authenticated via LDAP - if user.nil? || user.ldap_user? - # Second chance - try LDAP authentication - return nil unless Gitlab::LDAP::Config.enabled? + # If no user is found, or it's an LDAP server, try LDAP. + # LDAP users are only authenticated via LDAP + if user.nil? || user.ldap_user? + # Second chance - try LDAP authentication + return nil unless Gitlab::LDAP::Config.enabled? - Gitlab::LDAP::Authentication.login(login, password) - else - user if user.valid_password?(password) + Gitlab::LDAP::Authentication.login(login, password) + else + user if user.active? && user.valid_password?(password) + end end end @@ -99,14 +113,13 @@ module Gitlab end end - def personal_access_token_check(login, password) - if login && password - token = PersonalAccessToken.active.find_by_token(password) - validation = User.by_login(login) + def personal_access_token_check(password) + return unless password.present? - if valid_personal_access_token?(token, validation) - Gitlab::Auth::Result.new(validation, nil, :personal_token, full_authentication_abilities) - end + token = PersonalAccessTokensFinder.new(state: 'active').find_by(token: password) + + if token && valid_api_token?(token) + Gitlab::Auth::Result.new(token.user, nil, :personal_token, full_authentication_abilities) end end @@ -114,10 +127,6 @@ module Gitlab token && token.accessible? && valid_api_token?(token) end - def valid_personal_access_token?(token, user) - token && token.user == user && valid_api_token?(token) - end - def valid_api_token?(token) AccessTokenValidationService.new(token).include_any_scope?(['api']) end @@ -143,7 +152,9 @@ module Gitlab read_authentication_abilities end - Result.new(actor, nil, token_handler.type, authentication_abilities) if Devise.secure_compare(token_handler.token, password) + if Devise.secure_compare(token_handler.token, password) + Gitlab::Auth::Result.new(actor, nil, token_handler.type, authentication_abilities) + end end def build_access_token_check(login, password) diff --git a/lib/gitlab/auth/too_many_ips.rb b/lib/gitlab/auth/too_many_ips.rb new file mode 100644 index 00000000000..ed862791551 --- /dev/null +++ b/lib/gitlab/auth/too_many_ips.rb @@ -0,0 +1,17 @@ +module Gitlab + module Auth + class TooManyIps < StandardError + attr_reader :user_id, :ip, :unique_ips_count + + def initialize(user_id, ip, unique_ips_count) + @user_id = user_id + @ip = ip + @unique_ips_count = unique_ips_count + end + + def message + "User #{user_id} from IP: #{ip} tried logging from too many ips: #{unique_ips_count}" + end + end + end +end diff --git a/lib/gitlab/auth/unique_ips_limiter.rb b/lib/gitlab/auth/unique_ips_limiter.rb new file mode 100644 index 00000000000..bf2239ca150 --- /dev/null +++ b/lib/gitlab/auth/unique_ips_limiter.rb @@ -0,0 +1,43 @@ +module Gitlab + module Auth + class UniqueIpsLimiter + USER_UNIQUE_IPS_PREFIX = 'user_unique_ips'.freeze + + class << self + def limit_user_id!(user_id) + if config.unique_ips_limit_enabled + ip = RequestContext.client_ip + unique_ips = update_and_return_ips_count(user_id, ip) + + raise TooManyIps.new(user_id, ip, unique_ips) if unique_ips > config.unique_ips_limit_per_user + end + end + + def limit_user!(user = nil) + user ||= yield if block_given? + limit_user_id!(user.id) unless user.nil? + user + end + + def config + Gitlab::CurrentSettings.current_application_settings + end + + def update_and_return_ips_count(user_id, ip) + time = Time.now.utc.to_i + key = "#{USER_UNIQUE_IPS_PREFIX}:#{user_id}" + + Gitlab::Redis.with do |redis| + unique_ips_count = nil + redis.multi do |r| + r.zadd(key, time, ip) + r.zremrangebyscore(key, 0, time - config.unique_ips_limit_time_window) + unique_ips_count = r.zcard(key) + end + unique_ips_count.value + end + end + end + end + end +end diff --git a/lib/gitlab/award_emoji.rb b/lib/gitlab/award_emoji.rb deleted file mode 100644 index 39b43ab5489..00000000000 --- a/lib/gitlab/award_emoji.rb +++ /dev/null @@ -1,83 +0,0 @@ -module Gitlab - class AwardEmoji - CATEGORIES = { - objects: "Objects", - travel: "Travel", - symbols: "Symbols", - nature: "Nature", - people: "People", - activity: "Activity", - flags: "Flags", - food: "Food" - }.with_indifferent_access - - def self.normalize_emoji_name(name) - aliases[name] || name - end - - def self.emoji_by_category - unless @emoji_by_category - @emoji_by_category = Hash.new { |h, key| h[key] = [] } - - emojis.each do |emoji_name, data| - data["name"] = emoji_name - - # Skip Fitzpatrick(tone) modifiers - next if data["category"] == "modifier" - - category = data["category"] - - @emoji_by_category[category] << data - end - - @emoji_by_category = @emoji_by_category.sort.to_h - end - - @emoji_by_category - end - - def self.emojis - @emojis ||= - begin - json_path = File.join(Rails.root, 'fixtures', 'emojis', 'index.json' ) - JSON.parse(File.read(json_path)) - end - end - - def self.aliases - @aliases ||= - begin - json_path = File.join(Rails.root, 'fixtures', 'emojis', 'aliases.json') - JSON.parse(File.read(json_path)) - end - end - - # Returns an Array of Emoji names and their asset URLs. - def self.urls - @urls ||= begin - path = File.join(Rails.root, 'fixtures', 'emojis', 'digests.json') - # Construct the full asset path ourselves because - # ActionView::Helpers::AssetUrlHelper.asset_url is slow for hundreds - # of entries since it has to do a lot of extra work (e.g. regexps). - prefix = Gitlab::Application.config.assets.prefix - digest = Gitlab::Application.config.assets.digest - base = - if defined?(Gitlab::Application.config.relative_url_root) && Gitlab::Application.config.relative_url_root - Gitlab::Application.config.relative_url_root - else - '' - end - - JSON.parse(File.read(path)).map do |hash| - if digest - fname = "#{hash['unicode']}-#{hash['digest']}" - else - fname = hash['unicode'] - end - - { name: hash['name'], path: File.join(base, prefix, "#{fname}.png") } - end - end - end - end -end diff --git a/lib/gitlab/badge/build/template.rb b/lib/gitlab/badge/build/template.rb index 2b95ddfcb53..bc0e0cd441d 100644 --- a/lib/gitlab/badge/build/template.rb +++ b/lib/gitlab/badge/build/template.rb @@ -15,7 +15,7 @@ module Gitlab canceled: '#9f9f9f', skipped: '#9f9f9f', unknown: '#9f9f9f' - } + }.freeze def initialize(badge) @entity = badge.entity diff --git a/lib/gitlab/badge/coverage/template.rb b/lib/gitlab/badge/coverage/template.rb index 06e0d084e9f..fcecb1d9665 100644 --- a/lib/gitlab/badge/coverage/template.rb +++ b/lib/gitlab/badge/coverage/template.rb @@ -13,7 +13,7 @@ module Gitlab medium: '#dfb317', low: '#e05d44', unknown: '#9f9f9f' - } + }.freeze def initialize(badge) @entity = badge.entity diff --git a/lib/gitlab/badge/metadata.rb b/lib/gitlab/badge/metadata.rb index 548f85b78bb..4a049ef758d 100644 --- a/lib/gitlab/badge/metadata.rb +++ b/lib/gitlab/badge/metadata.rb @@ -20,6 +20,10 @@ module Gitlab "[](#{link_url})" end + def to_asciidoc + "image:#{image_url}[link=\"#{link_url}\",title=\"#{title}\"]" + end + def title raise NotImplementedError end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 44323b47dca..f4efa20374a 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -130,8 +130,13 @@ module Gitlab end def create_labels - LABELS.each do |label| - @labels[label[:title]] = project.labels.create!(label) + LABELS.each do |label_params| + label = ::Labels::CreateService.new(label_params).execute(project: project) + if label.valid? + @labels[label_params[:title]] = label + else + raise "Failed to create label \"#{label_params[:title]}\" for project \"#{project.name_with_namespace}\"" + end end end diff --git a/lib/gitlab/changes_list.rb b/lib/gitlab/changes_list.rb index 95308aca95f..5b32fca00a4 100644 --- a/lib/gitlab/changes_list.rb +++ b/lib/gitlab/changes_list.rb @@ -5,7 +5,7 @@ module Gitlab attr_reader :raw_changes def initialize(changes) - @raw_changes = changes.kind_of?(String) ? changes.lines : changes + @raw_changes = changes.is_a?(String) ? changes.lines : changes end def each(&block) diff --git a/lib/gitlab/chat_commands/base_command.rb b/lib/gitlab/chat_commands/base_command.rb index 4fe53ce93a9..25da8474e95 100644 --- a/lib/gitlab/chat_commands/base_command.rb +++ b/lib/gitlab/chat_commands/base_command.rb @@ -42,10 +42,6 @@ module Gitlab def find_by_iid(iid) collection.find_by(iid: iid) end - - def presenter - Gitlab::ChatCommands::Presenter.new - end end end end diff --git a/lib/gitlab/chat_commands/command.rb b/lib/gitlab/chat_commands/command.rb index 145086755e4..f34ed0f4cf2 100644 --- a/lib/gitlab/chat_commands/command.rb +++ b/lib/gitlab/chat_commands/command.rb @@ -3,7 +3,7 @@ module Gitlab class Command < BaseCommand COMMANDS = [ Gitlab::ChatCommands::IssueShow, - Gitlab::ChatCommands::IssueCreate, + Gitlab::ChatCommands::IssueNew, Gitlab::ChatCommands::IssueSearch, Gitlab::ChatCommands::Deploy, ].freeze @@ -13,51 +13,32 @@ module Gitlab if command if command.allowed?(project, current_user) - present command.new(project, current_user, params).execute(match) + command.new(project, current_user, params).execute(match) else - access_denied + Gitlab::ChatCommands::Presenters::Access.new.access_denied end else - help(help_messages) + Gitlab::ChatCommands::Help.new(project, current_user, params).execute(available_commands, params[:text]) end end def match_command match = nil - service = available_commands.find do |klass| - match = klass.match(command) - end + service = + available_commands.find do |klass| + match = klass.match(params[:text]) + end [service, match] end private - def help_messages - available_commands.map(&:help_message) - end - def available_commands COMMANDS.select do |klass| klass.available?(project) end end - - def command - params[:text] - end - - def help(messages) - presenter.help(messages, params[:command]) - end - - def access_denied - presenter.access_denied - end - - def present(resource) - presenter.present(resource) - end end end end diff --git a/lib/gitlab/chat_commands/deploy.rb b/lib/gitlab/chat_commands/deploy.rb index 7127d2f6d04..458d90f84e8 100644 --- a/lib/gitlab/chat_commands/deploy.rb +++ b/lib/gitlab/chat_commands/deploy.rb @@ -1,8 +1,6 @@ module Gitlab module ChatCommands class Deploy < BaseCommand - include Gitlab::Routing.url_helpers - def self.match(text) /\Adeploy\s+(?<from>\S+.*)\s+to+\s+(?<to>\S+.*)\z/.match(text) end @@ -24,35 +22,29 @@ module Gitlab to = match[:to] actions = find_actions(from, to) - return unless actions.present? - if actions.one? - play!(from, to, actions.first) + if actions.none? + Gitlab::ChatCommands::Presenters::Deploy.new(nil).no_actions + elsif actions.one? + action = play!(from, to, actions.first) + Gitlab::ChatCommands::Presenters::Deploy.new(action).present(from, to) else - Result.new(:error, 'Too many actions defined') + Gitlab::ChatCommands::Presenters::Deploy.new(actions).too_many_actions end end private def play!(from, to, action) - new_action = action.play(current_user) - - Result.new(:success, "Deployment from #{from} to #{to} started. Follow the progress: #{url(new_action)}.") + action.play(current_user) end def find_actions(from, to) environment = project.environments.find_by(name: from) - return unless environment + return [] unless environment environment.actions_for(to).select(&:starts_environment?) end - - def url(subject) - polymorphic_url( - [subject.project.namespace.becomes(Namespace), subject.project, subject] - ) - end end end end diff --git a/lib/gitlab/chat_commands/help.rb b/lib/gitlab/chat_commands/help.rb new file mode 100644 index 00000000000..6c0e4d304a4 --- /dev/null +++ b/lib/gitlab/chat_commands/help.rb @@ -0,0 +1,28 @@ +module Gitlab + module ChatCommands + class Help < BaseCommand + # This class has to be used last, as it always matches. It has to match + # because other commands were not triggered and we want to show the help + # command + def self.match(_text) + true + end + + def self.help_message + 'help' + end + + def self.allowed?(_project, _user) + true + end + + def execute(commands, text) + Gitlab::ChatCommands::Presenters::Help.new(commands).present(trigger, text) + end + + def trigger + params[:command] + end + end + end +end diff --git a/lib/gitlab/chat_commands/issue_create.rb b/lib/gitlab/chat_commands/issue_new.rb index cefb6775db8..016054ecd46 100644 --- a/lib/gitlab/chat_commands/issue_create.rb +++ b/lib/gitlab/chat_commands/issue_new.rb @@ -1,8 +1,8 @@ module Gitlab module ChatCommands - class IssueCreate < IssueCommand + class IssueNew < IssueCommand def self.match(text) - # we can not match \n with the dot by passing the m modifier as than + # we can not match \n with the dot by passing the m modifier as than # the title and description are not seperated /\Aissue\s+(new|create)\s+(?<title>[^\n]*)\n*(?<description>(.|\n)*)/.match(text) end @@ -19,8 +19,24 @@ module Gitlab title = match[:title] description = match[:description].to_s.rstrip + issue = create_issue(title: title, description: description) + + if issue.persisted? + presenter(issue).present + else + presenter(issue).display_errors + end + end + + private + + def create_issue(title:, description:) Issues::CreateService.new(project, current_user, title: title, description: description).execute end + + def presenter(issue) + Gitlab::ChatCommands::Presenters::IssueNew.new(issue) + end end end end diff --git a/lib/gitlab/chat_commands/issue_search.rb b/lib/gitlab/chat_commands/issue_search.rb index 51bf80c800b..3491b53093e 100644 --- a/lib/gitlab/chat_commands/issue_search.rb +++ b/lib/gitlab/chat_commands/issue_search.rb @@ -10,7 +10,13 @@ module Gitlab end def execute(match) - collection.search(match[:query]).limit(QUERY_LIMIT) + issues = collection.search(match[:query]).limit(QUERY_LIMIT) + + if issues.present? + Presenters::IssueSearch.new(issues).present + else + Presenters::Access.new(issues).not_found + end end end end diff --git a/lib/gitlab/chat_commands/issue_show.rb b/lib/gitlab/chat_commands/issue_show.rb index 2a45d49cf6b..d6013f4d10c 100644 --- a/lib/gitlab/chat_commands/issue_show.rb +++ b/lib/gitlab/chat_commands/issue_show.rb @@ -10,7 +10,13 @@ module Gitlab end def execute(match) - find_by_iid(match[:iid]) + issue = find_by_iid(match[:iid]) + + if issue + Gitlab::ChatCommands::Presenters::IssueShow.new(issue).present + else + Gitlab::ChatCommands::Presenters::Access.new.not_found + end end end end diff --git a/lib/gitlab/chat_commands/presenter.rb b/lib/gitlab/chat_commands/presenter.rb deleted file mode 100644 index 8930a21f406..00000000000 --- a/lib/gitlab/chat_commands/presenter.rb +++ /dev/null @@ -1,131 +0,0 @@ -module Gitlab - module ChatCommands - class Presenter - include Gitlab::Routing - - def authorize_chat_name(url) - message = if url - ":wave: Hi there! Before I do anything for you, please [connect your GitLab account](#{url})." - else - ":sweat_smile: Couldn't identify you, nor can I autorize you!" - end - - ephemeral_response(message) - end - - def help(commands, trigger) - if commands.none? - ephemeral_response("No commands configured") - else - commands.map! { |command| "#{trigger} #{command}" } - message = header_with_list("Available commands", commands) - - ephemeral_response(message) - end - end - - def present(subject) - return not_found unless subject - - if subject.is_a?(Gitlab::ChatCommands::Result) - show_result(subject) - elsif subject.respond_to?(:count) - if subject.none? - not_found - elsif subject.one? - single_resource(subject.first) - else - multiple_resources(subject) - end - else - single_resource(subject) - end - end - - def access_denied - ephemeral_response("Whoops! That action is not allowed. This incident will be [reported](https://xkcd.com/838/).") - end - - private - - def show_result(result) - case result.type - when :success - in_channel_response(result.message) - else - ephemeral_response(result.message) - end - end - - def not_found - ephemeral_response("404 not found! GitLab couldn't find what you were looking for! :boom:") - end - - def single_resource(resource) - return error(resource) if resource.errors.any? || !resource.persisted? - - message = "#{title(resource)}:" - message << "\n\n#{resource.description}" if resource.try(:description) - - in_channel_response(message) - end - - def multiple_resources(resources) - titles = resources.map { |resource| title(resource) } - - message = header_with_list("Multiple results were found:", titles) - - ephemeral_response(message) - end - - def error(resource) - message = header_with_list("The action was not successful, because:", resource.errors.messages) - - ephemeral_response(message) - end - - def title(resource) - reference = resource.try(:to_reference) || resource.try(:id) - title = resource.try(:title) || resource.try(:name) - - "[#{reference} #{title}](#{url(resource)})" - end - - def header_with_list(header, items) - message = [header] - - items.each do |item| - message << "- #{item}" - end - - message.join("\n") - end - - def url(resource) - url_for( - [ - resource.project.namespace.becomes(Namespace), - resource.project, - resource - ] - ) - end - - def ephemeral_response(message) - { - response_type: :ephemeral, - text: message, - status: 200 - } - end - - def in_channel_response(message) - { - response_type: :in_channel, - text: message, - status: 200 - } - end - end - end -end diff --git a/lib/gitlab/chat_commands/presenters/access.rb b/lib/gitlab/chat_commands/presenters/access.rb new file mode 100644 index 00000000000..92f4fa17f78 --- /dev/null +++ b/lib/gitlab/chat_commands/presenters/access.rb @@ -0,0 +1,40 @@ +module Gitlab + module ChatCommands + module Presenters + class Access < Presenters::Base + def access_denied + ephemeral_response(text: "Whoops! This action is not allowed. This incident will be [reported](https://xkcd.com/838/).") + end + + def not_found + ephemeral_response(text: "404 not found! GitLab couldn't find what you were looking for! :boom:") + end + + def authorize + message = + if @resource + ":wave: Hi there! Before I do anything for you, please [connect your GitLab account](#{@resource})." + else + ":sweat_smile: Couldn't identify you, nor can I autorize you!" + end + + ephemeral_response(text: message) + end + + def unknown_command(commands) + ephemeral_response(text: help_message(trigger)) + end + + private + + def help_message(trigger) + header_with_list("Command not found, these are the commands you can use", full_commands(trigger)) + end + + def full_commands(trigger) + @resource.map { |command| "#{trigger} #{command.help_message}" } + end + end + end + end +end diff --git a/lib/gitlab/chat_commands/presenters/base.rb b/lib/gitlab/chat_commands/presenters/base.rb new file mode 100644 index 00000000000..2700a5a2ad5 --- /dev/null +++ b/lib/gitlab/chat_commands/presenters/base.rb @@ -0,0 +1,77 @@ +module Gitlab + module ChatCommands + module Presenters + class Base + include Gitlab::Routing.url_helpers + + def initialize(resource = nil) + @resource = resource + end + + def display_errors + message = header_with_list("The action was not successful, because:", @resource.errors.full_messages) + + ephemeral_response(text: message) + end + + private + + def header_with_list(header, items) + message = [header] + + items.each do |item| + message << "- #{item}" + end + + message.join("\n") + end + + def ephemeral_response(message) + response = { + response_type: :ephemeral, + status: 200 + }.merge(message) + + format_response(response) + end + + def in_channel_response(message) + response = { + response_type: :in_channel, + status: 200 + }.merge(message) + + format_response(response) + end + + def format_response(response) + response[:text] = format(response[:text]) if response.has_key?(:text) + + if response.has_key?(:attachments) + response[:attachments].each do |attachment| + attachment[:pretext] = format(attachment[:pretext]) if attachment[:pretext] + attachment[:text] = format(attachment[:text]) if attachment[:text] + end + end + + response + end + + # Convert Markdown to slacks format + def format(string) + Slack::Notifier::LinkFormatter.format(string) + end + + def resource_url + url_for( + [ + @resource.project.namespace.becomes(Namespace), + @resource.project, + @resource + ] + ) + end + end + end + end +end diff --git a/lib/gitlab/chat_commands/presenters/deploy.rb b/lib/gitlab/chat_commands/presenters/deploy.rb new file mode 100644 index 00000000000..863d0bf99ca --- /dev/null +++ b/lib/gitlab/chat_commands/presenters/deploy.rb @@ -0,0 +1,21 @@ +module Gitlab + module ChatCommands + module Presenters + class Deploy < Presenters::Base + def present(from, to) + message = "Deployment started from #{from} to #{to}. [Follow its progress](#{resource_url})." + + in_channel_response(text: message) + end + + def no_actions + ephemeral_response(text: "No action found to be executed") + end + + def too_many_actions + ephemeral_response(text: "Too many actions defined") + end + end + end + end +end diff --git a/lib/gitlab/chat_commands/presenters/help.rb b/lib/gitlab/chat_commands/presenters/help.rb new file mode 100644 index 00000000000..cd47b7f4c6a --- /dev/null +++ b/lib/gitlab/chat_commands/presenters/help.rb @@ -0,0 +1,27 @@ +module Gitlab + module ChatCommands + module Presenters + class Help < Presenters::Base + def present(trigger, text) + ephemeral_response(text: help_message(trigger, text)) + end + + private + + def help_message(trigger, text) + return "No commands available :thinking_face:" unless @resource.present? + + if text.start_with?('help') + header_with_list("Available commands", full_commands(trigger)) + else + header_with_list("Unknown command, these commands are available", full_commands(trigger)) + end + end + + def full_commands(trigger) + @resource.map { |command| "#{trigger} #{command.help_message}" } + end + end + end + end +end diff --git a/lib/gitlab/chat_commands/presenters/issue_base.rb b/lib/gitlab/chat_commands/presenters/issue_base.rb new file mode 100644 index 00000000000..054f7f4be0c --- /dev/null +++ b/lib/gitlab/chat_commands/presenters/issue_base.rb @@ -0,0 +1,43 @@ +module Gitlab + module ChatCommands + module Presenters + module IssueBase + def color(issuable) + issuable.open? ? '#38ae67' : '#d22852' + end + + def status_text(issuable) + issuable.open? ? 'Open' : 'Closed' + end + + def project + @resource.project + end + + def author + @resource.author + end + + def fields + [ + { + title: "Assignee", + value: @resource.assignee ? @resource.assignee.name : "_None_", + short: true + }, + { + title: "Milestone", + value: @resource.milestone ? @resource.milestone.title : "_None_", + short: true + }, + { + title: "Labels", + value: @resource.labels.any? ? @resource.label_names.join(', ') : "_None_", + short: true + } + ] + end + end + end + end +end diff --git a/lib/gitlab/chat_commands/presenters/issue_new.rb b/lib/gitlab/chat_commands/presenters/issue_new.rb new file mode 100644 index 00000000000..3674ba25641 --- /dev/null +++ b/lib/gitlab/chat_commands/presenters/issue_new.rb @@ -0,0 +1,50 @@ +module Gitlab + module ChatCommands + module Presenters + class IssueNew < Presenters::Base + include Presenters::IssueBase + + def present + in_channel_response(new_issue) + end + + private + + def new_issue + { + attachments: [ + { + title: "#{@resource.title} · #{@resource.to_reference}", + title_link: resource_url, + author_name: author.name, + author_icon: author.avatar_url, + fallback: "New issue #{@resource.to_reference}: #{@resource.title}", + pretext: pretext, + color: color(@resource), + fields: fields, + mrkdwn_in: [ + :title, + :pretext, + :text, + :fields + ] + } + ] + } + end + + def pretext + "I created an issue on #{author_profile_link}'s behalf: **#{@resource.to_reference}** in #{project_link}" + end + + def project_link + "[#{project.name_with_namespace}](#{project.web_url})" + end + + def author_profile_link + "[#{author.to_reference}](#{url_for(author)})" + end + end + end + end +end diff --git a/lib/gitlab/chat_commands/presenters/issue_search.rb b/lib/gitlab/chat_commands/presenters/issue_search.rb new file mode 100644 index 00000000000..73788cf9662 --- /dev/null +++ b/lib/gitlab/chat_commands/presenters/issue_search.rb @@ -0,0 +1,47 @@ +module Gitlab + module ChatCommands + module Presenters + class IssueSearch < Presenters::Base + include Presenters::IssueBase + + def present + text = if @resource.count >= 5 + "Here are the first 5 issues I found:" + elsif @resource.one? + "Here is the only issue I found:" + else + "Here are the #{@resource.count} issues I found:" + end + + ephemeral_response(text: text, attachments: attachments) + end + + private + + def attachments + @resource.map do |issue| + url = "[#{issue.to_reference}](#{url_for([namespace, project, issue])})" + + { + color: color(issue), + fallback: "#{issue.to_reference} #{issue.title}", + text: "#{url} · #{issue.title} (#{status_text(issue)})", + + mrkdwn_in: [ + :text + ] + } + end + end + + def project + @project ||= @resource.first.project + end + + def namespace + @namespace ||= project.namespace.becomes(Namespace) + end + end + end + end +end diff --git a/lib/gitlab/chat_commands/presenters/issue_show.rb b/lib/gitlab/chat_commands/presenters/issue_show.rb new file mode 100644 index 00000000000..bd784ad241e --- /dev/null +++ b/lib/gitlab/chat_commands/presenters/issue_show.rb @@ -0,0 +1,61 @@ +module Gitlab + module ChatCommands + module Presenters + class IssueShow < Presenters::Base + include Presenters::IssueBase + + def present + if @resource.confidential? + ephemeral_response(show_issue) + else + in_channel_response(show_issue) + end + end + + private + + def show_issue + { + attachments: [ + { + title: "#{@resource.title} · #{@resource.to_reference}", + title_link: resource_url, + author_name: author.name, + author_icon: author.avatar_url, + fallback: "Issue #{@resource.to_reference}: #{@resource.title}", + pretext: pretext, + text: text, + color: color(@resource), + fields: fields, + mrkdwn_in: [ + :pretext, + :text, + :fields + ] + } + ] + } + end + + def text + message = "**#{status_text(@resource)}**" + + if @resource.upvotes.zero? && @resource.downvotes.zero? && @resource.user_notes_count.zero? + return message + end + + message << " · " + message << ":+1: #{@resource.upvotes} " unless @resource.upvotes.zero? + message << ":-1: #{@resource.downvotes} " unless @resource.downvotes.zero? + message << ":speech_balloon: #{@resource.user_notes_count}" unless @resource.user_notes_count.zero? + + message + end + + def pretext + "Issue *#{@resource.to_reference}* from #{project.name_with_namespace}" + end + end + end + end +end diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb index 273118135a9..c85f79127bc 100644 --- a/lib/gitlab/checks/change_access.rb +++ b/lib/gitlab/checks/change_access.rb @@ -1,16 +1,20 @@ module Gitlab module Checks class ChangeAccess - attr_reader :user_access, :project, :skip_authorization + # protocol is currently used only in EE + attr_reader :user_access, :project, :skip_authorization, :protocol def initialize( - change, user_access:, project:, env: {}, skip_authorization: false) + change, user_access:, project:, env: {}, skip_authorization: false, + protocol: + ) @oldrev, @newrev, @ref = change.values_at(:oldrev, :newrev, :ref) @branch_name = Gitlab::Git.branch_name(@ref) @user_access = user_access @project = project @env = env @skip_authorization = skip_authorization + @protocol = protocol end def exec diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index cd2e83b4c27..a375ccbece0 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -6,7 +6,7 @@ module Gitlab module Build module Artifacts class Metadata - class ParserError < StandardError; end + ParserError = Class.new(StandardError) VERSION_PATTERN = /^[\w\s]+(\d+\.\d+\.\d+)/ INVALID_PATH_PATTERN = %r{(^\.?\.?/)|(/\.?\.?/)} diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb index 7f4c750b6fd..6f799c2f031 100644 --- a/lib/gitlab/ci/build/artifacts/metadata/entry.rb +++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb @@ -27,6 +27,8 @@ module Gitlab end end + delegate :empty?, to: :children + def directory? blank_node? || @path.end_with?('/') end @@ -91,10 +93,6 @@ module Gitlab blank_node? || @entries.include?(@path) end - def empty? - children.empty? - end - def total_size descendant_pattern = %r{^#{Regexp.escape(@path)}} entries.sum do |path, entry| diff --git a/lib/gitlab/ci/build/image.rb b/lib/gitlab/ci/build/image.rb new file mode 100644 index 00000000000..c62aeb60fa9 --- /dev/null +++ b/lib/gitlab/ci/build/image.rb @@ -0,0 +1,33 @@ +module Gitlab + module Ci + module Build + class Image + attr_reader :name + + class << self + def from_image(job) + image = Gitlab::Ci::Build::Image.new(job.options[:image]) + return unless image.valid? + image + end + + def from_services(job) + services = job.options[:services].to_a.map do |service| + Gitlab::Ci::Build::Image.new(service) + end + + services.select(&:valid?).compact + end + end + + def initialize(image) + @name = image + end + + def valid? + @name.present? + end + end + end + end +end diff --git a/lib/gitlab/ci/build/step.rb b/lib/gitlab/ci/build/step.rb new file mode 100644 index 00000000000..ee034d9cc56 --- /dev/null +++ b/lib/gitlab/ci/build/step.rb @@ -0,0 +1,41 @@ +module Gitlab + module Ci + module Build + class Step + WHEN_ON_FAILURE = 'on_failure'.freeze + WHEN_ON_SUCCESS = 'on_success'.freeze + WHEN_ALWAYS = 'always'.freeze + + attr_reader :name + attr_accessor :script, :timeout, :when, :allow_failure + + class << self + def from_commands(job) + self.new(:script).tap do |step| + step.script = job.commands.split("\n") + step.timeout = job.timeout + step.when = WHEN_ON_SUCCESS + end + end + + def from_after_script(job) + after_script = job.options[:after_script] + return unless after_script + + self.new(:after_script).tap do |step| + step.script = after_script + step.timeout = job.timeout + step.when = WHEN_ALWAYS + step.allow_failure = true + end + end + end + + def initialize(name) + @name = name + @allow_failure = false + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/artifacts.rb b/lib/gitlab/ci/config/entry/artifacts.rb index b756b0d4555..8275aacee9b 100644 --- a/lib/gitlab/ci/config/entry/artifacts.rb +++ b/lib/gitlab/ci/config/entry/artifacts.rb @@ -9,7 +9,7 @@ module Gitlab include Validatable include Attributable - ALLOWED_KEYS = %i[name untracked paths when expire_in] + ALLOWED_KEYS = %i[name untracked paths when expire_in].freeze attributes ALLOWED_KEYS diff --git a/lib/gitlab/ci/config/entry/cache.rb b/lib/gitlab/ci/config/entry/cache.rb index 7653cab668b..f074df9c7a1 100644 --- a/lib/gitlab/ci/config/entry/cache.rb +++ b/lib/gitlab/ci/config/entry/cache.rb @@ -8,7 +8,7 @@ module Gitlab class Cache < Node include Configurable - ALLOWED_KEYS = %i[key untracked paths] + ALLOWED_KEYS = %i[key untracked paths].freeze validations do validates :config, allowed_keys: ALLOWED_KEYS @@ -22,6 +22,12 @@ module Gitlab entry :paths, Entry::Paths, description: 'Specify which paths should be cached across builds.' + + helpers :key + + def value + super.merge(key: key_value) + end end end end diff --git a/lib/gitlab/ci/config/entry/configurable.rb b/lib/gitlab/ci/config/entry/configurable.rb index 833ae4a0ff3..e05aca9881b 100644 --- a/lib/gitlab/ci/config/entry/configurable.rb +++ b/lib/gitlab/ci/config/entry/configurable.rb @@ -58,7 +58,7 @@ module Gitlab def helpers(*nodes) nodes.each do |symbol| define_method("#{symbol}_defined?") do - @entries[symbol].specified? if @entries[symbol] + @entries[symbol]&.specified? end define_method("#{symbol}_value") do diff --git a/lib/gitlab/ci/config/entry/coverage.rb b/lib/gitlab/ci/config/entry/coverage.rb new file mode 100644 index 00000000000..12a063059cb --- /dev/null +++ b/lib/gitlab/ci/config/entry/coverage.rb @@ -0,0 +1,22 @@ +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents Coverage settings. + # + class Coverage < Node + include Validatable + + validations do + validates :config, regexp: true + end + + def value + @config[1...-1] + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/environment.rb b/lib/gitlab/ci/config/entry/environment.rb index f7c530c7d9f..0c1f9eb7cbf 100644 --- a/lib/gitlab/ci/config/entry/environment.rb +++ b/lib/gitlab/ci/config/entry/environment.rb @@ -8,7 +8,7 @@ module Gitlab class Environment < Node include Validatable - ALLOWED_KEYS = %i[name url action on_stop] + ALLOWED_KEYS = %i[name url action on_stop].freeze validations do validate do @@ -21,12 +21,14 @@ module Gitlab validates :name, type: { with: String, - message: Gitlab::Regex.environment_name_regex_message } + message: Gitlab::Regex.environment_name_regex_message + } validates :name, format: { with: Gitlab::Regex.environment_name_regex, - message: Gitlab::Regex.environment_name_regex_message } + message: Gitlab::Regex.environment_name_regex_message + } with_options if: :hash? do validates :config, allowed_keys: ALLOWED_KEYS diff --git a/lib/gitlab/ci/config/entry/factory.rb b/lib/gitlab/ci/config/entry/factory.rb index 9f5e393d191..6be8288748f 100644 --- a/lib/gitlab/ci/config/entry/factory.rb +++ b/lib/gitlab/ci/config/entry/factory.rb @@ -6,7 +6,7 @@ module Gitlab # Factory class responsible for fabricating entry objects. # class Factory - class InvalidFactory < StandardError; end + InvalidFactory = Class.new(StandardError) def initialize(entry) @entry = entry diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index a55362f0b6b..176301bcca1 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -11,7 +11,7 @@ module Gitlab ALLOWED_KEYS = %i[tags script only except type image services allow_failure type stage when artifacts cache dependencies before_script - after_script variables environment] + after_script variables environment coverage].freeze validations do validates :config, allowed_keys: ALLOWED_KEYS @@ -71,9 +71,12 @@ module Gitlab entry :environment, Entry::Environment, description: 'Environment configuration for this job.' + entry :coverage, Entry::Coverage, + description: 'Coverage configuration for this job.' + helpers :before_script, :script, :stage, :type, :after_script, :cache, :image, :services, :only, :except, :variables, - :artifacts, :commands, :environment + :artifacts, :commands, :environment, :coverage attributes :script, :tags, :allow_failure, :when, :dependencies @@ -101,6 +104,14 @@ module Gitlab (before_script_value.to_a + script_value.to_a).join("\n") end + def manual_action? + self.when == 'manual' + end + + def ignored? + allow_failure.nil? ? manual_action? : allow_failure + end + private def inherit!(deps) @@ -130,8 +141,10 @@ module Gitlab variables: variables_defined? ? variables_value : nil, environment: environment_defined? ? environment_value : nil, environment_name: environment_defined? ? environment_value[:name] : nil, + coverage: coverage_defined? ? coverage_value : nil, artifacts: artifacts_value, - after_script: after_script_value } + after_script: after_script_value, + ignore: ignored? } end end end diff --git a/lib/gitlab/ci/config/entry/key.rb b/lib/gitlab/ci/config/entry/key.rb index 0e4c9fe6edc..f27ad0a7759 100644 --- a/lib/gitlab/ci/config/entry/key.rb +++ b/lib/gitlab/ci/config/entry/key.rb @@ -11,6 +11,10 @@ module Gitlab validations do validates :config, key: true end + + def self.default + 'default' + end end end end diff --git a/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb b/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb index f01975aab5c..9b9a0a8125a 100644 --- a/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb +++ b/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb @@ -28,17 +28,21 @@ module Gitlab value.is_a?(String) || value.is_a?(Symbol) end + def validate_regexp(value) + !value.nil? && Regexp.new(value.to_s) && true + rescue RegexpError, TypeError + false + end + def validate_string_or_regexp(value) return true if value.is_a?(Symbol) return false unless value.is_a?(String) if value.first == '/' && value.last == '/' - Regexp.new(value[1...-1]) + validate_regexp(value[1...-1]) else true end - rescue RegexpError - false end def validate_boolean(value) diff --git a/lib/gitlab/ci/config/entry/node.rb b/lib/gitlab/ci/config/entry/node.rb index 5eef2868cd6..a6a914d79c1 100644 --- a/lib/gitlab/ci/config/entry/node.rb +++ b/lib/gitlab/ci/config/entry/node.rb @@ -6,7 +6,7 @@ module Gitlab # Base abstract class for each configuration entry node. # class Node - class InvalidError < StandardError; end + InvalidError = Class.new(StandardError) attr_reader :config, :metadata attr_accessor :key, :parent, :description @@ -70,6 +70,12 @@ module Gitlab true end + def inspect + val = leaf? ? config : descendants + unspecified = specified? ? '' : '(unspecified) ' + "#<#{self.class.name} #{unspecified}{#{key}: #{val.inspect}}>" + end + def self.default end diff --git a/lib/gitlab/ci/config/entry/trigger.rb b/lib/gitlab/ci/config/entry/trigger.rb index 28b0a9ffe01..16b234e6c59 100644 --- a/lib/gitlab/ci/config/entry/trigger.rb +++ b/lib/gitlab/ci/config/entry/trigger.rb @@ -9,15 +9,7 @@ module Gitlab include Validatable validations do - include LegacyValidationHelpers - - validate :array_of_strings_or_regexps - - def array_of_strings_or_regexps - unless validate_array_of_strings_or_regexps(config) - errors.add(:config, 'should be an array of strings or regexps') - end - end + validates :config, array_of_strings_or_regexps: true end end end diff --git a/lib/gitlab/ci/config/entry/undefined.rb b/lib/gitlab/ci/config/entry/undefined.rb index b33b8238230..1171ac10f22 100644 --- a/lib/gitlab/ci/config/entry/undefined.rb +++ b/lib/gitlab/ci/config/entry/undefined.rb @@ -29,6 +29,10 @@ module Gitlab def relevant? false end + + def inspect + "#<#{self.class.name}>" + end end end end diff --git a/lib/gitlab/ci/config/entry/validators.rb b/lib/gitlab/ci/config/entry/validators.rb index 8632dd0e233..bd7428b1272 100644 --- a/lib/gitlab/ci/config/entry/validators.rb +++ b/lib/gitlab/ci/config/entry/validators.rb @@ -54,6 +54,51 @@ module Gitlab end end + class RegexpValidator < ActiveModel::EachValidator + include LegacyValidationHelpers + + def validate_each(record, attribute, value) + unless validate_regexp(value) + record.errors.add(attribute, 'must be a regular expression') + end + end + + private + + def look_like_regexp?(value) + value.is_a?(String) && value.start_with?('/') && + value.end_with?('/') + end + + def validate_regexp(value) + look_like_regexp?(value) && + Regexp.new(value.to_s[1...-1]) && + true + rescue RegexpError + false + end + end + + class ArrayOfStringsOrRegexpsValidator < RegexpValidator + def validate_each(record, attribute, value) + unless validate_array_of_strings_or_regexps(value) + record.errors.add(attribute, 'should be an array of strings or regexps') + end + end + + private + + def validate_array_of_strings_or_regexps(values) + values.is_a?(Array) && values.all?(&method(:validate_string_or_regexp)) + end + + def validate_string_or_regexp(value) + return false unless value.is_a?(String) + return validate_regexp(value) if look_like_regexp?(value) + true + end + end + class TypeValidator < ActiveModel::EachValidator def validate_each(record, attribute, value) type = options[:with] diff --git a/lib/gitlab/ci/config/loader.rb b/lib/gitlab/ci/config/loader.rb index dbf6eb0edbe..e7d9f6a7761 100644 --- a/lib/gitlab/ci/config/loader.rb +++ b/lib/gitlab/ci/config/loader.rb @@ -2,7 +2,7 @@ module Gitlab module Ci class Config class Loader - class FormatError < StandardError; end + FormatError = Class.new(StandardError) def initialize(config) @config = YAML.safe_load(config, [Symbol], [], true) diff --git a/lib/gitlab/ci/status/build/cancelable.rb b/lib/gitlab/ci/status/build/cancelable.rb index a979fe7d573..67bbc3c4849 100644 --- a/lib/gitlab/ci/status/build/cancelable.rb +++ b/lib/gitlab/ci/status/build/cancelable.rb @@ -10,7 +10,7 @@ module Gitlab end def action_icon - 'ban' + 'icon_action_cancel' end def action_path diff --git a/lib/gitlab/ci/status/build/factory.rb b/lib/gitlab/ci/status/build/factory.rb index eee9a64120b..38ac6edc9f1 100644 --- a/lib/gitlab/ci/status/build/factory.rb +++ b/lib/gitlab/ci/status/build/factory.rb @@ -4,8 +4,11 @@ module Gitlab module Build class Factory < Status::Factory def self.extended_statuses - [Status::Build::Stop, Status::Build::Play, - Status::Build::Cancelable, Status::Build::Retryable] + [[Status::Build::Cancelable, + Status::Build::Retryable], + [Status::Build::FailedAllowed, + Status::Build::Play, + Status::Build::Stop]] end def self.common_helpers diff --git a/lib/gitlab/ci/status/build/failed_allowed.rb b/lib/gitlab/ci/status/build/failed_allowed.rb new file mode 100644 index 00000000000..807afe24bd5 --- /dev/null +++ b/lib/gitlab/ci/status/build/failed_allowed.rb @@ -0,0 +1,27 @@ +module Gitlab + module Ci + module Status + module Build + class FailedAllowed < SimpleDelegator + include Status::Extended + + def label + 'failed (allowed to fail)' + end + + def icon + 'icon_status_warning' + end + + def group + 'failed_with_warnings' + end + + def self.matches?(build, user) + build.failed? && build.allow_failure? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/play.rb b/lib/gitlab/ci/status/build/play.rb index 1bf949c96dd..3495b8d0448 100644 --- a/lib/gitlab/ci/status/build/play.rb +++ b/lib/gitlab/ci/status/build/play.rb @@ -5,38 +5,22 @@ module Gitlab class Play < SimpleDelegator include Status::Extended - def text - 'manual' - end - def label 'manual play action' end - def icon - 'icon_status_manual' - end - - def group - 'manual' - end - def has_action? can?(user, :update_build, subject) end def action_icon - 'play' + 'icon_action_play' end def action_title 'Play' end - def action_class - 'ci-play-icon' - end - def action_path play_namespace_project_build_path(subject.project.namespace, subject.project, diff --git a/lib/gitlab/ci/status/build/retryable.rb b/lib/gitlab/ci/status/build/retryable.rb index 8e38d6a8523..6b362af7634 100644 --- a/lib/gitlab/ci/status/build/retryable.rb +++ b/lib/gitlab/ci/status/build/retryable.rb @@ -10,7 +10,7 @@ module Gitlab end def action_icon - 'refresh' + 'icon_action_retry' end def action_title diff --git a/lib/gitlab/ci/status/build/stop.rb b/lib/gitlab/ci/status/build/stop.rb index e1dfdb76d41..e8530f2aaae 100644 --- a/lib/gitlab/ci/status/build/stop.rb +++ b/lib/gitlab/ci/status/build/stop.rb @@ -5,28 +5,16 @@ module Gitlab class Stop < SimpleDelegator include Status::Extended - def text - 'manual' - end - def label 'manual stop action' end - def icon - 'icon_status_manual' - end - - def group - 'manual' - end - def has_action? can?(user, :update_build, subject) end def action_icon - 'stop' + 'icon_action_stop' end def action_title diff --git a/lib/gitlab/ci/status/canceled.rb b/lib/gitlab/ci/status/canceled.rb index dd6d99e9075..97c121ce7b9 100644 --- a/lib/gitlab/ci/status/canceled.rb +++ b/lib/gitlab/ci/status/canceled.rb @@ -13,6 +13,10 @@ module Gitlab def icon 'icon_status_canceled' end + + def favicon + 'favicon_status_canceled' + end end end end diff --git a/lib/gitlab/ci/status/core.rb b/lib/gitlab/ci/status/core.rb index 73b6ab5a635..d4fd83b93f8 100644 --- a/lib/gitlab/ci/status/core.rb +++ b/lib/gitlab/ci/status/core.rb @@ -18,6 +18,10 @@ module Gitlab raise NotImplementedError end + def favicon + raise NotImplementedError + end + def label raise NotImplementedError end @@ -42,9 +46,6 @@ module Gitlab raise NotImplementedError end - def action_class - end - def action_path raise NotImplementedError end diff --git a/lib/gitlab/ci/status/created.rb b/lib/gitlab/ci/status/created.rb index 6596d7e01ca..0721bf6ec7c 100644 --- a/lib/gitlab/ci/status/created.rb +++ b/lib/gitlab/ci/status/created.rb @@ -13,6 +13,10 @@ module Gitlab def icon 'icon_status_created' end + + def favicon + 'favicon_status_created' + end end end end diff --git a/lib/gitlab/ci/status/factory.rb b/lib/gitlab/ci/status/factory.rb index ae9ef895df4..15836c699c7 100644 --- a/lib/gitlab/ci/status/factory.rb +++ b/lib/gitlab/ci/status/factory.rb @@ -5,41 +5,46 @@ module Gitlab def initialize(subject, user) @subject = subject @user = user + @status = subject.status || HasStatus::DEFAULT_STATUS end def fabricate! - if extended_status - extended_status.new(core_status) - else + if extended_statuses.none? core_status + else + compound_extended_status end end - def self.extended_statuses - [] + def core_status + Gitlab::Ci::Status + .const_get(@status.capitalize) + .new(@subject, @user) + .extend(self.class.common_helpers) end - def self.common_helpers - Module.new + def compound_extended_status + extended_statuses.inject(core_status) do |status, extended| + extended.new(status) + end end - private + def extended_statuses + return @extended_statuses if defined?(@extended_statuses) - def simple_status - @simple_status ||= @subject.status || :created + groups = self.class.extended_statuses.map do |group| + Array(group).find { |status| status.matches?(@subject, @user) } + end + + @extended_statuses = groups.flatten.compact end - def core_status - Gitlab::Ci::Status - .const_get(simple_status.capitalize) - .new(@subject, @user) - .extend(self.class.common_helpers) + def self.extended_statuses + [] end - def extended_status - @extended ||= self.class.extended_statuses.find do |status| - status.matches?(@subject, @user) - end + def self.common_helpers + Module.new end end end diff --git a/lib/gitlab/ci/status/failed.rb b/lib/gitlab/ci/status/failed.rb index c5b5e3203ad..cb75e9383a8 100644 --- a/lib/gitlab/ci/status/failed.rb +++ b/lib/gitlab/ci/status/failed.rb @@ -13,6 +13,10 @@ module Gitlab def icon 'icon_status_failed' end + + def favicon + 'favicon_status_failed' + end end end end diff --git a/lib/gitlab/ci/status/manual.rb b/lib/gitlab/ci/status/manual.rb new file mode 100644 index 00000000000..f8f6c2903ba --- /dev/null +++ b/lib/gitlab/ci/status/manual.rb @@ -0,0 +1,23 @@ +module Gitlab + module Ci + module Status + class Manual < Status::Core + def text + 'manual' + end + + def label + 'manual action' + end + + def icon + 'icon_status_manual' + end + + def favicon + 'favicon_status_manual' + end + end + end + end +end diff --git a/lib/gitlab/ci/status/pending.rb b/lib/gitlab/ci/status/pending.rb index d30f35a59a2..f40cc1314dc 100644 --- a/lib/gitlab/ci/status/pending.rb +++ b/lib/gitlab/ci/status/pending.rb @@ -13,6 +13,10 @@ module Gitlab def icon 'icon_status_pending' end + + def favicon + 'favicon_status_pending' + end end end end diff --git a/lib/gitlab/ci/status/pipeline/blocked.rb b/lib/gitlab/ci/status/pipeline/blocked.rb new file mode 100644 index 00000000000..a250c3fcb41 --- /dev/null +++ b/lib/gitlab/ci/status/pipeline/blocked.rb @@ -0,0 +1,23 @@ +module Gitlab + module Ci + module Status + module Pipeline + class Blocked < SimpleDelegator + include Status::Extended + + def text + 'blocked' + end + + def label + 'waiting for manual action' + end + + def self.matches?(pipeline, user) + pipeline.blocked? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/pipeline/factory.rb b/lib/gitlab/ci/status/pipeline/factory.rb index 16dcb326be9..17f9a75f436 100644 --- a/lib/gitlab/ci/status/pipeline/factory.rb +++ b/lib/gitlab/ci/status/pipeline/factory.rb @@ -4,7 +4,8 @@ module Gitlab module Pipeline class Factory < Status::Factory def self.extended_statuses - [Pipeline::SuccessWithWarnings] + [[Status::SuccessWarning, + Status::Pipeline::Blocked]] end def self.common_helpers diff --git a/lib/gitlab/ci/status/pipeline/success_with_warnings.rb b/lib/gitlab/ci/status/pipeline/success_with_warnings.rb deleted file mode 100644 index 24bf8b869e0..00000000000 --- a/lib/gitlab/ci/status/pipeline/success_with_warnings.rb +++ /dev/null @@ -1,31 +0,0 @@ -module Gitlab - module Ci - module Status - module Pipeline - class SuccessWithWarnings < SimpleDelegator - include Status::Extended - - def text - 'passed' - end - - def label - 'passed with warnings' - end - - def icon - 'icon_status_warning' - end - - def group - 'success_with_warnings' - end - - def self.matches?(pipeline, user) - pipeline.success? && pipeline.has_warnings? - end - end - end - end - end -end diff --git a/lib/gitlab/ci/status/running.rb b/lib/gitlab/ci/status/running.rb index 2aba3c373c7..1237cd47dc8 100644 --- a/lib/gitlab/ci/status/running.rb +++ b/lib/gitlab/ci/status/running.rb @@ -13,6 +13,10 @@ module Gitlab def icon 'icon_status_running' end + + def favicon + 'favicon_status_running' + end end end end diff --git a/lib/gitlab/ci/status/skipped.rb b/lib/gitlab/ci/status/skipped.rb index 16282aefd03..28005d91503 100644 --- a/lib/gitlab/ci/status/skipped.rb +++ b/lib/gitlab/ci/status/skipped.rb @@ -13,6 +13,10 @@ module Gitlab def icon 'icon_status_skipped' end + + def favicon + 'favicon_status_skipped' + end end end end diff --git a/lib/gitlab/ci/status/stage/factory.rb b/lib/gitlab/ci/status/stage/factory.rb index 689a5dd45bc..4c37f084d07 100644 --- a/lib/gitlab/ci/status/stage/factory.rb +++ b/lib/gitlab/ci/status/stage/factory.rb @@ -3,6 +3,10 @@ module Gitlab module Status module Stage class Factory < Status::Factory + def self.extended_statuses + [Status::SuccessWarning] + end + def self.common_helpers Status::Stage::Common end diff --git a/lib/gitlab/ci/status/success.rb b/lib/gitlab/ci/status/success.rb index c09c5f006e3..88f7758a270 100644 --- a/lib/gitlab/ci/status/success.rb +++ b/lib/gitlab/ci/status/success.rb @@ -13,6 +13,10 @@ module Gitlab def icon 'icon_status_success' end + + def favicon + 'favicon_status_success' + end end end end diff --git a/lib/gitlab/ci/status/success_warning.rb b/lib/gitlab/ci/status/success_warning.rb new file mode 100644 index 00000000000..d4cdab6957a --- /dev/null +++ b/lib/gitlab/ci/status/success_warning.rb @@ -0,0 +1,33 @@ +module Gitlab + module Ci + module Status + ## + # Extended status used when pipeline or stage passed conditionally. + # This means that failed jobs that are allowed to fail were present. + # + class SuccessWarning < SimpleDelegator + include Status::Extended + + def text + 'passed' + end + + def label + 'passed with warnings' + end + + def icon + 'icon_status_warning' + end + + def group + 'success_with_warnings' + end + + def self.matches?(subject, user) + subject.success? && subject.has_warnings? + end + end + end + end +end diff --git a/lib/gitlab/ci/trace_reader.rb b/lib/gitlab/ci/trace_reader.rb index 37e51536e8f..1d7ddeb3e0f 100644 --- a/lib/gitlab/ci/trace_reader.rb +++ b/lib/gitlab/ci/trace_reader.rb @@ -42,6 +42,7 @@ module Gitlab end chunks.join.lines.last(max_lines).join + .force_encoding(Encoding.default_external) end end end diff --git a/lib/gitlab/conflict/file.rb b/lib/gitlab/conflict/file.rb index c843315782d..75a213ef752 100644 --- a/lib/gitlab/conflict/file.rb +++ b/lib/gitlab/conflict/file.rb @@ -4,8 +4,7 @@ module Gitlab include Gitlab::Routing.url_helpers include IconsHelper - class MissingResolution < ResolutionError - end + MissingResolution = Class.new(ResolutionError) CONTEXT_LINES = 3 @@ -91,11 +90,12 @@ module Gitlab our_highlight = Gitlab::Highlight.highlight(our_path, our_file, repository: repository).lines lines.each do |line| - if line.type == 'old' - line.rich_text = their_highlight[line.old_line - 1].try(:html_safe) - else - line.rich_text = our_highlight[line.new_line - 1].try(:html_safe) - end + line.rich_text = + if line.type == 'old' + their_highlight[line.old_line - 1].try(:html_safe) + else + our_highlight[line.new_line - 1].try(:html_safe) + end end end diff --git a/lib/gitlab/conflict/file_collection.rb b/lib/gitlab/conflict/file_collection.rb index fa5bd4649d4..990b719ecfd 100644 --- a/lib/gitlab/conflict/file_collection.rb +++ b/lib/gitlab/conflict/file_collection.rb @@ -1,8 +1,7 @@ module Gitlab module Conflict class FileCollection - class ConflictSideMissing < StandardError - end + ConflictSideMissing = Class.new(StandardError) attr_reader :merge_request, :our_commit, :their_commit diff --git a/lib/gitlab/conflict/parser.rb b/lib/gitlab/conflict/parser.rb index ddd657903fb..84f9ecd3d23 100644 --- a/lib/gitlab/conflict/parser.rb +++ b/lib/gitlab/conflict/parser.rb @@ -1,35 +1,23 @@ module Gitlab module Conflict class Parser - class UnresolvableError < StandardError - end - - class UnmergeableFile < UnresolvableError - end - - class UnsupportedEncoding < UnresolvableError - end + UnresolvableError = Class.new(StandardError) + UnmergeableFile = Class.new(UnresolvableError) + UnsupportedEncoding = Class.new(UnresolvableError) # Recoverable errors - the conflict can be resolved in an editor, but not with # sections. - class ParserError < StandardError - end - - class UnexpectedDelimiter < ParserError - end - - class MissingEndDelimiter < ParserError - end + ParserError = Class.new(StandardError) + UnexpectedDelimiter = Class.new(ParserError) + MissingEndDelimiter = Class.new(ParserError) def parse(text, our_path:, their_path:, parent_file: nil) raise UnmergeableFile if text.blank? # Typically a binary file raise UnmergeableFile if text.length > 200.kilobytes - begin - text.to_json - rescue Encoding::UndefinedConversionError - raise UnsupportedEncoding - end + text.force_encoding('UTF-8') + + raise UnsupportedEncoding unless text.valid_encoding? line_obj_index = 0 line_old = 1 diff --git a/lib/gitlab/conflict/resolution_error.rb b/lib/gitlab/conflict/resolution_error.rb index a0f2006bc24..0b61256b35a 100644 --- a/lib/gitlab/conflict/resolution_error.rb +++ b/lib/gitlab/conflict/resolution_error.rb @@ -1,6 +1,5 @@ module Gitlab module Conflict - class ResolutionError < StandardError - end + ResolutionError = Class.new(StandardError) end end diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index 7e3d5647b39..15992b77680 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -22,8 +22,10 @@ module Gitlab having(action: [Event::CREATED, Event::CLOSED], target_type: "Issue") mr_events = event_counts(date_from, :merge_requests). having(action: [Event::MERGED, Event::CREATED, Event::CLOSED], target_type: "MergeRequest") + note_events = event_counts(date_from, :merge_requests). + having(action: [Event::COMMENTED], target_type: "Note") - union = Gitlab::SQL::Union.new([repo_events, issue_events, mr_events]) + union = Gitlab::SQL::Union.new([repo_events, issue_events, mr_events, note_events]) events = Event.find_by_sql(union.to_sql).map(&:attributes) @activity_events = events.each_with_object(Hash.new {|h, k| h[k] = 0 }) do |event, activities| @@ -38,7 +40,7 @@ module Gitlab # Use visible_to_user? instead of the complicated logic in activity_dates # because we're only viewing the events for a single day. - events.select {|event| event.visible_to_user?(current_user) } + events.select { |event| event.visible_to_user?(current_user) } end def starting_year diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 2ff27e46d64..82576d197fe 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -9,7 +9,9 @@ module Gitlab end def ensure_application_settings! - if connect_to_db? + return fake_application_settings unless connect_to_db? + + unless ENV['IN_MEMORY_APPLICATION_SETTINGS'] == 'true' begin settings = ::ApplicationSetting.current # In case Redis isn't running or the Redis UNIX socket file is not available @@ -20,43 +22,21 @@ module Gitlab settings ||= ::ApplicationSetting.create_from_defaults unless ActiveRecord::Migrator.needs_migration? end - settings || fake_application_settings + settings || in_memory_application_settings end - def sidekiq_throttling_enabled? - current_application_settings.sidekiq_throttling_enabled? + delegate :sidekiq_throttling_enabled?, to: :current_application_settings + + def in_memory_application_settings + @in_memory_application_settings ||= ::ApplicationSetting.new(::ApplicationSetting.defaults) + # In case migrations the application_settings table is not created yet, + # we fallback to a simple OpenStruct + rescue ActiveRecord::StatementInvalid, ActiveRecord::UnknownAttributeError + fake_application_settings end def fake_application_settings - OpenStruct.new( - default_projects_limit: Settings.gitlab['default_projects_limit'], - default_branch_protection: Settings.gitlab['default_branch_protection'], - signup_enabled: Settings.gitlab['signup_enabled'], - signin_enabled: Settings.gitlab['signin_enabled'], - gravatar_enabled: Settings.gravatar['enabled'], - koding_enabled: false, - plantuml_enabled: false, - sign_in_text: nil, - after_sign_up_text: nil, - help_page_text: nil, - shared_runners_text: nil, - restricted_visibility_levels: Settings.gitlab['restricted_visibility_levels'], - max_attachment_size: Settings.gitlab['max_attachment_size'], - session_expire_delay: Settings.gitlab['session_expire_delay'], - default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'], - default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'], - domain_whitelist: Settings.gitlab['domain_whitelist'], - import_sources: %w[gitea github bitbucket gitlab google_code fogbugz git gitlab_project], - shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], - max_artifacts_size: Settings.artifacts['max_size'], - require_two_factor_authentication: false, - two_factor_grace_period: 48, - akismet_enabled: false, - repository_checks_enabled: true, - container_registry_token_expire_delay: 5, - user_default_external: false, - sidekiq_throttling_enabled: false, - ) + OpenStruct.new(::ApplicationSetting.defaults) end private diff --git a/lib/gitlab/cycle_analytics/base_event_fetcher.rb b/lib/gitlab/cycle_analytics/base_event_fetcher.rb index 0d8791d396b..ab115afcaa5 100644 --- a/lib/gitlab/cycle_analytics/base_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/base_event_fetcher.rb @@ -5,6 +5,8 @@ module Gitlab attr_reader :projections, :query, :stage, :order + MAX_EVENTS = 50 + def initialize(project:, stage:, options:) @project = project @stage = stage @@ -38,7 +40,7 @@ module Gitlab def events_query diff_fn = subtract_datetimes_diff(base_query, @options[:start_time_attrs], @options[:end_time_attrs]) - base_query.project(extract_diff_epoch(diff_fn).as('total_time'), *projections).order(order.desc) + base_query.project(extract_diff_epoch(diff_fn).as('total_time'), *projections).order(order.desc).take(MAX_EVENTS) end def default_order diff --git a/lib/gitlab/cycle_analytics/base_stage.rb b/lib/gitlab/cycle_analytics/base_stage.rb index 74bbcdcb3dd..559e3939da6 100644 --- a/lib/gitlab/cycle_analytics/base_stage.rb +++ b/lib/gitlab/cycle_analytics/base_stage.rb @@ -13,7 +13,7 @@ module Gitlab end def as_json - AnalyticsStageSerializer.new.represent(self).as_json + AnalyticsStageSerializer.new.represent(self) end def title diff --git a/lib/gitlab/cycle_analytics/code_event_fetcher.rb b/lib/gitlab/cycle_analytics/code_event_fetcher.rb index 5245b9ca8fc..d5bf6149749 100644 --- a/lib/gitlab/cycle_analytics/code_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/code_event_fetcher.rb @@ -18,7 +18,7 @@ module Gitlab private def serialize(event) - AnalyticsMergeRequestSerializer.new(project: @project).represent(event).as_json + AnalyticsMergeRequestSerializer.new(project: @project).represent(event) end end end diff --git a/lib/gitlab/cycle_analytics/code_stage.rb b/lib/gitlab/cycle_analytics/code_stage.rb index d1bc2055ba8..1e52b6614a1 100644 --- a/lib/gitlab/cycle_analytics/code_stage.rb +++ b/lib/gitlab/cycle_analytics/code_stage.rb @@ -13,6 +13,10 @@ module Gitlab :code end + def legend + "Related Merge Requests" + end + def description "Time until first merge request" end diff --git a/lib/gitlab/cycle_analytics/issue_event_fetcher.rb b/lib/gitlab/cycle_analytics/issue_event_fetcher.rb index 0d8da99455e..3df9cbdcfce 100644 --- a/lib/gitlab/cycle_analytics/issue_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/issue_event_fetcher.rb @@ -16,7 +16,7 @@ module Gitlab private def serialize(event) - AnalyticsIssueSerializer.new(project: @project).represent(event).as_json + AnalyticsIssueSerializer.new(project: @project).represent(event) end end end diff --git a/lib/gitlab/cycle_analytics/issue_stage.rb b/lib/gitlab/cycle_analytics/issue_stage.rb index d2068fbc38f..213994988a5 100644 --- a/lib/gitlab/cycle_analytics/issue_stage.rb +++ b/lib/gitlab/cycle_analytics/issue_stage.rb @@ -14,6 +14,10 @@ module Gitlab :issue end + def legend + "Related Issues" + end + def description "Time before an issue gets scheduled" end diff --git a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb index 88a8710dbe6..7d342a2d2cb 100644 --- a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb @@ -37,7 +37,7 @@ module Gitlab def serialize_commit(event, st_commit, query) commit = Commit.new(Gitlab::Git::Commit.new(st_commit), @project) - AnalyticsCommitSerializer.new(project: @project, total_time: event['total_time']).represent(commit).as_json + AnalyticsCommitSerializer.new(project: @project, total_time: event['total_time']).represent(commit) end end end diff --git a/lib/gitlab/cycle_analytics/plan_stage.rb b/lib/gitlab/cycle_analytics/plan_stage.rb index 3b4dfc6a30e..45d51d30ccc 100644 --- a/lib/gitlab/cycle_analytics/plan_stage.rb +++ b/lib/gitlab/cycle_analytics/plan_stage.rb @@ -14,6 +14,10 @@ module Gitlab :plan end + def legend + "Related Commits" + end + def description "Time before an issue starts implementation" end diff --git a/lib/gitlab/cycle_analytics/production_stage.rb b/lib/gitlab/cycle_analytics/production_stage.rb index 2a6bcc80116..9f387a02945 100644 --- a/lib/gitlab/cycle_analytics/production_stage.rb +++ b/lib/gitlab/cycle_analytics/production_stage.rb @@ -15,6 +15,10 @@ module Gitlab :production end + def legend + "Related Issues" + end + def description "From issue creation until deploy to production" end diff --git a/lib/gitlab/cycle_analytics/review_event_fetcher.rb b/lib/gitlab/cycle_analytics/review_event_fetcher.rb index 4df0bd06393..4c7b3f4467f 100644 --- a/lib/gitlab/cycle_analytics/review_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/review_event_fetcher.rb @@ -15,7 +15,7 @@ module Gitlab end def serialize(event) - AnalyticsMergeRequestSerializer.new(project: @project).represent(event).as_json + AnalyticsMergeRequestSerializer.new(project: @project).represent(event) end end end diff --git a/lib/gitlab/cycle_analytics/review_stage.rb b/lib/gitlab/cycle_analytics/review_stage.rb index fbaa3010d81..4744be834de 100644 --- a/lib/gitlab/cycle_analytics/review_stage.rb +++ b/lib/gitlab/cycle_analytics/review_stage.rb @@ -13,6 +13,10 @@ module Gitlab :review end + def legend + "Relative Merged Requests" + end + def description "Time between merge request creation and merge/close" end diff --git a/lib/gitlab/cycle_analytics/stage_summary.rb b/lib/gitlab/cycle_analytics/stage_summary.rb index b34baf5b081..fc77bd86097 100644 --- a/lib/gitlab/cycle_analytics/stage_summary.rb +++ b/lib/gitlab/cycle_analytics/stage_summary.rb @@ -16,7 +16,7 @@ module Gitlab private def serialize(summary_object) - AnalyticsSummarySerializer.new.represent(summary_object).as_json + AnalyticsSummarySerializer.new.represent(summary_object) end end end diff --git a/lib/gitlab/cycle_analytics/staging_event_fetcher.rb b/lib/gitlab/cycle_analytics/staging_event_fetcher.rb index a34731a5fcd..36c0260dbfe 100644 --- a/lib/gitlab/cycle_analytics/staging_event_fetcher.rb +++ b/lib/gitlab/cycle_analytics/staging_event_fetcher.rb @@ -23,7 +23,7 @@ module Gitlab private def serialize(event) - AnalyticsBuildSerializer.new.represent(event['build']).as_json + AnalyticsBuildSerializer.new.represent(event['build']) end end end diff --git a/lib/gitlab/cycle_analytics/staging_stage.rb b/lib/gitlab/cycle_analytics/staging_stage.rb index 945909a4d62..3cdbe04fbaf 100644 --- a/lib/gitlab/cycle_analytics/staging_stage.rb +++ b/lib/gitlab/cycle_analytics/staging_stage.rb @@ -14,6 +14,10 @@ module Gitlab :staging end + def legend + "Relative Deployed Builds" + end + def description "From merge request merge until deploy to production" end diff --git a/lib/gitlab/cycle_analytics/test_stage.rb b/lib/gitlab/cycle_analytics/test_stage.rb index 0079d56e0e4..e96943833bc 100644 --- a/lib/gitlab/cycle_analytics/test_stage.rb +++ b/lib/gitlab/cycle_analytics/test_stage.rb @@ -13,6 +13,10 @@ module Gitlab :test end + def legend + "Relative Builds Trigger by Commits" + end + def description "Total test time for all commits/merges" end diff --git a/lib/gitlab/data_builder/build.rb b/lib/gitlab/data_builder/build.rb index 6548e6475c6..f78106f5b10 100644 --- a/lib/gitlab/data_builder/build.rb +++ b/lib/gitlab/data_builder/build.rb @@ -8,6 +8,8 @@ module Gitlab commit = build.pipeline user = build.user + author_url = build_author_url(build.commit, commit) + data = { object_kind: 'build', @@ -43,6 +45,7 @@ module Gitlab message: commit.git_commit_message, author_name: commit.git_author_name, author_email: commit.git_author_email, + author_url: author_url, status: commit.status, duration: commit.duration, started_at: commit.started_at, @@ -62,6 +65,13 @@ module Gitlab data end + + private + + def build_author_url(commit, pipeline) + author = commit.try(:author) + author ? Gitlab::Routing.url_helpers.user_url(author) : "mailto:#{pipeline.git_author_email}" + end end end end diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb index e50e54b6e99..182a30fd74d 100644 --- a/lib/gitlab/data_builder/pipeline.rb +++ b/lib/gitlab/data_builder/pipeline.rb @@ -39,7 +39,7 @@ module Gitlab started_at: build.started_at, finished_at: build.finished_at, when: build.when, - manual: build.manual?, + manual: build.action?, user: build.user.try(:hook_attrs), runner: build.runner && runner_hook_attrs(build.runner), artifacts_file: { diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 55b8f888d53..63b8d0d3b9d 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -5,8 +5,12 @@ module Gitlab # http://dev.mysql.com/doc/refman/5.7/en/integer-types.html MAX_INT_VALUE = 2147483647 + def self.config + ActiveRecord::Base.configurations[Rails.env] + end + def self.adapter_name - connection.adapter_name + config['adapter'] end def self.mysql? @@ -24,7 +28,7 @@ module Gitlab def self.nulls_last_order(field, direction = 'ASC') order = "#{field} #{direction}" - if Gitlab::Database.postgresql? + if postgresql? order << ' NULLS LAST' else # `field IS NULL` will be `0` for non-NULL columns and `1` for NULL @@ -35,8 +39,22 @@ module Gitlab order end + def self.nulls_first_order(field, direction = 'ASC') + order = "#{field} #{direction}" + + if postgresql? + order << ' NULLS FIRST' + else + # `field IS NULL` will be `0` for non-NULL columns and `1` for NULL + # columns. In the (default) ascending order, `0` comes first. + order.prepend("#{field} IS NULL, ") if direction == 'DESC' + end + + order + end + def self.random - Gitlab::Database.postgresql? ? "RANDOM()" : "RAND()" + postgresql? ? "RANDOM()" : "RAND()" end def true_value @@ -55,6 +73,36 @@ module Gitlab end end + def self.with_connection_pool(pool_size) + pool = create_connection_pool(pool_size) + + begin + yield(pool) + ensure + pool.disconnect! + end + end + + # pool_size - The size of the DB pool. + # host - An optional host name to use instead of the default one. + def self.create_connection_pool(pool_size, host = nil) + # See activerecord-4.2.7.1/lib/active_record/connection_adapters/connection_specification.rb + env = Rails.env + original_config = ActiveRecord::Base.configurations + + env_config = original_config[env].merge('pool' => pool_size) + env_config['host'] = host if host + + config = original_config.merge(env => env_config) + + spec = + ActiveRecord:: + ConnectionAdapters:: + ConnectionSpecification::Resolver.new(config).spec(env.to_sym) + + ActiveRecord::ConnectionAdapters::ConnectionPool.new(spec) + end + def self.connection ActiveRecord::Base.connection end diff --git a/lib/gitlab/database/median.rb b/lib/gitlab/database/median.rb index 08607c27c09..23890e5f493 100644 --- a/lib/gitlab/database/median.rb +++ b/lib/gitlab/database/median.rb @@ -108,6 +108,7 @@ module Gitlab Arel.sql(%Q{EXTRACT(EPOCH FROM (#{diff.to_sql}))}) end + # Need to cast '0' to an INTERVAL before we can check if the interval is positive def zero_interval Arel::Nodes::NamedFunction.new("CAST", [Arel.sql("'0' AS INTERVAL")]) diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 0bd6e148ba8..fc445ab9483 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -26,11 +26,68 @@ module Gitlab add_index(table_name, column_name, options) end + # Adds a foreign key with only minimal locking on the tables involved. + # + # This method only requires minimal locking when using PostgreSQL. When + # using MySQL this method will use Rails' default `add_foreign_key`. + # + # source - The source table containing the foreign key. + # target - The target table the key points to. + # column - The name of the column to create the foreign key on. + # on_delete - The action to perform when associated data is removed, + # defaults to "CASCADE". + def add_concurrent_foreign_key(source, target, column:, on_delete: :cascade) + # Transactions would result in ALTER TABLE locks being held for the + # duration of the transaction, defeating the purpose of this method. + if transaction_open? + raise 'add_concurrent_foreign_key can not be run inside a transaction' + end + + # While MySQL does allow disabling of foreign keys it has no equivalent + # of PostgreSQL's "VALIDATE CONSTRAINT". As a result we'll just fall + # back to the normal foreign key procedure. + if Database.mysql? + return add_foreign_key(source, target, + column: column, + on_delete: on_delete) + end + + disable_statement_timeout + + key_name = concurrent_foreign_key_name(source, column) + + # Using NOT VALID allows us to create a key without immediately + # validating it. This means we keep the ALTER TABLE lock only for a + # short period of time. The key _is_ enforced for any newly created + # data. + execute <<-EOF.strip_heredoc + ALTER TABLE #{source} + ADD CONSTRAINT #{key_name} + FOREIGN KEY (#{column}) + REFERENCES #{target} (id) + ON DELETE #{on_delete} NOT VALID; + EOF + + # Validate the existing constraint. This can potentially take a very + # long time to complete, but fortunately does not lock the source table + # while running. + execute("ALTER TABLE #{source} VALIDATE CONSTRAINT #{key_name};") + end + + # Returns the name for a concurrent foreign key. + # + # PostgreSQL constraint names have a limit of 63 bytes. The logic used + # here is based on Rails' foreign_key_name() method, which unfortunately + # is private so we can't rely on it directly. + def concurrent_foreign_key_name(table, column) + "fk_#{Digest::SHA256.hexdigest("#{table}_#{column}_fk").first(10)}" + end + # Long-running migrations may take more than the timeout allowed by # the database. Disable the session's statement timeout to ensure # migrations don't get killed prematurely. (PostgreSQL only) def disable_statement_timeout - ActiveRecord::Base.connection.execute('SET statement_timeout TO 0') if Database.postgresql? + execute('SET statement_timeout TO 0') if Database.postgresql? end # Updates the value of a column in batches. diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index 9ea976e18fa..7db896522a9 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -50,7 +50,7 @@ module Gitlab # Only update text if line is found. This will prevent # issues with submodules given the line only exists in diff content. if rich_line - line_prefix = diff_line.text.match(/\A(.)/) ? $1 : ' ' + line_prefix = diff_line.text =~ /\A(.)/ ? $1 : ' ' "#{line_prefix}#{rich_line}".html_safe end end diff --git a/lib/gitlab/diff/inline_diff_marker.rb b/lib/gitlab/diff/inline_diff_marker.rb index 87a9b1e23ac..736933b1c4b 100644 --- a/lib/gitlab/diff/inline_diff_marker.rb +++ b/lib/gitlab/diff/inline_diff_marker.rb @@ -4,7 +4,7 @@ module Gitlab MARKDOWN_SYMBOLS = { addition: "+", deletion: "-" - } + }.freeze attr_accessor :raw_line, :rich_line diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index 80a146b4a5a..114656958e3 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -38,11 +38,11 @@ module Gitlab end def added? - type == 'new' + type == 'new' || type == 'new-nonewline' end def removed? - type == 'old' + type == 'old' || type == 'old-nonewline' end def rich_text @@ -52,7 +52,7 @@ module Gitlab end def meta? - type == 'match' || type == 'nonewline' + type == 'match' end def as_json(opts = nil) diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb index 59a2367b65d..742f989c50b 100644 --- a/lib/gitlab/diff/parser.rb +++ b/lib/gitlab/diff/parser.rb @@ -11,6 +11,7 @@ module Gitlab line_old = 1 line_new = 1 type = nil + context = nil # By returning an Enumerator we make it possible to search for a single line (with #find) # without having to instantiate all the others that come after it. @@ -20,7 +21,7 @@ module Gitlab full_line = line.delete("\n") - if line.match(/^@@ -/) + if line =~ /^@@ -/ type = "match" line_old = line.match(/\-[0-9]*/)[0].to_i.abs rescue 0 @@ -31,7 +32,8 @@ module Gitlab line_obj_index += 1 next elsif line[0] == '\\' - type = 'nonewline' + type = "#{context}-nonewline" + yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new) line_obj_index += 1 else @@ -43,9 +45,11 @@ module Gitlab case line[0] when "+" line_new += 1 + context = :new when "-" line_old += 1 - when "\\" + context = :old + when "\\" # rubocop:disable Lint/EmptyWhen # No increment else line_new += 1 diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb index ecf62dead35..fc728123c97 100644 --- a/lib/gitlab/diff/position.rb +++ b/lib/gitlab/diff/position.rb @@ -140,15 +140,16 @@ module Gitlab def find_diff_file(repository) # We're at the initial commit, so just get that as we can't compare to anything. - if Gitlab::Git.blank_ref?(start_sha) - compare = Gitlab::Git::Commit.find(repository.raw_repository, head_sha) - else - compare = Gitlab::Git::Compare.new( - repository.raw_repository, - start_sha, - head_sha - ) - end + compare = + if Gitlab::Git.blank_ref?(start_sha) + Gitlab::Git::Commit.find(repository.raw_repository, head_sha) + else + Gitlab::Git::Compare.new( + repository.raw_repository, + start_sha, + head_sha + ) + end diff = compare.diffs(paths: paths).first diff --git a/lib/gitlab/downtime_check/message.rb b/lib/gitlab/downtime_check/message.rb index 40a4815a9a0..543e62794c5 100644 --- a/lib/gitlab/downtime_check/message.rb +++ b/lib/gitlab/downtime_check/message.rb @@ -3,8 +3,8 @@ module Gitlab class Message attr_reader :path, :offline - OFFLINE = "\e[31moffline\e[0m" - ONLINE = "\e[32monline\e[0m" + OFFLINE = "\e[31moffline\e[0m".freeze + ONLINE = "\e[32monline\e[0m".freeze # path - The file path of the migration. # offline - When set to `true` the migration will require downtime. diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index c8e36d8ff4a..496ee0bdcb0 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -5,35 +5,44 @@ module Gitlab CE_REPO = 'https://gitlab.com/gitlab-org/gitlab-ce.git'.freeze EE_REPO = 'https://gitlab.com/gitlab-org/gitlab-ee.git'.freeze CHECK_DIR = Rails.root.join('ee_compat_check') - MAX_FETCH_DEPTH = 500 IGNORED_FILES_REGEX = /(VERSION|CHANGELOG\.md:\d+)/.freeze - - attr_reader :repo_dir, :patches_dir, :ce_repo, :ce_branch + PLEASE_READ_THIS_BANNER = %Q{ + ============================================================ + ===================== PLEASE READ THIS ===================== + ============================================================ + }.freeze + THANKS_FOR_READING_BANNER = %Q{ + ============================================================ + ==================== THANKS FOR READING ==================== + ============================================================\n + }.freeze + + attr_reader :ee_repo_dir, :patches_dir, :ce_repo, :ce_branch, :ee_branch_found + attr_reader :failed_files def initialize(branch:, ce_repo: CE_REPO) - @repo_dir = CHECK_DIR.join('repo') + @ee_repo_dir = CHECK_DIR.join('ee-repo') @patches_dir = CHECK_DIR.join('patches') @ce_branch = branch @ce_repo = ce_repo end def check - ensure_ee_repo ensure_patches_dir - generate_patch(ce_branch, ce_patch_full_path) - Dir.chdir(repo_dir) do - step("In the #{repo_dir} directory") + ensure_ee_repo + Dir.chdir(ee_repo_dir) do + step("In the #{ee_repo_dir} directory") status = catch(:halt_check) do ce_branch_compat_check! - delete_ee_branch_locally! + delete_ee_branches_locally! ee_branch_presence_check! ee_branch_compat_check! end - delete_ee_branch_locally! + delete_ee_branches_locally! if status.nil? true @@ -46,11 +55,13 @@ module Gitlab private def ensure_ee_repo - if Dir.exist?(repo_dir) - step("#{repo_dir} already exists") + if Dir.exist?(ee_repo_dir) + step("#{ee_repo_dir} already exists") else - cmd = %W[git clone --branch master --single-branch --depth 200 #{EE_REPO} #{repo_dir}] - step("Cloning #{EE_REPO} into #{repo_dir}", cmd) + step( + "Cloning #{EE_REPO} into #{ee_repo_dir}", + %W[git clone --branch master --single-branch --depth=200 #{EE_REPO} #{ee_repo_dir}] + ) end end @@ -61,23 +72,18 @@ module Gitlab def generate_patch(branch, patch_path) FileUtils.rm(patch_path, force: true) - depth = 0 - loop do - depth += 50 - cmd = %W[git fetch --depth #{depth} origin --prune +refs/heads/master:refs/remotes/origin/master] - Gitlab::Popen.popen(cmd) - _, status = Gitlab::Popen.popen(%w[git merge-base FETCH_HEAD HEAD]) + find_merge_base_with_master(branch: branch) - raise "#{branch} is too far behind master, please rebase it!" if depth >= MAX_FETCH_DEPTH - break if status.zero? - end + step( + "Generating the patch against origin/master in #{patch_path}", + %w[git format-patch origin/master --stdout] + ) do |output, status| + throw(:halt_check, :ko) unless status.zero? - step("Generating the patch against master in #{patch_path}") - output, status = Gitlab::Popen.popen(%w[git format-patch FETCH_HEAD --stdout]) - throw(:halt_check, :ko) unless status.zero? + File.write(patch_path, output) - File.write(patch_path, output) - throw(:halt_check, :ko) unless File.exist?(patch_path) + throw(:halt_check, :ko) unless File.exist?(patch_path) + end end def ce_branch_compat_check! @@ -88,9 +94,17 @@ module Gitlab end def ee_branch_presence_check! - status = step("Fetching origin/#{ee_branch}", %W[git fetch origin #{ee_branch}]) + _, status = step("Fetching origin/#{ee_branch_prefix}", %W[git fetch origin #{ee_branch_prefix}]) - unless status.zero? + if status.zero? + @ee_branch_found = ee_branch_prefix + else + _, status = step("Fetching origin/#{ee_branch_suffix}", %W[git fetch origin #{ee_branch_suffix}]) + end + + if status.zero? + @ee_branch_found = ee_branch_suffix + else puts puts ce_branch_doesnt_apply_cleanly_and_no_ee_branch_msg @@ -99,9 +113,9 @@ module Gitlab end def ee_branch_compat_check! - step("Checking out origin/#{ee_branch}", %W[git checkout -b #{ee_branch} FETCH_HEAD]) + step("Checking out origin/#{ee_branch_found}", %W[git checkout -b #{ee_branch_found} FETCH_HEAD]) - generate_patch(ee_branch, ee_patch_full_path) + generate_patch(ee_branch_found, ee_patch_full_path) unless check_patch(ee_patch_full_path).zero? puts @@ -111,41 +125,77 @@ module Gitlab end puts - puts applies_cleanly_msg(ee_branch) + puts applies_cleanly_msg(ee_branch_found) end def check_patch(patch_path) step("Checking out master", %w[git checkout master]) - step("Reseting to latest master", %w[git reset --hard origin/master]) - - step("Checking if #{patch_path} applies cleanly to EE/master") - output, status = Gitlab::Popen.popen(%W[git apply --check #{patch_path}]) - - unless status.zero? - failed_files = output.lines.reduce([]) do |memo, line| - if line.start_with?('error: patch failed:') - file = line.sub(/\Aerror: patch failed: /, '') - memo << file unless file =~ IGNORED_FILES_REGEX + step("Resetting to latest master", %w[git reset --hard origin/master]) + step( + "Checking if #{patch_path} applies cleanly to EE/master", + %W[git apply --check --3way #{patch_path}] + ) do |output, status| + unless status.zero? + @failed_files = output.lines.reduce([]) do |memo, line| + if line.start_with?('error: patch failed:') + file = line.sub(/\Aerror: patch failed: /, '') + memo << file unless file =~ IGNORED_FILES_REGEX + end + memo end - memo + + status = 0 if failed_files.empty? end - if failed_files.empty? - status = 0 - else - puts "\nConflicting files:" - failed_files.each do |file| - puts " - #{file}" - end + status + end + end + + def delete_ee_branches_locally! + command(%w[git checkout master]) + command(%W[git branch --delete --force #{ee_branch_prefix}]) + command(%W[git branch --delete --force #{ee_branch_suffix}]) + end + + def merge_base_found? + step( + "Finding merge base with master", + %w[git merge-base origin/master HEAD] + ) do |output, status| + if status.zero? + puts "Merge base was found: #{output}" + true end end + end + + def find_merge_base_with_master(branch:) + return if merge_base_found? + + # Start with (Math.exp(3).to_i = 20) until (Math.exp(6).to_i = 403) + # In total we go (20 + 54 + 148 + 403 = 625) commits deeper + depth = 20 + success = + (3..6).any? do |factor| + depth += Math.exp(factor).to_i + # Repository is initially cloned with a depth of 20 so we need to fetch + # deeper in the case the branch has more than 20 commits on top of master + fetch(branch: branch, depth: depth) + fetch(branch: 'master', depth: depth) + + merge_base_found? + end - status + raise "\n#{branch} is too far behind master, please rebase it!\n" unless success end - def delete_ee_branch_locally! - command(%w[git checkout master]) - step("Deleting the local #{ee_branch} branch", %W[git branch -D #{ee_branch}]) + def fetch(branch:, depth:) + step( + "Fetching deeper...", + %W[git fetch --depth=#{depth} --prune origin +refs/heads/#{branch}:refs/remotes/origin/#{branch}] + ) do |output, status| + raise "Fetch failed: #{output}" unless status.zero? + end end def ce_patch_name @@ -156,12 +206,16 @@ module Gitlab @ce_patch_full_path ||= patches_dir.join(ce_patch_name) end - def ee_branch - @ee_branch ||= "#{ce_branch}-ee" + def ee_branch_suffix + @ee_branch_suffix ||= "#{ce_branch}-ee" + end + + def ee_branch_prefix + @ee_branch_prefix ||= "ee-#{ce_branch}" end def ee_patch_name - @ee_patch_name ||= patch_name_from_branch(ee_branch) + @ee_patch_name ||= patch_name_from_branch(ee_branch_found) end def ee_patch_full_path @@ -178,98 +232,125 @@ module Gitlab if cmd start = Time.now puts "\n$ #{cmd.join(' ')}" - status = command(cmd) - puts "\nFinished in #{Time.now - start} seconds" - status + + output, status = command(cmd) + puts "\n==> Finished in #{Time.now - start} seconds" + + if block_given? + yield(output, status) + else + [output, status] + end end end def command(cmd) - output, status = Gitlab::Popen.popen(cmd) - puts output - - status + Gitlab::Popen.popen(cmd) end def applies_cleanly_msg(branch) - <<-MSG.strip_heredoc - ================================================================= + %Q{ + #{PLEASE_READ_THIS_BANNER} 🎉 Congratulations!! 🎉 - The #{branch} branch applies cleanly to EE/master! + The `#{branch}` branch applies cleanly to EE/master! - Much ❤️!! - =================================================================\n - MSG + Much ❤️! For more information, see + https://docs.gitlab.com/ce/development/limit_ee_conflicts.html#check-the-rake-ee_compat_check-in-your-merge-requests + #{THANKS_FOR_READING_BANNER} + } end def ce_branch_doesnt_apply_cleanly_and_no_ee_branch_msg - <<-MSG.strip_heredoc - ================================================================= + %Q{ + #{PLEASE_READ_THIS_BANNER} 💥 Oh no! 💥 - The #{ce_branch} branch does not apply cleanly to the current - EE/master, and no #{ee_branch} branch was found in the EE repository. + The `#{ce_branch}` branch does not apply cleanly to the current + EE/master, and no `#{ee_branch_prefix}` or `#{ee_branch_suffix}` branch + was found in the EE repository. - Please create a #{ee_branch} branch that includes changes from - #{ce_branch} but also specific changes than can be applied cleanly - to EE/master. + #{conflicting_files_msg} + + We advise you to create a `#{ee_branch_prefix}` or `#{ee_branch_suffix}` + branch that includes changes from `#{ce_branch}` but also specific changes + than can be applied cleanly to EE/master. In some cases, the conflicts + are trivial and you can ignore the warning from this job. As always, + use your best judgment! There are different ways to create such branch: - 1. Create a new branch based on the CE branch and rebase it on top of EE/master + 1. Create a new branch from master and cherry-pick your CE commits # In the EE repo - $ git fetch #{ce_repo} #{ce_branch} - $ git checkout -b #{ee_branch} FETCH_HEAD - - # You can squash the #{ce_branch} commits into a single "Port of #{ce_branch} to EE" commit - # before rebasing to limit the conflicts-resolving steps during the rebase $ git fetch origin - $ git rebase origin/master + $ git checkout -b #{ee_branch_prefix} origin/master + $ git fetch #{ce_repo} #{ce_branch} + $ git cherry-pick SHA # Repeat for all the commits you want to pick - At this point you will likely have conflicts. - Solve them, and continue/finish the rebase. + You can squash the `#{ce_branch}` commits into a single "Port of #{ce_branch} to EE" commit. - You can squash the #{ce_branch} commits into a single "Port of #{ce_branch} to EE". + 2. Apply your branch's patch to EE - 2. Create a new branch from master and cherry-pick your CE commits + # In the CE repo + $ git fetch origin master + $ git format-patch origin/master --stdout > #{ce_branch}.patch # In the EE repo - $ git fetch origin - $ git checkout -b #{ee_branch} origin/master - $ git fetch #{ce_repo} #{ce_branch} - $ git cherry-pick SHA # Repeat for all the commits you want to pick + $ git fetch origin master + $ git checkout -b #{ee_branch_prefix} origin/master + $ git apply --3way path/to/#{ce_branch}.patch + + At this point you might have conflicts such as: - You can squash the #{ce_branch} commits into a single "Port of #{ce_branch} to EE" commit. + error: patch failed: lib/gitlab/ee_compat_check.rb:5 + Falling back to three-way merge... + Applied patch to 'lib/gitlab/ee_compat_check.rb' with conflicts. + U lib/gitlab/ee_compat_check.rb - Don't forget to push your branch to #{EE_REPO}: + Resolve them, stage the changes and commit them. + + ⚠️ Don't forget to push your branch to gitlab-ee: # In the EE repo - $ git push origin #{ee_branch} + $ git push origin #{ee_branch_prefix} + + ⚠️ Also, don't forget to create a new merge request on gitlab-ce and + cross-link it with the CE merge request. - You can then retry this failed build, and hopefully it should pass. + Once this is done, you can retry this failed build, and it should pass. - Stay 💪 ! - =================================================================\n - MSG + Stay 💪 ! For more information, see + https://docs.gitlab.com/ce/development/limit_ee_conflicts.html#check-the-rake-ee_compat_check-in-your-merge-requests + #{THANKS_FOR_READING_BANNER} + } end def ee_branch_doesnt_apply_cleanly_msg - <<-MSG.strip_heredoc - ================================================================= + %Q{ + #{PLEASE_READ_THIS_BANNER} 💥 Oh no! 💥 - The #{ce_branch} does not apply cleanly to the current - EE/master, and even though a #{ee_branch} branch exists in the EE - repository, it does not apply cleanly either to EE/master! + The `#{ce_branch}` does not apply cleanly to the current EE/master, and + even though a `#{ee_branch_found}` branch + exists in the EE repository, it does not apply cleanly either to + EE/master! + + #{conflicting_files_msg} - Please update the #{ee_branch}, push it again to #{EE_REPO}, and + Please update the `#{ee_branch_found}`, push it again to gitlab-ee, and retry this build. - Stay 💪 ! - =================================================================\n - MSG + Stay 💪 ! For more information, see + https://docs.gitlab.com/ce/development/limit_ee_conflicts.html#check-the-rake-ee_compat_check-in-your-merge-requests + #{THANKS_FOR_READING_BANNER} + } + end + + def conflicting_files_msg + failed_files.reduce("The conflicts detected were as follows:\n") do |memo, file| + memo << "\n - #{file}" + end end end end diff --git a/lib/gitlab/email/handler.rb b/lib/gitlab/email/handler.rb index bd3267e2a80..35ea2e0ef59 100644 --- a/lib/gitlab/email/handler.rb +++ b/lib/gitlab/email/handler.rb @@ -1,10 +1,11 @@ require 'gitlab/email/handler/create_note_handler' require 'gitlab/email/handler/create_issue_handler' +require 'gitlab/email/handler/unsubscribe_handler' module Gitlab module Email module Handler - HANDLERS = [CreateNoteHandler, CreateIssueHandler] + HANDLERS = [UnsubscribeHandler, CreateNoteHandler, CreateIssueHandler].freeze def self.for(mail, mail_key) HANDLERS.find do |klass| diff --git a/lib/gitlab/email/handler/base_handler.rb b/lib/gitlab/email/handler/base_handler.rb index 7cccf465334..3f6ace0311a 100644 --- a/lib/gitlab/email/handler/base_handler.rb +++ b/lib/gitlab/email/handler/base_handler.rb @@ -9,52 +9,13 @@ module Gitlab @mail_key = mail_key end - def message - @message ||= process_message - end - - def author + def can_execute? raise NotImplementedError end - def project + def execute raise NotImplementedError end - - private - - def validate_permission!(permission) - raise UserNotFoundError unless author - raise UserBlockedError if author.blocked? - raise ProjectNotFound unless author.can?(:read_project, project) - raise UserNotAuthorizedError unless author.can?(permission, project) - end - - def process_message - message = ReplyParser.new(mail).execute.strip - add_attachments(message) - end - - def add_attachments(reply) - attachments = Email::AttachmentUploader.new(mail).execute(project) - - reply + attachments.map do |link| - "\n\n#{link[:markdown]}" - end.join - end - - def verify_record!(record:, invalid_exception:, record_name:) - return if record.persisted? - return if record.errors.key?(:commands_only) - - error_title = "The #{record_name} could not be created for the following reasons:" - - msg = error_title + record.errors.full_messages.map do |error| - "\n\n- #{error}" - end.join - - raise invalid_exception, msg - end end end end diff --git a/lib/gitlab/email/handler/create_issue_handler.rb b/lib/gitlab/email/handler/create_issue_handler.rb index 9f90a3ec2b2..b8ec9138c10 100644 --- a/lib/gitlab/email/handler/create_issue_handler.rb +++ b/lib/gitlab/email/handler/create_issue_handler.rb @@ -5,6 +5,7 @@ module Gitlab module Email module Handler class CreateIssueHandler < BaseHandler + include ReplyProcessing attr_reader :project_path, :incoming_email_token def initialize(mail, mail_key) @@ -33,7 +34,7 @@ module Gitlab end def project - @project ||= Project.find_with_namespace(project_path) + @project ||= Project.find_by_full_path(project_path) end private diff --git a/lib/gitlab/email/handler/create_note_handler.rb b/lib/gitlab/email/handler/create_note_handler.rb index 447c7a6a6b9..d87ba427f4b 100644 --- a/lib/gitlab/email/handler/create_note_handler.rb +++ b/lib/gitlab/email/handler/create_note_handler.rb @@ -1,10 +1,13 @@ require 'gitlab/email/handler/base_handler' +require 'gitlab/email/handler/reply_processing' module Gitlab module Email module Handler class CreateNoteHandler < BaseHandler + include ReplyProcessing + def can_handle? mail_key =~ /\A\w+\z/ end @@ -24,6 +27,8 @@ module Gitlab record_name: 'comment') end + private + def author sent_notification.recipient end @@ -36,8 +41,6 @@ module Gitlab @sent_notification ||= SentNotification.for(mail_key) end - private - def create_note Notes::CreateService.new( project, diff --git a/lib/gitlab/email/handler/reply_processing.rb b/lib/gitlab/email/handler/reply_processing.rb new file mode 100644 index 00000000000..32c5caf93e8 --- /dev/null +++ b/lib/gitlab/email/handler/reply_processing.rb @@ -0,0 +1,54 @@ +module Gitlab + module Email + module Handler + module ReplyProcessing + private + + def author + raise NotImplementedError + end + + def project + raise NotImplementedError + end + + def message + @message ||= process_message + end + + def process_message + message = ReplyParser.new(mail).execute.strip + add_attachments(message) + end + + def add_attachments(reply) + attachments = Email::AttachmentUploader.new(mail).execute(project) + + reply + attachments.map do |link| + "\n\n#{link[:markdown]}" + end.join + end + + def validate_permission!(permission) + raise UserNotFoundError unless author + raise UserBlockedError if author.blocked? + raise ProjectNotFound unless author.can?(:read_project, project) + raise UserNotAuthorizedError unless author.can?(permission, project) + end + + def verify_record!(record:, invalid_exception:, record_name:) + return if record.persisted? + return if record.errors.key?(:commands_only) + + error_title = "The #{record_name} could not be created for the following reasons:" + + msg = error_title + record.errors.full_messages.map do |error| + "\n\n- #{error}" + end.join + + raise invalid_exception, msg + end + end + end + end +end diff --git a/lib/gitlab/email/handler/unsubscribe_handler.rb b/lib/gitlab/email/handler/unsubscribe_handler.rb new file mode 100644 index 00000000000..97d7a8d65ff --- /dev/null +++ b/lib/gitlab/email/handler/unsubscribe_handler.rb @@ -0,0 +1,32 @@ +require 'gitlab/email/handler/base_handler' + +module Gitlab + module Email + module Handler + class UnsubscribeHandler < BaseHandler + def can_handle? + mail_key =~ /\A\w+#{Regexp.escape(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX)}\z/ + end + + def execute + raise SentNotificationNotFoundError unless sent_notification + return unless sent_notification.unsubscribable? + + noteable = sent_notification.noteable + raise NoteableNotFoundError unless noteable + noteable.unsubscribe(sent_notification.recipient) + end + + private + + def sent_notification + @sent_notification ||= SentNotification.for(reply_key) + end + + def reply_key + mail_key.sub(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX, '') + end + end + end + end +end diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb index 0e3b65fceb4..6c69cd9e6a9 100644 --- a/lib/gitlab/email/message/repository_push.rb +++ b/lib/gitlab/email/message/repository_push.rb @@ -46,7 +46,7 @@ module Gitlab end def diffs_count - diffs.size if diffs + diffs&.size end def compare @@ -58,7 +58,7 @@ module Gitlab end def compare_timeout - diffs.overflow? if diffs + diffs&.overflow? end def reverse_compare? diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index a40c44eb1bc..ec0529b5a4b 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -4,19 +4,19 @@ require_dependency 'gitlab/email/handler' # Inspired in great part by Discourse's Email::Receiver module Gitlab module Email - class ProcessingError < StandardError; end - class EmailUnparsableError < ProcessingError; end - class SentNotificationNotFoundError < ProcessingError; end - class ProjectNotFound < ProcessingError; end - class EmptyEmailError < ProcessingError; end - class AutoGeneratedEmailError < ProcessingError; end - class UserNotFoundError < ProcessingError; end - class UserBlockedError < ProcessingError; end - class UserNotAuthorizedError < ProcessingError; end - class NoteableNotFoundError < ProcessingError; end - class InvalidNoteError < ProcessingError; end - class InvalidIssueError < ProcessingError; end - class UnknownIncomingEmail < ProcessingError; end + ProcessingError = Class.new(StandardError) + EmailUnparsableError = Class.new(ProcessingError) + SentNotificationNotFoundError = Class.new(ProcessingError) + ProjectNotFound = Class.new(ProcessingError) + EmptyEmailError = Class.new(ProcessingError) + AutoGeneratedEmailError = Class.new(ProcessingError) + UserNotFoundError = Class.new(ProcessingError) + UserBlockedError = Class.new(ProcessingError) + UserNotAuthorizedError = Class.new(ProcessingError) + NoteableNotFoundError = Class.new(ProcessingError) + InvalidNoteError = Class.new(ProcessingError) + InvalidIssueError = Class.new(ProcessingError) + UnknownIncomingEmail = Class.new(ProcessingError) class Receiver def initialize(raw) @@ -35,6 +35,8 @@ module Gitlab handler.execute end + private + def build_mail Mail::Message.new(@raw) rescue Encoding::UndefinedConversionError, @@ -54,7 +56,24 @@ module Gitlab end def key_from_additional_headers(mail) - Array(mail.references).find do |mail_id| + references = ensure_references_array(mail.references) + + find_key_from_references(references) + end + + def ensure_references_array(references) + case references + when Array + references + when String + # Handle emails from clients which append with commas, + # example clients are Microsoft exchange and iOS app + Gitlab::IncomingEmail.scan_fallback_references(references) + end + end + + def find_key_from_references(references) + references.find do |mail_id| key = Gitlab::IncomingEmail.key_from_fallback_message_id(mail_id) break key if key end diff --git a/lib/gitlab/email/reply_parser.rb b/lib/gitlab/email/reply_parser.rb index 8c8dd1b9cef..558df87f36d 100644 --- a/lib/gitlab/email/reply_parser.rb +++ b/lib/gitlab/email/reply_parser.rb @@ -31,11 +31,12 @@ module Gitlab private def select_body(message) - if message.multipart? - part = message.text_part || message.html_part || message - else - part = message - end + part = + if message.multipart? + message.text_part || message.html_part || message + else + message + end decoded = fix_charset(part) diff --git a/lib/gitlab/emoji.rb b/lib/gitlab/emoji.rb index bbbca8acc40..a16d9fc2265 100644 --- a/lib/gitlab/emoji.rb +++ b/lib/gitlab/emoji.rb @@ -1,7 +1,7 @@ module Gitlab module Emoji extend self - + def emojis Gemojione.index.instance_variable_get(:@emoji_by_name) end @@ -18,6 +18,10 @@ module Gitlab emojis.keys end + def emojis_aliases + @emoji_aliases ||= JSON.parse(File.read(Rails.root.join('fixtures', 'emojis', 'aliases.json'))) + end + def emoji_filename(name) emojis[name]["unicode"] end @@ -25,5 +29,32 @@ module Gitlab def emoji_unicode_filename(moji) emojis_by_moji[moji]["unicode"] end + + def emoji_unicode_version(name) + @emoji_unicode_versions_by_name ||= JSON.parse(File.read(Rails.root.join('fixtures', 'emojis', 'emoji-unicode-version-map.json'))) + @emoji_unicode_versions_by_name[name] + end + + def normalize_emoji_name(name) + emojis_aliases[name] || name + end + + def emoji_image_tag(name, src) + "<img class='emoji' title=':#{name}:' alt=':#{name}:' src='#{src}' height='20' width='20' align='absmiddle' />" + end + + # CSS sprite fallback takes precedence over image fallback + def gl_emoji_tag(name) + emoji_name = emojis_aliases[name] || name + emoji_info = emojis[emoji_name] + return unless emoji_info + + data = { + name: emoji_name, + unicode_version: emoji_unicode_version(emoji_name) + } + + ActionController::Base.helpers.content_tag('gl-emoji', emoji_info['moji'], data: data) + end end end diff --git a/lib/gitlab/etag_caching/middleware.rb b/lib/gitlab/etag_caching/middleware.rb new file mode 100644 index 00000000000..9c98f0d1a30 --- /dev/null +++ b/lib/gitlab/etag_caching/middleware.rb @@ -0,0 +1,73 @@ +module Gitlab + module EtagCaching + class Middleware + RESERVED_WORDS = ProjectPathValidator::RESERVED.map { |word| "/#{word}/" }.join('|') + ROUTE_REGEXP = Regexp.union( + %r(^(?!.*(#{RESERVED_WORDS})).*/noteable/issue/\d+/notes\z) + ) + + def initialize(app) + @app = app + end + + def call(env) + return @app.call(env) unless enabled_for_current_route?(env) + Gitlab::Metrics.add_event(:etag_caching_middleware_used) + + etag, cached_value_present = get_etag(env) + if_none_match = env['HTTP_IF_NONE_MATCH'] + + if if_none_match == etag + handle_cache_hit(etag) + else + track_cache_miss(if_none_match, cached_value_present) + + status, headers, body = @app.call(env) + headers['ETag'] = etag + [status, headers, body] + end + end + + private + + def enabled_for_current_route?(env) + ROUTE_REGEXP.match(env['PATH_INFO']) + end + + def get_etag(env) + cache_key = env['PATH_INFO'] + store = Gitlab::EtagCaching::Store.new + current_value = store.get(cache_key) + cached_value_present = current_value.present? + + unless cached_value_present + current_value = store.touch(cache_key, only_if_missing: true) + end + + [weak_etag_format(current_value), cached_value_present] + end + + def weak_etag_format(value) + %Q{W/"#{value}"} + end + + def handle_cache_hit(etag) + Gitlab::Metrics.add_event(:etag_caching_cache_hit) + + status_code = Gitlab::PollingInterval.polling_enabled? ? 304 : 429 + + [status_code, { 'ETag' => etag }, ['']] + end + + def track_cache_miss(if_none_match, cached_value_present) + if if_none_match.blank? + Gitlab::Metrics.add_event(:etag_caching_header_missing) + elsif !cached_value_present + Gitlab::Metrics.add_event(:etag_caching_key_not_found) + else + Gitlab::Metrics.add_event(:etag_caching_resource_changed) + end + end + end + end +end diff --git a/lib/gitlab/etag_caching/store.rb b/lib/gitlab/etag_caching/store.rb new file mode 100644 index 00000000000..0039fc01c8f --- /dev/null +++ b/lib/gitlab/etag_caching/store.rb @@ -0,0 +1,32 @@ +module Gitlab + module EtagCaching + class Store + EXPIRY_TIME = 20.minutes + REDIS_NAMESPACE = 'etag:'.freeze + + def get(key) + Gitlab::Redis.with { |redis| redis.get(redis_key(key)) } + end + + def touch(key, only_if_missing: false) + etag = generate_etag + + Gitlab::Redis.with do |redis| + redis.set(redis_key(key), etag, ex: EXPIRY_TIME, nx: only_if_missing) + end + + etag + end + + private + + def generate_etag + SecureRandom.hex + end + + def redis_key(key) + "#{REDIS_NAMESPACE}#{key}" + end + end + end +end diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index 2dd42704396..62ddd45785d 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -10,7 +10,7 @@ module Gitlab # ExclusiveLease. # class ExclusiveLease - LUA_CANCEL_SCRIPT = <<-EOS + LUA_CANCEL_SCRIPT = <<-EOS.freeze local key, uuid = KEYS[1], ARGV[1] if redis.call("get", key) == uuid then redis.call("del", key) diff --git a/lib/gitlab/file_detector.rb b/lib/gitlab/file_detector.rb index 1d93a67dc56..c9ca4cadd1c 100644 --- a/lib/gitlab/file_detector.rb +++ b/lib/gitlab/file_detector.rb @@ -14,7 +14,7 @@ module Gitlab koding: '.koding.yml', gitlab_ci: '.gitlab-ci.yml', avatar: /\Alogo\.(png|jpg|gif)\z/ - } + }.freeze # Returns an Array of file types based on the given paths. # diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 3cd515e4a3a..d3df3f1bca1 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -6,7 +6,7 @@ module Gitlab class << self def ref_name(ref) - ref.gsub(/\Arefs\/(tags|heads)\//, '') + ref.sub(/\Arefs\/(tags|heads)\//, '') end def branch_name(ref) diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index b742d9e1e4b..e56eb0d3beb 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -93,163 +93,6 @@ module Gitlab commit_id: sha, ) end - - # Commit file in repository and return commit sha - # - # options should contain next structure: - # file: { - # content: 'Lorem ipsum...', - # path: 'documents/story.txt', - # update: true - # }, - # author: { - # email: 'user@example.com', - # name: 'Test User', - # time: Time.now - # }, - # committer: { - # email: 'user@example.com', - # name: 'Test User', - # time: Time.now - # }, - # commit: { - # message: 'Wow such commit', - # branch: 'master', - # update_ref: false - # } - # - # rubocop:disable Metrics/AbcSize - # rubocop:disable Metrics/CyclomaticComplexity - # rubocop:disable Metrics/PerceivedComplexity - def commit(repository, options, action = :add) - file = options[:file] - update = file[:update].nil? ? true : file[:update] - author = options[:author] - committer = options[:committer] - commit = options[:commit] - repo = repository.rugged - ref = commit[:branch] - update_ref = commit[:update_ref].nil? ? true : commit[:update_ref] - parents = [] - mode = 0o100644 - - unless ref.start_with?('refs/') - ref = 'refs/heads/' + ref - end - - path_name = Gitlab::Git::PathHelper.normalize_path(file[:path]) - # Abort if any invalid characters remain (e.g. ../foo) - raise Gitlab::Git::Repository::InvalidBlobName.new("Invalid path") if path_name.each_filename.to_a.include?('..') - - filename = path_name.to_s - index = repo.index - - unless repo.empty? - rugged_ref = repo.references[ref] - raise Gitlab::Git::Repository::InvalidRef.new("Invalid branch name") unless rugged_ref - last_commit = rugged_ref.target - index.read_tree(last_commit.tree) - parents = [last_commit] - end - - if action == :remove - index.remove(filename) - else - file_entry = index.get(filename) - - if action == :rename - old_path_name = Gitlab::Git::PathHelper.normalize_path(file[:previous_path]) - old_filename = old_path_name.to_s - file_entry = index.get(old_filename) - index.remove(old_filename) unless file_entry.blank? - end - - if file_entry - raise Gitlab::Git::Repository::InvalidBlobName.new("Filename already exists; update not allowed") unless update - - # Preserve the current file mode if one is available - mode = file_entry[:mode] if file_entry[:mode] - end - - content = file[:content] - detect = CharlockHolmes::EncodingDetector.new.detect(content) if content - - unless detect && detect[:type] == :binary - # When writing to the repo directly as we are doing here, - # the `core.autocrlf` config isn't taken into account. - content.gsub!("\r\n", "\n") if repository.autocrlf - end - - oid = repo.write(content, :blob) - index.add(path: filename, oid: oid, mode: mode) - end - - opts = {} - opts[:tree] = index.write_tree(repo) - opts[:author] = author - opts[:committer] = committer - opts[:message] = commit[:message] - opts[:parents] = parents - opts[:update_ref] = ref if update_ref - - Rugged::Commit.create(repo, opts) - end - # rubocop:enable Metrics/AbcSize - # rubocop:enable Metrics/CyclomaticComplexity - # rubocop:enable Metrics/PerceivedComplexity - - # Remove file from repository and return commit sha - # - # options should contain next structure: - # file: { - # path: 'documents/story.txt' - # }, - # author: { - # email: 'user@example.com', - # name: 'Test User', - # time: Time.now - # }, - # committer: { - # email: 'user@example.com', - # name: 'Test User', - # time: Time.now - # }, - # commit: { - # message: 'Remove FILENAME', - # branch: 'master' - # } - # - def remove(repository, options) - commit(repository, options, :remove) - end - - # Rename file from repository and return commit sha - # - # options should contain next structure: - # file: { - # previous_path: 'documents/old_story.txt' - # path: 'documents/story.txt' - # content: 'Lorem ipsum...', - # update: true - # }, - # author: { - # email: 'user@example.com', - # name: 'Test User', - # time: Time.now - # }, - # committer: { - # email: 'user@example.com', - # name: 'Test User', - # time: Time.now - # }, - # commit: { - # message: 'Rename FILENAME', - # branch: 'master' - # } - # - def rename(repository, options) - commit(repository, options, :rename) - end end def initialize(options) diff --git a/lib/gitlab/git/blob_snippet.rb b/lib/gitlab/git/blob_snippet.rb index e98de57fc22..d7975f88aaa 100644 --- a/lib/gitlab/git/blob_snippet.rb +++ b/lib/gitlab/git/blob_snippet.rb @@ -13,7 +13,7 @@ module Gitlab end def data - lines.join("\n") if lines + lines&.join("\n") end def name diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index d785516ebdd..3a73697dc5d 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -14,6 +14,8 @@ module Gitlab attr_accessor *SERIALIZE_KEYS # rubocop:disable Lint/AmbiguousOperator + delegate :tree, to: :raw_commit + def ==(other) return false unless other.is_a?(Gitlab::Git::Commit) @@ -218,10 +220,6 @@ module Gitlab raw_commit.parents.map { |c| Gitlab::Git::Commit.new(c) } end - def tree - raw_commit.tree - end - def stats Gitlab::Git::CommitStats.new(self) end diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index d6b3b5705a9..019be151353 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -2,7 +2,7 @@ module Gitlab module Git class Diff - class TimeoutError < StandardError; end + TimeoutError = Class.new(StandardError) include Gitlab::Git::EncodingHelper # Diff properties @@ -176,9 +176,13 @@ module Gitlab def initialize(raw_diff, collapse: false) case raw_diff when Hash - init_from_hash(raw_diff, collapse: collapse) + init_from_hash(raw_diff) + prune_diff_if_eligible(collapse) when Rugged::Patch, Rugged::Diff::Delta init_from_rugged(raw_diff, collapse: collapse) + when Gitaly::CommitDiffResponse + init_from_gitaly(raw_diff) + prune_diff_if_eligible(collapse) when nil raise "Nil as raw diff passed" else @@ -266,13 +270,26 @@ module Gitlab @diff = encode!(strip_diff_headers(patch.to_s)) end - def init_from_hash(hash, collapse: false) + def init_from_hash(hash) raw_diff = hash.symbolize_keys serialize_keys.each do |key| send(:"#{key}=", raw_diff[key.to_sym]) end + end + + def init_from_gitaly(diff_msg) + @diff = diff_msg.raw_chunks.join + @new_path = encode!(diff_msg.to_path.dup) + @old_path = encode!(diff_msg.from_path.dup) + @a_mode = diff_msg.old_mode.to_s(8) + @b_mode = diff_msg.new_mode.to_s(8) + @new_file = diff_msg.from_id == BLANK_SHA + @renamed_file = diff_msg.from_path != diff_msg.to_path + @deleted_file = diff_msg.to_id == BLANK_SHA + end + def prune_diff_if_eligible(collapse = false) prune_large_diff! if too_large? prune_collapsed_diff! if collapse && collapsible? end diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index 65e06f5065d..4e45ec7c174 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -30,7 +30,9 @@ module Gitlab elsif @deltas_only each_delta(&block) else - each_patch(&block) + Gitlab::GitalyClient.migrate(:commit_raw_diffs) do + each_patch(&block) + end end end diff --git a/lib/gitlab/git/index.rb b/lib/gitlab/git/index.rb new file mode 100644 index 00000000000..af1744c9c46 --- /dev/null +++ b/lib/gitlab/git/index.rb @@ -0,0 +1,126 @@ +module Gitlab + module Git + class Index + DEFAULT_MODE = 0o100644 + + attr_reader :repository, :raw_index + + def initialize(repository) + @repository = repository + @raw_index = repository.rugged.index + end + + delegate :read_tree, :get, to: :raw_index + + def write_tree + raw_index.write_tree(repository.rugged) + end + + def dir_exists?(path) + raw_index.find { |entry| entry[:path].start_with?("#{path}/") } + end + + def create(options) + options = normalize_options(options) + + file_entry = get(options[:file_path]) + if file_entry + raise Gitlab::Git::Repository::InvalidBlobName.new("Filename already exists") + end + + add_blob(options) + end + + def create_dir(options) + options = normalize_options(options) + + file_entry = get(options[:file_path]) + if file_entry + raise Gitlab::Git::Repository::InvalidBlobName.new("Directory already exists as a file") + end + + if dir_exists?(options[:file_path]) + raise Gitlab::Git::Repository::InvalidBlobName.new("Directory already exists") + end + + options = options.dup + options[:file_path] += '/.gitkeep' + options[:content] = '' + + add_blob(options) + end + + def update(options) + options = normalize_options(options) + + file_entry = get(options[:file_path]) + unless file_entry + raise Gitlab::Git::Repository::InvalidBlobName.new("File doesn't exist") + end + + add_blob(options, mode: file_entry[:mode]) + end + + def move(options) + options = normalize_options(options) + + file_entry = get(options[:previous_path]) + unless file_entry + raise Gitlab::Git::Repository::InvalidBlobName.new("File doesn't exist") + end + + raw_index.remove(options[:previous_path]) + + add_blob(options, mode: file_entry[:mode]) + end + + def delete(options) + options = normalize_options(options) + + file_entry = get(options[:file_path]) + unless file_entry + raise Gitlab::Git::Repository::InvalidBlobName.new("File doesn't exist") + end + + raw_index.remove(options[:file_path]) + end + + private + + def normalize_options(options) + options = options.dup + options[:file_path] = normalize_path(options[:file_path]) if options[:file_path] + options[:previous_path] = normalize_path(options[:previous_path]) if options[:previous_path] + options + end + + def normalize_path(path) + pathname = Gitlab::Git::PathHelper.normalize_path(path.dup) + + if pathname.each_filename.include?('..') + raise Gitlab::Git::Repository::InvalidBlobName.new('Invalid path') + end + + pathname.to_s + end + + def add_blob(options, mode: nil) + content = options[:content] + content = Base64.decode64(content) if options[:encoding] == 'base64' + + detect = CharlockHolmes::EncodingDetector.new.detect(content) + unless detect && detect[:type] == :binary + # When writing to the repo directly as we are doing here, + # the `core.autocrlf` config isn't taken into account. + content.gsub!("\r\n", "\n") if repository.autocrlf + end + + oid = repository.rugged.write(content, :blob) + + raw_index.add(path: options[:file_path], oid: oid, mode: mode || DEFAULT_MODE) + rescue Rugged::IndexError => e + raise Gitlab::Git::Repository::InvalidBlobName.new(e.message) + end + end + end +end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 7068e68a855..32aebb6f6f0 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -1,5 +1,4 @@ # Gitlab::Git::Repository is a wrapper around native Rugged::Repository object -require 'forwardable' require 'tempfile' require 'forwardable' require "rubygems/package" @@ -7,14 +6,13 @@ require "rubygems/package" module Gitlab module Git class Repository - extend Forwardable include Gitlab::Git::Popen SEARCH_CONTEXT_LINES = 3 - class NoRepository < StandardError; end - class InvalidBlobName < StandardError; end - class InvalidRef < StandardError; end + NoRepository = Class.new(StandardError) + InvalidBlobName = Class.new(StandardError) + InvalidRef = Class.new(StandardError) # Full path to repo attr_reader :path @@ -33,6 +31,10 @@ module Gitlab @attributes = Gitlab::Git::Attributes.new(path) end + delegate :empty?, + :bare?, + to: :rugged + # Default branch in the repository def root_ref @root_ref ||= discover_default_branch @@ -162,14 +164,6 @@ module Gitlab !empty? end - def empty? - rugged.empty? - end - - def bare? - rugged.bare? - end - def repo_exists? !!rugged end @@ -205,13 +199,17 @@ module Gitlab nil end + def archive_prefix(ref, sha) + project_name = self.name.chomp('.git') + "#{project_name}-#{ref.tr('/', '-')}-#{sha}" + end + def archive_metadata(ref, storage_path, format = "tar.gz") ref ||= root_ref commit = Gitlab::Git::Commit.find(self, ref) return {} if commit.nil? - project_name = self.name.chomp('.git') - prefix = "#{project_name}-#{ref}-#{commit.id}" + prefix = archive_prefix(ref, commit.id) { 'RepoPath' => path, @@ -322,7 +320,7 @@ module Gitlab def log_by_walk(sha, options) walk_options = { show: sha, - sort: Rugged::SORT_DATE, + sort: Rugged::SORT_NONE, limit: options[:limit], offset: options[:offset] } @@ -330,24 +328,47 @@ module Gitlab end def log_by_shell(sha, options) - cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path} log) - cmd += %W(-n #{options[:limit].to_i}) - cmd += %w(--format=%H) - cmd += %W(--skip=#{options[:offset].to_i}) - cmd += %w(--follow) if options[:follow] - cmd += %w(--no-merges) if options[:skip_merges] - cmd += %W(--after=#{options[:after].iso8601}) if options[:after] - cmd += %W(--before=#{options[:before].iso8601}) if options[:before] - cmd += [sha] - cmd += %W(-- #{options[:path]}) if options[:path].present? - - raw_output = IO.popen(cmd) {|io| io.read } - - log = raw_output.lines.map do |c| - Rugged::Commit.new(rugged, c.strip) + limit = options[:limit].to_i + offset = options[:offset].to_i + use_follow_flag = options[:follow] && options[:path].present? + + # We will perform the offset in Ruby because --follow doesn't play well with --skip. + # See: https://gitlab.com/gitlab-org/gitlab-ce/issues/3574#note_3040520 + offset_in_ruby = use_follow_flag && options[:offset].present? + limit += offset if offset_in_ruby + + cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} log] + cmd << "--max-count=#{limit}" + cmd << '--format=%H' + cmd << "--skip=#{offset}" unless offset_in_ruby + cmd << '--follow' if use_follow_flag + cmd << '--no-merges' if options[:skip_merges] + cmd << "--after=#{options[:after].iso8601}" if options[:after] + cmd << "--before=#{options[:before].iso8601}" if options[:before] + cmd << sha + + # :path can be a string or an array of strings + if options[:path].present? + cmd << '--' + cmd += Array(options[:path]) end - log.is_a?(Array) ? log : [] + raw_output = IO.popen(cmd) { |io| io.read } + lines = offset_in_ruby ? raw_output.lines.drop(offset) : raw_output.lines + + lines.map! { |c| Rugged::Commit.new(rugged, c.strip) } + end + + def count_commits(options) + cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} rev-list] + cmd << "--after=#{options[:after].iso8601}" if options[:after] + cmd << "--before=#{options[:before].iso8601}" if options[:before] + cmd += %W[--count #{options[:ref]}] + cmd += %W[-- #{options[:path]}] if options[:path].present? + + raw_output = IO.popen(cmd) { |io| io.read } + + raw_output.to_i end def sha_from_ref(ref) @@ -366,7 +387,7 @@ module Gitlab # a detailed list of valid arguments. def commits_between(from, to) walker = Rugged::Walker.new(rugged) - walker.sorting(Rugged::SORT_DATE | Rugged::SORT_REVERSE) + walker.sorting(Rugged::SORT_NONE | Rugged::SORT_REVERSE) sha_from = sha_from_ref(from) sha_to = sha_from_ref(to) @@ -390,6 +411,11 @@ module Gitlab rugged.merge_base(from, to) end + # Returns true is +from+ is direct ancestor to +to+, otherwise false + def is_ancestor?(from, to) + Gitlab::GitalyClient::Commit.is_ancestor(self, from, to) + end + # Return an array of Diff objects that represent the diff # between +from+ and +to+. See Diff::filter_diff_options for the allowed # diff options. The +options+ hash can also include :break_rewrites to @@ -444,7 +470,7 @@ module Gitlab if actual_options[:order] == :topo walker.sorting(Rugged::SORT_TOPO) else - walker.sorting(Rugged::SORT_DATE) + walker.sorting(Rugged::SORT_NONE) end commits = [] @@ -565,9 +591,7 @@ module Gitlab # will trigger a +:mixed+ reset and the working directory will be # replaced with the content of the index. (Untracked and ignored files # will be left alone) - def reset(ref, reset_type) - rugged.reset(ref, reset_type) - end + delegate :reset, to: :rugged # Mimic the `git clean` command and recursively delete untracked files. # Valid keys that can be passed in the +options+ hash are: @@ -814,23 +838,6 @@ module Gitlab Rugged::Commit.create(rugged, actual_options) end - def commits_since(from_date) - walker = Rugged::Walker.new(rugged) - walker.sorting(Rugged::SORT_DATE | Rugged::SORT_REVERSE) - - rugged.references.each("refs/heads/*") do |ref| - walker.push(ref.target_id) - end - - commits = [] - walker.each do |commit| - break if commit.author[:time].to_date < from_date - commits.push(commit) - end - - commits - end - AUTOCRLF_VALUES = { "true" => true, "false" => false, @@ -845,57 +852,6 @@ module Gitlab rugged.config['core.autocrlf'] = AUTOCRLF_VALUES.invert[value] end - # Create a new directory with a .gitkeep file. Creates - # all required nested directories (i.e. mkdir -p behavior) - # - # options should contain next structure: - # author: { - # email: 'user@example.com', - # name: 'Test User', - # time: Time.now - # }, - # committer: { - # email: 'user@example.com', - # name: 'Test User', - # time: Time.now - # }, - # commit: { - # message: 'Wow such commit', - # branch: 'master', - # update_ref: false - # } - def mkdir(path, options = {}) - # Check if this directory exists; if it does, then don't bother - # adding .gitkeep file. - ref = options[:commit][:branch] - path = Gitlab::Git::PathHelper.normalize_path(path).to_s - rugged_ref = rugged.ref(ref) - - raise InvalidRef.new("Invalid ref") if rugged_ref.nil? - - target_commit = rugged_ref.target - - raise InvalidRef.new("Invalid target commit") if target_commit.nil? - - entry = tree_entry(target_commit, path) - - if entry - if entry[:type] == :blob - raise InvalidBlobName.new("Directory already exists as a file") - else - raise InvalidBlobName.new("Directory already exists") - end - end - - options[:file] = { - content: '', - path: "#{path}/.gitkeep", - update: true - } - - Gitlab::Git::Blob.commit(self, options) - end - # Returns result like "git ls-files" , recursive and full file path # # Ex. diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index f7450e8b58f..b722d8a9f56 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -33,7 +33,7 @@ module Gitlab root_id: root_tree.oid, name: entry[:name], type: entry[:type], - mode: entry[:filemode], + mode: entry[:filemode].to_s(8), path: path ? File.join(path, entry[:name]) : entry[:name], commit_id: sha, ) diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 7e1484613f2..eea2f206902 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -10,10 +10,10 @@ module Gitlab deploy_key_upload: 'This deploy key does not have write access to this project.', no_repo: 'A repository for this project does not exist yet.' - } + }.freeze - DOWNLOAD_COMMANDS = %w{ git-upload-pack git-upload-archive } - PUSH_COMMANDS = %w{ git-receive-pack } + DOWNLOAD_COMMANDS = %w{ git-upload-pack git-upload-archive }.freeze + PUSH_COMMANDS = %w{ git-receive-pack }.freeze ALL_COMMANDS = DOWNLOAD_COMMANDS + PUSH_COMMANDS attr_reader :actor, :project, :protocol, :user_access, :authentication_abilities @@ -153,7 +153,9 @@ module Gitlab user_access: user_access, project: project, env: @env, - skip_authorization: deploy_key?).exec + skip_authorization: deploy_key?, + protocol: protocol + ).exec end def matching_merge_request?(newrev, branch_name) diff --git a/lib/gitlab/git_post_receive.rb b/lib/gitlab/git_post_receive.rb index d32bdd86427..6babea144c7 100644 --- a/lib/gitlab/git_post_receive.rb +++ b/lib/gitlab/git_post_receive.rb @@ -30,11 +30,11 @@ module Gitlab def retrieve_project_and_type @type = :project - @project = Project.find_with_namespace(@repo_path) + @project = Project.find_by_full_path(@repo_path) if @repo_path.end_with?('.wiki') && !@project @type = :wiki - @project = Project.find_with_namespace(@repo_path.gsub(/\.wiki\z/, '')) + @project = Project.find_by_full_path(@repo_path.gsub(/\.wiki\z/, '')) end end diff --git a/lib/gitlab/git_ref_validator.rb b/lib/gitlab/git_ref_validator.rb index 4d83d8e72a8..0e87ee30c98 100644 --- a/lib/gitlab/git_ref_validator.rb +++ b/lib/gitlab/git_ref_validator.rb @@ -5,6 +5,9 @@ module Gitlab # # Returns true for a valid reference name, false otherwise def validate(ref_name) + return false if ref_name.start_with?('refs/heads/') + return false if ref_name.start_with?('refs/remotes/') + Gitlab::Utils.system_silent( %W(#{Gitlab.config.git.bin_path} check-ref-format refs/#{ref_name})) end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb new file mode 100644 index 00000000000..fe15fb12adb --- /dev/null +++ b/lib/gitlab/gitaly_client.rb @@ -0,0 +1,54 @@ +require 'gitaly' + +module Gitlab + module GitalyClient + SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION'.freeze + + def self.configure_channel(storage, address) + @addresses ||= {} + @addresses[storage] = address + @channels ||= {} + @channels[storage] = new_channel(address) + end + + def self.new_channel(address) + address = address.sub(%r{^tcp://}, '') if URI(address).scheme == 'tcp' + # NOTE: When Gitaly runs on a Unix socket, permissions are + # handled using the file system and no additional authentication is + # required (therefore the :this_channel_is_insecure flag) + # TODO: Add authentication support when Gitaly is running on a TCP socket. + GRPC::Core::Channel.new(address, {}, :this_channel_is_insecure) + end + + def self.get_channel(storage) + @channels[storage] + end + + def self.get_address(storage) + @addresses[storage] + end + + def self.enabled? + Gitlab.config.gitaly.enabled + end + + def self.feature_enabled?(feature) + enabled? && ENV["GITALY_#{feature.upcase}"] == '1' + end + + def self.migrate(feature) + is_enabled = feature_enabled?(feature) + metric_name = feature.to_s + metric_name += "_gitaly" if is_enabled + + Gitlab::Metrics.measure(metric_name) do + yield is_enabled + end + end + + def self.expected_server_version + path = Rails.root.join(SERVER_VERSION_FILE) + path.read.chomp + end + end +end diff --git a/lib/gitlab/gitaly_client/commit.rb b/lib/gitlab/gitaly_client/commit.rb new file mode 100644 index 00000000000..f15faebe27e --- /dev/null +++ b/lib/gitlab/gitaly_client/commit.rb @@ -0,0 +1,41 @@ +module Gitlab + module GitalyClient + class Commit + # The ID of empty tree. + # See http://stackoverflow.com/a/40884093/1856239 and https://github.com/git/git/blob/3ad8b5bf26362ac67c9020bf8c30eee54a84f56d/cache.h#L1011-L1012 + EMPTY_TREE_ID = '4b825dc642cb6eb9a060e54bf8d69288fbee4904'.freeze + + class << self + def diff_from_parent(commit, options = {}) + project = commit.project + channel = GitalyClient.get_channel(project.repository_storage) + stub = Gitaly::Diff::Stub.new(nil, nil, channel_override: channel) + repo = Gitaly::Repository.new(path: project.repository.path_to_repo) + parent = commit.parents[0] + parent_id = parent ? parent.id : EMPTY_TREE_ID + request = Gitaly::CommitDiffRequest.new( + repository: repo, + left_commit_id: parent_id, + right_commit_id: commit.id + ) + + Gitlab::Git::DiffCollection.new(stub.commit_diff(request), options) + end + + def is_ancestor(repository, ancestor_id, child_id) + project = Project.find_by_path(repository.path) + channel = GitalyClient.get_channel(project.repository_storage) + stub = Gitaly::Commit::Stub.new(nil, nil, channel_override: channel) + repo = Gitaly::Repository.new(path: repository.path_to_repo) + request = Gitaly::CommitIsAncestorRequest.new( + repository: repo, + ancestor_id: ancestor_id, + child_id: child_id + ) + + stub.commit_is_ancestor(request).value + end + end + end + end +end diff --git a/lib/gitlab/gitaly_client/notifications.rb b/lib/gitlab/gitaly_client/notifications.rb new file mode 100644 index 00000000000..cbfb129c002 --- /dev/null +++ b/lib/gitlab/gitaly_client/notifications.rb @@ -0,0 +1,22 @@ +module Gitlab + module GitalyClient + class Notifications + attr_accessor :stub + + def initialize(repo_path) + full_path = Gitlab::RepoPath.strip_storage_path(repo_path). + sub(/\.git\z/, '').sub(/\.wiki\z/, '') + @project = Project.find_by_full_path(full_path) + + channel = GitalyClient.get_channel(@project.repository_storage) + @stub = Gitaly::Notifications::Stub.new(nil, nil, channel_override: channel) + end + + def post_receive + repository = Gitaly::Repository.new(path: @project.repository.path_to_repo) + request = Gitaly::PostReceiveRequest.new(repository: repository) + @stub.post_receive(request) + end + end + end +end diff --git a/lib/gitlab/github_import/base_formatter.rb b/lib/gitlab/github_import/base_formatter.rb index 95dba9a327b..8c80791e7c9 100644 --- a/lib/gitlab/github_import/base_formatter.rb +++ b/lib/gitlab/github_import/base_formatter.rb @@ -1,11 +1,12 @@ module Gitlab module GithubImport class BaseFormatter - attr_reader :formatter, :project, :raw_data + attr_reader :client, :formatter, :project, :raw_data - def initialize(project, raw_data) + def initialize(project, raw_data, client = nil) @project = project @raw_data = raw_data + @client = client @formatter = Gitlab::ImportFormatter.new end @@ -18,19 +19,6 @@ module Gitlab def url raw_data.url || '' end - - private - - def gitlab_user_id(github_id) - User.joins(:identities). - find_by("identities.extern_uid = ? AND identities.provider = 'github'", github_id.to_s). - try(:id) - end - - def gitlab_author_id - return @gitlab_author_id if defined?(@gitlab_author_id) - @gitlab_author_id = gitlab_user_id(raw_data.user.id) - end end end end diff --git a/lib/gitlab/github_import/branch_formatter.rb b/lib/gitlab/github_import/branch_formatter.rb index 0a8d05b5fe1..8aa885fb811 100644 --- a/lib/gitlab/github_import/branch_formatter.rb +++ b/lib/gitlab/github_import/branch_formatter.rb @@ -11,6 +11,14 @@ module Gitlab sha.present? && ref.present? end + def user + raw_data.user&.login || 'unknown' + end + + def short_sha + Commit.truncate_sha(sha) + end + private def branch_exists? @@ -18,7 +26,7 @@ module Gitlab end def commit_exists? - project.repository.commit(sha).present? + project.repository.branch_names_contains(sha).include?(ref) end def short_id diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index ba869faa92e..7dbeec5b010 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -10,6 +10,7 @@ module Gitlab @access_token = access_token @host = host.to_s.sub(%r{/+\z}, '') @api_version = api_version + @users = {} if access_token ::Octokit.auto_paginate = false @@ -64,6 +65,13 @@ module Gitlab api.respond_to?(method) || super end + def user(login) + return nil unless login.present? + return @users[login] if @users.key?(login) + + @users[login] = api.user(login) + end + private def api_endpoint diff --git a/lib/gitlab/github_import/comment_formatter.rb b/lib/gitlab/github_import/comment_formatter.rb index 2bddcde2b7c..e21922070c1 100644 --- a/lib/gitlab/github_import/comment_formatter.rb +++ b/lib/gitlab/github_import/comment_formatter.rb @@ -1,6 +1,8 @@ module Gitlab module GithubImport class CommentFormatter < BaseFormatter + attr_writer :author_id + def attributes { project: project, @@ -17,11 +19,11 @@ module Gitlab private def author - raw_data.user.login + @author ||= UserFormatter.new(client, raw_data.user) end def author_id - gitlab_author_id || project.creator_id + author.gitlab_id || project.creator_id end def body @@ -52,10 +54,10 @@ module Gitlab end def note - if gitlab_author_id + if author.gitlab_id body else - formatter.author_line(author) + body + formatter.author_line(author.login) + body end end diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index ec1318ab33c..a8c0b47e786 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -110,12 +110,12 @@ module Gitlab def import_issues fetch_resources(:issues, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |issues| issues.each do |raw| - gh_issue = IssueFormatter.new(project, raw) + gh_issue = IssueFormatter.new(project, raw, client) begin issuable = if gh_issue.pull_request? - MergeRequest.find_by_iid(gh_issue.number) + MergeRequest.find_by(target_project_id: project.id, iid: gh_issue.number) else gh_issue.create! end @@ -131,7 +131,8 @@ module Gitlab def import_pull_requests fetch_resources(:pull_requests, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |pull_requests| pull_requests.each do |raw| - gh_pull_request = PullRequestFormatter.new(project, raw) + gh_pull_request = PullRequestFormatter.new(project, raw, client) + next unless gh_pull_request.valid? begin @@ -156,7 +157,7 @@ module Gitlab end def restore_source_branch(pull_request) - project.repository.fetch_ref(repo_url, "pull/#{pull_request.number}/head", pull_request.source_branch_name) + project.repository.create_branch(pull_request.source_branch_name, pull_request.source_branch_sha) end def restore_target_branch(pull_request) @@ -170,6 +171,8 @@ module Gitlab end def clean_up_restored_branches(pull_request) + return if pull_request.opened? + remove_branch(pull_request.source_branch_name) unless pull_request.source_branch_exists? remove_branch(pull_request.target_branch_name) unless pull_request.target_branch_exists? end @@ -209,11 +212,17 @@ module Gitlab ActiveRecord::Base.no_touching do comments.each do |raw| begin - comment = CommentFormatter.new(project, raw) + comment = CommentFormatter.new(project, raw, client) + # GH does not return info about comment's parent, so we guess it by checking its URL! *_, parent, iid = URI(raw.html_url).path.split('/') - issuable_class = parent == 'issues' ? Issue : MergeRequest - issuable = issuable_class.find_by_iid(iid) + + issuable = if parent == 'issues' + Issue.find_by(project_id: project.id, iid: iid) + else + MergeRequest.find_by(target_project_id: project.id, iid: iid) + end + next unless issuable issuable.notes.create!(comment.attributes) @@ -278,7 +287,7 @@ module Gitlab def fetch_resources(resource_type, *opts) return if imported?(resource_type) - opts.last.merge!(page: current_page(resource_type)) + opts.last[:page] = current_page(resource_type) client.public_send(resource_type, *opts) do |resources| yield resources diff --git a/lib/gitlab/github_import/issuable_formatter.rb b/lib/gitlab/github_import/issuable_formatter.rb index 256f360efc7..27b171d6ddb 100644 --- a/lib/gitlab/github_import/issuable_formatter.rb +++ b/lib/gitlab/github_import/issuable_formatter.rb @@ -1,13 +1,13 @@ module Gitlab module GithubImport class IssuableFormatter < BaseFormatter + attr_writer :assignee_id, :author_id + def project_association raise NotImplementedError end - def number - raw_data.number - end + delegate :number, to: :raw_data def find_condition { iid: number } @@ -23,18 +23,24 @@ module Gitlab raw_data.assignee.present? end - def assignee_id + def author + @author ||= UserFormatter.new(client, raw_data.user) + end + + def author_id + @author_id ||= author.gitlab_id || project.creator_id + end + + def assignee if assigned? - gitlab_user_id(raw_data.assignee.id) + @assignee ||= UserFormatter.new(client, raw_data.assignee) end end - def author - raw_data.user.login - end + def assignee_id + return @assignee_id if defined?(@assignee_id) - def author_id - gitlab_author_id || project.creator_id + @assignee_id = assignee.try(:gitlab_id) end def body @@ -42,10 +48,10 @@ module Gitlab end def description - if gitlab_author_id + if author.gitlab_id body else - formatter.author_line(author) + body + formatter.author_line(author.login) + body end end diff --git a/lib/gitlab/github_import/project_creator.rb b/lib/gitlab/github_import/project_creator.rb index 3f635be22ba..a55adc9b1c8 100644 --- a/lib/gitlab/github_import/project_creator.rb +++ b/lib/gitlab/github_import/project_creator.rb @@ -1,6 +1,8 @@ module Gitlab module GithubImport class ProjectCreator + include Gitlab::CurrentSettings + attr_reader :repo, :name, :namespace, :current_user, :session_data, :type def initialize(repo, name, namespace, current_user, session_data, type: 'github') @@ -34,7 +36,7 @@ module Gitlab end def visibility_level - repo.private ? Gitlab::VisibilityLevel::PRIVATE : ApplicationSetting.current.default_project_visibility + repo.private ? Gitlab::VisibilityLevel::PRIVATE : current_application_settings.default_project_visibility end # diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb index 4ea0200e89b..150afa31432 100644 --- a/lib/gitlab/github_import/pull_request_formatter.rb +++ b/lib/gitlab/github_import/pull_request_formatter.rb @@ -1,8 +1,8 @@ module Gitlab module GithubImport class PullRequestFormatter < IssuableFormatter - delegate :exists?, :project, :ref, :repo, :sha, to: :source_branch, prefix: true - delegate :exists?, :project, :ref, :repo, :sha, to: :target_branch, prefix: true + delegate :user, :project, :ref, :repo, :sha, to: :source_branch, prefix: true + delegate :user, :exists?, :project, :ref, :repo, :sha, :short_sha, to: :target_branch, prefix: true def attributes { @@ -20,7 +20,8 @@ module Gitlab author_id: author_id, assignee_id: assignee_id, created_at: raw_data.created_at, - updated_at: raw_data.updated_at + updated_at: raw_data.updated_at, + imported: true } end @@ -37,9 +38,20 @@ module Gitlab end def source_branch_name - @source_branch_name ||= begin - source_branch_exists? ? source_branch_ref : "pull/#{number}/#{source_branch_ref}" - end + @source_branch_name ||= + if cross_project? || !source_branch_exists? + source_branch_name_prefixed + else + source_branch_ref + end + end + + def source_branch_name_prefixed + "gh-#{target_branch_short_sha}/#{number}/#{source_branch_user}/#{source_branch_ref}" + end + + def source_branch_exists? + !cross_project? && source_branch.exists? end def target_branch @@ -47,9 +59,21 @@ module Gitlab end def target_branch_name - @target_branch_name ||= begin - target_branch_exists? ? target_branch_ref : "pull/#{number}/#{target_branch_ref}" - end + @target_branch_name ||= target_branch_exists? ? target_branch_ref : target_branch_name_prefixed + end + + def target_branch_name_prefixed + "gl-#{target_branch_short_sha}/#{number}/#{target_branch_user}/#{target_branch_ref}" + end + + def cross_project? + return true if source_branch_repo.nil? + + source_branch_repo.id != target_branch_repo.id + end + + def opened? + state == 'opened' end private diff --git a/lib/gitlab/github_import/user_formatter.rb b/lib/gitlab/github_import/user_formatter.rb new file mode 100644 index 00000000000..04c2964da20 --- /dev/null +++ b/lib/gitlab/github_import/user_formatter.rb @@ -0,0 +1,45 @@ +module Gitlab + module GithubImport + class UserFormatter + attr_reader :client, :raw + + delegate :id, :login, to: :raw, allow_nil: true + + def initialize(client, raw) + @client = client + @raw = raw + end + + def gitlab_id + return @gitlab_id if defined?(@gitlab_id) + + @gitlab_id = find_by_external_uid || find_by_email + end + + private + + def email + @email ||= client.user(raw.login).try(:email) + end + + def find_by_email + return nil unless email + + User.find_by_any_email(email) + .try(:id) + end + + def find_by_external_uid + return nil unless id + + identities = ::Identity.arel_table + + User.select(:id) + .joins(:identities).where(identities[:provider].eq(:github) + .and(identities[:extern_uid].eq(id))) + .first + .try(:id) + end + end + end +end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 70ffb68c9ab..956e80b4a09 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -3,13 +3,13 @@ include SentryHelper module Gitlab module GonHelper def add_gon_variables - gon.api_version = API::API.version - gon.default_avatar_url = URI::join(Gitlab.config.gitlab.url, ActionController::Base.helpers.image_path('no_avatar.png')).to_s + gon.api_version = 'v4' + gon.default_avatar_url = URI.join(Gitlab.config.gitlab.url, ActionController::Base.helpers.image_path('no_avatar.png')).to_s gon.max_file_size = current_application_settings.max_attachment_size + gon.asset_host = ActionController::Base.asset_host gon.relative_url_root = Gitlab.config.gitlab.relative_url_root gon.shortcuts_path = help_page_path('shortcuts') gon.user_color_scheme = Gitlab::ColorSchemes.for_user(current_user).css_class - gon.award_menu_url = emojis_path gon.katex_css_url = ActionController::Base.helpers.asset_path('katex.css') gon.katex_js_url = ActionController::Base.helpers.asset_path('katex.js') gon.sentry_dsn = sentry_dsn_public if sentry_enabled? @@ -20,6 +20,7 @@ module Gitlab if current_user gon.current_user_id = current_user.id gon.current_username = current_user.username + gon.current_user_fullname = current_user.name end end end diff --git a/lib/gitlab/google_code_import/importer.rb b/lib/gitlab/google_code_import/importer.rb index 1f4edc36928..b02b9737493 100644 --- a/lib/gitlab/google_code_import/importer.rb +++ b/lib/gitlab/google_code_import/importer.rb @@ -310,7 +310,7 @@ module Gitlab if name == project.import_source "##{id}" else - "#{project.namespace.path}/#{name}##{id}" + "#{project.namespace.full_path}/#{name}##{id}" end text = "~~#{text}~~" if deleted text diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index 9360afedfcb..d787d5db4a0 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -14,7 +14,7 @@ module Gitlab end def initialize(blob_name, blob_content, repository: nil) - @formatter = Rouge::Formatters::HTMLGitlab.new + @formatter = Rouge::Formatters::HTMLGitlab @repository = repository @blob_name = blob_name @blob_content = blob_content @@ -28,7 +28,7 @@ module Gitlab hl_lexer = self.lexer end - @formatter.format(hl_lexer.lex(text, continue: continue)).html_safe + @formatter.format(hl_lexer.lex(text, continue: continue), tag: hl_lexer.tag).html_safe rescue @formatter.format(Rouge::Lexers::PlainText.lex(text)).html_safe end diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index eb667a85b78..8b327cfc226 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -3,7 +3,7 @@ module Gitlab extend self # For every version update, the version history in import_export.md has to be kept up to date. - VERSION = '0.1.5' + VERSION = '0.1.6'.freeze FILENAME_LIMIT = 50 def export_path(relative_path:) @@ -35,7 +35,7 @@ module Gitlab end def export_filename(project:) - basename = "#{Time.now.strftime('%Y-%m-%d_%H-%M-%3N')}_#{project.namespace.path}_#{project.path}" + basename = "#{Time.now.strftime('%Y-%m-%d_%H-%M-%3N')}_#{project.full_path.tr('/', '_')}" "#{basename[0..FILENAME_LIMIT]}_export.tar.gz" end diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index f00c7460e82..90942774a2e 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -15,14 +15,6 @@ module Gitlab execute(%W(#{git_bin_path} --git-dir=#{repo_path} bundle create #{bundle_path} --all)) end - def git_unbundle(repo_path:, bundle_path:) - execute(%W(#{git_bin_path} clone --bare #{bundle_path} #{repo_path})) - end - - def git_restore_hooks - execute(%W(#{Gitlab.config.gitlab_shell.path}/bin/create-hooks) + repository_storage_paths_args) - end - def mkdir_p(path) FileUtils.mkdir_p(path, mode: DEFAULT_MODE) FileUtils.chmod(DEFAULT_MODE, path) @@ -56,10 +48,6 @@ module Gitlab FileUtils.copy_entry(source, destination) true end - - def repository_storage_paths_args - Gitlab.config.repositories.storages.values - end end end end diff --git a/lib/gitlab/import_export/error.rb b/lib/gitlab/import_export/error.rb index e341c4d9cf8..788eedf2686 100644 --- a/lib/gitlab/import_export/error.rb +++ b/lib/gitlab/import_export/error.rb @@ -1,5 +1,5 @@ module Gitlab module ImportExport - class Error < StandardError; end + Error = Class.new(StandardError) end end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 08ad3274b38..ab74c8782f6 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -39,7 +39,6 @@ project_tree: - :author - :events - :statuses - - :variables - :triggers - :deploy_keys - :services @@ -74,6 +73,9 @@ excluded_attributes: - :milestone_id award_emoji: - :awardable_id + statuses: + - :trace + - :token methods: labels: @@ -82,6 +84,7 @@ methods: - :type statuses: - :type + - :gl_project_id services: - :type merge_request_diff: diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index e9ee47fc090..063ce74ecad 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -56,7 +56,7 @@ module Gitlab end def path_with_namespace - File.join(@project.namespace.path, @project.path) + File.join(@project.namespace.full_path, @project.path) end def repo_path diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index 2405b94db50..8b8e48aac76 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -32,6 +32,10 @@ module Gitlab @user.id end + def include?(old_author_id) + map.keys.include?(old_author_id) && map[old_author_id] != default_user_id + end + private def missing_keys_tracking_hash @@ -41,6 +45,8 @@ module Gitlab end def ensure_default_member! + @project.project_members.destroy_all + ProjectMember.create!(user: @user, access_level: ProjectMember::MASTER, source_id: @project.id, importing: true) end diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb index 2fbf437ec26..3473b466936 100644 --- a/lib/gitlab/import_export/project_tree_saver.rb +++ b/lib/gitlab/import_export/project_tree_saver.rb @@ -5,8 +5,9 @@ module Gitlab attr_reader :full_path - def initialize(project:, shared:) + def initialize(project:, current_user:, shared:) @project = project + @current_user = current_user @shared = shared @full_path = File.join(@shared.export_path, ImportExport.project_filename) end @@ -24,7 +25,35 @@ module Gitlab private def project_json_tree - @project.to_json(Gitlab::ImportExport::Reader.new(shared: @shared).project_tree) + project_json['project_members'] += group_members_json + + project_json.to_json + end + + def project_json + @project_json ||= @project.as_json(reader.project_tree) + end + + def reader + @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared) + end + + def group_members_json + group_members.as_json(reader.group_members_tree).each do |group_member| + group_member['source_type'] = 'Project' # Make group members project members of the future import + end + end + + def group_members + return [] unless @current_user.can?(:admin_group, @project.group) + + # We need `.where.not(user_id: nil)` here otherwise when a group has an + # invitee, it would make the following query return 0 rows since a NULL + # user_id would be present in the subquery + # See http://stackoverflow.com/questions/129077/not-in-clause-and-null-values + non_null_user_ids = @project.project_members.where.not(user_id: nil).select(:user_id) + + GroupMembersFinder.new(@project.group).execute.where.not(user_id: non_null_user_ids) end end end diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb index 5021a1a14ce..a1e7159fe42 100644 --- a/lib/gitlab/import_export/reader.rb +++ b/lib/gitlab/import_export/reader.rb @@ -21,6 +21,10 @@ module Gitlab false end + def group_members_tree + @attributes_finder.find_included(:project_members).merge(include: @attributes_finder.find(:user)) + end + private # Builds a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 19e43cce768..d44563333a5 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -4,7 +4,6 @@ module Gitlab OVERRIDES = { snippets: :project_snippets, pipelines: 'Ci::Pipeline', statuses: 'commit_status', - variables: 'Ci::Variable', triggers: 'Ci::Trigger', builds: 'Ci::Build', hooks: 'ProjectHook', @@ -16,7 +15,7 @@ module Gitlab USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id created_by_id merge_user_id resolved_by_id].freeze - PROJECT_REFERENCES = %w[project_id source_project_id gl_project_id target_project_id].freeze + PROJECT_REFERENCES = %w[project_id source_project_id target_project_id].freeze BUILD_MODELS = %w[Ci::Build commit_status].freeze @@ -24,6 +23,8 @@ module Gitlab EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels group_label group_labels].freeze + TOKEN_RESET_MODELS = %w[Ci::Trigger Ci::Build ProjectHook].freeze + def self.create(*args) new(*args).create end @@ -61,7 +62,9 @@ module Gitlab update_project_references handle_group_label if group_label? - reset_ci_tokens if @relation_name == 'Ci::Trigger' + reset_tokens! + remove_encrypted_attributes! + @relation_hash['data'].deep_symbolize_keys! if @relation_name == :events && @relation_hash['data'] set_st_diffs if @relation_name == :merge_request_diff end @@ -86,7 +89,7 @@ module Gitlab end def has_author?(old_author_id) - admin_user? && @members_mapper.map.keys.include?(old_author_id) + admin_user? && @members_mapper.include?(old_author_id) end def missing_author_note(updated_at, author_name) @@ -95,12 +98,11 @@ module Gitlab end def generate_imported_object - if BUILD_MODELS.include?(@relation_name) # call #trace= method after assigning the other attributes - trace = @relation_hash.delete('trace') + if BUILD_MODELS.include?(@relation_name) + @relation_hash.delete('trace') # old export files have trace @relation_hash.delete('token') imported_object do |object| - object.trace = trace object.commit_id = nil end else @@ -118,7 +120,6 @@ module Gitlab # project_id may not be part of the export, but we always need to populate it if required. @relation_hash['project_id'] = project_id - @relation_hash['gl_project_id'] = project_id if @relation_hash['gl_project_id'] @relation_hash['target_project_id'] = project_id if @relation_hash['target_project_id'] end @@ -140,11 +141,22 @@ module Gitlab end end - def reset_ci_tokens - return unless Gitlab::ImportExport.reset_tokens? + def reset_tokens! + return unless Gitlab::ImportExport.reset_tokens? && TOKEN_RESET_MODELS.include?(@relation_name.to_s) # If we import/export a project to the same instance, tokens will have to be reset. - @relation_hash['token'] = nil + # We also have to reset them to avoid issues when the gitlab secrets file cannot be copied across. + relation_class.attribute_names.select { |name| name.include?('token') }.each do |token| + @relation_hash[token] = nil + end + end + + def remove_encrypted_attributes! + return unless relation_class.respond_to?(:encrypted_attributes) && relation_class.encrypted_attributes.any? + + relation_class.encrypted_attributes.each_key do |key| + @relation_hash[key.to_s] = nil + end end def relation_class diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index 48a9a6fa5e2..c824d3ea9fc 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -2,6 +2,7 @@ module Gitlab module ImportExport class RepoRestorer include Gitlab::ImportExport::CommandLineUtil + include Gitlab::ShellAdapter def initialize(project:, shared:, path_to_bundle:) @project = project @@ -12,29 +13,11 @@ module Gitlab def restore return true unless File.exist?(@path_to_bundle) - mkdir_p(path_to_repo) - - git_unbundle(repo_path: path_to_repo, bundle_path: @path_to_bundle) && repo_restore_hooks + gitlab_shell.import_repository(@project.repository_storage_path, @project.path_with_namespace, @path_to_bundle) rescue => e @shared.error(e) false end - - private - - def path_to_repo - @project.repository.path_to_repo - end - - def repo_restore_hooks - return true if wiki? - - git_restore_hooks - end - - def wiki? - @project.class.name == 'ProjectWiki' - end end end end diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index 45958710c13..52276cbcd9a 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -5,8 +5,6 @@ # module Gitlab module ImportSources - extend CurrentSettings - ImportSource = Struct.new(:name, :title, :importer) ImportTable = [ diff --git a/lib/gitlab/incoming_email.rb b/lib/gitlab/incoming_email.rb index 801dfde9a36..c9122a23568 100644 --- a/lib/gitlab/incoming_email.rb +++ b/lib/gitlab/incoming_email.rb @@ -1,10 +1,9 @@ module Gitlab module IncomingEmail + UNSUBSCRIBE_SUFFIX = '+unsubscribe'.freeze WILDCARD_PLACEHOLDER = '%{key}'.freeze class << self - FALLBACK_MESSAGE_ID_REGEX = /\Areply\-(.+)@#{Gitlab.config.gitlab.host}\Z/.freeze - def enabled? config.enabled && config.address end @@ -18,7 +17,11 @@ module Gitlab end def reply_address(key) - config.address.gsub(WILDCARD_PLACEHOLDER, key) + config.address.sub(WILDCARD_PLACEHOLDER, key) + end + + def unsubscribe_address(key) + config.address.sub(WILDCARD_PLACEHOLDER, "#{key}#{UNSUBSCRIBE_SUFFIX}") end def key_from_address(address) @@ -32,10 +35,14 @@ module Gitlab end def key_from_fallback_message_id(mail_id) - match = mail_id.match(FALLBACK_MESSAGE_ID_REGEX) - return unless match + message_id_regexp = /\Areply\-(.+)@#{Gitlab.config.gitlab.host}\z/ - match[1] + mail_id[message_id_regexp, 1] + end + + def scan_fallback_references(references) + # It's looking for each <...> + references.scan(/(?!<)[^<>]+(?=>)/) end def config @@ -49,7 +56,7 @@ module Gitlab return nil unless wildcard_address regex = Regexp.escape(wildcard_address) - regex = regex.gsub(Regexp.escape('%{key}'), "(.+)") + regex = regex.sub(Regexp.escape(WILDCARD_PLACEHOLDER), '(.+)') Regexp.new(regex).freeze end end diff --git a/lib/gitlab/job_waiter.rb b/lib/gitlab/job_waiter.rb new file mode 100644 index 00000000000..8db91d25a4b --- /dev/null +++ b/lib/gitlab/job_waiter.rb @@ -0,0 +1,27 @@ +module Gitlab + # JobWaiter can be used to wait for a number of Sidekiq jobs to complete. + class JobWaiter + # The sleep interval between checking keys, in seconds. + INTERVAL = 0.1 + + # jobs - The job IDs to wait for. + def initialize(jobs) + @jobs = jobs + end + + # Waits for all the jobs to be completed. + # + # timeout - The maximum amount of seconds to block the caller for. This + # ensures we don't indefinitely block a caller in case a job takes + # long to process, or is never processed. + def wait(timeout = 60) + start = Time.current + + while (Time.current - start) <= timeout + break if SidekiqStatus.all_completed?(@jobs) + + sleep(INTERVAL) # to not overload Redis too much. + end + end + end +end diff --git a/lib/gitlab/kubernetes.rb b/lib/gitlab/kubernetes.rb index 288771c1c12..3a7af363548 100644 --- a/lib/gitlab/kubernetes.rb +++ b/lib/gitlab/kubernetes.rb @@ -43,10 +43,10 @@ module Gitlab end end - def add_terminal_auth(terminal, token, ca_pem = nil) + def add_terminal_auth(terminal, token:, max_session_time:, ca_pem: nil) terminal[:headers]['Authorization'] << "Bearer #{token}" + terminal[:max_session_time] = max_session_time terminal[:ca_pem] = ca_pem if ca_pem.present? - terminal end def container_exec_url(api_url, namespace, pod_name, container_name) diff --git a/lib/gitlab/ldap/config.rb b/lib/gitlab/ldap/config.rb index 28129198438..46deea3cc9f 100644 --- a/lib/gitlab/ldap/config.rb +++ b/lib/gitlab/ldap/config.rb @@ -124,9 +124,9 @@ module Gitlab def name_proc if allow_username_or_email_login - Proc.new { |name| name.gsub(/@.*\z/, '') } + proc { |name| name.gsub(/@.*\z/, '') } else - Proc.new { |name| name } + proc { |name| name } end end diff --git a/lib/gitlab/ldap/person.rb b/lib/gitlab/ldap/person.rb index 7084fd1767d..43eb73250b7 100644 --- a/lib/gitlab/ldap/person.rb +++ b/lib/gitlab/ldap/person.rb @@ -43,9 +43,7 @@ module Gitlab attribute_value(:email) end - def dn - entry.dn - end + delegate :dn, to: :entry private diff --git a/lib/gitlab/ldap/user.rb b/lib/gitlab/ldap/user.rb index b84c81f1a6c..2d5e47a6f3b 100644 --- a/lib/gitlab/ldap/user.rb +++ b/lib/gitlab/ldap/user.rb @@ -1,5 +1,3 @@ -require 'gitlab/o_auth/user' - # LDAP extension for User model # # * Find or create user from omniauth.auth data diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index 3d1ba33ec68..857e0abf710 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -112,7 +112,7 @@ module Gitlab def self.tag_transaction(name, value) trans = current_transaction - trans.add_tag(name, value) if trans + trans&.add_tag(name, value) end # Sets the action of the current transaction (if any) @@ -121,7 +121,7 @@ module Gitlab def self.action=(action) trans = current_transaction - trans.action = action if trans + trans&.action = action end # Tracks an event. @@ -130,7 +130,7 @@ module Gitlab def self.add_event(*args) trans = current_transaction - trans.add_event(*args) if trans + trans&.add_event(*args) end # Returns the prefix to use for the name of a series. diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb index 4b7a791e497..6aa38542cb4 100644 --- a/lib/gitlab/metrics/instrumentation.rb +++ b/lib/gitlab/metrics/instrumentation.rb @@ -143,11 +143,12 @@ module Gitlab # signature this would break things. As a result we'll make sure the # generated method _only_ accepts regular arguments if the underlying # method also accepts them. - if method.arity == 0 - args_signature = '' - else - args_signature = '*args' - end + args_signature = + if method.arity == 0 + '' + else + '*args' + end proxy_module.class_eval <<-EOF, __FILE__, __LINE__ + 1 def #{name}(#{args_signature}) diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb index 47f88727fc8..adc0db1a874 100644 --- a/lib/gitlab/metrics/rack_middleware.rb +++ b/lib/gitlab/metrics/rack_middleware.rb @@ -2,8 +2,8 @@ module Gitlab module Metrics # Rack middleware for tracking Rails and Grape requests. class RackMiddleware - CONTROLLER_KEY = 'action_controller.instance' - ENDPOINT_KEY = 'api.endpoint' + CONTROLLER_KEY = 'action_controller.instance'.freeze + ENDPOINT_KEY = 'api.endpoint'.freeze CONTENT_TYPES = { 'text/html' => :html, 'text/plain' => :txt, @@ -14,7 +14,7 @@ module Gitlab 'image/jpeg' => :jpeg, 'image/gif' => :gif, 'image/svg+xml' => :svg - } + }.freeze def initialize(app) @app = app diff --git a/lib/gitlab/metrics/subscribers/action_view.rb b/lib/gitlab/metrics/subscribers/action_view.rb index 2e9dd4645e3..d435a33e9c7 100644 --- a/lib/gitlab/metrics/subscribers/action_view.rb +++ b/lib/gitlab/metrics/subscribers/action_view.rb @@ -5,7 +5,7 @@ module Gitlab class ActionView < ActiveSupport::Subscriber attach_to :action_view - SERIES = 'views' + SERIES = 'views'.freeze def render_template(event) track(event) if current_transaction diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb index 287b7a83547..3aaebb3e9c3 100644 --- a/lib/gitlab/metrics/system.rb +++ b/lib/gitlab/metrics/system.rb @@ -11,7 +11,7 @@ module Gitlab mem = 0 match = File.read('/proc/self/status').match(/VmRSS:\s+(\d+)/) - if match and match[1] + if match && match[1] mem = match[1].to_f * 1024 end diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index 7bc16181be6..4f9fb1c7853 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -5,7 +5,7 @@ module Gitlab THREAD_KEY = :_gitlab_metrics_transaction # The series to store events (e.g. Git pushes) in. - EVENT_SERIES = 'events' + EVENT_SERIES = 'events'.freeze attr_reader :tags, :values, :method, :metrics diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb index 5764ab15652..6023fa1820f 100644 --- a/lib/gitlab/middleware/go.rb +++ b/lib/gitlab/middleware/go.rb @@ -30,21 +30,69 @@ module Gitlab end def go_body(request) - base_url = Gitlab.config.gitlab.url - # Go subpackages may be in the form of namespace/project/path1/path2/../pathN - # We can just ignore the paths and leave the namespace/project - path_info = request.env["PATH_INFO"] - path_info.sub!(/^\//, '') - project_path = path_info.split('/').first(2).join('/') - request_url = URI.join(base_url, project_path) - domain_path = strip_url(request_url.to_s) + project_url = URI.join(Gitlab.config.gitlab.url, project_path(request)) + import_prefix = strip_url(project_url.to_s) - "<!DOCTYPE html><html><head><meta content='#{domain_path} git #{request_url}.git' name='go-import'></head></html>\n" + "<!DOCTYPE html><html><head><meta content='#{import_prefix} git #{project_url}.git' name='go-import'></head></html>\n" end def strip_url(url) url.gsub(/\Ahttps?:\/\//, '') end + + def project_path(request) + path_info = request.env["PATH_INFO"] + path_info.sub!(/^\//, '') + + # Go subpackages may be in the form of `namespace/project/path1/path2/../pathN`. + # In a traditional project with a single namespace, this would denote repo + # `namespace/project` with subpath `path1/path2/../pathN`, but with nested + # groups, this could also be `namespace/project/path1` with subpath + # `path2/../pathN`, for example. + + # We find all potential project paths out of the path segments + path_segments = path_info.split('/') + simple_project_path = path_segments.first(2).join('/') + + # If the path is at most 2 segments long, it is a simple `namespace/project` path and we're done + return simple_project_path if path_segments.length <= 2 + + project_paths = [] + begin + project_paths << path_segments.join('/') + path_segments.pop + end while path_segments.length >= 2 + + # We see if a project exists with any of these potential paths + project = project_for_paths(project_paths, request) + + if project + # If a project is found and the user has access, we return the full project path + project.full_path + else + # If not, we return the first two components as if it were a simple `namespace/project` path, + # so that we don't reveal the existence of a nested project the user doesn't have access to. + # This means that for an unauthenticated request to `group/subgroup/project/subpackage` + # for a private `group/subgroup/project` with subpackage path `subpackage`, GitLab will respond + # as if the user is looking for project `group/subgroup`, with subpackage path `project/subpackage`. + # Since `go get` doesn't authenticate by default, this means that + # `go get gitlab.com/group/subgroup/project/subpackage` will not work for private projects. + # `go get gitlab.com/group/subgroup/project.git/subpackage` will work, since Go is smart enough + # to figure that out. `import 'gitlab.com/...'` behaves the same as `go get`. + simple_project_path + end + end + + def project_for_paths(paths, request) + project = Project.where_full_path_in(paths).first + return unless Ability.allowed?(current_user(request), :read_project, project) + + project + end + + def current_user(request) + request.env['warden']&.authenticate + end end end end diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb index dd99f9bb7d7..fee741b47be 100644 --- a/lib/gitlab/middleware/multipart.rb +++ b/lib/gitlab/middleware/multipart.rb @@ -26,7 +26,7 @@ module Gitlab module Middleware class Multipart - RACK_ENV_KEY = 'HTTP_GITLAB_WORKHORSE_MULTIPART_FIELDS' + RACK_ENV_KEY = 'HTTP_GITLAB_WORKHORSE_MULTIPART_FIELDS'.freeze class Handler def initialize(env, message) diff --git a/lib/gitlab/middleware/webpack_proxy.rb b/lib/gitlab/middleware/webpack_proxy.rb new file mode 100644 index 00000000000..6105d165810 --- /dev/null +++ b/lib/gitlab/middleware/webpack_proxy.rb @@ -0,0 +1,24 @@ +# This Rack middleware is intended to proxy the webpack assets directory to the +# webpack-dev-server. It is only intended for use in development. + +module Gitlab + module Middleware + class WebpackProxy < Rack::Proxy + def initialize(app = nil, opts = {}) + @proxy_host = opts.fetch(:proxy_host, 'localhost') + @proxy_port = opts.fetch(:proxy_port, 3808) + @proxy_path = opts[:proxy_path] if opts[:proxy_path] + + super(app, backend: "http://#{@proxy_host}:#{@proxy_port}", **opts) + end + + def perform_request(env) + if @proxy_path && env['PATH_INFO'].start_with?("/#{@proxy_path}") + super(env) + else + @app.call(env) + end + end + end + end +end diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index 96ed20af918..f98481c6d3a 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -5,7 +5,7 @@ # module Gitlab module OAuth - class SignupDisabledError < StandardError; end + SignupDisabledError = Class.new(StandardError) class User attr_accessor :auth_hash, :gl_user @@ -29,12 +29,11 @@ module Gitlab def save(provider = 'OAuth') unauthorized_to_create unless gl_user - if needs_blocking? - gl_user.save! - gl_user.block - else - gl_user.save! - end + block_after_save = needs_blocking? + + gl_user.save! + + gl_user.block if block_after_save log.info "(#{provider}) saving user #{auth_hash.email} from login with extern_uid => #{auth_hash.uid}" gl_user @@ -148,10 +147,8 @@ module Gitlab end def build_new_user - user = ::User.new(user_attributes) - user.skip_confirmation! - user.identities.new(extern_uid: auth_hash.uid, provider: auth_hash.provider) - user + user_params = user_attributes.merge(extern_uid: auth_hash.uid, provider: auth_hash.provider, skip_confirmation: true) + Users::CreateService.new(nil, user_params).build end def user_attributes diff --git a/lib/gitlab/optimistic_locking.rb b/lib/gitlab/optimistic_locking.rb index 879d46446b3..962ff4d3985 100644 --- a/lib/gitlab/optimistic_locking.rb +++ b/lib/gitlab/optimistic_locking.rb @@ -1,12 +1,12 @@ module Gitlab module OptimisticLocking - extend self + module_function def retry_lock(subject, retries = 100, &block) loop do begin ActiveRecord::Base.transaction do - return block.call(subject) + return yield(subject) end rescue ActiveRecord::StaleObjectError retries -= 1 @@ -15,5 +15,7 @@ module Gitlab end end end + + alias_method :retry_optimistic_lock, :retry_lock end end diff --git a/lib/gitlab/other_markup.rb b/lib/gitlab/other_markup.rb index 4e2f8ed5587..e67acf28c94 100644 --- a/lib/gitlab/other_markup.rb +++ b/lib/gitlab/other_markup.rb @@ -17,6 +17,9 @@ module Gitlab html = Banzai.post_process(html, context) + filter = Banzai::Filter::SanitizationFilter.new(html) + html = filter.call.to_s + html.html_safe end end diff --git a/lib/gitlab/pages_transfer.rb b/lib/gitlab/pages_transfer.rb new file mode 100644 index 00000000000..fb215f27cbd --- /dev/null +++ b/lib/gitlab/pages_transfer.rb @@ -0,0 +1,7 @@ +module Gitlab + class PagesTransfer < ProjectTransfer + def root_dir + Gitlab.config.pages.path + end + end +end diff --git a/lib/gitlab/polling_interval.rb b/lib/gitlab/polling_interval.rb new file mode 100644 index 00000000000..c44bb1cd14d --- /dev/null +++ b/lib/gitlab/polling_interval.rb @@ -0,0 +1,22 @@ +module Gitlab + class PollingInterval + include Gitlab::CurrentSettings + + HEADER_NAME = 'Poll-Interval'.freeze + + def self.set_header(response, interval:) + if polling_enabled? + multiplier = current_application_settings.polling_interval_multiplier + value = (interval * multiplier).to_i + else + value = -1 + end + + response.headers[HEADER_NAME] = value + end + + def self.polling_enabled? + !current_application_settings.polling_interval_multiplier.zero? + end + end +end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 6bdf3db9cb8..db325c00705 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -71,6 +71,14 @@ module Gitlab ) end + def single_commit_result? + commits_count == 1 && total_result_count == 1 + end + + def total_result_count + issues_count + merge_requests_count + milestones_count + notes_count + blobs_count + wiki_blobs_count + commits_count + end + private def blobs @@ -114,7 +122,25 @@ module Gitlab end def commits - @commits ||= project.repository.find_commits_by_message(query) + @commits ||= find_commits(query) + end + + def find_commits(query) + return [] unless Ability.allowed?(@current_user, :download_code, @project) + + commits = find_commits_by_message(query) + commit_by_sha = find_commit_by_sha(query) + commits |= [commit_by_sha] if commit_by_sha + commits + end + + def find_commits_by_message(query) + project.repository.find_commits_by_message(query) + end + + def find_commit_by_sha(query) + key = query.strip + project.repository.commit(key) if Commit.valid_hash?(key) end def project_ids_relation diff --git a/lib/gitlab/project_transfer.rb b/lib/gitlab/project_transfer.rb new file mode 100644 index 00000000000..1bba0b78e2f --- /dev/null +++ b/lib/gitlab/project_transfer.rb @@ -0,0 +1,35 @@ +module Gitlab + class ProjectTransfer + def move_project(project_path, namespace_path_was, namespace_path) + new_namespace_folder = File.join(root_dir, namespace_path) + FileUtils.mkdir_p(new_namespace_folder) unless Dir.exist?(new_namespace_folder) + from = File.join(root_dir, namespace_path_was, project_path) + to = File.join(root_dir, namespace_path, project_path) + move(from, to, "") + end + + def rename_project(path_was, path, namespace_path) + base_dir = File.join(root_dir, namespace_path) + move(path_was, path, base_dir) + end + + def rename_namespace(path_was, path) + move(path_was, path) + end + + def root_dir + raise NotImplementedError + end + + private + + def move(path_was, path, base_dir = nil) + base_dir = root_dir unless base_dir + from = File.join(base_dir, path_was) + to = File.join(base_dir, path) + FileUtils.mv(from, to) + rescue Errno::ENOENT + false + end + end +end diff --git a/lib/gitlab/prometheus.rb b/lib/gitlab/prometheus.rb new file mode 100644 index 00000000000..62239779454 --- /dev/null +++ b/lib/gitlab/prometheus.rb @@ -0,0 +1,70 @@ +module Gitlab + PrometheusError = Class.new(StandardError) + + # Helper methods to interact with Prometheus network services & resources + class Prometheus + attr_reader :api_url + + def initialize(api_url:) + @api_url = api_url + end + + def ping + json_api_get('query', query: '1') + end + + def query(query) + get_result('vector') do + json_api_get('query', query: query) + end + end + + def query_range(query, start: 8.hours.ago) + get_result('matrix') do + json_api_get('query_range', + query: query, + start: start.to_f, + end: Time.now.utc.to_f, + step: 1.minute.to_i) + end + end + + private + + def json_api_get(type, args = {}) + get(join_api_url(type, args)) + rescue Errno::ECONNREFUSED + raise PrometheusError, 'Connection refused' + end + + def join_api_url(type, args = {}) + url = URI.parse(api_url) + rescue URI::Error + raise PrometheusError, "Invalid API URL: #{api_url}" + else + url.path = [url.path.sub(%r{/+\z}, ''), 'api', 'v1', type].join('/') + url.query = args.to_query + + url.to_s + end + + def get(url) + handle_response(HTTParty.get(url)) + end + + def handle_response(response) + if response.code == 200 && response['status'] == 'success' + response['data'] || {} + elsif response.code == 400 + raise PrometheusError, response['error'] || 'Bad data received' + else + raise PrometheusError, "#{response.code} - #{response.body}" + end + end + + def get_result(expected_type) + data = yield + data['result'] if data['resultType'] == expected_type + end + end +end diff --git a/lib/gitlab/recaptcha.rb b/lib/gitlab/recaptcha.rb index 70e7f25d518..4bc76ea033f 100644 --- a/lib/gitlab/recaptcha.rb +++ b/lib/gitlab/recaptcha.rb @@ -10,5 +10,9 @@ module Gitlab true end end + + def self.enabled? + current_application_settings.recaptcha_enabled + end end end diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb index 9384102acec..bc5370de32a 100644 --- a/lib/gitlab/redis.rb +++ b/lib/gitlab/redis.rb @@ -1,27 +1,18 @@ # This file should not have any direct dependency on Rails environment # please require all dependencies below: require 'active_support/core_ext/hash/keys' +require 'active_support/core_ext/module/delegation' module Gitlab class Redis - CACHE_NAMESPACE = 'cache:gitlab' - SESSION_NAMESPACE = 'session:gitlab' - SIDEKIQ_NAMESPACE = 'resque:gitlab' - MAILROOM_NAMESPACE = 'mail_room:gitlab' - DEFAULT_REDIS_URL = 'redis://localhost:6379' - CONFIG_FILE = File.expand_path('../../config/resque.yml', __dir__) + CACHE_NAMESPACE = 'cache:gitlab'.freeze + SESSION_NAMESPACE = 'session:gitlab'.freeze + SIDEKIQ_NAMESPACE = 'resque:gitlab'.freeze + MAILROOM_NAMESPACE = 'mail_room:gitlab'.freeze + DEFAULT_REDIS_URL = 'redis://localhost:6379'.freeze class << self - # Do NOT cache in an instance variable. Result may be mutated by caller. - def params - new.params - end - - # Do NOT cache in an instance variable. Result may be mutated by caller. - # @deprecated Use .params instead to get sentinel support - def url - new.url - end + delegate :params, :url, to: :new def with @pool ||= ConnectionPool.new(size: pool_size) { ::Redis.new(params) } @@ -42,13 +33,17 @@ module Gitlab return @_raw_config if defined?(@_raw_config) begin - @_raw_config = ERB.new(File.read(CONFIG_FILE)).result.freeze + @_raw_config = ERB.new(File.read(config_file)).result.freeze rescue Errno::ENOENT @_raw_config = false end @_raw_config end + + def config_file + ENV['GITLAB_REDIS_CONFIG_FILE'] || File.expand_path('../../config/resque.yml', __dir__) + end end def initialize(rails_env = nil) diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index 11c0b01f0dc..7668ecacc4b 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -1,13 +1,12 @@ module Gitlab # Extract possible GFM references from an arbitrary String for further processing. class ReferenceExtractor < Banzai::ReferenceExtractor - REFERABLES = %i(user issue label milestone merge_request snippet commit commit_range) + REFERABLES = %i(user issue label milestone merge_request snippet commit commit_range directly_addressed_user).freeze attr_accessor :project, :current_user, :author def initialize(project, current_user = nil) @project = project @current_user = current_user - @references = {} super() @@ -21,6 +20,11 @@ module Gitlab super(type, project, current_user) end + def reset_memoized_values + @references = {} + super() + end + REFERABLES.each do |type| define_method("#{type}s") do @references[type] ||= references(type) diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index a3fa7c1331a..5e5f5ff1589 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -5,13 +5,18 @@ module Gitlab # The namespace regex is used in Javascript to validate usernames in the "Register" form. However, Javascript # does not support the negative lookbehind assertion (?<!) that disallows usernames ending in `.git` and `.atom`. # Since this is a non-trivial problem to solve in Javascript (heavily complicate the regex, modify view code to - # allow non-regex validatiions, etc), `NAMESPACE_REGEX_STR_SIMPLE` serves as a Javascript-compatible version of + # allow non-regex validatiions, etc), `NAMESPACE_REGEX_STR_JS` serves as a Javascript-compatible version of # `NAMESPACE_REGEX_STR`, with the negative lookbehind assertion removed. This means that the client-side validation # will pass for usernames ending in `.atom` and `.git`, but will be caught by the server-side validation. PATH_REGEX_STR = '[a-zA-Z0-9_\.][a-zA-Z0-9_\-\.]*'.freeze - NAMESPACE_REGEX_STR_SIMPLE = PATH_REGEX_STR + '[a-zA-Z0-9_\-]|[a-zA-Z0-9_]'.freeze - NAMESPACE_REGEX_STR = '(?:' + NAMESPACE_REGEX_STR_SIMPLE + ')(?<!\.git|\.atom)'.freeze - PROJECT_REGEX_STR = PATH_REGEX_STR + '(?<!\.git|\.atom)'.freeze + NAMESPACE_REGEX_STR_JS = PATH_REGEX_STR + '[a-zA-Z0-9_\-]|[a-zA-Z0-9_]'.freeze + NO_SUFFIX_REGEX_STR = '(?<!\.git|\.atom)'.freeze + NAMESPACE_REGEX_STR = "(?:#{NAMESPACE_REGEX_STR_JS})#{NO_SUFFIX_REGEX_STR}".freeze + PROJECT_REGEX_STR = "(?:#{PATH_REGEX_STR})#{NO_SUFFIX_REGEX_STR}".freeze + + # Same as NAMESPACE_REGEX_STR but allows `/` in the path. + # So `group/subgroup` will match this regex but not NAMESPACE_REGEX_STR + FULL_NAMESPACE_REGEX_STR = "(?:#{NAMESPACE_REGEX_STR}/)*#{NAMESPACE_REGEX_STR}".freeze def namespace_regex @namespace_regex ||= /\A#{NAMESPACE_REGEX_STR}\z/.freeze diff --git a/lib/gitlab/repo_path.rb b/lib/gitlab/repo_path.rb new file mode 100644 index 00000000000..4b1d828c45c --- /dev/null +++ b/lib/gitlab/repo_path.rb @@ -0,0 +1,23 @@ +module Gitlab + module RepoPath + NotFoundError = Class.new(StandardError) + + def self.strip_storage_path(repo_path) + result = nil + + Gitlab.config.repositories.storages.values.each do |params| + storage_path = params['path'] + if repo_path.start_with?(storage_path) + result = repo_path.sub(storage_path, '') + break + end + end + + if result.nil? + raise NotFoundError.new("No known storage path matches #{repo_path.inspect}") + end + + result.sub(/\A\/*/, '') + end + end +end diff --git a/lib/gitlab/request_context.rb b/lib/gitlab/request_context.rb new file mode 100644 index 00000000000..fef536ecb0b --- /dev/null +++ b/lib/gitlab/request_context.rb @@ -0,0 +1,21 @@ +module Gitlab + class RequestContext + class << self + def client_ip + RequestStore[:client_ip] + end + end + + def initialize(app) + @app = app + end + + def call(env) + req = Rack::Request.new(env) + + RequestStore[:client_ip] = req.ip + + @app.call(env) + end + end +end diff --git a/lib/gitlab/request_profiler.rb b/lib/gitlab/request_profiler.rb index 8130e55351e..0c9ab759e81 100644 --- a/lib/gitlab/request_profiler.rb +++ b/lib/gitlab/request_profiler.rb @@ -2,7 +2,7 @@ require 'fileutils' module Gitlab module RequestProfiler - PROFILES_DIR = "#{Gitlab.config.shared.path}/tmp/requests_profiles" + PROFILES_DIR = "#{Gitlab.config.shared.path}/tmp/requests_profiles".freeze def profile_token Rails.cache.fetch('profile-token') do diff --git a/lib/gitlab/request_profiler/middleware.rb b/lib/gitlab/request_profiler/middleware.rb index 786e1d49f5e..ef42b0557e0 100644 --- a/lib/gitlab/request_profiler/middleware.rb +++ b/lib/gitlab/request_profiler/middleware.rb @@ -1,5 +1,4 @@ require 'ruby-prof' -require_dependency 'gitlab/request_profiler' module Gitlab module RequestProfiler @@ -20,7 +19,7 @@ module Gitlab header_token = env['HTTP_X_PROFILE_TOKEN'] return unless header_token.present? - profile_token = RequestProfiler.profile_token + profile_token = Gitlab::RequestProfiler.profile_token return unless profile_token.present? header_token == profile_token diff --git a/lib/gitlab/route_map.rb b/lib/gitlab/route_map.rb new file mode 100644 index 00000000000..36791fae60f --- /dev/null +++ b/lib/gitlab/route_map.rb @@ -0,0 +1,50 @@ +module Gitlab + class RouteMap + FormatError = Class.new(StandardError) + + def initialize(data) + begin + entries = YAML.safe_load(data) + rescue + raise FormatError, 'Route map is not valid YAML' + end + + raise FormatError, 'Route map is not an array' unless entries.is_a?(Array) + + @map = entries.map { |entry| parse_entry(entry) } + end + + def public_path_for_source_path(path) + mapping = @map.find { |mapping| mapping[:source] === path } + return unless mapping + + path.sub(mapping[:source], mapping[:public]) + end + + private + + def parse_entry(entry) + raise FormatError, 'Route map entry is not a hash' unless entry.is_a?(Hash) + raise FormatError, 'Route map entry does not have a source key' unless entry.has_key?('source') + raise FormatError, 'Route map entry does not have a public key' unless entry.has_key?('public') + + source_pattern = entry['source'] + public_path = entry['public'] + + if source_pattern.start_with?('/') && source_pattern.end_with?('/') + source_pattern = source_pattern[1...-1].gsub('\/', '/') + + begin + source_pattern = /\A#{source_pattern}\z/ + rescue RegexpError => e + raise FormatError, "Route map entry source is not a valid regular expression: #{e}" + end + end + + { + source: source_pattern, + public: public_path + } + end + end +end diff --git a/lib/gitlab/saml/user.rb b/lib/gitlab/saml/user.rb index f253dc7477e..8a7cc690046 100644 --- a/lib/gitlab/saml/user.rb +++ b/lib/gitlab/saml/user.rb @@ -28,11 +28,12 @@ module Gitlab if external_users_enabled? && @user # Check if there is overlap between the user's groups and the external groups # setting then set user as external or internal. - if (auth_hash.groups & Gitlab::Saml::Config.external_groups).empty? - @user.external = false - else - @user.external = true - end + @user.external = + if (auth_hash.groups & Gitlab::Saml::Config.external_groups).empty? + false + else + true + end end @user diff --git a/lib/gitlab/sanitizers/svg/whitelist.rb b/lib/gitlab/sanitizers/svg/whitelist.rb index 7b6b70d8dbc..d50f826f924 100644 --- a/lib/gitlab/sanitizers/svg/whitelist.rb +++ b/lib/gitlab/sanitizers/svg/whitelist.rb @@ -6,18 +6,19 @@ module Gitlab module SVG class Whitelist ALLOWED_ELEMENTS = %w[ - a altGlyph altGlyphDef altGlyphItem animate - animateColor animateMotion animateTransform circle clipPath color-profile - cursor defs desc ellipse feBlend feColorMatrix feComponentTransfer - feComposite feConvolveMatrix feDiffuseLighting feDisplacementMap - feDistantLight feFlood feFuncA feFuncB feFuncG feFuncR feGaussianBlur - feImage feMerge feMergeNode feMorphology feOffset fePointLight - feSpecularLighting feSpotLight feTile feTurbulence filter font font-face - font-face-format font-face-name font-face-src font-face-uri foreignObject - g glyph glyphRef hkern image line linearGradient marker mask metadata - missing-glyph mpath path pattern polygon polyline radialGradient rect - script set stop style svg switch symbol text textPath title tref tspan use - view vkern].freeze + a altGlyph altGlyphDef altGlyphItem animate + animateColor animateMotion animateTransform circle clipPath color-profile + cursor defs desc ellipse feBlend feColorMatrix feComponentTransfer + feComposite feConvolveMatrix feDiffuseLighting feDisplacementMap + feDistantLight feFlood feFuncA feFuncB feFuncG feFuncR feGaussianBlur + feImage feMerge feMergeNode feMorphology feOffset fePointLight + feSpecularLighting feSpotLight feTile feTurbulence filter font font-face + font-face-format font-face-name font-face-src font-face-uri foreignObject + g glyph glyphRef hkern image line linearGradient marker mask metadata + missing-glyph mpath path pattern polygon polyline radialGradient rect + script set stop style svg switch symbol text textPath title tref tspan use + view vkern + ].freeze ALLOWED_DATA_ATTRIBUTES_IN_ELEMENTS = %w[svg].freeze diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 35212992698..ccfa517e04b 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -43,6 +43,10 @@ module Gitlab @milestones_count ||= milestones.count end + def single_commit_result? + false + end + private def projects @@ -52,11 +56,12 @@ module Gitlab def issues issues = IssuesFinder.new(current_user).execute.where(project_id: project_ids_relation) - if query =~ /#(\d+)\z/ - issues = issues.where(iid: $1) - else - issues = issues.full_search(query) - end + issues = + if query =~ /#(\d+)\z/ + issues.where(iid: $1) + else + issues.full_search(query) + end issues.order('updated_at DESC') end @@ -69,11 +74,12 @@ module Gitlab def merge_requests merge_requests = MergeRequestsFinder.new(current_user).execute.in_projects(project_ids_relation) - if query =~ /[#!](\d+)\z/ - merge_requests = merge_requests.where(iid: $1) - else - merge_requests = merge_requests.full_search(query) - end + merge_requests = + if query =~ /[#!](\d+)\z/ + merge_requests.where(iid: $1) + else + merge_requests.full_search(query) + end merge_requests.order('updated_at DESC') end diff --git a/lib/gitlab/seeder.rb b/lib/gitlab/seeder.rb index 7cf506ebe64..823f697f51c 100644 --- a/lib/gitlab/seeder.rb +++ b/lib/gitlab/seeder.rb @@ -1,24 +1,23 @@ +module DeliverNever + def deliver_later + self + end +end + module Gitlab class Seeder def self.quiet mute_mailer SeedFu.quiet = true + yield + SeedFu.quiet = false puts "\nOK".color(:green) end - def self.by_user(user) - yield - end - def self.mute_mailer - code = <<-eos -def Notify.deliver_later - self -end - eos - eval(code) + ActionMailer::MessageDelivery.prepend(DeliverNever) end end end diff --git a/lib/gitlab/serialize/ci/variables.rb b/lib/gitlab/serializer/ci/variables.rb index 3a9443bfcd9..c059c454eac 100644 --- a/lib/gitlab/serialize/ci/variables.rb +++ b/lib/gitlab/serializer/ci/variables.rb @@ -1,5 +1,5 @@ module Gitlab - module Serialize + module Serializer module Ci # This serializer could make sure our YAML variables' keys and values # are always strings. This is more for legacy build data because diff --git a/lib/gitlab/serializer/pagination.rb b/lib/gitlab/serializer/pagination.rb new file mode 100644 index 00000000000..9c92b83dddc --- /dev/null +++ b/lib/gitlab/serializer/pagination.rb @@ -0,0 +1,36 @@ +module Gitlab + module Serializer + class Pagination + InvalidResourceError = Class.new(StandardError) + include ::API::Helpers::Pagination + + def initialize(request, response) + @request = request + @response = response + end + + def paginate(resource) + if resource.respond_to?(:page) + super(resource) + else + raise InvalidResourceError + end + end + + private + + # Methods needed by `API::Helpers::Pagination` + # + + attr_reader :request + + def params + @request.query_parameters + end + + def header(header, value) + @response.headers[header] = value + end + end + end +end diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/shell.rb index 82e194c1af1..b631ef11ce7 100644 --- a/lib/gitlab/backend/shell.rb +++ b/lib/gitlab/shell.rb @@ -2,7 +2,7 @@ require 'securerandom' module Gitlab class Shell - class Error < StandardError; end + Error = Class.new(StandardError) KeyAdder = Struct.new(:io) do def add_key(id, key) @@ -80,8 +80,30 @@ module Gitlab # import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://github.com/randx/six.git") # def import_repository(storage, name, url) - output, status = Popen::popen([gitlab_shell_projects_path, 'import-project', - storage, "#{name}.git", url, '900']) + # Timeout should be less than 900 ideally, to prevent the memory killer + # to silently kill the process without knowing we are timing out here. + output, status = Popen.popen([gitlab_shell_projects_path, 'import-project', + storage, "#{name}.git", url, '800']) + raise Error, output unless status.zero? + true + end + + # Fetch remote for repository + # + # name - project path with namespace + # remote - remote name + # forced - should we use --force flag? + # no_tags - should we use --no-tags flag? + # + # Ex. + # fetch_remote("gitlab/gitlab-ci", "upstream") + # + def fetch_remote(storage, name, remote, forced: false, no_tags: false) + args = [gitlab_shell_projects_path, 'fetch-remote', storage, "#{name}.git", remote, '800'] + args << '--force' if forced + args << '--no-tags' if no_tags + + output, status = Popen.popen(args) raise Error, output unless status.zero? true end @@ -143,7 +165,7 @@ module Gitlab # batch_add_keys { |adder| adder.add_key("key-42", "sha-rsa ...") } def batch_add_keys(&block) IO.popen(%W(#{gitlab_shell_path}/bin/gitlab-keys batch-add-keys), 'w') do |io| - block.call(KeyAdder.new(io)) + yield(KeyAdder.new(io)) end end @@ -172,7 +194,10 @@ module Gitlab # add_namespace("/path/to/storage", "gitlab") # def add_namespace(storage, name) - FileUtils.mkdir(full_path(storage, name), mode: 0770) unless exists?(storage, name) + path = full_path(storage, name) + FileUtils.mkdir_p(path, mode: 0770) unless exists?(storage, name) + rescue Errno::EEXIST => e + Rails.logger.warn("Directory exists as a file: #{e} at: #{path}") end # Remove directory from repositories storage diff --git a/lib/gitlab/backend/shell_adapter.rb b/lib/gitlab/shell_adapter.rb index fbe2a7a0d72..fbe2a7a0d72 100644 --- a/lib/gitlab/backend/shell_adapter.rb +++ b/lib/gitlab/shell_adapter.rb diff --git a/lib/gitlab/sherlock/query.rb b/lib/gitlab/sherlock/query.rb index 4917c4ae2ac..99e56e923eb 100644 --- a/lib/gitlab/sherlock/query.rb +++ b/lib/gitlab/sherlock/query.rb @@ -94,11 +94,12 @@ module Gitlab private def raw_explain(query) - if Gitlab::Database.postgresql? - explain = "EXPLAIN ANALYZE #{query};" - else - explain = "EXPLAIN #{query};" - end + explain = + if Gitlab::Database.postgresql? + "EXPLAIN ANALYZE #{query};" + else + "EXPLAIN #{query};" + end ActiveRecord::Base.connection.execute(explain) end diff --git a/lib/gitlab/sidekiq_status.rb b/lib/gitlab/sidekiq_status.rb new file mode 100644 index 00000000000..11e5f1b645c --- /dev/null +++ b/lib/gitlab/sidekiq_status.rb @@ -0,0 +1,89 @@ +module Gitlab + # The SidekiqStatus module and its child classes can be used for checking if a + # Sidekiq job has been processed or not. + # + # To check if a job has been completed, simply pass the job ID to the + # `completed?` method: + # + # job_id = SomeWorker.perform_async(...) + # + # if Gitlab::SidekiqStatus.completed?(job_id) + # ... + # end + # + # For each job ID registered a separate key is stored in Redis, making lookups + # much faster than using Sidekiq's built-in job finding/status API. These keys + # expire after a certain period of time to prevent storing too many keys in + # Redis. + module SidekiqStatus + STATUS_KEY = 'gitlab-sidekiq-status:%s'.freeze + + # The default time (in seconds) after which a status key is expired + # automatically. The default of 30 minutes should be more than sufficient + # for most jobs. + DEFAULT_EXPIRATION = 30.minutes.to_i + + # Starts tracking of the given job. + # + # jid - The Sidekiq job ID + # expire - The expiration time of the Redis key. + def self.set(jid, expire = DEFAULT_EXPIRATION) + Sidekiq.redis do |redis| + redis.set(key_for(jid), 1, ex: expire) + end + end + + # Stops the tracking of the given job. + # + # jid - The Sidekiq job ID to remove. + def self.unset(jid) + Sidekiq.redis do |redis| + redis.del(key_for(jid)) + end + end + + # Returns true if all the given job have been completed. + # + # job_ids - The Sidekiq job IDs to check. + # + # Returns true or false. + def self.all_completed?(job_ids) + self.num_running(job_ids).zero? + end + + # Returns the number of jobs that are running. + # + # job_ids - The Sidekiq job IDs to check. + def self.num_running(job_ids) + responses = self.job_status(job_ids) + + responses.select(&:present?).count + end + + # Returns the number of jobs that have completed. + # + # job_ids - The Sidekiq job IDs to check. + def self.num_completed(job_ids) + job_ids.size - self.num_running(job_ids) + end + + # Returns the job status for each of the given job IDs. + # + # job_ids - The Sidekiq job IDs to check. + # + # Returns an array of true or false indicating job completion. + def self.job_status(job_ids) + keys = job_ids.map { |jid| key_for(jid) } + + Sidekiq.redis do |redis| + redis.pipelined do + keys.each { |key| redis.exists(key) } + end + end + end + + def self.key_for(jid) + STATUS_KEY % jid + end + end +end diff --git a/lib/gitlab/sidekiq_status/client_middleware.rb b/lib/gitlab/sidekiq_status/client_middleware.rb new file mode 100644 index 00000000000..d47609f490d --- /dev/null +++ b/lib/gitlab/sidekiq_status/client_middleware.rb @@ -0,0 +1,10 @@ +module Gitlab + module SidekiqStatus + class ClientMiddleware + def call(_, job, _, _) + Gitlab::SidekiqStatus.set(job['jid']) + yield + end + end + end +end diff --git a/lib/gitlab/sidekiq_status/server_middleware.rb b/lib/gitlab/sidekiq_status/server_middleware.rb new file mode 100644 index 00000000000..ceab10b8301 --- /dev/null +++ b/lib/gitlab/sidekiq_status/server_middleware.rb @@ -0,0 +1,13 @@ +module Gitlab + module SidekiqStatus + class ServerMiddleware + def call(worker, job, queue) + ret = yield + + Gitlab::SidekiqStatus.unset(job['jid']) + + ret + end + end + end +end diff --git a/lib/gitlab/slash_commands/extractor.rb b/lib/gitlab/slash_commands/extractor.rb index a672e5e4855..6dbb467d70d 100644 --- a/lib/gitlab/slash_commands/extractor.rb +++ b/lib/gitlab/slash_commands/extractor.rb @@ -103,7 +103,7 @@ module Gitlab (?<cmd>#{Regexp.union(names)}) (?: [ ] - (?<arg>[^\/\n]*) + (?<arg>[^\n]*) )? (?:\n|$) ) diff --git a/lib/gitlab/snippet_search_results.rb b/lib/gitlab/snippet_search_results.rb index 9e01f02029c..b85f70e450e 100644 --- a/lib/gitlab/snippet_search_results.rb +++ b/lib/gitlab/snippet_search_results.rb @@ -31,11 +31,11 @@ module Gitlab private def snippet_titles - limit_snippets.search(query).order('updated_at DESC') + limit_snippets.search(query).order('updated_at DESC').includes(:author) end def snippet_blobs - limit_snippets.search_code(query).order('updated_at DESC') + limit_snippets.search_code(query).order('updated_at DESC').includes(:author) end def default_scope diff --git a/lib/gitlab/template/finders/repo_template_finder.rb b/lib/gitlab/template/finders/repo_template_finder.rb index 22c39436cb2..cb7957e2af9 100644 --- a/lib/gitlab/template/finders/repo_template_finder.rb +++ b/lib/gitlab/template/finders/repo_template_finder.rb @@ -4,7 +4,7 @@ module Gitlab module Finders class RepoTemplateFinder < BaseTemplateFinder # Raised when file is not found - class FileNotFoundError < StandardError; end + FileNotFoundError = Class.new(StandardError) def initialize(project, base_dir, extension, categories = {}) @categories = categories diff --git a/lib/gitlab/template/gitlab_ci_yml_template.rb b/lib/gitlab/template/gitlab_ci_yml_template.rb index 9d2ecee9756..fd040148a1e 100644 --- a/lib/gitlab/template/gitlab_ci_yml_template.rb +++ b/lib/gitlab/template/gitlab_ci_yml_template.rb @@ -28,7 +28,7 @@ module Gitlab end def dropdown_names(context) - categories = context == 'autodeploy' ? ['Auto deploy'] : ['General', 'Pages'] + categories = context == 'autodeploy' ? ['Auto deploy'] : %w(General Pages) super().slice(*categories) end end diff --git a/lib/gitlab/testing/request_blocker_middleware.rb b/lib/gitlab/testing/request_blocker_middleware.rb new file mode 100644 index 00000000000..aa67fa08577 --- /dev/null +++ b/lib/gitlab/testing/request_blocker_middleware.rb @@ -0,0 +1,61 @@ +# rubocop:disable Style/ClassVars + +# This is inspired by http://www.salsify.com/blog/engineering/tearing-capybara-ajax-tests +# Rack middleware that keeps track of the number of active requests and can block new requests. +module Gitlab + module Testing + class RequestBlockerMiddleware + @@num_active_requests = Concurrent::AtomicFixnum.new(0) + @@block_requests = Concurrent::AtomicBoolean.new(false) + + # Returns the number of requests the server is currently processing. + def self.num_active_requests + @@num_active_requests.value + end + + # Prevents the server from accepting new requests. Any new requests will return an HTTP + # 503 status. + def self.block_requests! + @@block_requests.value = true + end + + # Allows the server to accept requests again. + def self.allow_requests! + @@block_requests.value = false + end + + def initialize(app) + @app = app + end + + def call(env) + increment_active_requests + if block_requests? + block_request(env) + else + @app.call(env) + end + ensure + decrement_active_requests + end + + private + + def block_requests? + @@block_requests.true? + end + + def block_request(env) + [503, {}, []] + end + + def increment_active_requests + @@num_active_requests.increment + end + + def decrement_active_requests + @@num_active_requests.decrement + end + end + end +end diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb deleted file mode 100644 index 19ab76ae80f..00000000000 --- a/lib/gitlab/themes.rb +++ /dev/null @@ -1,87 +0,0 @@ -module Gitlab - # Module containing GitLab's application theme definitions and helper methods - # for accessing them. - module Themes - extend self - - # Theme ID used when no `default_theme` configuration setting is provided. - APPLICATION_DEFAULT = 2 - - # Struct class representing a single Theme - Theme = Struct.new(:id, :name, :css_class) - - # All available Themes - THEMES = [ - Theme.new(1, 'Graphite', 'ui_graphite'), - Theme.new(2, 'Charcoal', 'ui_charcoal'), - Theme.new(3, 'Green', 'ui_green'), - Theme.new(4, 'Black', 'ui_black'), - Theme.new(5, 'Violet', 'ui_violet'), - Theme.new(6, 'Blue', 'ui_blue') - ].freeze - - # Convenience method to get a space-separated String of all the theme - # classes that might be applied to the `body` element - # - # Returns a String - def body_classes - THEMES.collect(&:css_class).uniq.join(' ') - end - - # Get a Theme by its ID - # - # If the ID is invalid, returns the default Theme. - # - # id - Integer ID - # - # Returns a Theme - def by_id(id) - THEMES.detect { |t| t.id == id } || default - end - - # Returns the number of defined Themes - def count - THEMES.size - end - - # Get the default Theme - # - # Returns a Theme - def default - by_id(default_id) - end - - # Iterate through each Theme - # - # Yields the Theme object - def each(&block) - THEMES.each(&block) - end - - # Get the Theme for the specified user, or the default - # - # user - User record - # - # Returns a Theme - def for_user(user) - if user - by_id(user.theme_id) - else - default - end - end - - private - - def default_id - id = Gitlab.config.gitlab.default_theme.to_i - - # Prevent an invalid configuration setting from causing an infinite loop - if id < THEMES.first.id || id > THEMES.last.id - APPLICATION_DEFAULT - else - id - end - end - end -end diff --git a/lib/gitlab/update_path_error.rb b/lib/gitlab/update_path_error.rb index ce14cc887d0..8947ecfb92e 100644 --- a/lib/gitlab/update_path_error.rb +++ b/lib/gitlab/update_path_error.rb @@ -1,3 +1,3 @@ module Gitlab - class UpdatePathError < StandardError; end + UpdatePathError = Class.new(StandardError) end diff --git a/lib/gitlab/upgrader.rb b/lib/gitlab/upgrader.rb index f3567f3ef85..961df0468a4 100644 --- a/lib/gitlab/upgrader.rb +++ b/lib/gitlab/upgrader.rb @@ -46,7 +46,7 @@ module Gitlab git_tags = fetch_git_tags git_tags = git_tags.select { |version| version =~ /v\d+\.\d+\.\d+\Z/ } git_versions = git_tags.map { |tag| Gitlab::VersionInfo.parse(tag.match(/v\d+\.\d+\.\d+/).to_s) } - "v#{git_versions.sort.last.to_s}" + "v#{git_versions.sort.last}" end def fetch_git_tags @@ -59,15 +59,18 @@ module Gitlab "Stash changed files" => %W(#{Gitlab.config.git.bin_path} stash), "Get latest code" => %W(#{Gitlab.config.git.bin_path} fetch), "Switch to new version" => %W(#{Gitlab.config.git.bin_path} checkout v#{latest_version}), - "Install gems" => %W(bundle), - "Migrate DB" => %W(bundle exec rake db:migrate), - "Recompile assets" => %W(bundle exec rake assets:clean assets:precompile), - "Clear cache" => %W(bundle exec rake cache:clear) + "Install gems" => %w(bundle), + "Migrate DB" => %w(bundle exec rake db:migrate), + "Recompile assets" => %w(bundle exec rake yarn:install gitlab:assets:clean gitlab:assets:compile), + "Clear cache" => %w(bundle exec rake cache:clear) } end def env - { 'RAILS_ENV' => 'production' } + { + 'RAILS_ENV' => 'production', + 'NODE_ENV' => 'production' + } end def upgrade diff --git a/lib/gitlab/uploads_transfer.rb b/lib/gitlab/uploads_transfer.rb index be8fcc7b2d2..7d0c47c5361 100644 --- a/lib/gitlab/uploads_transfer.rb +++ b/lib/gitlab/uploads_transfer.rb @@ -1,35 +1,7 @@ module Gitlab - class UploadsTransfer - def move_project(project_path, namespace_path_was, namespace_path) - new_namespace_folder = File.join(root_dir, namespace_path) - FileUtils.mkdir_p(new_namespace_folder) unless Dir.exist?(new_namespace_folder) - from = File.join(root_dir, namespace_path_was, project_path) - to = File.join(root_dir, namespace_path, project_path) - move(from, to, "") - end - - def rename_project(path_was, path, namespace_path) - base_dir = File.join(root_dir, namespace_path) - move(path_was, path, base_dir) - end - - def rename_namespace(path_was, path) - move(path_was, path) - end - - private - - def move(path_was, path, base_dir = nil) - base_dir = root_dir unless base_dir - from = File.join(base_dir, path_was) - to = File.join(base_dir, path) - FileUtils.mv(from, to) - rescue Errno::ENOENT - false - end - + class UploadsTransfer < ProjectTransfer def root_dir - File.join(Rails.root, "public", "uploads") + File.join(CarrierWave.root, GitlabUploader.base_dir) end end end diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb new file mode 100644 index 00000000000..7e14a566696 --- /dev/null +++ b/lib/gitlab/url_blocker.rb @@ -0,0 +1,59 @@ +require 'resolv' + +module Gitlab + class UrlBlocker + class << self + # Used to specify what hosts and port numbers should be prohibited for project + # imports. + VALID_PORTS = [22, 80, 443].freeze + + def blocked_url?(url) + return false if url.nil? + + blocked_ips = ["127.0.0.1", "::1", "0.0.0.0"] + blocked_ips.concat(Socket.ip_address_list.map(&:ip_address)) + + begin + uri = Addressable::URI.parse(url) + # Allow imports from the GitLab instance itself but only from the configured ports + return false if internal?(uri) + + return true if blocked_port?(uri.port) + + server_ips = Resolv.getaddresses(uri.hostname) + return true if (blocked_ips & server_ips).any? + rescue Addressable::URI::InvalidURIError + return true + end + + false + end + + private + + def blocked_port?(port) + return false if port.blank? + + port < 1024 && !VALID_PORTS.include?(port) + end + + def internal?(uri) + internal_web?(uri) || internal_shell?(uri) + end + + def internal_web?(uri) + uri.hostname == config.gitlab.host && + (uri.port.blank? || uri.port == config.gitlab.port) + end + + def internal_shell?(uri) + uri.hostname == config.gitlab_shell.ssh_host && + (uri.port.blank? || uri.port == config.gitlab_shell.ssh_port) + end + + def config + Gitlab.config + end + end + end +end diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb index 19dad699edf..9ce13feb79a 100644 --- a/lib/gitlab/url_sanitizer.rb +++ b/lib/gitlab/url_sanitizer.rb @@ -1,7 +1,7 @@ module Gitlab class UrlSanitizer def self.sanitize(content) - regexp = URI::Parser.new.make_regexp(['http', 'https', 'ssh', 'git']) + regexp = URI::Parser.new.make_regexp(%w(http https ssh git)) content.gsub(regexp) { |url| new(url).masked_url } rescue Addressable::URI::InvalidURIError @@ -9,6 +9,8 @@ module Gitlab end def self.valid?(url) + return false unless url + Addressable::URI.parse(url.strip) true @@ -16,6 +18,12 @@ module Gitlab false end + def self.http_credentials_for_user(user) + return {} unless user.respond_to?(:username) + + { user: user.username } + end + def initialize(url, credentials: nil) @url = Addressable::URI.parse(url.strip) @credentials = credentials diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb index 6c7e673fb9f..f260c0c535f 100644 --- a/lib/gitlab/user_access.rb +++ b/lib/gitlab/user_access.rb @@ -8,7 +8,7 @@ module Gitlab end def can_do_action?(action) - return false if no_user_or_blocked? + return false unless can_access_git? @permission_cache ||= {} @permission_cache[action] ||= user.can?(action, project) @@ -19,7 +19,7 @@ module Gitlab end def allowed? - return false if no_user_or_blocked? + return false unless can_access_git? if user.requires_ldap_check? && user.try_obtain_ldap_lease return false unless Gitlab::LDAP::Access.allowed?(user) @@ -29,20 +29,22 @@ module Gitlab end def can_push_to_branch?(ref) - return false if no_user_or_blocked? + return false unless can_access_git? if project.protected_branch?(ref) return true if project.empty_repo? && project.user_can_push_to_empty_repo?(user) access_levels = project.protected_branches.matching(ref).map(&:push_access_levels).flatten - access_levels.any? { |access_level| access_level.check_access(user) } + has_access = access_levels.any? { |access_level| access_level.check_access(user) } + + has_access || !project.repository.branch_exists?(ref) && can_merge_to_branch?(ref) else user.can?(:push_code, project) end end def can_merge_to_branch?(ref) - return false if no_user_or_blocked? + return false unless can_access_git? if project.protected_branch?(ref) access_levels = project.protected_branches.matching(ref).map(&:merge_access_levels).flatten @@ -53,15 +55,15 @@ module Gitlab end def can_read_project? - return false if no_user_or_blocked? + return false unless can_access_git? user.can?(:read_project, project) end private - def no_user_or_blocked? - user.nil? || user.blocked? + def can_access_git? + user && user.can?(:access_git) end end end diff --git a/lib/gitlab/view/presenter/base.rb b/lib/gitlab/view/presenter/base.rb index 83c8ba5c1cf..dbfe0941e4d 100644 --- a/lib/gitlab/view/presenter/base.rb +++ b/lib/gitlab/view/presenter/base.rb @@ -1,6 +1,8 @@ module Gitlab module View module Presenter + CannotOverrideMethodError = Class.new(StandardError) + module Base extend ActiveSupport::Concern diff --git a/lib/gitlab/view/presenter/delegated.rb b/lib/gitlab/view/presenter/delegated.rb index f4d330c590e..387ff0f5d43 100644 --- a/lib/gitlab/view/presenter/delegated.rb +++ b/lib/gitlab/view/presenter/delegated.rb @@ -8,6 +8,10 @@ module Gitlab @subject = subject attributes.each do |key, value| + if subject.respond_to?(key) + raise CannotOverrideMethodError.new("#{subject} already respond to #{key}!") + end + define_singleton_method(key) { value } end diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index 9462f3368e6..8f1d1fdc02e 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -11,8 +11,21 @@ module Gitlab included do scope :public_only, -> { where(visibility_level: PUBLIC) } scope :public_and_internal_only, -> { where(visibility_level: [PUBLIC, INTERNAL] ) } - - scope :public_to_user, -> (user) { user && !user.external ? public_and_internal_only : public_only } + scope :non_public_only, -> { where.not(visibility_level: PUBLIC) } + + scope :public_to_user, -> (user) do + if user + if user.admin? + all + elsif !user.external? + public_and_internal_only + else + public_only + end + else + public_only + end + end end PRIVATE = 0 unless const_defined?(:PRIVATE) @@ -20,8 +33,10 @@ module Gitlab PUBLIC = 20 unless const_defined?(:PUBLIC) class << self - def values - options.values + delegate :values, to: :options + + def string_values + string_options.keys end def options @@ -32,6 +47,14 @@ module Gitlab } end + def string_options + { + 'private' => PRIVATE, + 'internal' => INTERNAL, + 'public' => PUBLIC + } + end + def highest_allowed_level restricted_levels = current_application_settings.restricted_visibility_levels @@ -71,18 +94,39 @@ module Gitlab level_name end + + def level_value(level) + return level.to_i if level.to_i.to_s == level.to_s && string_options.key(level.to_i) + string_options[level] || PRIVATE + end + + def string_level(level) + string_options.key(level) + end end def private? - visibility_level_field == PRIVATE + visibility_level_value == PRIVATE end def internal? - visibility_level_field == INTERNAL + visibility_level_value == INTERNAL end def public? - visibility_level_field == PUBLIC + visibility_level_value == PUBLIC + end + + def visibility_level_value + self[visibility_level_field] + end + + def visibility + Gitlab::VisibilityLevel.string_level(visibility_level_value) + end + + def visibility=(level) + self[visibility_level_field] = Gitlab::VisibilityLevel.level_value(level) end end end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index a3b502ffd6a..d0637f8b394 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -1,29 +1,51 @@ require 'base64' require 'json' require 'securerandom' +require 'uri' module Gitlab class Workhorse - SEND_DATA_HEADER = 'Gitlab-Workhorse-Send-Data' - VERSION_FILE = 'GITLAB_WORKHORSE_VERSION' - INTERNAL_API_CONTENT_TYPE = 'application/vnd.gitlab-workhorse+json' - INTERNAL_API_REQUEST_HEADER = 'Gitlab-Workhorse-Api-Request' + SEND_DATA_HEADER = 'Gitlab-Workhorse-Send-Data'.freeze + VERSION_FILE = 'GITLAB_WORKHORSE_VERSION'.freeze + INTERNAL_API_CONTENT_TYPE = 'application/vnd.gitlab-workhorse+json'.freeze + INTERNAL_API_REQUEST_HEADER = 'Gitlab-Workhorse-Api-Request'.freeze + NOTIFICATION_CHANNEL = 'workhorse:notifications'.freeze # Supposedly the effective key size for HMAC-SHA256 is 256 bits, i.e. 32 # bytes https://tools.ietf.org/html/rfc4868#section-2.6 SECRET_LENGTH = 32 class << self - def git_http_ok(repository, user) + def git_http_ok(repository, user, action) + repo_path = repository.path_to_repo params = { GL_ID: Gitlab::GlId.gl_id(user), - RepoPath: repository.path_to_repo, + RepoPath: repo_path, } - params.merge!( - GitalySocketPath: Gitlab.config.gitaly.socket_path, - GitalyResourcePath: "/projects/#{repository.project.id}/git-http/info-refs", - ) if Gitlab.config.gitaly.socket_path.present? + if Gitlab.config.gitaly.enabled + storage = repository.project.repository_storage + address = Gitlab::GitalyClient.get_address(storage) + # TODO: use GitalyClient code to assemble the Repository message + params[:Repository] = Gitaly::Repository.new( + path: repo_path, + storage_name: storage, + relative_path: Gitlab::RepoPath.strip_storage_path(repo_path), + ).to_h + + feature_enabled = case action.to_s + when 'git_receive_pack' + Gitlab::GitalyClient.feature_enabled?(:post_receive_pack) + when 'git_upload_pack' + Gitlab::GitalyClient.feature_enabled?(:post_upload_pack) + when 'info_refs' + true + else + raise "Unsupported action: #{action}" + end + + params[:GitalySocketPath] = URI(address).path if feature_enabled + end params end @@ -107,7 +129,8 @@ module Gitlab 'Terminal' => { 'Subprotocols' => terminal[:subprotocols], 'Url' => terminal[:url], - 'Header' => terminal[:headers] + 'Header' => terminal[:headers], + 'MaxSessionTime' => terminal[:max_session_time], } } details['Terminal']['CAPem'] = terminal[:ca_pem] if terminal.has_key?(:ca_pem) @@ -153,6 +176,18 @@ module Gitlab Rails.root.join('.gitlab_workhorse_secret') end + def set_key_and_notify(key, value, expire: nil, overwrite: true) + Gitlab::Redis.with do |redis| + result = redis.set(key, value, ex: expire, nx: !overwrite) + if result + redis.publish(NOTIFICATION_CHANNEL, "#{key}=#{value}") + value + else + redis.get(key) + end + end + end + protected def encode(hash) diff --git a/lib/mattermost/client.rb b/lib/mattermost/client.rb index ec2903b7ec6..3d60618006c 100644 --- a/lib/mattermost/client.rb +++ b/lib/mattermost/client.rb @@ -1,5 +1,5 @@ module Mattermost - class ClientError < Mattermost::Error; end + ClientError = Class.new(Mattermost::Error) class Client attr_reader :user @@ -8,21 +8,31 @@ module Mattermost @user = user end - private - def with_session(&blk) Mattermost::Session.new(user).with_session(&blk) end - def json_get(path, options = {}) + private + + # Should be used in a session manually + def get(session, path, options = {}) + json_response session.get(path, options) + end + + # Should be used in a session manually + def post(session, path, options = {}) + json_response session.post(path, options) + end + + def session_get(path, options = {}) with_session do |session| - json_response session.get(path, options) + get(session, path, options) end end - def json_post(path, options = {}) + def session_post(path, options = {}) with_session do |session| - json_response session.post(path, options) + post(session, path, options) end end diff --git a/lib/mattermost/command.rb b/lib/mattermost/command.rb index d1e4bb0eccf..33e450d7f0a 100644 --- a/lib/mattermost/command.rb +++ b/lib/mattermost/command.rb @@ -1,7 +1,7 @@ module Mattermost class Command < Client def create(params) - response = json_post("/api/v3/teams/#{params[:team_id]}/commands/create", + response = session_post("/api/v3/teams/#{params[:team_id]}/commands/create", body: params.to_json) response['token'] diff --git a/lib/mattermost/error.rb b/lib/mattermost/error.rb index 014df175be0..dee6deb7974 100644 --- a/lib/mattermost/error.rb +++ b/lib/mattermost/error.rb @@ -1,3 +1,3 @@ module Mattermost - class Error < StandardError; end + Error = Class.new(StandardError) end diff --git a/lib/mattermost/session.rb b/lib/mattermost/session.rb index 377cb7b1021..688a79c0441 100644 --- a/lib/mattermost/session.rb +++ b/lib/mattermost/session.rb @@ -5,7 +5,7 @@ module Mattermost end end - class ConnectionError < Mattermost::Error; end + ConnectionError = Class.new(Mattermost::Error) # This class' prime objective is to obtain a session token on a Mattermost # instance with SSO configured where this GitLab instance is the provider. @@ -153,7 +153,7 @@ module Mattermost yield rescue HTTParty::Error => e raise Mattermost::ConnectionError.new(e.message) - rescue Errno::ECONNREFUSED + rescue Errno::ECONNREFUSED => e raise Mattermost::ConnectionError.new(e.message) end end diff --git a/lib/mattermost/team.rb b/lib/mattermost/team.rb index 784eca6ab5a..2cdbbdece16 100644 --- a/lib/mattermost/team.rb +++ b/lib/mattermost/team.rb @@ -1,7 +1,18 @@ module Mattermost class Team < Client + # Returns **all** teams for an admin def all - json_get('/api/v3/teams/all') + session_get('/api/v3/teams/all').values + end + + # Creates a team on the linked Mattermost instance, the team admin will be the + # `current_user` passed to the Mattermost::Client instance + def create(name:, display_name:, type:) + session_post('/api/v3/teams/create', body: { + name: name, + display_name: display_name, + type: type + }.to_json) end end end diff --git a/lib/omniauth/strategies/bitbucket.rb b/lib/omni_auth/strategies/bitbucket.rb index 5a7d67c2390..5a7d67c2390 100644 --- a/lib/omniauth/strategies/bitbucket.rb +++ b/lib/omni_auth/strategies/bitbucket.rb diff --git a/lib/rouge/formatters/html_gitlab.rb b/lib/rouge/formatters/html_gitlab.rb index 4edfd015074..be0d97370d0 100644 --- a/lib/rouge/formatters/html_gitlab.rb +++ b/lib/rouge/formatters/html_gitlab.rb @@ -5,10 +5,10 @@ module Rouge # Creates a new <tt>Rouge::Formatter::HTMLGitlab</tt> instance. # - # [+linenostart+] The line number for the first line (default: 1). - def initialize(linenostart: 1) - @linenostart = linenostart - @line_number = linenostart + # [+tag+] The tag (language) of the lexer used to generate the formatted tokens + def initialize(tag: nil) + @line_number = 1 + @tag = tag end def stream(tokens, &b) @@ -17,7 +17,7 @@ module Rouge yield "\n" unless is_first is_first = false - yield %(<span id="LC#{@line_number}" class="line">) + yield %(<span id="LC#{@line_number}" class="line" lang="#{@tag}">) line.each { |token, value| yield span(token, value.chomp) } yield %(</span>) diff --git a/lib/rouge/lexers/plantuml.rb b/lib/rouge/lexers/plantuml.rb new file mode 100644 index 00000000000..7d5700b7f6d --- /dev/null +++ b/lib/rouge/lexers/plantuml.rb @@ -0,0 +1,21 @@ +module Rouge + module Lexers + class Plantuml < Lexer + title "A passthrough lexer used for PlantUML input" + desc "A boring lexer that doesn't highlight anything" + + tag 'plantuml' + mimetypes 'text/plain' + + default_options token: 'Text' + + def token + @token ||= Token[option :token] + end + + def stream_tokens(string, &b) + yield self.token, string + end + end + end +end diff --git a/lib/support/deploy/deploy.sh b/lib/support/deploy/deploy.sh index adea4c7a747..ab46c47d8f5 100755 --- a/lib/support/deploy/deploy.sh +++ b/lib/support/deploy/deploy.sh @@ -31,8 +31,8 @@ echo 'Deploy: Bundle and migrate' sudo -u git -H bundle --without aws development test mysql --deployment sudo -u git -H bundle exec rake db:migrate RAILS_ENV=production -sudo -u git -H bundle exec rake assets:clean RAILS_ENV=production -sudo -u git -H bundle exec rake assets:precompile RAILS_ENV=production +sudo -u git -H bundle exec rake gitlab:assets:clean RAILS_ENV=production +sudo -u git -H bundle exec rake gitlab:assets:compile RAILS_ENV=production sudo -u git -H bundle exec rake cache:clear RAILS_ENV=production # return stashed changes (if necessary) diff --git a/lib/support/init.d/gitlab b/lib/support/init.d/gitlab index 31b00ff128a..09e121e5120 100755 --- a/lib/support/init.d/gitlab +++ b/lib/support/init.d/gitlab @@ -42,7 +42,16 @@ gitlab_workhorse_dir=$(cd $app_root/../gitlab-workhorse 2> /dev/null && pwd) gitlab_workhorse_pid_path="$pid_path/gitlab-workhorse.pid" gitlab_workhorse_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-workhorse.socket -authBackend http://127.0.0.1:8080 -authSocket $rails_socket -documentRoot $app_root/public" gitlab_workhorse_log="$app_root/log/gitlab-workhorse.log" +gitlab_pages_enabled=false +gitlab_pages_dir=$(cd $app_root/../gitlab-pages 2> /dev/null && pwd) +gitlab_pages_pid_path="$pid_path/gitlab-pages.pid" +gitlab_pages_options="-pages-domain example.com -pages-root $app_root/shared/pages -listen-proxy 127.0.0.1:8090" +gitlab_pages_log="$app_root/log/gitlab-pages.log" shell_path="/bin/bash" +gitaly_enabled=false +gitaly_dir=$(cd $app_root/../gitaly 2> /dev/null && pwd) +gitaly_pid_path="$pid_path/gitaly.pid" +gitaly_log="$app_root/log/gitaly.log" # Read configuration variable file if it is present test -f /etc/default/gitlab && . /etc/default/gitlab @@ -89,13 +98,27 @@ check_pids(){ mpid=0 fi fi + if [ "$gitlab_pages_enabled" = true ]; then + if [ -f "$gitlab_pages_pid_path" ]; then + gppid=$(cat "$gitlab_pages_pid_path") + else + gppid=0 + fi + fi + if [ "$gitaly_enabled" = true ]; then + if [ -f "$gitaly_pid_path" ]; then + gapid=$(cat "$gitaly_pid_path") + else + gapid=0 + fi + fi } ## Called when we have started the two processes and are waiting for their pid files. wait_for_pids(){ # We are sleeping a bit here mostly because sidekiq is slow at writing its pid i=0; - while [ ! -f $web_server_pid_path ] || [ ! -f $sidekiq_pid_path ] || [ ! -f $gitlab_workhorse_pid_path ] || { [ "$mail_room_enabled" = true ] && [ ! -f $mail_room_pid_path ]; }; do + while [ ! -f $web_server_pid_path ] || [ ! -f $sidekiq_pid_path ] || [ ! -f $gitlab_workhorse_pid_path ] || { [ "$mail_room_enabled" = true ] && [ ! -f $mail_room_pid_path ]; } || { [ "$gitlab_pages_enabled" = true ] && [ ! -f $gitlab_pages_pid_path ]; } || { [ "$gitaly_enabled" = true ] && [ ! -f $gitaly_pid_path ]; }; do sleep 0.1; i=$((i+1)) if [ $((i%10)) = 0 ]; then @@ -144,7 +167,23 @@ check_status(){ mail_room_status="-1" fi fi - if [ $web_status = 0 ] && [ $sidekiq_status = 0 ] && [ $gitlab_workhorse_status = 0 ] && { [ "$mail_room_enabled" != true ] || [ $mail_room_status = 0 ]; }; then + if [ "$gitlab_pages_enabled" = true ]; then + if [ $gppid -ne 0 ]; then + kill -0 "$gppid" 2>/dev/null + gitlab_pages_status="$?" + else + gitlab_pages_status="-1" + fi + fi + if [ "$gitaly_enabled" = true ]; then + if [ $gapid -ne 0 ]; then + kill -0 "$gapid" 2>/dev/null + gitaly_status="$?" + else + gitaly_status="-1" + fi + fi + if [ $web_status = 0 ] && [ $sidekiq_status = 0 ] && [ $gitlab_workhorse_status = 0 ] && { [ "$mail_room_enabled" != true ] || [ $mail_room_status = 0 ]; } && { [ "$gitlab_pages_enabled" != true ] || [ $gitlab_pages_status = 0 ]; } && { [ "$gitaly_enabled" != true ] || [ $gitaly_status = 0 ]; }; then gitlab_status=0 else # http://refspecs.linuxbase.org/LSB_4.1.0/LSB-Core-generic/LSB-Core-generic/iniscrptact.html @@ -186,12 +225,26 @@ check_stale_pids(){ exit 1 fi fi + if [ "$gitlab_pages_enabled" = true ] && [ "$gppid" != "0" ] && [ "$gitlab_pages_status" != "0" ]; then + echo "Removing stale GitLab Pages job dispatcher pid. This is most likely caused by GitLab Pages crashing the last time it ran." + if ! rm "$gitlab_pages_pid_path"; then + echo "Unable to remove stale pid, exiting" + exit 1 + fi + fi + if [ "$gitaly_enabled" = true ] && [ "$gapid" != "0" ] && [ "$gitaly_status" != "0" ]; then + echo "Removing stale Gitaly pid. This is most likely caused by Gitaly crashing the last time it ran." + if ! rm "$gitaly_pid_path"; then + echo "Unable to remove stale pid, exiting" + exit 1 + fi + fi } ## If no parts of the service is running, bail out. exit_if_not_running(){ check_stale_pids - if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then + if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; } && { [ "$gitlab_pages_enabled" != true ] || [ "$gitlab_pages_status" != "0" ]; } && { [ "$gitaly_enabled" != true ] || [ "$gitaly_status" != "0" ]; }; then echo "GitLab is not running." exit fi @@ -213,6 +266,12 @@ start_gitlab() { if [ "$mail_room_enabled" = true ] && [ "$mail_room_status" != "0" ]; then echo "Starting GitLab MailRoom" fi + if [ "$gitlab_pages_enabled" = true ] && [ "$gitlab_pages_status" != "0" ]; then + echo "Starting GitLab Pages" + fi + if [ "$gitaly_enabled" = true ] && [ "$gitaly_status" != "0" ]; then + echo "Starting Gitaly" + fi # Then check if the service is running. If it is: don't start again. if [ "$web_status" = "0" ]; then @@ -252,6 +311,26 @@ start_gitlab() { fi fi + if [ "$gitlab_pages_enabled" = true ]; then + if [ "$gitlab_pages_status" = "0" ]; then + echo "The GitLab Pages is already running with pid $spid, not restarting" + else + $app_root/bin/daemon_with_pidfile $gitlab_pages_pid_path \ + $gitlab_pages_dir/gitlab-pages $gitlab_pages_options \ + >> $gitlab_pages_log 2>&1 & + fi + fi + + if [ "$gitaly_enabled" = true ]; then + if [ "$gitaly_status" = "0" ]; then + echo "Gitaly is already running with pid $gapid, not restarting" + else + $app_root/bin/daemon_with_pidfile $gitaly_pid_path \ + $app_root/bin/with_env $gitaly_dir/env \ + $gitaly_dir/gitaly >> $gitaly_log 2>&1 & + fi + fi + # Wait for the pids to be planted wait_for_pids # Finally check the status to tell wether or not GitLab is running @@ -278,13 +357,21 @@ stop_gitlab() { echo "Shutting down GitLab MailRoom" RAILS_ENV=$RAILS_ENV bin/mail_room stop fi + if [ "$gitlab_pages_status" = "0" ]; then + echo "Shutting down gitlab-pages" + kill -- $(cat $gitlab_pages_pid_path) + fi + if [ "$gitaly_status" = "0" ]; then + echo "Shutting down Gitaly" + kill -- $(cat $gitaly_pid_path) + fi # If something needs to be stopped, lets wait for it to stop. Never use SIGKILL in a script. - while [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_workhorse_status" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; }; do + while [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_workhorse_status" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; } || { [ "$gitlab_pages_enabled" = true ] && [ "$gitlab_pages_status" = "0" ]; } || { [ "$gitaly_enabled" = true ] && [ "$gitaly_status" = "0" ]; }; do sleep 1 check_status printf "." - if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then + if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; } && { [ "$gitlab_pages_enabled" != true ] || [ "$gitlab_pages_status" != "0" ]; } && { [ "$gitaly_enabled" != true ] || [ "$gitaly_status" != "0" ]; }; then printf "\n" break fi @@ -298,6 +385,8 @@ stop_gitlab() { if [ "$mail_room_enabled" = true ]; then rm "$mail_room_pid_path" 2>/dev/null fi + rm -f "$gitlab_pages_pid_path" + rm -f "$gitaly_pid_path" print_status } @@ -305,7 +394,7 @@ stop_gitlab() { ## Prints the status of GitLab and its components. print_status() { check_status - if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then + if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; } && { [ "$gitlab_pages_enabled" != true ] || [ "$gitlab_pages_status" != "0" ]; } && { [ "$gitaly_enabled" != true ] || [ "$gitaly_status" != "0" ]; }; then echo "GitLab is not running." return fi @@ -331,7 +420,21 @@ print_status() { printf "The GitLab MailRoom email processor is \033[31mnot running\033[0m.\n" fi fi - if [ "$web_status" = "0" ] && [ "$sidekiq_status" = "0" ] && [ "$gitlab_workhorse_status" = "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" = "0" ]; }; then + if [ "$gitlab_pages_enabled" = true ]; then + if [ "$gitlab_pages_status" = "0" ]; then + echo "The GitLab Pages with pid $mpid is running." + else + printf "The GitLab Pages is \033[31mnot running\033[0m.\n" + fi + fi + if [ "$gitaly_enabled" = true ]; then + if [ "$gitaly_status" = "0" ]; then + echo "Gitaly with pid $gapid is running." + else + printf "Gitaly is \033[31mnot running\033[0m.\n" + fi + fi + if [ "$web_status" = "0" ] && [ "$sidekiq_status" = "0" ] && [ "$gitlab_workhorse_status" = "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" = "0" ]; } && { [ "$gitlab_pages_enabled" != true ] || [ "$gitlab_pages_status" = "0" ]; } && { [ "$gitaly_enabled" != true ] || [ "$gitaly_status" = "0" ]; }; then printf "GitLab and all its components are \033[32mup and running\033[0m.\n" fi } @@ -362,7 +465,7 @@ reload_gitlab(){ ## Restarts Sidekiq and Unicorn. restart_gitlab(){ check_status - if [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_workhorse" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; }; then + if [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_workhorse" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; } || { [ "$gitlab_pages_enabled" = true ] && [ "$gitlab_pages_status" = "0" ]; } || { [ "$gitaly_enabled" = true ] && [ "$gitaly_status" = "0" ]; }; then stop_gitlab fi start_gitlab diff --git a/lib/support/init.d/gitlab.default.example b/lib/support/init.d/gitlab.default.example index cc8617b72ca..9472c3c992f 100755..100644 --- a/lib/support/init.d/gitlab.default.example +++ b/lib/support/init.d/gitlab.default.example @@ -47,6 +47,30 @@ gitlab_workhorse_pid_path="$pid_path/gitlab-workhorse.pid" gitlab_workhorse_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-workhorse.socket -authBackend http://127.0.0.1:8080 -authSocket $socket_path/gitlab.socket -documentRoot $app_root/public" gitlab_workhorse_log="$app_root/log/gitlab-workhorse.log" +# The GitLab Pages Daemon needs either a separate IP address on which it will +# listen or use different ports than 80 or 443 that will be forwarded to GitLab +# Pages Daemon. +# +# To enable HTTP support for custom domains add the `-listen-http` directive +# in `gitlab_pages_options` below. +# The value of -listen-http must be set to `gitlab.yml > pages > external_http` +# as well. For example: +# +# -listen-http 1.1.1.1:80 -listen-http [2001::1]:80 +# +# To enable HTTPS support for custom domains add the `-listen-https`, +# `-root-cert` and `-root-key` directives in `gitlab_pages_options` below. +# The value of -listen-https must be set to `gitlab.yml > pages > external_https` +# as well. For example: +# +# -listen-https 1.1.1.1:443 -listen-http [2001::1]:443 -root-cert /path/to/example.com.crt -root-key /path/to/example.com.key +# +# The -pages-domain must be specified the same as in `gitlab.yml > pages > host`. +# Set `gitlab_pages_enabled=true` if you want to enable the Pages feature. +gitlab_pages_enabled=false +gitlab_pages_options="-pages-domain example.com -pages-root $app_root/shared/pages -listen-proxy 127.0.0.1:8090" +gitlab_pages_log="$app_root/log/gitlab-pages.log" + # mail_room_enabled specifies whether mail_room, which is used to process incoming email, is enabled. # This is required for the Reply by email feature. # The default is "false" @@ -60,3 +84,7 @@ mail_room_pid_path="$pid_path/mail_room.pid" # shell other than "bash" # The default is "/bin/bash" shell_path="/bin/bash" + +# This variable controls whether the init script starts/stops Gitaly +gitaly_enabled=false +gitaly_log="$app_root/log/gitaly.log" diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab index 2f7c34a3f31..f25e66d54c8 100644 --- a/lib/support/nginx/gitlab +++ b/lib/support/nginx/gitlab @@ -38,6 +38,13 @@ server { ## See app/controllers/application_controller.rb for headers set + ## Real IP Module Config + ## http://nginx.org/en/docs/http/ngx_http_realip_module.html + real_ip_header X-Real-IP; ## X-Real-IP or X-Forwarded-For or proxy_protocol + real_ip_recursive off; ## If you enable 'on' + ## If you have a trusted IP address, uncomment it and set it + # set_real_ip_from YOUR_TRUSTED_ADDRESS; ## Replace this with something like 192.168.1.0/24 + ## Individual nginx logs for this GitLab vhost access_log /var/log/nginx/gitlab_access.log; error_log /var/log/nginx/gitlab_error.log; diff --git a/lib/support/nginx/gitlab-pages b/lib/support/nginx/gitlab-pages new file mode 100644 index 00000000000..d9746c5c1aa --- /dev/null +++ b/lib/support/nginx/gitlab-pages @@ -0,0 +1,28 @@ +## GitLab +## + +## Pages serving host +server { + listen 0.0.0.0:80; + listen [::]:80 ipv6only=on; + + ## Replace this with something like pages.gitlab.com + server_name ~^.*\.YOUR_GITLAB_PAGES\.DOMAIN$; + + ## Individual nginx logs for GitLab pages + access_log /var/log/nginx/gitlab_pages_access.log; + error_log /var/log/nginx/gitlab_pages_error.log; + + location / { + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + # The same address as passed to GitLab Pages: `-listen-proxy` + proxy_pass http://localhost:8090/; + } + + # Define custom error pages + error_page 403 /403.html; + error_page 404 /404.html; +} diff --git a/lib/support/nginx/gitlab-pages-ssl b/lib/support/nginx/gitlab-pages-ssl new file mode 100644 index 00000000000..a1ccf266835 --- /dev/null +++ b/lib/support/nginx/gitlab-pages-ssl @@ -0,0 +1,77 @@ +## GitLab +## + +## Redirects all HTTP traffic to the HTTPS host +server { + ## Either remove "default_server" from the listen line below, + ## or delete the /etc/nginx/sites-enabled/default file. This will cause gitlab + ## to be served if you visit any address that your server responds to, eg. + ## the ip address of the server (http://x.x.x.x/) + listen 0.0.0.0:80; + listen [::]:80 ipv6only=on; + + ## Replace this with something like pages.gitlab.com + server_name ~^.*\.YOUR_GITLAB_PAGES\.DOMAIN$; + server_tokens off; ## Don't show the nginx version number, a security best practice + + return 301 https://$http_host$request_uri; + + access_log /var/log/nginx/gitlab_pages_access.log; + error_log /var/log/nginx/gitlab_pages_access.log; +} + +## Pages serving host +server { + listen 0.0.0.0:443 ssl; + listen [::]:443 ipv6only=on ssl http2; + + ## Replace this with something like pages.gitlab.com + server_name ~^.*\.YOUR_GITLAB_PAGES\.DOMAIN$; + server_tokens off; ## Don't show the nginx version number, a security best practice + + ## Strong SSL Security + ## https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html & https://cipherli.st/ + ssl on; + ssl_certificate /etc/nginx/ssl/gitlab-pages.crt; + ssl_certificate_key /etc/nginx/ssl/gitlab-pages.key; + + # GitLab needs backwards compatible ciphers to retain compatibility with Java IDEs + ssl_ciphers "ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-DES-CBC3-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4"; + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + ssl_prefer_server_ciphers on; + ssl_session_cache shared:SSL:10m; + ssl_session_timeout 5m; + + ## See app/controllers/application_controller.rb for headers set + + ## [Optional] If your certficate has OCSP, enable OCSP stapling to reduce the overhead and latency of running SSL. + ## Replace with your ssl_trusted_certificate. For more info see: + ## - https://medium.com/devops-programming/4445f4862461 + ## - https://www.ruby-forum.com/topic/4419319 + ## - https://www.digitalocean.com/community/tutorials/how-to-configure-ocsp-stapling-on-apache-and-nginx + # ssl_stapling on; + # ssl_stapling_verify on; + # ssl_trusted_certificate /etc/nginx/ssl/stapling.trusted.crt; + + ## [Optional] Generate a stronger DHE parameter: + ## sudo openssl dhparam -out /etc/ssl/certs/dhparam.pem 4096 + ## + # ssl_dhparam /etc/ssl/certs/dhparam.pem; + + ## Individual nginx logs for GitLab pages + access_log /var/log/nginx/gitlab_pages_access.log; + error_log /var/log/nginx/gitlab_pages_error.log; + + location / { + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + # The same address as passed to GitLab Pages: `-listen-proxy` + proxy_pass http://localhost:8090/; + } + + # Define custom error pages + error_page 403 /403.html; + error_page 404 /404.html; +} diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl index 5661394058d..2b40da18bab 100644 --- a/lib/support/nginx/gitlab-ssl +++ b/lib/support/nginx/gitlab-ssl @@ -82,6 +82,16 @@ server { ## # ssl_dhparam /etc/ssl/certs/dhparam.pem; + ## [Optional] Enable HTTP Strict Transport Security + # add_header Strict-Transport-Security "max-age=31536000; includeSubDomains"; + + ## Real IP Module Config + ## http://nginx.org/en/docs/http/ngx_http_realip_module.html + real_ip_header X-Real-IP; ## X-Real-IP or X-Forwarded-For or proxy_protocol + real_ip_recursive off; ## If you enable 'on' + ## If you have a trusted IP address, uncomment it and set it + # set_real_ip_from YOUR_TRUSTED_ADDRESS; ## Replace this with something like 192.168.1.0/24 + ## Individual nginx logs for this GitLab vhost access_log /var/log/nginx/gitlab_access.log; error_log /var/log/nginx/gitlab_error.log; diff --git a/lib/tasks/brakeman.rake b/lib/tasks/brakeman.rake index d5a402907d8..2301ec9b228 100644 --- a/lib/tasks/brakeman.rake +++ b/lib/tasks/brakeman.rake @@ -2,7 +2,7 @@ desc 'Security check via brakeman' task :brakeman do # We get 0 warnings at level 'w3' but we would like to reach 'w2'. Merge # requests are welcome! - if system(*%W(brakeman --no-progress --skip-files lib/backup/repository.rb -w3 -z)) + if system(*%w(brakeman --no-progress --skip-files lib/backup/repository.rb -w3 -z)) puts 'Security check succeed' else puts 'Security check failed' diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake index 78ae187817a..d55923673b1 100644 --- a/lib/tasks/cache.rake +++ b/lib/tasks/cache.rake @@ -1,7 +1,7 @@ namespace :cache do namespace :clear do REDIS_CLEAR_BATCH_SIZE = 1000 # There seems to be no speedup when pushing beyond 1,000 - REDIS_SCAN_START_STOP = '0' # Magic value, see http://redis.io/commands/scan + REDIS_SCAN_START_STOP = '0'.freeze # Magic value, see http://redis.io/commands/scan desc "GitLab | Clear redis cache" task redis: :environment do diff --git a/lib/tasks/config_lint.rake b/lib/tasks/config_lint.rake new file mode 100644 index 00000000000..ddbcf1e1eb8 --- /dev/null +++ b/lib/tasks/config_lint.rake @@ -0,0 +1,25 @@ +module ConfigLint + def self.run(files) + failures = files.reject do |file| + yield(file) + end + + if failures.present? + puts failures + exit failures.count + end + end +end + +desc "Checks syntax for shell scripts and nginx config files in 'lib/support/'" +task :config_lint do + shell_scripts = [ + 'lib/support/init.d/gitlab', + 'lib/support/init.d/gitlab.default.example', + 'lib/support/deploy/deploy.sh' + ] + + ConfigLint.run(shell_scripts) do |file| + Kernel.system('bash', '-n', file) + end +end diff --git a/lib/tasks/dev.rake b/lib/tasks/dev.rake index 5e94fba97bf..e65609d7001 100644 --- a/lib/tasks/dev.rake +++ b/lib/tasks/dev.rake @@ -2,7 +2,7 @@ task dev: ["dev:setup"] namespace :dev do desc "GitLab | Setup developer environment (db, fixtures)" - task :setup => :environment do + task setup: :environment do ENV['force'] = 'yes' Rake::Task["gitlab:setup"].invoke Rake::Task["gitlab:shell:setup"].invoke diff --git a/lib/tasks/downtime_check.rake b/lib/tasks/downtime_check.rake index afe5d42910c..557f4fef10b 100644 --- a/lib/tasks/downtime_check.rake +++ b/lib/tasks/downtime_check.rake @@ -1,10 +1,10 @@ desc 'Checks if migrations in a branch require downtime' task downtime_check: :environment do - if defined?(Gitlab::License) - repo = 'gitlab-ee' - else - repo = 'gitlab-ce' - end + repo = if defined?(Gitlab::License) + 'gitlab-ee' + else + 'gitlab-ce' + end `git fetch https://gitlab.com/gitlab-org/#{repo}.git --depth 1` diff --git a/lib/tasks/eslint.rake b/lib/tasks/eslint.rake index d43cbad1909..51f5d768102 100644 --- a/lib/tasks/eslint.rake +++ b/lib/tasks/eslint.rake @@ -1,7 +1,8 @@ unless Rails.env.production? desc "GitLab | Run ESLint" - task :eslint do - system("npm", "run", "eslint") + task eslint: ['yarn:check'] do + unless system('yarn run eslint') + abort('rake eslint failed') + end end end - diff --git a/lib/tasks/flay.rake b/lib/tasks/flay.rake index e9587595fef..7ad2b2e4d39 100644 --- a/lib/tasks/flay.rake +++ b/lib/tasks/flay.rake @@ -1,6 +1,6 @@ desc 'Code duplication analyze via flay' task :flay do - output = %x(bundle exec flay --mass 35 app/ lib/gitlab/) + output = `bundle exec flay --mass 35 app/ lib/gitlab/` if output.include? "Similar code found" puts output diff --git a/lib/tasks/gemojione.rake b/lib/tasks/gemojione.rake index 993112aee3b..5293f5af12d 100644 --- a/lib/tasks/gemojione.rake +++ b/lib/tasks/gemojione.rake @@ -1,33 +1,36 @@ namespace :gemojione do desc 'Generates Emoji SHA256 digests' - task digests: :environment do + task digests: ['yarn:check', 'environment'] do require 'digest/sha2' require 'json' - dir = Gemojione.images_path - digests = [] - aliases = Hash.new { |hash, key| hash[key] = [] } - aliases_path = File.join(Rails.root, 'fixtures', 'emojis', 'aliases.json') - - JSON.parse(File.read(aliases_path)).each do |alias_name, real_name| - aliases[real_name] << alias_name - end - - Gitlab::AwardEmoji.emojis.map do |name, emoji_hash| - fpath = File.join(dir, "#{emoji_hash['unicode']}.png") - digest = Digest::SHA256.file(fpath).hexdigest + # We don't have `node_modules` available in built versions of GitLab + FileUtils.cp_r(Rails.root.join('node_modules', 'emoji-unicode-version', 'emoji-unicode-version-map.json'), File.join(Rails.root, 'fixtures', 'emojis')) - digests << { name: name, unicode: emoji_hash['unicode'], digest: digest } + dir = Gemojione.images_path + resultant_emoji_map = {} + + Gitlab::Emoji.emojis.each do |name, emoji_hash| + # Ignore aliases + unless Gitlab::Emoji.emojis_aliases.key?(name) + fpath = File.join(dir, "#{emoji_hash['unicode']}.png") + hash_digest = Digest::SHA256.file(fpath).hexdigest + + entry = { + category: emoji_hash['category'], + moji: emoji_hash['moji'], + unicodeVersion: Gitlab::Emoji.emoji_unicode_version(name), + digest: hash_digest, + } - aliases[name].each do |alias_name| - digests << { name: alias_name, unicode: emoji_hash['unicode'], digest: digest } + resultant_emoji_map[name] = entry end end out = File.join(Rails.root, 'fixtures', 'emojis', 'digests.json') File.open(out, 'w') do |handle| - handle.write(JSON.pretty_generate(digests)) + handle.write(JSON.pretty_generate(resultant_emoji_map)) end end @@ -55,21 +58,40 @@ namespace :gemojione do SPRITESHEET_WIDTH = 860 SPRITESHEET_HEIGHT = 840 + # Setup a map to rename image files + emoji_unicode_string_to_name_map = {} + Gitlab::Emoji.emojis.each do |name, emoji_hash| + # Ignore aliases + unless Gitlab::Emoji.emojis_aliases.key?(name) + emoji_unicode_string_to_name_map[emoji_hash['unicode']] = name + end + end + + # Copy the Gemojione assets to the temporary folder for renaming + emoji_dir = "app/assets/images/emoji" + FileUtils.rm_rf(emoji_dir) + FileUtils.mkdir_p(emoji_dir, mode: 0700) + FileUtils.cp_r(File.join(Gemojione.images_path, '.'), emoji_dir) + Dir[File.join(emoji_dir, "**/*.png")].each do |png| + image_path = png + rename_to_named_emoji_image!(emoji_unicode_string_to_name_map, image_path) + end + Dir.mktmpdir do |tmpdir| - # Copy the Gemojione assets to the temporary folder for resizing - FileUtils.cp_r(Gemojione.images_path, tmpdir) + FileUtils.cp_r(File.join(emoji_dir, '.'), tmpdir) Dir.chdir(tmpdir) do Dir["**/*.png"].each do |png| - resize!(File.join(tmpdir, png), SIZE) + tmp_image_path = File.join(tmpdir, png) + resize!(tmp_image_path, SIZE) end end - style_path = Rails.root.join(*%w(app assets stylesheets pages emojis.scss)) + style_path = Rails.root.join(*%w(app assets stylesheets framework emoji-sprites.scss)) # Combine the resized assets into a packed sprite and re-generate the SCSS SpriteFactory.cssurl = "image-url('$IMAGE')" - SpriteFactory.run!(File.join(tmpdir, 'png'), { + SpriteFactory.run!(tmpdir, { output_style: style_path, output_image: "app/assets/images/emoji.png", selector: '.emoji-', @@ -83,7 +105,7 @@ namespace :gemojione do # let's simplify it system(%Q(sed -i '' "s/width: #{SIZE}px; height: #{SIZE}px; background: image-url('emoji.png')/background-position:/" #{style_path})) system(%Q(sed -i '' "s/ no-repeat//" #{style_path})) - system(%Q(sed -i '' "s/ 0px/ 0/" #{style_path})) + system(%Q(sed -i '' "s/ 0px/ 0/g" #{style_path})) # Append a generic rule that applies to all Emojis File.open(style_path, 'a') do |f| @@ -92,6 +114,8 @@ namespace :gemojione do .emoji-icon { background-image: image-url('emoji.png'); background-repeat: no-repeat; + color: transparent; + text-indent: -99em; height: #{SIZE}px; width: #{SIZE}px; @@ -112,16 +136,17 @@ namespace :gemojione do # Now do it again but for Retina Dir.mktmpdir do |tmpdir| # Copy the Gemojione assets to the temporary folder for resizing - FileUtils.cp_r(Gemojione.images_path, tmpdir) + FileUtils.cp_r(File.join(emoji_dir, '.'), tmpdir) Dir.chdir(tmpdir) do Dir["**/*.png"].each do |png| - resize!(File.join(tmpdir, png), RETINA) + tmp_image_path = File.join(tmpdir, png) + resize!(tmp_image_path, RETINA) end end # Combine the resized assets into a packed sprite and re-generate the SCSS - SpriteFactory.run!(File.join(tmpdir), { + SpriteFactory.run!(tmpdir, { output_image: "app/assets/images/emoji@2x.png", style: false, nocomments: true, @@ -155,4 +180,20 @@ namespace :gemojione do image.write(image_path) { self.quality = 100 } image.destroy! end + + EMOJI_IMAGE_PATH_RE = /(.*?)(([0-9a-f]-?)+)\.png$/i + def rename_to_named_emoji_image!(emoji_unicode_string_to_name_map, image_path) + # Rename file from unicode to emoji name + matches = EMOJI_IMAGE_PATH_RE.match(image_path) + preceding_path = matches[1] + unicode_string = matches[2] + name = emoji_unicode_string_to_name_map[unicode_string] + if name + new_png_path = File.join(preceding_path, "#{name}.png") + FileUtils.mv(image_path, new_png_path) + new_png_path + else + puts "Warning: emoji_unicode_string_to_name_map missing entry for #{unicode_string}. Full path: #{image_path}" + end + end end diff --git a/lib/tasks/gitlab/assets.rake b/lib/tasks/gitlab/assets.rake new file mode 100644 index 00000000000..003d57adbbd --- /dev/null +++ b/lib/tasks/gitlab/assets.rake @@ -0,0 +1,48 @@ +namespace :gitlab do + namespace :assets do + desc 'GitLab | Assets | Compile all frontend assets' + task compile: [ + 'yarn:check', + 'rake:assets:precompile', + 'webpack:compile', + 'fix_urls' + ] + + desc 'GitLab | Assets | Clean up old compiled frontend assets' + task clean: ['rake:assets:clean'] + + desc 'GitLab | Assets | Remove all compiled frontend assets' + task purge: ['rake:assets:clobber'] + + desc 'GitLab | Assets | Uninstall frontend dependencies' + task purge_modules: ['yarn:clobber'] + + desc 'GitLab | Assets | Fix all absolute url references in CSS' + task :fix_urls do + css_files = Dir['public/assets/*.css'] + css_files.each do |file| + # replace url(/assets/*) with url(./*) + puts "Fixing #{file}" + system "sed", "-i", "-e", 's/url(\([\"\']\?\)\/assets\//url(\1.\//g', file + + # rewrite the corresponding gzip file (if it exists) + gzip = "#{file}.gz" + if File.exist?(gzip) + puts "Fixing #{gzip}" + + FileUtils.rm(gzip) + mtime = File.stat(file).mtime + + File.open(gzip, 'wb+') do |f| + gz = Zlib::GzipWriter.new(f, Zlib::BEST_COMPRESSION) + gz.mtime = mtime + gz.write IO.binread(file) + gz.close + + File.utime(mtime, mtime, f.path) + end + end + end + end + end +end diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index a9f1255e8cf..1650263b98d 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -13,6 +13,7 @@ namespace :gitlab do Rake::Task["gitlab:backup:uploads:create"].invoke Rake::Task["gitlab:backup:builds:create"].invoke Rake::Task["gitlab:backup:artifacts:create"].invoke + Rake::Task["gitlab:backup:pages:create"].invoke Rake::Task["gitlab:backup:lfs:create"].invoke Rake::Task["gitlab:backup:registry:create"].invoke @@ -56,6 +57,7 @@ namespace :gitlab do Rake::Task['gitlab:backup:uploads:restore'].invoke unless backup.skipped?('uploads') Rake::Task['gitlab:backup:builds:restore'].invoke unless backup.skipped?('builds') Rake::Task['gitlab:backup:artifacts:restore'].invoke unless backup.skipped?('artifacts') + Rake::Task["gitlab:backup:pages:restore"].invoke unless backup.skipped?('pages') Rake::Task['gitlab:backup:lfs:restore'].invoke unless backup.skipped?('lfs') Rake::Task['gitlab:backup:registry:restore'].invoke unless backup.skipped?('registry') Rake::Task['gitlab:shell:setup'].invoke @@ -159,6 +161,25 @@ namespace :gitlab do end end + namespace :pages do + task create: :environment do + $progress.puts "Dumping pages ... ".color(:blue) + + if ENV["SKIP"] && ENV["SKIP"].include?("pages") + $progress.puts "[SKIPPED]".color(:cyan) + else + Backup::Pages.new.dump + $progress.puts "done".color(:green) + end + end + + task restore: :environment do + $progress.puts "Restoring pages ... ".color(:blue) + Backup::Pages.new.restore + $progress.puts "done".color(:green) + end + end + namespace :lfs do task create: :environment do $progress.puts "Dumping lfs objects ... ".color(:blue) diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 35c4194e87c..a9a48f7188f 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -6,8 +6,6 @@ namespace :gitlab do gitlab:ldap:check gitlab:app:check} - - namespace :app do desc "GitLab | Check the configuration of the GitLab Rails app" task check: :environment do @@ -34,7 +32,6 @@ namespace :gitlab do finished_checking "GitLab" end - # Checks ######################## @@ -194,7 +191,7 @@ namespace :gitlab do def check_migrations_are_up print "All migrations up? ... " - migration_status, _ = Gitlab::Popen.popen(%W(bundle exec rake db:migrate:status)) + migration_status, _ = Gitlab::Popen.popen(%w(bundle exec rake db:migrate:status)) unless migration_status =~ /down\s+\d{14}/ puts "yes".color(:green) @@ -279,7 +276,7 @@ namespace :gitlab do upload_path_tmp = File.join(upload_path, 'tmp') if File.stat(upload_path).mode == 040700 - unless Dir.exists?(upload_path_tmp) + unless Dir.exist?(upload_path_tmp) puts 'skipped (no tmp uploads folder yet)'.color(:magenta) return end @@ -316,7 +313,7 @@ namespace :gitlab do min_redis_version = "2.8.0" print "Redis version >= #{min_redis_version}? ... " - redis_version = run_command(%W(redis-cli --version)) + redis_version = run_command(%w(redis-cli --version)) redis_version = redis_version.try(:match, /redis-cli (\d+\.\d+\.\d+)/) if redis_version && (Gem::Version.new(redis_version[1]) > Gem::Version.new(min_redis_version)) @@ -351,14 +348,14 @@ namespace :gitlab do finished_checking "GitLab Shell" end - # Checks ######################## def check_repo_base_exists puts "Repo base directory exists?" - Gitlab.config.repositories.storages.each do |name, repo_base_path| + Gitlab.config.repositories.storages.each do |name, repository_storage| + repo_base_path = repository_storage['path'] print "#{name}... " if File.exist?(repo_base_path) @@ -382,12 +379,13 @@ namespace :gitlab do def check_repo_base_is_not_symlink puts "Repo storage directories are symlinks?" - Gitlab.config.repositories.storages.each do |name, repo_base_path| + Gitlab.config.repositories.storages.each do |name, repository_storage| + repo_base_path = repository_storage['path'] print "#{name}... " unless File.exist?(repo_base_path) puts "can't check because of previous errors".color(:magenta) - return + break end unless File.symlink?(repo_base_path) @@ -405,12 +403,13 @@ namespace :gitlab do def check_repo_base_permissions puts "Repo paths access is drwxrws---?" - Gitlab.config.repositories.storages.each do |name, repo_base_path| + Gitlab.config.repositories.storages.each do |name, repository_storage| + repo_base_path = repository_storage['path'] print "#{name}... " unless File.exist?(repo_base_path) puts "can't check because of previous errors".color(:magenta) - return + break end if File.stat(repo_base_path).mode.to_s(8).ends_with?("2770") @@ -435,12 +434,13 @@ namespace :gitlab do gitlab_shell_owner_group = Gitlab.config.gitlab_shell.owner_group puts "Repo paths owned by #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group}?" - Gitlab.config.repositories.storages.each do |name, repo_base_path| + Gitlab.config.repositories.storages.each do |name, repository_storage| + repo_base_path = repository_storage['path'] print "#{name}... " unless File.exist?(repo_base_path) puts "can't check because of previous errors".color(:magenta) - return + break end uid = uid_for(gitlab_shell_ssh_user) @@ -493,7 +493,6 @@ namespace :gitlab do ) fix_and_rerun end - end end @@ -565,8 +564,6 @@ namespace :gitlab do end end - - namespace :sidekiq do desc "GitLab | Check the configuration of Sidekiq" task check: :environment do @@ -579,7 +576,6 @@ namespace :gitlab do finished_checking "Sidekiq" end - # Checks ######################## @@ -621,12 +617,11 @@ namespace :gitlab do end def sidekiq_process_count - ps_ux, _ = Gitlab::Popen.popen(%W(ps ux)) + ps_ux, _ = Gitlab::Popen.popen(%w(ps uxww)) ps_ux.scan(/sidekiq \d+\.\d+\.\d+/).count end end - namespace :incoming_email do desc "GitLab | Check the configuration of Reply by email" task check: :environment do @@ -649,7 +644,6 @@ namespace :gitlab do finished_checking "Reply by email" end - # Checks ######################## @@ -724,8 +718,11 @@ namespace :gitlab do def check_imap_authentication print "IMAP server credentials are correct? ... " - config_path = Rails.root.join('config', 'mail_room.yml') - config_file = YAML.load(ERB.new(File.read(config_path)).result) + config_path = Rails.root.join('config', 'mail_room.yml').to_s + erb = ERB.new(File.read(config_path)) + erb.filename = config_path + config_file = YAML.load(erb.result) + config = config_file[:mailboxes].first if config @@ -754,7 +751,7 @@ namespace :gitlab do end def mail_room_running? - ps_ux, _ = Gitlab::Popen.popen(%W(ps ux)) + ps_ux, _ = Gitlab::Popen.popen(%w(ps uxww)) ps_ux.include?("mail_room") end end @@ -802,13 +799,13 @@ namespace :gitlab do def check_ldap_auth(adapter) auth = adapter.config.has_auth? - if auth && adapter.ldap.bind - message = 'Success'.color(:green) - elsif auth - message = 'Failed. Check `bind_dn` and `password` configuration values'.color(:red) - else - message = 'Anonymous. No `bind_dn` or `password` configured'.color(:yellow) - end + message = if auth && adapter.ldap.bind + 'Success'.color(:green) + elsif auth + 'Failed. Check `bind_dn` and `password` configuration values'.color(:red) + else + 'Anonymous. No `bind_dn` or `password` configured'.color(:yellow) + end puts "LDAP authentication... #{message}" end @@ -817,8 +814,8 @@ namespace :gitlab do namespace :repo do desc "GitLab | Check the integrity of the repositories managed by GitLab" task check: :environment do - Gitlab.config.repositories.storages.each do |name, path| - namespace_dirs = Dir.glob(File.join(path, '*')) + Gitlab.config.repositories.storages.each do |name, repository_storage| + namespace_dirs = Dir.glob(File.join(repository_storage['path'], '*')) namespace_dirs.each do |namespace_dir| repo_dirs = Dir.glob(File.join(namespace_dir, '*')) @@ -835,11 +832,11 @@ namespace :gitlab do user = User.find_by(username: username) if user repo_dirs = user.authorized_projects.map do |p| - File.join( - p.repository_storage_path, - "#{p.path_with_namespace}.git" - ) - end + File.join( + p.repository_storage_path, + "#{p.path_with_namespace}.git" + ) + end repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) } else @@ -852,7 +849,7 @@ namespace :gitlab do ########################## def fix_and_rerun - puts " Please #{"fix the error above"} and rerun the checks.".color(:red) + puts " Please fix the error above and rerun the checks.".color(:red) end def for_more_information(*sources) @@ -914,7 +911,7 @@ namespace :gitlab do def check_ruby_version required_version = Gitlab::VersionInfo.new(2, 1, 0) - current_version = Gitlab::VersionInfo.parse(run_command(%W(ruby --version))) + current_version = Gitlab::VersionInfo.parse(run_command(%w(ruby --version))) print "Ruby version >= #{required_version} ? ... " @@ -985,13 +982,13 @@ namespace :gitlab do end def check_config_lock(repo_dir) - config_exists = File.exist?(File.join(repo_dir,'config.lock')) + config_exists = File.exist?(File.join(repo_dir, 'config.lock')) config_output = config_exists ? 'yes'.color(:red) : 'no'.color(:green) puts "'config.lock' file exists?".color(:yellow) + " ... #{config_output}" end def check_ref_locks(repo_dir) - lock_files = Dir.glob(File.join(repo_dir,'refs/heads/*.lock')) + lock_files = Dir.glob(File.join(repo_dir, 'refs/heads/*.lock')) if lock_files.present? puts "Ref lock files exist:".color(:red) lock_files.each do |lock_file| diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index 4a696a52b4d..f76bef5f4bf 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -6,7 +6,8 @@ namespace :gitlab do remove_flag = ENV['REMOVE'] namespaces = Namespace.pluck(:path) - Gitlab.config.repositories.storages.each do |name, git_base_path| + Gitlab.config.repositories.storages.each do |name, repository_storage| + git_base_path = repository_storage['path'] all_dirs = Dir.glob(git_base_path + '/*') puts git_base_path.color(:yellow) @@ -25,7 +26,6 @@ namespace :gitlab do end all_dirs.each do |dir_path| - if remove_flag if FileUtils.rm_rf dir_path puts "Removed...#{dir_path}".color(:red) @@ -48,17 +48,18 @@ namespace :gitlab do warn_user_is_not_gitlab move_suffix = "+orphaned+#{Time.now.to_i}" - Gitlab.config.repositories.storages.each do |name, repo_root| + Gitlab.config.repositories.storages.each do |name, repository_storage| + repo_root = repository_storage['path'] # Look for global repos (legacy, depth 1) and normal repos (depth 2) IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find| find.each_line do |path| path.chomp! - repo_with_namespace = path. - sub(repo_root, ''). - sub(%r{^/*}, ''). - chomp('.git'). - chomp('.wiki') - next if Project.find_with_namespace(repo_with_namespace) + repo_with_namespace = path + .sub(repo_root, '') + .sub(%r{^/*}, '') + .chomp('.git') + .chomp('.wiki') + next if Project.find_by_full_path(repo_with_namespace) new_path = path + move_suffix puts path.inspect + ' -> ' + new_path.inspect File.rename(path, new_path) diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 7c96bc864ce..5476438b8fa 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -23,7 +23,7 @@ namespace :gitlab do end desc 'Drop all tables' - task :drop_tables => :environment do + task drop_tables: :environment do connection = ActiveRecord::Base.connection # If MySQL, turn off foreign key checks @@ -62,9 +62,9 @@ namespace :gitlab do ref = Shellwords.escape(args[:ref]) - migrations = `git diff #{ref}.. --name-only -- db/migrate`.lines. - map { |file| Rails.root.join(file.strip).to_s }. - select { |file| File.file?(file) } + migrations = `git diff #{ref}.. --diff-filter=A --name-only -- db/migrate`.lines + .map { |file| Rails.root.join(file.strip).to_s } + .select { |file| File.file?(file) } Gitlab::DowntimeCheck.new.check_and_print(migrations) end diff --git a/lib/tasks/gitlab/dev.rake b/lib/tasks/gitlab/dev.rake index 7db0779def8..7ccda04a35f 100644 --- a/lib/tasks/gitlab/dev.rake +++ b/lib/tasks/gitlab/dev.rake @@ -4,7 +4,7 @@ namespace :gitlab do task :ee_compat_check, [:branch] => :environment do |_, args| opts = if ENV['CI'] - { branch: ENV['CI_BUILD_REF_NAME'] } + { branch: ENV['CI_COMMIT_REF_NAME'] } else unless args[:branch] puts "Must specify a branch as an argument".color(:red) diff --git a/lib/tasks/gitlab/git.rake b/lib/tasks/gitlab/git.rake index a67c1fe1f27..cf82134d97e 100644 --- a/lib/tasks/gitlab/git.rake +++ b/lib/tasks/gitlab/git.rake @@ -1,6 +1,5 @@ namespace :gitlab do namespace :git do - desc "GitLab | Git | Repack" task repack: :environment do failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} repack -a --quiet), "Repacking repo") @@ -50,6 +49,5 @@ namespace :gitlab do puts "The following repositories reported errors:".color(:red) failures.each { |f| puts "- #{f}" } end - end end diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake new file mode 100644 index 00000000000..c288e17ac8d --- /dev/null +++ b/lib/tasks/gitlab/gitaly.rake @@ -0,0 +1,23 @@ +namespace :gitlab do + namespace :gitaly do + desc "GitLab | Install or upgrade gitaly" + task :install, [:dir] => :environment do |t, args| + warn_user_is_not_gitlab + unless args.dir.present? + abort %(Please specify the directory where you want to install gitaly:\n rake "gitlab:gitaly:install[/home/git/gitaly]") + end + + tag = "v#{Gitlab::GitalyClient.expected_server_version}" + repo = 'https://gitlab.com/gitlab-org/gitaly.git' + + checkout_or_clone_tag(tag: tag, repo: repo, target_dir: args.dir) + + _, status = Gitlab::Popen.popen(%w[which gmake]) + command = status.zero? ? 'gmake' : 'make' + + Dir.chdir(args.dir) do + run_command!([command]) + end + end + end +end diff --git a/lib/tasks/gitlab/import.rake b/lib/tasks/gitlab/import.rake index a2eca74a3c8..48bd9139ce8 100644 --- a/lib/tasks/gitlab/import.rake +++ b/lib/tasks/gitlab/import.rake @@ -11,7 +11,8 @@ namespace :gitlab do # desc "GitLab | Import bare repositories from repositories -> storages into GitLab project instance" task repos: :environment do - Gitlab.config.repositories.storages.each do |name, git_base_path| + Gitlab.config.repositories.storages.each_value do |repository_storage| + git_base_path = repository_storage['path'] repos_to_import = Dir.glob(git_base_path + '/**/*.git') repos_to_import.each do |repo_path| @@ -29,7 +30,7 @@ namespace :gitlab do next end - project = Project.find_with_namespace(path) + project = Project.find_by_full_path(path) if project puts " * #{project.name} (#{repo_path}) exists" @@ -46,7 +47,7 @@ namespace :gitlab do group = Namespace.find_by(path: group_name) # create group namespace unless group - group = Group.new(:name => group_name) + group = Group.new(name: group_name) group.path = group_name group.owner = user if group.save @@ -63,7 +64,7 @@ namespace :gitlab do if project.persisted? puts " * Created #{project.name} (#{repo_path})".color(:green) - ProjectCacheWorker.perform(project.id) + ProjectCacheWorker.perform_async(project.id) else puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red) puts " Errors: #{project.errors.messages}".color(:red) diff --git a/lib/tasks/gitlab/import_export.rake b/lib/tasks/gitlab/import_export.rake index c2c6031db67..dd1825c8a9e 100644 --- a/lib/tasks/gitlab/import_export.rake +++ b/lib/tasks/gitlab/import_export.rake @@ -7,7 +7,7 @@ namespace :gitlab do desc "GitLab | Display exported DB structure" task data: :environment do - puts YAML.load_file(Gitlab::ImportExport.config_file)['project_tree'].to_yaml(:SortKeys => true) + puts YAML.load_file(Gitlab::ImportExport.config_file)['project_tree'].to_yaml(SortKeys: true) end end end diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake index dffea8ed155..a2a2db487b7 100644 --- a/lib/tasks/gitlab/info.rake +++ b/lib/tasks/gitlab/info.rake @@ -2,31 +2,35 @@ namespace :gitlab do namespace :env do desc "GitLab | Show information about GitLab and its environment" task info: :environment do - # check if there is an RVM environment - rvm_version = run_and_match(%W(rvm --version), /[\d\.]+/).try(:to_s) + rvm_version = run_and_match(%w(rvm --version), /[\d\.]+/).try(:to_s) # check Ruby version - ruby_version = run_and_match(%W(ruby --version), /[\d\.p]+/).try(:to_s) + ruby_version = run_and_match(%w(ruby --version), /[\d\.p]+/).try(:to_s) # check Gem version - gem_version = run_command(%W(gem --version)) - # check Bundler version - bunder_version = run_and_match(%W(bundle --version), /[\d\.]+/).try(:to_s) + gem_version = run_command(%w(gem --version)) # check Bundler version - rake_version = run_and_match(%W(rake --version), /[\d\.]+/).try(:to_s) + bunder_version = run_and_match(%w(bundle --version), /[\d\.]+/).try(:to_s) + # check Rake version + rake_version = run_and_match(%w(rake --version), /[\d\.]+/).try(:to_s) + # check redis version + redis_version = run_and_match(%w(redis-cli --version), /redis-cli (\d+\.\d+\.\d+)/).to_a + # check Git version + git_version = run_and_match([Gitlab.config.git.bin_path, '--version'], /git version ([\d\.]+)/).to_a puts "" puts "System information".color(:yellow) puts "System:\t\t#{os_name || "unknown".color(:red)}" - puts "Current User:\t#{run_command(%W(whoami))}" + puts "Current User:\t#{run_command(%w(whoami))}" puts "Using RVM:\t#{rvm_version.present? ? "yes".color(:green) : "no"}" puts "RVM Version:\t#{rvm_version}" if rvm_version.present? puts "Ruby Version:\t#{ruby_version || "unknown".color(:red)}" puts "Gem Version:\t#{gem_version || "unknown".color(:red)}" puts "Bundler Version:#{bunder_version || "unknown".color(:red)}" puts "Rake Version:\t#{rake_version || "unknown".color(:red)}" + puts "Redis Version:\t#{redis_version[1] || "unknown".color(:red)}" + puts "Git Version:\t#{git_version[1] || "unknown".color(:red)}" puts "Sidekiq Version:#{Sidekiq::VERSION}" - # check database adapter database_adapter = ActiveRecord::Base.connection.adapter_name.downcase @@ -51,8 +55,6 @@ namespace :gitlab do puts "Using Omniauth:\t#{Gitlab.config.omniauth.enabled ? "yes".color(:green) : "no"}" puts "Omniauth Providers: #{omniauth_providers.join(', ')}" if Gitlab.config.omniauth.enabled - - # check Gitolite version gitlab_shell_version_file = "#{Gitlab.config.gitlab_shell.hooks_path}/../VERSION" if File.readable?(gitlab_shell_version_file) @@ -63,12 +65,11 @@ namespace :gitlab do puts "GitLab Shell".color(:yellow) puts "Version:\t#{gitlab_shell_version || "unknown".color(:red)}" puts "Repository storage paths:" - Gitlab.config.repositories.storages.each do |name, path| - puts "- #{name}: \t#{path}" + Gitlab.config.repositories.storages.each do |name, repository_storage| + puts "- #{name}: \t#{repository_storage['path']}" end puts "Hooks:\t\t#{Gitlab.config.gitlab_shell.hooks_path}" puts "Git:\t\t#{Gitlab.config.git.bin_path}" - end end end diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index 5a09cd7ce41..dd2fda54e62 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -20,10 +20,10 @@ namespace :gitlab do config = { user: Gitlab.config.gitlab.user, gitlab_url: gitlab_url, - http_settings: {self_signed_cert: false}.stringify_keys, + http_settings: { self_signed_cert: false }.stringify_keys, auth_file: File.join(user_home, ".ssh", "authorized_keys"), redis: { - bin: %x{which redis-cli}.chomp, + bin: `which redis-cli`.chomp, namespace: "resque:gitlab" }.stringify_keys, log_level: "INFO", @@ -43,7 +43,7 @@ namespace :gitlab do File.open("config.yml", "w+") {|f| f.puts config.to_yaml} # Launch installation process - system(*%W(bin/install) + repository_storage_paths_args) + system(*%w(bin/install) + repository_storage_paths_args) end # (Re)create hooks diff --git a/lib/tasks/gitlab/sidekiq.rake b/lib/tasks/gitlab/sidekiq.rake index 7e2a6668e59..6cbc83b8973 100644 --- a/lib/tasks/gitlab/sidekiq.rake +++ b/lib/tasks/gitlab/sidekiq.rake @@ -1,13 +1,13 @@ namespace :gitlab do namespace :sidekiq do - QUEUE = 'queue:post_receive' + QUEUE = 'queue:post_receive'.freeze desc 'Drop all Sidekiq PostReceive jobs for a given project' - task :drop_post_receive , [:project] => :environment do |t, args| + task :drop_post_receive, [:project] => :environment do |t, args| unless args.project.present? abort "Please specify the project you want to drop PostReceive jobs for:\n rake gitlab:sidekiq:drop_post_receive[group/project]" end - project_path = Project.find_with_namespace(args.project).repository.path_to_repo + project_path = Project.find_by_full_path(args.project).repository.path_to_repo Sidekiq.redis do |redis| unless redis.exists(QUEUE) @@ -21,7 +21,7 @@ namespace :gitlab do # new jobs already. We will repopulate it with the old jobs, skipping the # ones we want to drop. dropped = 0 - while (job = redis.lpop(temp_queue)) do + while (job = redis.lpop(temp_queue)) if repo_path(job) == project_path dropped += 1 else diff --git a/lib/tasks/gitlab/task_helpers.rb b/lib/tasks/gitlab/task_helpers.rb index e128738b5f8..cdba2262bc2 100644 --- a/lib/tasks/gitlab/task_helpers.rb +++ b/lib/tasks/gitlab/task_helpers.rb @@ -19,23 +19,20 @@ module Gitlab # It will primarily use lsb_relase to determine the OS. # It has fallbacks to Debian, SuSE, OS X and systems running systemd. def os_name - os_name = run_command(%W(lsb_release -irs)) - os_name ||= if File.readable?('/etc/system-release') - File.read('/etc/system-release') - end - os_name ||= if File.readable?('/etc/debian_version') - debian_version = File.read('/etc/debian_version') - "Debian #{debian_version}" - end - os_name ||= if File.readable?('/etc/SuSE-release') - File.read('/etc/SuSE-release') - end - os_name ||= if os_x_version = run_command(%W(sw_vers -productVersion)) - "Mac OS X #{os_x_version}" - end - os_name ||= if File.readable?('/etc/os-release') - File.read('/etc/os-release').match(/PRETTY_NAME=\"(.+)\"/)[1] - end + os_name = run_command(%w(lsb_release -irs)) + os_name ||= + if File.readable?('/etc/system-release') + File.read('/etc/system-release') + elsif File.readable?('/etc/debian_version') + "Debian #{File.read('/etc/debian_version')}" + elsif File.readable?('/etc/SuSE-release') + File.read('/etc/SuSE-release') + elsif os_x_version = run_command(%w(sw_vers -productVersion)) + "Mac OS X #{os_x_version}" + elsif File.readable?('/etc/os-release') + File.read('/etc/os-release').match(/PRETTY_NAME=\"(.+)\"/)[1] + end + os_name.try(:squish!) end @@ -84,7 +81,7 @@ module Gitlab def run_command!(command) output, status = Gitlab::Popen.popen(command) - raise Gitlab::TaskFailedError unless status.zero? + raise Gitlab::TaskFailedError.new(output) unless status.zero? output end @@ -104,7 +101,7 @@ module Gitlab def warn_user_is_not_gitlab unless @warned_user_not_gitlab gitlab_user = Gitlab.config.gitlab.user - current_user = run_command(%W(whoami)).chomp + current_user = run_command(%w(whoami)).chomp unless current_user == gitlab_user puts " Warning ".color(:black).background(:yellow) puts " You are running as user #{current_user.color(:magenta)}, we hope you know what you are doing." @@ -133,8 +130,8 @@ module Gitlab end def all_repos - Gitlab.config.repositories.storages.each do |name, path| - IO.popen(%W(find #{path} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find| + Gitlab.config.repositories.storages.each_value do |repository_storage| + IO.popen(%W(find #{repository_storage['path']} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find| find.each_line do |path| yield path.chomp end @@ -143,7 +140,7 @@ module Gitlab end def repository_storage_paths_args - Gitlab.config.repositories.storages.values + Gitlab.config.repositories.storages.values.map { |rs| rs['path'] } end def user_home @@ -171,14 +168,14 @@ module Gitlab def reset_to_tag(tag_wanted, target_dir) tag = - begin - # First try to checkout without fetching - # to avoid stalling tests if the Internet is down. - run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} describe -- #{tag_wanted}]) - rescue Gitlab::TaskFailedError - run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} fetch origin]) - run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} describe -- origin/#{tag_wanted}]) - end + begin + # First try to checkout without fetching + # to avoid stalling tests if the Internet is down. + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} describe -- #{tag_wanted}]) + rescue Gitlab::TaskFailedError + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} fetch origin]) + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} describe -- origin/#{tag_wanted}]) + end if tag run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} reset --hard #{tag.strip}]) diff --git a/lib/tasks/gitlab/test.rake b/lib/tasks/gitlab/test.rake index 4d4e746503a..523b0fa055b 100644 --- a/lib/tasks/gitlab/test.rake +++ b/lib/tasks/gitlab/test.rake @@ -2,15 +2,15 @@ namespace :gitlab do desc "GitLab | Run all tests" task :test do cmds = [ - %W(rake brakeman), - %W(rake rubocop), - %W(rake spinach), - %W(rake spec), - %W(rake teaspoon) + %w(rake brakeman), + %w(rake rubocop), + %w(rake spinach), + %w(rake spec), + %w(rake karma) ] cmds.each do |cmd| - system({'RAILS_ENV' => 'test', 'force' => 'yes'}, *cmd) or raise("#{cmd} failed!") + system({ 'RAILS_ENV' => 'test', 'force' => 'yes' }, *cmd) || raise("#{cmd} failed!") end end end diff --git a/lib/tasks/gitlab/track_deployment.rake b/lib/tasks/gitlab/track_deployment.rake index 84aa2e8507a..6f101aea303 100644 --- a/lib/tasks/gitlab/track_deployment.rake +++ b/lib/tasks/gitlab/track_deployment.rake @@ -1,8 +1,8 @@ namespace :gitlab do desc 'GitLab | Tracks a deployment in GitLab Performance Monitoring' task track_deployment: :environment do - metric = Gitlab::Metrics::Metric. - new('deployments', version: Gitlab::VERSION) + metric = Gitlab::Metrics::Metric + .new('deployments', version: Gitlab::VERSION) Gitlab::Metrics.submit_metrics([metric.to_hash]) end diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake index b77a5bb62d1..dbdfb335a5c 100644 --- a/lib/tasks/gitlab/update_templates.rake +++ b/lib/tasks/gitlab/update_templates.rake @@ -46,7 +46,7 @@ namespace :gitlab do "https://gitlab.com/gitlab-org/gitlab-ci-yml.git", /(\.{1,2}|LICENSE|Pages|autodeploy|\.gitlab-ci.yml)\z/ ) - ] + ].freeze def vendor_directory Rails.root.join('vendor') diff --git a/lib/tasks/gitlab/web_hook.rake b/lib/tasks/gitlab/web_hook.rake index 49530e7a372..5a1c8006052 100644 --- a/lib/tasks/gitlab/web_hook.rake +++ b/lib/tasks/gitlab/web_hook.rake @@ -1,7 +1,7 @@ namespace :gitlab do namespace :web_hook do desc "GitLab | Adds a webhook to the projects" - task :add => :environment do + task add: :environment do web_hook_url = ENV['URL'] namespace_path = ENV['NAMESPACE'] @@ -21,7 +21,7 @@ namespace :gitlab do end desc "GitLab | Remove a webhook from the projects" - task :rm => :environment do + task rm: :environment do web_hook_url = ENV['URL'] namespace_path = ENV['NAMESPACE'] @@ -34,7 +34,7 @@ namespace :gitlab do end desc "GitLab | List webhooks" - task :list => :environment do + task list: :environment do namespace_path = ENV['NAMESPACE'] projects = find_projects(namespace_path) diff --git a/lib/tasks/grape.rake b/lib/tasks/grape.rake index 9980e0b7984..ea2698da606 100644 --- a/lib/tasks/grape.rake +++ b/lib/tasks/grape.rake @@ -2,7 +2,11 @@ namespace :grape do desc 'Print compiled grape routes' task routes: :environment do API::API.routes.each do |route| - puts route + puts "#{route.options[:method]} #{route.path} - #{route_description(route.options)}" end end + + def route_description(options) + options[:settings][:description][:description] if options[:settings][:description] + end end diff --git a/lib/tasks/karma.rake b/lib/tasks/karma.rake new file mode 100644 index 00000000000..62a12174efa --- /dev/null +++ b/lib/tasks/karma.rake @@ -0,0 +1,21 @@ +unless Rails.env.production? + namespace :karma do + desc 'GitLab | Karma | Generate fixtures for JavaScript tests' + RSpec::Core::RakeTask.new(:fixtures, [:pattern]) do |t, args| + args.with_defaults(pattern: 'spec/javascripts/fixtures/*.rb') + ENV['NO_KNAPSACK'] = 'true' + t.pattern = args[:pattern] + t.rspec_opts = '--format documentation' + end + + desc 'GitLab | Karma | Run JavaScript tests' + task tests: ['yarn:check'] do + sh "yarn run karma" do |ok, res| + abort('rake karma:tests failed') unless ok + end + end + end + + desc 'GitLab | Karma | Shortcut for karma:fixtures and karma:tests' + task karma: ['karma:fixtures', 'karma:tests'] +end diff --git a/lib/tasks/lint.rake b/lib/tasks/lint.rake index 32b668df3bf..7b63e93db0e 100644 --- a/lib/tasks/lint.rake +++ b/lib/tasks/lint.rake @@ -6,4 +6,3 @@ unless Rails.env.production? end end end - diff --git a/lib/tasks/migrate/migrate_iids.rake b/lib/tasks/migrate/migrate_iids.rake index 4f2486157b7..fc2cea8c016 100644 --- a/lib/tasks/migrate/migrate_iids.rake +++ b/lib/tasks/migrate/migrate_iids.rake @@ -24,7 +24,7 @@ task migrate_iids: :environment do else print 'F' end - rescue => ex + rescue print 'F' end end diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake index f5caca3ddbf..8938bc515f5 100644 --- a/lib/tasks/migrate/setup_postgresql.rake +++ b/lib/tasks/migrate/setup_postgresql.rake @@ -3,10 +3,12 @@ require Rails.root.join('lib/gitlab/database/migration_helpers') require Rails.root.join('db/migrate/20151007120511_namespaces_projects_path_lower_indexes') require Rails.root.join('db/migrate/20151008110232_add_users_lower_username_email_indexes') require Rails.root.join('db/migrate/20161212142807_add_lower_path_index_to_routes') +require Rails.root.join('db/migrate/20170317203554_index_routes_path_for_like') desc 'GitLab | Sets up PostgreSQL' task setup_postgresql: :environment do NamespacesProjectsPathLowerIndexes.new.up AddUsersLowerUsernameEmailIndexes.new.up AddLowerPathIndexToRoutes.new.up + IndexRoutesPathForLike.new.up end diff --git a/lib/tasks/services.rake b/lib/tasks/services.rake index 39541c0b9c6..56b81106c5f 100644 --- a/lib/tasks/services.rake +++ b/lib/tasks/services.rake @@ -76,23 +76,23 @@ namespace :services do end param_hash - end.sort_by { |p| p[:required] ? 0 : 1 } + end + service_hash[:params].sort_by! { |p| p[:required] ? 0 : 1 } - puts "Collected data for: #{service.title}, #{Time.now-service_start}" + puts "Collected data for: #{service.title}, #{Time.now - service_start}" service_hash end doc_start = Time.now doc_path = File.join(Rails.root, 'doc', 'api', 'services.md') - result = ERB.new(services_template, 0 , '>') + result = ERB.new(services_template, 0, '>') .result(OpenStruct.new(services: services).instance_eval { binding }) File.open(doc_path, 'w') do |f| f.write result end - puts "write a new service.md to: #{doc_path.to_s}, #{Time.now-doc_start}" - + puts "write a new service.md to: #{doc_path}, #{Time.now - doc_start}" end end diff --git a/lib/tasks/sidekiq.rake b/lib/tasks/sidekiq.rake index d1f6ed87704..dd9ce86f7ca 100644 --- a/lib/tasks/sidekiq.rake +++ b/lib/tasks/sidekiq.rake @@ -1,21 +1,21 @@ namespace :sidekiq do desc "GitLab | Stop sidekiq" task :stop do - system *%W(bin/background_jobs stop) + system(*%w(bin/background_jobs stop)) end desc "GitLab | Start sidekiq" task :start do - system *%W(bin/background_jobs start) + system(*%w(bin/background_jobs start)) end desc 'GitLab | Restart sidekiq' task :restart do - system *%W(bin/background_jobs restart) + system(*%w(bin/background_jobs restart)) end desc "GitLab | Start sidekiq with launchd on Mac OS X" task :launchd do - system *%W(bin/background_jobs start_no_deamonize) + system(*%w(bin/background_jobs start_no_deamonize)) end end diff --git a/lib/tasks/spec.rake b/lib/tasks/spec.rake index 2cf7a25a0fd..602c60be828 100644 --- a/lib/tasks/spec.rake +++ b/lib/tasks/spec.rake @@ -4,8 +4,8 @@ namespace :spec do desc 'GitLab | Rspec | Run request specs' task :api do cmds = [ - %W(rake gitlab:setup), - %W(rspec spec --tag @api) + %w(rake gitlab:setup), + %w(rspec spec --tag @api) ] run_commands(cmds) end @@ -13,8 +13,8 @@ namespace :spec do desc 'GitLab | Rspec | Run feature specs' task :feature do cmds = [ - %W(rake gitlab:setup), - %W(rspec spec --tag @feature) + %w(rake gitlab:setup), + %w(rspec spec --tag @feature) ] run_commands(cmds) end @@ -22,8 +22,8 @@ namespace :spec do desc 'GitLab | Rspec | Run model specs' task :models do cmds = [ - %W(rake gitlab:setup), - %W(rspec spec --tag @models) + %w(rake gitlab:setup), + %w(rspec spec --tag @models) ] run_commands(cmds) end @@ -31,8 +31,8 @@ namespace :spec do desc 'GitLab | Rspec | Run service specs' task :services do cmds = [ - %W(rake gitlab:setup), - %W(rspec spec --tag @services) + %w(rake gitlab:setup), + %w(rspec spec --tag @services) ] run_commands(cmds) end @@ -40,8 +40,8 @@ namespace :spec do desc 'GitLab | Rspec | Run lib specs' task :lib do cmds = [ - %W(rake gitlab:setup), - %W(rspec spec --tag @lib) + %w(rake gitlab:setup), + %w(rspec spec --tag @lib) ] run_commands(cmds) end @@ -49,8 +49,8 @@ namespace :spec do desc 'GitLab | Rspec | Run other specs' task :other do cmds = [ - %W(rake gitlab:setup), - %W(rspec spec --tag ~@api --tag ~@feature --tag ~@models --tag ~@lib --tag ~@services) + %w(rake gitlab:setup), + %w(rspec spec --tag ~@api --tag ~@feature --tag ~@models --tag ~@lib --tag ~@services) ] run_commands(cmds) end @@ -59,14 +59,14 @@ end desc "GitLab | Run specs" task :spec do cmds = [ - %W(rake gitlab:setup), - %W(rspec spec), + %w(rake gitlab:setup), + %w(rspec spec), ] run_commands(cmds) end def run_commands(cmds) cmds.each do |cmd| - system({'RAILS_ENV' => 'test', 'force' => 'yes'}, *cmd) or raise("#{cmd} failed!") + system({ 'RAILS_ENV' => 'test', 'force' => 'yes' }, *cmd) || raise("#{cmd} failed!") end end diff --git a/lib/tasks/spinach.rake b/lib/tasks/spinach.rake index 8dbfa7751dc..19ff13f06c0 100644 --- a/lib/tasks/spinach.rake +++ b/lib/tasks/spinach.rake @@ -35,7 +35,7 @@ task :spinach do end def run_system_command(cmd) - system({'RAILS_ENV' => 'test', 'force' => 'yes'}, *cmd) + system({ 'RAILS_ENV' => 'test', 'force' => 'yes' }, *cmd) end def run_spinach_command(args) diff --git a/lib/tasks/teaspoon.rake b/lib/tasks/teaspoon.rake deleted file mode 100644 index 08caedd7ff3..00000000000 --- a/lib/tasks/teaspoon.rake +++ /dev/null @@ -1,25 +0,0 @@ -unless Rails.env.production? - Rake::Task['teaspoon'].clear if Rake::Task.task_defined?('teaspoon') - - namespace :teaspoon do - desc 'GitLab | Teaspoon | Generate fixtures for JavaScript tests' - RSpec::Core::RakeTask.new(:fixtures) do |t| - ENV['NO_KNAPSACK'] = 'true' - t.pattern = 'spec/javascripts/fixtures/*.rb' - t.rspec_opts = '--format documentation' - end - - desc 'GitLab | Teaspoon | Run JavaScript tests' - task :tests do - require "teaspoon/console" - options = {} - abort('rake teaspoon:tests failed') if Teaspoon::Console.new(options).failures? - end - end - - desc 'GitLab | Teaspoon | Shortcut for teaspoon:fixtures and teaspoon:tests' - task :teaspoon do - Rake::Task['teaspoon:fixtures'].invoke - Rake::Task['teaspoon:tests'].invoke - end -end diff --git a/lib/tasks/test.rake b/lib/tasks/test.rake index d3dcbd2c29b..3e01f91d32c 100644 --- a/lib/tasks/test.rake +++ b/lib/tasks/test.rake @@ -7,5 +7,5 @@ end unless Rails.env.production? desc "GitLab | Run all tests on CI with simplecov" - task test_ci: [:rubocop, :brakeman, :teaspoon, :spinach, :spec] + task test_ci: [:rubocop, :brakeman, :karma, :spinach, :spec] end diff --git a/lib/tasks/tokens.rake b/lib/tasks/tokens.rake new file mode 100644 index 00000000000..95735f43802 --- /dev/null +++ b/lib/tasks/tokens.rake @@ -0,0 +1,38 @@ +require_relative '../../app/models/concerns/token_authenticatable.rb' + +namespace :tokens do + desc "Reset all GitLab user auth tokens" + task reset_all_auth: :environment do + reset_all_users_token(:reset_authentication_token!) + end + + desc "Reset all GitLab email tokens" + task reset_all_email: :environment do + reset_all_users_token(:reset_incoming_email_token!) + end + + def reset_all_users_token(reset_token_method) + TmpUser.find_in_batches do |batch| + puts "Processing batch starting with user ID: #{batch.first.id}" + STDOUT.flush + + batch.each(&reset_token_method) + end + end +end + +class TmpUser < ActiveRecord::Base + include TokenAuthenticatable + + self.table_name = 'users' + + def reset_authentication_token! + write_new_token(:authentication_token) + save!(validate: false) + end + + def reset_incoming_email_token! + write_new_token(:incoming_email_token) + save!(validate: false) + end +end diff --git a/lib/tasks/yarn.rake b/lib/tasks/yarn.rake new file mode 100644 index 00000000000..2ac88a039e7 --- /dev/null +++ b/lib/tasks/yarn.rake @@ -0,0 +1,40 @@ + +namespace :yarn do + desc 'Ensure Yarn is installed' + task :available do + unless system('yarn --version', out: File::NULL) + warn( + 'Error: Yarn executable was not detected in the system.'.color(:red), + 'Download Yarn at https://yarnpkg.com/en/docs/install'.color(:green) + ) + abort + end + end + + desc 'Ensure Node dependencies are installed' + task check: ['yarn:available'] do + unless system('yarn check --ignore-engines', out: File::NULL) + warn( + 'Error: You have unmet dependencies. (`yarn check` command failed)'.color(:red), + 'Run `yarn install` to install missing modules.'.color(:green) + ) + abort + end + end + + desc 'Install Node dependencies with Yarn' + task install: ['yarn:available'] do + unless system('yarn install --pure-lockfile --ignore-engines') + abort 'Error: Unable to install node modules.'.color(:red) + end + end + + desc 'Remove Node dependencies' + task :clobber do + warn 'Purging ./node_modules directory'.color(:red) + FileUtils.rm_rf 'node_modules' + end +end + +desc 'Install Node dependencies with Yarn' +task yarn: ['yarn:install'] |