diff options
Diffstat (limited to 'lib')
135 files changed, 3794 insertions, 1493 deletions
diff --git a/lib/api/access_requests.rb b/lib/api/access_requests.rb index ed723b94cfd..789f45489eb 100644 --- a/lib/api/access_requests.rb +++ b/lib/api/access_requests.rb @@ -1,5 +1,7 @@ module API class AccessRequests < Grape::API + include PaginationParams + before { authenticate! } helpers ::API::Helpers::MembersHelpers @@ -13,6 +15,9 @@ module API detail 'This feature was introduced in GitLab 8.11.' success Entities::AccessRequester end + params do + use :pagination + end get ":id/access_requests" do source = find_source(source_type, params[:id]) diff --git a/lib/api/api.rb b/lib/api/api.rb index 67109ceeef9..9d5adffd8f4 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -3,6 +3,8 @@ module API include APIGuard version 'v3', using: :path + before { allow_access_with_scope :api } + rescue_from Gitlab::Access::AccessDeniedError do rack_response({ 'message' => '403 Forbidden' }.to_json, 403) end @@ -64,6 +66,7 @@ module API mount ::API::Session mount ::API::Settings mount ::API::SidekiqMetrics + mount ::API::Snippets mount ::API::Subscriptions mount ::API::SystemHooks mount ::API::Tags diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index 8cc7a26f1fa..df6db140d0e 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -6,6 +6,9 @@ module API module APIGuard extend ActiveSupport::Concern + PRIVATE_TOKEN_HEADER = "HTTP_PRIVATE_TOKEN" + PRIVATE_TOKEN_PARAM = :private_token + included do |base| # OAuth2 Resource Server Authentication use Rack::OAuth2::Server::Resource::Bearer, 'The API' do |request| @@ -44,27 +47,60 @@ module API access_token = find_access_token return nil unless access_token - case validate_access_token(access_token, scopes) - when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE + case AccessTokenValidationService.new(access_token).validate(scopes: scopes) + when AccessTokenValidationService::INSUFFICIENT_SCOPE raise InsufficientScopeError.new(scopes) - when Oauth2::AccessTokenValidationService::EXPIRED + when AccessTokenValidationService::EXPIRED raise ExpiredError - when Oauth2::AccessTokenValidationService::REVOKED + when AccessTokenValidationService::REVOKED raise RevokedError - when Oauth2::AccessTokenValidationService::VALID + when AccessTokenValidationService::VALID @current_user = User.find(access_token.resource_owner_id) end end + def find_user_by_private_token(scopes: []) + token_string = (params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]).to_s + + return nil unless token_string.present? + + find_user_by_authentication_token(token_string) || find_user_by_personal_access_token(token_string, scopes) + end + def current_user @current_user end + # Set the authorization scope(s) allowed for the current request. + # + # Note: A call to this method adds to any previous scopes in place. This is done because + # `Grape` callbacks run from the outside-in: the top-level callback (API::API) runs first, then + # the next-level callback (API::API::Users, for example) runs. All these scopes are valid for the + # given endpoint (GET `/api/users` is accessible by the `api` and `read_user` scopes), and so they + # need to be stored. + def allow_access_with_scope(*scopes) + @scopes ||= [] + @scopes.concat(scopes.map(&:to_s)) + end + private + def find_user_by_authentication_token(token_string) + User.find_by_authentication_token(token_string) + end + + def find_user_by_personal_access_token(token_string, scopes) + access_token = PersonalAccessToken.active.find_by_token(token_string) + return unless access_token + + if AccessTokenValidationService.new(access_token).include_any_scope?(scopes) + User.find(access_token.user_id) + end + end + def find_access_token @access_token ||= Doorkeeper.authenticate(doorkeeper_request, Doorkeeper.configuration.access_token_methods) end @@ -72,10 +108,6 @@ module API def doorkeeper_request @doorkeeper_request ||= ActionDispatch::Request.new(env) end - - def validate_access_token(access_token, scopes) - Oauth2::AccessTokenValidationService.validate(access_token, scopes: scopes) - end end module ClassMethods diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb index e9ccba3b465..58a4df54bea 100644 --- a/lib/api/award_emoji.rb +++ b/lib/api/award_emoji.rb @@ -1,5 +1,7 @@ module API class AwardEmoji < Grape::API + include PaginationParams + before { authenticate! } AWARDABLES = %w[issue merge_request snippet] @@ -21,6 +23,9 @@ module API detail 'This feature was introduced in 8.9' success Entities::AwardEmoji end + params do + use :pagination + end get endpoint do if can_read_awardable? awards = paginate(awardable.award_emoji) diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 73aed624ea7..0950c3d2e88 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -23,9 +23,9 @@ module API success Entities::RepoBranch end params do - requires :branch, type: String, regexp: /.+/, desc: 'The name of the branch' + requires :branch, type: String, desc: 'The name of the branch' end - get ':id/repository/branches/:branch' do + get ':id/repository/branches/:branch', requirements: { branch: /.+/ } do branch = user_project.repository.find_branch(params[:branch]) not_found!("Branch") unless branch @@ -39,11 +39,11 @@ module API success Entities::RepoBranch end params do - requires :branch, type: String, regexp: /.+/, desc: 'The name of the branch' + requires :branch, type: String, desc: 'The name of the branch' optional :developers_can_push, type: Boolean, desc: 'Flag if developers can push to that branch' optional :developers_can_merge, type: Boolean, desc: 'Flag if developers can merge to that branch' end - put ':id/repository/branches/:branch/protect' do + put ':id/repository/branches/:branch/protect', requirements: { branch: /.+/ } do authorize_admin_project branch = user_project.repository.find_branch(params[:branch]) @@ -76,9 +76,9 @@ module API success Entities::RepoBranch end params do - requires :branch, type: String, regexp: /.+/, desc: 'The name of the branch' + requires :branch, type: String, desc: 'The name of the branch' end - put ':id/repository/branches/:branch/unprotect' do + put ':id/repository/branches/:branch/unprotect', requirements: { branch: /.+/ } do authorize_admin_project branch = user_project.repository.find_branch(params[:branch]) @@ -112,9 +112,9 @@ module API desc 'Delete a branch' params do - requires :branch, type: String, regexp: /.+/, desc: 'The name of the branch' + requires :branch, type: String, desc: 'The name of the branch' end - delete ":id/repository/branches/:branch" do + delete ":id/repository/branches/:branch", requirements: { branch: /.+/ } do authorize_push_project result = DeleteBranchService.new(user_project, current_user). diff --git a/lib/api/broadcast_messages.rb b/lib/api/broadcast_messages.rb index b6281a7f0ac..1217002bf8e 100644 --- a/lib/api/broadcast_messages.rb +++ b/lib/api/broadcast_messages.rb @@ -1,5 +1,7 @@ module API class BroadcastMessages < Grape::API + include PaginationParams + before { authenticate! } before { authenticated_as_admin! } @@ -15,8 +17,7 @@ module API success Entities::BroadcastMessage end params do - optional :page, type: Integer, desc: 'Current page number' - optional :per_page, type: Integer, desc: 'Number of messages per page' + use :pagination end get do messages = BroadcastMessage.all diff --git a/lib/api/builds.rb b/lib/api/builds.rb index 67adca6605f..af61be343be 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -1,6 +1,7 @@ module API - # Projects builds API class Builds < Grape::API + include PaginationParams + before { authenticate! } params do @@ -28,6 +29,7 @@ module API end params do use :optional_scope + use :pagination end get ':id/builds' do builds = user_project.builds.order('id DESC') @@ -41,8 +43,9 @@ module API success Entities::Build end params do - requires :sha, type: String, desc: 'The SHA id of a commit' + requires :sha, type: String, desc: 'The SHA id of a commit' use :optional_scope + use :pagination end get ':id/repository/commits/:sha/builds' do authorize_read_builds! diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index f54d4f06627..4bbdf06a49c 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -1,9 +1,10 @@ require 'mime/types' module API - # Project commit statuses API class CommitStatuses < Grape::API resource :projects do + include PaginationParams + before { authenticate! } desc "Get a commit's statuses" do @@ -16,6 +17,7 @@ module API optional :stage, type: String, desc: 'The stage' optional :name, type: String, desc: 'The name' optional :all, type: String, desc: 'Show all statuses, default: false' + use :pagination end get ':id/repository/commits/:sha/statuses' do authorize!(:read_commit_status, user_project) @@ -77,7 +79,7 @@ module API ) begin - case params[:state].to_s + case params[:state] when 'pending' status.enqueue! when 'running' diff --git a/lib/api/commits.rb b/lib/api/commits.rb index f412e1da1bf..cf2489dbb67 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -1,8 +1,9 @@ require 'mime/types' module API - # Projects commits API class Commits < Grape::API + include PaginationParams + before { authenticate! } before { authorize! :download_code, user_project } @@ -46,7 +47,7 @@ module API requires :id, type: Integer, desc: 'The project ID' requires :branch_name, type: String, desc: 'The name of branch' requires :commit_message, type: String, desc: 'Commit message' - requires :actions, type: Array, desc: 'Actions to perform in commit' + requires :actions, type: Array[Hash], desc: 'Actions to perform in commit' optional :author_email, type: String, desc: 'Author email for commit' optional :author_name, type: String, desc: 'Author name for commit' end @@ -107,9 +108,8 @@ module API failure [[404, 'Not Found']] end params do + use :pagination requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' - optional :per_page, type: Integer, desc: 'The amount of items per page for paginaion' - optional :page, type: Integer, desc: 'The page number for pagination' end get ':id/repository/commits/:sha/comments' do commit = user_project.commit(params[:sha]) @@ -120,6 +120,41 @@ module API present paginate(notes), with: Entities::CommitNote end + desc 'Cherry pick commit into a branch' do + detail 'This feature was introduced in GitLab 8.15' + success Entities::RepoCommit + end + params do + requires :sha, type: String, desc: 'A commit sha to be cherry picked' + requires :branch, type: String, desc: 'The name of the branch' + end + post ':id/repository/commits/:sha/cherry_pick' do + authorize! :push_code, user_project + + commit = user_project.commit(params[:sha]) + not_found!('Commit') unless commit + + branch = user_project.repository.find_branch(params[:branch]) + not_found!('Branch') unless branch + + commit_params = { + commit: commit, + create_merge_request: false, + source_project: user_project, + source_branch: commit.cherry_pick_branch_name, + target_branch: params[:branch] + } + + result = ::Commits::CherryPickService.new(user_project, current_user, commit_params).execute + + if result[:status] == :success + branch = user_project.repository.find_branch(params[:branch]) + present user_project.repository.commit(branch.dereferenced_target), with: Entities::RepoCommit + else + render_api_error!(result[:message], 400) + end + end + desc 'Post comment to commit' do success Entities::CommitNote end diff --git a/lib/api/deployments.rb b/lib/api/deployments.rb index f782bcaf7e9..c5feb49b22f 100644 --- a/lib/api/deployments.rb +++ b/lib/api/deployments.rb @@ -1,6 +1,8 @@ module API # Deployments RESTfull API endpoints class Deployments < Grape::API + include PaginationParams + before { authenticate! } params do @@ -12,8 +14,7 @@ module API success Entities::Deployment end params do - optional :page, type: Integer, desc: 'Page number of the current request' - optional :per_page, type: Integer, desc: 'Number of items per page' + use :pagination end get ':id/deployments' do authorize! :read_deployment, user_project diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 33cb6fd3704..dfbb3ab86dd 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -22,7 +22,7 @@ module API expose :provider, :extern_uid end - class UserFull < User + class UserPublic < User expose :last_sign_in_at expose :confirmed_at expose :email @@ -34,7 +34,7 @@ module API expose :external end - class UserLogin < UserFull + class UserWithPrivateToken < UserPublic expose :private_token end @@ -141,8 +141,12 @@ module API options[:project].repository.commit(repo_branch.dereferenced_target) end + expose :merged do |repo_branch, options| + options[:project].repository.merged_to_root_ref?(repo_branch.name) + end + expose :protected do |repo_branch, options| - options[:project].protected_branch? repo_branch.name + options[:project].protected_branch?(repo_branch.name) end expose :developers_can_push do |repo_branch, options| @@ -170,6 +174,7 @@ module API class RepoCommit < Grape::Entity expose :id, :short_id, :title, :author_name, :author_email, :created_at + expose :committer_name, :committer_email expose :safe_message, as: :message end @@ -196,6 +201,19 @@ module API end end + class PersonalSnippet < Grape::Entity + expose :id, :title, :file_name + expose :author, using: Entities::UserBasic + expose :updated_at, :created_at + + expose :web_url do |snippet| + Gitlab::UrlBuilder.build(snippet) + end + expose :raw_url do |snippet| + Gitlab::UrlBuilder.build(snippet) + "/raw" + end + end + class ProjectEntity < Grape::Entity expose :id, :iid expose(:project_id) { |entity| entity.project.id } @@ -210,6 +228,7 @@ module API class Milestone < ProjectEntity expose :due_date + expose :start_date end class Issue < ProjectEntity @@ -283,7 +302,7 @@ module API end class SSHKeyWithUser < SSHKey - expose :user, using: Entities::UserFull + expose :user, using: Entities::UserPublic end class Note < Grape::Entity @@ -606,10 +625,11 @@ module API expose :user, with: Entities::UserBasic expose :created_at, :updated_at, :started_at, :finished_at, :committed_at expose :duration + expose :coverage end class EnvironmentBasic < Grape::Entity - expose :id, :name, :external_url + expose :id, :name, :slug, :external_url end class Environment < EnvironmentBasic diff --git a/lib/api/environments.rb b/lib/api/environments.rb index 00c901937b1..1a7e68f0528 100644 --- a/lib/api/environments.rb +++ b/lib/api/environments.rb @@ -1,6 +1,9 @@ module API # Environments RESTfull API endpoints class Environments < Grape::API + include ::API::Helpers::CustomValidators + include PaginationParams + before { authenticate! } params do @@ -12,8 +15,7 @@ module API success Entities::Environment end params do - optional :page, type: Integer, desc: 'Page number of the current request' - optional :per_page, type: Integer, desc: 'Number of items per page' + use :pagination end get ':id/environments' do authorize! :read_environment, user_project @@ -28,6 +30,7 @@ module API params do requires :name, type: String, desc: 'The name of the environment to be created' optional :external_url, type: String, desc: 'URL on which this deployment is viewable' + optional :slug, absence: { message: "is automatically generated and cannot be changed" } end post ':id/environments' do authorize! :create_environment, user_project @@ -49,6 +52,7 @@ module API requires :environment_id, type: Integer, desc: 'The environment ID' optional :name, type: String, desc: 'The new environment name' optional :external_url, type: String, desc: 'The new URL on which this deployment is viewable' + optional :slug, absence: { message: "is automatically generated and cannot be changed" } end put ':id/environments/:environment_id' do authorize! :update_environment, user_project diff --git a/lib/api/files.rb b/lib/api/files.rb index 96510e651a3..28f306e45f3 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -23,140 +23,107 @@ module API branch_name: attrs[:branch_name] } end + + params :simple_file_params do + requires :file_path, type: String, desc: 'The path to new file. Ex. lib/class.rb' + requires :branch_name, type: String, desc: 'The name of branch' + requires :commit_message, type: String, desc: 'Commit Message' + optional :author_email, type: String, desc: 'The email of the author' + optional :author_name, type: String, desc: 'The name of the author' + end + + params :extended_file_params do + use :simple_file_params + requires :content, type: String, desc: 'File content' + optional :encoding, type: String, values: %w[base64], desc: 'File encoding' + end end + params do + requires :id, type: String, desc: 'The project ID' + end resource :projects do - # Get file from repository - # File content is Base64 encoded - # - # Parameters: - # file_path (required) - The path to the file. Ex. lib/class.rb - # ref (required) - The name of branch, tag or commit - # - # Example Request: - # GET /projects/:id/repository/files - # - # Example response: - # { - # "file_name": "key.rb", - # "file_path": "app/models/key.rb", - # "size": 1476, - # "encoding": "base64", - # "content": "IyA9PSBTY2hlbWEgSW5mb3...", - # "ref": "master", - # "blob_id": "79f7bbd25901e8334750839545a9bd021f0e4c83", - # "commit_id": "d5a3ff139356ce33e37e73add446f16869741b50", - # "last_commit_id": "570e7b2abdd848b95f2f578043fc23bd6f6fd24d", - # } - # + desc 'Get a file from repository' + params do + requires :file_path, type: String, desc: 'The path to the file. Ex. lib/class.rb' + requires :ref, type: String, desc: 'The name of branch, tag, or commit' + end get ":id/repository/files" do authorize! :download_code, user_project - required_attributes! [:file_path, :ref] - attrs = attributes_for_keys [:file_path, :ref] - ref = attrs.delete(:ref) - file_path = attrs.delete(:file_path) - - commit = user_project.commit(ref) - not_found! 'Commit' unless commit + commit = user_project.commit(params[:ref]) + not_found!('Commit') unless commit repo = user_project.repository - blob = repo.blob_at(commit.sha, file_path) + blob = repo.blob_at(commit.sha, params[:file_path]) + not_found!('File') unless blob - if blob - blob.load_all_data!(repo) - status(200) + blob.load_all_data!(repo) + status(200) - { - file_name: blob.name, - file_path: blob.path, - size: blob.size, - encoding: "base64", - content: Base64.strict_encode64(blob.data), - ref: ref, - blob_id: blob.id, - commit_id: commit.id, - last_commit_id: repo.last_commit_for_path(commit.sha, file_path).id - } - else - not_found! 'File' - end + { + file_name: blob.name, + file_path: blob.path, + size: blob.size, + encoding: "base64", + content: Base64.strict_encode64(blob.data), + ref: params[:ref], + blob_id: blob.id, + commit_id: commit.id, + last_commit_id: repo.last_commit_for_path(commit.sha, params[:file_path]).id + } end - # Create new file in repository - # - # Parameters: - # file_path (required) - The path to new file. Ex. lib/class.rb - # branch_name (required) - The name of branch - # content (required) - File content - # commit_message (required) - Commit message - # - # Example Request: - # POST /projects/:id/repository/files - # + desc 'Create new file in repository' + params do + use :extended_file_params + end post ":id/repository/files" do authorize! :push_code, user_project - required_attributes! [:file_path, :branch_name, :content, :commit_message] - attrs = attributes_for_keys [:file_path, :branch_name, :content, :commit_message, :encoding, :author_email, :author_name] - result = ::Files::CreateService.new(user_project, current_user, commit_params(attrs)).execute + file_params = declared_params(include_missing: false) + result = ::Files::CreateService.new(user_project, current_user, commit_params(file_params)).execute if result[:status] == :success status(201) - commit_response(attrs) + commit_response(file_params) else render_api_error!(result[:message], 400) end end - # Update existing file in repository - # - # Parameters: - # file_path (optional) - The path to file. Ex. lib/class.rb - # branch_name (required) - The name of branch - # content (required) - File content - # commit_message (required) - Commit message - # - # Example Request: - # PUT /projects/:id/repository/files - # + desc 'Update existing file in repository' + params do + use :extended_file_params + end put ":id/repository/files" do authorize! :push_code, user_project - required_attributes! [:file_path, :branch_name, :content, :commit_message] - attrs = attributes_for_keys [:file_path, :branch_name, :content, :commit_message, :encoding, :author_email, :author_name] - result = ::Files::UpdateService.new(user_project, current_user, commit_params(attrs)).execute + file_params = declared_params(include_missing: false) + result = ::Files::UpdateService.new(user_project, current_user, commit_params(file_params)).execute if result[:status] == :success status(200) - commit_response(attrs) + commit_response(file_params) else http_status = result[:http_status] || 400 render_api_error!(result[:message], http_status) end end - # Delete existing file in repository - # - # Parameters: - # file_path (optional) - The path to file. Ex. lib/class.rb - # branch_name (required) - The name of branch - # content (required) - File content - # commit_message (required) - Commit message - # - # Example Request: - # DELETE /projects/:id/repository/files - # + desc 'Delete an existing file in repository' + params do + use :simple_file_params + end delete ":id/repository/files" do authorize! :push_code, user_project - required_attributes! [:file_path, :branch_name, :commit_message] - attrs = attributes_for_keys [:file_path, :branch_name, :commit_message, :author_email, :author_name] - result = ::Files::DeleteService.new(user_project, current_user, commit_params(attrs)).execute + file_params = declared_params(include_missing: false) + result = ::Files::DeleteService.new(user_project, current_user, commit_params(file_params)).execute if result[:status] == :success status(200) - commit_response(attrs) + commit_response(file_params) else render_api_error!(result[:message], 400) end diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 48ad3b80ae0..9b9d3df7435 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -1,5 +1,7 @@ module API class Groups < Grape::API + include PaginationParams + before { authenticate! } helpers do @@ -21,6 +23,7 @@ module API optional :search, type: String, desc: 'Search for a specific group' optional :order_by, type: String, values: %w[name path], default: 'name', desc: 'Order by name or path' optional :sort, type: String, values: %w[asc desc], default: 'asc', desc: 'Sort by asc (ascending) or desc (descending)' + use :pagination end get do groups = if current_user.admin @@ -33,7 +36,7 @@ module API groups = groups.search(params[:search]) if params[:search].present? groups = groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present? - groups = groups.reorder(params[:order_by] => params[:sort].to_sym) + groups = groups.reorder(params[:order_by] => params[:sort]) present paginate(groups), with: Entities::Group end @@ -41,6 +44,9 @@ module API desc 'Get list of owned groups for authenticated user' do success Entities::Group end + params do + use :pagination + end get '/owned' do groups = current_user.owned_groups present paginate(groups), with: Entities::Group, user: current_user @@ -82,7 +88,7 @@ module API :lfs_enabled, :request_access_enabled end put ':id' do - group = find_group(params[:id]) + group = find_group!(params[:id]) authorize! :admin_group, group if ::Groups::UpdateService.new(group, current_user, declared_params(include_missing: false)).execute @@ -96,13 +102,13 @@ module API success Entities::GroupDetail end get ":id" do - group = find_group(params[:id]) + group = find_group!(params[:id]) present group, with: Entities::GroupDetail end desc 'Remove a group.' delete ":id" do - group = find_group(params[:id]) + group = find_group!(params[:id]) authorize! :admin_group, group DestroyGroupService.new(group, current_user).execute end @@ -110,11 +116,25 @@ module API desc 'Get a list of projects in this group.' do success Entities::Project end + params do + optional :archived, type: Boolean, default: false, desc: 'Limit by archived status' + optional :visibility, type: String, values: %w[public internal private], + desc: 'Limit by visibility' + optional :search, type: String, desc: 'Return list of authorized projects matching the search criteria' + optional :order_by, type: String, values: %w[id name path created_at updated_at last_activity_at], + default: 'created_at', desc: 'Return projects ordered by field' + optional :sort, type: String, values: %w[asc desc], default: 'desc', + desc: 'Return projects sorted in ascending and descending order' + optional :simple, type: Boolean, default: false, + desc: 'Return only the ID, URL, name, and path of each project' + use :pagination + end get ":id/projects" do - group = find_group(params[:id]) + group = find_group!(params[:id]) projects = GroupProjectsFinder.new(group).execute(current_user) - projects = paginate projects - present projects, with: Entities::Project, user: current_user + projects = filter_projects(projects) + entity = params[:simple] ? Entities::BasicProjectDetails : Entities::Project + present paginate(projects), with: entity, user: current_user end desc 'Transfer a project to the group namespace. Available only for admin.' do diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 2c593dbb4ea..4be659fc20b 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -2,81 +2,54 @@ module API module Helpers include Gitlab::Utils - PRIVATE_TOKEN_HEADER = "HTTP_PRIVATE_TOKEN" - PRIVATE_TOKEN_PARAM = :private_token SUDO_HEADER = "HTTP_SUDO" SUDO_PARAM = :sudo - def private_token - params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER] - end - - def warden - env['warden'] - end - - # Check the Rails session for valid authentication details - # - # Until CSRF protection is added to the API, disallow this method for - # state-changing endpoints - def find_user_from_warden - warden.try(:authenticate) if %w[GET HEAD].include?(env['REQUEST_METHOD']) - end - def declared_params(options = {}) options = { include_parent_namespaces: false }.merge(options) declared(params, options).to_h.symbolize_keys end - def find_user_by_private_token - token = private_token - return nil unless token.present? - - User.find_by_authentication_token(token) || User.find_by_personal_access_token(token) - end - def current_user - @current_user ||= find_user_by_private_token - @current_user ||= doorkeeper_guard - @current_user ||= find_user_from_warden - - unless @current_user && Gitlab::UserAccess.new(@current_user).allowed? - return nil - end + return @current_user if defined?(@current_user) - identifier = sudo_identifier() + @current_user = initial_current_user - # If the sudo is the current user do nothing - if identifier && !(@current_user.id == identifier || @current_user.username == identifier) - forbidden!('Must be admin to use sudo') unless @current_user.is_admin? - @current_user = User.by_username_or_id(identifier) - not_found!("No user id or username for: #{identifier}") if @current_user.nil? - end + sudo! @current_user end - def sudo_identifier - identifier ||= params[SUDO_PARAM] || env[SUDO_HEADER] - - # Regex for integers - if !!(identifier =~ /\A[0-9]+\z/) - identifier.to_i - else - identifier - end + def sudo? + initial_current_user != current_user end def user_project - @project ||= find_project(params[:id]) + @project ||= find_project!(params[:id]) end def available_labels @available_labels ||= LabelsFinder.new(current_user, project_id: user_project.id).execute end + def find_user(id) + if id =~ /^\d+$/ + User.find_by(id: id) + else + User.find_by(username: id) + end + end + def find_project(id) - project = Project.find_with_namespace(id) || Project.find_by(id: id) + if id =~ /^\d+$/ + Project.find_by(id: id) + else + Project.find_with_namespace(id) + end + end + + def find_project!(id) + project = find_project(id) if can?(current_user, :read_project, project) project @@ -85,19 +58,16 @@ module API end end - def project_service(project = user_project) - @project_service ||= project.find_or_initialize_service(params[:service_slug].underscore) - @project_service || not_found!("Service") - end - - def service_attributes - @service_attributes ||= project_service.fields.inject([]) do |arr, hash| - arr << hash[:name].to_sym + def find_group(id) + if id =~ /^\d+$/ + Group.find_by(id: id) + else + Group.find_by_full_path(id) end end - def find_group(id) - group = Group.find_by(path: id) || Group.find_by(id: id) + def find_group!(id) + group = find_group(id) if can?(current_user, :read_group, group) group @@ -112,9 +82,7 @@ module API end def find_project_issue(id) - issue = user_project.issues.find(id) - not_found! unless can?(current_user, :read_issue, issue) - issue + IssuesFinder.new(current_user, project_id: user_project.id).find(id) end def paginate(relation) @@ -127,6 +95,10 @@ module API unauthorized! unless current_user end + def authenticate_non_get! + authenticate! unless %w[GET HEAD].include?(route.route_method) + end + def authenticate_by_gitlab_shell_token! input = params['secret_token'].try(:chomp) unless Devise.secure_compare(secret_token, input) @@ -135,6 +107,7 @@ module API end def authenticated_as_admin! + authenticate! forbidden! unless current_user.is_admin? end @@ -182,20 +155,6 @@ module API ActionController::Parameters.new(attrs).permit! end - # Helper method for validating all labels against its names - def validate_label_params(params) - errors = {} - - params[:labels].to_s.split(',').each do |label_name| - label = available_labels.find_or_initialize_by(title: label_name.strip) - next if label.valid? - - errors[label.title] = label.errors - end - - errors - end - # Checks the occurrences of datetime attributes, each attribute if present in the params hash must be in ISO 8601 # format (YYYY-MM-DDTHH:MM:SSZ) or a Bad Request error is invoked. # @@ -212,22 +171,6 @@ module API end end - def issuable_order_by - if params["order_by"] == 'updated_at' - 'updated_at' - else - 'created_at' - end - end - - def issuable_sort - if params["sort"] == 'asc' - :asc - else - :desc - end - end - def filter_by_iid(items, iid) items.where(iid: iid) end @@ -308,11 +251,6 @@ module API # Projects helpers def filter_projects(projects) - # If the archived parameter is passed, limit results accordingly - if params[:archived].present? - projects = projects.where(archived: to_boolean(params[:archived])) - end - if params[:search].present? projects = projects.search(params[:search]) end @@ -321,25 +259,8 @@ module API projects = projects.search_by_visibility(params[:visibility]) end - projects.reorder(project_order_by => project_sort) - end - - def project_order_by - order_fields = %w(id name path created_at updated_at last_activity_at) - - if order_fields.include?(params['order_by']) - params['order_by'] - else - 'created_at' - end - end - - def project_sort - if params["sort"] == 'asc' - :asc - else - :desc - end + projects = projects.where(archived: params[:archived]) + projects.reorder(params[:order_by] => params[:sort]) end # file helpers @@ -384,6 +305,62 @@ module API private + def private_token + params[APIGuard::PRIVATE_TOKEN_PARAM] || env[APIGuard::PRIVATE_TOKEN_HEADER] + end + + def warden + env['warden'] + end + + # Check the Rails session for valid authentication details + # + # Until CSRF protection is added to the API, disallow this method for + # state-changing endpoints + def find_user_from_warden + warden.try(:authenticate) if %w[GET HEAD].include?(env['REQUEST_METHOD']) + end + + def initial_current_user + return @initial_current_user if defined?(@initial_current_user) + + @initial_current_user ||= find_user_by_private_token(scopes: @scopes) + @initial_current_user ||= doorkeeper_guard(scopes: @scopes) + @initial_current_user ||= find_user_from_warden + + unless @initial_current_user && Gitlab::UserAccess.new(@initial_current_user).allowed? + @initial_current_user = nil + end + + @initial_current_user + end + + def sudo! + return unless sudo_identifier + return unless initial_current_user + + unless initial_current_user.is_admin? + forbidden!('Must be admin to use sudo') + end + + # Only private tokens should be used for the SUDO feature + unless private_token == initial_current_user.private_token + forbidden!('Private token must be specified in order to use sudo') + end + + sudoed_user = find_user(sudo_identifier) + + if sudoed_user + @current_user = sudoed_user + else + not_found!("No user id or username for: #{sudo_identifier}") + end + end + + def sudo_identifier + @sudo_identifier ||= params[SUDO_PARAM] || env[SUDO_HEADER] + end + def add_pagination_headers(paginated_data) header 'X-Total', paginated_data.total_count.to_s header 'X-Total-Pages', paginated_data.total_pages.to_s diff --git a/lib/api/helpers/custom_validators.rb b/lib/api/helpers/custom_validators.rb new file mode 100644 index 00000000000..0a8f3073a50 --- /dev/null +++ b/lib/api/helpers/custom_validators.rb @@ -0,0 +1,14 @@ +module API + module Helpers + module CustomValidators + class Absence < Grape::Validations::Base + def validate_param!(attr_name, params) + return if params.respond_to?(:key?) && !params.key?(attr_name) + raise Grape::Exceptions::Validation, params: [@scope.full_name(attr_name)], message: message(:absence) + end + end + end + end +end + +Grape::Validations.register_validator(:absence, ::API::Helpers::CustomValidators::Absence) diff --git a/lib/api/helpers/members_helpers.rb b/lib/api/helpers/members_helpers.rb index 90114f6f667..d9cae1501f8 100644 --- a/lib/api/helpers/members_helpers.rb +++ b/lib/api/helpers/members_helpers.rb @@ -2,7 +2,7 @@ module API module Helpers module MembersHelpers def find_source(source_type, id) - public_send("find_#{source_type}", id) + public_send("find_#{source_type}!", id) end def authorize_admin_source!(source_type, source) diff --git a/lib/api/issues.rb b/lib/api/issues.rb index eea5b91d4f9..c9124649cbb 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -1,6 +1,7 @@ module API - # Issues API class Issues < Grape::API + include PaginationParams + before { authenticate! } helpers do @@ -19,98 +20,89 @@ module API def filter_issues_milestone(issues, milestone) issues.includes(:milestone).where('milestones.title' => milestone) end + + params :issues_params do + optional :labels, type: String, desc: 'Comma-separated list of label names' + optional :order_by, type: String, values: %w[created_at updated_at], default: 'created_at', + desc: 'Return issues ordered by `created_at` or `updated_at` fields.' + optional :sort, type: String, values: %w[asc desc], default: 'desc', + desc: 'Return issues sorted in `asc` or `desc` order.' + use :pagination + end + + params :issue_params do + optional :description, type: String, desc: 'The description of an issue' + optional :assignee_id, type: Integer, desc: 'The ID of a user to assign issue' + optional :milestone_id, type: Integer, desc: 'The ID of a milestone to assign issue' + optional :labels, type: String, desc: 'Comma-separated list of label names' + optional :due_date, type: String, desc: 'Date time string in the format YEAR-MONTH-DAY' + optional :confidential, type: Boolean, desc: 'Boolean parameter if the issue should be confidential' + optional :state_event, type: String, values: %w[open close], + desc: 'State of the issue' + end end resource :issues do - # Get currently authenticated user's issues - # - # Parameters: - # state (optional) - Return "opened" or "closed" issues - # labels (optional) - Comma-separated list of label names - # order_by (optional) - Return requests ordered by `created_at` or `updated_at` fields. Default is `created_at` - # sort (optional) - Return requests sorted in `asc` or `desc` order. Default is `desc` - # - # Example Requests: - # GET /issues - # GET /issues?state=opened - # GET /issues?state=closed - # GET /issues?labels=foo - # GET /issues?labels=foo,bar - # GET /issues?labels=foo,bar&state=opened + desc "Get currently authenticated user's issues" do + success Entities::Issue + end + params do + optional :state, type: String, values: %w[opened closed all], default: 'all', + desc: 'Return opened, closed, or all issues' + use :issues_params + end get do issues = current_user.issues.inc_notes_with_associations - issues = filter_issues_state(issues, params[:state]) unless params[:state].nil? + issues = filter_issues_state(issues, params[:state]) issues = filter_issues_labels(issues, params[:labels]) unless params[:labels].nil? - issues = issues.reorder(issuable_order_by => issuable_sort) + issues = issues.reorder(params[:order_by] => params[:sort]) present paginate(issues), with: Entities::Issue, current_user: current_user end end + params do + requires :id, type: String, desc: 'The ID of a group' + end resource :groups do - # Get a list of group issues - # - # Parameters: - # id (required) - The ID of a group - # state (optional) - Return "opened" or "closed" issues - # labels (optional) - Comma-separated list of label names - # milestone (optional) - Milestone title - # order_by (optional) - Return requests ordered by `created_at` or `updated_at` fields. Default is `created_at` - # sort (optional) - Return requests sorted in `asc` or `desc` order. Default is `desc` - # - # Example Requests: - # GET /groups/:id/issues - # GET /groups/:id/issues?state=opened - # GET /groups/:id/issues?state=closed - # GET /groups/:id/issues?labels=foo - # GET /groups/:id/issues?labels=foo,bar - # GET /groups/:id/issues?labels=foo,bar&state=opened - # GET /groups/:id/issues?milestone=1.0.0 - # GET /groups/:id/issues?milestone=1.0.0&state=closed + desc 'Get a list of group issues' do + success Entities::Issue + end + params do + optional :state, type: String, values: %w[opened closed all], default: 'opened', + desc: 'Return opened, closed, or all issues' + use :issues_params + end get ":id/issues" do - group = find_group(params[:id]) + group = find_group!(params.delete(:id)) - params[:state] ||= 'opened' params[:group_id] = group.id params[:milestone_title] = params.delete(:milestone) params[:label_name] = params.delete(:labels) - if params[:order_by] || params[:sort] - # The Sortable concern takes 'created_desc', not 'created_at_desc' (for example) - params[:sort] = "#{issuable_order_by.sub('_at', '')}_#{issuable_sort}" - end - issues = IssuesFinder.new(current_user, params).execute + issues = issues.reorder(params[:order_by] => params[:sort]) present paginate(issues), with: Entities::Issue, current_user: current_user end end + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do - # Get a list of project issues - # - # Parameters: - # id (required) - The ID of a project - # iid (optional) - Return the project issue having the given `iid` - # state (optional) - Return "opened" or "closed" issues - # labels (optional) - Comma-separated list of label names - # milestone (optional) - Milestone title - # order_by (optional) - Return requests ordered by `created_at` or `updated_at` fields. Default is `created_at` - # sort (optional) - Return requests sorted in `asc` or `desc` order. Default is `desc` - # - # Example Requests: - # GET /projects/:id/issues - # GET /projects/:id/issues?state=opened - # GET /projects/:id/issues?state=closed - # GET /projects/:id/issues?labels=foo - # GET /projects/:id/issues?labels=foo,bar - # GET /projects/:id/issues?labels=foo,bar&state=opened - # GET /projects/:id/issues?milestone=1.0.0 - # GET /projects/:id/issues?milestone=1.0.0&state=closed - # GET /issues?iid=42 + desc 'Get a list of project issues' do + success Entities::Issue + end + params do + optional :state, type: String, values: %w[opened closed all], default: 'all', + desc: 'Return opened, closed, or all issues' + optional :iid, type: Integer, desc: 'The IID of the issue' + use :issues_params + end get ":id/issues" do - issues = user_project.issues.inc_notes_with_associations.visible_to_user(current_user) - issues = filter_issues_state(issues, params[:state]) unless params[:state].nil? + issues = IssuesFinder.new(current_user, project_id: user_project.id).execute.inc_notes_with_associations + issues = filter_issues_state(issues, params[:state]) issues = filter_issues_labels(issues, params[:labels]) unless params[:labels].nil? issues = filter_by_iid(issues, params[:iid]) unless params[:iid].nil? @@ -118,57 +110,49 @@ module API issues = filter_issues_milestone(issues, params[:milestone]) end - issues = issues.reorder(issuable_order_by => issuable_sort) - + issues = issues.reorder(params[:order_by] => params[:sort]) present paginate(issues), with: Entities::Issue, current_user: current_user, project: user_project end - # Get a single project issue - # - # Parameters: - # id (required) - The ID of a project - # issue_id (required) - The ID of a project issue - # Example Request: - # GET /projects/:id/issues/:issue_id + desc 'Get a single project issue' do + success Entities::Issue + end + params do + requires :issue_id, type: Integer, desc: 'The ID of a project issue' + end get ":id/issues/:issue_id" do - @issue = find_project_issue(params[:issue_id]) - present @issue, with: Entities::Issue, current_user: current_user, project: user_project + issue = find_project_issue(params[:issue_id]) + present issue, with: Entities::Issue, current_user: current_user, project: user_project end - # Create a new project issue - # - # Parameters: - # id (required) - The ID of a project - # title (required) - The title of an issue - # description (optional) - The description of an issue - # assignee_id (optional) - The ID of a user to assign issue - # milestone_id (optional) - The ID of a milestone to assign issue - # labels (optional) - The labels of an issue - # created_at (optional) - Date time string, ISO 8601 formatted - # due_date (optional) - Date time string in the format YEAR-MONTH-DAY - # confidential (optional) - Boolean parameter if the issue should be confidential - # Example Request: - # POST /projects/:id/issues + desc 'Create a new project issue' do + success Entities::Issue + end + params do + requires :title, type: String, desc: 'The title of an issue' + optional :created_at, type: DateTime, + desc: 'Date time when the issue was created. Available only for admins and project owners.' + optional :merge_request_for_resolving_discussions, type: Integer, + desc: 'The IID of a merge request for which to resolve discussions' + use :issue_params + end post ':id/issues' do - required_attributes! [:title] - - keys = [:title, :description, :assignee_id, :milestone_id, :due_date, :confidential] - keys << :created_at if current_user.admin? || user_project.owner == current_user - attrs = attributes_for_keys(keys) - - # Validate label names in advance - if (errors = validate_label_params(params)).any? - render_api_error!({ labels: errors }, 400) + # Setting created_at time only allowed for admins and project owners + unless current_user.admin? || user_project.owner == current_user + params.delete(:created_at) end - attrs[:labels] = params[:labels] if params[:labels] + issue_params = declared_params(include_missing: false) - # Convert and filter out invalid confidential flags - attrs['confidential'] = to_boolean(attrs['confidential']) - attrs.delete('confidential') if attrs['confidential'].nil? - - issue = ::Issues::CreateService.new(user_project, current_user, attrs.merge(request: request, api: true)).execute + if merge_request_iid = params[:merge_request_for_resolving_discussions] + issue_params[:merge_request_for_resolving_discussions] = MergeRequestsFinder.new(current_user, project_id: user_project.id). + execute. + find_by(iid: merge_request_iid) + end + issue = ::Issues::CreateService.new(user_project, + current_user, + issue_params.merge(request: request, api: true)).execute if issue.spam? render_api_error!({ error: 'Spam detected' }, 400) end @@ -180,41 +164,30 @@ module API end end - # Update an existing issue - # - # Parameters: - # id (required) - The ID of a project - # issue_id (required) - The ID of a project issue - # title (optional) - The title of an issue - # description (optional) - The description of an issue - # assignee_id (optional) - The ID of a user to assign issue - # milestone_id (optional) - The ID of a milestone to assign issue - # labels (optional) - The labels of an issue - # state_event (optional) - The state event of an issue (close|reopen) - # updated_at (optional) - Date time string, ISO 8601 formatted - # due_date (optional) - Date time string in the format YEAR-MONTH-DAY - # confidential (optional) - Boolean parameter if the issue should be confidential - # Example Request: - # PUT /projects/:id/issues/:issue_id + desc 'Update an existing issue' do + success Entities::Issue + end + params do + requires :issue_id, type: Integer, desc: 'The ID of a project issue' + optional :title, type: String, desc: 'The title of an issue' + optional :updated_at, type: DateTime, + desc: 'Date time when the issue was updated. Available only for admins and project owners.' + use :issue_params + at_least_one_of :title, :description, :assignee_id, :milestone_id, + :labels, :created_at, :due_date, :confidential, :state_event + end put ':id/issues/:issue_id' do - issue = user_project.issues.find(params[:issue_id]) + issue = user_project.issues.find(params.delete(:issue_id)) authorize! :update_issue, issue - keys = [:title, :description, :assignee_id, :milestone_id, :state_event, :due_date, :confidential] - keys << :updated_at if current_user.admin? || user_project.owner == current_user - attrs = attributes_for_keys(keys) - # Validate label names in advance - if (errors = validate_label_params(params)).any? - render_api_error!({ labels: errors }, 400) + # Setting created_at time only allowed for admins and project owners + unless current_user.admin? || user_project.owner == current_user + params.delete(:updated_at) end - attrs[:labels] = params[:labels] if params[:labels] - - # Convert and filter out invalid confidential flags - attrs['confidential'] = to_boolean(attrs['confidential']) - attrs.delete('confidential') if attrs['confidential'].nil? - - issue = ::Issues::UpdateService.new(user_project, current_user, attrs).execute(issue) + issue = ::Issues::UpdateService.new(user_project, + current_user, + declared_params(include_missing: false)).execute(issue) if issue.valid? present issue, with: Entities::Issue, current_user: current_user, project: user_project @@ -223,19 +196,19 @@ module API end end - # Move an existing issue - # - # Parameters: - # id (required) - The ID of a project - # issue_id (required) - The ID of a project issue - # to_project_id (required) - The ID of the new project - # Example Request: - # POST /projects/:id/issues/:issue_id/move + desc 'Move an existing issue' do + success Entities::Issue + end + params do + requires :issue_id, type: Integer, desc: 'The ID of a project issue' + requires :to_project_id, type: Integer, desc: 'The ID of the new project' + end post ':id/issues/:issue_id/move' do - required_attributes! [:to_project_id] + issue = user_project.issues.find_by(id: params[:issue_id]) + not_found!('Issue') unless issue - issue = user_project.issues.find(params[:issue_id]) - new_project = Project.find(params[:to_project_id]) + new_project = Project.find_by(id: params[:to_project_id]) + not_found!('Project') unless new_project begin issue = ::Issues::MoveService.new(user_project, current_user).execute(issue, new_project) @@ -245,16 +218,13 @@ module API end end - # - # Delete a project issue - # - # Parameters: - # id (required) - The ID of a project - # issue_id (required) - The ID of a project issue - # Example Request: - # DELETE /projects/:id/issues/:issue_id + desc 'Delete a project issue' + params do + requires :issue_id, type: Integer, desc: 'The ID of a project issue' + end delete ":id/issues/:issue_id" do issue = user_project.issues.find_by(id: params[:issue_id]) + not_found!('Issue') unless issue authorize!(:destroy_issue, issue) issue.destroy diff --git a/lib/api/members.rb b/lib/api/members.rb index 2d4d5cedf20..d85f1f78cd6 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -1,5 +1,7 @@ module API class Members < Grape::API + include PaginationParams + before { authenticate! } helpers ::API::Helpers::MembersHelpers @@ -14,15 +16,15 @@ module API end params do optional :query, type: String, desc: 'A query string to search for members' + use :pagination end get ":id/members" do source = find_source(source_type, params[:id]) users = source.users users = users.merge(User.search(params[:query])) if params[:query] - users = paginate(users) - present users, with: Entities::Member, source: source + present paginate(users), with: Entities::Member, source: source end desc 'Gets a member of a group or project.' do diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 15488b33f31..5d1fe22f2df 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -1,5 +1,7 @@ module API class MergeRequests < Grape::API + include PaginationParams + DEPRECATION_MESSAGE = 'This endpoint is deprecated and will be removed in GitLab 9.0.'.freeze before { authenticate! } @@ -28,6 +30,7 @@ module API optional :assignee_id, type: Integer, desc: 'The ID of a user to assign the merge request' optional :milestone_id, type: Integer, desc: 'The ID of a milestone to assign the merge request' optional :labels, type: String, desc: 'Comma-separated list of label names' + optional :remove_source_branch, type: Boolean, desc: 'Remove source branch when merging' end end @@ -41,15 +44,14 @@ module API desc: 'Return merge requests ordered by `created_at` or `updated_at` fields.' optional :sort, type: String, values: %w[asc desc], default: 'desc', desc: 'Return merge requests sorted in `asc` or `desc` order.' - optional :iid, type: Integer, desc: 'The IID of the merge requests' + optional :iid, type: Array[Integer], desc: 'The IID of the merge requests' + use :pagination end get ":id/merge_requests" do authorize! :read_merge_request, user_project - merge_requests = user_project.merge_requests.inc_notes_with_associations - unless params[:iid].nil? - merge_requests = filter_by_iid(merge_requests, params[:iid]) - end + merge_requests = user_project.merge_requests.inc_notes_with_associations + merge_requests = filter_by_iid(merge_requests, params[:iid]) if params[:iid].present? merge_requests = case params[:state] @@ -77,12 +79,8 @@ module API post ":id/merge_requests" do authorize! :create_merge_request, user_project - mr_params = declared_params - - # Validate label names in advance - if (errors = validate_label_params(mr_params)).any? - render_api_error!({ labels: errors }, 400) - end + mr_params = declared_params(include_missing: false) + mr_params[:force_remove_source_branch] = mr_params.delete(:remove_source_branch) if mr_params[:remove_source_branch].present? merge_request = ::MergeRequests::CreateService.new(user_project, current_user, mr_params).execute @@ -145,24 +143,21 @@ module API success Entities::MergeRequest end params do - optional :title, type: String, desc: 'The title of the merge request' - optional :target_branch, type: String, desc: 'The target branch' + optional :title, type: String, allow_blank: false, desc: 'The title of the merge request' + optional :target_branch, type: String, allow_blank: false, desc: 'The target branch' optional :state_event, type: String, values: %w[close reopen merge], desc: 'Status of the merge request' use :optional_params at_least_one_of :title, :target_branch, :description, :assignee_id, - :milestone_id, :labels, :state_event + :milestone_id, :labels, :state_event, + :remove_source_branch end put path do merge_request = user_project.merge_requests.find(params.delete(:merge_request_id)) authorize! :update_merge_request, merge_request mr_params = declared_params(include_missing: false) - - # Validate label names in advance - if (errors = validate_label_params(mr_params)).any? - render_api_error!({ labels: errors }, 400) - end + mr_params[:force_remove_source_branch] = mr_params.delete(:remove_source_branch) if mr_params[:remove_source_branch].present? merge_request = ::MergeRequests::UpdateService.new(user_project, current_user, mr_params).execute(merge_request) @@ -181,7 +176,7 @@ module API optional :should_remove_source_branch, type: Boolean, desc: 'When true, the source branch will be deleted if possible' optional :merge_when_build_succeeds, type: Boolean, - desc: 'When true, this merge request will be merged when the build succeeds' + desc: 'When true, this merge request will be merged when the pipeline succeeds' optional :sha, type: String, desc: 'When present, must have the HEAD SHA of the source branch' end put "#{path}/merge" do @@ -204,18 +199,20 @@ module API should_remove_source_branch: params[:should_remove_source_branch] } - if params[:merge_when_build_succeeds] && merge_request.pipeline && merge_request.pipeline.active? - ::MergeRequests::MergeWhenBuildSucceedsService.new(merge_request.target_project, current_user, merge_params). - execute(merge_request) + if params[:merge_when_build_succeeds] && merge_request.head_pipeline && merge_request.head_pipeline.active? + ::MergeRequests::MergeWhenPipelineSucceedsService + .new(merge_request.target_project, current_user, merge_params) + .execute(merge_request) else - ::MergeRequests::MergeService.new(merge_request.target_project, current_user, merge_params). - execute(merge_request) + ::MergeRequests::MergeService + .new(merge_request.target_project, current_user, merge_params) + .execute(merge_request) end present merge_request, with: Entities::MergeRequest, current_user: current_user, project: user_project end - desc 'Cancel merge if "Merge when build succeeds" is enabled' do + desc 'Cancel merge if "Merge When Pipeline Succeeds" is enabled' do success Entities::MergeRequest end post "#{path}/cancel_merge_when_build_succeeds" do @@ -223,13 +220,18 @@ module API unauthorized! unless merge_request.can_cancel_merge_when_build_succeeds?(current_user) - ::MergeRequest::MergeWhenBuildSucceedsService.new(merge_request.target_project, current_user).cancel(merge_request) + ::MergeRequest::MergeWhenPipelineSucceedsService + .new(merge_request.target_project, current_user) + .cancel(merge_request) end desc 'Get the comments of a merge request' do detail 'Duplicate. DEPRECATED and WILL BE REMOVED in 9.0' success Entities::MRNote end + params do + use :pagination + end get "#{path}/comments" do merge_request = user_project.merge_requests.find(params[:merge_request_id]) @@ -267,6 +269,9 @@ module API desc 'List issues that will be closed on merge' do success Entities::MRNote end + params do + use :pagination + end get "#{path}/closes_issues" do merge_request = user_project.merge_requests.find(params[:merge_request_id]) issues = ::Kaminari.paginate_array(merge_request.closes_issues(current_user)) diff --git a/lib/api/milestones.rb b/lib/api/milestones.rb index 937c118779d..3c373a84ec5 100644 --- a/lib/api/milestones.rb +++ b/lib/api/milestones.rb @@ -1,6 +1,7 @@ module API - # Milestones API class Milestones < Grape::API + include PaginationParams + before { authenticate! } helpers do @@ -14,7 +15,8 @@ module API params :optional_params do optional :description, type: String, desc: 'The description of the milestone' - optional :due_date, type: String, desc: 'The due date of the milestone' + optional :due_date, type: String, desc: 'The due date of the milestone. The ISO 8601 date format (%Y-%m-%d)' + optional :start_date, type: String, desc: 'The start date of the milestone. The ISO 8601 date format (%Y-%m-%d)' end end @@ -28,7 +30,8 @@ module API params do optional :state, type: String, values: %w[active closed all], default: 'all', desc: 'Return "active", "closed", or "all" milestones' - optional :iid, type: Integer, desc: 'The IID of the milestone' + optional :iid, type: Array[Integer], desc: 'The IID of the milestone' + use :pagination end get ":id/milestones" do authorize! :read_milestone, user_project @@ -102,6 +105,7 @@ module API end params do requires :milestone_id, type: Integer, desc: 'The ID of a project milestone' + use :pagination end get ":id/milestones/:milestone_id/issues" do authorize! :read_milestone, user_project diff --git a/lib/api/namespaces.rb b/lib/api/namespaces.rb index fe981d7b9fa..30761cb9b55 100644 --- a/lib/api/namespaces.rb +++ b/lib/api/namespaces.rb @@ -1,6 +1,7 @@ module API - # namespaces API class Namespaces < Grape::API + include PaginationParams + before { authenticate! } resource :namespaces do @@ -9,6 +10,7 @@ module API end params do optional :search, type: String, desc: "Search query for namespaces" + use :pagination end get do namespaces = current_user.admin ? Namespace.all : current_user.namespaces diff --git a/lib/api/notes.rb b/lib/api/notes.rb index b255b47742b..d0faf17714b 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -1,6 +1,7 @@ module API - # Notes API class Notes < Grape::API + include PaginationParams + before { authenticate! } NOTEABLE_TYPES = [Issue, MergeRequest, Snippet] @@ -17,6 +18,7 @@ module API end params do requires :noteable_id, type: Integer, desc: 'The ID of the noteable' + use :pagination end get ":id/#{noteables_str}/:noteable_id/notes" do noteable = user_project.send(noteables_str.to_sym).find(params[:noteable_id]) diff --git a/lib/api/pagination_params.rb b/lib/api/pagination_params.rb new file mode 100644 index 00000000000..8c1e4381a74 --- /dev/null +++ b/lib/api/pagination_params.rb @@ -0,0 +1,24 @@ +module API + # Concern for declare pagination params. + # + # @example + # class CustomApiResource < Grape::API + # include PaginationParams + # + # params do + # use :pagination + # end + # end + module PaginationParams + extend ActiveSupport::Concern + + included do + helpers do + params :pagination do + optional :page, type: Integer, desc: 'Current page number' + optional :per_page, type: Integer, desc: 'Number of items per page' + end + end + end + end +end diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb index e69b0569612..b634b1d0222 100644 --- a/lib/api/pipelines.rb +++ b/lib/api/pipelines.rb @@ -1,5 +1,7 @@ module API class Pipelines < Grape::API + include PaginationParams + before { authenticate! } params do @@ -11,8 +13,7 @@ module API success Entities::Pipeline end params do - optional :page, type: Integer, desc: 'Page number of the current request' - optional :per_page, type: Integer, desc: 'Number of items per page' + use :pagination optional :scope, type: String, values: ['running', 'branches', 'tags'], desc: 'Either running, branches, or tags' end diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb index 2b36ef7c426..dcc0fb7a911 100644 --- a/lib/api/project_hooks.rb +++ b/lib/api/project_hooks.rb @@ -1,6 +1,10 @@ module API - # Projects API class ProjectHooks < Grape::API + include PaginationParams + + before { authenticate! } + before { authorize_admin_project } + helpers do params :project_hook_properties do requires :url, type: String, desc: "The URL to send the request to" @@ -17,9 +21,6 @@ module API end end - before { authenticate! } - before { authorize_admin_project } - params do requires :id, type: String, desc: 'The ID of a project' end @@ -27,6 +28,9 @@ module API desc 'Get project hooks' do success Entities::ProjectHook end + params do + use :pagination + end get ":id/hooks" do hooks = paginate user_project.hooks diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb index d0ee9c9a5b2..9d8c5b63685 100644 --- a/lib/api/project_snippets.rb +++ b/lib/api/project_snippets.rb @@ -1,6 +1,7 @@ module API - # Projects API class ProjectSnippets < Grape::API + include PaginationParams + before { authenticate! } params do @@ -24,6 +25,9 @@ module API desc 'Get all project snippets' do success Entities::ProjectSnippet end + params do + use :pagination + end get ":id/snippets" do present paginate(snippets_for_current_user), with: Entities::ProjectSnippet end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 6b856128c2e..2929d2157dc 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -1,293 +1,295 @@ module API # Projects API class Projects < Grape::API - before { authenticate! } + include PaginationParams + + before { authenticate_non_get! } + + helpers do + params :optional_params do + optional :description, type: String, desc: 'The description of the project' + optional :issues_enabled, type: Boolean, desc: 'Flag indication if the issue tracker is enabled' + optional :merge_requests_enabled, type: Boolean, desc: 'Flag indication if merge requests are enabled' + optional :wiki_enabled, type: Boolean, desc: 'Flag indication if the wiki is enabled' + optional :builds_enabled, type: Boolean, desc: 'Flag indication if builds are enabled' + optional :snippets_enabled, type: Boolean, desc: 'Flag indication if snippets are enabled' + optional :shared_runners_enabled, type: Boolean, desc: 'Flag indication if shared runners are enabled for that project' + optional :container_registry_enabled, type: Boolean, desc: 'Flag indication if the container registry is enabled for that project' + optional :lfs_enabled, type: Boolean, desc: 'Flag indication if Git LFS is enabled for that project' + optional :public, type: Boolean, desc: 'Create a public project. The same as visibility_level = 20.' + optional :visibility_level, type: Integer, values: [ + Gitlab::VisibilityLevel::PRIVATE, + Gitlab::VisibilityLevel::INTERNAL, + Gitlab::VisibilityLevel::PUBLIC ], desc: 'Create a public project. The same as visibility_level = 20.' + optional :public_builds, type: Boolean, desc: 'Perform public builds' + optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access' + optional :only_allow_merge_if_build_succeeds, type: Boolean, desc: 'Only allow to merge if builds succeed' + optional :only_allow_merge_if_all_discussions_are_resolved, type: Boolean, desc: 'Only allow to merge if all discussions are resolved' + end - resource :projects, requirements: { id: /[^\/]+/ } do - helpers do - def map_public_to_visibility_level(attrs) - publik = attrs.delete(:public) - if publik.present? && !attrs[:visibility_level].present? - publik = to_boolean(publik) - # Since setting the public attribute to private could mean either - # private or internal, use the more conservative option, private. - attrs[:visibility_level] = (publik == true) ? Gitlab::VisibilityLevel::PUBLIC : Gitlab::VisibilityLevel::PRIVATE - end - attrs + def map_public_to_visibility_level(attrs) + publik = attrs.delete(:public) + if !publik.nil? && !attrs[:visibility_level].present? + # Since setting the public attribute to private could mean either + # private or internal, use the more conservative option, private. + attrs[:visibility_level] = (publik == true) ? Gitlab::VisibilityLevel::PUBLIC : Gitlab::VisibilityLevel::PRIVATE end + attrs end + end - # Get a projects list for authenticated user - # - # Example Request: - # GET /projects - get do - projects = current_user.authorized_projects - projects = filter_projects(projects) - projects = paginate projects - entity = params[:simple] ? Entities::BasicProjectDetails : Entities::ProjectWithAccess + resource :projects do + helpers do + params :sort_params do + optional :order_by, type: String, values: %w[id name path created_at updated_at last_activity_at], + default: 'created_at', desc: 'Return projects ordered by field' + optional :sort, type: String, values: %w[asc desc], default: 'desc', + desc: 'Return projects sorted in ascending and descending order' + end + + params :filter_params do + optional :archived, type: Boolean, default: false, desc: 'Limit by archived status' + optional :visibility, type: String, values: %w[public internal private], + desc: 'Limit by visibility' + optional :search, type: String, desc: 'Return list of authorized projects matching the search criteria' + use :sort_params + end - present projects, with: entity, user: current_user + params :create_params do + optional :namespace_id, type: Integer, desc: 'Namespace ID for the new project. Default to the user namespace.' + optional :import_url, type: String, desc: 'URL from which the project is imported' + end end - # Get a list of visible projects for authenticated user - # - # Example Request: - # GET /projects/visible + desc 'Get a list of visible projects for authenticated user' do + success Entities::BasicProjectDetails + end + params do + optional :simple, type: Boolean, default: false, + desc: 'Return only the ID, URL, name, and path of each project' + use :filter_params + use :pagination + end get '/visible' do projects = ProjectsFinder.new.execute(current_user) projects = filter_projects(projects) - projects = paginate projects + entity = params[:simple] || !current_user ? Entities::BasicProjectDetails : Entities::ProjectWithAccess + + present paginate(projects), with: entity, user: current_user + end + + desc 'Get a projects list for authenticated user' do + success Entities::BasicProjectDetails + end + params do + optional :simple, type: Boolean, default: false, + desc: 'Return only the ID, URL, name, and path of each project' + use :filter_params + use :pagination + end + get do + authenticate! + + projects = current_user.authorized_projects + projects = filter_projects(projects) entity = params[:simple] ? Entities::BasicProjectDetails : Entities::ProjectWithAccess - present projects, with: entity, user: current_user + present paginate(projects), with: entity, user: current_user end - # Get an owned projects list for authenticated user - # - # Example Request: - # GET /projects/owned + desc 'Get an owned projects list for authenticated user' do + success Entities::BasicProjectDetails + end + params do + use :filter_params + use :pagination + end get '/owned' do + authenticate! + projects = current_user.owned_projects projects = filter_projects(projects) - projects = paginate projects - present projects, with: Entities::ProjectWithAccess, user: current_user + + present paginate(projects), with: Entities::ProjectWithAccess, user: current_user end - # Gets starred project for the authenticated user - # - # Example Request: - # GET /projects/starred + desc 'Gets starred project for the authenticated user' do + success Entities::BasicProjectDetails + end + params do + use :filter_params + use :pagination + end get '/starred' do + authenticate! + projects = current_user.viewable_starred_projects projects = filter_projects(projects) - projects = paginate projects - present projects, with: Entities::Project, user: current_user + + present paginate(projects), with: Entities::Project, user: current_user end - # Get all projects for admin user - # - # Example Request: - # GET /projects/all + desc 'Get all projects for admin user' do + success Entities::BasicProjectDetails + end + params do + use :filter_params + use :pagination + end get '/all' do authenticated_as_admin! + projects = Project.all projects = filter_projects(projects) - projects = paginate projects - present projects, with: Entities::ProjectWithAccess, user: current_user + + present paginate(projects), with: Entities::ProjectWithAccess, user: current_user end - # Get a single project - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # GET /projects/:id - get ":id" do - present user_project, with: Entities::ProjectWithAccess, user: current_user, - user_can_admin_project: can?(current_user, :admin_project, user_project) + desc 'Search for projects the current user has access to' do + success Entities::Project end + params do + requires :query, type: String, desc: 'The project name to be searched' + use :sort_params + use :pagination + end + get "/search/:query" do + search_service = Search::GlobalService.new(current_user, search: params[:query]).execute + projects = search_service.objects('projects', params[:page]) + projects = projects.reorder(params[:order_by] => params[:sort]) - # Get events for a single project - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # GET /projects/:id/events - get ":id/events" do - events = paginate user_project.events.recent - present events, with: Entities::Event - end - - # Create new project - # - # Parameters: - # name (required) - name for new project - # description (optional) - short project description - # issues_enabled (optional) - # merge_requests_enabled (optional) - # builds_enabled (optional) - # wiki_enabled (optional) - # snippets_enabled (optional) - # container_registry_enabled (optional) - # shared_runners_enabled (optional) - # namespace_id (optional) - defaults to user namespace - # public (optional) - if true same as setting visibility_level = 20 - # visibility_level (optional) - 0 by default - # import_url (optional) - # public_builds (optional) - # lfs_enabled (optional) - # request_access_enabled (optional) - Allow users to request member access - # Example Request - # POST /projects + present paginate(projects), with: Entities::Project + end + + desc 'Create new project' do + success Entities::Project + end + params do + requires :name, type: String, desc: 'The name of the project' + optional :path, type: String, desc: 'The path of the repository' + use :optional_params + use :create_params + end post do - required_attributes! [:name] - attrs = attributes_for_keys [:builds_enabled, - :container_registry_enabled, - :description, - :import_url, - :issues_enabled, - :lfs_enabled, - :merge_requests_enabled, - :name, - :namespace_id, - :only_allow_merge_if_build_succeeds, - :path, - :public, - :public_builds, - :request_access_enabled, - :shared_runners_enabled, - :snippets_enabled, - :visibility_level, - :wiki_enabled, - :only_allow_merge_if_all_discussions_are_resolved] - attrs = map_public_to_visibility_level(attrs) - @project = ::Projects::CreateService.new(current_user, attrs).execute - if @project.saved? - present @project, with: Entities::Project, - user_can_admin_project: can?(current_user, :admin_project, @project) + attrs = map_public_to_visibility_level(declared_params(include_missing: false)) + project = ::Projects::CreateService.new(current_user, attrs).execute + + if project.saved? + present project, with: Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, project) else - if @project.errors[:limit_reached].present? - error!(@project.errors[:limit_reached], 403) + if project.errors[:limit_reached].present? + error!(project.errors[:limit_reached], 403) end - render_validation_error!(@project) + render_validation_error!(project) end end - # Create new project for a specified user. Only available to admin users. - # - # Parameters: - # user_id (required) - The ID of a user - # name (required) - name for new project - # description (optional) - short project description - # default_branch (optional) - 'master' by default - # issues_enabled (optional) - # merge_requests_enabled (optional) - # builds_enabled (optional) - # wiki_enabled (optional) - # snippets_enabled (optional) - # container_registry_enabled (optional) - # shared_runners_enabled (optional) - # public (optional) - if true same as setting visibility_level = 20 - # visibility_level (optional) - # import_url (optional) - # public_builds (optional) - # lfs_enabled (optional) - # request_access_enabled (optional) - Allow users to request member access - # Example Request - # POST /projects/user/:user_id + desc 'Create new project for a specified user. Only available to admin users.' do + success Entities::Project + end + params do + requires :name, type: String, desc: 'The name of the project' + requires :user_id, type: Integer, desc: 'The ID of a user' + optional :default_branch, type: String, desc: 'The default branch of the project' + use :optional_params + use :create_params + end post "user/:user_id" do authenticated_as_admin! - user = User.find(params[:user_id]) - attrs = attributes_for_keys [:builds_enabled, - :default_branch, - :description, - :import_url, - :issues_enabled, - :lfs_enabled, - :merge_requests_enabled, - :name, - :only_allow_merge_if_build_succeeds, - :public, - :public_builds, - :request_access_enabled, - :shared_runners_enabled, - :snippets_enabled, - :visibility_level, - :wiki_enabled, - :only_allow_merge_if_all_discussions_are_resolved] - attrs = map_public_to_visibility_level(attrs) - @project = ::Projects::CreateService.new(user, attrs).execute - if @project.saved? - present @project, with: Entities::Project, - user_can_admin_project: can?(current_user, :admin_project, @project) + user = User.find_by(id: params.delete(:user_id)) + not_found!('User') unless user + + attrs = map_public_to_visibility_level(declared_params(include_missing: false)) + project = ::Projects::CreateService.new(user, attrs).execute + + if project.saved? + present project, with: Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, project) else - render_validation_error!(@project) + render_validation_error!(project) end end + end + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: { id: /[^\/]+/ } do + desc 'Get a single project' do + success Entities::ProjectWithAccess + end + get ":id" do + entity = current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails + present user_project, with: entity, user: current_user, + user_can_admin_project: can?(current_user, :admin_project, user_project) + end + + desc 'Get events for a single project' do + success Entities::Event + end + params do + use :pagination + end + get ":id/events" do + present paginate(user_project.events.recent), with: Entities::Event + end - # Fork new project for the current user or provided namespace. - # - # Parameters: - # id (required) - The ID of a project - # namespace (optional) - The ID or name of the namespace that the project will be forked into. - # Example Request - # POST /projects/fork/:id + desc 'Fork new project for the current user or provided namespace.' do + success Entities::Project + end + params do + optional :namespace, type: String, desc: 'The ID or name of the namespace that the project will be forked into' + end post 'fork/:id' do - attrs = {} - namespace_id = params[:namespace] + fork_params = declared_params(include_missing: false) + namespace_id = fork_params[:namespace] if namespace_id.present? - namespace = Namespace.find_by(id: namespace_id) || Namespace.find_by_path_or_name(namespace_id) + fork_params[:namespace] = if namespace_id =~ /^\d+$/ + Namespace.find_by(id: namespace_id) + else + Namespace.find_by_path_or_name(namespace_id) + end - unless namespace && can?(current_user, :create_projects, namespace) + unless fork_params[:namespace] && can?(current_user, :create_projects, fork_params[:namespace]) not_found!('Target Namespace') end - - attrs[:namespace] = namespace end - @forked_project = - ::Projects::ForkService.new(user_project, - current_user, - attrs).execute + forked_project = ::Projects::ForkService.new(user_project, current_user, fork_params).execute - if @forked_project.errors.any? - conflict!(@forked_project.errors.messages) + if forked_project.errors.any? + conflict!(forked_project.errors.messages) else - present @forked_project, with: Entities::Project, - user_can_admin_project: can?(current_user, :admin_project, @forked_project) + present forked_project, with: Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, forked_project) end end - # Update an existing project - # - # Parameters: - # id (required) - the id of a project - # name (optional) - name of a project - # path (optional) - path of a project - # description (optional) - short project description - # issues_enabled (optional) - # merge_requests_enabled (optional) - # builds_enabled (optional) - # wiki_enabled (optional) - # snippets_enabled (optional) - # container_registry_enabled (optional) - # shared_runners_enabled (optional) - # public (optional) - if true same as setting visibility_level = 20 - # visibility_level (optional) - visibility level of a project - # public_builds (optional) - # lfs_enabled (optional) - # Example Request - # PUT /projects/:id + desc 'Update an existing project' do + success Entities::Project + end + params do + optional :name, type: String, desc: 'The name of the project' + optional :default_branch, type: String, desc: 'The default branch of the project' + optional :path, type: String, desc: 'The path of the repository' + use :optional_params + at_least_one_of :name, :description, :issues_enabled, :merge_requests_enabled, + :wiki_enabled, :builds_enabled, :snippets_enabled, + :shared_runners_enabled, :container_registry_enabled, + :lfs_enabled, :public, :visibility_level, :public_builds, + :request_access_enabled, :only_allow_merge_if_build_succeeds, + :only_allow_merge_if_all_discussions_are_resolved, :path, + :default_branch + end put ':id' do - attrs = attributes_for_keys [:builds_enabled, - :container_registry_enabled, - :default_branch, - :description, - :issues_enabled, - :lfs_enabled, - :merge_requests_enabled, - :name, - :only_allow_merge_if_build_succeeds, - :path, - :public, - :public_builds, - :request_access_enabled, - :shared_runners_enabled, - :snippets_enabled, - :visibility_level, - :wiki_enabled, - :only_allow_merge_if_all_discussions_are_resolved] - attrs = map_public_to_visibility_level(attrs) authorize_admin_project + attrs = map_public_to_visibility_level(declared_params(include_missing: false)) authorize! :rename_project, user_project if attrs[:name].present? - if attrs[:visibility_level].present? - authorize! :change_visibility_level, user_project - end + authorize! :change_visibility_level, user_project if attrs[:visibility_level].present? - ::Projects::UpdateService.new(user_project, - current_user, attrs).execute + ::Projects::UpdateService.new(user_project, current_user, attrs).execute if user_project.errors.any? render_validation_error!(user_project) @@ -297,12 +299,9 @@ module API end end - # Archive project - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # PUT /projects/:id/archive + desc 'Archive a project' do + success Entities::Project + end post ':id/archive' do authorize!(:archive_project, user_project) @@ -311,12 +310,9 @@ module API present user_project, with: Entities::Project end - # Unarchive project - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # PUT /projects/:id/unarchive + desc 'Unarchive a project' do + success Entities::Project + end post ':id/unarchive' do authorize!(:archive_project, user_project) @@ -325,12 +321,9 @@ module API present user_project, with: Entities::Project end - # Star project - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # POST /projects/:id/star + desc 'Star a project' do + success Entities::Project + end post ':id/star' do if current_user.starred?(user_project) not_modified! @@ -342,12 +335,9 @@ module API end end - # Unstar project - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # DELETE /projects/:id/star + desc 'Unstar a project' do + success Entities::Project + end delete ':id/star' do if current_user.starred?(user_project) current_user.toggle_star(user_project) @@ -359,67 +349,51 @@ module API end end - # Remove project - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # DELETE /projects/:id + desc 'Remove a project' delete ":id" do authorize! :remove_project, user_project ::Projects::DestroyService.new(user_project, current_user, {}).async_execute end - # Mark this project as forked from another - # - # Parameters: - # id: (required) - The ID of the project being marked as a fork - # forked_from_id: (required) - The ID of the project it was forked from - # Example Request: - # POST /projects/:id/fork/:forked_from_id + desc 'Mark this project as forked from another' + params do + requires :forked_from_id, type: String, desc: 'The ID of the project it was forked from' + end post ":id/fork/:forked_from_id" do authenticated_as_admin! - forked_from_project = find_project(params[:forked_from_id]) - unless forked_from_project.nil? - if user_project.forked_from_project.nil? - user_project.create_forked_project_link(forked_to_project_id: user_project.id, forked_from_project_id: forked_from_project.id) - else - render_api_error!("Project already forked", 409) - end + + forked_from_project = find_project!(params[:forked_from_id]) + not_found!("Source Project") unless forked_from_project + + if user_project.forked_from_project.nil? + user_project.create_forked_project_link(forked_to_project_id: user_project.id, forked_from_project_id: forked_from_project.id) else - not_found!("Source Project") + render_api_error!("Project already forked", 409) end end - # Remove a forked_from relationship - # - # Parameters: - # id: (required) - The ID of the project being marked as a fork - # Example Request: - # DELETE /projects/:id/fork + desc 'Remove a forked_from relationship' delete ":id/fork" do authorize! :remove_fork_project, user_project + if user_project.forked? user_project.forked_project_link.destroy + else + not_modified! end end - # Share project with group - # - # Parameters: - # id (required) - The ID of a project - # group_id (required) - The ID of a group - # group_access (required) - Level of permissions for sharing - # expires_at (optional) - Share expiration date - # - # Example Request: - # POST /projects/:id/share + desc 'Share the project with a group' do + success Entities::ProjectGroupLink + end + params do + requires :group_id, type: Integer, desc: 'The ID of a group' + requires :group_access, type: Integer, values: Gitlab::Access.values, desc: 'The group access level' + optional :expires_at, type: Date, desc: 'Share expiration date' + end post ":id/share" do authorize! :admin_project, user_project - required_attributes! [:group_id, :group_access] - attrs = attributes_for_keys [:group_id, :group_access, :expires_at] - - group = Group.find_by_id(attrs[:group_id]) + group = Group.find_by_id(params[:group_id]) unless group && can?(current_user, :read_group, group) not_found!('Group') @@ -429,7 +403,7 @@ module API return render_api_error!("The project sharing with group is disabled", 400) end - link = user_project.project_group_links.new(attrs) + link = user_project.project_group_links.new(declared_params(include_missing: false)) if link.save present link, with: Entities::ProjectGroupLink @@ -438,40 +412,39 @@ module API end end - # Upload a file - # - # Parameters: - # id: (required) - The ID of the project - # file: (required) - The file to be uploaded - post ":id/uploads" do - ::Projects::UploadService.new(user_project, params[:file]).execute + params do + requires :group_id, type: Integer, desc: 'The ID of the group' end + delete ":id/share/:group_id" do + authorize! :admin_project, user_project - # search for projects current_user has access to - # - # Parameters: - # query (required) - A string contained in the project name - # per_page (optional) - number of projects to return per page - # page (optional) - the page to retrieve - # Example Request: - # GET /projects/search/:query - get "/search/:query" do - search_service = Search::GlobalService.new(current_user, search: params[:query]).execute - projects = search_service.objects('projects', params[:page]) - projects = projects.reorder(project_order_by => project_sort) + link = user_project.project_group_links.find_by(group_id: params[:group_id]) + not_found!('Group Link') unless link - present paginate(projects), with: Entities::Project + link.destroy + no_content! + end + + desc 'Upload a file' + params do + requires :file, type: File, desc: 'The file to be uploaded' + end + post ":id/uploads" do + ::Projects::UploadService.new(user_project, params[:file]).execute end - # Get a users list - # - # Example Request: - # GET /users + desc 'Get the users list of a project' do + success Entities::UserBasic + end + params do + optional :search, type: String, desc: 'Return list of users matching the search criteria' + use :pagination + end get ':id/users' do - @users = User.where(id: user_project.team.users.map(&:id)) - @users = @users.search(params[:search]) if params[:search].present? - @users = paginate @users - present @users, with: Entities::UserBasic + users = user_project.team.users + users = users.search(params[:search]) if params[:search].present? + + present paginate(users), with: Entities::UserBasic end end end diff --git a/lib/api/runners.rb b/lib/api/runners.rb index b145cce7e3e..4816b5ed1b7 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -1,5 +1,7 @@ module API class Runners < Grape::API + include PaginationParams + before { authenticate! } resource :runners do @@ -9,6 +11,7 @@ module API params do optional :scope, type: String, values: %w[active paused online], desc: 'The scope of specific runners to show' + use :pagination end get do runners = filter_runners(current_user.ci_authorized_runners, params[:scope], without: ['specific', 'shared']) @@ -21,6 +24,7 @@ module API params do optional :scope, type: String, values: %w[active paused online specific shared], desc: 'The scope of specific runners to show' + use :pagination end get 'all' do authenticated_as_admin! @@ -91,6 +95,7 @@ module API params do optional :scope, type: String, values: %w[active paused online specific shared], desc: 'The scope of specific runners to show' + use :pagination end get ':id/runners' do runners = filter_runners(Ci::Runner.owned_or_shared(user_project.id), params[:scope]) diff --git a/lib/api/services.rb b/lib/api/services.rb index 4d23499aa39..59232c84c24 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -1,84 +1,638 @@ module API - # Projects API class Services < Grape::API + services = { + 'asana' => [ + { + required: true, + name: :api_key, + type: String, + desc: 'User API token' + }, + { + required: false, + name: :restrict_to_branch, + type: String, + desc: 'Comma-separated list of branches which will be automatically inspected. Leave blank to include all branches' + } + ], + 'assembla' => [ + { + required: true, + name: :token, + type: String, + desc: 'The authentication token' + }, + { + required: false, + name: :subdomain, + type: String, + desc: 'Subdomain setting' + } + ], + 'bamboo' => [ + { + required: true, + name: :bamboo_url, + type: String, + desc: 'Bamboo root URL like https://bamboo.example.com' + }, + { + required: true, + name: :build_key, + type: String, + desc: 'Bamboo build plan key like' + }, + { + required: true, + name: :username, + type: String, + desc: 'A user with API access, if applicable' + }, + { + required: true, + name: :password, + type: String, + desc: 'Passord of the user' + } + ], + 'bugzilla' => [ + { + required: true, + name: :new_issue_url, + type: String, + desc: 'New issue URL' + }, + { + required: true, + name: :issues_url, + type: String, + desc: 'Issues URL' + }, + { + required: true, + name: :project_url, + type: String, + desc: 'Project URL' + }, + { + required: false, + name: :description, + type: String, + desc: 'Description' + }, + { + required: false, + name: :title, + type: String, + desc: 'Title' + } + ], + 'buildkite' => [ + { + required: true, + name: :token, + type: String, + desc: 'Buildkite project GitLab token' + }, + { + required: true, + name: :project_url, + type: String, + desc: 'The buildkite project URL' + }, + { + required: false, + name: :enable_ssl_verification, + type: Boolean, + desc: 'Enable SSL verification for communication' + } + ], + 'builds-email' => [ + { + required: true, + name: :recipients, + type: String, + desc: 'Comma-separated list of recipient email addresses' + }, + { + required: false, + name: :add_pusher, + type: Boolean, + desc: 'Add pusher to recipients list' + }, + { + required: false, + name: :notify_only_broken_builds, + type: Boolean, + desc: 'Notify only broken builds' + } + ], + 'campfire' => [ + { + required: true, + name: :token, + type: String, + desc: 'Campfire token' + }, + { + required: false, + name: :subdomain, + type: String, + desc: 'Campfire subdomain' + }, + { + required: false, + name: :room, + type: String, + desc: 'Campfire room' + }, + ], + 'custom-issue-tracker' => [ + { + required: true, + name: :new_issue_url, + type: String, + desc: 'New issue URL' + }, + { + required: true, + name: :issues_url, + type: String, + desc: 'Issues URL' + }, + { + required: true, + name: :project_url, + type: String, + desc: 'Project URL' + }, + { + required: false, + name: :description, + type: String, + desc: 'Description' + }, + { + required: false, + name: :title, + type: String, + desc: 'Title' + } + ], + 'drone-ci' => [ + { + required: true, + name: :token, + type: String, + desc: 'Drone CI token' + }, + { + required: true, + name: :drone_url, + type: String, + desc: 'Drone CI URL' + }, + { + required: false, + name: :enable_ssl_verification, + type: Boolean, + desc: 'Enable SSL verification for communication' + } + ], + 'emails-on-push' => [ + { + required: true, + name: :recipients, + type: String, + desc: 'Comma-separated list of recipient email addresses' + }, + { + required: false, + name: :disable_diffs, + type: Boolean, + desc: 'Disable code diffs' + }, + { + required: false, + name: :send_from_committer_email, + type: Boolean, + desc: 'Send from committer' + } + ], + 'external-wiki' => [ + { + required: true, + name: :external_wiki_url, + type: String, + desc: 'The URL of the external Wiki' + } + ], + 'flowdock' => [ + { + required: true, + name: :token, + type: String, + desc: 'Flowdock token' + } + ], + 'gemnasium' => [ + { + required: true, + name: :api_key, + type: String, + desc: 'Your personal API key on gemnasium.com' + }, + { + required: true, + name: :token, + type: String, + desc: "The project's slug on gemnasium.com" + } + ], + 'hipchat' => [ + { + required: true, + name: :token, + type: String, + desc: 'The room token' + }, + { + required: false, + name: :room, + type: String, + desc: 'The room name or ID' + }, + { + required: false, + name: :color, + type: String, + desc: 'The room color' + }, + { + required: false, + name: :notify, + type: Boolean, + desc: 'Enable notifications' + }, + { + required: false, + name: :api_version, + type: String, + desc: 'Leave blank for default (v2)' + }, + { + required: false, + name: :server, + type: String, + desc: 'Leave blank for default. https://hipchat.example.com' + } + ], + 'irker' => [ + { + required: true, + name: :recipients, + type: String, + desc: 'Recipients/channels separated by whitespaces' + }, + { + required: false, + name: :default_irc_uri, + type: String, + desc: 'Default: irc://irc.network.net:6697' + }, + { + required: false, + name: :server_host, + type: String, + desc: 'Server host. Default localhost' + }, + { + required: false, + name: :server_port, + type: Integer, + desc: 'Server port. Default 6659' + }, + { + required: false, + name: :colorize_messages, + type: Boolean, + desc: 'Colorize messages' + } + ], + 'jira' => [ + { + required: true, + name: :url, + type: String, + desc: 'The URL to the JIRA project which is being linked to this GitLab project, e.g., https://jira.example.com' + }, + { + required: true, + name: :project_key, + type: String, + desc: 'The short identifier for your JIRA project, all uppercase, e.g., PROJ' + }, + { + required: false, + name: :username, + type: String, + desc: 'The username of the user created to be used with GitLab/JIRA' + }, + { + required: false, + name: :password, + type: String, + desc: 'The password of the user created to be used with GitLab/JIRA' + }, + { + required: false, + name: :jira_issue_transition_id, + type: Integer, + desc: 'The ID of a transition that moves issues to a closed state. You can find this number under the JIRA workflow administration (**Administration > Issues > Workflows**) by selecting **View** under **Operations** of the desired workflow of your project. The ID of each state can be found inside the parenthesis of each transition name under the **Transitions (id)** column ([see screenshot][trans]). By default, this ID is set to `2`' + } + ], + + 'kubernetes' => [ + { + required: true, + name: :namespace, + type: String, + desc: 'The Kubernetes namespace to use' + }, + { + required: true, + name: :api_url, + type: String, + desc: 'The URL to the Kubernetes cluster API, e.g., https://kubernetes.example.com' + }, + { + required: true, + name: :token, + type: String, + desc: 'The service token to authenticate against the Kubernetes cluster with' + }, + { + required: false, + name: :ca_pem, + type: String, + desc: 'A custom certificate authority bundle to verify the Kubernetes cluster with (PEM format)' + }, + ], + + 'mattermost-slash-commands' => [ + { + required: true, + name: :token, + type: String, + desc: 'The Mattermost token' + } + ], + 'pipelines-email' => [ + { + required: true, + name: :recipients, + type: String, + desc: 'Comma-separated list of recipient email addresses' + }, + { + required: false, + name: :notify_only_broken_builds, + type: Boolean, + desc: 'Notify only broken builds' + } + ], + 'pivotaltracker' => [ + { + required: true, + name: :token, + type: String, + desc: 'The Pivotaltracker token' + }, + { + required: false, + name: :restrict_to_branch, + type: String, + desc: 'Comma-separated list of branches which will be automatically inspected. Leave blank to include all branches.' + } + ], + 'pushover' => [ + { + required: true, + name: :api_key, + type: String, + desc: 'The application key' + }, + { + required: true, + name: :user_key, + type: String, + desc: 'The user key' + }, + { + required: true, + name: :priority, + type: String, + desc: 'The priority' + }, + { + required: true, + name: :device, + type: String, + desc: 'Leave blank for all active devices' + }, + { + required: true, + name: :sound, + type: String, + desc: 'The sound of the notification' + } + ], + 'redmine' => [ + { + required: true, + name: :new_issue_url, + type: String, + desc: 'The new issue URL' + }, + { + required: true, + name: :project_url, + type: String, + desc: 'The project URL' + }, + { + required: true, + name: :issues_url, + type: String, + desc: 'The issues URL' + }, + { + required: false, + name: :description, + type: String, + desc: 'The description of the tracker' + } + ], + 'slack-notification' => [ + { + required: true, + name: :webhook, + type: String, + desc: 'The Slack webhook. e.g. https://hooks.slack.com/services/...' + }, + { + required: false, + name: :new_issue_url, + type: String, + desc: 'The user name' + }, + { + required: false, + name: :channel, + type: String, + desc: 'The channel name' + } + ], + 'mattermost-notification' => [ + { + required: true, + name: :webhook, + type: String, + desc: 'The Mattermost webhook. e.g. http://mattermost_host/hooks/...' + } + ], + 'teamcity' => [ + { + required: true, + name: :teamcity_url, + type: String, + desc: 'TeamCity root URL like https://teamcity.example.com' + }, + { + required: true, + name: :build_type, + type: String, + desc: 'Build configuration ID' + }, + { + required: true, + name: :username, + type: String, + desc: 'A user with permissions to trigger a manual build' + }, + { + required: true, + name: :password, + type: String, + desc: 'The password of the user' + } + ] + }.freeze + + trigger_services = { + 'mattermost-slash-commands' => [ + { + name: :token, + type: String, + desc: 'The Mattermost token' + } + ] + }.freeze + resource :projects do before { authenticate! } before { authorize_admin_project } - # Set <service_slug> service for project - # - # Example Request: - # - # PUT /projects/:id/services/gitlab-ci - # - put ':id/services/:service_slug' do - if project_service - validators = project_service.class.validators.select do |s| - s.class == ActiveRecord::Validations::PresenceValidator && - s.attributes != [:project_id] + helpers do + def service_attributes(service) + service.fields.inject([]) do |arr, hash| + arr << hash[:name].to_sym end + end + end - required_attributes! validators.map(&:attributes).flatten.uniq - attrs = attributes_for_keys service_attributes + services.each do |service_slug, settings| + desc "Set #{service_slug} service for project" + params do + settings.each do |setting| + if setting[:required] + requires setting[:name], type: setting[:type], desc: setting[:desc] + else + optional setting[:name], type: setting[:type], desc: setting[:desc] + end + end + end + put ":id/services/#{service_slug}" do + service = user_project.find_or_initialize_service(service_slug.underscore) + service_params = declared_params(include_missing: false).merge(active: true) - if project_service.update_attributes(attrs.merge(active: true)) + if service.update_attributes(service_params) true else - not_found! + render_api_error!('400 Bad Request', 400) end end end - # Delete <service_slug> service for project - # - # Example Request: - # - # DELETE /project/:id/services/gitlab-ci - # - delete ':id/services/:service_slug' do - if project_service - attrs = service_attributes.inject({}) do |hash, key| - hash.merge!(key => nil) - end + desc "Delete a service for project" + params do + requires :service_slug, type: String, values: services.keys, desc: 'The name of the service' + end + delete ":id/services/:service_slug" do + service = user_project.find_or_initialize_service(params[:service_slug].underscore) - if project_service.update_attributes(attrs.merge(active: false)) - true - else - not_found! - end + attrs = service_attributes(service).inject({}) do |hash, key| + hash.merge!(key => nil) + end + + if service.update_attributes(attrs.merge(active: false)) + true + else + render_api_error!('400 Bad Request', 400) end end - # Get <service_slug> service settings for project - # - # Example Request: - # - # GET /project/:id/services/gitlab-ci - # - get ':id/services/:service_slug' do - present project_service, with: Entities::ProjectService, include_passwords: current_user.is_admin? + desc 'Get the service settings for project' do + success Entities::ProjectService + end + params do + requires :service_slug, type: String, values: services.keys, desc: 'The name of the service' + end + get ":id/services/:service_slug" do + service = user_project.find_or_initialize_service(params[:service_slug].underscore) + present service, with: Entities::ProjectService, include_passwords: current_user.is_admin? end end - resource :projects do - desc 'Trigger a slash command' do - detail 'Added in GitLab 8.13' + trigger_services.each do |service_slug, settings| + params do + requires :id, type: String, desc: 'The ID of a project' end - post ':id/services/:service_slug/trigger' do - project = Project.find_with_namespace(params[:id]) || Project.find_by(id: params[:id]) + resource :projects do + desc "Trigger a slash command for #{service_slug}" do + detail 'Added in GitLab 8.13' + end + params do + settings.each do |setting| + requires setting[:name], type: setting[:type], desc: setting[:desc] + end + end + post ":id/services/#{service_slug.underscore}/trigger" do + project = find_project(params[:id]) - # This is not accurate, but done to prevent leakage of the project names - not_found!('Service') unless project + # This is not accurate, but done to prevent leakage of the project names + not_found!('Service') unless project - service = project_service(project) + service = project.find_or_initialize_service(service_slug.underscore) - result = service.try(:active?) && service.try(:trigger, params) + result = service.try(:active?) && service.try(:trigger, params) - if result - status result[:status] || 200 - present result - else - not_found!('Service') + if result + status result[:status] || 200 + present result + else + not_found!('Service') + end end end end diff --git a/lib/api/session.rb b/lib/api/session.rb index d09400b81f5..002ffd1d154 100644 --- a/lib/api/session.rb +++ b/lib/api/session.rb @@ -1,7 +1,7 @@ module API class Session < Grape::API desc 'Login to get token' do - success Entities::UserLogin + success Entities::UserWithPrivateToken end params do optional :login, type: String, desc: 'The username' @@ -14,7 +14,7 @@ module API return unauthorized! unless user return render_api_error!('401 Unauthorized. You have 2FA enabled. Please use a personal access token to access the API', 401) if user.two_factor_enabled? - present user, with: Entities::UserLogin + present user, with: Entities::UserWithPrivateToken end end end diff --git a/lib/api/sidekiq_metrics.rb b/lib/api/sidekiq_metrics.rb index d3d6827dc54..11f2b40269a 100644 --- a/lib/api/sidekiq_metrics.rb +++ b/lib/api/sidekiq_metrics.rb @@ -39,50 +39,22 @@ module API end end - # Get Sidekiq Queue metrics - # - # Parameters: - # None - # - # Example: - # GET /sidekiq/queue_metrics - # + desc 'Get the Sidekiq queue metrics' get 'sidekiq/queue_metrics' do { queues: queue_metrics } end - # Get Sidekiq Process metrics - # - # Parameters: - # None - # - # Example: - # GET /sidekiq/process_metrics - # + desc 'Get the Sidekiq process metrics' get 'sidekiq/process_metrics' do { processes: process_metrics } end - # Get Sidekiq Job statistics - # - # Parameters: - # None - # - # Example: - # GET /sidekiq/job_stats - # + desc 'Get the Sidekiq job statistics' get 'sidekiq/job_stats' do { jobs: job_stats } end - # Get Sidekiq Compound metrics. Includes all previous metrics - # - # Parameters: - # None - # - # Example: - # GET /sidekiq/compound_metrics - # + desc 'Get the Sidekiq Compound metrics. Includes queue, process, and job statistics' get 'sidekiq/compound_metrics' do { queues: queue_metrics, processes: process_metrics, jobs: job_stats } end diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb new file mode 100644 index 00000000000..e096e636806 --- /dev/null +++ b/lib/api/snippets.rb @@ -0,0 +1,137 @@ +module API + # Snippets API + class Snippets < Grape::API + include PaginationParams + + before { authenticate! } + + resource :snippets do + helpers do + def snippets_for_current_user + SnippetsFinder.new.execute(current_user, filter: :by_user, user: current_user) + end + + def public_snippets + SnippetsFinder.new.execute(current_user, filter: :public) + end + end + + desc 'Get a snippets list for authenticated user' do + detail 'This feature was introduced in GitLab 8.15.' + success Entities::PersonalSnippet + end + params do + use :pagination + end + get do + present paginate(snippets_for_current_user), with: Entities::PersonalSnippet + end + + desc 'List all public snippets current_user has access to' do + detail 'This feature was introduced in GitLab 8.15.' + success Entities::PersonalSnippet + end + params do + use :pagination + end + get 'public' do + present paginate(public_snippets), with: Entities::PersonalSnippet + end + + desc 'Get a single snippet' do + detail 'This feature was introduced in GitLab 8.15.' + success Entities::PersonalSnippet + end + params do + requires :id, type: Integer, desc: 'The ID of a snippet' + end + get ':id' do + snippet = snippets_for_current_user.find(params[:id]) + present snippet, with: Entities::PersonalSnippet + end + + desc 'Create new snippet' do + detail 'This feature was introduced in GitLab 8.15.' + success Entities::PersonalSnippet + end + params do + requires :title, type: String, desc: 'The title of a snippet' + requires :file_name, type: String, desc: 'The name of a snippet file' + requires :content, type: String, desc: 'The content of a snippet' + optional :visibility_level, type: Integer, + values: Gitlab::VisibilityLevel.values, + default: Gitlab::VisibilityLevel::INTERNAL, + desc: 'The visibility level of the snippet' + end + post do + attrs = declared_params(include_missing: false) + snippet = CreateSnippetService.new(nil, current_user, attrs).execute + + if snippet.persisted? + present snippet, with: Entities::PersonalSnippet + else + render_validation_error!(snippet) + end + end + + desc 'Update an existing snippet' do + detail 'This feature was introduced in GitLab 8.15.' + success Entities::PersonalSnippet + end + params do + requires :id, type: Integer, desc: 'The ID of a snippet' + optional :title, type: String, desc: 'The title of a snippet' + optional :file_name, type: String, desc: 'The name of a snippet file' + optional :content, type: String, desc: 'The content of a snippet' + optional :visibility_level, type: Integer, + values: Gitlab::VisibilityLevel.values, + desc: 'The visibility level of the snippet' + at_least_one_of :title, :file_name, :content, :visibility_level + end + put ':id' do + snippet = snippets_for_current_user.find_by(id: params.delete(:id)) + return not_found!('Snippet') unless snippet + authorize! :update_personal_snippet, snippet + + attrs = declared_params(include_missing: false) + + UpdateSnippetService.new(nil, current_user, snippet, attrs).execute + if snippet.persisted? + present snippet, with: Entities::PersonalSnippet + else + render_validation_error!(snippet) + end + end + + desc 'Remove snippet' do + detail 'This feature was introduced in GitLab 8.15.' + success Entities::PersonalSnippet + end + params do + requires :id, type: Integer, desc: 'The ID of a snippet' + end + delete ':id' do + snippet = snippets_for_current_user.find_by(id: params.delete(:id)) + return not_found!('Snippet') unless snippet + authorize! :destroy_personal_snippet, snippet + snippet.destroy + no_content! + end + + desc 'Get a raw snippet' do + detail 'This feature was introduced in GitLab 8.15.' + end + params do + requires :id, type: Integer, desc: 'The ID of a snippet' + end + get ":id/raw" do + snippet = snippets_for_current_user.find_by(id: params.delete(:id)) + return not_found!('Snippet') unless snippet + + env['api.format'] = :txt + content_type 'text/plain' + present snippet.content + end + end + end +end diff --git a/lib/api/tags.rb b/lib/api/tags.rb index cd33f9a9903..5b345db3a41 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -1,7 +1,6 @@ module API # Git Tags API class Tags < Grape::API - before { authenticate! } before { authorize! :download_code, user_project } params do diff --git a/lib/api/todos.rb b/lib/api/todos.rb index 832b04a3bb1..ed8f48aa1e3 100644 --- a/lib/api/todos.rb +++ b/lib/api/todos.rb @@ -1,6 +1,7 @@ module API - # Todos API class Todos < Grape::API + include PaginationParams + before { authenticate! } ISSUABLE_TYPES = { @@ -44,10 +45,11 @@ module API desc 'Get a todo list' do success Entities::Todo end + params do + use :pagination + end get do - todos = find_todos - - present paginate(todos), with: Entities::Todo, current_user: current_user + present paginate(find_todos), with: Entities::Todo, current_user: current_user end desc 'Mark a todo as done' do diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb index 569598fbd2c..87a717ba751 100644 --- a/lib/api/triggers.rb +++ b/lib/api/triggers.rb @@ -1,5 +1,7 @@ module API class Triggers < Grape::API + include PaginationParams + params do requires :id, type: String, desc: 'The ID of a project' end @@ -13,7 +15,7 @@ module API optional :variables, type: Hash, desc: 'The list of variables to be injected into build' end post ":id/(ref/:ref/)trigger/builds" do - project = Project.find_with_namespace(params[:id]) || Project.find_by(id: params[:id]) + project = find_project(params[:id]) trigger = Ci::Trigger.find_by_token(params[:token].to_s) not_found! unless project && trigger unauthorized! unless trigger.project == project @@ -42,6 +44,9 @@ module API desc 'Get triggers list' do success Entities::Trigger end + params do + use :pagination + end get ':id/triggers' do authenticate! authorize! :admin_build, user_project diff --git a/lib/api/users.rb b/lib/api/users.rb index a73650dc361..0842c3874c5 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -1,7 +1,11 @@ module API - # Users API class Users < Grape::API - before { authenticate! } + include PaginationParams + + before do + allow_access_with_scope :read_user if request.get? + authenticate! + end resource :users, requirements: { uid: /[0-9]*/, id: /[0-9]*/ } do helpers do @@ -33,6 +37,7 @@ module API optional :active, type: Boolean, default: false, desc: 'Filters only active users' optional :external, type: Boolean, default: false, desc: 'Filters only external users' optional :blocked, type: Boolean, default: false, desc: 'Filters only blocked users' + use :pagination end get do unless can?(current_user, :read_users_list, nil) @@ -49,7 +54,7 @@ module API users = users.external if params[:external] && current_user.is_admin? end - entity = current_user.is_admin? ? Entities::UserFull : Entities::UserBasic + entity = current_user.is_admin? ? Entities::UserPublic : Entities::UserBasic present paginate(users), with: entity end @@ -64,7 +69,7 @@ module API not_found!('User') unless user if current_user && current_user.is_admin? - present user, with: Entities::UserFull + present user, with: Entities::UserPublic elsif can?(current_user, :read_user, user) present user, with: Entities::User else @@ -73,7 +78,7 @@ module API end desc 'Create a user. Available only for admins.' do - success Entities::UserFull + success Entities::UserPublic end params do requires :email, type: String, desc: 'The email of the user' @@ -97,7 +102,7 @@ module API end if user.save - present user, with: Entities::UserFull + present user, with: Entities::UserPublic else conflict!('Email has already been taken') if User. where(email: user.email). @@ -112,7 +117,7 @@ module API end desc 'Update a user. Available only for admins.' do - success Entities::UserFull + success Entities::UserPublic end params do requires :id, type: Integer, desc: 'The ID of the user' @@ -159,7 +164,7 @@ module API user_params.delete(:provider) if user.update_attributes(user_params) - present user, with: Entities::UserFull + present user, with: Entities::UserPublic else render_validation_error!(user) end @@ -330,6 +335,7 @@ module API end params do requires :id, type: Integer, desc: 'The ID of the user' + use :pagination end get ':id/events' do user = User.find_by(id: params[:id]) @@ -347,10 +353,10 @@ module API resource :user do desc 'Get the currently authenticated user' do - success Entities::UserFull + success Entities::UserPublic end get do - present current_user, with: Entities::UserFull + present current_user, with: sudo? ? Entities::UserWithPrivateToken : Entities::UserPublic end desc "Get the currently authenticated user's SSH keys" do diff --git a/lib/api/variables.rb b/lib/api/variables.rb index b9fb3c21dbb..f623b1dfe9f 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -1,6 +1,8 @@ module API # Projects variables API class Variables < Grape::API + include PaginationParams + before { authenticate! } before { authorize! :admin_build, user_project } @@ -13,8 +15,7 @@ module API success Entities::Variable end params do - optional :page, type: Integer, desc: 'The page number for pagination' - optional :per_page, type: Integer, desc: 'The value of items per page to show' + use :pagination end get ':id/variables' do variables = user_project.variables @@ -29,7 +30,7 @@ module API end get ':id/variables/:key' do key = params[:key] - variable = user_project.variables.find_by(key: key.to_s) + variable = user_project.variables.find_by(key: key) return not_found!('Variable') unless variable diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 0dfffaf0bc6..7e6537e3d9e 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -14,7 +14,7 @@ module Backup s[:gitlab_version] = Gitlab::VERSION s[:tar_version] = tar_version s[:skipped] = ENV["SKIP"] - tar_file = "#{s[:backup_created_at].to_i}_gitlab_backup.tar" + tar_file = s[:backup_created_at].strftime('%s_%Y_%m_%d') + '_gitlab_backup.tar' Dir.chdir(Gitlab.config.backup.path) do File.open("#{Gitlab.config.backup.path}/backup_information.yml", @@ -82,12 +82,17 @@ module Backup removed = 0 Dir.chdir(Gitlab.config.backup.path) do - file_list = Dir.glob('*_gitlab_backup.tar') - file_list.map! { |f| $1.to_i if f =~ /(\d+)_gitlab_backup.tar/ } - file_list.sort.each do |timestamp| + Dir.glob('*_gitlab_backup.tar').each do |file| + next unless file =~ /(\d+)(?:_\d{4}_\d{2}_\d{2})?_gitlab_backup\.tar/ + + timestamp = $1.to_i + if Time.at(timestamp) < (Time.now - keep_time) - if Kernel.system(*%W(rm #{timestamp}_gitlab_backup.tar)) + begin + FileUtils.rm(file) removed += 1 + rescue => e + $progress.puts "Deleting #{file} failed: #{e.message}".color(:red) end end end @@ -103,7 +108,7 @@ module Backup Dir.chdir(Gitlab.config.backup.path) # check for existing backups in the backup dir - file_list = Dir.glob("*_gitlab_backup.tar").each.map { |f| f.split(/_/).first.to_i } + file_list = Dir.glob("*_gitlab_backup.tar") puts "no backups found" if file_list.count == 0 if file_list.count > 1 && ENV["BACKUP"].nil? @@ -112,7 +117,7 @@ module Backup exit 1 end - tar_file = ENV["BACKUP"].nil? ? File.join("#{file_list.first}_gitlab_backup.tar") : File.join(ENV["BACKUP"] + "_gitlab_backup.tar") + tar_file = ENV["BACKUP"].nil? ? file_list.first : file_list.grep(ENV['BACKUP']).first unless File.exist?(tar_file) puts "The specified backup doesn't exist!" diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 3740d4fb4cd..fd74eeaebe7 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -33,7 +33,7 @@ module Banzai # Returns a String replaced with the return of the block. def self.references_in(text, pattern = object_class.reference_pattern) text.gsub(pattern) do |match| - yield match, $~[object_sym].to_i, $~[:project], $~ + yield match, $~[object_sym].to_i, $~[:project], $~[:namespace], $~ end end @@ -145,8 +145,9 @@ module Banzai # Returns a String with references replaced with links. All links # have `gfm` and `gfm-OBJECT_NAME` class names attached for styling. def object_link_filter(text, pattern, link_content: nil) - references_in(text, pattern) do |match, id, project_ref, matches| - project = project_from_ref_cached(project_ref) + references_in(text, pattern) do |match, id, project_ref, namespace_ref, matches| + project_path = full_project_path(namespace_ref, project_ref) + project = project_from_ref_cached(project_path) if project && object = find_object_cached(project, id) title = object_link_title(object) @@ -217,10 +218,9 @@ module Banzai nodes.each do |node| node.to_html.scan(regex) do - project = $~[:project] || current_project_path + project_path = full_project_path($~[:namespace], $~[:project]) symbol = $~[object_sym] - - refs[project] << symbol if object_class.reference_valid?(symbol) + refs[project_path] << symbol if object_class.reference_valid?(symbol) end end @@ -248,7 +248,7 @@ module Banzai end def projects_relation_for_paths(paths) - Project.where_paths_in(paths).includes(:namespace) + Project.where_full_path_in(paths).includes(:namespace) end # Returns projects for the given paths. @@ -272,8 +272,19 @@ module Banzai @current_project_path ||= project.path_with_namespace end + def current_project_namespace_path + @current_project_namespace_path ||= project.namespace.path + end + private + def full_project_path(namespace, project_ref) + return current_project_path unless project_ref + + namespace_ref = namespace || current_project_namespace_path + "#{namespace_ref}/#{project_ref}" + end + def project_refs_cache RequestStore[:banzai_project_refs] ||= {} end diff --git a/lib/banzai/filter/commit_range_reference_filter.rb b/lib/banzai/filter/commit_range_reference_filter.rb index 4358bf45549..eaacb9591b1 100644 --- a/lib/banzai/filter/commit_range_reference_filter.rb +++ b/lib/banzai/filter/commit_range_reference_filter.rb @@ -12,7 +12,7 @@ module Banzai def self.references_in(text, pattern = CommitRange.reference_pattern) text.gsub(pattern) do |match| - yield match, $~[:commit_range], $~[:project], $~ + yield match, $~[:commit_range], $~[:project], $~[:namespace], $~ end end diff --git a/lib/banzai/filter/commit_reference_filter.rb b/lib/banzai/filter/commit_reference_filter.rb index a26dd09c25a..69c06117eda 100644 --- a/lib/banzai/filter/commit_reference_filter.rb +++ b/lib/banzai/filter/commit_reference_filter.rb @@ -12,7 +12,7 @@ module Banzai def self.references_in(text, pattern = Commit.reference_pattern) text.gsub(pattern) do |match| - yield match, $~[:commit], $~[:project], $~ + yield match, $~[:commit], $~[:project], $~[:namespace], $~ end end diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb index 9f9a96cdc65..a605dea149e 100644 --- a/lib/banzai/filter/label_reference_filter.rb +++ b/lib/banzai/filter/label_reference_filter.rb @@ -14,16 +14,18 @@ module Banzai def self.references_in(text, pattern = Label.reference_pattern) unescape_html_entities(text).gsub(pattern) do |match| - yield match, $~[:label_id].to_i, $~[:label_name], $~[:project], $~ + yield match, $~[:label_id].to_i, $~[:label_name], $~[:project], $~[:namespace], $~ end end def references_in(text, pattern = Label.reference_pattern) unescape_html_entities(text).gsub(pattern) do |match| - label = find_label($~[:project], $~[:label_id], $~[:label_name]) + namespace, project = $~[:namespace], $~[:project] + project_path = full_project_path(namespace, project) + label = find_label(project_path, $~[:label_id], $~[:label_name]) if label - yield match, label.id, $~[:project], $~ + yield match, label.id, project, namespace, $~ else match end @@ -64,48 +66,12 @@ module Banzai end def object_link_text(object, matches) - if same_group?(object) && namespace_match?(matches) - render_same_project_label(object) - elsif same_project?(object) - render_same_project_label(object) - else - render_cross_project_label(object, matches) - end - end - - def same_group?(object) - object.is_a?(GroupLabel) && object.group == project.group - end - - def namespace_match?(matches) - matches[:project].blank? || matches[:project] == project.path_with_namespace - end - - def same_project?(object) - object.is_a?(ProjectLabel) && object.project == project - end - - def user - context[:current_user] || context[:author] - end - - def project - context[:project] - end - - def render_same_project_label(object) - LabelsHelper.render_colored_label(object) - end - - def render_cross_project_label(object, matches) - source_project = - if matches[:project] - Project.find_with_namespace(matches[:project]) - else - object.project - end + project_path = full_project_path(matches[:namespace], matches[:project]) + project_from_ref = project_from_ref_cached(project_path) + reference = project_from_ref.to_human_reference(project) + label_suffix = " <i>in #{reference}</i>" if reference.present? - LabelsHelper.render_colored_cross_project_label(object, source_project) + LabelsHelper.render_colored_label(object, label_suffix) end def unescape_html_entities(text) diff --git a/lib/banzai/filter/math_filter.rb b/lib/banzai/filter/math_filter.rb new file mode 100644 index 00000000000..cb037f89337 --- /dev/null +++ b/lib/banzai/filter/math_filter.rb @@ -0,0 +1,51 @@ +require 'uri' + +module Banzai + module Filter + # HTML filter that adds class="code math" and removes the dollar sign in $`2+2`$. + # + class MathFilter < HTML::Pipeline::Filter + # This picks out <code>...</code>. + INLINE_MATH = 'descendant-or-self::code'.freeze + + # Pick out a code block which is declared math + DISPLAY_MATH = "descendant-or-self::pre[contains(@class, 'math') and contains(@class, 'code')]".freeze + + # Attribute indicating inline or display math. + STYLE_ATTRIBUTE = 'data-math-style'.freeze + + # Class used for tagging elements that should be rendered + TAG_CLASS = 'js-render-math'.freeze + + INLINE_CLASSES = "code math #{TAG_CLASS}".freeze + + DOLLAR_SIGN = '$'.freeze + + def call + doc.xpath(INLINE_MATH).each do |code| + closing = code.next + opening = code.previous + + # We need a sibling before and after. + # They should end and start with $ respectively. + if closing && opening && + closing.content.first == DOLLAR_SIGN && + opening.content.last == DOLLAR_SIGN + + code[:class] = INLINE_CLASSES + code[STYLE_ATTRIBUTE] = 'inline' + closing.content = closing.content[1..-1] + opening.content = opening.content[0..-2] + end + end + + doc.xpath(DISPLAY_MATH).each do |el| + el[STYLE_ATTRIBUTE] = 'display' + el[:class] += " #{TAG_CLASS}" + end + + doc + end + end + end +end diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb index 58fff496d00..f12014e191f 100644 --- a/lib/banzai/filter/milestone_reference_filter.rb +++ b/lib/banzai/filter/milestone_reference_filter.rb @@ -19,18 +19,20 @@ module Banzai return super(text, pattern) if pattern != Milestone.reference_pattern text.gsub(pattern) do |match| - milestone = find_milestone($~[:project], $~[:milestone_iid], $~[:milestone_name]) + milestone = find_milestone($~[:project], $~[:namespace], $~[:milestone_iid], $~[:milestone_name]) if milestone - yield match, milestone.iid, $~[:project], $~ + yield match, milestone.iid, $~[:project], $~[:namespace], $~ else match end end end - def find_milestone(project_ref, milestone_id, milestone_name) - project = project_from_ref(project_ref) + def find_milestone(project_ref, namespace_ref, milestone_id, milestone_name) + project_path = full_project_path(namespace_ref, project_ref) + project = project_from_ref(project_path) + return unless project milestone_params = milestone_params(milestone_id, milestone_name) @@ -52,11 +54,13 @@ module Banzai end def object_link_text(object, matches) - if context[:project] == object.project - super + milestone_link = escape_once(super) + reference = object.project.to_reference(project) + + if reference.present? + "#{milestone_link} <i>in #{reference}</i>".html_safe else - "#{escape_once(super)} <i>in #{escape_once(object.project.path_with_namespace)}</i>". - html_safe + milestone_link end end diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb index f09d78be0ce..9e23c8f8c55 100644 --- a/lib/banzai/filter/relative_link_filter.rb +++ b/lib/banzai/filter/relative_link_filter.rb @@ -46,7 +46,7 @@ module Banzai end def rebuild_relative_uri(uri) - file_path = relative_file_path(uri.path) + file_path = relative_file_path(uri) uri.path = [ relative_url_root, @@ -59,8 +59,10 @@ module Banzai uri end - def relative_file_path(path) - nested_path = build_relative_path(path, context[:requested_path]) + def relative_file_path(uri) + path = Addressable::URI.unescape(uri.path) + request_path = Addressable::URI.unescape(context[:requested_path]) + nested_path = build_relative_path(path, request_path) file_exists?(nested_path) ? nested_path : path end @@ -108,11 +110,7 @@ module Banzai end def uri_type(path) - @uri_types[path] ||= begin - unescaped_path = Addressable::URI.unescape(path) - - current_commit.uri_type(unescaped_path) - end + @uri_types[path] ||= current_commit.uri_type(path) end def current_commit diff --git a/lib/banzai/filter/table_of_contents_filter.rb b/lib/banzai/filter/table_of_contents_filter.rb index a4eda6fdf76..8e7084f2543 100644 --- a/lib/banzai/filter/table_of_contents_filter.rb +++ b/lib/banzai/filter/table_of_contents_filter.rb @@ -35,9 +35,11 @@ module Banzai headers[id] += 1 if header_content = node.children.first + # namespace detection will be automatically handled via javascript (see issue #22781) + namespace = "user-content-" href = "#{id}#{uniq}" push_toc(href, text) - header_content.add_previous_sibling(anchor_tag(href)) + header_content.add_previous_sibling(anchor_tag("#{namespace}#{href}", href)) end end @@ -48,8 +50,8 @@ module Banzai private - def anchor_tag(href) - %Q{<a id="#{href}" class="anchor" href="##{href}" aria-hidden="true"></a>} + def anchor_tag(id, href) + %Q{<a id="#{id}" class="anchor" href="##{href}" aria-hidden="true"></a>} end def push_toc(href, text) diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index 5da2d0b008c..5a1f873496c 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -6,6 +6,7 @@ module Banzai Filter::SyntaxHighlightFilter, Filter::SanitizationFilter, + Filter::MathFilter, Filter::UploadLinkFilter, Filter::VideoLinkFilter, Filter::ImageLinkFilter, diff --git a/lib/bitbucket/client.rb b/lib/bitbucket/client.rb new file mode 100644 index 00000000000..f8ee7e0f9ae --- /dev/null +++ b/lib/bitbucket/client.rb @@ -0,0 +1,58 @@ +module Bitbucket + class Client + attr_reader :connection + + def initialize(options = {}) + @connection = Connection.new(options) + end + + def issues(repo) + path = "/repositories/#{repo}/issues" + get_collection(path, :issue) + end + + def issue_comments(repo, issue_id) + path = "/repositories/#{repo}/issues/#{issue_id}/comments" + get_collection(path, :comment) + end + + def pull_requests(repo) + path = "/repositories/#{repo}/pullrequests?state=ALL" + get_collection(path, :pull_request) + end + + def pull_request_comments(repo, pull_request) + path = "/repositories/#{repo}/pullrequests/#{pull_request}/comments" + get_collection(path, :pull_request_comment) + end + + def pull_request_diff(repo, pull_request) + path = "/repositories/#{repo}/pullrequests/#{pull_request}/diff" + connection.get(path) + end + + def repo(name) + parsed_response = connection.get("/repositories/#{name}") + Representation::Repo.new(parsed_response) + end + + def repos + path = "/repositories?role=member" + get_collection(path, :repo) + end + + def user + @user ||= begin + parsed_response = connection.get('/user') + Representation::User.new(parsed_response) + end + end + + private + + def get_collection(path, type) + paginator = Paginator.new(connection, path, type) + Collection.new(paginator) + end + end +end diff --git a/lib/bitbucket/collection.rb b/lib/bitbucket/collection.rb new file mode 100644 index 00000000000..3a9379ff680 --- /dev/null +++ b/lib/bitbucket/collection.rb @@ -0,0 +1,21 @@ +module Bitbucket + class Collection < Enumerator + def initialize(paginator) + super() do |yielder| + loop do + paginator.items.each { |item| yielder << item } + end + end + + lazy + end + + def method_missing(method, *args) + return super unless self.respond_to?(method) + + self.send(method, *args) do |item| + block_given? ? yield(item) : item + end + end + end +end diff --git a/lib/bitbucket/connection.rb b/lib/bitbucket/connection.rb new file mode 100644 index 00000000000..7e55cf4deab --- /dev/null +++ b/lib/bitbucket/connection.rb @@ -0,0 +1,69 @@ +module Bitbucket + class Connection + DEFAULT_API_VERSION = '2.0' + DEFAULT_BASE_URI = 'https://api.bitbucket.org/' + DEFAULT_QUERY = {} + + attr_reader :expires_at, :expires_in, :refresh_token, :token + + def initialize(options = {}) + @api_version = options.fetch(:api_version, DEFAULT_API_VERSION) + @base_uri = options.fetch(:base_uri, DEFAULT_BASE_URI) + @default_query = options.fetch(:query, DEFAULT_QUERY) + + @token = options[:token] + @expires_at = options[:expires_at] + @expires_in = options[:expires_in] + @refresh_token = options[:refresh_token] + end + + def get(path, extra_query = {}) + refresh! if expired? + + response = connection.get(build_url(path), params: @default_query.merge(extra_query)) + response.parsed + end + + def expired? + connection.expired? + end + + def refresh! + response = connection.refresh! + + @token = response.token + @expires_at = response.expires_at + @expires_in = response.expires_in + @refresh_token = response.refresh_token + @connection = nil + end + + private + + def client + @client ||= OAuth2::Client.new(provider.app_id, provider.app_secret, options) + end + + def connection + @connection ||= OAuth2::AccessToken.new(client, @token, refresh_token: @refresh_token, expires_at: @expires_at, expires_in: @expires_in) + end + + def build_url(path) + return path if path.starts_with?(root_url) + + "#{root_url}#{path}" + end + + def root_url + @root_url ||= "#{@base_uri}#{@api_version}" + end + + def provider + Gitlab::OAuth::Provider.config_for('bitbucket') + end + + def options + OmniAuth::Strategies::Bitbucket.default_options[:client_options].deep_symbolize_keys + end + end +end diff --git a/lib/bitbucket/error/unauthorized.rb b/lib/bitbucket/error/unauthorized.rb new file mode 100644 index 00000000000..5e2eb57bb0e --- /dev/null +++ b/lib/bitbucket/error/unauthorized.rb @@ -0,0 +1,6 @@ +module Bitbucket + module Error + class Unauthorized < StandardError + end + end +end diff --git a/lib/bitbucket/page.rb b/lib/bitbucket/page.rb new file mode 100644 index 00000000000..2b0a3fe7b1a --- /dev/null +++ b/lib/bitbucket/page.rb @@ -0,0 +1,34 @@ +module Bitbucket + class Page + attr_reader :attrs, :items + + def initialize(raw, type) + @attrs = parse_attrs(raw) + @items = parse_values(raw, representation_class(type)) + end + + def next? + attrs.fetch(:next, false) + end + + def next + attrs.fetch(:next) + end + + private + + def parse_attrs(raw) + raw.slice(*%w(size page pagelen next previous)).symbolize_keys + end + + def parse_values(raw, bitbucket_rep_class) + return [] unless raw['values'] && raw['values'].is_a?(Array) + + bitbucket_rep_class.decorate(raw['values']) + end + + def representation_class(type) + Bitbucket::Representation.const_get(type.to_s.camelize) + end + end +end diff --git a/lib/bitbucket/paginator.rb b/lib/bitbucket/paginator.rb new file mode 100644 index 00000000000..135d0d55674 --- /dev/null +++ b/lib/bitbucket/paginator.rb @@ -0,0 +1,36 @@ +module Bitbucket + class Paginator + PAGE_LENGTH = 50 # The minimum length is 10 and the maximum is 100. + + def initialize(connection, url, type) + @connection = connection + @type = type + @url = url + @page = nil + end + + def items + raise StopIteration unless has_next_page? + + @page = fetch_next_page + @page.items + end + + private + + attr_reader :connection, :page, :url, :type + + def has_next_page? + page.nil? || page.next? + end + + def next_url + page.nil? ? url : page.next + end + + def fetch_next_page + parsed_response = connection.get(next_url, pagelen: PAGE_LENGTH, sort: :created_on) + Page.new(parsed_response, type) + end + end +end diff --git a/lib/bitbucket/representation/base.rb b/lib/bitbucket/representation/base.rb new file mode 100644 index 00000000000..94adaacc9b5 --- /dev/null +++ b/lib/bitbucket/representation/base.rb @@ -0,0 +1,17 @@ +module Bitbucket + module Representation + class Base + def initialize(raw) + @raw = raw + end + + def self.decorate(entries) + entries.map { |entry| new(entry)} + end + + private + + attr_reader :raw + end + end +end diff --git a/lib/bitbucket/representation/comment.rb b/lib/bitbucket/representation/comment.rb new file mode 100644 index 00000000000..4937aa9728f --- /dev/null +++ b/lib/bitbucket/representation/comment.rb @@ -0,0 +1,27 @@ +module Bitbucket + module Representation + class Comment < Representation::Base + def author + user['username'] + end + + def note + raw.fetch('content', {}).fetch('raw', nil) + end + + def created_at + raw['created_on'] + end + + def updated_at + raw['updated_on'] || raw['created_on'] + end + + private + + def user + raw.fetch('user', {}) + end + end + end +end diff --git a/lib/bitbucket/representation/issue.rb b/lib/bitbucket/representation/issue.rb new file mode 100644 index 00000000000..054064395c3 --- /dev/null +++ b/lib/bitbucket/representation/issue.rb @@ -0,0 +1,53 @@ +module Bitbucket + module Representation + class Issue < Representation::Base + CLOSED_STATUS = %w(resolved invalid duplicate wontfix closed).freeze + + def iid + raw['id'] + end + + def kind + raw['kind'] + end + + def author + raw.fetch('reporter', {}).fetch('username', nil) + end + + def description + raw.fetch('content', {}).fetch('raw', nil) + end + + def state + closed? ? 'closed' : 'opened' + end + + def title + raw['title'] + end + + def milestone + raw['milestone']['name'] if raw['milestone'].present? + end + + def created_at + raw['created_on'] + end + + def updated_at + raw['edited_on'] + end + + def to_s + iid + end + + private + + def closed? + CLOSED_STATUS.include?(raw['state']) + end + end + end +end diff --git a/lib/bitbucket/representation/pull_request.rb b/lib/bitbucket/representation/pull_request.rb new file mode 100644 index 00000000000..eebf8093380 --- /dev/null +++ b/lib/bitbucket/representation/pull_request.rb @@ -0,0 +1,65 @@ +module Bitbucket + module Representation + class PullRequest < Representation::Base + def author + raw.fetch('author', {}).fetch('username', nil) + end + + def description + raw['description'] + end + + def iid + raw['id'] + end + + def state + if raw['state'] == 'MERGED' + 'merged' + elsif raw['state'] == 'DECLINED' + 'closed' + else + 'opened' + end + end + + def created_at + raw['created_on'] + end + + def updated_at + raw['updated_on'] + end + + def title + raw['title'] + end + + def source_branch_name + source_branch.fetch('branch', {}).fetch('name', nil) + end + + def source_branch_sha + source_branch.fetch('commit', {}).fetch('hash', nil) + end + + def target_branch_name + target_branch.fetch('branch', {}).fetch('name', nil) + end + + def target_branch_sha + target_branch.fetch('commit', {}).fetch('hash', nil) + end + + private + + def source_branch + raw['source'] + end + + def target_branch + raw['destination'] + end + end + end +end diff --git a/lib/bitbucket/representation/pull_request_comment.rb b/lib/bitbucket/representation/pull_request_comment.rb new file mode 100644 index 00000000000..4f8efe03bae --- /dev/null +++ b/lib/bitbucket/representation/pull_request_comment.rb @@ -0,0 +1,39 @@ +module Bitbucket + module Representation + class PullRequestComment < Comment + def iid + raw['id'] + end + + def file_path + inline.fetch('path') + end + + def old_pos + inline.fetch('from') + end + + def new_pos + inline.fetch('to') + end + + def parent_id + raw.fetch('parent', {}).fetch('id', nil) + end + + def inline? + raw.has_key?('inline') + end + + def has_parent? + raw.has_key?('parent') + end + + private + + def inline + raw.fetch('inline', {}) + end + end + end +end diff --git a/lib/bitbucket/representation/repo.rb b/lib/bitbucket/representation/repo.rb new file mode 100644 index 00000000000..8969ecd1c19 --- /dev/null +++ b/lib/bitbucket/representation/repo.rb @@ -0,0 +1,67 @@ +module Bitbucket + module Representation + class Repo < Representation::Base + attr_reader :owner, :slug + + def initialize(raw) + super(raw) + end + + def owner_and_slug + @owner_and_slug ||= full_name.split('/', 2) + end + + def owner + owner_and_slug.first + end + + def slug + owner_and_slug.last + end + + def clone_url(token = nil) + url = raw['links']['clone'].find { |link| link['name'] == 'https' }.fetch('href') + + if token.present? + clone_url = URI::parse(url) + clone_url.user = "x-token-auth:#{token}" + clone_url.to_s + else + url + end + end + + def description + raw['description'] + end + + def full_name + raw['full_name'] + end + + def issues_enabled? + raw['has_issues'] + end + + def name + raw['name'] + end + + def valid? + raw['scm'] == 'git' + end + + def visibility_level + if raw['is_private'] + Gitlab::VisibilityLevel::PRIVATE + else + Gitlab::VisibilityLevel::PUBLIC + end + end + + def to_s + full_name + end + end + end +end diff --git a/lib/bitbucket/representation/user.rb b/lib/bitbucket/representation/user.rb new file mode 100644 index 00000000000..ba6b7667b49 --- /dev/null +++ b/lib/bitbucket/representation/user.rb @@ -0,0 +1,9 @@ +module Bitbucket + module Representation + class User < Representation::Base + def username + raw['username'] + end + end + end +end diff --git a/lib/constraints/constrainer_helper.rb b/lib/constraints/constrainer_helper.rb deleted file mode 100644 index ab07a6793d9..00000000000 --- a/lib/constraints/constrainer_helper.rb +++ /dev/null @@ -1,15 +0,0 @@ -module ConstrainerHelper - def extract_resource_path(path) - id = path.dup - id.sub!(/\A#{relative_url_root}/, '') if relative_url_root - id.sub(/\A\/+/, '').sub(/\/+\z/, '').sub(/.atom\z/, '') - end - - private - - def relative_url_root - if defined?(Gitlab::Application.config.relative_url_root) - Gitlab::Application.config.relative_url_root - end - end -end diff --git a/lib/constraints/group_url_constrainer.rb b/lib/constraints/group_url_constrainer.rb index 2af6e1a11c8..bae4db1ca4d 100644 --- a/lib/constraints/group_url_constrainer.rb +++ b/lib/constraints/group_url_constrainer.rb @@ -1,15 +1,17 @@ -require_relative 'constrainer_helper' - class GroupUrlConstrainer - include ConstrainerHelper - def matches?(request) - id = extract_resource_path(request.path) + id = request.params[:id] + + return false unless valid?(id) + + Group.find_by_full_path(id).present? + end + + private - if id =~ Gitlab::Regex.namespace_regex - Group.find_by(path: id).present? - else - false + def valid?(id) + id.split('/').all? do |namespace| + NamespaceValidator.valid?(namespace) end end end diff --git a/lib/constraints/project_url_constrainer.rb b/lib/constraints/project_url_constrainer.rb new file mode 100644 index 00000000000..730b05bed97 --- /dev/null +++ b/lib/constraints/project_url_constrainer.rb @@ -0,0 +1,13 @@ +class ProjectUrlConstrainer + def matches?(request) + namespace_path = request.params[:namespace_id] + project_path = request.params[:project_id] || request.params[:id] + full_path = namespace_path + '/' + project_path + + unless ProjectPathValidator.valid?(project_path) + return false + end + + Project.find_with_namespace(full_path).present? + end +end diff --git a/lib/constraints/user_url_constrainer.rb b/lib/constraints/user_url_constrainer.rb index 4d722ad5af2..9ab5bcb12ff 100644 --- a/lib/constraints/user_url_constrainer.rb +++ b/lib/constraints/user_url_constrainer.rb @@ -1,15 +1,5 @@ -require_relative 'constrainer_helper' - class UserUrlConstrainer - include ConstrainerHelper - def matches?(request) - id = extract_resource_path(request.path) - - if id =~ Gitlab::Regex.namespace_regex - User.find_by('lower(username) = ?', id.downcase).present? - else - false - end + User.find_by_username(request.params[:username]).present? end end diff --git a/lib/email_template_interceptor.rb b/lib/email_template_interceptor.rb new file mode 100644 index 00000000000..fb04a7824b8 --- /dev/null +++ b/lib/email_template_interceptor.rb @@ -0,0 +1,13 @@ +# Read about interceptors in http://guides.rubyonrails.org/action_mailer_basics.html#intercepting-emails +class EmailTemplateInterceptor + include Gitlab::CurrentSettings + + def self.delivering_email(message) + # Remove HTML part if HTML emails are disabled. + unless current_application_settings.html_emails_enabled + message.part.delete_if do |part| + part.content_type.try(:start_with?, 'text/html') + end + end + end +end diff --git a/lib/event_filter.rb b/lib/event_filter.rb index 21f6a9a762b..515095af1c2 100644 --- a/lib/event_filter.rb +++ b/lib/event_filter.rb @@ -14,6 +14,10 @@ class EventFilter 'merged' end + def issue + 'issue' + end + def comments 'comments' end @@ -32,32 +36,20 @@ class EventFilter end def apply_filter(events) - return events unless params.present? - - filter = params.dup - actions = [] + return events if params.blank? || params == EventFilter.all - case filter + case params when EventFilter.push - actions = [Event::PUSHED] + events.where(action: Event::PUSHED) when EventFilter.merged - actions = [Event::MERGED] + events.where(action: Event::MERGED) when EventFilter.comments - actions = [Event::COMMENTED] + events.where(action: Event::COMMENTED) when EventFilter.team - actions = [Event::JOINED, Event::LEFT, Event::EXPIRED] - when EventFilter.all - actions = [ - Event::PUSHED, - Event::MERGED, - Event::COMMENTED, - Event::JOINED, - Event::LEFT, - Event::EXPIRED - ] + events.where(action: [Event::JOINED, Event::LEFT, Event::EXPIRED]) + when EventFilter.issue + events.where(action: [Event::CREATED, Event::UPDATED, Event::CLOSED, Event::REOPENED]) end - - events.where(action: actions) end def options(key) @@ -73,6 +65,10 @@ class EventFilter end def active?(key) - params.include? key + if params.present? + params.include? key + else + key == EventFilter.all + end end end diff --git a/lib/gitlab/allowable.rb b/lib/gitlab/allowable.rb new file mode 100644 index 00000000000..f48abcc86d5 --- /dev/null +++ b/lib/gitlab/allowable.rb @@ -0,0 +1,7 @@ +module Gitlab + module Allowable + def can?(user, action, subject) + Ability.allowed?(user, action, subject) + end + end +end diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb index 1a22ad9acf5..fa234284361 100644 --- a/lib/gitlab/asciidoc.rb +++ b/lib/gitlab/asciidoc.rb @@ -1,4 +1,5 @@ require 'asciidoctor' +require 'asciidoctor/converter/html5' module Gitlab # Parser/renderer for the AsciiDoc format that uses Asciidoctor and filters @@ -6,7 +7,7 @@ module Gitlab module Asciidoc DEFAULT_ADOC_ATTRS = [ 'showtitle', 'idprefix=user-content-', 'idseparator=-', 'env=gitlab', - 'env-gitlab', 'source-highlighter=html-pipeline' + 'env-gitlab', 'source-highlighter=html-pipeline', 'icons=font' ].freeze # Public: Converts the provided Asciidoc markup into HTML. @@ -23,7 +24,7 @@ module Gitlab def self.render(input, context, asciidoc_opts = {}) asciidoc_opts.reverse_merge!( safe: :secure, - backend: :html5, + backend: :gitlab_html5, attributes: [] ) asciidoc_opts[:attributes].unshift(*DEFAULT_ADOC_ATTRS) @@ -34,5 +35,29 @@ module Gitlab html.html_safe end + + class Html5Converter < Asciidoctor::Converter::Html5Converter + extend Asciidoctor::Converter::Config + + register_for 'gitlab_html5' + + def stem(node) + return super unless node.style.to_sym == :latexmath + + %(<pre#{id_attribute(node)} class="code math js-render-math #{node.role}" data-math-style="display"><code>#{node.content}</code></pre>) + end + + def inline_quoted(node) + return super unless node.type.to_sym == :latexmath + + %(<code#{id_attribute(node)} class="code math js-render-math #{node.role}" data-math-style="inline">#{node.text}</code>) + end + + private + + def id_attribute(node) + node.id ? %( id="#{node.id}") : nil + end + end end end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index aca5d0020cf..8dda65c71ef 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -2,6 +2,10 @@ module Gitlab module Auth class MissingPersonalTokenError < StandardError; end + SCOPES = [:api, :read_user] + DEFAULT_SCOPES = [:api] + OPTIONAL_SCOPES = SCOPES - DEFAULT_SCOPES + class << self def find_for_git_client(login, password, project:, ip:) raise "Must provide an IP for rate limiting" if ip.nil? @@ -88,7 +92,7 @@ module Gitlab def oauth_access_token_check(login, password) if login == "oauth2" && password.present? token = Doorkeeper::AccessToken.by_token(password) - if token && token.accessible? + if valid_oauth_token?(token) user = User.find_by(id: token.resource_owner_id) Gitlab::Auth::Result.new(user, nil, :oauth, read_authentication_abilities) end @@ -97,12 +101,27 @@ module Gitlab def personal_access_token_check(login, password) if login && password - user = User.find_by_personal_access_token(password) + token = PersonalAccessToken.active.find_by_token(password) validation = User.by_login(login) - Gitlab::Auth::Result.new(user, nil, :personal_token, full_authentication_abilities) if user.present? && user == validation + + if valid_personal_access_token?(token, validation) + Gitlab::Auth::Result.new(validation, nil, :personal_token, full_authentication_abilities) + end end end + def valid_oauth_token?(token) + token && token.accessible? && valid_api_token?(token) + end + + def valid_personal_access_token?(token, user) + token && token.user == user && valid_api_token?(token) + end + + def valid_api_token?(token) + AccessTokenValidationService.new(token).include_any_scope?(['api']) + end + def lfs_token_check(login, password) deploy_key_matches = login.match(/\Alfs\+deploy-key-(\d+)\z/) diff --git a/lib/gitlab/badge/build/status.rb b/lib/gitlab/badge/build/status.rb index 50aa45e5406..b762d85b6e5 100644 --- a/lib/gitlab/badge/build/status.rb +++ b/lib/gitlab/badge/build/status.rb @@ -20,8 +20,8 @@ module Gitlab def status @project.pipelines - .where(sha: @sha, ref: @ref) - .status || 'unknown' + .where(sha: @sha) + .latest_status(@ref) || 'unknown' end def metadata diff --git a/lib/gitlab/bitbucket_import.rb b/lib/gitlab/bitbucket_import.rb deleted file mode 100644 index 7298152e7e9..00000000000 --- a/lib/gitlab/bitbucket_import.rb +++ /dev/null @@ -1,6 +0,0 @@ -module Gitlab - module BitbucketImport - mattr_accessor :public_key - @public_key = nil - end -end diff --git a/lib/gitlab/bitbucket_import/client.rb b/lib/gitlab/bitbucket_import/client.rb deleted file mode 100644 index 8d1ad62fae0..00000000000 --- a/lib/gitlab/bitbucket_import/client.rb +++ /dev/null @@ -1,142 +0,0 @@ -module Gitlab - module BitbucketImport - class Client - class Unauthorized < StandardError; end - - attr_reader :consumer, :api - - def self.from_project(project) - import_data_credentials = project.import_data.credentials if project.import_data - if import_data_credentials && import_data_credentials[:bb_session] - token = import_data_credentials[:bb_session][:bitbucket_access_token] - token_secret = import_data_credentials[:bb_session][:bitbucket_access_token_secret] - new(token, token_secret) - else - raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{project.id}" - end - end - - def initialize(access_token = nil, access_token_secret = nil) - @consumer = ::OAuth::Consumer.new( - config.app_id, - config.app_secret, - bitbucket_options - ) - - if access_token && access_token_secret - @api = ::OAuth::AccessToken.new(@consumer, access_token, access_token_secret) - end - end - - def request_token(redirect_uri) - request_token = consumer.get_request_token(oauth_callback: redirect_uri) - - { - oauth_token: request_token.token, - oauth_token_secret: request_token.secret, - oauth_callback_confirmed: request_token.callback_confirmed?.to_s - } - end - - def authorize_url(request_token, redirect_uri) - request_token = ::OAuth::RequestToken.from_hash(consumer, request_token) if request_token.is_a?(Hash) - - if request_token.callback_confirmed? - request_token.authorize_url - else - request_token.authorize_url(oauth_callback: redirect_uri) - end - end - - def get_token(request_token, oauth_verifier, redirect_uri) - request_token = ::OAuth::RequestToken.from_hash(consumer, request_token) if request_token.is_a?(Hash) - - if request_token.callback_confirmed? - request_token.get_access_token(oauth_verifier: oauth_verifier) - else - request_token.get_access_token(oauth_callback: redirect_uri) - end - end - - def user - JSON.parse(get("/api/1.0/user").body) - end - - def issues(project_identifier) - all_issues = [] - offset = 0 - per_page = 50 # Maximum number allowed by Bitbucket - index = 0 - - begin - issues = JSON.parse(get(issue_api_endpoint(project_identifier, per_page, offset)).body) - # Find out how many total issues are present - total = issues["count"] if index == 0 - all_issues.concat(issues["issues"]) - offset += issues["issues"].count - index += 1 - end while all_issues.count < total - - all_issues - end - - def issue_comments(project_identifier, issue_id) - comments = JSON.parse(get("/api/1.0/repositories/#{project_identifier}/issues/#{issue_id}/comments").body) - comments.sort_by { |comment| comment["utc_created_on"] } - end - - def project(project_identifier) - JSON.parse(get("/api/1.0/repositories/#{project_identifier}").body) - end - - def find_deploy_key(project_identifier, key) - JSON.parse(get("/api/1.0/repositories/#{project_identifier}/deploy-keys").body).find do |deploy_key| - deploy_key["key"].chomp == key.chomp - end - end - - def add_deploy_key(project_identifier, key) - deploy_key = find_deploy_key(project_identifier, key) - return if deploy_key - - JSON.parse(api.post("/api/1.0/repositories/#{project_identifier}/deploy-keys", key: key, label: "GitLab import key").body) - end - - def delete_deploy_key(project_identifier, key) - deploy_key = find_deploy_key(project_identifier, key) - return unless deploy_key - - api.delete("/api/1.0/repositories/#{project_identifier}/deploy-keys/#{deploy_key["pk"]}").code == "204" - end - - def projects - JSON.parse(get("/api/1.0/user/repositories").body).select { |repo| repo["scm"] == "git" } - end - - def incompatible_projects - JSON.parse(get("/api/1.0/user/repositories").body).reject { |repo| repo["scm"] == "git" } - end - - private - - def get(url) - response = api.get(url) - raise Unauthorized if (400..499).cover?(response.code.to_i) - - response - end - - def issue_api_endpoint(project_identifier, per_page, offset) - "/api/1.0/repositories/#{project_identifier}/issues?sort=utc_created_on&limit=#{per_page}&start=#{offset}" - end - - def config - Gitlab.config.omniauth.providers.find { |provider| provider.name == "bitbucket" } - end - - def bitbucket_options - OmniAuth::Strategies::Bitbucket.default_options[:client_options].symbolize_keys - end - end - end -end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index f4b5097adb1..7d2f92d577a 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -1,84 +1,234 @@ module Gitlab module BitbucketImport class Importer - attr_reader :project, :client + LABELS = [{ title: 'bug', color: '#FF0000' }, + { title: 'enhancement', color: '#428BCA' }, + { title: 'proposal', color: '#69D100' }, + { title: 'task', color: '#7F8C8D' }].freeze + + attr_reader :project, :client, :errors, :users def initialize(project) @project = project - @client = Client.from_project(@project) + @client = Bitbucket::Client.new(project.import_data.credentials) @formatter = Gitlab::ImportFormatter.new + @labels = {} + @errors = [] + @users = {} end def execute - import_issues if has_issues? + import_issues + import_pull_requests + handle_errors true - rescue ActiveRecord::RecordInvalid => e - raise Projects::ImportService::Error.new, e.message - ensure - Gitlab::BitbucketImport::KeyDeleter.new(project).execute end private - def gitlab_user_id(project, bitbucket_id) - if bitbucket_id - user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", bitbucket_id.to_s) - (user && user.id) || project.creator_id - else - project.creator_id - end + def handle_errors + return unless errors.any? + + project.update_column(:import_error, { + message: 'The remote data could not be fully imported.', + errors: errors + }.to_json) end - def identifier - project.import_source + def gitlab_user_id(project, username) + find_user_id(username) || project.creator_id end - def has_issues? - client.project(identifier)["has_issues"] + def find_user_id(username) + return nil unless username + + return users[username] if users.key?(username) + + users[username] = User.select(:id) + .joins(:identities) + .find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", username) + .try(:id) + end + + def repo + @repo ||= client.repo(project.import_source) end def import_issues - issues = client.issues(identifier) + return unless repo.issues_enabled? + + create_labels + + client.issues(repo).each do |issue| + begin + description = '' + description += @formatter.author_line(issue.author) unless find_user_id(issue.author) + description += issue.description + + label_name = issue.kind + milestone = issue.milestone ? project.milestones.find_or_create_by(title: issue.milestone) : nil + + gitlab_issue = project.issues.create!( + iid: issue.iid, + title: issue.title, + description: description, + state: issue.state, + author_id: gitlab_user_id(project, issue.author), + milestone: milestone, + created_at: issue.created_at, + updated_at: issue.updated_at + ) + + gitlab_issue.labels << @labels[label_name] + + import_issue_comments(issue, gitlab_issue) if gitlab_issue.persisted? + rescue StandardError => e + errors << { type: :issue, iid: issue.iid, errors: e.message } + end + end + end + + def import_issue_comments(issue, gitlab_issue) + client.issue_comments(repo, issue.iid).each do |comment| + # The note can be blank for issue service messages like "Changed title: ..." + # We would like to import those comments as well but there is no any + # specific parameter that would allow to process them, it's just an empty comment. + # To prevent our importer from just crashing or from creating useless empty comments + # we do this check. + next unless comment.note.present? + + note = '' + note += @formatter.author_line(comment.author) unless find_user_id(comment.author) + note += comment.note + + begin + gitlab_issue.notes.create!( + project: project, + note: note, + author_id: gitlab_user_id(project, comment.author), + created_at: comment.created_at, + updated_at: comment.updated_at + ) + rescue StandardError => e + errors << { type: :issue_comment, iid: issue.iid, errors: e.message } + end + end + end - issues.each do |issue| - body = '' - reporter = nil - author = 'Anonymous' + def create_labels + LABELS.each do |label| + @labels[label[:title]] = project.labels.create!(label) + end + end - if issue["reported_by"] && issue["reported_by"]["username"] - reporter = issue["reported_by"]["username"] - author = reporter + def import_pull_requests + pull_requests = client.pull_requests(repo) + + pull_requests.each do |pull_request| + begin + description = '' + description += @formatter.author_line(pull_request.author) unless find_user_id(pull_request.author) + description += pull_request.description + + merge_request = project.merge_requests.create( + iid: pull_request.iid, + title: pull_request.title, + description: description, + source_project: project, + source_branch: pull_request.source_branch_name, + source_branch_sha: pull_request.source_branch_sha, + target_project: project, + target_branch: pull_request.target_branch_name, + target_branch_sha: pull_request.target_branch_sha, + state: pull_request.state, + author_id: gitlab_user_id(project, pull_request.author), + assignee_id: nil, + created_at: pull_request.created_at, + updated_at: pull_request.updated_at + ) + + import_pull_request_comments(pull_request, merge_request) if merge_request.persisted? + rescue StandardError => e + errors << { type: :pull_request, iid: pull_request.iid, errors: e.message } end + end + end - body = @formatter.author_line(author) - body += issue["content"] + def import_pull_request_comments(pull_request, merge_request) + comments = client.pull_request_comments(repo, pull_request.iid) - comments = client.issue_comments(identifier, issue["local_id"]) + inline_comments, pr_comments = comments.partition(&:inline?) - if comments.any? - body += @formatter.comments_header - end + import_inline_comments(inline_comments, pull_request, merge_request) + import_standalone_pr_comments(pr_comments, merge_request) + end - comments.each do |comment| - author = 'Anonymous' + def import_inline_comments(inline_comments, pull_request, merge_request) + line_code_map = {} - if comment["author_info"] && comment["author_info"]["username"] - author = comment["author_info"]["username"] - end + children, parents = inline_comments.partition(&:has_parent?) + + # The Bitbucket API returns threaded replies as parent-child + # relationships. We assume that the child can appear in any order in + # the JSON. + parents.each do |comment| + line_code_map[comment.iid] = generate_line_code(comment) + end - body += @formatter.comment(author, comment["utc_created_on"], comment["content"]) + children.each do |comment| + line_code_map[comment.iid] = line_code_map.fetch(comment.parent_id, nil) + end + + inline_comments.each do |comment| + begin + attributes = pull_request_comment_attributes(comment) + attributes.merge!( + position: build_position(merge_request, comment), + line_code: line_code_map.fetch(comment.iid), + type: 'DiffNote') + + merge_request.notes.create!(attributes) + rescue StandardError => e + errors << { type: :pull_request, iid: comment.iid, errors: e.message } end + end + end - project.issues.create!( - description: body, - title: issue["title"], - state: %w(resolved invalid duplicate wontfix closed).include?(issue["status"]) ? 'closed' : 'opened', - author_id: gitlab_user_id(project, reporter) - ) + def build_position(merge_request, pr_comment) + params = { + diff_refs: merge_request.diff_refs, + old_path: pr_comment.file_path, + new_path: pr_comment.file_path, + old_line: pr_comment.old_pos, + new_line: pr_comment.new_pos + } + + Gitlab::Diff::Position.new(params) + end + + def import_standalone_pr_comments(pr_comments, merge_request) + pr_comments.each do |comment| + begin + merge_request.notes.create!(pull_request_comment_attributes(comment)) + rescue StandardError => e + errors << { type: :pull_request, iid: comment.iid, errors: e.message } + end end - rescue ActiveRecord::RecordInvalid => e - raise Projects::ImportService::Error, e.message + end + + def generate_line_code(pr_comment) + Gitlab::Diff::LineCode.generate(pr_comment.file_path, pr_comment.new_pos, pr_comment.old_pos) + end + + def pull_request_comment_attributes(comment) + { + project: project, + note: comment.note, + author_id: gitlab_user_id(project, comment.author), + created_at: comment.created_at, + updated_at: comment.updated_at + } end end end diff --git a/lib/gitlab/bitbucket_import/key_adder.rb b/lib/gitlab/bitbucket_import/key_adder.rb deleted file mode 100644 index 0b63f025d0a..00000000000 --- a/lib/gitlab/bitbucket_import/key_adder.rb +++ /dev/null @@ -1,24 +0,0 @@ -module Gitlab - module BitbucketImport - class KeyAdder - attr_reader :repo, :current_user, :client - - def initialize(repo, current_user, access_params) - @repo, @current_user = repo, current_user - @client = Client.new(access_params[:bitbucket_access_token], - access_params[:bitbucket_access_token_secret]) - end - - def execute - return false unless BitbucketImport.public_key.present? - - project_identifier = "#{repo["owner"]}/#{repo["slug"]}" - client.add_deploy_key(project_identifier, BitbucketImport.public_key) - - true - rescue - false - end - end - end -end diff --git a/lib/gitlab/bitbucket_import/key_deleter.rb b/lib/gitlab/bitbucket_import/key_deleter.rb deleted file mode 100644 index e03c3155b3e..00000000000 --- a/lib/gitlab/bitbucket_import/key_deleter.rb +++ /dev/null @@ -1,23 +0,0 @@ -module Gitlab - module BitbucketImport - class KeyDeleter - attr_reader :project, :current_user, :client - - def initialize(project) - @project = project - @current_user = project.creator - @client = Client.from_project(@project) - end - - def execute - return false unless BitbucketImport.public_key.present? - - client.delete_deploy_key(project.import_source, BitbucketImport.public_key) - - true - rescue - false - end - end - end -end diff --git a/lib/gitlab/bitbucket_import/project_creator.rb b/lib/gitlab/bitbucket_import/project_creator.rb index b90ef0b0fba..eb03882ab26 100644 --- a/lib/gitlab/bitbucket_import/project_creator.rb +++ b/lib/gitlab/bitbucket_import/project_creator.rb @@ -1,10 +1,11 @@ module Gitlab module BitbucketImport class ProjectCreator - attr_reader :repo, :namespace, :current_user, :session_data + attr_reader :repo, :name, :namespace, :current_user, :session_data - def initialize(repo, namespace, current_user, session_data) + def initialize(repo, name, namespace, current_user, session_data) @repo = repo + @name = name @namespace = namespace @current_user = current_user @session_data = session_data @@ -13,15 +14,15 @@ module Gitlab def execute ::Projects::CreateService.new( current_user, - name: repo["name"], - path: repo["slug"], - description: repo["description"], + name: name, + path: name, + description: repo.description, namespace_id: namespace.id, - visibility_level: repo["is_private"] ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::VisibilityLevel::PUBLIC, - import_type: "bitbucket", - import_source: "#{repo["owner"]}/#{repo["slug"]}", - import_url: "ssh://git@bitbucket.org/#{repo["owner"]}/#{repo["slug"]}.git", - import_data: { credentials: { bb_session: session_data } } + visibility_level: repo.visibility_level, + import_type: 'bitbucket', + import_source: repo.full_name, + import_url: repo.clone_url(session_data[:token]), + import_data: { credentials: session_data } ).execute end end diff --git a/lib/gitlab/chat_commands/base_command.rb b/lib/gitlab/chat_commands/base_command.rb index e59d69b72b9..25da8474e95 100644 --- a/lib/gitlab/chat_commands/base_command.rb +++ b/lib/gitlab/chat_commands/base_command.rb @@ -40,9 +40,7 @@ module Gitlab private def find_by_iid(iid) - resource = collection.find_by(iid: iid) - - readable?(resource) ? resource : nil + collection.find_by(iid: iid) end end end diff --git a/lib/gitlab/chat_commands/command.rb b/lib/gitlab/chat_commands/command.rb index 0ec358debc7..b0d3fdbc48a 100644 --- a/lib/gitlab/chat_commands/command.rb +++ b/lib/gitlab/chat_commands/command.rb @@ -4,6 +4,7 @@ module Gitlab COMMANDS = [ Gitlab::ChatCommands::IssueShow, Gitlab::ChatCommands::IssueCreate, + Gitlab::ChatCommands::IssueSearch, Gitlab::ChatCommands::Deploy, ].freeze diff --git a/lib/gitlab/chat_commands/issue_command.rb b/lib/gitlab/chat_commands/issue_command.rb index f1bc36239d5..84de3e44c70 100644 --- a/lib/gitlab/chat_commands/issue_command.rb +++ b/lib/gitlab/chat_commands/issue_command.rb @@ -6,11 +6,7 @@ module Gitlab end def collection - project.issues - end - - def readable?(issue) - self.class.can?(current_user, :read_issue, issue) + IssuesFinder.new(current_user, project_id: project.id).execute end end end diff --git a/lib/gitlab/chat_commands/issue_create.rb b/lib/gitlab/chat_commands/issue_create.rb index 98338ebfa27..cefb6775db8 100644 --- a/lib/gitlab/chat_commands/issue_create.rb +++ b/lib/gitlab/chat_commands/issue_create.rb @@ -2,11 +2,13 @@ module Gitlab module ChatCommands class IssueCreate < IssueCommand def self.match(text) - /\Aissue\s+create\s+(?<title>[^\n]*)\n*(?<description>.*)\z/.match(text) + # we can not match \n with the dot by passing the m modifier as than + # the title and description are not seperated + /\Aissue\s+(new|create)\s+(?<title>[^\n]*)\n*(?<description>(.|\n)*)/.match(text) end def self.help_message - 'issue create <title>\n<description>' + 'issue new <title> *`⇧ Shift`*+*`↵ Enter`* <description>' end def self.allowed?(project, user) @@ -15,7 +17,7 @@ module Gitlab def execute(match) title = match[:title] - description = match[:description] + description = match[:description].to_s.rstrip Issues::CreateService.new(project, current_user, title: title, description: description).execute end diff --git a/lib/gitlab/chat_commands/issue_search.rb b/lib/gitlab/chat_commands/issue_search.rb new file mode 100644 index 00000000000..51bf80c800b --- /dev/null +++ b/lib/gitlab/chat_commands/issue_search.rb @@ -0,0 +1,17 @@ +module Gitlab + module ChatCommands + class IssueSearch < IssueCommand + def self.match(text) + /\Aissue\s+search\s+(?<query>.*)/.match(text) + end + + def self.help_message + "issue search <your query>" + end + + def execute(match) + collection.search(match[:query]).limit(QUERY_LIMIT) + end + end + end +end diff --git a/lib/gitlab/chat_commands/issue_show.rb b/lib/gitlab/chat_commands/issue_show.rb index f5bceb038e5..2a45d49cf6b 100644 --- a/lib/gitlab/chat_commands/issue_show.rb +++ b/lib/gitlab/chat_commands/issue_show.rb @@ -2,7 +2,7 @@ module Gitlab module ChatCommands class IssueShow < IssueCommand def self.match(text) - /\Aissue\s+show\s+(?<iid>\d+)/.match(text) + /\Aissue\s+show\s+#{Issue.reference_prefix}?(?<iid>\d+)/.match(text) end def self.help_message diff --git a/lib/gitlab/ci/status/build/cancelable.rb b/lib/gitlab/ci/status/build/cancelable.rb new file mode 100644 index 00000000000..a979fe7d573 --- /dev/null +++ b/lib/gitlab/ci/status/build/cancelable.rb @@ -0,0 +1,37 @@ +module Gitlab + module Ci + module Status + module Build + class Cancelable < SimpleDelegator + include Status::Extended + + def has_action? + can?(user, :update_build, subject) + end + + def action_icon + 'ban' + end + + def action_path + cancel_namespace_project_build_path(subject.project.namespace, + subject.project, + subject) + end + + def action_method + :post + end + + def action_title + 'Cancel' + end + + def self.matches?(build, user) + build.cancelable? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/common.rb b/lib/gitlab/ci/status/build/common.rb new file mode 100644 index 00000000000..3fec2c5d4db --- /dev/null +++ b/lib/gitlab/ci/status/build/common.rb @@ -0,0 +1,19 @@ +module Gitlab + module Ci + module Status + module Build + module Common + def has_details? + can?(user, :read_build, subject) + end + + def details_path + namespace_project_build_path(subject.project.namespace, + subject.project, + subject) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/factory.rb b/lib/gitlab/ci/status/build/factory.rb new file mode 100644 index 00000000000..eee9a64120b --- /dev/null +++ b/lib/gitlab/ci/status/build/factory.rb @@ -0,0 +1,18 @@ +module Gitlab + module Ci + module Status + module Build + class Factory < Status::Factory + def self.extended_statuses + [Status::Build::Stop, Status::Build::Play, + Status::Build::Cancelable, Status::Build::Retryable] + end + + def self.common_helpers + Status::Build::Common + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/play.rb b/lib/gitlab/ci/status/build/play.rb new file mode 100644 index 00000000000..5c506e6d59f --- /dev/null +++ b/lib/gitlab/ci/status/build/play.rb @@ -0,0 +1,53 @@ +module Gitlab + module Ci + module Status + module Build + class Play < SimpleDelegator + include Status::Extended + + def text + 'manual' + end + + def label + 'manual play action' + end + + def icon + 'icon_status_manual' + end + + def has_action? + can?(user, :update_build, subject) + end + + def action_icon + 'play' + end + + def action_title + 'Play' + end + + def action_class + 'ci-play-icon' + end + + def action_path + play_namespace_project_build_path(subject.project.namespace, + subject.project, + subject) + end + + def action_method + :post + end + + def self.matches?(build, user) + build.playable? && !build.stops_environment? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/retryable.rb b/lib/gitlab/ci/status/build/retryable.rb new file mode 100644 index 00000000000..8e38d6a8523 --- /dev/null +++ b/lib/gitlab/ci/status/build/retryable.rb @@ -0,0 +1,37 @@ +module Gitlab + module Ci + module Status + module Build + class Retryable < SimpleDelegator + include Status::Extended + + def has_action? + can?(user, :update_build, subject) + end + + def action_icon + 'refresh' + end + + def action_title + 'Retry' + end + + def action_path + retry_namespace_project_build_path(subject.project.namespace, + subject.project, + subject) + end + + def action_method + :post + end + + def self.matches?(build, user) + build.retryable? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/stop.rb b/lib/gitlab/ci/status/build/stop.rb new file mode 100644 index 00000000000..f8ffa95cde4 --- /dev/null +++ b/lib/gitlab/ci/status/build/stop.rb @@ -0,0 +1,49 @@ +module Gitlab + module Ci + module Status + module Build + class Stop < SimpleDelegator + include Status::Extended + + def text + 'manual' + end + + def label + 'manual stop action' + end + + def icon + 'icon_status_manual' + end + + def has_action? + can?(user, :update_build, subject) + end + + def action_icon + 'stop' + end + + def action_title + 'Stop' + end + + def action_path + play_namespace_project_build_path(subject.project.namespace, + subject.project, + subject) + end + + def action_method + :post + end + + def self.matches?(build, user) + build.playable? && build.stops_environment? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/canceled.rb b/lib/gitlab/ci/status/canceled.rb new file mode 100644 index 00000000000..dd6d99e9075 --- /dev/null +++ b/lib/gitlab/ci/status/canceled.rb @@ -0,0 +1,19 @@ +module Gitlab + module Ci + module Status + class Canceled < Status::Core + def text + 'canceled' + end + + def label + 'canceled' + end + + def icon + 'icon_status_canceled' + end + end + end + end +end diff --git a/lib/gitlab/ci/status/core.rb b/lib/gitlab/ci/status/core.rb new file mode 100644 index 00000000000..46fef8262c1 --- /dev/null +++ b/lib/gitlab/ci/status/core.rb @@ -0,0 +1,69 @@ +module Gitlab + module Ci + module Status + # Base abstract class fore core status + # + class Core + include Gitlab::Routing + include Gitlab::Allowable + + attr_reader :subject, :user + + def initialize(subject, user) + @subject = subject + @user = user + end + + def icon + raise NotImplementedError + end + + def label + raise NotImplementedError + end + + # Deprecation warning: this method is here because we need to maintain + # backwards compatibility with legacy statuses. We often do something + # like "ci-status ci-status-#{status}" to set CSS class. + # + # `to_s` method should be renamed to `group` at some point, after + # phasing legacy satuses out. + # + def to_s + self.class.name.demodulize.downcase.underscore + end + + def has_details? + false + end + + def details_path + raise NotImplementedError + end + + def has_action? + false + end + + def action_icon + raise NotImplementedError + end + + def action_class + end + + def action_path + raise NotImplementedError + end + + def action_method + raise NotImplementedError + end + + def action_title + raise NotImplementedError + end + end + end + end +end diff --git a/lib/gitlab/ci/status/created.rb b/lib/gitlab/ci/status/created.rb new file mode 100644 index 00000000000..6596d7e01ca --- /dev/null +++ b/lib/gitlab/ci/status/created.rb @@ -0,0 +1,19 @@ +module Gitlab + module Ci + module Status + class Created < Status::Core + def text + 'created' + end + + def label + 'created' + end + + def icon + 'icon_status_created' + end + end + end + end +end diff --git a/lib/gitlab/ci/status/extended.rb b/lib/gitlab/ci/status/extended.rb new file mode 100644 index 00000000000..d367c9bda69 --- /dev/null +++ b/lib/gitlab/ci/status/extended.rb @@ -0,0 +1,15 @@ +module Gitlab + module Ci + module Status + module Extended + extend ActiveSupport::Concern + + class_methods do + def matches?(_subject, _user) + raise NotImplementedError + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/factory.rb b/lib/gitlab/ci/status/factory.rb new file mode 100644 index 00000000000..ae9ef895df4 --- /dev/null +++ b/lib/gitlab/ci/status/factory.rb @@ -0,0 +1,47 @@ +module Gitlab + module Ci + module Status + class Factory + def initialize(subject, user) + @subject = subject + @user = user + end + + def fabricate! + if extended_status + extended_status.new(core_status) + else + core_status + end + end + + def self.extended_statuses + [] + end + + def self.common_helpers + Module.new + end + + private + + def simple_status + @simple_status ||= @subject.status || :created + end + + def core_status + Gitlab::Ci::Status + .const_get(simple_status.capitalize) + .new(@subject, @user) + .extend(self.class.common_helpers) + end + + def extended_status + @extended ||= self.class.extended_statuses.find do |status| + status.matches?(@subject, @user) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/failed.rb b/lib/gitlab/ci/status/failed.rb new file mode 100644 index 00000000000..c5b5e3203ad --- /dev/null +++ b/lib/gitlab/ci/status/failed.rb @@ -0,0 +1,19 @@ +module Gitlab + module Ci + module Status + class Failed < Status::Core + def text + 'failed' + end + + def label + 'failed' + end + + def icon + 'icon_status_failed' + end + end + end + end +end diff --git a/lib/gitlab/ci/status/pending.rb b/lib/gitlab/ci/status/pending.rb new file mode 100644 index 00000000000..d30f35a59a2 --- /dev/null +++ b/lib/gitlab/ci/status/pending.rb @@ -0,0 +1,19 @@ +module Gitlab + module Ci + module Status + class Pending < Status::Core + def text + 'pending' + end + + def label + 'pending' + end + + def icon + 'icon_status_pending' + end + end + end + end +end diff --git a/lib/gitlab/ci/status/pipeline/common.rb b/lib/gitlab/ci/status/pipeline/common.rb new file mode 100644 index 00000000000..76bfd18bf40 --- /dev/null +++ b/lib/gitlab/ci/status/pipeline/common.rb @@ -0,0 +1,23 @@ +module Gitlab + module Ci + module Status + module Pipeline + module Common + def has_details? + can?(user, :read_pipeline, subject) + end + + def details_path + namespace_project_pipeline_path(subject.project.namespace, + subject.project, + subject) + end + + def has_action? + false + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/pipeline/factory.rb b/lib/gitlab/ci/status/pipeline/factory.rb new file mode 100644 index 00000000000..16dcb326be9 --- /dev/null +++ b/lib/gitlab/ci/status/pipeline/factory.rb @@ -0,0 +1,17 @@ +module Gitlab + module Ci + module Status + module Pipeline + class Factory < Status::Factory + def self.extended_statuses + [Pipeline::SuccessWithWarnings] + end + + def self.common_helpers + Status::Pipeline::Common + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/pipeline/success_with_warnings.rb b/lib/gitlab/ci/status/pipeline/success_with_warnings.rb new file mode 100644 index 00000000000..a7c98f9e909 --- /dev/null +++ b/lib/gitlab/ci/status/pipeline/success_with_warnings.rb @@ -0,0 +1,31 @@ +module Gitlab + module Ci + module Status + module Pipeline + class SuccessWithWarnings < SimpleDelegator + include Status::Extended + + def text + 'passed' + end + + def label + 'passed with warnings' + end + + def icon + 'icon_status_warning' + end + + def to_s + 'success_with_warnings' + end + + def self.matches?(pipeline, user) + pipeline.success? && pipeline.has_warnings? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/running.rb b/lib/gitlab/ci/status/running.rb new file mode 100644 index 00000000000..2aba3c373c7 --- /dev/null +++ b/lib/gitlab/ci/status/running.rb @@ -0,0 +1,19 @@ +module Gitlab + module Ci + module Status + class Running < Status::Core + def text + 'running' + end + + def label + 'running' + end + + def icon + 'icon_status_running' + end + end + end + end +end diff --git a/lib/gitlab/ci/status/skipped.rb b/lib/gitlab/ci/status/skipped.rb new file mode 100644 index 00000000000..16282aefd03 --- /dev/null +++ b/lib/gitlab/ci/status/skipped.rb @@ -0,0 +1,19 @@ +module Gitlab + module Ci + module Status + class Skipped < Status::Core + def text + 'skipped' + end + + def label + 'skipped' + end + + def icon + 'icon_status_skipped' + end + end + end + end +end diff --git a/lib/gitlab/ci/status/stage/common.rb b/lib/gitlab/ci/status/stage/common.rb new file mode 100644 index 00000000000..7852f492e1d --- /dev/null +++ b/lib/gitlab/ci/status/stage/common.rb @@ -0,0 +1,24 @@ +module Gitlab + module Ci + module Status + module Stage + module Common + def has_details? + can?(user, :read_pipeline, subject.pipeline) + end + + def details_path + namespace_project_pipeline_path(subject.project.namespace, + subject.project, + subject.pipeline, + anchor: subject.name) + end + + def has_action? + false + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/stage/factory.rb b/lib/gitlab/ci/status/stage/factory.rb new file mode 100644 index 00000000000..689a5dd45bc --- /dev/null +++ b/lib/gitlab/ci/status/stage/factory.rb @@ -0,0 +1,13 @@ +module Gitlab + module Ci + module Status + module Stage + class Factory < Status::Factory + def self.common_helpers + Status::Stage::Common + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/success.rb b/lib/gitlab/ci/status/success.rb new file mode 100644 index 00000000000..c09c5f006e3 --- /dev/null +++ b/lib/gitlab/ci/status/success.rb @@ -0,0 +1,19 @@ +module Gitlab + module Ci + module Status + class Success < Status::Core + def text + 'passed' + end + + def label + 'passed' + end + + def icon + 'icon_status_success' + end + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/base_event.rb b/lib/gitlab/cycle_analytics/base_event.rb index 486139b1687..53a148ad703 100644 --- a/lib/gitlab/cycle_analytics/base_event.rb +++ b/lib/gitlab/cycle_analytics/base_event.rb @@ -16,7 +16,7 @@ module Gitlab event_result.map do |event| serialize(event) if has_permission?(event['id']) - end + end.compact end def custom_query(_base_query); end diff --git a/lib/gitlab/cycle_analytics/plan_event.rb b/lib/gitlab/cycle_analytics/plan_event.rb index b1ae215f348..7c3f0e9989f 100644 --- a/lib/gitlab/cycle_analytics/plan_event.rb +++ b/lib/gitlab/cycle_analytics/plan_event.rb @@ -27,6 +27,8 @@ module Gitlab end def first_time_reference_commit(commits, event) + return nil if commits.blank? + YAML.load(commits).find do |commit| next unless commit[:committed_date] && event['first_mentioned_in_commit_at'] diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb index 06a783ebc1c..e50e54b6e99 100644 --- a/lib/gitlab/data_builder/pipeline.rb +++ b/lib/gitlab/data_builder/pipeline.rb @@ -22,7 +22,7 @@ module Gitlab sha: pipeline.sha, before_sha: pipeline.before_sha, status: pipeline.status, - stages: pipeline.stages, + stages: pipeline.stages_name, created_at: pipeline.created_at, finished_at: pipeline.finished_at, duration: pipeline.duration diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 2d5c9232425..55b8f888d53 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -35,13 +35,6 @@ module Gitlab order end - def self.serialized_transaction - opts = {} - opts[:isolation] = :serializable unless Rails.env.test? && connection.transaction_open? - - connection.transaction(opts) { yield } - end - def self.random Gitlab::Database.postgresql? ? "RANDOM()" : "RAND()" end diff --git a/lib/gitlab/diff/file_collection/merge_request_diff.rb b/lib/gitlab/diff/file_collection/merge_request_diff.rb index fe7adb7bed6..56530448f36 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff.rb @@ -20,7 +20,7 @@ module Gitlab # Extracted method to highlight in the same iteration to the diff_collection. def decorate_diff!(diff) diff_file = super - cache_highlight!(diff_file) if cacheable? + cache_highlight!(diff_file) if cacheable?(diff_file) diff_file end @@ -60,8 +60,8 @@ module Gitlab Rails.cache.write(cache_key, highlight_cache) if @highlight_cache_was_empty end - def cacheable? - @merge_request_diff.present? + def cacheable?(diff_file) + @merge_request_diff.present? && diff_file.blob && diff_file.blob.text? end def cache_key diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index f4d1505ea91..c8e36d8ff4a 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -149,7 +149,7 @@ module Gitlab end def ce_patch_name - @ce_patch_name ||= "#{ce_branch}.patch" + @ce_patch_name ||= patch_name_from_branch(ce_branch) end def ce_patch_full_path @@ -161,13 +161,17 @@ module Gitlab end def ee_patch_name - @ee_patch_name ||= "#{ee_branch}.patch" + @ee_patch_name ||= patch_name_from_branch(ee_branch) end def ee_patch_full_path @ee_patch_full_path ||= patches_dir.join(ee_patch_name) end + def patch_name_from_branch(branch_name) + branch_name.parameterize << '.patch' + end + def step(desc, cmd = nil) puts "\n=> #{desc}\n" diff --git a/lib/gitlab/email/reply_parser.rb b/lib/gitlab/email/reply_parser.rb index 85402c2a278..f586c5ab062 100644 --- a/lib/gitlab/email/reply_parser.rb +++ b/lib/gitlab/email/reply_parser.rb @@ -69,7 +69,7 @@ module Gitlab # This one might be controversial but so many reply lines have years, times and end with a colon. # Let's try it and see how well it works. break if (l =~ /\d{4}/ && l =~ /\d:\d\d/ && l =~ /\:$/) || - (l =~ /On \w+ \d+,? \d+,?.*wrote:/) + (l =~ /On \w+ \d+,? \d+,?.*wrote:/) # Headers on subsequent lines break if (0..2).all? { |off| lines[idx + off] =~ REPLYING_HEADER_REGEX } diff --git a/lib/gitlab/gfm/uploads_rewriter.rb b/lib/gitlab/gfm/uploads_rewriter.rb index abc8c8c55e6..8fab5489616 100644 --- a/lib/gitlab/gfm/uploads_rewriter.rb +++ b/lib/gitlab/gfm/uploads_rewriter.rb @@ -1,3 +1,5 @@ +require 'fileutils' + module Gitlab module Gfm ## @@ -22,7 +24,9 @@ module Gitlab return markdown unless file.try(:exists?) new_uploader = FileUploader.new(target_project) - new_uploader.store!(file) + with_link_in_tmp_dir(file.file) do |open_tmp_file| + new_uploader.store!(open_tmp_file) + end new_uploader.to_markdown end end @@ -46,6 +50,19 @@ module Gitlab uploader.retrieve_from_store!(file) uploader.file end + + # Because the uploaders use 'move_to_store' we must have a temporary + # file that is allowed to be (re)moved. + def with_link_in_tmp_dir(file) + dir = Dir.mktmpdir('UploadsRewriter', File.dirname(file)) + # The filename matters to Carrierwave so we make sure to preserve it + tmp_file = File.join(dir, File.basename(file)) + File.link(file, tmp_file) + # Open the file to placate Carrierwave + File.open(tmp_file) { |open_file| yield open_file } + ensure + FileUtils.rm_rf(dir) + end end end end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index bcbf6455998..db07b7c5fcc 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -46,7 +46,7 @@ module Gitlab def download_access_check if user user_download_access_check - elsif deploy_key.nil? && !Guest.can?(:download_code, project) + elsif deploy_key.nil? && !guest_can_downlod_code? raise UnauthorizedError, ERROR_MESSAGES[:download] end end @@ -59,6 +59,10 @@ module Gitlab end end + def guest_can_downlod_code? + Guest.can?(:download_code, project) + end + def user_download_access_check unless user_can_download_code? || build_can_download_code? raise UnauthorizedError, ERROR_MESSAGES[:download] diff --git a/lib/gitlab/git_access_wiki.rb b/lib/gitlab/git_access_wiki.rb index f71d3575909..2c06c4ff1ef 100644 --- a/lib/gitlab/git_access_wiki.rb +++ b/lib/gitlab/git_access_wiki.rb @@ -1,5 +1,13 @@ module Gitlab class GitAccessWiki < GitAccess + def guest_can_downlod_code? + Guest.can?(:download_wiki_code, project) + end + + def user_can_download_code? + authentication_abilities.include?(:download_code) && user_access.can_do_action?(:download_wiki_code) + end + def change_access_check(change) if user_access.can_do_action?(:create_wiki) build_status_object(true) diff --git a/lib/gitlab/github_import/branch_formatter.rb b/lib/gitlab/github_import/branch_formatter.rb index 4750675ae9d..0a8d05b5fe1 100644 --- a/lib/gitlab/github_import/branch_formatter.rb +++ b/lib/gitlab/github_import/branch_formatter.rb @@ -8,7 +8,7 @@ module Gitlab end def valid? - repo.present? + sha.present? && ref.present? end private diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 2c21804fe7a..4d4e04e9e35 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -8,6 +8,8 @@ module Gitlab gon.shortcuts_path = help_page_path('shortcuts') gon.user_color_scheme = Gitlab::ColorSchemes.for_user(current_user).css_class gon.award_menu_url = emojis_path + gon.katex_css_url = ActionController::Base.helpers.asset_path('katex.css') + gon.katex_js_url = ActionController::Base.helpers.asset_path('katex.js') if current_user gon.current_user_id = current_user.id diff --git a/lib/gitlab/identifier.rb b/lib/gitlab/identifier.rb index f8809db21aa..94678b6ec40 100644 --- a/lib/gitlab/identifier.rb +++ b/lib/gitlab/identifier.rb @@ -21,10 +21,8 @@ module Gitlab return if !commit || !commit.author_email - email = commit.author_email - - identify_with_cache(:email, email) do - User.find_by(email: email) + identify_with_cache(:email, commit.author_email) do + commit.author end end diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb new file mode 100644 index 00000000000..65713e73a59 --- /dev/null +++ b/lib/gitlab/middleware/multipart.rb @@ -0,0 +1,99 @@ +# Gitlab::Middleware::Multipart - a Rack::Multipart replacement +# +# Rack::Multipart leaves behind tempfiles in /tmp and uses valuable Ruby +# process time to copy files around. This alternative solution uses +# gitlab-workhorse to clean up the tempfiles and puts the tempfiles in a +# location where copying should not be needed. +# +# When gitlab-workhorse finds files in a multipart MIME body it sends +# a signed message via a request header. This message lists the names of +# the multipart entries that gitlab-workhorse filtered out of the +# multipart structure and saved to tempfiles. Workhorse adds new entries +# in the multipart structure with paths to the tempfiles. +# +# The job of this Rack middleware is to detect and decode the message +# from workhorse. If present, it walks the Rack 'params' hash for the +# current request, opens the respective tempfiles, and inserts the open +# Ruby File objects in the params hash where Rack::Multipart would have +# put them. The goal is that application code deeper down can keep +# working the way it did with Rack::Multipart without changes. +# +# CAVEAT: the code that modifies the params hash is a bit complex. It is +# conceivable that certain Rack params structures will not be modified +# correctly. We are not aware of such bugs at this time though. +# + +module Gitlab + module Middleware + class Multipart + RACK_ENV_KEY = 'HTTP_GITLAB_WORKHORSE_MULTIPART_FIELDS' + + class Handler + def initialize(env, message) + @request = Rack::Request.new(env) + @rewritten_fields = message['rewritten_fields'] + @open_files = [] + end + + def with_open_files + @rewritten_fields.each do |field, tmp_path| + parsed_field = Rack::Utils.parse_nested_query(field) + raise "unexpected field: #{field.inspect}" unless parsed_field.count == 1 + + key, value = parsed_field.first + if value.nil? + value = File.open(tmp_path) + @open_files << value + else + value = decorate_params_value(value, @request.params[key], tmp_path) + end + @request.update_param(key, value) + end + + yield + ensure + @open_files.each(&:close) + end + + # This function calls itself recursively + def decorate_params_value(path_hash, value_hash, tmp_path) + unless path_hash.is_a?(Hash) && path_hash.count == 1 + raise "invalid path: #{path_hash.inspect}" + end + path_key, path_value = path_hash.first + + unless value_hash.is_a?(Hash) && value_hash[path_key] + raise "invalid value hash: #{value_hash.inspect}" + end + + case path_value + when nil + value_hash[path_key] = File.open(tmp_path) + @open_files << value_hash[path_key] + value_hash + when Hash + decorate_params_value(path_value, value_hash[path_key], tmp_path) + value_hash + else + raise "unexpected path value: #{path_value.inspect}" + end + end + end + + def initialize(app) + @app = app + end + + def call(env) + encoded_message = env.delete(RACK_ENV_KEY) + return @app.call(env) if encoded_message.blank? + + message = Gitlab::Workhorse.decode_jwt(encoded_message)[0] + + Handler.new(env, message).with_open_files do + @app.call(env) + end + end + end + end +end diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index a8b4dc2a83f..96ed20af918 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -39,7 +39,7 @@ module Gitlab log.info "(#{provider}) saving user #{auth_hash.email} from login with extern_uid => #{auth_hash.uid}" gl_user rescue ActiveRecord::RecordInvalid => e - log.info "(#{provider}) Error saving user: #{gl_user.errors.full_messages}" + log.info "(#{provider}) Error saving user #{auth_hash.uid} (#{auth_hash.email}): #{gl_user.errors.full_messages}" return self, e.record.errors end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 66e6b29e798..6bdf3db9cb8 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -110,7 +110,7 @@ module Gitlab end def notes - @notes ||= project.notes.user.search(query, as_user: @current_user).order('updated_at DESC') + @notes ||= NotesFinder.new(project, @current_user, search: query).execute.user.order('updated_at DESC') end def commits diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index c12358ceef4..9e0b0e5ea98 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -8,8 +8,10 @@ module Gitlab # allow non-regex validatiions, etc), `NAMESPACE_REGEX_STR_SIMPLE` serves as a Javascript-compatible version of # `NAMESPACE_REGEX_STR`, with the negative lookbehind assertion removed. This means that the client-side validation # will pass for usernames ending in `.atom` and `.git`, but will be caught by the server-side validation. - NAMESPACE_REGEX_STR_SIMPLE = '[a-zA-Z0-9_\.][a-zA-Z0-9_\-\.]*[a-zA-Z0-9_\-]|[a-zA-Z0-9_]'.freeze + PATH_REGEX_STR = '[a-zA-Z0-9_\.][a-zA-Z0-9_\-\.]*'.freeze + NAMESPACE_REGEX_STR_SIMPLE = PATH_REGEX_STR + '[a-zA-Z0-9_\-]|[a-zA-Z0-9_]'.freeze NAMESPACE_REGEX_STR = '(?:' + NAMESPACE_REGEX_STR_SIMPLE + ')(?<!\.git|\.atom)'.freeze + PROJECT_REGEX_STR = PATH_REGEX_STR + '(?<!\.git|\.atom)'.freeze def namespace_regex @namespace_regex ||= /\A#{NAMESPACE_REGEX_STR}\z/.freeze @@ -42,7 +44,15 @@ module Gitlab end def project_path_regex - @project_path_regex ||= /\A[a-zA-Z0-9_.][a-zA-Z0-9_\-\.]*(?<!\.git|\.atom)\z/.freeze + @project_path_regex ||= /\A#{PROJECT_REGEX_STR}\z/.freeze + end + + def project_route_regex + @project_route_regex ||= /#{PROJECT_REGEX_STR}/.freeze + end + + def project_git_route_regex + @project_route_git_regex ||= /#{PATH_REGEX_STR}\.git/.freeze end def project_path_regex_message @@ -51,7 +61,7 @@ module Gitlab end def file_name_regex - @file_name_regex ||= /\A[a-zA-Z0-9_\-\.\@]*\z/.freeze + @file_name_regex ||= /\A[[[:alnum:]]_\-\.\@]*\z/.freeze end def file_name_regex_message @@ -59,7 +69,7 @@ module Gitlab end def file_path_regex - @file_path_regex ||= /\A[a-zA-Z0-9_\-\.\/\@]*\z/.freeze + @file_path_regex ||= /\A[[[:alnum:]]_\-\.\/\@]*\z/.freeze end def file_path_regex_message @@ -113,5 +123,22 @@ module Gitlab def environment_name_regex_message "can contain only letters, digits, '-', '_', '/', '$', '{', '}', '.' and spaces" end + + def kubernetes_namespace_regex + /\A[a-z0-9]([-a-z0-9]*[a-z0-9])?\z/ + end + + def kubernetes_namespace_regex_message + "can contain only letters, digits or '-', and cannot start or end with '-'" + end + + def environment_slug_regex + @environment_slug_regex ||= /\A[a-z]([a-z0-9-]*[a-z0-9])?\z/.freeze + end + + def environment_slug_regex_message + "can contain only lowercase letters, digits, and '-'. " \ + "Must start with a letter, and cannot end with '-'" + end end end diff --git a/lib/gitlab/routing.rb b/lib/gitlab/routing.rb index 5132177de51..632e2d87500 100644 --- a/lib/gitlab/routing.rb +++ b/lib/gitlab/routing.rb @@ -1,5 +1,11 @@ module Gitlab module Routing + extend ActiveSupport::Concern + + included do + include Gitlab::Routing.url_helpers + end + # Returns the URL helpers Module. # # This method caches the output as Rails' "url_helpers" method creates an diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 2690938fe82..35212992698 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -50,7 +50,7 @@ module Gitlab end def issues - issues = Issue.visible_to_user(current_user).where(project_id: project_ids_relation) + issues = IssuesFinder.new(current_user).execute.where(project_id: project_ids_relation) if query =~ /#(\d+)\z/ issues = issues.where(iid: $1) @@ -68,7 +68,7 @@ module Gitlab end def merge_requests - merge_requests = MergeRequest.in_projects(project_ids_relation) + merge_requests = MergeRequestsFinder.new(current_user).execute.in_projects(project_ids_relation) if query =~ /[#!](\d+)\z/ merge_requests = merge_requests.where(iid: $1) else diff --git a/lib/gitlab/sql/union.rb b/lib/gitlab/sql/union.rb index 1cd89b3a9c4..222021e8802 100644 --- a/lib/gitlab/sql/union.rb +++ b/lib/gitlab/sql/union.rb @@ -22,9 +22,7 @@ module Gitlab # By using "unprepared_statements" we remove the usage of placeholders # (thus fixing this problem), at a slight performance cost. fragments = ActiveRecord::Base.connection.unprepared_statement do - @relations.map do |rel| - rel.reorder(nil).to_sql - end + @relations.map { |rel| rel.reorder(nil).to_sql }.reject(&:blank?) end fragments.join("\nUNION\n") diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index 99d0c28e749..ccb456bcc94 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -24,6 +24,8 @@ module Gitlab wiki_page_url when ProjectSnippet project_snippet_url(object) + when Snippet + personal_snippet_url(object) else raise NotImplementedError.new("No URL builder defined for #{object.class}") end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 594439a5d4b..aeb1a26e1ba 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -117,8 +117,12 @@ module Gitlab end def verify_api_request!(request_headers) + decode_jwt(request_headers[INTERNAL_API_REQUEST_HEADER]) + end + + def decode_jwt(encoded_message) JWT.decode( - request_headers[INTERNAL_API_REQUEST_HEADER], + encoded_message, secret, true, { iss: 'gitlab-workhorse', verify_iss: true, algorithm: 'HS256' }, diff --git a/lib/mattermost/session.rb b/lib/mattermost/session.rb new file mode 100644 index 00000000000..fb8d7d97f8a --- /dev/null +++ b/lib/mattermost/session.rb @@ -0,0 +1,115 @@ +module Mattermost + class NoSessionError < StandardError; end + # This class' prime objective is to obtain a session token on a Mattermost + # instance with SSO configured where this GitLab instance is the provider. + # + # The process depends on OAuth, but skips a step in the authentication cycle. + # For example, usually a user would click the 'login in GitLab' button on + # Mattermost, which would yield a 302 status code and redirects you to GitLab + # to approve the use of your account on Mattermost. Which would trigger a + # callback so Mattermost knows this request is approved and gets the required + # data to create the user account etc. + # + # This class however skips the button click, and also the approval phase to + # speed up the process and keep it without manual action and get a session + # going. + class Session + include Doorkeeper::Helpers::Controller + include HTTParty + + base_uri Settings.mattermost.host + + attr_accessor :current_resource_owner, :token + + def initialize(current_user) + @current_resource_owner = current_user + end + + def with_session + raise NoSessionError unless create + + begin + yield self + ensure + destroy + end + end + + # Next methods are needed for Doorkeeper + def pre_auth + @pre_auth ||= Doorkeeper::OAuth::PreAuthorization.new( + Doorkeeper.configuration, server.client_via_uid, params) + end + + def authorization + @authorization ||= strategy.request + end + + def strategy + @strategy ||= server.authorization_request(pre_auth.response_type) + end + + def request + @request ||= OpenStruct.new(parameters: params) + end + + def params + Rack::Utils.parse_query(oauth_uri.query).symbolize_keys + end + + def get(path, options = {}) + self.class.get(path, options.merge(headers: @headers)) + end + + def post(path, options = {}) + self.class.post(path, options.merge(headers: @headers)) + end + + private + + def create + return unless oauth_uri + return unless token_uri + + @token = request_token + @headers = { + Authorization: "Bearer #{@token}" + } + + @token + end + + def destroy + post('/api/v3/users/logout') + end + + def oauth_uri + return @oauth_uri if defined?(@oauth_uri) + + @oauth_uri = nil + + response = get("/api/v3/oauth/gitlab/login", follow_redirects: false) + return unless 300 <= response.code && response.code < 400 + + redirect_uri = response.headers['location'] + return unless redirect_uri + + @oauth_uri = URI.parse(redirect_uri) + end + + def token_uri + @token_uri ||= + if oauth_uri + authorization.authorize.redirect_uri if pre_auth.authorizable? + end + end + + def request_token + response = get(token_uri, follow_redirects: false) + + if 200 <= response.code && response.code < 400 + response.headers['token'] + end + end + end +end diff --git a/lib/omniauth/strategies/bitbucket.rb b/lib/omniauth/strategies/bitbucket.rb new file mode 100644 index 00000000000..5a7d67c2390 --- /dev/null +++ b/lib/omniauth/strategies/bitbucket.rb @@ -0,0 +1,41 @@ +require 'omniauth-oauth2' + +module OmniAuth + module Strategies + class Bitbucket < OmniAuth::Strategies::OAuth2 + option :name, 'bitbucket' + + option :client_options, { + site: 'https://bitbucket.org', + authorize_url: 'https://bitbucket.org/site/oauth2/authorize', + token_url: 'https://bitbucket.org/site/oauth2/access_token' + } + + uid do + raw_info['username'] + end + + info do + { + name: raw_info['display_name'], + avatar: raw_info['links']['avatar']['href'], + email: primary_email + } + end + + def raw_info + @raw_info ||= access_token.get('api/2.0/user').parsed + end + + def primary_email + primary = emails.find { |i| i['is_primary'] && i['is_confirmed'] } + primary && primary['email'] || nil + end + + def emails + email_response = access_token.get('api/2.0/user/emails').parsed + @emails ||= email_response && email_response['values'] || nil + end + end + end +end diff --git a/lib/rouge/lexers/math.rb b/lib/rouge/lexers/math.rb new file mode 100644 index 00000000000..80784adfd76 --- /dev/null +++ b/lib/rouge/lexers/math.rb @@ -0,0 +1,21 @@ +module Rouge + module Lexers + class Math < Lexer + title "A passthrough lexer used for LaTeX input" + desc "A boring lexer that doesn't highlight anything" + + tag 'math' + mimetypes 'text/plain' + + default_options token: 'Text' + + def token + @token ||= Token[option :token] + end + + def stream_tokens(string, &b) + yield self.token, string + end + end + end +end diff --git a/lib/tasks/gitlab/helpers.rake b/lib/tasks/gitlab/helpers.rake new file mode 100644 index 00000000000..dd2d5861481 --- /dev/null +++ b/lib/tasks/gitlab/helpers.rake @@ -0,0 +1,8 @@ +require 'tasks/gitlab/task_helpers' + +# Prevent StateMachine warnings from outputting during a cron task +StateMachines::Machine.ignore_method_conflicts = true if ENV['CRON'] + +namespace :gitlab do + include Gitlab::TaskHelpers +end diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index 58761a129d4..5a09cd7ce41 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -5,42 +5,23 @@ namespace :gitlab do warn_user_is_not_gitlab default_version = Gitlab::Shell.version_required - default_version_tag = 'v' + default_version - args.with_defaults(tag: default_version_tag, repo: "https://gitlab.com/gitlab-org/gitlab-shell.git") + default_version_tag = "v#{default_version}" + args.with_defaults(tag: default_version_tag, repo: 'https://gitlab.com/gitlab-org/gitlab-shell.git') - user = Gitlab.config.gitlab.user - home_dir = Rails.env.test? ? Rails.root.join('tmp/tests') : Gitlab.config.gitlab.user_home gitlab_url = Gitlab.config.gitlab.url # gitlab-shell requires a / at the end of the url gitlab_url += '/' unless gitlab_url.end_with?('/') target_dir = Gitlab.config.gitlab_shell.path - # Clone if needed - if File.directory?(target_dir) - Dir.chdir(target_dir) do - system(*%W(Gitlab.config.git.bin_path} fetch --tags --quiet)) - system(*%W(Gitlab.config.git.bin_path} checkout --quiet #{default_version_tag})) - end - else - system(*%W(#{Gitlab.config.git.bin_path} clone -- #{args.repo} #{target_dir})) - end + checkout_or_clone_tag(tag: default_version_tag, repo: args.repo, target_dir: target_dir) # Make sure we're on the right tag Dir.chdir(target_dir) do - # First try to checkout without fetching - # to avoid stalling tests if the Internet is down. - reseted = reset_to_commit(args) - - unless reseted - system(*%W(#{Gitlab.config.git.bin_path} fetch origin)) - reset_to_commit(args) - end - config = { - user: user, + user: Gitlab.config.gitlab.user, gitlab_url: gitlab_url, http_settings: {self_signed_cert: false}.stringify_keys, - auth_file: File.join(home_dir, ".ssh", "authorized_keys"), + auth_file: File.join(user_home, ".ssh", "authorized_keys"), redis: { bin: %x{which redis-cli}.chomp, namespace: "resque:gitlab" @@ -74,7 +55,7 @@ namespace :gitlab do # be an issue since it is more than likely that there are no "normal" # user accounts on a gitlab server). The alternative is for the admin to # install a ruby (1.9.3+) in the global path. - File.open(File.join(home_dir, ".ssh", "environment"), "w+") do |f| + File.open(File.join(user_home, ".ssh", "environment"), "w+") do |f| f.puts "PATH=#{ENV['PATH']}" end @@ -142,15 +123,4 @@ namespace :gitlab do puts "Quitting...".color(:red) exit 1 end - - def reset_to_commit(args) - tag, status = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} describe -- #{args.tag})) - - unless status.zero? - tag, status = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} describe -- origin/#{args.tag})) - end - - tag = tag.strip - system(*%W(#{Gitlab.config.git.bin_path} reset --hard #{tag})) - end end diff --git a/lib/tasks/gitlab/task_helpers.rake b/lib/tasks/gitlab/task_helpers.rake deleted file mode 100644 index 74be413423a..00000000000 --- a/lib/tasks/gitlab/task_helpers.rake +++ /dev/null @@ -1,140 +0,0 @@ -module Gitlab - class TaskAbortedByUserError < StandardError; end -end - -require 'rainbow/ext/string' - -# Prevent StateMachine warnings from outputting during a cron task -StateMachines::Machine.ignore_method_conflicts = true if ENV['CRON'] - -namespace :gitlab do - - # Ask if the user wants to continue - # - # Returns "yes" the user chose to continue - # Raises Gitlab::TaskAbortedByUserError if the user chose *not* to continue - def ask_to_continue - answer = prompt("Do you want to continue (yes/no)? ".color(:blue), %w{yes no}) - raise Gitlab::TaskAbortedByUserError unless answer == "yes" - end - - # Check which OS is running - # - # It will primarily use lsb_relase to determine the OS. - # It has fallbacks to Debian, SuSE, OS X and systems running systemd. - def os_name - os_name = run_command(%W(lsb_release -irs)) - os_name ||= if File.readable?('/etc/system-release') - File.read('/etc/system-release') - end - os_name ||= if File.readable?('/etc/debian_version') - debian_version = File.read('/etc/debian_version') - "Debian #{debian_version}" - end - os_name ||= if File.readable?('/etc/SuSE-release') - File.read('/etc/SuSE-release') - end - os_name ||= if os_x_version = run_command(%W(sw_vers -productVersion)) - "Mac OS X #{os_x_version}" - end - os_name ||= if File.readable?('/etc/os-release') - File.read('/etc/os-release').match(/PRETTY_NAME=\"(.+)\"/)[1] - end - os_name.try(:squish!) - end - - # Prompt the user to input something - # - # message - the message to display before input - # choices - array of strings of acceptable answers or nil for any answer - # - # Returns the user's answer - def prompt(message, choices = nil) - begin - print(message) - answer = STDIN.gets.chomp - end while choices.present? && !choices.include?(answer) - answer - end - - # Runs the given command and matches the output against the given pattern - # - # Returns nil if nothing matched - # Returns the MatchData if the pattern matched - # - # see also #run_command - # see also String#match - def run_and_match(command, regexp) - run_command(command).try(:match, regexp) - end - - # Runs the given command - # - # Returns nil if the command was not found - # Returns the output of the command otherwise - # - # see also #run_and_match - def run_command(command) - output, _ = Gitlab::Popen.popen(command) - output - rescue Errno::ENOENT - '' # if the command does not exist, return an empty string - end - - def uid_for(user_name) - run_command(%W(id -u #{user_name})).chomp.to_i - end - - def gid_for(group_name) - begin - Etc.getgrnam(group_name).gid - rescue ArgumentError # no group - "group #{group_name} doesn't exist" - end - end - - def warn_user_is_not_gitlab - unless @warned_user_not_gitlab - gitlab_user = Gitlab.config.gitlab.user - current_user = run_command(%W(whoami)).chomp - unless current_user == gitlab_user - puts " Warning ".color(:black).background(:yellow) - puts " You are running as user #{current_user.color(:magenta)}, we hope you know what you are doing." - puts " Things may work\/fail for the wrong reasons." - puts " For correct results you should run this as user #{gitlab_user.color(:magenta)}." - puts "" - end - @warned_user_not_gitlab = true - end - end - - # Tries to configure git itself - # - # Returns true if all subcommands were successfull (according to their exit code) - # Returns false if any or all subcommands failed. - def auto_fix_git_config(options) - if !@warned_user_not_gitlab - command_success = options.map do |name, value| - system(*%W(#{Gitlab.config.git.bin_path} config --global #{name} #{value})) - end - - command_success.all? - else - false - end - end - - def all_repos - Gitlab.config.repositories.storages.each do |name, path| - IO.popen(%W(find #{path} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find| - find.each_line do |path| - yield path.chomp - end - end - end - end - - def repository_storage_paths_args - Gitlab.config.repositories.storages.values - end -end diff --git a/lib/tasks/gitlab/task_helpers.rb b/lib/tasks/gitlab/task_helpers.rb new file mode 100644 index 00000000000..e128738b5f8 --- /dev/null +++ b/lib/tasks/gitlab/task_helpers.rb @@ -0,0 +1,190 @@ +require 'rainbow/ext/string' + +module Gitlab + TaskFailedError = Class.new(StandardError) + TaskAbortedByUserError = Class.new(StandardError) + + module TaskHelpers + # Ask if the user wants to continue + # + # Returns "yes" the user chose to continue + # Raises Gitlab::TaskAbortedByUserError if the user chose *not* to continue + def ask_to_continue + answer = prompt("Do you want to continue (yes/no)? ".color(:blue), %w{yes no}) + raise Gitlab::TaskAbortedByUserError unless answer == "yes" + end + + # Check which OS is running + # + # It will primarily use lsb_relase to determine the OS. + # It has fallbacks to Debian, SuSE, OS X and systems running systemd. + def os_name + os_name = run_command(%W(lsb_release -irs)) + os_name ||= if File.readable?('/etc/system-release') + File.read('/etc/system-release') + end + os_name ||= if File.readable?('/etc/debian_version') + debian_version = File.read('/etc/debian_version') + "Debian #{debian_version}" + end + os_name ||= if File.readable?('/etc/SuSE-release') + File.read('/etc/SuSE-release') + end + os_name ||= if os_x_version = run_command(%W(sw_vers -productVersion)) + "Mac OS X #{os_x_version}" + end + os_name ||= if File.readable?('/etc/os-release') + File.read('/etc/os-release').match(/PRETTY_NAME=\"(.+)\"/)[1] + end + os_name.try(:squish!) + end + + # Prompt the user to input something + # + # message - the message to display before input + # choices - array of strings of acceptable answers or nil for any answer + # + # Returns the user's answer + def prompt(message, choices = nil) + begin + print(message) + answer = STDIN.gets.chomp + end while choices.present? && !choices.include?(answer) + answer + end + + # Runs the given command and matches the output against the given pattern + # + # Returns nil if nothing matched + # Returns the MatchData if the pattern matched + # + # see also #run_command + # see also String#match + def run_and_match(command, regexp) + run_command(command).try(:match, regexp) + end + + # Runs the given command + # + # Returns '' if the command was not found + # Returns the output of the command otherwise + # + # see also #run_and_match + def run_command(command) + output, _ = Gitlab::Popen.popen(command) + output + rescue Errno::ENOENT + '' # if the command does not exist, return an empty string + end + + # Runs the given command and raises a Gitlab::TaskFailedError exception if + # the command does not exit with 0 + # + # Returns the output of the command otherwise + def run_command!(command) + output, status = Gitlab::Popen.popen(command) + + raise Gitlab::TaskFailedError unless status.zero? + + output + end + + def uid_for(user_name) + run_command(%W(id -u #{user_name})).chomp.to_i + end + + def gid_for(group_name) + begin + Etc.getgrnam(group_name).gid + rescue ArgumentError # no group + "group #{group_name} doesn't exist" + end + end + + def warn_user_is_not_gitlab + unless @warned_user_not_gitlab + gitlab_user = Gitlab.config.gitlab.user + current_user = run_command(%W(whoami)).chomp + unless current_user == gitlab_user + puts " Warning ".color(:black).background(:yellow) + puts " You are running as user #{current_user.color(:magenta)}, we hope you know what you are doing." + puts " Things may work\/fail for the wrong reasons." + puts " For correct results you should run this as user #{gitlab_user.color(:magenta)}." + puts "" + end + @warned_user_not_gitlab = true + end + end + + # Tries to configure git itself + # + # Returns true if all subcommands were successfull (according to their exit code) + # Returns false if any or all subcommands failed. + def auto_fix_git_config(options) + if !@warned_user_not_gitlab + command_success = options.map do |name, value| + system(*%W(#{Gitlab.config.git.bin_path} config --global #{name} #{value})) + end + + command_success.all? + else + false + end + end + + def all_repos + Gitlab.config.repositories.storages.each do |name, path| + IO.popen(%W(find #{path} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find| + find.each_line do |path| + yield path.chomp + end + end + end + end + + def repository_storage_paths_args + Gitlab.config.repositories.storages.values + end + + def user_home + Rails.env.test? ? Rails.root.join('tmp/tests') : Gitlab.config.gitlab.user_home + end + + def checkout_or_clone_tag(tag:, repo:, target_dir:) + if Dir.exist?(target_dir) + checkout_tag(tag, target_dir) + else + clone_repo(repo, target_dir) + end + + reset_to_tag(tag, target_dir) + end + + def clone_repo(repo, target_dir) + run_command!(%W[#{Gitlab.config.git.bin_path} clone -- #{repo} #{target_dir}]) + end + + def checkout_tag(tag, target_dir) + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} fetch --tags --quiet]) + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} checkout --quiet #{tag}]) + end + + def reset_to_tag(tag_wanted, target_dir) + tag = + begin + # First try to checkout without fetching + # to avoid stalling tests if the Internet is down. + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} describe -- #{tag_wanted}]) + rescue Gitlab::TaskFailedError + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} fetch origin]) + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} describe -- origin/#{tag_wanted}]) + end + + if tag + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} reset --hard #{tag.strip}]) + else + raise Gitlab::TaskFailedError + end + end + end +end diff --git a/lib/tasks/gitlab/workhorse.rake b/lib/tasks/gitlab/workhorse.rake new file mode 100644 index 00000000000..baea94bf8ca --- /dev/null +++ b/lib/tasks/gitlab/workhorse.rake @@ -0,0 +1,23 @@ +namespace :gitlab do + namespace :workhorse do + desc "GitLab | Install or upgrade gitlab-workhorse" + task :install, [:dir] => :environment do |t, args| + warn_user_is_not_gitlab + unless args.dir.present? + abort %(Please specify the directory where you want to install gitlab-workhorse:\n rake "gitlab:workhorse:install[/home/git/gitlab-workhorse]") + end + + tag = "v#{Gitlab::Workhorse.version}" + repo = 'https://gitlab.com/gitlab-org/gitlab-workhorse.git' + + checkout_or_clone_tag(tag: tag, repo: repo, target_dir: args.dir) + + _, status = Gitlab::Popen.popen(%w[which gmake]) + command = status.zero? ? 'gmake' : 'make' + + Dir.chdir(args.dir) do + run_command!([command]) + end + end + end +end diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake index 141a0b74ec0..f5caca3ddbf 100644 --- a/lib/tasks/migrate/setup_postgresql.rake +++ b/lib/tasks/migrate/setup_postgresql.rake @@ -1,8 +1,12 @@ +require Rails.root.join('lib/gitlab/database') +require Rails.root.join('lib/gitlab/database/migration_helpers') require Rails.root.join('db/migrate/20151007120511_namespaces_projects_path_lower_indexes') require Rails.root.join('db/migrate/20151008110232_add_users_lower_username_email_indexes') +require Rails.root.join('db/migrate/20161212142807_add_lower_path_index_to_routes') desc 'GitLab | Sets up PostgreSQL' task setup_postgresql: :environment do NamespacesProjectsPathLowerIndexes.new.up AddUsersLowerUsernameEmailIndexes.new.up + AddLowerPathIndexToRoutes.new.up end |