diff options
Diffstat (limited to 'lib')
547 files changed, 21465 insertions, 5896 deletions
diff --git a/lib/additional_email_headers_interceptor.rb b/lib/additional_email_headers_interceptor.rb new file mode 100644 index 00000000000..2358fa6bbfd --- /dev/null +++ b/lib/additional_email_headers_interceptor.rb @@ -0,0 +1,8 @@ +class AdditionalEmailHeadersInterceptor + def self.delivering_email(message) + message.headers( + 'Auto-Submitted' => 'auto-generated', + 'X-Auto-Response-Suppress' => 'All' + ) + end +end diff --git a/lib/api/access_requests.rb b/lib/api/access_requests.rb index d02b469dac8..789f45489eb 100644 --- a/lib/api/access_requests.rb +++ b/lib/api/access_requests.rb @@ -1,36 +1,36 @@ module API class AccessRequests < Grape::API + include PaginationParams + before { authenticate! } helpers ::API::Helpers::MembersHelpers %w[group project].each do |source_type| + params do + requires :id, type: String, desc: "The #{source_type} ID" + end resource source_type.pluralize do - # Get a list of group/project access requests viewable by the authenticated user. - # - # Parameters: - # id (required) - The group/project ID - # - # Example Request: - # GET /groups/:id/access_requests - # GET /projects/:id/access_requests + desc "Gets a list of access requests for a #{source_type}." do + detail 'This feature was introduced in GitLab 8.11.' + success Entities::AccessRequester + end + params do + use :pagination + end get ":id/access_requests" do source = find_source(source_type, params[:id]) - authorize_admin_source!(source_type, source) - access_requesters = paginate(source.requesters.includes(:user)) + access_requesters = AccessRequestsFinder.new(source).execute!(current_user) + access_requesters = paginate(access_requesters.includes(:user)) - present access_requesters.map(&:user), with: Entities::AccessRequester, access_requesters: access_requesters + present access_requesters.map(&:user), with: Entities::AccessRequester, source: source end - # Request access to the group/project - # - # Parameters: - # id (required) - The group/project ID - # - # Example Request: - # POST /groups/:id/access_requests - # POST /projects/:id/access_requests + desc "Requests access for the authenticated user to a #{source_type}." do + detail 'This feature was introduced in GitLab 8.11.' + success Entities::AccessRequester + end post ":id/access_requests" do source = find_source(source_type, params[:id]) access_requester = source.request_access(current_user) @@ -42,47 +42,34 @@ module API end end - # Approve a group/project access request - # - # Parameters: - # id (required) - The group/project ID - # user_id (required) - The user ID of the access requester - # access_level (optional) - Access level - # - # Example Request: - # PUT /groups/:id/access_requests/:user_id/approve - # PUT /projects/:id/access_requests/:user_id/approve + desc 'Approves an access request for the given user.' do + detail 'This feature was introduced in GitLab 8.11.' + success Entities::Member + end + params do + requires :user_id, type: Integer, desc: 'The user ID of the access requester' + optional :access_level, type: Integer, desc: 'A valid access level (defaults: `30`, developer access level)' + end put ':id/access_requests/:user_id/approve' do - required_attributes! [:user_id] source = find_source(source_type, params[:id]) - authorize_admin_source!(source_type, source) - member = source.requesters.find_by!(user_id: params[:user_id]) - if params[:access_level] - member.update(access_level: params[:access_level]) - end - member.accept_request + member = ::Members::ApproveAccessRequestService.new(source, current_user, declared_params).execute status :created present member.user, with: Entities::Member, member: member end - # Deny a group/project access request - # - # Parameters: - # id (required) - The group/project ID - # user_id (required) - The user ID of the access requester - # - # Example Request: - # DELETE /groups/:id/access_requests/:user_id - # DELETE /projects/:id/access_requests/:user_id + desc 'Denies an access request for the given user.' do + detail 'This feature was introduced in GitLab 8.11.' + end + params do + requires :user_id, type: Integer, desc: 'The user ID of the access requester' + end delete ":id/access_requests/:user_id" do - required_attributes! [:user_id] source = find_source(source_type, params[:id]) - access_requester = source.requesters.find_by!(user_id: params[:user_id]) - - ::Members::DestroyService.new(access_requester, current_user).execute + ::Members::DestroyService.new(source, current_user, params). + execute(:requesters) end end end diff --git a/lib/api/api.rb b/lib/api/api.rb index d43af3f24e9..7aa95a4a3c1 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -1,7 +1,35 @@ module API class API < Grape::API include APIGuard - version 'v3', using: :path + + version %w(v3 v4), using: :path + + version 'v3', using: :path do + mount ::API::V3::Boards + mount ::API::V3::Branches + mount ::API::V3::Commits + mount ::API::V3::DeployKeys + mount ::API::V3::Files + mount ::API::V3::Groups + mount ::API::V3::Issues + mount ::API::V3::Labels + mount ::API::V3::Members + mount ::API::V3::MergeRequestDiffs + mount ::API::V3::MergeRequests + mount ::API::V3::Notes + mount ::API::V3::ProjectHooks + mount ::API::V3::Projects + mount ::API::V3::ProjectSnippets + mount ::API::V3::Repositories + mount ::API::V3::Subscriptions + mount ::API::V3::SystemHooks + mount ::API::V3::Tags + mount ::API::V3::Todos + mount ::API::V3::Templates + mount ::API::V3::Users + end + + before { allow_access_with_scope :api } rescue_from Gitlab::Access::AccessDeniedError do rack_response({ 'message' => '403 Forbidden' }.to_json, 403) @@ -12,37 +40,37 @@ module API end # Retain 405 error rather than a 500 error for Grape 0.15.0+. - # See: https://github.com/ruby-grape/grape/commit/252bfd27c320466ec3c0751812cf44245e97e5de + # https://github.com/ruby-grape/grape/blob/a3a28f5b5dfbb2797442e006dbffd750b27f2a76/UPGRADING.md#changes-to-method-not-allowed-routes + rescue_from Grape::Exceptions::MethodNotAllowed do |e| + error! e.message, e.status, e.headers + end + rescue_from Grape::Exceptions::Base do |e| error! e.message, e.status, e.headers end rescue_from :all do |exception| - # lifted from https://github.com/rails/rails/blob/master/actionpack/lib/action_dispatch/middleware/debug_exceptions.rb#L60 - # why is this not wrapped in something reusable? - trace = exception.backtrace - - message = "\n#{exception.class} (#{exception.message}):\n" - message << exception.annoted_source_code.to_s if exception.respond_to?(:annoted_source_code) - message << " " << trace.join("\n ") - - API.logger.add Logger::FATAL, message - rack_response({ 'message' => '500 Internal Server Error' }.to_json, 500) + handle_api_exception(exception) end format :json content_type :txt, "text/plain" # Ensure the namespace is right, otherwise we might load Grape::API::Helpers + helpers ::SentryHelper helpers ::API::Helpers + # Keep in alphabetical order mount ::API::AccessRequests mount ::API::AwardEmoji + mount ::API::Boards mount ::API::Branches + mount ::API::BroadcastMessages mount ::API::Builds - mount ::API::CommitStatuses mount ::API::Commits + mount ::API::CommitStatuses mount ::API::DeployKeys + mount ::API::Deployments mount ::API::Environments mount ::API::Files mount ::API::Groups @@ -50,21 +78,26 @@ module API mount ::API::Issues mount ::API::Keys mount ::API::Labels - mount ::API::LicenseTemplates + mount ::API::Lint mount ::API::Members + mount ::API::MergeRequestDiffs mount ::API::MergeRequests mount ::API::Milestones mount ::API::Namespaces mount ::API::Notes + mount ::API::NotificationSettings + mount ::API::Pipelines mount ::API::ProjectHooks - mount ::API::ProjectSnippets mount ::API::Projects + mount ::API::ProjectSnippets mount ::API::Repositories + mount ::API::Runner mount ::API::Runners mount ::API::Services mount ::API::Session mount ::API::Settings mount ::API::SidekiqMetrics + mount ::API::Snippets mount ::API::Subscriptions mount ::API::SystemHooks mount ::API::Tags @@ -73,5 +106,10 @@ module API mount ::API::Triggers mount ::API::Users mount ::API::Variables + mount ::API::Version + + route :any, '*path' do + error!('404 Not Found', 404) + end end end diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index 7e67edb203a..c11f8529183 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -6,6 +6,9 @@ module API module APIGuard extend ActiveSupport::Concern + PRIVATE_TOKEN_HEADER = "HTTP_PRIVATE_TOKEN".freeze + PRIVATE_TOKEN_PARAM = :private_token + included do |base| # OAuth2 Resource Server Authentication use Rack::OAuth2::Server::Resource::Bearer, 'The API' do |request| @@ -33,55 +36,71 @@ module API # # If the token is revoked, then it raises RevokedError. # - # If the token is not found (nil), then it raises TokenNotFoundError. + # If the token is not found (nil), then it returns nil # # Arguments: # # scopes: (optional) scopes required for this guard. # Defaults to empty array. # - def doorkeeper_guard!(scopes: []) - if (access_token = find_access_token).nil? - raise TokenNotFoundError - - else - case validate_access_token(access_token, scopes) - when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE - raise InsufficientScopeError.new(scopes) - when Oauth2::AccessTokenValidationService::EXPIRED - raise ExpiredError - when Oauth2::AccessTokenValidationService::REVOKED - raise RevokedError - when Oauth2::AccessTokenValidationService::VALID - @current_user = User.find(access_token.resource_owner_id) - end - end - end - def doorkeeper_guard(scopes: []) - if access_token = find_access_token - case validate_access_token(access_token, scopes) - when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE - raise InsufficientScopeError.new(scopes) + access_token = find_access_token + return nil unless access_token - when Oauth2::AccessTokenValidationService::EXPIRED - raise ExpiredError + case AccessTokenValidationService.new(access_token).validate(scopes: scopes) + when AccessTokenValidationService::INSUFFICIENT_SCOPE + raise InsufficientScopeError.new(scopes) - when Oauth2::AccessTokenValidationService::REVOKED - raise RevokedError + when AccessTokenValidationService::EXPIRED + raise ExpiredError - when Oauth2::AccessTokenValidationService::VALID - @current_user = User.find(access_token.resource_owner_id) - end + when AccessTokenValidationService::REVOKED + raise RevokedError + + when AccessTokenValidationService::VALID + @current_user = User.find(access_token.resource_owner_id) end end + def find_user_by_private_token(scopes: []) + token_string = (params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]).to_s + + return nil unless token_string.present? + + find_user_by_authentication_token(token_string) || find_user_by_personal_access_token(token_string, scopes) + end + def current_user @current_user end + # Set the authorization scope(s) allowed for the current request. + # + # Note: A call to this method adds to any previous scopes in place. This is done because + # `Grape` callbacks run from the outside-in: the top-level callback (API::API) runs first, then + # the next-level callback (API::API::Users, for example) runs. All these scopes are valid for the + # given endpoint (GET `/api/users` is accessible by the `api` and `read_user` scopes), and so they + # need to be stored. + def allow_access_with_scope(*scopes) + @scopes ||= [] + @scopes.concat(scopes.map(&:to_s)) + end + private + def find_user_by_authentication_token(token_string) + User.find_by_authentication_token(token_string) + end + + def find_user_by_personal_access_token(token_string, scopes) + access_token = PersonalAccessToken.active.find_by_token(token_string) + return unless access_token + + if AccessTokenValidationService.new(access_token).include_any_scope?(scopes) + User.find(access_token.user_id) + end + end + def find_access_token @access_token ||= Doorkeeper.authenticate(doorkeeper_request, Doorkeeper.configuration.access_token_methods) end @@ -89,31 +108,14 @@ module API def doorkeeper_request @doorkeeper_request ||= ActionDispatch::Request.new(env) end - - def validate_access_token(access_token, scopes) - Oauth2::AccessTokenValidationService.validate(access_token, scopes: scopes) - end end module ClassMethods - # Installs the doorkeeper guard on the whole Grape API endpoint. - # - # Arguments: - # - # scopes: (optional) scopes required for this guard. - # Defaults to empty array. - # - def guard_all!(scopes: []) - before do - guard! scopes: scopes - end - end - private def install_error_responders(base) - error_classes = [ MissingTokenError, TokenNotFoundError, - ExpiredError, RevokedError, InsufficientScopeError] + error_classes = [MissingTokenError, TokenNotFoundError, + ExpiredError, RevokedError, InsufficientScopeError] base.send :rescue_from, *error_classes, oauth2_bearer_token_error_handler end diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb index 2efe7e3adf3..301271118d4 100644 --- a/lib/api/award_emoji.rb +++ b/lib/api/award_emoji.rb @@ -1,40 +1,48 @@ module API class AwardEmoji < Grape::API + include PaginationParams + before { authenticate! } - AWARDABLES = [Issue, MergeRequest] + AWARDABLES = %w[issue merge_request snippet].freeze resource :projects do AWARDABLES.each do |awardable_type| - awardable_string = awardable_type.to_s.underscore.pluralize - awardable_id_string = "#{awardable_type.to_s.underscore}_id" + awardable_string = awardable_type.pluralize + awardable_id_string = "#{awardable_type}_id" + + params do + requires :id, type: String, desc: 'The ID of a project' + requires :"#{awardable_id_string}", type: Integer, desc: "The ID of an Issue, Merge Request or Snippet" + end - [ ":id/#{awardable_string}/:#{awardable_id_string}/award_emoji", + [ + ":id/#{awardable_string}/:#{awardable_id_string}/award_emoji", ":id/#{awardable_string}/:#{awardable_id_string}/notes/:note_id/award_emoji" ].each do |endpoint| - # Get a list of project +awardable+ award emoji - # - # Parameters: - # id (required) - The ID of a project - # awardable_id (required) - The ID of an issue or MR - # Example Request: - # GET /projects/:id/issues/:awardable_id/award_emoji + + desc 'Get a list of project +awardable+ award emoji' do + detail 'This feature was introduced in 8.9' + success Entities::AwardEmoji + end + params do + use :pagination + end get endpoint do if can_read_awardable? - awards = paginate(awardable.award_emoji) - present awards, with: Entities::AwardEmoji + awards = awardable.award_emoji + present paginate(awards), with: Entities::AwardEmoji else not_found!("Award Emoji") end end - # Get a specific award emoji - # - # Parameters: - # id (required) - The ID of a project - # awardable_id (required) - The ID of an issue or MR - # award_id (required) - The ID of the award - # Example Request: - # GET /projects/:id/issues/:awardable_id/award_emoji/:award_id + desc 'Get a specific award emoji' do + detail 'This feature was introduced in 8.9' + success Entities::AwardEmoji + end + params do + requires :award_id, type: Integer, desc: 'The ID of the award' + end get "#{endpoint}/:award_id" do if can_read_awardable? present awardable.award_emoji.find(params[:award_id]), with: Entities::AwardEmoji @@ -43,18 +51,15 @@ module API end end - # Award a new Emoji - # - # Parameters: - # id (required) - The ID of a project - # awardable_id (required) - The ID of an issue or mr - # name (required) - The name of a award_emoji (without colons) - # Example Request: - # POST /projects/:id/issues/:awardable_id/award_emoji + desc 'Award a new Emoji' do + detail 'This feature was introduced in 8.9' + success Entities::AwardEmoji + end + params do + requires :name, type: String, desc: 'The name of a award_emoji (without colons)' + end post endpoint do - required_attributes! [:name] - - not_found!('Award Emoji') unless can_read_awardable? + not_found!('Award Emoji') unless can_read_awardable? && can_award_awardable? award = awardable.create_award_emoji(params[:name], current_user) @@ -65,14 +70,13 @@ module API end end - # Delete a +awardables+ award emoji - # - # Parameters: - # id (required) - The ID of a project - # awardable_id (required) - The ID of an issue or MR - # award_emoji_id (required) - The ID of an award emoji - # Example Request: - # DELETE /projects/:id/issues/:issue_id/notes/:note_id/award_emoji/:award_id + desc 'Delete a +awardables+ award emoji' do + detail 'This feature was introduced in 8.9' + success Entities::AwardEmoji + end + params do + requires :award_id, type: Integer, desc: 'The ID of an award emoji' + end delete "#{endpoint}/:award_id" do award = awardable.award_emoji.find(params[:award_id]) @@ -87,27 +91,36 @@ module API helpers do def can_read_awardable? - ability = "read_#{awardable.class.to_s.underscore}".to_sym + can?(current_user, read_ability(awardable), awardable) + end - can?(current_user, ability, awardable) + def can_award_awardable? + awardable.user_can_award?(current_user, params[:name]) end def awardable @awardable ||= begin if params.include?(:note_id) - noteable.notes.find(params[:note_id]) + note_id = params.delete(:note_id) + + awardable.notes.find(note_id) + elsif params.include?(:issue_id) + user_project.issues.find(params[:issue_id]) + elsif params.include?(:merge_request_id) + user_project.merge_requests.find(params[:merge_request_id]) else - noteable + user_project.snippets.find(params[:snippet_id]) end end end - def noteable - if params.include?(:issue_id) - user_project.issues.find(params[:issue_id]) + def read_ability(awardable) + case awardable + when Note + read_ability(awardable.noteable) else - user_project.merge_requests.find(params[:merge_request_id]) + :"read_#{awardable.class.to_s.underscore}" end end end diff --git a/lib/api/boards.rb b/lib/api/boards.rb new file mode 100644 index 00000000000..f4226e5a89d --- /dev/null +++ b/lib/api/boards.rb @@ -0,0 +1,139 @@ +module API + class Boards < Grape::API + include PaginationParams + + before { authenticate! } + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects do + desc 'Get all project boards' do + detail 'This feature was introduced in 8.13' + success Entities::Board + end + params do + use :pagination + end + get ':id/boards' do + authorize!(:read_board, user_project) + present paginate(user_project.boards), with: Entities::Board + end + + params do + requires :board_id, type: Integer, desc: 'The ID of a board' + end + segment ':id/boards/:board_id' do + helpers do + def project_board + board = user_project.boards.first + + if params[:board_id] == board.id + board + else + not_found!('Board') + end + end + + def board_lists + project_board.lists.destroyable + end + end + + desc 'Get the lists of a project board' do + detail 'Does not include `done` list. This feature was introduced in 8.13' + success Entities::List + end + params do + use :pagination + end + get '/lists' do + authorize!(:read_board, user_project) + present paginate(board_lists), with: Entities::List + end + + desc 'Get a list of a project board' do + detail 'This feature was introduced in 8.13' + success Entities::List + end + params do + requires :list_id, type: Integer, desc: 'The ID of a list' + end + get '/lists/:list_id' do + authorize!(:read_board, user_project) + present board_lists.find(params[:list_id]), with: Entities::List + end + + desc 'Create a new board list' do + detail 'This feature was introduced in 8.13' + success Entities::List + end + params do + requires :label_id, type: Integer, desc: 'The ID of an existing label' + end + post '/lists' do + unless available_labels.exists?(params[:label_id]) + render_api_error!({ error: 'Label not found!' }, 400) + end + + authorize!(:admin_list, user_project) + + service = ::Boards::Lists::CreateService.new(user_project, current_user, + { label_id: params[:label_id] }) + + list = service.execute(project_board) + + if list.valid? + present list, with: Entities::List + else + render_validation_error!(list) + end + end + + desc 'Moves a board list to a new position' do + detail 'This feature was introduced in 8.13' + success Entities::List + end + params do + requires :list_id, type: Integer, desc: 'The ID of a list' + requires :position, type: Integer, desc: 'The position of the list' + end + put '/lists/:list_id' do + list = project_board.lists.movable.find(params[:list_id]) + + authorize!(:admin_list, user_project) + + service = ::Boards::Lists::MoveService.new(user_project, current_user, + { position: params[:position] }) + + if service.execute(list) + present list, with: Entities::List + else + render_api_error!({ error: "List could not be moved!" }, 400) + end + end + + desc 'Delete a board list' do + detail 'This feature was introduced in 8.13' + success Entities::List + end + params do + requires :list_id, type: Integer, desc: 'The ID of a board list' + end + delete "/lists/:list_id" do + authorize!(:admin_list, user_project) + + list = board_lists.find(params[:list_id]) + + service = ::Boards::Lists::DestroyService.new(user_project, current_user) + + if service.execute(list) + present list, with: Entities::List + else + render_api_error!({ error: 'List could not be deleted!' }, 400) + end + end + end + end + end +end diff --git a/lib/api/branches.rb b/lib/api/branches.rb index a77afe634f6..34f136948c2 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -1,124 +1,109 @@ require 'mime/types' module API - # Projects API class Branches < Grape::API + include PaginationParams + before { authenticate! } before { authorize! :download_code, user_project } + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do - # Get a project repository branches - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # GET /projects/:id/repository/branches + desc 'Get a project repository branches' do + success Entities::RepoBranch + end + params do + use :pagination + end get ":id/repository/branches" do - branches = user_project.repository.branches.sort_by(&:name) + branches = ::Kaminari.paginate_array(user_project.repository.branches.sort_by(&:name)) - present branches, with: Entities::RepoBranch, project: user_project + present paginate(branches), with: Entities::RepoBranch, project: user_project end - # Get a single branch - # - # Parameters: - # id (required) - The ID of a project - # branch (required) - The name of the branch - # Example Request: - # GET /projects/:id/repository/branches/:branch + desc 'Get a single branch' do + success Entities::RepoBranch + end + params do + requires :branch, type: String, desc: 'The name of the branch' + end get ':id/repository/branches/:branch', requirements: { branch: /.+/ } do - @branch = user_project.repository.branches.find { |item| item.name == params[:branch] } - not_found!("Branch") unless @branch + branch = user_project.repository.find_branch(params[:branch]) + not_found!("Branch") unless branch - present @branch, with: Entities::RepoBranch, project: user_project + present branch, with: Entities::RepoBranch, project: user_project end - # Protect a single branch - # # Note: The internal data model moved from `developers_can_{merge,push}` to `allowed_to_{merge,push}` # in `gitlab-org/gitlab-ce!5081`. The API interface has not been changed (to maintain compatibility), # but it works with the changed data model to infer `developers_can_merge` and `developers_can_push`. - # - # Parameters: - # id (required) - The ID of a project - # branch (required) - The name of the branch - # developers_can_push (optional) - Flag if developers can push to that branch - # developers_can_merge (optional) - Flag if developers can merge to that branch - # Example Request: - # PUT /projects/:id/repository/branches/:branch/protect - put ':id/repository/branches/:branch/protect', - requirements: { branch: /.+/ } do + desc 'Protect a single branch' do + success Entities::RepoBranch + end + params do + requires :branch, type: String, desc: 'The name of the branch' + optional :developers_can_push, type: Boolean, desc: 'Flag if developers can push to that branch' + optional :developers_can_merge, type: Boolean, desc: 'Flag if developers can merge to that branch' + end + put ':id/repository/branches/:branch/protect', requirements: { branch: /.+/ } do authorize_admin_project - @branch = user_project.repository.find_branch(params[:branch]) - not_found!('Branch') unless @branch - protected_branch = user_project.protected_branches.find_by(name: @branch.name) + branch = user_project.repository.find_branch(params[:branch]) + not_found!('Branch') unless branch - developers_can_merge = to_boolean(params[:developers_can_merge]) - developers_can_push = to_boolean(params[:developers_can_push]) + protected_branch = user_project.protected_branches.find_by(name: branch.name) protected_branch_params = { - name: @branch.name + name: branch.name, + developers_can_push: params[:developers_can_push], + developers_can_merge: params[:developers_can_merge] } - unless developers_can_merge.nil? - protected_branch_params.merge!({ - merge_access_level_attributes: { - access_level: developers_can_merge ? Gitlab::Access::DEVELOPER : Gitlab::Access::MASTER - } - }) - end + service_args = [user_project, current_user, protected_branch_params] - unless developers_can_push.nil? - protected_branch_params.merge!({ - push_access_level_attributes: { - access_level: developers_can_push ? Gitlab::Access::DEVELOPER : Gitlab::Access::MASTER - } - }) - end + protected_branch = if protected_branch + ProtectedBranches::ApiUpdateService.new(*service_args).execute(protected_branch) + else + ProtectedBranches::ApiCreateService.new(*service_args).execute + end - if protected_branch - service = ProtectedBranches::UpdateService.new(user_project, current_user, protected_branch_params) - service.execute(protected_branch) + if protected_branch.valid? + present branch, with: Entities::RepoBranch, project: user_project else - service = ProtectedBranches::CreateService.new(user_project, current_user, protected_branch_params) - service.execute + render_api_error!(protected_branch.errors.full_messages, 422) end - - present @branch, with: Entities::RepoBranch, project: user_project end - # Unprotect a single branch - # - # Parameters: - # id (required) - The ID of a project - # branch (required) - The name of the branch - # Example Request: - # PUT /projects/:id/repository/branches/:branch/unprotect - put ':id/repository/branches/:branch/unprotect', - requirements: { branch: /.+/ } do + desc 'Unprotect a single branch' do + success Entities::RepoBranch + end + params do + requires :branch, type: String, desc: 'The name of the branch' + end + put ':id/repository/branches/:branch/unprotect', requirements: { branch: /.+/ } do authorize_admin_project - @branch = user_project.repository.find_branch(params[:branch]) - not_found!("Branch") unless @branch - protected_branch = user_project.protected_branches.find_by(name: @branch.name) - protected_branch.destroy if protected_branch + branch = user_project.repository.find_branch(params[:branch]) + not_found!("Branch") unless branch + protected_branch = user_project.protected_branches.find_by(name: branch.name) + protected_branch&.destroy - present @branch, with: Entities::RepoBranch, project: user_project + present branch, with: Entities::RepoBranch, project: user_project end - # Create branch - # - # Parameters: - # id (required) - The ID of a project - # branch_name (required) - The name of the branch - # ref (required) - Create branch from commit sha or existing branch - # Example Request: - # POST /projects/:id/repository/branches + desc 'Create branch' do + success Entities::RepoBranch + end + params do + requires :branch, type: String, desc: 'The name of the branch' + requires :ref, type: String, desc: 'Create branch from commit sha or existing branch' + end post ":id/repository/branches" do authorize_push_project result = CreateBranchService.new(user_project, current_user). - execute(params[:branch_name], params[:ref]) + execute(params[:branch], params[:ref]) if result[:status] == :success present result[:branch], @@ -129,27 +114,31 @@ module API end end - # Delete branch - # - # Parameters: - # id (required) - The ID of a project - # branch (required) - The name of the branch - # Example Request: - # DELETE /projects/:id/repository/branches/:branch - delete ":id/repository/branches/:branch", - requirements: { branch: /.+/ } do + desc 'Delete a branch' + params do + requires :branch, type: String, desc: 'The name of the branch' + end + delete ":id/repository/branches/:branch", requirements: { branch: /.+/ } do authorize_push_project + result = DeleteBranchService.new(user_project, current_user). - execute(params[:branch]) + execute(params[:branch]) if result[:status] == :success { - branch_name: params[:branch] + branch: params[:branch] } else render_api_error!(result[:message], result[:return_code]) end end + + desc 'Delete all merged branches' + delete ":id/repository/merged_branches" do + DeleteMergedBranchesService.new(user_project, current_user).async_execute + + accepted! + end end end end diff --git a/lib/api/broadcast_messages.rb b/lib/api/broadcast_messages.rb new file mode 100644 index 00000000000..1217002bf8e --- /dev/null +++ b/lib/api/broadcast_messages.rb @@ -0,0 +1,98 @@ +module API + class BroadcastMessages < Grape::API + include PaginationParams + + before { authenticate! } + before { authenticated_as_admin! } + + resource :broadcast_messages do + helpers do + def find_message + BroadcastMessage.find(params[:id]) + end + end + + desc 'Get all broadcast messages' do + detail 'This feature was introduced in GitLab 8.12.' + success Entities::BroadcastMessage + end + params do + use :pagination + end + get do + messages = BroadcastMessage.all + + present paginate(messages), with: Entities::BroadcastMessage + end + + desc 'Create a broadcast message' do + detail 'This feature was introduced in GitLab 8.12.' + success Entities::BroadcastMessage + end + params do + requires :message, type: String, desc: 'Message to display' + optional :starts_at, type: DateTime, desc: 'Starting time', default: -> { Time.zone.now } + optional :ends_at, type: DateTime, desc: 'Ending time', default: -> { 1.hour.from_now } + optional :color, type: String, desc: 'Background color' + optional :font, type: String, desc: 'Foreground color' + end + post do + message = BroadcastMessage.create(declared_params(include_missing: false)) + + if message.persisted? + present message, with: Entities::BroadcastMessage + else + render_validation_error!(message) + end + end + + desc 'Get a specific broadcast message' do + detail 'This feature was introduced in GitLab 8.12.' + success Entities::BroadcastMessage + end + params do + requires :id, type: Integer, desc: 'Broadcast message ID' + end + get ':id' do + message = find_message + + present message, with: Entities::BroadcastMessage + end + + desc 'Update a broadcast message' do + detail 'This feature was introduced in GitLab 8.12.' + success Entities::BroadcastMessage + end + params do + requires :id, type: Integer, desc: 'Broadcast message ID' + optional :message, type: String, desc: 'Message to display' + optional :starts_at, type: DateTime, desc: 'Starting time' + optional :ends_at, type: DateTime, desc: 'Ending time' + optional :color, type: String, desc: 'Background color' + optional :font, type: String, desc: 'Foreground color' + end + put ':id' do + message = find_message + + if message.update(declared_params(include_missing: false)) + present message, with: Entities::BroadcastMessage + else + render_validation_error!(message) + end + end + + desc 'Delete a broadcast message' do + detail 'This feature was introduced in GitLab 8.12.' + success Entities::BroadcastMessage + end + params do + requires :id, type: Integer, desc: 'Broadcast message ID' + end + delete ':id' do + message = find_message + + present message.destroy, with: Entities::BroadcastMessage + end + end + end +end diff --git a/lib/api/builds.rb b/lib/api/builds.rb index be5a3484ec8..5b76913fe45 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -1,17 +1,36 @@ module API - # Projects builds API class Builds < Grape::API + include PaginationParams + before { authenticate! } + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do - # Get a project builds - # - # Parameters: - # id (required) - The ID of a project - # scope (optional) - The scope of builds to show (one or array of: pending, running, failed, success, canceled; - # if none provided showing all builds) - # Example Request: - # GET /projects/:id/builds + helpers do + params :optional_scope do + optional :scope, types: [String, Array[String]], desc: 'The scope of builds to show', + values: ::CommitStatus::AVAILABLE_STATUSES, + coerce_with: ->(scope) { + if scope.is_a?(String) + [scope] + elsif scope.is_a?(Hashie::Mash) + scope.values + else + ['unknown'] + end + } + end + end + + desc 'Get a project builds' do + success Entities::Build + end + params do + use :optional_scope + use :pagination + end get ':id/builds' do builds = user_project.builds.order('id DESC') builds = filter_builds(builds, params[:scope]) @@ -20,15 +39,14 @@ module API user_can_download_artifacts: can?(current_user, :read_build, user_project) end - # Get builds for a specific commit of a project - # - # Parameters: - # id (required) - The ID of a project - # sha (required) - The SHA id of a commit - # scope (optional) - The scope of builds to show (one or array of: pending, running, failed, success, canceled; - # if none provided showing all builds) - # Example Request: - # GET /projects/:id/repository/commits/:sha/builds + desc 'Get builds for a specific commit of a project' do + success Entities::Build + end + params do + requires :sha, type: String, desc: 'The SHA id of a commit' + use :optional_scope + use :pagination + end get ':id/repository/commits/:sha/builds' do authorize_read_builds! @@ -42,13 +60,12 @@ module API user_can_download_artifacts: can?(current_user, :read_build, user_project) end - # Get a specific build of a project - # - # Parameters: - # id (required) - The ID of a project - # build_id (required) - The ID of a build - # Example Request: - # GET /projects/:id/builds/:build_id + desc 'Get a specific build of a project' do + success Entities::Build + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end get ':id/builds/:build_id' do authorize_read_builds! @@ -58,13 +75,12 @@ module API user_can_download_artifacts: can?(current_user, :read_build, user_project) end - # Download the artifacts file from build - # - # Parameters: - # id (required) - The ID of a build - # token (required) - The build authorization token - # Example Request: - # GET /projects/:id/builds/:build_id/artifacts + desc 'Download the artifacts file from build' do + detail 'This feature was introduced in GitLab 8.5' + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end get ':id/builds/:build_id/artifacts' do authorize_read_builds! @@ -73,14 +89,13 @@ module API present_artifacts!(build.artifacts_file) end - # Download the artifacts file from ref_name and job - # - # Parameters: - # id (required) - The ID of a project - # ref_name (required) - The ref from repository - # job (required) - The name for the build - # Example Request: - # GET /projects/:id/builds/artifacts/:ref_name/download?job=name + desc 'Download the artifacts file from build' do + detail 'This feature was introduced in GitLab 8.10' + end + params do + requires :ref_name, type: String, desc: 'The ref from repository' + requires :job, type: String, desc: 'The name for the build' + end get ':id/builds/artifacts/:ref_name/download', requirements: { ref_name: /.+/ } do authorize_read_builds! @@ -91,17 +106,13 @@ module API present_artifacts!(latest_build.artifacts_file) end - # Get a trace of a specific build of a project - # - # Parameters: - # id (required) - The ID of a project - # build_id (required) - The ID of a build - # Example Request: - # GET /projects/:id/build/:build_id/trace - # # TODO: We should use `present_file!` and leave this implementation for backward compatibility (when build trace # is saved in the DB instead of file). But before that, we need to consider how to replace the value of # `runners_token` with some mask (like `xxxxxx`) when sending trace file directly by workhorse. + desc 'Get a trace of a specific build of a project' + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end get ':id/builds/:build_id/trace' do authorize_read_builds! @@ -115,13 +126,12 @@ module API body trace end - # Cancel a specific build of a project - # - # parameters: - # id (required) - the id of a project - # build_id (required) - the id of a build - # example request: - # post /projects/:id/build/:build_id/cancel + desc 'Cancel a specific build of a project' do + success Entities::Build + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end post ':id/builds/:build_id/cancel' do authorize_update_builds! @@ -133,13 +143,12 @@ module API user_can_download_artifacts: can?(current_user, :read_build, user_project) end - # Retry a specific build of a project - # - # parameters: - # id (required) - the id of a project - # build_id (required) - the id of a build - # example request: - # post /projects/:id/build/:build_id/retry + desc 'Retry a specific build of a project' do + success Entities::Build + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end post ':id/builds/:build_id/retry' do authorize_update_builds! @@ -152,13 +161,12 @@ module API user_can_download_artifacts: can?(current_user, :read_build, user_project) end - # Erase build (remove artifacts and build trace) - # - # Parameters: - # id (required) - the id of a project - # build_id (required) - the id of a build - # example Request: - # post /projects/:id/build/:build_id/erase + desc 'Erase build (remove artifacts and build trace)' do + success Entities::Build + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end post ':id/builds/:build_id/erase' do authorize_update_builds! @@ -170,13 +178,12 @@ module API user_can_download_artifacts: can?(current_user, :download_build_artifacts, user_project) end - # Keep the artifacts to prevent them from being deleted - # - # Parameters: - # id (required) - the id of a project - # build_id (required) - The ID of a build - # Example Request: - # POST /projects/:id/builds/:build_id/artifacts/keep + desc 'Keep the artifacts to prevent them from being deleted' do + success Entities::Build + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end post ':id/builds/:build_id/artifacts/keep' do authorize_update_builds! @@ -189,6 +196,27 @@ module API present build, with: Entities::Build, user_can_download_artifacts: can?(current_user, :read_build, user_project) end + + desc 'Trigger a manual build' do + success Entities::Build + detail 'This feature was added in GitLab 8.11' + end + params do + requires :build_id, type: Integer, desc: 'The ID of a Build' + end + post ":id/builds/:build_id/play" do + authorize_read_builds! + + build = get_build!(params[:build_id]) + + bad_request!("Unplayable Job") unless build.playable? + + build.play(current_user) + + status 200 + present build, with: Entities::Build, + user_can_download_artifacts: can?(current_user, :read_build, user_project) + end end helpers do @@ -214,14 +242,6 @@ module API return builds if scope.nil? || scope.empty? available_statuses = ::CommitStatus::AVAILABLE_STATUSES - scope = - if scope.is_a?(String) - [scope] - elsif scope.is_a?(Hashie::Mash) - scope.values - else - ['unknown'] - end unknown = scope - available_statuses render_api_error!('Scope contains invalid value(s)', 400) unless unknown.empty? diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index 4df6ca8333e..dba0831664c 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -1,22 +1,24 @@ require 'mime/types' module API - # Project commit statuses API class CommitStatuses < Grape::API resource :projects do + include PaginationParams + before { authenticate! } - # Get a commit's statuses - # - # Parameters: - # id (required) - The ID of a project - # sha (required) - The commit hash - # ref (optional) - The ref - # stage (optional) - The stage - # name (optional) - The name - # all (optional) - Show all statuses, default: false - # Examples: - # GET /projects/:id/repository/commits/:sha/statuses + desc "Get a commit's statuses" do + success Entities::CommitStatus + end + params do + requires :id, type: String, desc: 'The ID of a project' + requires :sha, type: String, desc: 'The commit hash' + optional :ref, type: String, desc: 'The ref' + optional :stage, type: String, desc: 'The stage' + optional :name, type: String, desc: 'The name' + optional :all, type: String, desc: 'Show all statuses, default: false' + use :pagination + end get ':id/repository/commits/:sha/statuses' do authorize!(:read_commit_status, user_project) @@ -31,22 +33,24 @@ module API present paginate(statuses), with: Entities::CommitStatus end - # Post status to commit - # - # Parameters: - # id (required) - The ID of a project - # sha (required) - The commit hash - # ref (optional) - The ref - # state (required) - The state of the status. Can be: pending, running, success, error or failure - # target_url (optional) - The target URL to associate with this status - # description (optional) - A short description of the status - # name or context (optional) - A string label to differentiate this status from the status of other systems. Default: "default" - # Examples: - # POST /projects/:id/statuses/:sha + desc 'Post status to a commit' do + success Entities::CommitStatus + end + params do + requires :id, type: String, desc: 'The ID of a project' + requires :sha, type: String, desc: 'The commit hash' + requires :state, type: String, desc: 'The state of the status', + values: %w(pending running success failed canceled) + optional :ref, type: String, desc: 'The ref' + optional :target_url, type: String, desc: 'The target URL to associate with this status' + optional :description, type: String, desc: 'A short description of the status' + optional :name, type: String, desc: 'A string label to differentiate this status from the status of other systems. Default: "default"' + optional :context, type: String, desc: 'A string label to differentiate this status from the status of other systems. Default: "default"' + optional :coverage, type: Float, desc: 'The total code coverage' + end post ':id/statuses/:sha' do authorize! :create_commit_status, user_project - required_attributes! [:state] - attrs = attributes_for_keys [:ref, :target_url, :description, :context, :name] + commit = @project.commit(params[:sha]) not_found! 'Commit' unless commit @@ -58,36 +62,46 @@ module API # the first found branch on that commit ref = params[:ref] - unless ref - branches = @project.repository.branch_names_contains(commit.sha) - not_found! 'References for commit' if branches.none? - ref = branches.first - end + ref ||= @project.repository.branch_names_contains(commit.sha).first + not_found! 'References for commit' unless ref - pipeline = @project.ensure_pipeline(commit.sha, ref, current_user) + name = params[:name] || params[:context] || 'default' - name = params[:name] || params[:context] - status = GenericCommitStatus.running_or_pending.find_by(pipeline: pipeline, name: name, ref: params[:ref]) - status ||= GenericCommitStatus.new(project: @project, pipeline: pipeline, user: current_user) - status.update(attrs) + pipeline = @project.ensure_pipeline(ref, commit.sha, current_user) - case params[:state].to_s - when 'running' - status.run - when 'success' - status.success - when 'failed' - status.drop - when 'canceled' - status.cancel - else - status.status = params[:state].to_s - end + status = GenericCommitStatus.running_or_pending.find_or_initialize_by( + project: @project, + pipeline: pipeline, + user: current_user, + name: name, + ref: ref, + target_url: params[:target_url], + description: params[:description], + coverage: params[:coverage] + ) + + render_validation_error!(status) if status.invalid? + + begin + case params[:state] + when 'pending' + status.enqueue! + when 'running' + status.enqueue + status.run! + when 'success' + status.success! + when 'failed' + status.drop! + when 'canceled' + status.cancel! + else + render_api_error!('invalid state', 400) + end - if status.save present status, with: Entities::CommitStatus - else - render_validation_error!(status) + rescue StateMachines::InvalidTransition => e + render_api_error!(e.message, 400) end end end diff --git a/lib/api/commits.rb b/lib/api/commits.rb index b4eaf1813d4..fd03e92264d 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -1,107 +1,175 @@ require 'mime/types' module API - # Projects commits API class Commits < Grape::API + include PaginationParams + before { authenticate! } before { authorize! :download_code, user_project } + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do - # Get a project repository commits - # - # Parameters: - # id (required) - The ID of a project - # ref_name (optional) - The name of a repository branch or tag, if not given the default branch is used - # since (optional) - Only commits after or in this date will be returned - # until (optional) - Only commits before or in this date will be returned - # Example Request: - # GET /projects/:id/repository/commits + desc 'Get a project repository commits' do + success Entities::RepoCommit + end + params do + optional :ref_name, type: String, desc: 'The name of a repository branch or tag, if not given the default branch is used' + optional :since, type: DateTime, desc: 'Only commits after or on this date will be returned' + optional :until, type: DateTime, desc: 'Only commits before or on this date will be returned' + optional :page, type: Integer, default: 0, desc: 'The page for pagination' + optional :per_page, type: Integer, default: 20, desc: 'The number of results per page' + optional :path, type: String, desc: 'The file path' + end get ":id/repository/commits" do - datetime_attributes! :since, :until - - page = (params[:page] || 0).to_i - per_page = (params[:per_page] || 20).to_i ref = params[:ref_name] || user_project.try(:default_branch) || 'master' - after = params[:since] - before = params[:until] + offset = params[:page] * params[:per_page] + + commits = user_project.repository.commits(ref, + path: params[:path], + limit: params[:per_page], + offset: offset, + after: params[:since], + before: params[:until]) - commits = user_project.repository.commits(ref, limit: per_page, offset: page * per_page, after: after, before: before) present commits, with: Entities::RepoCommit end - # Get a specific commit of a project - # - # Parameters: - # id (required) - The ID of a project - # sha (required) - The commit hash or name of a repository branch or tag - # Example Request: - # GET /projects/:id/repository/commits/:sha + desc 'Commit multiple file changes as one commit' do + success Entities::RepoCommitDetail + detail 'This feature was introduced in GitLab 8.13' + end + params do + requires :branch, type: String, desc: 'The name of branch' + requires :commit_message, type: String, desc: 'Commit message' + requires :actions, type: Array[Hash], desc: 'Actions to perform in commit' + optional :author_email, type: String, desc: 'Author email for commit' + optional :author_name, type: String, desc: 'Author name for commit' + end + post ":id/repository/commits" do + authorize! :push_code, user_project + + attrs = declared_params.merge(start_branch: declared_params[:branch], target_branch: declared_params[:branch]) + + result = ::Files::MultiService.new(user_project, current_user, attrs).execute + + if result[:status] == :success + commit_detail = user_project.repository.commits(result[:result], limit: 1).first + present commit_detail, with: Entities::RepoCommitDetail + else + render_api_error!(result[:message], 400) + end + end + + desc 'Get a specific commit of a project' do + success Entities::RepoCommitDetail + failure [[404, 'Not Found']] + end + params do + requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' + end get ":id/repository/commits/:sha" do - sha = params[:sha] - commit = user_project.commit(sha) + commit = user_project.commit(params[:sha]) + not_found! "Commit" unless commit + present commit, with: Entities::RepoCommitDetail end - # Get the diff for a specific commit of a project - # - # Parameters: - # id (required) - The ID of a project - # sha (required) - The commit or branch name - # Example Request: - # GET /projects/:id/repository/commits/:sha/diff + desc 'Get the diff for a specific commit of a project' do + failure [[404, 'Not Found']] + end + params do + requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' + end get ":id/repository/commits/:sha/diff" do - sha = params[:sha] - commit = user_project.commit(sha) + commit = user_project.commit(params[:sha]) + not_found! "Commit" unless commit + commit.raw_diffs.to_a end - # Get a commit's comments - # - # Parameters: - # id (required) - The ID of a project - # sha (required) - The commit hash - # Examples: - # GET /projects/:id/repository/commits/:sha/comments + desc "Get a commit's comments" do + success Entities::CommitNote + failure [[404, 'Not Found']] + end + params do + use :pagination + requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' + end get ':id/repository/commits/:sha/comments' do - sha = params[:sha] - commit = user_project.commit(sha) + commit = user_project.commit(params[:sha]) + not_found! 'Commit' unless commit - notes = Note.where(commit_id: commit.id).order(:created_at) + notes = user_project.notes.where(commit_id: commit.id).order(:created_at) + present paginate(notes), with: Entities::CommitNote end - # Post comment to commit - # - # Parameters: - # id (required) - The ID of a project - # sha (required) - The commit hash - # note (required) - Text of comment - # path (optional) - The file path - # line (optional) - The line number - # line_type (optional) - The type of line (new or old) - # Examples: - # POST /projects/:id/repository/commits/:sha/comments - post ':id/repository/commits/:sha/comments' do - required_attributes! [:note] + desc 'Cherry pick commit into a branch' do + detail 'This feature was introduced in GitLab 8.15' + success Entities::RepoCommit + end + params do + requires :sha, type: String, desc: 'A commit sha to be cherry picked' + requires :branch, type: String, desc: 'The name of the branch' + end + post ':id/repository/commits/:sha/cherry_pick' do + authorize! :push_code, user_project + + commit = user_project.commit(params[:sha]) + not_found!('Commit') unless commit + + branch = user_project.repository.find_branch(params[:branch]) + not_found!('Branch') unless branch + + commit_params = { + commit: commit, + create_merge_request: false, + target_branch: params[:branch] + } + + result = ::Commits::CherryPickService.new(user_project, current_user, commit_params).execute + + if result[:status] == :success + branch = user_project.repository.find_branch(params[:branch]) + present user_project.repository.commit(branch.dereferenced_target), with: Entities::RepoCommit + else + render_api_error!(result[:message], 400) + end + end - sha = params[:sha] - commit = user_project.commit(sha) + desc 'Post comment to commit' do + success Entities::CommitNote + end + params do + requires :sha, type: String, regexp: /\A\h{6,40}\z/, desc: "The commit's SHA" + requires :note, type: String, desc: 'The text of the comment' + optional :path, type: String, desc: 'The file path' + given :path do + requires :line, type: Integer, desc: 'The line number' + requires :line_type, type: String, values: %w(new old), default: 'new', desc: 'The type of the line' + end + end + post ':id/repository/commits/:sha/comments' do + commit = user_project.commit(params[:sha]) not_found! 'Commit' unless commit + opts = { note: params[:note], noteable_type: 'Commit', commit_id: commit.id } - if params[:path] && params[:line] && params[:line_type] + if params[:path] commit.raw_diffs(all_diffs: true).each do |diff| next unless diff.new_path == params[:path] lines = Gitlab::Diff::Parser.new.parse(diff.diff.each_line) lines.each do |line| - next unless line.new_pos == params[:line].to_i && line.type == params[:line_type] + next unless line.new_pos == params[:line] && line.type == params[:line_type] break opts[:line_code] = Gitlab::Diff::LineCode.generate(diff.new_path, line.new_pos, line.old_pos) end diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb index 825e05fbae3..69e85c27a65 100644 --- a/lib/api/deploy_keys.rb +++ b/lib/api/deploy_keys.rb @@ -1,13 +1,17 @@ module API - # Projects API class DeployKeys < Grape::API + include PaginationParams + before { authenticate! } + desc 'Return all deploy keys' + params do + use :pagination + end get "deploy_keys" do authenticated_as_admin! - keys = DeployKey.all - present keys, with: Entities::SSHKey + present paginate(DeployKey.all), with: Entities::SSHKey end params do @@ -16,100 +20,90 @@ module API resource :projects do before { authorize_admin_project } - # Routing "projects/:id/keys/..." is DEPRECATED and WILL BE REMOVED in version 9.0 - # Use "projects/:id/deploy_keys/..." instead. - # - %w(keys deploy_keys).each do |path| - desc "Get a specific project's deploy keys" do - success Entities::SSHKey - end - get ":id/#{path}" do - present user_project.deploy_keys, with: Entities::SSHKey - end - - desc 'Get single deploy key' do - success Entities::SSHKey - end - params do - requires :key_id, type: Integer, desc: 'The ID of the deploy key' - end - get ":id/#{path}/:key_id" do - key = user_project.deploy_keys.find params[:key_id] - present key, with: Entities::SSHKey - end - - # TODO: for 9.0 we should check if params are there with the params block - # grape provides, at this point we'd change behaviour so we can't - # Behaviour now if you don't provide all required params: it renders a - # validation error or two. - desc 'Add new deploy key to currently authenticated user' do - success Entities::SSHKey - end - post ":id/#{path}" do - attrs = attributes_for_keys [:title, :key] - attrs[:key].strip! if attrs[:key] - - key = user_project.deploy_keys.find_by(key: attrs[:key]) - present key, with: Entities::SSHKey if key + desc "Get a specific project's deploy keys" do + success Entities::SSHKey + end + params do + use :pagination + end + get ":id/deploy_keys" do + present paginate(user_project.deploy_keys), with: Entities::SSHKey + end - # Check for available deploy keys in other projects - key = current_user.accessible_deploy_keys.find_by(key: attrs[:key]) - if key - user_project.deploy_keys << key - present key, with: Entities::SSHKey - end + desc 'Get single deploy key' do + success Entities::SSHKey + end + params do + requires :key_id, type: Integer, desc: 'The ID of the deploy key' + end + get ":id/deploy_keys/:key_id" do + key = user_project.deploy_keys.find params[:key_id] + present key, with: Entities::SSHKey + end - key = DeployKey.new attrs + desc 'Add new deploy key to currently authenticated user' do + success Entities::SSHKey + end + params do + requires :key, type: String, desc: 'The new deploy key' + requires :title, type: String, desc: 'The name of the deploy key' + end + post ":id/deploy_keys" do + params[:key].strip! - if key.valid? && user_project.deploy_keys << key - present key, with: Entities::SSHKey - else - render_validation_error!(key) - end + # Check for an existing key joined to this project + key = user_project.deploy_keys.find_by(key: params[:key]) + if key + present key, with: Entities::SSHKey + break end - desc 'Enable a deploy key for a project' do - detail 'This feature was added in GitLab 8.11' - success Entities::SSHKey - end - params do - requires :key_id, type: Integer, desc: 'The ID of the deploy key' + # Check for available deploy keys in other projects + key = current_user.accessible_deploy_keys.find_by(key: params[:key]) + if key + user_project.deploy_keys << key + present key, with: Entities::SSHKey + break end - post ":id/#{path}/:key_id/enable" do - key = ::Projects::EnableDeployKeyService.new(user_project, - current_user, declared(params)).execute - if key - present key, with: Entities::SSHKey - else - not_found!('Deploy Key') - end + # Create a new deploy key + key = DeployKey.new(declared_params(include_missing: false)) + if key.valid? && user_project.deploy_keys << key + present key, with: Entities::SSHKey + else + render_validation_error!(key) end + end - desc 'Disable a deploy key for a project' do - detail 'This feature was added in GitLab 8.11' - success Entities::SSHKey - end - params do - requires :key_id, type: Integer, desc: 'The ID of the deploy key' - end - delete ":id/#{path}/:key_id/disable" do - key = user_project.deploy_keys_projects.find_by(deploy_key_id: params[:key_id]) - key.destroy + desc 'Enable a deploy key for a project' do + detail 'This feature was added in GitLab 8.11' + success Entities::SSHKey + end + params do + requires :key_id, type: Integer, desc: 'The ID of the deploy key' + end + post ":id/deploy_keys/:key_id/enable" do + key = ::Projects::EnableDeployKeyService.new(user_project, + current_user, declared_params).execute - present key.deploy_key, with: Entities::SSHKey + if key + present key, with: Entities::SSHKey + else + not_found!('Deploy Key') end + end - desc 'Delete existing deploy key of currently authenticated user' do - success Key - end - params do - requires :key_id, type: Integer, desc: 'The ID of the deploy key' - end - delete ":id/#{path}/:key_id" do - key = user_project.deploy_keys.find(params[:key_id]) - key.destroy - end + desc 'Delete deploy key for a project' do + success Key + end + params do + requires :key_id, type: Integer, desc: 'The ID of the deploy key' + end + delete ":id/deploy_keys/:key_id" do + key = user_project.deploy_keys_projects.find_by(deploy_key_id: params[:key_id]) + not_found!('Deploy Key') unless key + + key.destroy end end end diff --git a/lib/api/deployments.rb b/lib/api/deployments.rb new file mode 100644 index 00000000000..c5feb49b22f --- /dev/null +++ b/lib/api/deployments.rb @@ -0,0 +1,41 @@ +module API + # Deployments RESTfull API endpoints + class Deployments < Grape::API + include PaginationParams + + before { authenticate! } + + params do + requires :id, type: String, desc: 'The project ID' + end + resource :projects do + desc 'Get all deployments of the project' do + detail 'This feature was introduced in GitLab 8.11.' + success Entities::Deployment + end + params do + use :pagination + end + get ':id/deployments' do + authorize! :read_deployment, user_project + + present paginate(user_project.deployments), with: Entities::Deployment + end + + desc 'Gets a specific deployment' do + detail 'This feature was introduced in GitLab 8.11.' + success Entities::Deployment + end + params do + requires :deployment_id, type: Integer, desc: 'The deployment ID' + end + get ':id/deployments/:deployment_id' do + authorize! :read_deployment, user_project + + deployment = user_project.deployments.find(params[:deployment_id]) + + present deployment, with: Entities::Deployment + end + end + end +end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index ae74d14a4bb..a99d9cadc8a 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -15,18 +15,18 @@ module API class User < UserBasic expose :created_at expose :is_admin?, as: :is_admin - expose :bio, :location, :skype, :linkedin, :twitter, :website_url + expose :bio, :location, :skype, :linkedin, :twitter, :website_url, :organization end class Identity < Grape::Entity expose :provider, :extern_uid end - class UserFull < User + class UserPublic < User expose :last_sign_in_at expose :confirmed_at expose :email - expose :theme_id, :color_scheme_id, :projects_limit, :current_sign_in_at + expose :color_scheme_id, :projects_limit, :current_sign_in_at expose :identities, using: Entities::Identity expose :can_create_group?, as: :can_create_group expose :can_create_project?, as: :can_create_project @@ -34,7 +34,7 @@ module API expose :external end - class UserLogin < UserFull + class UserWithPrivateToken < UserPublic expose :private_token end @@ -43,13 +43,13 @@ module API end class Hook < Grape::Entity - expose :id, :url, :created_at + expose :id, :url, :created_at, :push_events, :tag_push_events + expose :enable_ssl_verification end class ProjectHook < Hook - expose :project_id, :push_events - expose :issues_events, :merge_requests_events, :tag_push_events, :note_events, :build_events - expose :enable_ssl_verification + expose :project_id, :issues_events, :merge_requests_events + expose :note_events, :build_events, :pipeline_events, :wiki_page_events end class BasicProjectDetails < Grape::Entity @@ -75,40 +75,79 @@ module API expose :owner, using: Entities::UserBasic, unless: ->(project, options) { project.group } expose :name, :name_with_namespace expose :path, :path_with_namespace - expose :issues_enabled, :merge_requests_enabled, :wiki_enabled, :builds_enabled, :snippets_enabled, :container_registry_enabled + expose :container_registry_enabled + + # Expose old field names with the new permissions methods to keep API compatible + expose(:issues_enabled) { |project, options| project.feature_available?(:issues, options[:current_user]) } + expose(:merge_requests_enabled) { |project, options| project.feature_available?(:merge_requests, options[:current_user]) } + expose(:wiki_enabled) { |project, options| project.feature_available?(:wiki, options[:current_user]) } + expose(:builds_enabled) { |project, options| project.feature_available?(:builds, options[:current_user]) } + expose(:snippets_enabled) { |project, options| project.feature_available?(:snippets, options[:current_user]) } + expose :created_at, :last_activity_at expose :shared_runners_enabled + expose :lfs_enabled?, as: :lfs_enabled expose :creator_id - expose :namespace + expose :namespace, using: 'API::Entities::Namespace' expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda{ |project, options| project.forked? } expose :avatar_url expose :star_count, :forks_count - expose :open_issues_count, if: lambda { |project, options| project.issues_enabled? && project.default_issues_tracker? } + expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:current_user]) && project.default_issues_tracker? } expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] } expose :public_builds expose :shared_with_groups do |project, options| SharedGroup.represent(project.project_group_links.all, options) end + expose :only_allow_merge_if_build_succeeds + expose :request_access_enabled + expose :only_allow_merge_if_all_discussions_are_resolved + + expose :statistics, using: 'API::Entities::ProjectStatistics', if: :statistics + end + + class ProjectStatistics < Grape::Entity + expose :commit_count + expose :storage_size + expose :repository_size + expose :lfs_objects_size + expose :build_artifacts_size end class Member < UserBasic expose :access_level do |user, options| - member = options[:member] || options[:members].find { |m| m.user_id == user.id } + member = options[:member] || options[:source].members.find_by(user_id: user.id) member.access_level end + expose :expires_at do |user, options| + member = options[:member] || options[:source].members.find_by(user_id: user.id) + member.expires_at + end end class AccessRequester < UserBasic expose :requested_at do |user, options| - access_requester = options[:access_requester] || options[:access_requesters].find { |m| m.user_id == user.id } + access_requester = options[:access_requester] || options[:source].requesters.find_by(user_id: user.id) access_requester.requested_at end end class Group < Grape::Entity expose :id, :name, :path, :description, :visibility_level + expose :lfs_enabled?, as: :lfs_enabled expose :avatar_url expose :web_url + expose :request_access_enabled + expose :full_name, :full_path + expose :parent_id + + expose :statistics, if: :statistics do + with_options format_with: -> (value) { value.to_i } do + expose :storage_size + expose :repository_size + expose :lfs_objects_size + expose :build_artifacts_size + end + end end class GroupDetail < Group @@ -116,30 +155,53 @@ module API expose :shared_projects, using: Entities::Project end + class RepoCommit < Grape::Entity + expose :id, :short_id, :title, :created_at + expose :parent_ids + expose :safe_message, as: :message + expose :author_name, :author_email, :authored_date + expose :committer_name, :committer_email, :committed_date + end + + class RepoCommitStats < Grape::Entity + expose :additions, :deletions, :total + end + + class RepoCommitDetail < RepoCommit + expose :stats, using: Entities::RepoCommitStats + expose :status + end + class RepoBranch < Grape::Entity expose :name - expose :commit do |repo_branch, options| - options[:project].repository.commit(repo_branch.target) + expose :commit, using: Entities::RepoCommit do |repo_branch, options| + options[:project].repository.commit(repo_branch.dereferenced_target) + end + + expose :merged do |repo_branch, options| + options[:project].repository.merged_to_root_ref?(repo_branch.name) end expose :protected do |repo_branch, options| - options[:project].protected_branch? repo_branch.name + options[:project].protected_branch?(repo_branch.name) end expose :developers_can_push do |repo_branch, options| project = options[:project] - project.protected_branches.matching(repo_branch.name).any? { |protected_branch| protected_branch.push_access_level.access_level == Gitlab::Access::DEVELOPER } + access_levels = project.protected_branches.matching(repo_branch.name).map(&:push_access_levels).flatten + access_levels.any? { |access_level| access_level.access_level == Gitlab::Access::DEVELOPER } end expose :developers_can_merge do |repo_branch, options| project = options[:project] - project.protected_branches.matching(repo_branch.name).any? { |protected_branch| protected_branch.merge_access_level.access_level == Gitlab::Access::DEVELOPER } + access_levels = project.protected_branches.matching(repo_branch.name).map(&:merge_access_levels).flatten + access_levels.any? { |access_level| access_level.access_level == Gitlab::Access::DEVELOPER } end end class RepoTreeObject < Grape::Entity - expose :id, :name, :type + expose :id, :name, :type, :path expose :mode do |obj, options| filemode = obj.mode.to_s(8) @@ -148,28 +210,27 @@ module API end end - class RepoCommit < Grape::Entity - expose :id, :short_id, :title, :author_name, :author_email, :created_at - expose :safe_message, as: :message - end - - class RepoCommitStats < Grape::Entity - expose :additions, :deletions, :total - end + class ProjectSnippet < Grape::Entity + expose :id, :title, :file_name + expose :author, using: Entities::UserBasic + expose :updated_at, :created_at - class RepoCommitDetail < RepoCommit - expose :parent_ids, :committed_date, :authored_date - expose :stats, using: Entities::RepoCommitStats - expose :status + expose :web_url do |snippet, options| + Gitlab::UrlBuilder.build(snippet) + end end - class ProjectSnippet < Grape::Entity + class PersonalSnippet < Grape::Entity expose :id, :title, :file_name expose :author, using: Entities::UserBasic expose :updated_at, :created_at - # TODO (rspeicher): Deprecated; remove in 9.0 - expose(:expires_at) { |snippet| nil } + expose :web_url do |snippet| + Gitlab::UrlBuilder.build(snippet) + end + expose :raw_url do |snippet| + Gitlab::UrlBuilder.build(snippet) + "/raw" + end end class ProjectEntity < Grape::Entity @@ -186,6 +247,7 @@ module API class Milestone < ProjectEntity expose :due_date + expose :start_date end class Issue < ProjectEntity @@ -194,11 +256,23 @@ module API expose :assignee, :author, using: Entities::UserBasic expose :subscribed do |issue, options| - issue.subscribed?(options[:current_user]) + issue.subscribed?(options[:current_user], options[:project] || issue.project) end expose :user_notes_count expose :upvotes, :downvotes expose :due_date + expose :confidential + + expose :web_url do |issue, options| + Gitlab::UrlBuilder.build(issue) + end + end + + class IssuableTimeStats < Grape::Entity + expose :time_estimate + expose :total_time_spent + expose :human_time_estimate + expose :human_total_time_spent end class ExternalIssue < Grape::Entity @@ -216,12 +290,18 @@ module API expose :milestone, using: Entities::Milestone expose :merge_when_build_succeeds expose :merge_status + expose :diff_head_sha, as: :sha + expose :merge_commit_sha expose :subscribed do |merge_request, options| - merge_request.subscribed?(options[:current_user]) + merge_request.subscribed?(options[:current_user], options[:project]) end expose :user_notes_count expose :should_remove_source_branch?, as: :should_remove_source_branch expose :force_remove_source_branch?, as: :force_remove_source_branch + + expose :web_url do |merge_request, options| + Gitlab::UrlBuilder.build(merge_request) + end end class MergeRequestChanges < MergeRequest @@ -230,12 +310,25 @@ module API end end + class MergeRequestDiff < Grape::Entity + expose :id, :head_commit_sha, :base_commit_sha, :start_commit_sha, + :created_at, :merge_request_id, :state, :real_size + end + + class MergeRequestDiffFull < MergeRequestDiff + expose :commits, using: Entities::RepoCommit + + expose :diffs, using: Entities::RepoDiff do |compare, _| + compare.raw_diffs(all_diffs: true).to_a + end + end + class SSHKey < Grape::Entity - expose :id, :title, :key, :created_at + expose :id, :title, :key, :created_at, :can_push end class SSHKeyWithUser < SSHKey - expose :user, using: Entities::UserFull + expose :user, using: Entities::UserPublic end class Note < Grape::Entity @@ -246,9 +339,6 @@ module API expose :created_at, :updated_at expose :system?, as: :system expose :noteable_id, :noteable_type - # upvote? and downvote? are deprecated, always return false - expose(:upvote?) { |note| false } - expose(:downvote?) { |note| false } end class AwardEmoji < Grape::Entity @@ -275,7 +365,7 @@ module API class CommitStatus < Grape::Entity expose :id, :sha, :ref, :status, :name, :target_url, :description, - :created_at, :started_at, :finished_at, :allow_failure + :created_at, :started_at, :finished_at, :allow_failure, :coverage expose :author, using: Entities::UserBasic end @@ -288,14 +378,12 @@ module API expose :author, using: Entities::UserBasic, if: ->(event, options) { event.author } expose :author_username do |event, options| - if event.author - event.author.username - end + event.author&.username end end class ProjectGroupLink < Grape::Entity - expose :id, :project_id, :group_id, :group_access + expose :id, :project_id, :group_id, :group_access, :expires_at end class Todo < Grape::Entity @@ -324,14 +412,14 @@ module API end class Namespace < Grape::Entity - expose :id, :path, :kind + expose :id, :name, :path, :kind, :full_path end class MemberAccess < Grape::Entity expose :access_level expose :notification_level do |member, options| if member.notification_setting - NotificationSetting.levels[member.notification_setting.level] + ::NotificationSetting.levels[member.notification_setting.level] end end end @@ -342,9 +430,25 @@ module API class GroupAccess < MemberAccess end + class NotificationSetting < Grape::Entity + expose :level + expose :events, if: ->(notification_setting, _) { notification_setting.custom? } do + ::NotificationSetting::EMAIL_EVENTS.each do |event| + expose event + end + end + end + + class GlobalNotificationSetting < NotificationSetting + expose :notification_email do |notification_setting, options| + notification_setting.user.notification_email + end + end + class ProjectService < Grape::Entity expose :id, :title, :created_at, :updated_at, :active - expose :push_events, :issues_events, :merge_requests_events, :tag_push_events, :note_events, :build_events + expose :push_events, :issues_events, :merge_requests_events + expose :tag_push_events, :note_events, :build_events, :pipeline_events # Expose serialized properties expose :properties do |service, options| field_names = service.fields. @@ -357,23 +461,53 @@ module API class ProjectWithAccess < Project expose :permissions do expose :project_access, using: Entities::ProjectAccess do |project, options| - project.project_members.find_by(user_id: options[:user].id) + project.project_members.find_by(user_id: options[:current_user].id) end expose :group_access, using: Entities::GroupAccess do |project, options| if project.group - project.group.group_members.find_by(user_id: options[:user].id) + project.group.group_members.find_by(user_id: options[:current_user].id) end end end end - class Label < Grape::Entity - expose :name, :color, :description - expose :open_issues_count, :closed_issues_count, :open_merge_requests_count + class LabelBasic < Grape::Entity + expose :id, :name, :color, :description + end + + class Label < LabelBasic + expose :open_issues_count do |label, options| + label.open_issues_count(options[:current_user]) + end + + expose :closed_issues_count do |label, options| + label.closed_issues_count(options[:current_user]) + end + + expose :open_merge_requests_count do |label, options| + label.open_merge_requests_count(options[:current_user]) + end + + expose :priority do |label, options| + label.priority(options[:project]) + end expose :subscribed do |label, options| - label.subscribed?(options[:current_user]) + label.subscribed?(options[:current_user], options[:project]) + end + end + + class List < Grape::Entity + expose :id + expose :label, using: Entities::LabelBasic + expose :position + end + + class Board < Grape::Entity + expose :id + expose :lists, using: Entities::List do |board| + board.lists.destroyable end end @@ -423,6 +557,7 @@ module API expose :default_project_visibility expose :default_snippet_visibility expose :default_group_visibility + expose :default_artifacts_expire_in expose :domain_whitelist expose :domain_blacklist_enabled expose :domain_blacklist @@ -430,6 +565,12 @@ module API expose :after_sign_out_path expose :container_registry_token_expire_delay expose :repository_storage + expose :repository_storages + expose :koding_enabled + expose :koding_url + expose :plantuml_enabled + expose :plantuml_url + expose :terminal_max_session_time end class Release < Grape::Entity @@ -441,7 +582,7 @@ module API expose :name, :message expose :commit do |repo_tag, options| - options[:project].repository.commit(repo_tag.target) + options[:project].repository.commit(repo_tag.dereferenced_target) end expose :release, using: Entities::Release do |repo_tag, options| @@ -477,10 +618,18 @@ module API end end + class RunnerRegistrationDetails < Grape::Entity + expose :id, :token + end + class BuildArtifactFile < Grape::Entity expose :filename, :size end + class PipelineBasic < Grape::Entity + expose :id, :sha, :ref, :status + end + class Build < Grape::Entity expose :id, :status, :stage, :name, :ref, :tag, :coverage expose :created_at, :started_at, :finished_at @@ -488,6 +637,7 @@ module API expose :artifacts_file, using: BuildArtifactFile, if: -> (build, opts) { build.artifacts? } expose :commit, with: RepoCommit expose :runner, with: Runner + expose :pipeline, with: PipelineBasic end class Trigger < Grape::Entity @@ -498,8 +648,28 @@ module API expose :key, :value end - class Environment < Grape::Entity - expose :id, :name, :external_url + class Pipeline < PipelineBasic + expose :before_sha, :tag, :yaml_errors + + expose :user, with: Entities::UserBasic + expose :created_at, :updated_at, :started_at, :finished_at, :committed_at + expose :duration + expose :coverage + end + + class EnvironmentBasic < Grape::Entity + expose :id, :name, :slug, :external_url + end + + class Environment < EnvironmentBasic + expose :project, using: Entities::Project + end + + class Deployment < Grape::Entity + expose :id, :iid, :ref, :sha, :created_at + expose :user, using: Entities::UserBasic + expose :environment, using: Entities::EnvironmentBasic + expose :deployable, using: Entities::Build end class RepoLicense < Grape::Entity @@ -521,5 +691,10 @@ module API class Template < Grape::Entity expose :name, :content end + + class BroadcastMessage < Grape::Entity + expose :id, :message, :starts_at, :ends_at, :color, :font + expose :active?, as: :active + end end end diff --git a/lib/api/environments.rb b/lib/api/environments.rb index 819f80d8365..1a7e68f0528 100644 --- a/lib/api/environments.rb +++ b/lib/api/environments.rb @@ -1,6 +1,9 @@ module API # Environments RESTfull API endpoints class Environments < Grape::API + include ::API::Helpers::CustomValidators + include PaginationParams + before { authenticate! } params do @@ -12,8 +15,7 @@ module API success Entities::Environment end params do - optional :page, type: Integer, desc: 'Page number of the current request' - optional :per_page, type: Integer, desc: 'Number of items per page' + use :pagination end get ':id/environments' do authorize! :read_environment, user_project @@ -28,12 +30,12 @@ module API params do requires :name, type: String, desc: 'The name of the environment to be created' optional :external_url, type: String, desc: 'URL on which this deployment is viewable' + optional :slug, absence: { message: "is automatically generated and cannot be changed" } end post ':id/environments' do authorize! :create_environment, user_project - create_params = declared(params, include_parent_namespaces: false).to_h - environment = user_project.environments.create(create_params) + environment = user_project.environments.create(declared_params) if environment.persisted? present environment, with: Entities::Environment @@ -50,13 +52,14 @@ module API requires :environment_id, type: Integer, desc: 'The environment ID' optional :name, type: String, desc: 'The new environment name' optional :external_url, type: String, desc: 'The new URL on which this deployment is viewable' + optional :slug, absence: { message: "is automatically generated and cannot be changed" } end put ':id/environments/:environment_id' do authorize! :update_environment, user_project environment = user_project.environments.find(params[:environment_id]) - - update_params = declared(params, include_missing: false).extract!(:name, :external_url).to_h + + update_params = declared_params(include_missing: false).extract!(:name, :external_url) if environment.update(update_params) present environment, with: Entities::Environment else diff --git a/lib/api/files.rb b/lib/api/files.rb index c1d86f313b0..500f9d3c787 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -1,160 +1,126 @@ module API - # Projects API class Files < Grape::API - before { authenticate! } - helpers do def commit_params(attrs) { file_path: attrs[:file_path], - source_branch: attrs[:branch_name], - target_branch: attrs[:branch_name], + start_branch: attrs[:branch], + target_branch: attrs[:branch], commit_message: attrs[:commit_message], file_content: attrs[:content], - file_content_encoding: attrs[:encoding] + file_content_encoding: attrs[:encoding], + author_email: attrs[:author_email], + author_name: attrs[:author_name] } end def commit_response(attrs) { file_path: attrs[:file_path], - branch_name: attrs[:branch_name], + branch: attrs[:branch] } end + + params :simple_file_params do + requires :file_path, type: String, desc: 'The path to new file. Ex. lib/class.rb' + requires :branch, type: String, desc: 'The name of branch' + requires :commit_message, type: String, desc: 'Commit Message' + optional :author_email, type: String, desc: 'The email of the author' + optional :author_name, type: String, desc: 'The name of the author' + end + + params :extended_file_params do + use :simple_file_params + requires :content, type: String, desc: 'File content' + optional :encoding, type: String, values: %w[base64], desc: 'File encoding' + end end + params do + requires :id, type: String, desc: 'The project ID' + end resource :projects do - # Get file from repository - # File content is Base64 encoded - # - # Parameters: - # file_path (required) - The path to the file. Ex. lib/class.rb - # ref (required) - The name of branch, tag or commit - # - # Example Request: - # GET /projects/:id/repository/files - # - # Example response: - # { - # "file_name": "key.rb", - # "file_path": "app/models/key.rb", - # "size": 1476, - # "encoding": "base64", - # "content": "IyA9PSBTY2hlbWEgSW5mb3...", - # "ref": "master", - # "blob_id": "79f7bbd25901e8334750839545a9bd021f0e4c83", - # "commit_id": "d5a3ff139356ce33e37e73add446f16869741b50", - # "last_commit_id": "570e7b2abdd848b95f2f578043fc23bd6f6fd24d", - # } - # + desc 'Get a file from repository' + params do + requires :file_path, type: String, desc: 'The path to the file. Ex. lib/class.rb' + requires :ref, type: String, desc: 'The name of branch, tag, or commit' + end get ":id/repository/files" do authorize! :download_code, user_project - required_attributes! [:file_path, :ref] - attrs = attributes_for_keys [:file_path, :ref] - ref = attrs.delete(:ref) - file_path = attrs.delete(:file_path) - - commit = user_project.commit(ref) - not_found! 'Commit' unless commit + commit = user_project.commit(params[:ref]) + not_found!('Commit') unless commit repo = user_project.repository - blob = repo.blob_at(commit.sha, file_path) + blob = repo.blob_at(commit.sha, params[:file_path]) + not_found!('File') unless blob - if blob - blob.load_all_data!(repo) - status(200) + blob.load_all_data!(repo) + status(200) - { - file_name: blob.name, - file_path: blob.path, - size: blob.size, - encoding: "base64", - content: Base64.strict_encode64(blob.data), - ref: ref, - blob_id: blob.id, - commit_id: commit.id, - last_commit_id: repo.last_commit_for_path(commit.sha, file_path).id - } - else - not_found! 'File' - end + { + file_name: blob.name, + file_path: blob.path, + size: blob.size, + encoding: "base64", + content: Base64.strict_encode64(blob.data), + ref: params[:ref], + blob_id: blob.id, + commit_id: commit.id, + last_commit_id: repo.last_commit_id_for_path(commit.sha, params[:file_path]) + } end - # Create new file in repository - # - # Parameters: - # file_path (required) - The path to new file. Ex. lib/class.rb - # branch_name (required) - The name of branch - # content (required) - File content - # commit_message (required) - Commit message - # - # Example Request: - # POST /projects/:id/repository/files - # + desc 'Create new file in repository' + params do + use :extended_file_params + end post ":id/repository/files" do authorize! :push_code, user_project - required_attributes! [:file_path, :branch_name, :content, :commit_message] - attrs = attributes_for_keys [:file_path, :branch_name, :content, :commit_message, :encoding] - result = ::Files::CreateService.new(user_project, current_user, commit_params(attrs)).execute + file_params = declared_params(include_missing: false) + result = ::Files::CreateService.new(user_project, current_user, commit_params(file_params)).execute if result[:status] == :success status(201) - commit_response(attrs) + commit_response(file_params) else render_api_error!(result[:message], 400) end end - # Update existing file in repository - # - # Parameters: - # file_path (optional) - The path to file. Ex. lib/class.rb - # branch_name (required) - The name of branch - # content (required) - File content - # commit_message (required) - Commit message - # - # Example Request: - # PUT /projects/:id/repository/files - # + desc 'Update existing file in repository' + params do + use :extended_file_params + end put ":id/repository/files" do authorize! :push_code, user_project - required_attributes! [:file_path, :branch_name, :content, :commit_message] - attrs = attributes_for_keys [:file_path, :branch_name, :content, :commit_message, :encoding] - result = ::Files::UpdateService.new(user_project, current_user, commit_params(attrs)).execute + file_params = declared_params(include_missing: false) + result = ::Files::UpdateService.new(user_project, current_user, commit_params(file_params)).execute if result[:status] == :success status(200) - commit_response(attrs) + commit_response(file_params) else http_status = result[:http_status] || 400 render_api_error!(result[:message], http_status) end end - # Delete existing file in repository - # - # Parameters: - # file_path (optional) - The path to file. Ex. lib/class.rb - # branch_name (required) - The name of branch - # content (required) - File content - # commit_message (required) - Commit message - # - # Example Request: - # DELETE /projects/:id/repository/files - # + desc 'Delete an existing file in repository' + params do + use :simple_file_params + end delete ":id/repository/files" do authorize! :push_code, user_project - required_attributes! [:file_path, :branch_name, :commit_message] - attrs = attributes_for_keys [:file_path, :branch_name, :commit_message] - result = ::Files::DeleteService.new(user_project, current_user, commit_params(attrs)).execute + file_params = declared_params(include_missing: false) + result = ::Files::DestroyService.new(user_project, current_user, commit_params(file_params)).execute if result[:status] == :success status(200) - commit_response(attrs) + commit_response(file_params) else render_api_error!(result[:message], 400) end diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 9d8b8d737a9..9cffd6180ae 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -1,120 +1,167 @@ module API - # groups API class Groups < Grape::API + include PaginationParams + before { authenticate! } + helpers do + params :optional_params do + optional :description, type: String, desc: 'The description of the group' + optional :visibility_level, type: Integer, desc: 'The visibility level of the group' + optional :lfs_enabled, type: Boolean, desc: 'Enable/disable LFS for the projects in this group' + optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access' + end + + params :statistics_params do + optional :statistics, type: Boolean, default: false, desc: 'Include project statistics' + end + + def present_groups(groups, options = {}) + options = options.reverse_merge( + with: Entities::Group, + current_user: current_user, + ) + + groups = groups.with_statistics if options[:statistics] + present paginate(groups), options + end + end + resource :groups do - # Get a groups list - # - # Example Request: - # GET /groups + desc 'Get a groups list' do + success Entities::Group + end + params do + use :statistics_params + optional :skip_groups, type: Array[Integer], desc: 'Array of group ids to exclude from list' + optional :all_available, type: Boolean, desc: 'Show all group that you have access to' + optional :search, type: String, desc: 'Search for a specific group' + optional :owned, type: Boolean, default: false, desc: 'Limit by owned by authenticated user' + optional :order_by, type: String, values: %w[name path], default: 'name', desc: 'Order by name or path' + optional :sort, type: String, values: %w[asc desc], default: 'asc', desc: 'Sort by asc (ascending) or desc (descending)' + use :pagination + end get do - @groups = if current_user.admin - Group.all - else - current_user.groups - end - - @groups = @groups.search(params[:search]) if params[:search].present? - @groups = paginate @groups - present @groups, with: Entities::Group - end - - # Create group. Available only for users who can create groups. - # - # Parameters: - # name (required) - The name of the group - # path (required) - The path of the group - # description (optional) - The description of the group - # visibility_level (optional) - The visibility level of the group - # Example Request: - # POST /groups + groups = if params[:owned] + current_user.owned_groups + elsif current_user.admin + Group.all + elsif params[:all_available] + GroupsFinder.new.execute(current_user) + else + current_user.groups + end + + groups = groups.search(params[:search]) if params[:search].present? + groups = groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present? + groups = groups.reorder(params[:order_by] => params[:sort]) + + present_groups groups, statistics: params[:statistics] && current_user.is_admin? + end + + desc 'Create a group. Available only for users who can create groups.' do + success Entities::Group + end + params do + requires :name, type: String, desc: 'The name of the group' + requires :path, type: String, desc: 'The path of the group' + optional :parent_id, type: Integer, desc: 'The parent group id for creating nested group' + use :optional_params + end post do - authorize! :create_group, current_user - required_attributes! [:name, :path] + authorize! :create_group - attrs = attributes_for_keys [:name, :path, :description, :visibility_level] - @group = Group.new(attrs) + group = ::Groups::CreateService.new(current_user, declared_params(include_missing: false)).execute - if @group.save - @group.add_owner(current_user) - present @group, with: Entities::Group + if group.persisted? + present group, with: Entities::Group, current_user: current_user else - render_api_error!("Failed to save group #{@group.errors.messages}", 400) + render_api_error!("Failed to save group #{group.errors.messages}", 400) end end + end - # Update group. Available only for users who can administrate groups. - # - # Parameters: - # id (required) - The ID of a group - # path (optional) - The path of the group - # description (optional) - The description of the group - # visibility_level (optional) - The visibility level of the group - # Example Request: - # PUT /groups/:id + params do + requires :id, type: String, desc: 'The ID of a group' + end + resource :groups do + desc 'Update a group. Available only for users who can administrate groups.' do + success Entities::Group + end + params do + optional :name, type: String, desc: 'The name of the group' + optional :path, type: String, desc: 'The path of the group' + use :optional_params + at_least_one_of :name, :path, :description, :visibility_level, + :lfs_enabled, :request_access_enabled + end put ':id' do - group = find_group(params[:id]) + group = find_group!(params[:id]) authorize! :admin_group, group - attrs = attributes_for_keys [:name, :path, :description, :visibility_level] - - if ::Groups::UpdateService.new(group, current_user, attrs).execute - present group, with: Entities::GroupDetail + if ::Groups::UpdateService.new(group, current_user, declared_params(include_missing: false)).execute + present group, with: Entities::GroupDetail, current_user: current_user else render_validation_error!(group) end end - # Get a single group, with containing projects - # - # Parameters: - # id (required) - The ID of a group - # Example Request: - # GET /groups/:id + desc 'Get a single group, with containing projects.' do + success Entities::GroupDetail + end get ":id" do - group = find_group(params[:id]) - present group, with: Entities::GroupDetail + group = find_group!(params[:id]) + present group, with: Entities::GroupDetail, current_user: current_user end - # Remove group - # - # Parameters: - # id (required) - The ID of a group - # Example Request: - # DELETE /groups/:id + desc 'Remove a group.' delete ":id" do - group = find_group(params[:id]) + group = find_group!(params[:id]) authorize! :admin_group, group - DestroyGroupService.new(group, current_user).execute + ::Groups::DestroyService.new(group, current_user).execute + end + + desc 'Get a list of projects in this group.' do + success Entities::Project end + params do + optional :archived, type: Boolean, default: false, desc: 'Limit by archived status' + optional :visibility, type: String, values: %w[public internal private], + desc: 'Limit by visibility' + optional :search, type: String, desc: 'Return list of authorized projects matching the search criteria' + optional :order_by, type: String, values: %w[id name path created_at updated_at last_activity_at], + default: 'created_at', desc: 'Return projects ordered by field' + optional :sort, type: String, values: %w[asc desc], default: 'desc', + desc: 'Return projects sorted in ascending and descending order' + optional :simple, type: Boolean, default: false, + desc: 'Return only the ID, URL, name, and path of each project' + optional :owned, type: Boolean, default: false, desc: 'Limit by owned by authenticated user' + optional :starred, type: Boolean, default: false, desc: 'Limit by starred status' - # Get a list of projects in this group - # - # Example Request: - # GET /groups/:id/projects + use :pagination + end get ":id/projects" do - group = find_group(params[:id]) + group = find_group!(params[:id]) projects = GroupProjectsFinder.new(group).execute(current_user) - projects = paginate projects - present projects, with: Entities::Project + projects = filter_projects(projects) + entity = params[:simple] ? Entities::BasicProjectDetails : Entities::Project + present paginate(projects), with: entity, current_user: current_user end - # Transfer a project to the Group namespace - # - # Parameters: - # id - group id - # project_id - project id - # Example Request: - # POST /groups/:id/projects/:project_id + desc 'Transfer a project to the group namespace. Available only for admin.' do + success Entities::GroupDetail + end + params do + requires :project_id, type: String, desc: 'The ID or path of the project' + end post ":id/projects/:project_id" do authenticated_as_admin! - group = Group.find_by(id: params[:id]) - project = Project.find(params[:project_id]) + group = find_group!(params[:id]) + project = find_project!(params[:project_id]) result = ::Projects::TransferService.new(project, current_user).execute(group) if result - present group + present group, with: Entities::GroupDetail, current_user: current_user else render_api_error!("Failed to transfer project #{project.errors.messages}", 400) end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index d0469d6602d..72d2b320077 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -1,94 +1,74 @@ module API module Helpers - PRIVATE_TOKEN_HEADER = "HTTP_PRIVATE_TOKEN" - PRIVATE_TOKEN_PARAM = :private_token - SUDO_HEADER = "HTTP_SUDO" - SUDO_PARAM = :sudo - - def to_boolean(value) - return true if value =~ /^(true|t|yes|y|1|on)$/i - return false if value =~ /^(false|f|no|n|0|off)$/i + include Gitlab::Utils + include Helpers::Pagination - nil - end + SUDO_HEADER = "HTTP_SUDO".freeze + SUDO_PARAM = :sudo - def find_user_by_private_token - token_string = (params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]).to_s - User.find_by_authentication_token(token_string) || User.find_by_personal_access_token(token_string) + def declared_params(options = {}) + options = { include_parent_namespaces: false }.merge(options) + declared(params, options).to_h.symbolize_keys end def current_user - @current_user ||= (find_user_by_private_token || doorkeeper_guard) + return @current_user if defined?(@current_user) - unless @current_user && Gitlab::UserAccess.new(@current_user).allowed? - return nil - end - - identifier = sudo_identifier() + @current_user = initial_current_user - # If the sudo is the current user do nothing - if identifier && !(@current_user.id == identifier || @current_user.username == identifier) - forbidden!('Must be admin to use sudo') unless @current_user.is_admin? - @current_user = User.by_username_or_id(identifier) - not_found!("No user id or username for: #{identifier}") if @current_user.nil? - end + sudo! @current_user end - def sudo_identifier - identifier ||= params[SUDO_PARAM] || env[SUDO_HEADER] - - # Regex for integers - if !!(identifier =~ /\A[0-9]+\z/) - identifier.to_i - else - identifier - end + def sudo? + initial_current_user != current_user end def user_project - @project ||= find_project(params[:id]) + @project ||= find_project!(params[:id]) end - def find_project(id) - project = Project.find_with_namespace(id) || Project.find_by(id: id) + def available_labels + @available_labels ||= LabelsFinder.new(current_user, project_id: user_project.id).execute + end - if can?(current_user, :read_project, project) - project + def find_user(id) + if id =~ /^\d+$/ + User.find_by(id: id) else - not_found!('Project') + User.find_by(username: id) end end - def project_service - @project_service ||= begin - underscored_service = params[:service_slug].underscore - - if Service.available_services_names.include?(underscored_service) - user_project.build_missing_services - - service_method = "#{underscored_service}_service" - - send_service(service_method) - end + def find_project(id) + if id =~ /^\d+$/ + Project.find_by(id: id) + else + Project.find_by_full_path(id) end - - @project_service || not_found!("Service") end - def send_service(service_method) - user_project.send(service_method) - end + def find_project!(id) + project = find_project(id) - def service_attributes - @service_attributes ||= project_service.fields.inject([]) do |arr, hash| - arr << hash[:name].to_sym + if can?(current_user, :read_project, project) + project + else + not_found!('Project') end end def find_group(id) - group = Group.find_by(path: id) || Group.find_by(id: id) + if id =~ /^\d+$/ + Group.find_by(id: id) + else + Group.find_by_full_path(id) + end + end + + def find_group!(id) + group = find_group(id) if can?(current_user, :read_group, group) group @@ -98,26 +78,32 @@ module API end def find_project_label(id) - label = user_project.labels.find_by_id(id) || user_project.labels.find_by_title(id) + label = available_labels.find_by_id(id) || available_labels.find_by_title(id) label || not_found!('Label') end def find_project_issue(id) - issue = user_project.issues.find(id) - not_found! unless can?(current_user, :read_issue, issue) - issue + IssuesFinder.new(current_user, project_id: user_project.id).find(id) end - def paginate(relation) - relation.page(params[:page]).per(params[:per_page].to_i).tap do |data| - add_pagination_headers(data) - end + def find_project_merge_request(id) + MergeRequestsFinder.new(current_user, project_id: user_project.id).find(id) + end + + def find_merge_request_with_access(id, access_level = :read_merge_request) + merge_request = user_project.merge_requests.find(id) + authorize! access_level, merge_request + merge_request end def authenticate! unauthorized! unless current_user end + def authenticate_non_get! + authenticate! unless %w[GET HEAD].include?(route.request_method) + end + def authenticate_by_gitlab_shell_token! input = params['secret_token'].try(:chomp) unless Devise.secure_compare(secret_token, input) @@ -126,10 +112,11 @@ module API end def authenticated_as_admin! + authenticate! forbidden! unless current_user.is_admin? end - def authorize!(action, subject) + def authorize!(action, subject = nil) forbidden! unless can?(current_user, action, subject) end @@ -148,7 +135,7 @@ module API end def can?(object, action, subject) - abilities.allowed?(object, action, subject) + Ability.allowed?(object, action, subject) end # Checks the occurrences of required attributes, each attribute must be present in the params hash @@ -166,64 +153,13 @@ module API params_hash = custom_params || params attrs = {} keys.each do |key| - if params_hash[key].present? or (params_hash.has_key?(key) and params_hash[key] == false) + if params_hash[key].present? || (params_hash.has_key?(key) && params_hash[key] == false) attrs[key] = params_hash[key] end end ActionController::Parameters.new(attrs).permit! end - # Helper method for validating all labels against its names - def validate_label_params(params) - errors = {} - - if params[:labels].present? - params[:labels].split(',').each do |label_name| - label = user_project.labels.create_with( - color: Label::DEFAULT_COLOR).find_or_initialize_by( - title: label_name.strip) - - if label.invalid? - errors[label.title] = label.errors - end - end - end - - errors - end - - # Checks the occurrences of datetime attributes, each attribute if present in the params hash must be in ISO 8601 - # format (YYYY-MM-DDTHH:MM:SSZ) or a Bad Request error is invoked. - # - # Parameters: - # keys (required) - An array consisting of elements that must be parseable as dates from the params hash - def datetime_attributes!(*keys) - keys.each do |key| - begin - params[key] = Time.xmlschema(params[key]) if params[key].present? - rescue ArgumentError - message = "\"" + key.to_s + "\" must be a timestamp in ISO 8601 format: YYYY-MM-DDTHH:MM:SSZ" - render_api_error!(message, 400) - end - end - end - - def issuable_order_by - if params["order_by"] == 'updated_at' - 'updated_at' - else - 'created_at' - end - end - - def issuable_sort - if params["sort"] == 'asc' - :asc - else - :desc - end - end - def filter_by_iid(items, iid) items.where(iid: iid) end @@ -269,22 +205,55 @@ module API render_api_error!('304 Not Modified', 304) end + def no_content! + render_api_error!('204 No Content', 204) + end + + def accepted! + render_api_error!('202 Accepted', 202) + end + def render_validation_error!(model) if model.errors.any? render_api_error!(model.errors.messages || '400 Bad Request', 400) end end + def render_spam_error! + render_api_error!({ error: 'Spam detected' }, 400) + end + def render_api_error!(message, status) - error!({ 'message' => message }, status) + error!({ 'message' => message }, status, header) + end + + def handle_api_exception(exception) + if sentry_enabled? && report_exception?(exception) + define_params_for_grape_middleware + sentry_context + Raven.capture_exception(exception) + end + + # lifted from https://github.com/rails/rails/blob/master/actionpack/lib/action_dispatch/middleware/debug_exceptions.rb#L60 + trace = exception.backtrace + + message = "\n#{exception.class} (#{exception.message}):\n" + message << exception.annoted_source_code.to_s if exception.respond_to?(:annoted_source_code) + message << " " << trace.join("\n ") + + API.logger.add Logger::FATAL, message + rack_response({ 'message' => '500 Internal Server Error' }.to_json, 500) end - # Projects helpers + # project helpers def filter_projects(projects) - # If the archived parameter is passed, limit results accordingly - if params[:archived].present? - projects = projects.where(archived: to_boolean(params[:archived])) + if params[:owned] + projects = projects.merge(current_user.owned_projects) + end + + if params[:starred] + projects = projects.merge(current_user.starred_projects) end if params[:search].present? @@ -295,25 +264,8 @@ module API projects = projects.search_by_visibility(params[:visibility]) end - projects.reorder(project_order_by => project_sort) - end - - def project_order_by - order_fields = %w(id name path created_at updated_at last_activity_at) - - if order_fields.include?(params['order_by']) - params['order_by'] - else - 'created_at' - end - end - - def project_sort - if params["sort"] == 'asc' - :asc - else - :desc - end + projects = projects.where(archived: params[:archived]) + projects.reorder(params[:order_by] => params[:sort]) end # file helpers @@ -352,54 +304,70 @@ module API header['X-Sendfile'] = path body else - file FileStreamer.new(path) + file path end end private - def add_pagination_headers(paginated_data) - header 'X-Total', paginated_data.total_count.to_s - header 'X-Total-Pages', paginated_data.total_pages.to_s - header 'X-Per-Page', paginated_data.limit_value.to_s - header 'X-Page', paginated_data.current_page.to_s - header 'X-Next-Page', paginated_data.next_page.to_s - header 'X-Prev-Page', paginated_data.prev_page.to_s - header 'Link', pagination_links(paginated_data) + def private_token + params[APIGuard::PRIVATE_TOKEN_PARAM] || env[APIGuard::PRIVATE_TOKEN_HEADER] end - def pagination_links(paginated_data) - request_url = request.url.split('?').first - request_params = params.clone - request_params[:per_page] = paginated_data.limit_value + def warden + env['warden'] + end - links = [] + # Check the Rails session for valid authentication details + # + # Until CSRF protection is added to the API, disallow this method for + # state-changing endpoints + def find_user_from_warden + warden.try(:authenticate) if %w[GET HEAD].include?(env['REQUEST_METHOD']) + end + + def initial_current_user + return @initial_current_user if defined?(@initial_current_user) - request_params[:page] = paginated_data.current_page - 1 - links << %(<#{request_url}?#{request_params.to_query}>; rel="prev") unless paginated_data.first_page? + @initial_current_user ||= find_user_by_private_token(scopes: @scopes) + @initial_current_user ||= doorkeeper_guard(scopes: @scopes) + @initial_current_user ||= find_user_from_warden - request_params[:page] = paginated_data.current_page + 1 - links << %(<#{request_url}?#{request_params.to_query}>; rel="next") unless paginated_data.last_page? + unless @initial_current_user && Gitlab::UserAccess.new(@initial_current_user).allowed? + @initial_current_user = nil + end - request_params[:page] = 1 - links << %(<#{request_url}?#{request_params.to_query}>; rel="first") + @initial_current_user + end - request_params[:page] = paginated_data.total_pages - links << %(<#{request_url}?#{request_params.to_query}>; rel="last") + def sudo! + return unless sudo_identifier + return unless initial_current_user - links.join(', ') + unless initial_current_user.is_admin? + forbidden!('Must be admin to use sudo') + end + + # Only private tokens should be used for the SUDO feature + unless private_token == initial_current_user.private_token + forbidden!('Private token must be specified in order to use sudo') + end + + sudoed_user = find_user(sudo_identifier) + + if sudoed_user + @current_user = sudoed_user + else + not_found!("No user id or username for: #{sudo_identifier}") + end end - def abilities - @abilities ||= begin - abilities = Six.new - abilities << Ability - abilities - end + def sudo_identifier + @sudo_identifier ||= params[SUDO_PARAM] || env[SUDO_HEADER] end def secret_token - File.read(Gitlab.config.gitlab_shell.secret_file).chomp + Gitlab::Shell.secret_token end def send_git_blob(repository, blob) @@ -419,5 +387,19 @@ module API Entities::Issue end end + + # The Grape Error Middleware only has access to env but no params. We workaround this by + # defining a method that returns the right value. + def define_params_for_grape_middleware + self.define_singleton_method(:params) { Rack::Request.new(env).params.symbolize_keys } + end + + # We could get a Grape or a standard Ruby exception. We should only report anything that + # is clearly an error. + def report_exception?(exception) + return true unless exception.respond_to?(:status) + + exception.status == 500 + end end end diff --git a/lib/api/helpers/custom_validators.rb b/lib/api/helpers/custom_validators.rb new file mode 100644 index 00000000000..0a8f3073a50 --- /dev/null +++ b/lib/api/helpers/custom_validators.rb @@ -0,0 +1,14 @@ +module API + module Helpers + module CustomValidators + class Absence < Grape::Validations::Base + def validate_param!(attr_name, params) + return if params.respond_to?(:key?) && !params.key?(attr_name) + raise Grape::Exceptions::Validation, params: [@scope.full_name(attr_name)], message: message(:absence) + end + end + end + end +end + +Grape::Validations.register_validator(:absence, ::API::Helpers::CustomValidators::Absence) diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb new file mode 100644 index 00000000000..080a6274957 --- /dev/null +++ b/lib/api/helpers/internal_helpers.rb @@ -0,0 +1,65 @@ +module API + module Helpers + module InternalHelpers + # Project paths may be any of the following: + # * /repository/storage/path/namespace/project + # * /namespace/project + # * namespace/project + # + # In addition, they may have a '.git' extension and multiple namespaces + # + # Transform all these cases to 'namespace/project' + def clean_project_path(project_path, storage_paths = Repository.storages.values) + project_path = project_path.sub(/\.git\z/, '') + + storage_paths.each do |storage_path| + storage_path = File.expand_path(storage_path) + + if project_path.start_with?(storage_path) + project_path = project_path.sub(storage_path, '') + break + end + end + + project_path.sub(/\A\//, '') + end + + def project_path + @project_path ||= clean_project_path(params[:project]) + end + + def wiki? + @wiki ||= project_path.end_with?('.wiki') && + !Project.find_by_full_path(project_path) + end + + def project + @project ||= begin + # Check for *.wiki repositories. + # Strip out the .wiki from the pathname before finding the + # project. This applies the correct project permissions to + # the wiki repository as well. + project_path.chomp!('.wiki') if wiki? + + Project.find_by_full_path(project_path) + end + end + + def ssh_authentication_abilities + [ + :read_project, + :download_code, + :push_code + ] + end + + def parse_allowed_environment_variables + return if params[:env].blank? + + JSON.parse(params[:env]) + + rescue JSON::ParserError + end + end + end +end diff --git a/lib/api/helpers/members_helpers.rb b/lib/api/helpers/members_helpers.rb index 90114f6f667..d9cae1501f8 100644 --- a/lib/api/helpers/members_helpers.rb +++ b/lib/api/helpers/members_helpers.rb @@ -2,7 +2,7 @@ module API module Helpers module MembersHelpers def find_source(source_type, id) - public_send("find_#{source_type}", id) + public_send("find_#{source_type}!", id) end def authorize_admin_source!(source_type, source) diff --git a/lib/api/helpers/pagination.rb b/lib/api/helpers/pagination.rb new file mode 100644 index 00000000000..0764b58fb4c --- /dev/null +++ b/lib/api/helpers/pagination.rb @@ -0,0 +1,45 @@ +module API + module Helpers + module Pagination + def paginate(relation) + relation.page(params[:page]).per(params[:per_page]).tap do |data| + add_pagination_headers(data) + end + end + + private + + def add_pagination_headers(paginated_data) + header 'X-Total', paginated_data.total_count.to_s + header 'X-Total-Pages', paginated_data.total_pages.to_s + header 'X-Per-Page', paginated_data.limit_value.to_s + header 'X-Page', paginated_data.current_page.to_s + header 'X-Next-Page', paginated_data.next_page.to_s + header 'X-Prev-Page', paginated_data.prev_page.to_s + header 'Link', pagination_links(paginated_data) + end + + def pagination_links(paginated_data) + request_url = request.url.split('?').first + request_params = params.clone + request_params[:per_page] = paginated_data.limit_value + + links = [] + + request_params[:page] = paginated_data.current_page - 1 + links << %(<#{request_url}?#{request_params.to_query}>; rel="prev") unless paginated_data.first_page? + + request_params[:page] = paginated_data.current_page + 1 + links << %(<#{request_url}?#{request_params.to_query}>; rel="next") unless paginated_data.last_page? + + request_params[:page] = 1 + links << %(<#{request_url}?#{request_params.to_query}>; rel="first") + + request_params[:page] = paginated_data.total_pages + links << %(<#{request_url}?#{request_params.to_query}>; rel="last") + + links.join(', ') + end + end + end +end diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb new file mode 100644 index 00000000000..119ca81b883 --- /dev/null +++ b/lib/api/helpers/runner.rb @@ -0,0 +1,23 @@ +module API + module Helpers + module Runner + def runner_registration_token_valid? + ActiveSupport::SecurityUtils.variable_size_secure_compare(params[:token], + current_application_settings.runners_registration_token) + end + + def get_runner_version_from_params + return unless params['info'].present? + attributes_for_keys(%w(name version revision platform architecture), params['info']) + end + + def authenticate_runner! + forbidden! unless current_runner + end + + def current_runner + @runner ||= ::Ci::Runner.find_by_token(params[:token].to_s) + end + end + end +end diff --git a/lib/api/internal.rb b/lib/api/internal.rb index d8e9ac406c4..d235977fbd8 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -3,6 +3,8 @@ module API class Internal < Grape::API before { authenticate_by_gitlab_shell_token! } + helpers ::API::Helpers::InternalHelpers + namespace 'internal' do # Check if git command is allowed to project # @@ -14,29 +16,6 @@ module API # ref - branch name # forced_push - forced_push # protocol - Git access protocol being used, e.g. HTTP or SSH - # - - helpers do - def wiki? - @wiki ||= params[:project].end_with?('.wiki') && - !Project.find_with_namespace(params[:project]) - end - - def project - @project ||= begin - project_path = params[:project] - - # Check for *.wiki repositories. - # Strip out the .wiki from the pathname before finding the - # project. This applies the correct project permissions to - # the wiki repository as well. - project_path.chomp!('.wiki') if wiki? - - Project.find_with_namespace(project_path) - end - end - end - post "/allowed" do status 200 @@ -49,11 +28,17 @@ module API protocol = params[:protocol] + actor.update_last_used_at if actor.is_a?(Key) + access = if wiki? - Gitlab::GitAccessWiki.new(actor, project, protocol) + Gitlab::GitAccessWiki.new(actor, project, protocol, authentication_abilities: ssh_authentication_abilities) else - Gitlab::GitAccess.new(actor, project, protocol) + Gitlab::GitAccess.new(actor, + project, + protocol, + authentication_abilities: ssh_authentication_abilities, + env: parse_allowed_environment_variables) end access_status = access.check(params[:action], params[:changes]) @@ -74,6 +59,21 @@ module API response end + post "/lfs_authenticate" do + status 200 + + key = Key.find(params[:key_id]) + key.update_last_used_at + + token_handler = Gitlab::LfsToken.new(key) + + { + username: token_handler.actor_name, + lfs_token: token_handler.token, + repository_http_path: project.http_url_to_repo + } + end + get "/merge_request_urls" do ::MergeRequests::GetUrlsService.new(project).execute(params[:changes]) end @@ -101,6 +101,37 @@ module API {} end end + + post '/two_factor_recovery_codes' do + status 200 + + key = Key.find_by(id: params[:key_id]) + + if key + key.update_last_used_at + else + return { 'success' => false, 'message' => 'Could not find the given key' } + end + + if key.is_a?(DeployKey) + return { success: false, message: 'Deploy keys cannot be used to retrieve recovery codes' } + end + + user = key.user + + unless user + return { success: false, message: 'Could not find a user for the given key' } + end + + unless user.two_factor_enabled? + return { success: false, message: 'Two-factor authentication is not enabled for this user' } + end + + codes = user.generate_otp_backup_codes! + user.save! + + { success: true, recovery_codes: codes } + end end end end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index c4d3134da6c..6d30c5d81b1 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -1,258 +1,218 @@ module API - # Issues API class Issues < Grape::API - before { authenticate! } + include PaginationParams - helpers ::Gitlab::AkismetHelper + before { authenticate! } helpers do - def filter_issues_state(issues, state) - case state - when 'opened' then issues.opened - when 'closed' then issues.closed - else issues - end + def find_issues(args = {}) + args = params.merge(args) + + args.delete(:id) + args[:milestone_title] = args.delete(:milestone) + args[:label_name] = args.delete(:labels) + + issues = IssuesFinder.new(current_user, args).execute + + issues.reorder(args[:order_by] => args[:sort]) end - def filter_issues_labels(issues, labels) - issues.includes(:labels).where('labels.title' => labels.split(',')) + params :issues_params do + optional :labels, type: String, desc: 'Comma-separated list of label names' + optional :milestone, type: String, desc: 'Milestone title' + optional :order_by, type: String, values: %w[created_at updated_at], default: 'created_at', + desc: 'Return issues ordered by `created_at` or `updated_at` fields.' + optional :sort, type: String, values: %w[asc desc], default: 'desc', + desc: 'Return issues sorted in `asc` or `desc` order.' + optional :milestone, type: String, desc: 'Return issues for a specific milestone' + use :pagination end - def filter_issues_milestone(issues, milestone) - issues.includes(:milestone).where('milestones.title' => milestone) + params :issue_params do + optional :description, type: String, desc: 'The description of an issue' + optional :assignee_id, type: Integer, desc: 'The ID of a user to assign issue' + optional :milestone_id, type: Integer, desc: 'The ID of a milestone to assign issue' + optional :labels, type: String, desc: 'Comma-separated list of label names' + optional :due_date, type: String, desc: 'Date time string in the format YEAR-MONTH-DAY' + optional :confidential, type: Boolean, desc: 'Boolean parameter if the issue should be confidential' end end resource :issues do - # Get currently authenticated user's issues - # - # Parameters: - # state (optional) - Return "opened" or "closed" issues - # labels (optional) - Comma-separated list of label names - # order_by (optional) - Return requests ordered by `created_at` or `updated_at` fields. Default is `created_at` - # sort (optional) - Return requests sorted in `asc` or `desc` order. Default is `desc` - # - # Example Requests: - # GET /issues - # GET /issues?state=opened - # GET /issues?state=closed - # GET /issues?labels=foo - # GET /issues?labels=foo,bar - # GET /issues?labels=foo,bar&state=opened + desc "Get currently authenticated user's issues" do + success Entities::Issue + end + params do + optional :state, type: String, values: %w[opened closed all], default: 'all', + desc: 'Return opened, closed, or all issues' + use :issues_params + end get do - issues = current_user.issues.inc_notes_with_associations - issues = filter_issues_state(issues, params[:state]) unless params[:state].nil? - issues = filter_issues_labels(issues, params[:labels]) unless params[:labels].nil? - issues.reorder(issuable_order_by => issuable_sort) + issues = find_issues(scope: 'authored') + present paginate(issues), with: Entities::Issue, current_user: current_user end end + params do + requires :id, type: String, desc: 'The ID of a group' + end resource :groups do - # Get a list of group issues - # - # Parameters: - # id (required) - The ID of a group - # state (optional) - Return "opened" or "closed" issues - # labels (optional) - Comma-separated list of label names - # milestone (optional) - Milestone title - # order_by (optional) - Return requests ordered by `created_at` or `updated_at` fields. Default is `created_at` - # sort (optional) - Return requests sorted in `asc` or `desc` order. Default is `desc` - # - # Example Requests: - # GET /groups/:id/issues - # GET /groups/:id/issues?state=opened - # GET /groups/:id/issues?state=closed - # GET /groups/:id/issues?labels=foo - # GET /groups/:id/issues?labels=foo,bar - # GET /groups/:id/issues?labels=foo,bar&state=opened - # GET /groups/:id/issues?milestone=1.0.0 - # GET /groups/:id/issues?milestone=1.0.0&state=closed + desc 'Get a list of group issues' do + success Entities::Issue + end + params do + optional :state, type: String, values: %w[opened closed all], default: 'opened', + desc: 'Return opened, closed, or all issues' + use :issues_params + end get ":id/issues" do - group = find_group(params[:id]) + group = find_group!(params[:id]) - params[:state] ||= 'opened' - params[:group_id] = group.id - params[:milestone_title] = params.delete(:milestone) - params[:label_name] = params.delete(:labels) - params[:sort] = "#{params.delete(:order_by)}_#{params.delete(:sort)}" if params[:order_by] && params[:sort] - - issues = IssuesFinder.new(current_user, params).execute + issues = find_issues(group_id: group.id, state: params[:state] || 'opened') present paginate(issues), with: Entities::Issue, current_user: current_user end end + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do - # Get a list of project issues - # - # Parameters: - # id (required) - The ID of a project - # iid (optional) - Return the project issue having the given `iid` - # state (optional) - Return "opened" or "closed" issues - # labels (optional) - Comma-separated list of label names - # milestone (optional) - Milestone title - # order_by (optional) - Return requests ordered by `created_at` or `updated_at` fields. Default is `created_at` - # sort (optional) - Return requests sorted in `asc` or `desc` order. Default is `desc` - # - # Example Requests: - # GET /projects/:id/issues - # GET /projects/:id/issues?state=opened - # GET /projects/:id/issues?state=closed - # GET /projects/:id/issues?labels=foo - # GET /projects/:id/issues?labels=foo,bar - # GET /projects/:id/issues?labels=foo,bar&state=opened - # GET /projects/:id/issues?milestone=1.0.0 - # GET /projects/:id/issues?milestone=1.0.0&state=closed - # GET /issues?iid=42 + include TimeTrackingEndpoints + + desc 'Get a list of project issues' do + success Entities::Issue + end + params do + optional :state, type: String, values: %w[opened closed all], default: 'all', + desc: 'Return opened, closed, or all issues' + use :issues_params + end get ":id/issues" do - issues = user_project.issues.inc_notes_with_associations.visible_to_user(current_user) - issues = filter_issues_state(issues, params[:state]) unless params[:state].nil? - issues = filter_issues_labels(issues, params[:labels]) unless params[:labels].nil? - issues = filter_by_iid(issues, params[:iid]) unless params[:iid].nil? + project = find_project(params[:id]) - unless params[:milestone].nil? - issues = filter_issues_milestone(issues, params[:milestone]) - end + issues = find_issues(project_id: project.id) - issues.reorder(issuable_order_by => issuable_sort) - present paginate(issues), with: Entities::Issue, current_user: current_user + present paginate(issues), with: Entities::Issue, current_user: current_user, project: user_project end - # Get a single project issue - # - # Parameters: - # id (required) - The ID of a project - # issue_id (required) - The ID of a project issue - # Example Request: - # GET /projects/:id/issues/:issue_id + desc 'Get a single project issue' do + success Entities::Issue + end + params do + requires :issue_id, type: Integer, desc: 'The ID of a project issue' + end get ":id/issues/:issue_id" do - @issue = find_project_issue(params[:issue_id]) - present @issue, with: Entities::Issue, current_user: current_user + issue = find_project_issue(params[:issue_id]) + present issue, with: Entities::Issue, current_user: current_user, project: user_project end - # Create a new project issue - # - # Parameters: - # id (required) - The ID of a project - # title (required) - The title of an issue - # description (optional) - The description of an issue - # assignee_id (optional) - The ID of a user to assign issue - # milestone_id (optional) - The ID of a milestone to assign issue - # labels (optional) - The labels of an issue - # created_at (optional) - Date time string, ISO 8601 formatted - # due_date (optional) - Date time string in the format YEAR-MONTH-DAY - # Example Request: - # POST /projects/:id/issues + desc 'Create a new project issue' do + success Entities::Issue + end + params do + requires :title, type: String, desc: 'The title of an issue' + optional :created_at, type: DateTime, + desc: 'Date time when the issue was created. Available only for admins and project owners.' + optional :merge_request_for_resolving_discussions, type: Integer, + desc: 'The IID of a merge request for which to resolve discussions' + use :issue_params + end post ':id/issues' do - required_attributes! [:title] - - keys = [:title, :description, :assignee_id, :milestone_id, :due_date] - keys << :created_at if current_user.admin? || user_project.owner == current_user - attrs = attributes_for_keys(keys) - - # Validate label names in advance - if (errors = validate_label_params(params)).any? - render_api_error!({ labels: errors }, 400) + # Setting created_at time only allowed for admins and project owners + unless current_user.admin? || user_project.owner == current_user + params.delete(:created_at) end - project = user_project + issue_params = declared_params(include_missing: false) - issue = ::Issues::CreateService.new(project, current_user, attrs.merge(request: request, api: true)).execute + if merge_request_iid = params[:merge_request_for_resolving_discussions] + issue_params[:merge_request_for_resolving_discussions] = MergeRequestsFinder.new(current_user, project_id: user_project.id). + execute. + find_by(iid: merge_request_iid) + end + issue = ::Issues::CreateService.new(user_project, + current_user, + issue_params.merge(request: request, api: true)).execute if issue.spam? render_api_error!({ error: 'Spam detected' }, 400) end if issue.valid? - # Find or create labels and attach to issue. Labels are valid because - # we already checked its name, so there can't be an error here - if params[:labels].present? - issue.add_labels_by_names(params[:labels].split(',')) - end - - present issue, with: Entities::Issue, current_user: current_user + present issue, with: Entities::Issue, current_user: current_user, project: user_project else render_validation_error!(issue) end end - # Update an existing issue - # - # Parameters: - # id (required) - The ID of a project - # issue_id (required) - The ID of a project issue - # title (optional) - The title of an issue - # description (optional) - The description of an issue - # assignee_id (optional) - The ID of a user to assign issue - # milestone_id (optional) - The ID of a milestone to assign issue - # labels (optional) - The labels of an issue - # state_event (optional) - The state event of an issue (close|reopen) - # updated_at (optional) - Date time string, ISO 8601 formatted - # due_date (optional) - Date time string in the format YEAR-MONTH-DAY - # Example Request: - # PUT /projects/:id/issues/:issue_id + desc 'Update an existing issue' do + success Entities::Issue + end + params do + requires :issue_id, type: Integer, desc: 'The ID of a project issue' + optional :title, type: String, desc: 'The title of an issue' + optional :updated_at, type: DateTime, + desc: 'Date time when the issue was updated. Available only for admins and project owners.' + optional :state_event, type: String, values: %w[reopen close], desc: 'State of the issue' + use :issue_params + at_least_one_of :title, :description, :assignee_id, :milestone_id, + :labels, :created_at, :due_date, :confidential, :state_event + end put ':id/issues/:issue_id' do - issue = user_project.issues.find(params[:issue_id]) + issue = user_project.issues.find(params.delete(:issue_id)) authorize! :update_issue, issue - keys = [:title, :description, :assignee_id, :milestone_id, :state_event, :due_date] - keys << :updated_at if current_user.admin? || user_project.owner == current_user - attrs = attributes_for_keys(keys) - # Validate label names in advance - if (errors = validate_label_params(params)).any? - render_api_error!({ labels: errors }, 400) + # Setting created_at time only allowed for admins and project owners + unless current_user.admin? || user_project.owner == current_user + params.delete(:updated_at) end - issue = ::Issues::UpdateService.new(user_project, current_user, attrs).execute(issue) + update_params = declared_params(include_missing: false).merge(request: request, api: true) + + issue = ::Issues::UpdateService.new(user_project, + current_user, + update_params).execute(issue) + + render_spam_error! if issue.spam? if issue.valid? - # Find or create labels and attach to issue. Labels are valid because - # we already checked its name, so there can't be an error here - if params[:labels] && can?(current_user, :admin_issue, user_project) - issue.remove_labels - # Create and add labels to the new created issue - issue.add_labels_by_names(params[:labels].split(',')) - end - - present issue, with: Entities::Issue, current_user: current_user + present issue, with: Entities::Issue, current_user: current_user, project: user_project else render_validation_error!(issue) end end - # Move an existing issue - # - # Parameters: - # id (required) - The ID of a project - # issue_id (required) - The ID of a project issue - # to_project_id (required) - The ID of the new project - # Example Request: - # POST /projects/:id/issues/:issue_id/move + desc 'Move an existing issue' do + success Entities::Issue + end + params do + requires :issue_id, type: Integer, desc: 'The ID of a project issue' + requires :to_project_id, type: Integer, desc: 'The ID of the new project' + end post ':id/issues/:issue_id/move' do - required_attributes! [:to_project_id] + issue = user_project.issues.find_by(id: params[:issue_id]) + not_found!('Issue') unless issue - issue = user_project.issues.find(params[:issue_id]) - new_project = Project.find(params[:to_project_id]) + new_project = Project.find_by(id: params[:to_project_id]) + not_found!('Project') unless new_project begin issue = ::Issues::MoveService.new(user_project, current_user).execute(issue, new_project) - present issue, with: Entities::Issue, current_user: current_user + present issue, with: Entities::Issue, current_user: current_user, project: user_project rescue ::Issues::MoveService::MoveError => error render_api_error!(error.message, 400) end end - # - # Delete a project issue - # - # Parameters: - # id (required) - The ID of a project - # issue_id (required) - The ID of a project issue - # Example Request: - # DELETE /projects/:id/issues/:issue_id + desc 'Delete a project issue' + params do + requires :issue_id, type: Integer, desc: 'The ID of a project issue' + end delete ":id/issues/:issue_id" do issue = user_project.issues.find_by(id: params[:issue_id]) + not_found!('Issue') unless issue authorize!(:destroy_issue, issue) issue.destroy diff --git a/lib/api/keys.rb b/lib/api/keys.rb index 2b723b79504..767f27ef334 100644 --- a/lib/api/keys.rb +++ b/lib/api/keys.rb @@ -4,10 +4,9 @@ module API before { authenticate! } resource :keys do - # Get single ssh key by id. Only available to admin users. - # - # Example Request: - # GET /keys/:id + desc 'Get single ssh key by id. Only available to admin users' do + success Entities::SSHKeyWithUser + end get ":id" do authenticated_as_admin! diff --git a/lib/api/labels.rb b/lib/api/labels.rb index c806829d69e..d2955af3f95 100644 --- a/lib/api/labels.rb +++ b/lib/api/labels.rb @@ -1,99 +1,98 @@ module API - # Labels API class Labels < Grape::API + include PaginationParams + before { authenticate! } + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do - # Get all labels of the project - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # GET /projects/:id/labels + desc 'Get all labels of the project' do + success Entities::Label + end + params do + use :pagination + end get ':id/labels' do - present user_project.labels, with: Entities::Label, current_user: current_user + present paginate(available_labels), with: Entities::Label, current_user: current_user, project: user_project end - # Creates a new label - # - # Parameters: - # id (required) - The ID of a project - # name (required) - The name of the label to be created - # color (required) - Color of the label given in 6-digit hex - # notation with leading '#' sign (e.g. #FFAABB) - # description (optional) - The description of label to be created - # Example Request: - # POST /projects/:id/labels + desc 'Create a new label' do + success Entities::Label + end + params do + requires :name, type: String, desc: 'The name of the label to be created' + requires :color, type: String, desc: "The color of the label given in 6-digit hex notation with leading '#' sign (e.g. #FFAABB)" + optional :description, type: String, desc: 'The description of label to be created' + optional :priority, type: Integer, desc: 'The priority of the label', allow_blank: true + end post ':id/labels' do authorize! :admin_label, user_project - required_attributes! [:name, :color] - - attrs = attributes_for_keys [:name, :color, :description] - label = user_project.find_label(attrs[:name]) + label = available_labels.find_by(title: params[:name]) conflict!('Label already exists') if label - label = user_project.labels.create(attrs) + priority = params.delete(:priority) + label = user_project.labels.create(declared_params(include_missing: false)) if label.valid? - present label, with: Entities::Label, current_user: current_user + label.prioritize!(user_project, priority) if priority + present label, with: Entities::Label, current_user: current_user, project: user_project else render_validation_error!(label) end end - # Deletes an existing label - # - # Parameters: - # id (required) - The ID of a project - # name (required) - The name of the label to be deleted - # - # Example Request: - # DELETE /projects/:id/labels + desc 'Delete an existing label' do + success Entities::Label + end + params do + requires :name, type: String, desc: 'The name of the label to be deleted' + end delete ':id/labels' do authorize! :admin_label, user_project - required_attributes! [:name] - label = user_project.find_label(params[:name]) + label = user_project.labels.find_by(title: params[:name]) not_found!('Label') unless label - label.destroy + present label.destroy, with: Entities::Label, current_user: current_user, project: user_project end - # Updates an existing label. At least one optional parameter is required. - # - # Parameters: - # id (required) - The ID of a project - # name (required) - The name of the label to be deleted - # new_name (optional) - The new name of the label - # color (optional) - Color of the label given in 6-digit hex - # notation with leading '#' sign (e.g. #FFAABB) - # description (optional) - The description of label to be created - # Example Request: - # PUT /projects/:id/labels + desc 'Update an existing label. At least one optional parameter is required.' do + success Entities::Label + end + params do + requires :name, type: String, desc: 'The name of the label to be updated' + optional :new_name, type: String, desc: 'The new name of the label' + optional :color, type: String, desc: "The new color of the label given in 6-digit hex notation with leading '#' sign (e.g. #FFAABB)" + optional :description, type: String, desc: 'The new description of label' + optional :priority, type: Integer, desc: 'The priority of the label', allow_blank: true + at_least_one_of :new_name, :color, :description, :priority + end put ':id/labels' do authorize! :admin_label, user_project - required_attributes! [:name] - label = user_project.find_label(params[:name]) + label = user_project.labels.find_by(title: params[:name]) not_found!('Label not found') unless label - attrs = attributes_for_keys [:new_name, :color, :description] - - if attrs.empty? - render_api_error!('Required parameters "new_name" or "color" ' \ - 'missing', - 400) - end - + update_priority = params.key?(:priority) + priority = params.delete(:priority) + label_params = declared_params(include_missing: false) # Rename new name to the actual label attribute name - attrs[:name] = attrs.delete(:new_name) if attrs.key?(:new_name) + label_params[:name] = label_params.delete(:new_name) if label_params.key?(:new_name) - if label.update(attrs) - present label, with: Entities::Label, current_user: current_user - else - render_validation_error!(label) + render_validation_error!(label) unless label.update(label_params) + + if update_priority + if priority.nil? + label.unprioritize!(user_project) + else + label.prioritize!(user_project, priority) + end end + + present label, with: Entities::Label, current_user: current_user, project: user_project end end end diff --git a/lib/api/license_templates.rb b/lib/api/license_templates.rb deleted file mode 100644 index d0552299ed0..00000000000 --- a/lib/api/license_templates.rb +++ /dev/null @@ -1,58 +0,0 @@ -module API - # License Templates API - class LicenseTemplates < Grape::API - PROJECT_TEMPLATE_REGEX = - /[\<\{\[] - (project|description| - one\sline\s.+\swhat\sit\sdoes\.) # matching the start and end is enough here - [\>\}\]]/xi.freeze - YEAR_TEMPLATE_REGEX = /[<{\[](year|yyyy)[>}\]]/i.freeze - FULLNAME_TEMPLATE_REGEX = - /[\<\{\[] - (fullname|name\sof\s(author|copyright\sowner)) - [\>\}\]]/xi.freeze - - # Get the list of the available license templates - # - # Parameters: - # popular - Filter licenses to only the popular ones - # - # Example Request: - # GET /licenses - # GET /licenses?popular=1 - get 'licenses' do - options = { - featured: params[:popular].present? ? true : nil - } - present Licensee::License.all(options), with: Entities::RepoLicense - end - - # Get text for specific license - # - # Parameters: - # key (required) - The key of a license - # project - Copyrighted project name - # fullname - Full name of copyright holder - # - # Example Request: - # GET /licenses/mit - # - get 'licenses/:key', requirements: { key: /[\w\.-]+/ } do - required_attributes! [:key] - - not_found!('License') unless Licensee::License.find(params[:key]) - - # We create a fresh Licensee::License object since we'll modify its - # content in place below. - license = Licensee::License.new(params[:key]) - - license.content.gsub!(YEAR_TEMPLATE_REGEX, Time.now.year.to_s) - license.content.gsub!(PROJECT_TEMPLATE_REGEX, params[:project]) if params[:project].present? - - fullname = params[:fullname].presence || current_user.try(:name) - license.content.gsub!(FULLNAME_TEMPLATE_REGEX, fullname) if fullname - - present license, with: Entities::RepoLicense - end - end -end diff --git a/lib/api/lint.rb b/lib/api/lint.rb new file mode 100644 index 00000000000..ae43a4a3237 --- /dev/null +++ b/lib/api/lint.rb @@ -0,0 +1,21 @@ +module API + class Lint < Grape::API + namespace :ci do + desc 'Validation of .gitlab-ci.yml content' + params do + requires :content, type: String, desc: 'Content of .gitlab-ci.yml' + end + post '/lint' do + error = Ci::GitlabCiYamlProcessor.validation_message(params[:content]) + + status 200 + + if error.blank? + { status: 'valid', errors: [] } + else + { status: 'invalid', errors: [error] } + end + end + end + end +end diff --git a/lib/api/members.rb b/lib/api/members.rb index 2fae83f60b2..5f6913d1a27 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -1,39 +1,38 @@ module API class Members < Grape::API + include PaginationParams + before { authenticate! } helpers ::API::Helpers::MembersHelpers %w[group project].each do |source_type| + params do + requires :id, type: String, desc: "The #{source_type} ID" + end resource source_type.pluralize do - # Get a list of group/project members viewable by the authenticated user. - # - # Parameters: - # id (required) - The group/project ID - # query - Query string - # - # Example Request: - # GET /groups/:id/members - # GET /projects/:id/members + desc 'Gets a list of group or project members viewable by the authenticated user.' do + success Entities::Member + end + params do + optional :query, type: String, desc: 'A query string to search for members' + use :pagination + end get ":id/members" do source = find_source(source_type, params[:id]) - members = source.members.includes(:user) - members = members.joins(:user).merge(User.search(params[:query])) if params[:query] - members = paginate(members) + users = source.users + users = users.merge(User.search(params[:query])) if params[:query] - present members.map(&:user), with: Entities::Member, members: members + present paginate(users), with: Entities::Member, source: source end - # Get a group/project member - # - # Parameters: - # id (required) - The group/project ID - # user_id (required) - The user ID of the member - # - # Example Request: - # GET /groups/:id/members/:user_id - # GET /projects/:id/members/:user_id + desc 'Gets a member of a group or project.' do + success Entities::Member + end + params do + requires :user_id, type: Integer, desc: 'The user ID of the member' + end get ":id/members/:user_id" do source = find_source(source_type, params[:id]) @@ -43,93 +42,57 @@ module API present member.user, with: Entities::Member, member: member end - # Add a new group/project member - # - # Parameters: - # id (required) - The group/project ID - # user_id (required) - The user ID of the new member - # access_level (required) - A valid access level - # - # Example Request: - # POST /groups/:id/members - # POST /projects/:id/members + desc 'Adds a member to a group or project.' do + success Entities::Member + end + params do + requires :user_id, type: Integer, desc: 'The user ID of the new member' + requires :access_level, type: Integer, desc: 'A valid access level (defaults: `30`, developer access level)' + optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY' + end post ":id/members" do source = find_source(source_type, params[:id]) authorize_admin_source!(source_type, source) - required_attributes! [:user_id, :access_level] - - access_requester = source.requesters.find_by(user_id: params[:user_id]) - if access_requester - # We pass current_user = access_requester so that the requester doesn't - # receive a "access denied" email - ::Members::DestroyService.new(access_requester, access_requester.user).execute - end member = source.members.find_by(user_id: params[:user_id]) + conflict!('Member already exists') if member - # This is to ensure back-compatibility but 409 behavior should be used - # for both project and group members in 9.0! - conflict!('Member already exists') if source_type == 'group' && member - - unless member - source.add_user(params[:user_id], params[:access_level], current_user) - member = source.members.find_by(user_id: params[:user_id]) - end + member = source.add_user(params[:user_id], params[:access_level], current_user: current_user, expires_at: params[:expires_at]) - if member + if member.persisted? && member.valid? present member.user, with: Entities::Member, member: member else - # Since `source.add_user` doesn't return a member object, we have to - # build a new one and populate its errors in order to render them. - member = source.members.build(attributes_for_keys([:user_id, :access_level])) - member.valid? # populate the errors - - # This is to ensure back-compatibility but 400 behavior should be used - # for all validation errors in 9.0! - render_api_error!('Access level is not known', 422) if member.errors.key?(:access_level) render_validation_error!(member) end end - # Update a group/project member - # - # Parameters: - # id (required) - The group/project ID - # user_id (required) - The user ID of the member - # access_level (required) - A valid access level - # - # Example Request: - # PUT /groups/:id/members/:user_id - # PUT /projects/:id/members/:user_id + desc 'Updates a member of a group or project.' do + success Entities::Member + end + params do + requires :user_id, type: Integer, desc: 'The user ID of the new member' + requires :access_level, type: Integer, desc: 'A valid access level' + optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY' + end put ":id/members/:user_id" do - source = find_source(source_type, params[:id]) + source = find_source(source_type, params.delete(:id)) authorize_admin_source!(source_type, source) - required_attributes! [:user_id, :access_level] - member = source.members.find_by!(user_id: params[:user_id]) + member = source.members.find_by!(user_id: params.delete(:user_id)) - if member.update_attributes(access_level: params[:access_level]) + if member.update_attributes(declared_params(include_missing: false)) present member.user, with: Entities::Member, member: member else - # This is to ensure back-compatibility but 400 behavior should be used - # for all validation errors in 9.0! - render_api_error!('Access level is not known', 422) if member.errors.key?(:access_level) render_validation_error!(member) end end - # Remove a group/project member - # - # Parameters: - # id (required) - The group/project ID - # user_id (required) - The user ID of the member - # - # Example Request: - # DELETE /groups/:id/members/:user_id - # DELETE /projects/:id/members/:user_id + desc 'Removes a user from a group or project.' + params do + requires :user_id, type: Integer, desc: 'The user ID of the member' + end delete ":id/members/:user_id" do source = find_source(source_type, params[:id]) - required_attributes! [:user_id] # This is to ensure back-compatibility but find_by! should be used # in that casse in 9.0! @@ -144,7 +107,7 @@ module API if member.nil? { message: "Access revoked", id: params[:user_id].to_i } else - ::Members::DestroyService.new(member, current_user).execute + ::Members::DestroyService.new(source, current_user, declared_params).execute present member.user, with: Entities::Member, member: member end diff --git a/lib/api/merge_request_diffs.rb b/lib/api/merge_request_diffs.rb new file mode 100644 index 00000000000..4901a7cfea6 --- /dev/null +++ b/lib/api/merge_request_diffs.rb @@ -0,0 +1,43 @@ +module API + # MergeRequestDiff API + class MergeRequestDiffs < Grape::API + include PaginationParams + + before { authenticate! } + + resource :projects do + desc 'Get a list of merge request diff versions' do + detail 'This feature was introduced in GitLab 8.12.' + success Entities::MergeRequestDiff + end + + params do + requires :id, type: String, desc: 'The ID of a project' + requires :merge_request_id, type: Integer, desc: 'The ID of a merge request' + use :pagination + end + get ":id/merge_requests/:merge_request_id/versions" do + merge_request = find_merge_request_with_access(params[:merge_request_id]) + + present paginate(merge_request.merge_request_diffs), with: Entities::MergeRequestDiff + end + + desc 'Get a single merge request diff version' do + detail 'This feature was introduced in GitLab 8.12.' + success Entities::MergeRequestDiffFull + end + + params do + requires :id, type: String, desc: 'The ID of a project' + requires :merge_request_id, type: Integer, desc: 'The ID of a merge request' + requires :version_id, type: Integer, desc: 'The ID of a merge request diff version' + end + + get ":id/merge_requests/:merge_request_id/versions/:version_id" do + merge_request = find_merge_request_with_access(params[:merge_request_id]) + + present merge_request.merge_request_diffs.find(params[:version_id]), with: Entities::MergeRequestDiffFull + end + end + end +end diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 2b685621da9..bdd764abfeb 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -1,9 +1,15 @@ module API - # MergeRequest API class MergeRequests < Grape::API + include PaginationParams + before { authenticate! } + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do + include TimeTrackingEndpoints + helpers do def handle_merge_request_errors!(errors) if errors[:project_access].any? @@ -18,320 +24,246 @@ module API render_api_error!(errors, 400) end + + params :optional_params do + optional :description, type: String, desc: 'The description of the merge request' + optional :assignee_id, type: Integer, desc: 'The ID of a user to assign the merge request' + optional :milestone_id, type: Integer, desc: 'The ID of a milestone to assign the merge request' + optional :labels, type: String, desc: 'Comma-separated list of label names' + optional :remove_source_branch, type: Boolean, desc: 'Remove source branch when merging' + end end - # List merge requests - # - # Parameters: - # id (required) - The ID of a project - # iid (optional) - Return the project MR having the given `iid` - # state (optional) - Return requests "merged", "opened" or "closed" - # order_by (optional) - Return requests ordered by `created_at` or `updated_at` fields. Default is `created_at` - # sort (optional) - Return requests sorted in `asc` or `desc` order. Default is `desc` - # - # Example: - # GET /projects/:id/merge_requests - # GET /projects/:id/merge_requests?state=opened - # GET /projects/:id/merge_requests?state=closed - # GET /projects/:id/merge_requests?order_by=created_at - # GET /projects/:id/merge_requests?order_by=updated_at - # GET /projects/:id/merge_requests?sort=desc - # GET /projects/:id/merge_requests?sort=asc - # GET /projects/:id/merge_requests?iid=42 - # + desc 'List merge requests' do + success Entities::MergeRequest + end + params do + optional :state, type: String, values: %w[opened closed merged all], default: 'all', + desc: 'Return opened, closed, merged, or all merge requests' + optional :order_by, type: String, values: %w[created_at updated_at], default: 'created_at', + desc: 'Return merge requests ordered by `created_at` or `updated_at` fields.' + optional :sort, type: String, values: %w[asc desc], default: 'desc', + desc: 'Return merge requests sorted in `asc` or `desc` order.' + optional :iids, type: Array[Integer], desc: 'The IID array of merge requests' + use :pagination + end get ":id/merge_requests" do authorize! :read_merge_request, user_project - merge_requests = user_project.merge_requests.inc_notes_with_associations - unless params[:iid].nil? - merge_requests = filter_by_iid(merge_requests, params[:iid]) - end + merge_requests = user_project.merge_requests.inc_notes_with_associations + merge_requests = filter_by_iid(merge_requests, params[:iids]) if params[:iids].present? merge_requests = - case params["state"] - when "opened" then merge_requests.opened - when "closed" then merge_requests.closed - when "merged" then merge_requests.merged + case params[:state] + when 'opened' then merge_requests.opened + when 'closed' then merge_requests.closed + when 'merged' then merge_requests.merged else merge_requests end - merge_requests = merge_requests.reorder(issuable_order_by => issuable_sort) - present paginate(merge_requests), with: Entities::MergeRequest, current_user: current_user - end - - # Create MR - # - # Parameters: - # - # id (required) - The ID of a project - this will be the source of the merge request - # source_branch (required) - The source branch - # target_branch (required) - The target branch - # target_project_id - The target project of the merge request defaults to the :id of the project - # assignee_id - Assignee user ID - # title (required) - Title of MR - # description - Description of MR - # labels (optional) - Labels for MR as a comma-separated list - # milestone_id (optional) - Milestone ID - # - # Example: - # POST /projects/:id/merge_requests - # + merge_requests = merge_requests.reorder(params[:order_by] => params[:sort]) + present paginate(merge_requests), with: Entities::MergeRequest, current_user: current_user, project: user_project + end + + desc 'Create a merge request' do + success Entities::MergeRequest + end + params do + requires :title, type: String, desc: 'The title of the merge request' + requires :source_branch, type: String, desc: 'The source branch' + requires :target_branch, type: String, desc: 'The target branch' + optional :target_project_id, type: Integer, + desc: 'The target project of the merge request defaults to the :id of the project' + use :optional_params + end post ":id/merge_requests" do authorize! :create_merge_request, user_project - required_attributes! [:source_branch, :target_branch, :title] - attrs = attributes_for_keys [:source_branch, :target_branch, :assignee_id, :title, :target_project_id, :description, :milestone_id] - # Validate label names in advance - if (errors = validate_label_params(params)).any? - render_api_error!({ labels: errors }, 400) - end + mr_params = declared_params(include_missing: false) + mr_params[:force_remove_source_branch] = mr_params.delete(:remove_source_branch) if mr_params[:remove_source_branch].present? - merge_request = ::MergeRequests::CreateService.new(user_project, current_user, attrs).execute + merge_request = ::MergeRequests::CreateService.new(user_project, current_user, mr_params).execute if merge_request.valid? - # Find or create labels and attach to issue - if params[:labels].present? - merge_request.add_labels_by_names(params[:labels].split(",")) - end - - present merge_request, with: Entities::MergeRequest, current_user: current_user + present merge_request, with: Entities::MergeRequest, current_user: current_user, project: user_project else handle_merge_request_errors! merge_request.errors end end - # Delete a MR - # - # Parameters: - # id (required) - The ID of the project - # merge_request_id (required) - The MR id + desc 'Delete a merge request' + params do + requires :merge_request_id, type: Integer, desc: 'The ID of a merge request' + end delete ":id/merge_requests/:merge_request_id" do - merge_request = user_project.merge_requests.find_by(id: params[:merge_request_id]) + merge_request = find_project_merge_request(params[:merge_request_id]) authorize!(:destroy_merge_request, merge_request) merge_request.destroy end - # Routing "merge_request/:merge_request_id/..." is DEPRECATED and WILL BE REMOVED in version 9.0 - # Use "merge_requests/:merge_request_id/..." instead. - # - [":id/merge_request/:merge_request_id", ":id/merge_requests/:merge_request_id"].each do |path| - # Show MR - # - # Parameters: - # id (required) - The ID of a project - # merge_request_id (required) - The ID of MR - # - # Example: - # GET /projects/:id/merge_requests/:merge_request_id - # - get path do - merge_request = user_project.merge_requests.find(params[:merge_request_id]) - - authorize! :read_merge_request, merge_request - - present merge_request, with: Entities::MergeRequest, current_user: current_user - end + params do + requires :merge_request_id, type: Integer, desc: 'The ID of a merge request' + end + desc 'Get a single merge request' do + success Entities::MergeRequest + end + get ':id/merge_requests/:merge_request_id' do + merge_request = find_merge_request_with_access(params[:merge_request_id]) - # Show MR commits - # - # Parameters: - # id (required) - The ID of a project - # merge_request_id (required) - The ID of MR - # - # Example: - # GET /projects/:id/merge_requests/:merge_request_id/commits - # - get "#{path}/commits" do - merge_request = user_project.merge_requests. - find(params[:merge_request_id]) - authorize! :read_merge_request, merge_request - present merge_request.commits, with: Entities::RepoCommit - end + present merge_request, with: Entities::MergeRequest, current_user: current_user, project: user_project + end - # Show MR changes - # - # Parameters: - # id (required) - The ID of a project - # merge_request_id (required) - The ID of MR - # - # Example: - # GET /projects/:id/merge_requests/:merge_request_id/changes - # - get "#{path}/changes" do - merge_request = user_project.merge_requests. - find(params[:merge_request_id]) - authorize! :read_merge_request, merge_request - present merge_request, with: Entities::MergeRequestChanges, current_user: current_user - end + desc 'Get the commits of a merge request' do + success Entities::RepoCommit + end + get ':id/merge_requests/:merge_request_id/commits' do + merge_request = find_merge_request_with_access(params[:merge_request_id]) + commits = ::Kaminari.paginate_array(merge_request.commits) - # Update MR - # - # Parameters: - # id (required) - The ID of a project - # merge_request_id (required) - ID of MR - # target_branch - The target branch - # assignee_id - Assignee user ID - # title - Title of MR - # state_event - Status of MR. (close|reopen|merge) - # description - Description of MR - # labels (optional) - Labels for a MR as a comma-separated list - # milestone_id (optional) - Milestone ID - # Example: - # PUT /projects/:id/merge_requests/:merge_request_id - # - put path do - attrs = attributes_for_keys [:target_branch, :assignee_id, :title, :state_event, :description, :milestone_id] - merge_request = user_project.merge_requests.find(params[:merge_request_id]) - authorize! :update_merge_request, merge_request - - # Ensure source_branch is not specified - if params[:source_branch].present? - render_api_error!('Source branch cannot be changed', 400) - end + present paginate(commits), with: Entities::RepoCommit + end - # Validate label names in advance - if (errors = validate_label_params(params)).any? - render_api_error!({ labels: errors }, 400) - end + desc 'Show the merge request changes' do + success Entities::MergeRequestChanges + end + get ':id/merge_requests/:merge_request_id/changes' do + merge_request = find_merge_request_with_access(params[:merge_request_id]) - merge_request = ::MergeRequests::UpdateService.new(user_project, current_user, attrs).execute(merge_request) + present merge_request, with: Entities::MergeRequestChanges, current_user: current_user + end - if merge_request.valid? - # Find or create labels and attach to issue - unless params[:labels].nil? - merge_request.remove_labels - merge_request.add_labels_by_names(params[:labels].split(",")) - end + desc 'Update a merge request' do + success Entities::MergeRequest + end + params do + optional :title, type: String, allow_blank: false, desc: 'The title of the merge request' + optional :target_branch, type: String, allow_blank: false, desc: 'The target branch' + optional :state_event, type: String, values: %w[close reopen], + desc: 'Status of the merge request' + use :optional_params + at_least_one_of :title, :target_branch, :description, :assignee_id, + :milestone_id, :labels, :state_event, + :remove_source_branch + end + put ':id/merge_requests/:merge_request_id' do + merge_request = find_merge_request_with_access(params.delete(:merge_request_id), :update_merge_request) - present merge_request, with: Entities::MergeRequest, current_user: current_user - else - handle_merge_request_errors! merge_request.errors - end - end + mr_params = declared_params(include_missing: false) + mr_params[:force_remove_source_branch] = mr_params.delete(:remove_source_branch) if mr_params[:remove_source_branch].present? - # Merge MR - # - # Parameters: - # id (required) - The ID of a project - # merge_request_id (required) - ID of MR - # merge_commit_message (optional) - Custom merge commit message - # should_remove_source_branch (optional) - When true, the source branch will be deleted if possible - # merge_when_build_succeeds (optional) - When true, this MR will be merged when the build succeeds - # sha (optional) - When present, must have the HEAD SHA of the source branch - # Example: - # PUT /projects/:id/merge_requests/:merge_request_id/merge - # - put "#{path}/merge" do - merge_request = user_project.merge_requests.find(params[:merge_request_id]) - - # Merge request can not be merged - # because user dont have permissions to push into target branch - unauthorized! unless merge_request.can_be_merged_by?(current_user) - - not_allowed! unless merge_request.mergeable_state? - - render_api_error!('Branch cannot be merged', 406) unless merge_request.mergeable? - - if params[:sha] && merge_request.diff_head_sha != params[:sha] - render_api_error!("SHA does not match HEAD of source branch: #{merge_request.diff_head_sha}", 409) - end + merge_request = ::MergeRequests::UpdateService.new(user_project, current_user, mr_params).execute(merge_request) - merge_params = { - commit_message: params[:merge_commit_message], - should_remove_source_branch: params[:should_remove_source_branch] - } - - if to_boolean(params[:merge_when_build_succeeds]) && merge_request.pipeline && merge_request.pipeline.active? - ::MergeRequests::MergeWhenBuildSucceedsService.new(merge_request.target_project, current_user, merge_params). - execute(merge_request) - else - ::MergeRequests::MergeService.new(merge_request.target_project, current_user, merge_params). - execute(merge_request) - end - - present merge_request, with: Entities::MergeRequest, current_user: current_user + if merge_request.valid? + present merge_request, with: Entities::MergeRequest, current_user: current_user, project: user_project + else + handle_merge_request_errors! merge_request.errors end + end - # Cancel Merge if Merge When build succeeds is enabled - # Parameters: - # id (required) - The ID of a project - # merge_request_id (required) - ID of MR - # - post "#{path}/cancel_merge_when_build_succeeds" do - merge_request = user_project.merge_requests.find(params[:merge_request_id]) + desc 'Merge a merge request' do + success Entities::MergeRequest + end + params do + optional :merge_commit_message, type: String, desc: 'Custom merge commit message' + optional :should_remove_source_branch, type: Boolean, + desc: 'When true, the source branch will be deleted if possible' + optional :merge_when_build_succeeds, type: Boolean, + desc: 'When true, this merge request will be merged when the pipeline succeeds' + optional :sha, type: String, desc: 'When present, must have the HEAD SHA of the source branch' + end + put ':id/merge_requests/:merge_request_id/merge' do + merge_request = find_project_merge_request(params[:merge_request_id]) - unauthorized! unless merge_request.can_cancel_merge_when_build_succeeds?(current_user) + # Merge request can not be merged + # because user dont have permissions to push into target branch + unauthorized! unless merge_request.can_be_merged_by?(current_user) - ::MergeRequest::MergeWhenBuildSucceedsService.new(merge_request.target_project, current_user).cancel(merge_request) - end + not_allowed! unless merge_request.mergeable_state? - # Duplicate. DEPRECATED and WILL BE REMOVED in 9.0. - # Use GET "/projects/:id/merge_requests/:merge_request_id/notes" instead - # - # Get a merge request's comments - # - # Parameters: - # id (required) - The ID of a project - # merge_request_id (required) - ID of MR - # Examples: - # GET /projects/:id/merge_requests/:merge_request_id/comments - # - get "#{path}/comments" do - merge_request = user_project.merge_requests.find(params[:merge_request_id]) - - authorize! :read_merge_request, merge_request - - present paginate(merge_request.notes.fresh), with: Entities::MRNote + render_api_error!('Branch cannot be merged', 406) unless merge_request.mergeable? + + if params[:sha] && merge_request.diff_head_sha != params[:sha] + render_api_error!("SHA does not match HEAD of source branch: #{merge_request.diff_head_sha}", 409) end - # Duplicate. DEPRECATED and WILL BE REMOVED in 9.0. - # Use POST "/projects/:id/merge_requests/:merge_request_id/notes" instead - # - # Post comment to merge request - # - # Parameters: - # id (required) - The ID of a project - # merge_request_id (required) - ID of MR - # note (required) - Text of comment - # Examples: - # POST /projects/:id/merge_requests/:merge_request_id/comments - # - post "#{path}/comments" do - required_attributes! [:note] - - merge_request = user_project.merge_requests.find(params[:merge_request_id]) - - authorize! :create_note, merge_request - - opts = { - note: params[:note], - noteable_type: 'MergeRequest', - noteable_id: merge_request.id - } - - note = ::Notes::CreateService.new(user_project, current_user, opts).execute - - if note.save - present note, with: Entities::MRNote - else - render_api_error!("Failed to save note #{note.errors.messages}", 400) - end + merge_params = { + commit_message: params[:merge_commit_message], + should_remove_source_branch: params[:should_remove_source_branch] + } + + if params[:merge_when_build_succeeds] && merge_request.head_pipeline && merge_request.head_pipeline.active? + ::MergeRequests::MergeWhenPipelineSucceedsService + .new(merge_request.target_project, current_user, merge_params) + .execute(merge_request) + else + ::MergeRequests::MergeService + .new(merge_request.target_project, current_user, merge_params) + .execute(merge_request) end - # List issues that will close on merge - # - # Parameters: - # id (required) - The ID of a project - # merge_request_id (required) - ID of MR - # Examples: - # GET /projects/:id/merge_requests/:merge_request_id/closes_issues - get "#{path}/closes_issues" do - merge_request = user_project.merge_requests.find(params[:merge_request_id]) - issues = ::Kaminari.paginate_array(merge_request.closes_issues(current_user)) - present paginate(issues), with: issue_entity(user_project), current_user: current_user + present merge_request, with: Entities::MergeRequest, current_user: current_user, project: user_project + end + + desc 'Cancel merge if "Merge When Pipeline Succeeds" is enabled' do + success Entities::MergeRequest + end + post ':id/merge_requests/:merge_request_id/cancel_merge_when_build_succeeds' do + merge_request = find_project_merge_request(params[:merge_request_id]) + + unauthorized! unless merge_request.can_cancel_merge_when_build_succeeds?(current_user) + + ::MergeRequest::MergeWhenPipelineSucceedsService + .new(merge_request.target_project, current_user) + .cancel(merge_request) + end + + desc 'Get the comments of a merge request' do + success Entities::MRNote + end + params do + use :pagination + end + get ':id/merge_requests/:merge_request_id/comments' do + merge_request = find_merge_request_with_access(params[:merge_request_id]) + present paginate(merge_request.notes.fresh), with: Entities::MRNote + end + + desc 'Post a comment to a merge request' do + success Entities::MRNote + end + params do + requires :note, type: String, desc: 'The text of the comment' + end + post ':id/merge_requests/:merge_request_id/comments' do + merge_request = find_merge_request_with_access(params[:merge_request_id], :create_note) + + opts = { + note: params[:note], + noteable_type: 'MergeRequest', + noteable_id: merge_request.id + } + + note = ::Notes::CreateService.new(user_project, current_user, opts).execute + + if note.save + present note, with: Entities::MRNote + else + render_api_error!("Failed to save note #{note.errors.messages}", 400) end end + + desc 'List issues that will be closed on merge' do + success Entities::MRNote + end + params do + use :pagination + end + get ':id/merge_requests/:merge_request_id/closes_issues' do + merge_request = find_merge_request_with_access(params[:merge_request_id]) + issues = ::Kaminari.paginate_array(merge_request.closes_issues(current_user)) + present paginate(issues), with: issue_entity(user_project), current_user: current_user + end end end end diff --git a/lib/api/milestones.rb b/lib/api/milestones.rb index 7a0cb7c99f3..0b4ed76b35c 100644 --- a/lib/api/milestones.rb +++ b/lib/api/milestones.rb @@ -1,6 +1,7 @@ module API - # Milestones API class Milestones < Grape::API + include PaginationParams + before { authenticate! } helpers do @@ -11,19 +12,27 @@ module API else milestones end end + + params :optional_params do + optional :description, type: String, desc: 'The description of the milestone' + optional :due_date, type: String, desc: 'The due date of the milestone. The ISO 8601 date format (%Y-%m-%d)' + optional :start_date, type: String, desc: 'The start date of the milestone. The ISO 8601 date format (%Y-%m-%d)' + end end + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do - # Get a list of project milestones - # - # Parameters: - # id (required) - The ID of a project - # state (optional) - Return "active" or "closed" milestones - # Example Request: - # GET /projects/:id/milestones - # GET /projects/:id/milestones?iid=42 - # GET /projects/:id/milestones?state=active - # GET /projects/:id/milestones?state=closed + desc 'Get a list of project milestones' do + success Entities::Milestone + end + params do + optional :state, type: String, values: %w[active closed all], default: 'all', + desc: 'Return "active", "closed", or "all" milestones' + optional :iid, type: Array[Integer], desc: 'The IID of the milestone' + use :pagination + end get ":id/milestones" do authorize! :read_milestone, user_project @@ -34,34 +43,30 @@ module API present paginate(milestones), with: Entities::Milestone end - # Get a single project milestone - # - # Parameters: - # id (required) - The ID of a project - # milestone_id (required) - The ID of a project milestone - # Example Request: - # GET /projects/:id/milestones/:milestone_id + desc 'Get a single project milestone' do + success Entities::Milestone + end + params do + requires :milestone_id, type: Integer, desc: 'The ID of a project milestone' + end get ":id/milestones/:milestone_id" do authorize! :read_milestone, user_project - @milestone = user_project.milestones.find(params[:milestone_id]) - present @milestone, with: Entities::Milestone + milestone = user_project.milestones.find(params[:milestone_id]) + present milestone, with: Entities::Milestone end - # Create a new project milestone - # - # Parameters: - # id (required) - The ID of the project - # title (required) - The title of the milestone - # description (optional) - The description of the milestone - # due_date (optional) - The due date of the milestone - # Example Request: - # POST /projects/:id/milestones + desc 'Create a new project milestone' do + success Entities::Milestone + end + params do + requires :title, type: String, desc: 'The title of the milestone' + use :optional_params + end post ":id/milestones" do authorize! :admin_milestone, user_project - required_attributes! [:title] - attrs = attributes_for_keys [:title, :description, :due_date] - milestone = ::Milestones::CreateService.new(user_project, current_user, attrs).execute + + milestone = ::Milestones::CreateService.new(user_project, current_user, declared_params).execute if milestone.valid? present milestone, with: Entities::Milestone @@ -70,22 +75,23 @@ module API end end - # Update an existing project milestone - # - # Parameters: - # id (required) - The ID of a project - # milestone_id (required) - The ID of a project milestone - # title (optional) - The title of a milestone - # description (optional) - The description of a milestone - # due_date (optional) - The due date of a milestone - # state_event (optional) - The state event of the milestone (close|activate) - # Example Request: - # PUT /projects/:id/milestones/:milestone_id + desc 'Update an existing project milestone' do + success Entities::Milestone + end + params do + requires :milestone_id, type: Integer, desc: 'The ID of a project milestone' + optional :title, type: String, desc: 'The title of the milestone' + optional :state_event, type: String, values: %w[close activate], + desc: 'The state event of the milestone ' + use :optional_params + at_least_one_of :title, :description, :due_date, :state_event + end put ":id/milestones/:milestone_id" do authorize! :admin_milestone, user_project - attrs = attributes_for_keys [:title, :description, :due_date, :state_event] - milestone = user_project.milestones.find(params[:milestone_id]) - milestone = ::Milestones::UpdateService.new(user_project, current_user, attrs).execute(milestone) + milestone = user_project.milestones.find(params.delete(:milestone_id)) + + milestone_params = declared_params(include_missing: false) + milestone = ::Milestones::UpdateService.new(user_project, current_user, milestone_params).execute(milestone) if milestone.valid? present milestone, with: Entities::Milestone @@ -94,26 +100,47 @@ module API end end - # Get all issues for a single project milestone - # - # Parameters: - # id (required) - The ID of a project - # milestone_id (required) - The ID of a project milestone - # Example Request: - # GET /projects/:id/milestones/:milestone_id/issues + desc 'Get all issues for a single project milestone' do + success Entities::Issue + end + params do + requires :milestone_id, type: Integer, desc: 'The ID of a project milestone' + use :pagination + end get ":id/milestones/:milestone_id/issues" do authorize! :read_milestone, user_project - @milestone = user_project.milestones.find(params[:milestone_id]) + milestone = user_project.milestones.find(params[:milestone_id]) finder_params = { project_id: user_project.id, - milestone_title: @milestone.title, - state: 'all' + milestone_title: milestone.title } issues = IssuesFinder.new(current_user, finder_params).execute - present paginate(issues), with: Entities::Issue, current_user: current_user + present paginate(issues), with: Entities::Issue, current_user: current_user, project: user_project + end + + desc 'Get all merge requests for a single project milestone' do + detail 'This feature was introduced in GitLab 9.' + success Entities::MergeRequest + end + params do + requires :milestone_id, type: Integer, desc: 'The ID of a project milestone' + use :pagination + end + get ':id/milestones/:milestone_id/merge_requests' do + authorize! :read_milestone, user_project + + milestone = user_project.milestones.find(params[:milestone_id]) + + finder_params = { + project_id: user_project.id, + milestone_id: milestone.id + } + + merge_requests = MergeRequestsFinder.new(current_user, finder_params).execute + present paginate(merge_requests), with: Entities::MergeRequest, current_user: current_user, project: user_project end end end diff --git a/lib/api/namespaces.rb b/lib/api/namespaces.rb index 50d3729449e..30761cb9b55 100644 --- a/lib/api/namespaces.rb +++ b/lib/api/namespaces.rb @@ -1,23 +1,23 @@ module API - # namespaces API class Namespaces < Grape::API + include PaginationParams + before { authenticate! } resource :namespaces do - # Get a namespaces list - # - # Example Request: - # GET /namespaces + desc 'Get a namespaces list' do + success Entities::Namespace + end + params do + optional :search, type: String, desc: "Search query for namespaces" + use :pagination + end get do - @namespaces = if current_user.admin - Namespace.all - else - current_user.namespaces - end - @namespaces = @namespaces.search(params[:search]) if params[:search].present? - @namespaces = paginate @namespaces + namespaces = current_user.admin ? Namespace.all : current_user.namespaces + + namespaces = namespaces.search(params[:search]) if params[:search].present? - present @namespaces, with: Entities::Namespace + present paginate(namespaces), with: Entities::Namespace end end end diff --git a/lib/api/notes.rb b/lib/api/notes.rb index 8bfa998dc53..f559a7f74a0 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -1,27 +1,29 @@ module API - # Notes API class Notes < Grape::API + include PaginationParams + before { authenticate! } - NOTEABLE_TYPES = [Issue, MergeRequest, Snippet] + NOTEABLE_TYPES = [Issue, MergeRequest, Snippet].freeze + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do NOTEABLE_TYPES.each do |noteable_type| noteables_str = noteable_type.to_s.underscore.pluralize - noteable_id_str = "#{noteable_type.to_s.underscore}_id" - - # Get a list of project +noteable+ notes - # - # Parameters: - # id (required) - The ID of a project - # noteable_id (required) - The ID of an issue or snippet - # Example Request: - # GET /projects/:id/issues/:noteable_id/notes - # GET /projects/:id/snippets/:noteable_id/notes - get ":id/#{noteables_str}/:#{noteable_id_str}/notes" do - @noteable = user_project.send(noteables_str.to_sym).find(params[noteable_id_str.to_sym]) - - if can?(current_user, noteable_read_ability_name(@noteable), @noteable) + + desc 'Get a list of project +noteable+ notes' do + success Entities::Note + end + params do + requires :noteable_id, type: Integer, desc: 'The ID of the noteable' + use :pagination + end + get ":id/#{noteables_str}/:noteable_id/notes" do + noteable = user_project.send(noteables_str.to_sym).find(params[:noteable_id]) + + if can?(current_user, noteable_read_ability_name(noteable), noteable) # We exclude notes that are cross-references and that cannot be viewed # by the current user. By doing this exclusion at this level and not # at the DB query level (which we cannot in that case), the current @@ -31,7 +33,7 @@ module API # paginate() only works with a relation. This could lead to a # mismatch between the pagination headers info and the actual notes # array returned, but this is really a edge-case. - paginate(@noteable.notes). + paginate(noteable.notes). reject { |n| n.cross_reference_not_visible_for?(current_user) } present notes, with: Entities::Note else @@ -39,72 +41,68 @@ module API end end - # Get a single +noteable+ note - # - # Parameters: - # id (required) - The ID of a project - # noteable_id (required) - The ID of an issue or snippet - # note_id (required) - The ID of a note - # Example Request: - # GET /projects/:id/issues/:noteable_id/notes/:note_id - # GET /projects/:id/snippets/:noteable_id/notes/:note_id - get ":id/#{noteables_str}/:#{noteable_id_str}/notes/:note_id" do - @noteable = user_project.send(noteables_str.to_sym).find(params[noteable_id_str.to_sym]) - @note = @noteable.notes.find(params[:note_id]) - can_read_note = can?(current_user, noteable_read_ability_name(@noteable), @noteable) && !@note.cross_reference_not_visible_for?(current_user) + desc 'Get a single +noteable+ note' do + success Entities::Note + end + params do + requires :note_id, type: Integer, desc: 'The ID of a note' + requires :noteable_id, type: Integer, desc: 'The ID of the noteable' + end + get ":id/#{noteables_str}/:noteable_id/notes/:note_id" do + noteable = user_project.send(noteables_str.to_sym).find(params[:noteable_id]) + note = noteable.notes.find(params[:note_id]) + can_read_note = can?(current_user, noteable_read_ability_name(noteable), noteable) && !note.cross_reference_not_visible_for?(current_user) if can_read_note - present @note, with: Entities::Note + present note, with: Entities::Note else not_found!("Note") end end - # Create a new +noteable+ note - # - # Parameters: - # id (required) - The ID of a project - # noteable_id (required) - The ID of an issue or snippet - # body (required) - The content of a note - # created_at (optional) - The date - # Example Request: - # POST /projects/:id/issues/:noteable_id/notes - # POST /projects/:id/snippets/:noteable_id/notes - post ":id/#{noteables_str}/:#{noteable_id_str}/notes" do - required_attributes! [:body] - + desc 'Create a new +noteable+ note' do + success Entities::Note + end + params do + requires :noteable_id, type: Integer, desc: 'The ID of the noteable' + requires :body, type: String, desc: 'The content of a note' + optional :created_at, type: String, desc: 'The creation date of the note' + end + post ":id/#{noteables_str}/:noteable_id/notes" do opts = { - note: params[:body], - noteable_type: noteables_str.classify, - noteable_id: params[noteable_id_str] + note: params[:body], + noteable_type: noteables_str.classify, + noteable_id: params[:noteable_id] } - if params[:created_at] && (current_user.is_admin? || user_project.owner == current_user) - opts[:created_at] = params[:created_at] - end + noteable = user_project.send(noteables_str.to_sym).find(params[:noteable_id]) + + if can?(current_user, noteable_read_ability_name(noteable), noteable) + if params[:created_at] && (current_user.is_admin? || user_project.owner == current_user) + opts[:created_at] = params[:created_at] + end - @note = ::Notes::CreateService.new(user_project, current_user, opts).execute + note = ::Notes::CreateService.new(user_project, current_user, opts).execute - if @note.valid? - present @note, with: Entities::Note + if note.valid? + present note, with: Entities.const_get(note.class.name) + else + not_found!("Note #{note.errors.messages}") + end else - not_found!("Note #{@note.errors.messages}") + not_found!("Note") end end - # Modify existing +noteable+ note - # - # Parameters: - # id (required) - The ID of a project - # noteable_id (required) - The ID of an issue or snippet - # node_id (required) - The ID of a note - # body (required) - New content of a note - # Example Request: - # PUT /projects/:id/issues/:noteable_id/notes/:note_id - # PUT /projects/:id/snippets/:noteable_id/notes/:node_id - put ":id/#{noteables_str}/:#{noteable_id_str}/notes/:note_id" do - required_attributes! [:body] - + desc 'Update an existing +noteable+ note' do + success Entities::Note + end + params do + requires :noteable_id, type: Integer, desc: 'The ID of the noteable' + requires :note_id, type: Integer, desc: 'The ID of a note' + requires :body, type: String, desc: 'The content of a note' + end + put ":id/#{noteables_str}/:noteable_id/notes/:note_id" do note = user_project.notes.find(params[:note_id]) authorize! :admin_note, note @@ -113,29 +111,27 @@ module API note: params[:body] } - @note = ::Notes::UpdateService.new(user_project, current_user, opts).execute(note) + note = ::Notes::UpdateService.new(user_project, current_user, opts).execute(note) - if @note.valid? - present @note, with: Entities::Note + if note.valid? + present note, with: Entities::Note else render_api_error!("Failed to save note #{note.errors.messages}", 400) end end - # Delete a +noteable+ note - # - # Parameters: - # id (required) - The ID of a project - # noteable_id (required) - The ID of an issue, MR, or snippet - # node_id (required) - The ID of a note - # Example Request: - # DELETE /projects/:id/issues/:noteable_id/notes/:note_id - # DELETE /projects/:id/snippets/:noteable_id/notes/:node_id - delete ":id/#{noteables_str}/:#{noteable_id_str}/notes/:note_id" do + desc 'Delete a +noteable+ note' do + success Entities::Note + end + params do + requires :noteable_id, type: Integer, desc: 'The ID of the noteable' + requires :note_id, type: Integer, desc: 'The ID of a note' + end + delete ":id/#{noteables_str}/:noteable_id/notes/:note_id" do note = user_project.notes.find(params[:note_id]) authorize! :admin_note, note - ::Notes::DeleteService.new(user_project, current_user).execute(note) + ::Notes::DestroyService.new(user_project, current_user).execute(note) present note, with: Entities::Note end diff --git a/lib/api/notification_settings.rb b/lib/api/notification_settings.rb new file mode 100644 index 00000000000..c5e9b3ad69b --- /dev/null +++ b/lib/api/notification_settings.rb @@ -0,0 +1,94 @@ +module API + # notification_settings API + class NotificationSettings < Grape::API + before { authenticate! } + + helpers ::API::Helpers::MembersHelpers + + resource :notification_settings do + desc 'Get global notification level settings and email, defaults to Participate' do + detail 'This feature was introduced in GitLab 8.12' + success Entities::GlobalNotificationSetting + end + get do + notification_setting = current_user.global_notification_setting + + present notification_setting, with: Entities::GlobalNotificationSetting + end + + desc 'Update global notification level settings and email, defaults to Participate' do + detail 'This feature was introduced in GitLab 8.12' + success Entities::GlobalNotificationSetting + end + params do + optional :level, type: String, desc: 'The global notification level' + optional :notification_email, type: String, desc: 'The email address to send notifications' + NotificationSetting::EMAIL_EVENTS.each do |event| + optional event, type: Boolean, desc: 'Enable/disable this notification' + end + end + put do + notification_setting = current_user.global_notification_setting + + begin + notification_setting.transaction do + new_notification_email = params.delete(:notification_email) + + current_user.update(notification_email: new_notification_email) if new_notification_email + notification_setting.update(declared_params(include_missing: false)) + end + rescue ArgumentError => e # catch level enum error + render_api_error! e.to_s, 400 + end + + render_validation_error! current_user + render_validation_error! notification_setting + present notification_setting, with: Entities::GlobalNotificationSetting + end + end + + %w[group project].each do |source_type| + resource source_type.pluralize do + desc "Get #{source_type} level notification level settings, defaults to Global" do + detail 'This feature was introduced in GitLab 8.12' + success Entities::NotificationSetting + end + params do + requires :id, type: String, desc: 'The group ID or project ID or project NAMESPACE/PROJECT_NAME' + end + get ":id/notification_settings" do + source = find_source(source_type, params[:id]) + + notification_setting = current_user.notification_settings_for(source) + + present notification_setting, with: Entities::NotificationSetting + end + + desc "Update #{source_type} level notification level settings, defaults to Global" do + detail 'This feature was introduced in GitLab 8.12' + success Entities::NotificationSetting + end + params do + requires :id, type: String, desc: 'The group ID or project ID or project NAMESPACE/PROJECT_NAME' + optional :level, type: String, desc: "The #{source_type} notification level" + NotificationSetting::EMAIL_EVENTS.each do |event| + optional event, type: Boolean, desc: 'Enable/disable this notification' + end + end + put ":id/notification_settings" do + source = find_source(source_type, params.delete(:id)) + notification_setting = current_user.notification_settings_for(source) + + begin + notification_setting.update(declared_params(include_missing: false)) + rescue ArgumentError => e # catch level enum error + render_api_error! e.to_s, 400 + end + + render_validation_error! notification_setting + present notification_setting, with: Entities::NotificationSetting + end + end + end + end +end diff --git a/lib/api/pagination_params.rb b/lib/api/pagination_params.rb new file mode 100644 index 00000000000..f566eb3ed2b --- /dev/null +++ b/lib/api/pagination_params.rb @@ -0,0 +1,24 @@ +module API + # Concern for declare pagination params. + # + # @example + # class CustomApiResource < Grape::API + # include PaginationParams + # + # params do + # use :pagination + # end + # end + module PaginationParams + extend ActiveSupport::Concern + + included do + helpers do + params :pagination do + optional :page, type: Integer, default: 1, desc: 'Current page number' + optional :per_page, type: Integer, default: 20, desc: 'Number of items per page' + end + end + end + end +end diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb new file mode 100644 index 00000000000..3afc1e385fe --- /dev/null +++ b/lib/api/pipelines.rb @@ -0,0 +1,99 @@ +module API + class Pipelines < Grape::API + include PaginationParams + + before { authenticate! } + + params do + requires :id, type: String, desc: 'The project ID' + end + resource :projects do + desc 'Get all Pipelines of the project' do + detail 'This feature was introduced in GitLab 8.11.' + success Entities::Pipeline + end + params do + use :pagination + optional :scope, type: String, values: %w(running branches tags), + desc: 'Either running, branches, or tags' + end + get ':id/pipelines' do + authorize! :read_pipeline, user_project + + pipelines = PipelinesFinder.new(user_project).execute(scope: params[:scope]) + present paginate(pipelines), with: Entities::Pipeline + end + + desc 'Create a new pipeline' do + detail 'This feature was introduced in GitLab 8.14' + success Entities::Pipeline + end + params do + requires :ref, type: String, desc: 'Reference' + end + post ':id/pipeline' do + authorize! :create_pipeline, user_project + + new_pipeline = Ci::CreatePipelineService.new(user_project, + current_user, + declared_params(include_missing: false)) + .execute(ignore_skip_ci: true, save_on_errors: false) + if new_pipeline.persisted? + present new_pipeline, with: Entities::Pipeline + else + render_validation_error!(new_pipeline) + end + end + + desc 'Gets a specific pipeline for the project' do + detail 'This feature was introduced in GitLab 8.11' + success Entities::Pipeline + end + params do + requires :pipeline_id, type: Integer, desc: 'The pipeline ID' + end + get ':id/pipelines/:pipeline_id' do + authorize! :read_pipeline, user_project + + present pipeline, with: Entities::Pipeline + end + + desc 'Retry builds in the pipeline' do + detail 'This feature was introduced in GitLab 8.11.' + success Entities::Pipeline + end + params do + requires :pipeline_id, type: Integer, desc: 'The pipeline ID' + end + post ':id/pipelines/:pipeline_id/retry' do + authorize! :update_pipeline, user_project + + pipeline.retry_failed(current_user) + + present pipeline, with: Entities::Pipeline + end + + desc 'Cancel all builds in the pipeline' do + detail 'This feature was introduced in GitLab 8.11.' + success Entities::Pipeline + end + params do + requires :pipeline_id, type: Integer, desc: 'The pipeline ID' + end + post ':id/pipelines/:pipeline_id/cancel' do + authorize! :update_pipeline, user_project + + pipeline.cancel_running + + status 200 + present pipeline.reload, with: Entities::Pipeline + end + end + + helpers do + def pipeline + @pipeline ||= user_project.pipelines.find(params[:pipeline_id]) + end + end + end +end diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb index 6bb70bc8bc3..f7a28d7ad10 100644 --- a/lib/api/project_hooks.rb +++ b/lib/api/project_hooks.rb @@ -1,108 +1,97 @@ module API - # Projects API class ProjectHooks < Grape::API + include PaginationParams + before { authenticate! } before { authorize_admin_project } + helpers do + params :project_hook_properties do + requires :url, type: String, desc: "The URL to send the request to" + optional :push_events, type: Boolean, desc: "Trigger hook on push events" + optional :issues_events, type: Boolean, desc: "Trigger hook on issues events" + optional :merge_requests_events, type: Boolean, desc: "Trigger hook on merge request events" + optional :tag_push_events, type: Boolean, desc: "Trigger hook on tag push events" + optional :note_events, type: Boolean, desc: "Trigger hook on note(comment) events" + optional :build_events, type: Boolean, desc: "Trigger hook on build events" + optional :pipeline_events, type: Boolean, desc: "Trigger hook on pipeline events" + optional :wiki_page_events, type: Boolean, desc: "Trigger hook on wiki events" + optional :enable_ssl_verification, type: Boolean, desc: "Do SSL verification when triggering the hook" + optional :token, type: String, desc: "Secret token to validate received payloads; this will not be returned in the response" + end + end + + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do - # Get project hooks - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # GET /projects/:id/hooks + desc 'Get project hooks' do + success Entities::ProjectHook + end + params do + use :pagination + end get ":id/hooks" do - @hooks = paginate user_project.hooks - present @hooks, with: Entities::ProjectHook + present paginate(user_project.hooks), with: Entities::ProjectHook end - # Get a project hook - # - # Parameters: - # id (required) - The ID of a project - # hook_id (required) - The ID of a project hook - # Example Request: - # GET /projects/:id/hooks/:hook_id + desc 'Get a project hook' do + success Entities::ProjectHook + end + params do + requires :hook_id, type: Integer, desc: 'The ID of a project hook' + end get ":id/hooks/:hook_id" do - @hook = user_project.hooks.find(params[:hook_id]) - present @hook, with: Entities::ProjectHook + hook = user_project.hooks.find(params[:hook_id]) + present hook, with: Entities::ProjectHook end - # Add hook to project - # - # Parameters: - # id (required) - The ID of a project - # url (required) - The hook URL - # Example Request: - # POST /projects/:id/hooks + desc 'Add hook to project' do + success Entities::ProjectHook + end + params do + use :project_hook_properties + end post ":id/hooks" do - required_attributes! [:url] - attrs = attributes_for_keys [ - :url, - :push_events, - :issues_events, - :merge_requests_events, - :tag_push_events, - :note_events, - :build_events, - :enable_ssl_verification - ] - @hook = user_project.hooks.new(attrs) + hook = user_project.hooks.new(declared_params(include_missing: false)) - if @hook.save - present @hook, with: Entities::ProjectHook + if hook.save + present hook, with: Entities::ProjectHook else - if @hook.errors[:url].present? - error!("Invalid url given", 422) - end - not_found!("Project hook #{@hook.errors.messages}") + error!("Invalid url given", 422) if hook.errors[:url].present? + + not_found!("Project hook #{hook.errors.messages}") end end - # Update an existing project hook - # - # Parameters: - # id (required) - The ID of a project - # hook_id (required) - The ID of a project hook - # url (required) - The hook URL - # Example Request: - # PUT /projects/:id/hooks/:hook_id + desc 'Update an existing project hook' do + success Entities::ProjectHook + end + params do + requires :hook_id, type: Integer, desc: "The ID of the hook to update" + use :project_hook_properties + end put ":id/hooks/:hook_id" do - @hook = user_project.hooks.find(params[:hook_id]) - required_attributes! [:url] - attrs = attributes_for_keys [ - :url, - :push_events, - :issues_events, - :merge_requests_events, - :tag_push_events, - :note_events, - :build_events, - :enable_ssl_verification - ] + hook = user_project.hooks.find(params.delete(:hook_id)) - if @hook.update_attributes attrs - present @hook, with: Entities::ProjectHook + if hook.update_attributes(declared_params(include_missing: false)) + present hook, with: Entities::ProjectHook else - if @hook.errors[:url].present? - error!("Invalid url given", 422) - end - not_found!("Project hook #{@hook.errors.messages}") + error!("Invalid url given", 422) if hook.errors[:url].present? + + not_found!("Project hook #{hook.errors.messages}") end end - # Deletes project hook. This is an idempotent function. - # - # Parameters: - # id (required) - The ID of a project - # hook_id (required) - The ID of hook to delete - # Example Request: - # DELETE /projects/:id/hooks/:hook_id + desc 'Deletes project hook' do + success Entities::ProjectHook + end + params do + requires :hook_id, type: Integer, desc: 'The ID of the hook to delete' + end delete ":id/hooks/:hook_id" do - required_attributes! [:hook_id] - begin - @hook = user_project.hooks.destroy(params[:hook_id]) + present user_project.hooks.destroy(params[:hook_id]), with: Entities::ProjectHook rescue # ProjectHook can raise Error if hook_id not found not_found!("Error deleting hook #{params[:hook_id]}") diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb index ce1bf0d26d2..2a1cce73f3f 100644 --- a/lib/api/project_snippets.rb +++ b/lib/api/project_snippets.rb @@ -1,8 +1,12 @@ module API - # Projects API class ProjectSnippets < Grape::API + include PaginationParams + before { authenticate! } + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do helpers do def handle_project_member_errors(errors) @@ -18,111 +22,117 @@ module API end end - # Get a project snippets - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # GET /projects/:id/snippets + desc 'Get all project snippets' do + success Entities::ProjectSnippet + end + params do + use :pagination + end get ":id/snippets" do present paginate(snippets_for_current_user), with: Entities::ProjectSnippet end - # Get a project snippet - # - # Parameters: - # id (required) - The ID of a project - # snippet_id (required) - The ID of a project snippet - # Example Request: - # GET /projects/:id/snippets/:snippet_id + desc 'Get a single project snippet' do + success Entities::ProjectSnippet + end + params do + requires :snippet_id, type: Integer, desc: 'The ID of a project snippet' + end get ":id/snippets/:snippet_id" do - @snippet = snippets_for_current_user.find(params[:snippet_id]) - present @snippet, with: Entities::ProjectSnippet - end - - # Create a new project snippet - # - # Parameters: - # id (required) - The ID of a project - # title (required) - The title of a snippet - # file_name (required) - The name of a snippet file - # code (required) - The content of a snippet - # visibility_level (required) - The snippet's visibility - # Example Request: - # POST /projects/:id/snippets + snippet = snippets_for_current_user.find(params[:snippet_id]) + present snippet, with: Entities::ProjectSnippet + end + + desc 'Create a new project snippet' do + success Entities::ProjectSnippet + end + params do + requires :title, type: String, desc: 'The title of the snippet' + requires :file_name, type: String, desc: 'The file name of the snippet' + requires :code, type: String, desc: 'The content of the snippet' + requires :visibility_level, type: Integer, + values: [Gitlab::VisibilityLevel::PRIVATE, + Gitlab::VisibilityLevel::INTERNAL, + Gitlab::VisibilityLevel::PUBLIC], + desc: 'The visibility level of the snippet' + end post ":id/snippets" do authorize! :create_project_snippet, user_project - required_attributes! [:title, :file_name, :code, :visibility_level] + snippet_params = declared_params.merge(request: request, api: true) + snippet_params[:content] = snippet_params.delete(:code) + + snippet = CreateSnippetService.new(user_project, current_user, snippet_params).execute - attrs = attributes_for_keys [:title, :file_name, :visibility_level] - attrs[:content] = params[:code] if params[:code].present? - @snippet = CreateSnippetService.new(user_project, current_user, - attrs).execute + render_spam_error! if snippet.spam? - if @snippet.errors.any? - render_validation_error!(@snippet) + if snippet.persisted? + present snippet, with: Entities::ProjectSnippet else - present @snippet, with: Entities::ProjectSnippet + render_validation_error!(snippet) end end - # Update an existing project snippet - # - # Parameters: - # id (required) - The ID of a project - # snippet_id (required) - The ID of a project snippet - # title (optional) - The title of a snippet - # file_name (optional) - The name of a snippet file - # code (optional) - The content of a snippet - # visibility_level (optional) - The snippet's visibility - # Example Request: - # PUT /projects/:id/snippets/:snippet_id + desc 'Update an existing project snippet' do + success Entities::ProjectSnippet + end + params do + requires :snippet_id, type: Integer, desc: 'The ID of a project snippet' + optional :title, type: String, desc: 'The title of the snippet' + optional :file_name, type: String, desc: 'The file name of the snippet' + optional :code, type: String, desc: 'The content of the snippet' + optional :visibility_level, type: Integer, + values: [Gitlab::VisibilityLevel::PRIVATE, + Gitlab::VisibilityLevel::INTERNAL, + Gitlab::VisibilityLevel::PUBLIC], + desc: 'The visibility level of the snippet' + at_least_one_of :title, :file_name, :code, :visibility_level + end put ":id/snippets/:snippet_id" do - @snippet = snippets_for_current_user.find(params[:snippet_id]) - authorize! :update_project_snippet, @snippet + snippet = snippets_for_current_user.find_by(id: params.delete(:snippet_id)) + not_found!('Snippet') unless snippet + + authorize! :update_project_snippet, snippet + + snippet_params = declared_params(include_missing: false) + .merge(request: request, api: true) + + snippet_params[:content] = snippet_params.delete(:code) if snippet_params[:code].present? - attrs = attributes_for_keys [:title, :file_name, :visibility_level] - attrs[:content] = params[:code] if params[:code].present? + UpdateSnippetService.new(user_project, current_user, snippet, + snippet_params).execute - UpdateSnippetService.new(user_project, current_user, @snippet, - attrs).execute - if @snippet.errors.any? - render_validation_error!(@snippet) + render_spam_error! if snippet.spam? + + if snippet.valid? + present snippet, with: Entities::ProjectSnippet else - present @snippet, with: Entities::ProjectSnippet + render_validation_error!(snippet) end end - # Delete a project snippet - # - # Parameters: - # id (required) - The ID of a project - # snippet_id (required) - The ID of a project snippet - # Example Request: - # DELETE /projects/:id/snippets/:snippet_id + desc 'Delete a project snippet' + params do + requires :snippet_id, type: Integer, desc: 'The ID of a project snippet' + end delete ":id/snippets/:snippet_id" do - begin - @snippet = snippets_for_current_user.find(params[:snippet_id]) - authorize! :update_project_snippet, @snippet - @snippet.destroy - rescue - not_found!('Snippet') - end + snippet = snippets_for_current_user.find_by(id: params[:snippet_id]) + not_found!('Snippet') unless snippet + + authorize! :admin_project_snippet, snippet + snippet.destroy end - # Get a raw project snippet - # - # Parameters: - # id (required) - The ID of a project - # snippet_id (required) - The ID of a project snippet - # Example Request: - # GET /projects/:id/snippets/:snippet_id/raw + desc 'Get a raw project snippet' + params do + requires :snippet_id, type: Integer, desc: 'The ID of a project snippet' + end get ":id/snippets/:snippet_id/raw" do - @snippet = snippets_for_current_user.find(params[:snippet_id]) + snippet = snippets_for_current_user.find_by(id: params[:snippet_id]) + not_found!('Snippet') unless snippet env['api.format'] = :txt content_type 'text/plain' - present @snippet.content + present snippet.content end end end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 60cfc103afd..b89bddc7e29 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -1,264 +1,236 @@ module API # Projects API class Projects < Grape::API - before { authenticate! } + include PaginationParams + + before { authenticate_non_get! } + + helpers do + params :optional_params do + optional :description, type: String, desc: 'The description of the project' + optional :issues_enabled, type: Boolean, desc: 'Flag indication if the issue tracker is enabled' + optional :merge_requests_enabled, type: Boolean, desc: 'Flag indication if merge requests are enabled' + optional :wiki_enabled, type: Boolean, desc: 'Flag indication if the wiki is enabled' + optional :builds_enabled, type: Boolean, desc: 'Flag indication if builds are enabled' + optional :snippets_enabled, type: Boolean, desc: 'Flag indication if snippets are enabled' + optional :shared_runners_enabled, type: Boolean, desc: 'Flag indication if shared runners are enabled for that project' + optional :container_registry_enabled, type: Boolean, desc: 'Flag indication if the container registry is enabled for that project' + optional :lfs_enabled, type: Boolean, desc: 'Flag indication if Git LFS is enabled for that project' + optional :visibility_level, type: Integer, values: [ + Gitlab::VisibilityLevel::PRIVATE, + Gitlab::VisibilityLevel::INTERNAL, + Gitlab::VisibilityLevel::PUBLIC + ], desc: 'Create a public project. The same as visibility_level = 20.' + optional :public_builds, type: Boolean, desc: 'Perform public builds' + optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access' + optional :only_allow_merge_if_build_succeeds, type: Boolean, desc: 'Only allow to merge if builds succeed' + optional :only_allow_merge_if_all_discussions_are_resolved, type: Boolean, desc: 'Only allow to merge if all discussions are resolved' + end + end - resource :projects, requirements: { id: /[^\/]+/ } do + resource :projects do helpers do - def map_public_to_visibility_level(attrs) - publik = attrs.delete(:public) - if publik.present? && !attrs[:visibility_level].present? - publik = to_boolean(publik) - # Since setting the public attribute to private could mean either - # private or internal, use the more conservative option, private. - attrs[:visibility_level] = (publik == true) ? Gitlab::VisibilityLevel::PUBLIC : Gitlab::VisibilityLevel::PRIVATE - end - attrs + params :collection_params do + use :sort_params + use :filter_params + use :pagination + + optional :simple, type: Boolean, default: false, + desc: 'Return only the ID, URL, name, and path of each project' end - end - # Get a projects list for authenticated user - # - # Example Request: - # GET /projects - get do - @projects = current_user.authorized_projects - @projects = filter_projects(@projects) - @projects = paginate @projects - if params[:simple] - present @projects, with: Entities::BasicProjectDetails, user: current_user - else - present @projects, with: Entities::ProjectWithAccess, user: current_user + params :sort_params do + optional :order_by, type: String, values: %w[id name path created_at updated_at last_activity_at], + default: 'created_at', desc: 'Return projects ordered by field' + optional :sort, type: String, values: %w[asc desc], default: 'desc', + desc: 'Return projects sorted in ascending and descending order' + end + + params :filter_params do + optional :archived, type: Boolean, default: false, desc: 'Limit by archived status' + optional :visibility, type: String, values: %w[public internal private], + desc: 'Limit by visibility' + optional :search, type: String, desc: 'Return list of authorized projects matching the search criteria' + optional :owned, type: Boolean, default: false, desc: 'Limit by owned by authenticated user' + optional :starred, type: Boolean, default: false, desc: 'Limit by starred status' + end + + params :statistics_params do + optional :statistics, type: Boolean, default: false, desc: 'Include project statistics' + end + + params :create_params do + optional :namespace_id, type: Integer, desc: 'Namespace ID for the new project. Default to the user namespace.' + optional :import_url, type: String, desc: 'URL from which the project is imported' + end + + def present_projects(projects, options = {}) + options = options.reverse_merge( + with: Entities::Project, + current_user: current_user, + simple: params[:simple], + ) + + projects = filter_projects(projects) + projects = projects.with_statistics if options[:statistics] + options[:with] = Entities::BasicProjectDetails if options[:simple] + + present paginate(projects), options end end - # Get an owned projects list for authenticated user - # - # Example Request: - # GET /projects/owned - get '/owned' do - @projects = current_user.owned_projects - @projects = filter_projects(@projects) - @projects = paginate @projects - present @projects, with: Entities::ProjectWithAccess, user: current_user - end - - # Gets starred project for the authenticated user - # - # Example Request: - # GET /projects/starred - get '/starred' do - @projects = current_user.viewable_starred_projects - @projects = filter_projects(@projects) - @projects = paginate @projects - present @projects, with: Entities::Project - end - - # Get all projects for admin user - # - # Example Request: - # GET /projects/all - get '/all' do - authenticated_as_admin! - @projects = Project.all - @projects = filter_projects(@projects) - @projects = paginate @projects - present @projects, with: Entities::ProjectWithAccess, user: current_user - end - - # Get a single project - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # GET /projects/:id - get ":id" do - present user_project, with: Entities::ProjectWithAccess, user: current_user, - user_can_admin_project: can?(current_user, :admin_project, user_project) + desc 'Get a list of visible projects for authenticated user' do + success Entities::BasicProjectDetails + end + params do + use :collection_params + end + get do + entity = current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails + present_projects ProjectsFinder.new.execute(current_user), with: entity, statistics: params[:statistics] end - # Get events for a single project - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # GET /projects/:id/events - get ":id/events" do - events = paginate user_project.events.recent - present events, with: Entities::Event - end - - # Create new project - # - # Parameters: - # name (required) - name for new project - # description (optional) - short project description - # issues_enabled (optional) - # merge_requests_enabled (optional) - # builds_enabled (optional) - # wiki_enabled (optional) - # snippets_enabled (optional) - # container_registry_enabled (optional) - # shared_runners_enabled (optional) - # namespace_id (optional) - defaults to user namespace - # public (optional) - if true same as setting visibility_level = 20 - # visibility_level (optional) - 0 by default - # import_url (optional) - # public_builds (optional) - # Example Request - # POST /projects + desc 'Create new project' do + success Entities::Project + end + params do + requires :name, type: String, desc: 'The name of the project' + optional :path, type: String, desc: 'The path of the repository' + use :optional_params + use :create_params + end post do - required_attributes! [:name] - attrs = attributes_for_keys [:name, - :path, - :description, - :issues_enabled, - :merge_requests_enabled, - :builds_enabled, - :wiki_enabled, - :snippets_enabled, - :container_registry_enabled, - :shared_runners_enabled, - :namespace_id, - :public, - :visibility_level, - :import_url, - :public_builds] - attrs = map_public_to_visibility_level(attrs) - @project = ::Projects::CreateService.new(current_user, attrs).execute - if @project.saved? - present @project, with: Entities::Project, - user_can_admin_project: can?(current_user, :admin_project, @project) + attrs = declared_params(include_missing: false) + project = ::Projects::CreateService.new(current_user, attrs).execute + + if project.saved? + present project, with: Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, project) else - if @project.errors[:limit_reached].present? - error!(@project.errors[:limit_reached], 403) + if project.errors[:limit_reached].present? + error!(project.errors[:limit_reached], 403) end - render_validation_error!(@project) + render_validation_error!(project) end end - # Create new project for a specified user. Only available to admin users. - # - # Parameters: - # user_id (required) - The ID of a user - # name (required) - name for new project - # description (optional) - short project description - # default_branch (optional) - 'master' by default - # issues_enabled (optional) - # merge_requests_enabled (optional) - # builds_enabled (optional) - # wiki_enabled (optional) - # snippets_enabled (optional) - # container_registry_enabled (optional) - # shared_runners_enabled (optional) - # public (optional) - if true same as setting visibility_level = 20 - # visibility_level (optional) - # import_url (optional) - # public_builds (optional) - # Example Request - # POST /projects/user/:user_id + desc 'Create new project for a specified user. Only available to admin users.' do + success Entities::Project + end + params do + requires :name, type: String, desc: 'The name of the project' + requires :user_id, type: Integer, desc: 'The ID of a user' + optional :default_branch, type: String, desc: 'The default branch of the project' + use :optional_params + use :create_params + end post "user/:user_id" do authenticated_as_admin! - user = User.find(params[:user_id]) - attrs = attributes_for_keys [:name, - :description, - :default_branch, - :issues_enabled, - :merge_requests_enabled, - :builds_enabled, - :wiki_enabled, - :snippets_enabled, - :shared_runners_enabled, - :public, - :visibility_level, - :import_url, - :public_builds] - attrs = map_public_to_visibility_level(attrs) - @project = ::Projects::CreateService.new(user, attrs).execute - if @project.saved? - present @project, with: Entities::Project, - user_can_admin_project: can?(current_user, :admin_project, @project) + user = User.find_by(id: params.delete(:user_id)) + not_found!('User') unless user + + attrs = declared_params(include_missing: false) + project = ::Projects::CreateService.new(user, attrs).execute + + if project.saved? + present project, with: Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, project) else - render_validation_error!(@project) + render_validation_error!(project) end end + end - # Fork new project for the current user. - # - # Parameters: - # id (required) - The ID of a project - # Example Request - # POST /projects/fork/:id - post 'fork/:id' do - @forked_project = - ::Projects::ForkService.new(user_project, - current_user).execute - if @forked_project.errors.any? - conflict!(@forked_project.errors.messages) + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: { id: /[^\/]+/ } do + desc 'Get a single project' do + success Entities::ProjectWithAccess + end + get ":id" do + entity = current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails + present user_project, with: entity, current_user: current_user, + user_can_admin_project: can?(current_user, :admin_project, user_project) + end + + desc 'Get events for a single project' do + success Entities::Event + end + params do + use :pagination + end + get ":id/events" do + present paginate(user_project.events.recent), with: Entities::Event + end + + desc 'Fork new project for the current user or provided namespace.' do + success Entities::Project + end + params do + optional :namespace, type: String, desc: 'The ID or name of the namespace that the project will be forked into' + end + post ':id/fork' do + fork_params = declared_params(include_missing: false) + namespace_id = fork_params[:namespace] + + if namespace_id.present? + fork_params[:namespace] = if namespace_id =~ /^\d+$/ + Namespace.find_by(id: namespace_id) + else + Namespace.find_by_path_or_name(namespace_id) + end + + unless fork_params[:namespace] && can?(current_user, :create_projects, fork_params[:namespace]) + not_found!('Target Namespace') + end + end + + forked_project = ::Projects::ForkService.new(user_project, current_user, fork_params).execute + + if forked_project.errors.any? + conflict!(forked_project.errors.messages) else - present @forked_project, with: Entities::Project, - user_can_admin_project: can?(current_user, :admin_project, @forked_project) + present forked_project, with: Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, forked_project) end end - # Update an existing project - # - # Parameters: - # id (required) - the id of a project - # name (optional) - name of a project - # path (optional) - path of a project - # description (optional) - short project description - # issues_enabled (optional) - # merge_requests_enabled (optional) - # builds_enabled (optional) - # wiki_enabled (optional) - # snippets_enabled (optional) - # container_registry_enabled (optional) - # shared_runners_enabled (optional) - # public (optional) - if true same as setting visibility_level = 20 - # visibility_level (optional) - visibility level of a project - # public_builds (optional) - # Example Request - # PUT /projects/:id + desc 'Update an existing project' do + success Entities::Project + end + params do + optional :name, type: String, desc: 'The name of the project' + optional :default_branch, type: String, desc: 'The default branch of the project' + optional :path, type: String, desc: 'The path of the repository' + use :optional_params + at_least_one_of :name, :description, :issues_enabled, :merge_requests_enabled, + :wiki_enabled, :builds_enabled, :snippets_enabled, + :shared_runners_enabled, :container_registry_enabled, + :lfs_enabled, :visibility_level, :public_builds, + :request_access_enabled, :only_allow_merge_if_build_succeeds, + :only_allow_merge_if_all_discussions_are_resolved, :path, + :default_branch + end put ':id' do - attrs = attributes_for_keys [:name, - :path, - :description, - :default_branch, - :issues_enabled, - :merge_requests_enabled, - :builds_enabled, - :wiki_enabled, - :snippets_enabled, - :container_registry_enabled, - :shared_runners_enabled, - :public, - :visibility_level, - :public_builds] - attrs = map_public_to_visibility_level(attrs) authorize_admin_project + attrs = declared_params(include_missing: false) authorize! :rename_project, user_project if attrs[:name].present? - if attrs[:visibility_level].present? - authorize! :change_visibility_level, user_project - end + authorize! :change_visibility_level, user_project if attrs[:visibility_level].present? - ::Projects::UpdateService.new(user_project, - current_user, attrs).execute + result = ::Projects::UpdateService.new(user_project, current_user, attrs).execute - if user_project.errors.any? - render_validation_error!(user_project) - else + if result[:status] == :success present user_project, with: Entities::Project, user_can_admin_project: can?(current_user, :admin_project, user_project) + else + render_validation_error!(user_project) end end - # Archive project - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # PUT /projects/:id/archive + desc 'Archive a project' do + success Entities::Project + end post ':id/archive' do authorize!(:archive_project, user_project) @@ -267,12 +239,9 @@ module API present user_project, with: Entities::Project end - # Unarchive project - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # PUT /projects/:id/unarchive + desc 'Unarchive a project' do + success Entities::Project + end post ':id/unarchive' do authorize!(:archive_project, user_project) @@ -281,12 +250,9 @@ module API present user_project, with: Entities::Project end - # Star project - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # POST /projects/:id/star + desc 'Star a project' do + success Entities::Project + end post ':id/star' do if current_user.starred?(user_project) not_modified! @@ -298,13 +264,10 @@ module API end end - # Unstar project - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # DELETE /projects/:id/star - delete ':id/star' do + desc 'Unstar a project' do + success Entities::Project + end + post ':id/unstar' do if current_user.starred?(user_project) current_user.toggle_star(user_project) user_project.reload @@ -315,71 +278,64 @@ module API end end - # Remove project - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # DELETE /projects/:id + desc 'Remove a project' delete ":id" do authorize! :remove_project, user_project ::Projects::DestroyService.new(user_project, current_user, {}).async_execute + + accepted! end - # Mark this project as forked from another - # - # Parameters: - # id: (required) - The ID of the project being marked as a fork - # forked_from_id: (required) - The ID of the project it was forked from - # Example Request: - # POST /projects/:id/fork/:forked_from_id + desc 'Mark this project as forked from another' + params do + requires :forked_from_id, type: String, desc: 'The ID of the project it was forked from' + end post ":id/fork/:forked_from_id" do authenticated_as_admin! - forked_from_project = find_project(params[:forked_from_id]) - unless forked_from_project.nil? - if user_project.forked_from_project.nil? - user_project.create_forked_project_link(forked_to_project_id: user_project.id, forked_from_project_id: forked_from_project.id) - else - render_api_error!("Project already forked", 409) - end + + forked_from_project = find_project!(params[:forked_from_id]) + not_found!("Source Project") unless forked_from_project + + if user_project.forked_from_project.nil? + user_project.create_forked_project_link(forked_to_project_id: user_project.id, forked_from_project_id: forked_from_project.id) else - not_found!("Source Project") + render_api_error!("Project already forked", 409) end end - # Remove a forked_from relationship - # - # Parameters: - # id: (required) - The ID of the project being marked as a fork - # Example Request: - # DELETE /projects/:id/fork + desc 'Remove a forked_from relationship' delete ":id/fork" do authorize! :remove_fork_project, user_project + if user_project.forked? user_project.forked_project_link.destroy + else + not_modified! end end - # Share project with group - # - # Parameters: - # id (required) - The ID of a project - # group_id (required) - The ID of a group - # group_access (required) - Level of permissions for sharing - # - # Example Request: - # POST /projects/:id/share + desc 'Share the project with a group' do + success Entities::ProjectGroupLink + end + params do + requires :group_id, type: Integer, desc: 'The ID of a group' + requires :group_access, type: Integer, values: Gitlab::Access.values, desc: 'The group access level' + optional :expires_at, type: Date, desc: 'Share expiration date' + end post ":id/share" do authorize! :admin_project, user_project - required_attributes! [:group_id, :group_access] + group = Group.find_by_id(params[:group_id]) + + unless group && can?(current_user, :read_group, group) + not_found!('Group') + end unless user_project.allowed_to_share_with_group? return render_api_error!("The project sharing with group is disabled", 400) end - link = user_project.project_group_links.new - link.group_id = params[:group_id] - link.group_access = params[:group_access] + link = user_project.project_group_links.new(declared_params(include_missing: false)) + if link.save present link, with: Entities::ProjectGroupLink else @@ -387,49 +343,52 @@ module API end end - # Upload a file - # - # Parameters: - # id: (required) - The ID of the project - # file: (required) - The file to be uploaded + params do + requires :group_id, type: Integer, desc: 'The ID of the group' + end + delete ":id/share/:group_id" do + authorize! :admin_project, user_project + + link = user_project.project_group_links.find_by(group_id: params[:group_id]) + not_found!('Group Link') unless link + + link.destroy + no_content! + end + + desc 'Upload a file' + params do + requires :file, type: File, desc: 'The file to be uploaded' + end post ":id/uploads" do ::Projects::UploadService.new(user_project, params[:file]).execute end - # search for projects current_user has access to - # - # Parameters: - # query (required) - A string contained in the project name - # per_page (optional) - number of projects to return per page - # page (optional) - the page to retrieve - # Example Request: - # GET /projects/search/:query - get "/search/:query" do - ids = current_user.authorized_projects.map(&:id) - visibility_levels = [ Gitlab::VisibilityLevel::INTERNAL, Gitlab::VisibilityLevel::PUBLIC ] - projects = Project.where("(id in (?) OR visibility_level in (?)) AND (name LIKE (?))", ids, visibility_levels, "%#{params[:query]}%") - sort = params[:sort] == 'desc' ? 'desc' : 'asc' - - projects = case params["order_by"] - when 'id' then projects.order("id #{sort}") - when 'name' then projects.order("name #{sort}") - when 'created_at' then projects.order("created_at #{sort}") - when 'last_activity_at' then projects.order("last_activity_at #{sort}") - else projects - end - - present paginate(projects), with: Entities::Project - end - - # Get a users list - # - # Example Request: - # GET /users + desc 'Get the users list of a project' do + success Entities::UserBasic + end + params do + optional :search, type: String, desc: 'Return list of users matching the search criteria' + use :pagination + end get ':id/users' do - @users = User.where(id: user_project.team.users.map(&:id)) - @users = @users.search(params[:search]) if params[:search].present? - @users = paginate @users - present @users, with: Entities::UserBasic + users = user_project.team.users + users = users.search(params[:search]) if params[:search].present? + + present paginate(users), with: Entities::UserBasic + end + + desc 'Start the housekeeping task for a project' do + detail 'This feature was introduced in GitLab 9.0.' + end + post ':id/housekeeping' do + authorize_admin_project + + begin + ::Projects::HousekeepingService.new(user_project).execute + rescue ::Projects::HousekeepingService::LeaseTaken => error + conflict!(error.message) + end end end end diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index f55aceed92c..36166780149 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -1,11 +1,14 @@ require 'mime/types' module API - # Projects API class Repositories < Grape::API - before { authenticate! } + include PaginationParams + before { authorize! :download_code, user_project } + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do helpers do def handle_project_member_errors(errors) @@ -16,13 +19,15 @@ module API end end - # Get a project repository tree - # - # Parameters: - # id (required) - The ID of a project - # ref_name (optional) - The name of a repository branch or tag, if not given the default branch is used - # Example Request: - # GET /projects/:id/repository/tree + desc 'Get a project repository tree' do + success Entities::RepoTreeObject + end + params do + optional :ref_name, type: String, desc: 'The name of a repository branch or tag, if not given the default branch is used' + optional :path, type: String, desc: 'The path of the tree' + optional :recursive, type: Boolean, default: false, desc: 'Used to get a recursive tree' + use :pagination + end get ':id/repository/tree' do ref = params[:ref_name] || user_project.try(:default_branch) || 'master' path = params[:path] || nil @@ -30,27 +35,20 @@ module API commit = user_project.commit(ref) not_found!('Tree') unless commit - tree = user_project.repository.tree(commit.id, path) - - present tree.sorted_entries, with: Entities::RepoTreeObject + tree = user_project.repository.tree(commit.id, path, recursive: params[:recursive]) + entries = ::Kaminari.paginate_array(tree.sorted_entries) + present paginate(entries), with: Entities::RepoTreeObject end - # Get a raw file contents - # - # Parameters: - # id (required) - The ID of a project - # sha (required) - The commit or branch name - # filepath (required) - The path to the file to display - # Example Request: - # GET /projects/:id/repository/blobs/:sha - get [ ":id/repository/blobs/:sha", ":id/repository/commits/:sha/blob" ] do - required_attributes! [:filepath] - - ref = params[:sha] - + desc 'Get a raw file contents' + params do + requires :sha, type: String, desc: 'The commit, branch name, or tag name' + requires :filepath, type: String, desc: 'The path to the file to display' + end + get [":id/repository/blobs/:sha", ":id/repository/commits/:sha/blob"] do repo = user_project.repository - commit = repo.commit(ref) + commit = repo.commit(params[:sha]) not_found! "Commit" unless commit blob = Gitlab::Git::Blob.find(repo, commit.id, params[:filepath]) @@ -59,20 +57,15 @@ module API send_git_blob repo, blob end - # Get a raw blob contents by blob sha - # - # Parameters: - # id (required) - The ID of a project - # sha (required) - The blob's sha - # Example Request: - # GET /projects/:id/repository/raw_blobs/:sha + desc 'Get a raw blob contents by blob sha' + params do + requires :sha, type: String, desc: 'The commit, branch name, or tag name' + end get ':id/repository/raw_blobs/:sha' do - ref = params[:sha] - repo = user_project.repository begin - blob = Gitlab::Git::Blob.raw(repo, ref) + blob = Gitlab::Git::Blob.raw(repo, params[:sha]) rescue not_found! 'Blob' end @@ -82,17 +75,12 @@ module API send_git_blob repo, blob end - # Get a an archive of the repository - # - # Parameters: - # id (required) - The ID of a project - # sha (optional) - the commit sha to download defaults to the tip of the default branch - # Example Request: - # GET /projects/:id/repository/archive - get ':id/repository/archive', - requirements: { format: Gitlab::Regex.archive_formats_regex } do - authorize! :download_code, user_project - + desc 'Get an archive of the repository' + params do + optional :sha, type: String, desc: 'The commit sha of the archive to be downloaded' + optional :format, type: String, desc: 'The archive format' + end + get ':id/repository/archive', requirements: { format: Gitlab::Regex.archive_formats_regex } do begin send_git_archive user_project.repository, ref: params[:sha], format: params[:format] rescue @@ -100,33 +88,28 @@ module API end end - # Compare two branches, tags or commits - # - # Parameters: - # id (required) - The ID of a project - # from (required) - the commit sha or branch name - # to (required) - the commit sha or branch name - # Example Request: - # GET /projects/:id/repository/compare?from=master&to=feature + desc 'Compare two branches, tags, or commits' do + success Entities::Compare + end + params do + requires :from, type: String, desc: 'The commit, branch name, or tag name to start comparison' + requires :to, type: String, desc: 'The commit, branch name, or tag name to stop comparison' + end get ':id/repository/compare' do - authorize! :download_code, user_project - required_attributes! [:from, :to] compare = Gitlab::Git::Compare.new(user_project.repository.raw_repository, params[:from], params[:to]) present compare, with: Entities::Compare end - # Get repository contributors - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # GET /projects/:id/repository/contributors + desc 'Get repository contributors' do + success Entities::Contributor + end + params do + use :pagination + end get ':id/repository/contributors' do - authorize! :download_code, user_project - begin - present user_project.repository.contributors, - with: Entities::Contributor + contributors = ::Kaminari.paginate_array(user_project.repository.contributors) + present paginate(contributors), with: Entities::Contributor rescue not_found! end diff --git a/lib/api/runner.rb b/lib/api/runner.rb new file mode 100644 index 00000000000..804b27d40a7 --- /dev/null +++ b/lib/api/runner.rb @@ -0,0 +1,52 @@ +module API + class Runner < Grape::API + helpers ::API::Helpers::Runner + + resource :runners do + desc 'Registers a new Runner' do + success Entities::RunnerRegistrationDetails + http_codes [[201, 'Runner was created'], [403, 'Forbidden']] + end + params do + requires :token, type: String, desc: 'Registration token' + optional :description, type: String, desc: %q(Runner's description) + optional :info, type: Hash, desc: %q(Runner's metadata) + optional :locked, type: Boolean, desc: 'Should Runner be locked for current project' + optional :run_untagged, type: Boolean, desc: 'Should Runner handle untagged jobs' + optional :tag_list, type: Array[String], desc: %q(List of Runner's tags) + end + post '/' do + attributes = attributes_for_keys [:description, :locked, :run_untagged, :tag_list] + + runner = + if runner_registration_token_valid? + # Create shared runner. Requires admin access + Ci::Runner.create(attributes.merge(is_shared: true)) + elsif project = Project.find_by(runners_token: params[:token]) + # Create a specific runner for project. + project.runners.create(attributes) + end + + return forbidden! unless runner + + if runner.id + runner.update(get_runner_version_from_params) + present runner, with: Entities::RunnerRegistrationDetails + else + not_found! + end + end + + desc 'Deletes a registered Runner' do + http_codes [[200, 'Runner was deleted'], [403, 'Forbidden']] + end + params do + requires :token, type: String, desc: %q(Runner's authentication token) + end + delete '/' do + authenticate_runner! + Ci::Runner.find_by_token(params[:token]).destroy + end + end + end +end diff --git a/lib/api/runners.rb b/lib/api/runners.rb index ecc8f2fc5a2..252e59bfa58 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -1,34 +1,43 @@ module API - # Runners API class Runners < Grape::API + include PaginationParams + before { authenticate! } resource :runners do - # Get runners available for user - # - # Example Request: - # GET /runners + desc 'Get runners available for user' do + success Entities::Runner + end + params do + optional :scope, type: String, values: %w[active paused online], + desc: 'The scope of specific runners to show' + use :pagination + end get do - runners = filter_runners(current_user.ci_authorized_runners, params[:scope], without: ['specific', 'shared']) + runners = filter_runners(current_user.ci_authorized_runners, params[:scope], without: %w(specific shared)) present paginate(runners), with: Entities::Runner end - # Get all runners - shared and specific - # - # Example Request: - # GET /runners/all + desc 'Get all runners - shared and specific' do + success Entities::Runner + end + params do + optional :scope, type: String, values: %w[active paused online specific shared], + desc: 'The scope of specific runners to show' + use :pagination + end get 'all' do authenticated_as_admin! runners = filter_runners(Ci::Runner.all, params[:scope]) present paginate(runners), with: Entities::Runner end - # Get runner's details - # - # Parameters: - # id (required) - The ID of ther runner - # Example Request: - # GET /runners/:id + desc "Get runner's details" do + success Entities::RunnerDetails + end + params do + requires :id, type: Integer, desc: 'The ID of the runner' + end get ':id' do runner = get_runner(params[:id]) authenticate_show_runner!(runner) @@ -36,33 +45,36 @@ module API present runner, with: Entities::RunnerDetails, current_user: current_user end - # Update runner's details - # - # Parameters: - # id (required) - The ID of ther runner - # description (optional) - Runner's description - # active (optional) - Runner's status - # tag_list (optional) - Array of tags for runner - # Example Request: - # PUT /runners/:id + desc "Update runner's details" do + success Entities::RunnerDetails + end + params do + requires :id, type: Integer, desc: 'The ID of the runner' + optional :description, type: String, desc: 'The description of the runner' + optional :active, type: Boolean, desc: 'The state of a runner' + optional :tag_list, type: Array[String], desc: 'The list of tags for a runner' + optional :run_untagged, type: Boolean, desc: 'Flag indicating the runner can execute untagged jobs' + optional :locked, type: Boolean, desc: 'Flag indicating the runner is locked' + at_least_one_of :description, :active, :tag_list, :run_untagged, :locked + end put ':id' do - runner = get_runner(params[:id]) + runner = get_runner(params.delete(:id)) authenticate_update_runner!(runner) + update_service = Ci::UpdateRunnerService.new(runner) - attrs = attributes_for_keys [:description, :active, :tag_list, :run_untagged, :locked] - if runner.update(attrs) + if update_service.update(declared_params(include_missing: false)) present runner, with: Entities::RunnerDetails, current_user: current_user else render_validation_error!(runner) end end - # Remove runner - # - # Parameters: - # id (required) - The ID of ther runner - # Example Request: - # DELETE /runners/:id + desc 'Remove a runner' do + success Entities::Runner + end + params do + requires :id, type: Integer, desc: 'The ID of the runner' + end delete ':id' do runner = get_runner(params[:id]) authenticate_delete_runner!(runner) @@ -72,28 +84,32 @@ module API end end + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do before { authorize_admin_project } - # Get runners available for project - # - # Example Request: - # GET /projects/:id/runners + desc 'Get runners available for project' do + success Entities::Runner + end + params do + optional :scope, type: String, values: %w[active paused online specific shared], + desc: 'The scope of specific runners to show' + use :pagination + end get ':id/runners' do runners = filter_runners(Ci::Runner.owned_or_shared(user_project.id), params[:scope]) present paginate(runners), with: Entities::Runner end - # Enable runner for project - # - # Parameters: - # id (required) - The ID of the project - # runner_id (required) - The ID of the runner - # Example Request: - # POST /projects/:id/runners/:runner_id + desc 'Enable a runner for a project' do + success Entities::Runner + end + params do + requires :runner_id, type: Integer, desc: 'The ID of the runner' + end post ':id/runners' do - required_attributes! [:runner_id] - runner = get_runner(params[:runner_id]) authenticate_enable_runner!(runner) @@ -106,13 +122,12 @@ module API end end - # Disable project's runner - # - # Parameters: - # id (required) - The ID of the project - # runner_id (required) - The ID of the runner - # Example Request: - # DELETE /projects/:id/runners/:runner_id + desc "Disable project's runner" do + success Entities::Runner + end + params do + requires :runner_id, type: Integer, desc: 'The ID of the runner' + end delete ':id/runners/:runner_id' do runner_project = user_project.runner_projects.find_by(runner_id: params[:runner_id]) not_found!('Runner') unless runner_project diff --git a/lib/api/services.rb b/lib/api/services.rb index fc8598daa32..ad856115485 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -1,62 +1,715 @@ module API - # Projects API class Services < Grape::API - before { authenticate! } - before { authorize_admin_project } + services = { + 'asana' => [ + { + required: true, + name: :api_key, + type: String, + desc: 'User API token' + }, + { + required: false, + name: :restrict_to_branch, + type: String, + desc: 'Comma-separated list of branches which will be automatically inspected. Leave blank to include all branches' + } + ], + 'assembla' => [ + { + required: true, + name: :token, + type: String, + desc: 'The authentication token' + }, + { + required: false, + name: :subdomain, + type: String, + desc: 'Subdomain setting' + } + ], + 'bamboo' => [ + { + required: true, + name: :bamboo_url, + type: String, + desc: 'Bamboo root URL like https://bamboo.example.com' + }, + { + required: true, + name: :build_key, + type: String, + desc: 'Bamboo build plan key like' + }, + { + required: true, + name: :username, + type: String, + desc: 'A user with API access, if applicable' + }, + { + required: true, + name: :password, + type: String, + desc: 'Passord of the user' + } + ], + 'bugzilla' => [ + { + required: true, + name: :new_issue_url, + type: String, + desc: 'New issue URL' + }, + { + required: true, + name: :issues_url, + type: String, + desc: 'Issues URL' + }, + { + required: true, + name: :project_url, + type: String, + desc: 'Project URL' + }, + { + required: false, + name: :description, + type: String, + desc: 'Description' + }, + { + required: false, + name: :title, + type: String, + desc: 'Title' + } + ], + 'buildkite' => [ + { + required: true, + name: :token, + type: String, + desc: 'Buildkite project GitLab token' + }, + { + required: true, + name: :project_url, + type: String, + desc: 'The buildkite project URL' + }, + { + required: false, + name: :enable_ssl_verification, + type: Boolean, + desc: 'Enable SSL verification for communication' + } + ], + 'builds-email' => [ + { + required: true, + name: :recipients, + type: String, + desc: 'Comma-separated list of recipient email addresses' + }, + { + required: false, + name: :add_pusher, + type: Boolean, + desc: 'Add pusher to recipients list' + }, + { + required: false, + name: :notify_only_broken_builds, + type: Boolean, + desc: 'Notify only broken builds' + } + ], + 'campfire' => [ + { + required: true, + name: :token, + type: String, + desc: 'Campfire token' + }, + { + required: false, + name: :subdomain, + type: String, + desc: 'Campfire subdomain' + }, + { + required: false, + name: :room, + type: String, + desc: 'Campfire room' + } + ], + 'custom-issue-tracker' => [ + { + required: true, + name: :new_issue_url, + type: String, + desc: 'New issue URL' + }, + { + required: true, + name: :issues_url, + type: String, + desc: 'Issues URL' + }, + { + required: true, + name: :project_url, + type: String, + desc: 'Project URL' + }, + { + required: false, + name: :description, + type: String, + desc: 'Description' + }, + { + required: false, + name: :title, + type: String, + desc: 'Title' + } + ], + 'drone-ci' => [ + { + required: true, + name: :token, + type: String, + desc: 'Drone CI token' + }, + { + required: true, + name: :drone_url, + type: String, + desc: 'Drone CI URL' + }, + { + required: false, + name: :enable_ssl_verification, + type: Boolean, + desc: 'Enable SSL verification for communication' + } + ], + 'emails-on-push' => [ + { + required: true, + name: :recipients, + type: String, + desc: 'Comma-separated list of recipient email addresses' + }, + { + required: false, + name: :disable_diffs, + type: Boolean, + desc: 'Disable code diffs' + }, + { + required: false, + name: :send_from_committer_email, + type: Boolean, + desc: 'Send from committer' + } + ], + 'external-wiki' => [ + { + required: true, + name: :external_wiki_url, + type: String, + desc: 'The URL of the external Wiki' + } + ], + 'flowdock' => [ + { + required: true, + name: :token, + type: String, + desc: 'Flowdock token' + } + ], + 'gemnasium' => [ + { + required: true, + name: :api_key, + type: String, + desc: 'Your personal API key on gemnasium.com' + }, + { + required: true, + name: :token, + type: String, + desc: "The project's slug on gemnasium.com" + } + ], + 'hipchat' => [ + { + required: true, + name: :token, + type: String, + desc: 'The room token' + }, + { + required: false, + name: :room, + type: String, + desc: 'The room name or ID' + }, + { + required: false, + name: :color, + type: String, + desc: 'The room color' + }, + { + required: false, + name: :notify, + type: Boolean, + desc: 'Enable notifications' + }, + { + required: false, + name: :api_version, + type: String, + desc: 'Leave blank for default (v2)' + }, + { + required: false, + name: :server, + type: String, + desc: 'Leave blank for default. https://hipchat.example.com' + } + ], + 'irker' => [ + { + required: true, + name: :recipients, + type: String, + desc: 'Recipients/channels separated by whitespaces' + }, + { + required: false, + name: :default_irc_uri, + type: String, + desc: 'Default: irc://irc.network.net:6697' + }, + { + required: false, + name: :server_host, + type: String, + desc: 'Server host. Default localhost' + }, + { + required: false, + name: :server_port, + type: Integer, + desc: 'Server port. Default 6659' + }, + { + required: false, + name: :colorize_messages, + type: Boolean, + desc: 'Colorize messages' + } + ], + 'jira' => [ + { + required: true, + name: :url, + type: String, + desc: 'The URL to the JIRA project which is being linked to this GitLab project, e.g., https://jira.example.com' + }, + { + required: true, + name: :project_key, + type: String, + desc: 'The short identifier for your JIRA project, all uppercase, e.g., PROJ' + }, + { + required: false, + name: :username, + type: String, + desc: 'The username of the user created to be used with GitLab/JIRA' + }, + { + required: false, + name: :password, + type: String, + desc: 'The password of the user created to be used with GitLab/JIRA' + }, + { + required: false, + name: :jira_issue_transition_id, + type: Integer, + desc: 'The ID of a transition that moves issues to a closed state. You can find this number under the JIRA workflow administration (**Administration > Issues > Workflows**) by selecting **View** under **Operations** of the desired workflow of your project. The ID of each state can be found inside the parenthesis of each transition name under the **Transitions (id)** column ([see screenshot][trans]). By default, this ID is set to `2`' + } + ], + + 'kubernetes' => [ + { + required: true, + name: :namespace, + type: String, + desc: 'The Kubernetes namespace to use' + }, + { + required: true, + name: :api_url, + type: String, + desc: 'The URL to the Kubernetes cluster API, e.g., https://kubernetes.example.com' + }, + { + required: true, + name: :token, + type: String, + desc: 'The service token to authenticate against the Kubernetes cluster with' + }, + { + required: false, + name: :ca_pem, + type: String, + desc: 'A custom certificate authority bundle to verify the Kubernetes cluster with (PEM format)' + }, + ], + 'mattermost-slash-commands' => [ + { + required: true, + name: :token, + type: String, + desc: 'The Mattermost token' + } + ], + 'slack-slash-commands' => [ + { + required: true, + name: :token, + type: String, + desc: 'The Slack token' + } + ], + 'pipelines-email' => [ + { + required: true, + name: :recipients, + type: String, + desc: 'Comma-separated list of recipient email addresses' + }, + { + required: false, + name: :notify_only_broken_builds, + type: Boolean, + desc: 'Notify only broken builds' + } + ], + 'pivotaltracker' => [ + { + required: true, + name: :token, + type: String, + desc: 'The Pivotaltracker token' + }, + { + required: false, + name: :restrict_to_branch, + type: String, + desc: 'Comma-separated list of branches which will be automatically inspected. Leave blank to include all branches.' + } + ], + 'pushover' => [ + { + required: true, + name: :api_key, + type: String, + desc: 'The application key' + }, + { + required: true, + name: :user_key, + type: String, + desc: 'The user key' + }, + { + required: true, + name: :priority, + type: String, + desc: 'The priority' + }, + { + required: true, + name: :device, + type: String, + desc: 'Leave blank for all active devices' + }, + { + required: true, + name: :sound, + type: String, + desc: 'The sound of the notification' + } + ], + 'redmine' => [ + { + required: true, + name: :new_issue_url, + type: String, + desc: 'The new issue URL' + }, + { + required: true, + name: :project_url, + type: String, + desc: 'The project URL' + }, + { + required: true, + name: :issues_url, + type: String, + desc: 'The issues URL' + }, + { + required: false, + name: :description, + type: String, + desc: 'The description of the tracker' + } + ], + 'slack' => [ + { + required: true, + name: :webhook, + type: String, + desc: 'The Slack webhook. e.g. https://hooks.slack.com/services/...' + }, + { + required: false, + name: :new_issue_url, + type: String, + desc: 'The user name' + }, + { + required: false, + name: :channel, + type: String, + desc: 'The channel name' + } + ], + 'mattermost' => [ + { + required: true, + name: :webhook, + type: String, + desc: 'The Mattermost webhook. e.g. http://mattermost_host/hooks/...' + } + ], + 'teamcity' => [ + { + required: true, + name: :teamcity_url, + type: String, + desc: 'TeamCity root URL like https://teamcity.example.com' + }, + { + required: true, + name: :build_type, + type: String, + desc: 'Build configuration ID' + }, + { + required: true, + name: :username, + type: String, + desc: 'A user with permissions to trigger a manual build' + }, + { + required: true, + name: :password, + type: String, + desc: 'The password of the user' + } + ] + } + + service_classes = [ + AsanaService, + AssemblaService, + BambooService, + BugzillaService, + BuildkiteService, + BuildsEmailService, + CampfireService, + CustomIssueTrackerService, + DroneCiService, + EmailsOnPushService, + ExternalWikiService, + FlowdockService, + GemnasiumService, + HipchatService, + IrkerService, + JiraService, + KubernetesService, + MattermostSlashCommandsService, + SlackSlashCommandsService, + PipelinesEmailService, + PivotaltrackerService, + PushoverService, + RedmineService, + SlackService, + MattermostService, + TeamcityService, + ] + + if Rails.env.development? + services['mock-ci'] = [ + { + required: true, + name: :mock_service_url, + type: String, + desc: 'URL to the mock service' + } + ] + + service_classes << MockCiService + end + + trigger_services = { + 'mattermost-slash-commands' => [ + { + name: :token, + type: String, + desc: 'The Mattermost token' + } + ], + 'slack-slash-commands' => [ + { + name: :token, + type: String, + desc: 'The Slack token' + } + ] + }.freeze resource :projects do - # Set <service_slug> service for project - # - # Example Request: - # - # PUT /projects/:id/services/gitlab-ci - # - put ':id/services/:service_slug' do - if project_service - validators = project_service.class.validators.select do |s| - s.class == ActiveRecord::Validations::PresenceValidator && - s.attributes != [:project_id] + before { authenticate! } + before { authorize_admin_project } + + helpers do + def service_attributes(service) + service.fields.inject([]) do |arr, hash| + arr << hash[:name].to_sym + end + end + end + + services.each do |service_slug, settings| + desc "Set #{service_slug} service for project" + params do + service_classes.each do |service| + event_names = service.try(:event_names) || [] + event_names.each do |event_name| + services[service.to_param.tr("_", "-")] << { + required: false, + name: event_name.to_sym, + type: String, + desc: ServicesHelper.service_event_description(event_name) + } + end end + services.freeze - required_attributes! validators.map(&:attributes).flatten.uniq - attrs = attributes_for_keys service_attributes + settings.each do |setting| + if setting[:required] + requires setting[:name], type: setting[:type], desc: setting[:desc] + else + optional setting[:name], type: setting[:type], desc: setting[:desc] + end + end + end + put ":id/services/#{service_slug}" do + service = user_project.find_or_initialize_service(service_slug.underscore) + service_params = declared_params(include_missing: false).merge(active: true) - if project_service.update_attributes(attrs.merge(active: true)) - true + if service.update_attributes(service_params) + present service, with: Entities::ProjectService, include_passwords: current_user.is_admin? else - not_found! + render_api_error!('400 Bad Request', 400) end end end - # Delete <service_slug> service for project - # - # Example Request: - # - # DELETE /project/:id/services/gitlab-ci - # - delete ':id/services/:service_slug' do - if project_service - attrs = service_attributes.inject({}) do |hash, key| - hash.merge!(key => nil) - end + desc "Delete a service for project" + params do + requires :service_slug, type: String, values: services.keys, desc: 'The name of the service' + end + delete ":id/services/:service_slug" do + service = user_project.find_or_initialize_service(params[:service_slug].underscore) - if project_service.update_attributes(attrs.merge(active: false)) - true - else - not_found! + attrs = service_attributes(service).inject({}) do |hash, key| + hash.merge!(key => nil) + end + + if service.update_attributes(attrs.merge(active: false)) + true + else + render_api_error!('400 Bad Request', 400) + end + end + + desc 'Get the service settings for project' do + success Entities::ProjectService + end + params do + requires :service_slug, type: String, values: services.keys, desc: 'The name of the service' + end + get ":id/services/:service_slug" do + service = user_project.find_or_initialize_service(params[:service_slug].underscore) + present service, with: Entities::ProjectService, include_passwords: current_user.is_admin? + end + end + + trigger_services.each do |service_slug, settings| + helpers do + def chat_command_service(project, service_slug, params) + project.services.active.where(template: false).find do |service| + service.try(:token) == params[:token] && service.to_param == service_slug.underscore end end end - # Get <service_slug> service settings for project - # - # Example Request: - # - # GET /project/:id/services/gitlab-ci - # - get ':id/services/:service_slug' do - present project_service, with: Entities::ProjectService, include_passwords: current_user.is_admin? + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects do + desc "Trigger a slash command for #{service_slug}" do + detail 'Added in GitLab 8.13' + end + params do + settings.each do |setting| + requires setting[:name], type: setting[:type], desc: setting[:desc] + end + end + post ":id/services/#{service_slug.underscore}/trigger" do + project = find_project(params[:id]) + + # This is not accurate, but done to prevent leakage of the project names + not_found!('Service') unless project + + service = chat_command_service(project, service_slug, params) + result = service.try(:trigger, params) + + if result + status result[:status] || 200 + present result + else + not_found!('Service') + end + end end end end diff --git a/lib/api/session.rb b/lib/api/session.rb index 56c202f1294..002ffd1d154 100644 --- a/lib/api/session.rb +++ b/lib/api/session.rb @@ -1,20 +1,20 @@ module API - # Users API class Session < Grape::API - # Login to get token - # - # Parameters: - # login (*required) - user login - # email (*required) - user email - # password (required) - user password - # - # Example Request: - # POST /session + desc 'Login to get token' do + success Entities::UserWithPrivateToken + end + params do + optional :login, type: String, desc: 'The username' + optional :email, type: String, desc: 'The email of the user' + requires :password, type: String, desc: 'The password of the user' + at_least_one_of :login, :email + end post "/session" do user = Gitlab::Auth.find_with_user_password(params[:email] || params[:login], params[:password]) return unauthorized! unless user - present user, with: Entities::UserLogin + return render_api_error!('401 Unauthorized. You have 2FA enabled. Please use a personal access token to access the API', 401) if user.two_factor_enabled? + present user, with: Entities::UserWithPrivateToken end end end diff --git a/lib/api/settings.rb b/lib/api/settings.rb index c885fcd7ea3..936c7e0930b 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -9,23 +9,126 @@ module API end end - # Get current applicaiton settings - # - # Example Request: - # GET /application/settings + desc 'Get the current application settings' do + success Entities::ApplicationSetting + end get "application/settings" do present current_settings, with: Entities::ApplicationSetting end - # Modify applicaiton settings - # - # Example Request: - # PUT /application/settings + desc 'Modify application settings' do + success Entities::ApplicationSetting + end + params do + optional :default_branch_protection, type: Integer, values: [0, 1, 2], desc: 'Determine if developers can push to master' + optional :default_project_visibility, type: Integer, values: Gitlab::VisibilityLevel.values, desc: 'The default project visibility' + optional :default_snippet_visibility, type: Integer, values: Gitlab::VisibilityLevel.values, desc: 'The default snippet visibility' + optional :default_group_visibility, type: Integer, values: Gitlab::VisibilityLevel.values, desc: 'The default group visibility' + optional :restricted_visibility_levels, type: Array[String], desc: 'Selected levels cannot be used by non-admin users for projects or snippets. If the public level is restricted, user profiles are only visible to logged in users.' + optional :import_sources, type: Array[String], values: %w[github bitbucket gitlab google_code fogbugz git gitlab_project], + desc: 'Enabled sources for code import during project creation. OmniAuth must be configured for GitHub, Bitbucket, and GitLab.com' + optional :disabled_oauth_sign_in_sources, type: Array[String], desc: 'Disable certain OAuth sign-in sources' + optional :enabled_git_access_protocol, type: String, values: %w[ssh http nil], desc: 'Allow only the selected protocols to be used for Git access.' + optional :gravatar_enabled, type: Boolean, desc: 'Flag indicating if the Gravatar service is enabled' + optional :default_projects_limit, type: Integer, desc: 'The maximum number of personal projects' + optional :max_attachment_size, type: Integer, desc: 'Maximum attachment size in MB' + optional :session_expire_delay, type: Integer, desc: 'Session duration in minutes. GitLab restart is required to apply changes.' + optional :user_oauth_applications, type: Boolean, desc: 'Allow users to register any application to use GitLab as an OAuth provider' + optional :user_default_external, type: Boolean, desc: 'Newly registered users will by default be external' + optional :signup_enabled, type: Boolean, desc: 'Flag indicating if sign up is enabled' + optional :send_user_confirmation_email, type: Boolean, desc: 'Send confirmation email on sign-up' + optional :domain_whitelist, type: String, desc: 'ONLY users with e-mail addresses that match these domain(s) will be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com' + optional :domain_blacklist_enabled, type: Boolean, desc: 'Enable domain blacklist for sign ups' + given domain_blacklist_enabled: ->(val) { val } do + requires :domain_blacklist, type: String, desc: 'Users with e-mail addresses that match these domain(s) will NOT be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com' + end + optional :after_sign_up_text, type: String, desc: 'Text shown after sign up' + optional :signin_enabled, type: Boolean, desc: 'Flag indicating if sign in is enabled' + optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users to setup Two-factor authentication' + given require_two_factor_authentication: ->(val) { val } do + requires :two_factor_grace_period, type: Integer, desc: 'Amount of time (in hours) that users are allowed to skip forced configuration of two-factor authentication' + end + optional :home_page_url, type: String, desc: 'We will redirect non-logged in users to this page' + optional :after_sign_out_path, type: String, desc: 'We will redirect users to this page after they sign out' + optional :sign_in_text, type: String, desc: 'The sign in text of the GitLab application' + optional :help_page_text, type: String, desc: 'Custom text displayed on the help page' + optional :shared_runners_enabled, type: Boolean, desc: 'Enable shared runners for new projects' + given shared_runners_enabled: ->(val) { val } do + requires :shared_runners_text, type: String, desc: 'Shared runners text ' + end + optional :max_artifacts_size, type: Integer, desc: "Set the maximum file size for each job's artifacts" + optional :default_artifacts_expire_in, type: String, desc: "Set the default expiration time for each job's artifacts" + optional :max_pages_size, type: Integer, desc: 'Maximum size of pages in MB' + optional :container_registry_token_expire_delay, type: Integer, desc: 'Authorization token duration (minutes)' + optional :metrics_enabled, type: Boolean, desc: 'Enable the InfluxDB metrics' + given metrics_enabled: ->(val) { val } do + requires :metrics_host, type: String, desc: 'The InfluxDB host' + requires :metrics_port, type: Integer, desc: 'The UDP port to use for connecting to InfluxDB' + requires :metrics_pool_size, type: Integer, desc: 'The amount of InfluxDB connections to open' + requires :metrics_timeout, type: Integer, desc: 'The amount of seconds after which an InfluxDB connection will time out' + requires :metrics_method_call_threshold, type: Integer, desc: 'A method call is only tracked when it takes longer to complete than the given amount of milliseconds.' + requires :metrics_sample_interval, type: Integer, desc: 'The sampling interval in seconds' + requires :metrics_packet_size, type: Integer, desc: 'The amount of points to store in a single UDP packet' + end + optional :sidekiq_throttling_enabled, type: Boolean, desc: 'Enable Sidekiq Job Throttling' + given sidekiq_throttling_enabled: ->(val) { val } do + requires :sidekiq_throttling_queus, type: Array[String], desc: 'Choose which queues you wish to throttle' + requires :sidekiq_throttling_factor, type: Float, desc: 'The factor by which the queues should be throttled. A value between 0.0 and 1.0, exclusive.' + end + optional :recaptcha_enabled, type: Boolean, desc: 'Helps prevent bots from creating accounts' + given recaptcha_enabled: ->(val) { val } do + requires :recaptcha_site_key, type: String, desc: 'Generate site key at http://www.google.com/recaptcha' + requires :recaptcha_private_key, type: String, desc: 'Generate private key at http://www.google.com/recaptcha' + end + optional :akismet_enabled, type: Boolean, desc: 'Helps prevent bots from creating issues' + given akismet_enabled: ->(val) { val } do + requires :akismet_api_key, type: String, desc: 'Generate API key at http://www.akismet.com' + end + optional :admin_notification_email, type: String, desc: 'Abuse reports will be sent to this address if it is set. Abuse reports are always available in the admin area.' + optional :sentry_enabled, type: Boolean, desc: 'Sentry is an error reporting and logging tool which is currently not shipped with GitLab, get it here: https://getsentry.com' + given sentry_enabled: ->(val) { val } do + requires :sentry_dsn, type: String, desc: 'Sentry Data Source Name' + end + optional :repository_storage, type: String, desc: 'Storage paths for new projects' + optional :repository_checks_enabled, type: Boolean, desc: "GitLab will periodically run 'git fsck' in all project and wiki repositories to look for silent disk corruption issues." + optional :koding_enabled, type: Boolean, desc: 'Enable Koding' + given koding_enabled: ->(val) { val } do + requires :koding_url, type: String, desc: 'The Koding team URL' + end + optional :plantuml_enabled, type: Boolean, desc: 'Enable PlantUML' + given plantuml_enabled: ->(val) { val } do + requires :plantuml_url, type: String, desc: 'The PlantUML server URL' + end + optional :version_check_enabled, type: Boolean, desc: 'Let GitLab inform you when an update is available.' + optional :email_author_in_body, type: Boolean, desc: 'Some email servers do not support overriding the email sender name. Enable this option to include the name of the author of the issue, merge request or comment in the email body instead.' + optional :html_emails_enabled, type: Boolean, desc: 'By default GitLab sends emails in HTML and plain text formats so mail clients can choose what format to use. Disable this option if you only want to send emails in plain text format.' + optional :housekeeping_enabled, type: Boolean, desc: 'Enable automatic repository housekeeping (git repack, git gc)' + given housekeeping_enabled: ->(val) { val } do + requires :housekeeping_bitmaps_enabled, type: Boolean, desc: "Creating pack file bitmaps makes housekeeping take a little longer but bitmaps should accelerate 'git clone' performance." + requires :housekeeping_incremental_repack_period, type: Integer, desc: "Number of Git pushes after which an incremental 'git repack' is run." + requires :housekeeping_full_repack_period, type: Integer, desc: "Number of Git pushes after which a full 'git repack' is run." + requires :housekeeping_gc_period, type: Integer, desc: "Number of Git pushes after which 'git gc' is run." + end + optional :terminal_max_session_time, type: Integer, desc: 'Maximum time for web terminal websocket connection (in seconds). Set to 0 for unlimited time.' + at_least_one_of :default_branch_protection, :default_project_visibility, :default_snippet_visibility, + :default_group_visibility, :restricted_visibility_levels, :import_sources, + :enabled_git_access_protocol, :gravatar_enabled, :default_projects_limit, + :max_attachment_size, :session_expire_delay, :disabled_oauth_sign_in_sources, + :user_oauth_applications, :user_default_external, :signup_enabled, + :send_user_confirmation_email, :domain_whitelist, :domain_blacklist_enabled, + :after_sign_up_text, :signin_enabled, :require_two_factor_authentication, + :home_page_url, :after_sign_out_path, :sign_in_text, :help_page_text, + :shared_runners_enabled, :max_artifacts_size, + :default_artifacts_expire_in, :max_pages_size, + :container_registry_token_expire_delay, + :metrics_enabled, :sidekiq_throttling_enabled, :recaptcha_enabled, + :akismet_enabled, :admin_notification_email, :sentry_enabled, + :repository_storage, :repository_checks_enabled, :koding_enabled, :plantuml_enabled, + :version_check_enabled, :email_author_in_body, :html_emails_enabled, + :housekeeping_enabled, :terminal_max_session_time + end put "application/settings" do - attributes = current_settings.attributes.keys - ["id"] - attrs = attributes_for_keys(attributes) - - if current_settings.update_attributes(attrs) + if current_settings.update_attributes(declared_params(include_missing: false)) present current_settings, with: Entities::ApplicationSetting else render_validation_error!(current_settings) diff --git a/lib/api/sidekiq_metrics.rb b/lib/api/sidekiq_metrics.rb index d3d6827dc54..11f2b40269a 100644 --- a/lib/api/sidekiq_metrics.rb +++ b/lib/api/sidekiq_metrics.rb @@ -39,50 +39,22 @@ module API end end - # Get Sidekiq Queue metrics - # - # Parameters: - # None - # - # Example: - # GET /sidekiq/queue_metrics - # + desc 'Get the Sidekiq queue metrics' get 'sidekiq/queue_metrics' do { queues: queue_metrics } end - # Get Sidekiq Process metrics - # - # Parameters: - # None - # - # Example: - # GET /sidekiq/process_metrics - # + desc 'Get the Sidekiq process metrics' get 'sidekiq/process_metrics' do { processes: process_metrics } end - # Get Sidekiq Job statistics - # - # Parameters: - # None - # - # Example: - # GET /sidekiq/job_stats - # + desc 'Get the Sidekiq job statistics' get 'sidekiq/job_stats' do { jobs: job_stats } end - # Get Sidekiq Compound metrics. Includes all previous metrics - # - # Parameters: - # None - # - # Example: - # GET /sidekiq/compound_metrics - # + desc 'Get the Sidekiq Compound metrics. Includes queue, process, and job statistics' get 'sidekiq/compound_metrics' do { queues: queue_metrics, processes: process_metrics, jobs: job_stats } end diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb new file mode 100644 index 00000000000..ac03fbd2a3d --- /dev/null +++ b/lib/api/snippets.rb @@ -0,0 +1,142 @@ +module API + # Snippets API + class Snippets < Grape::API + include PaginationParams + + before { authenticate! } + + resource :snippets do + helpers do + def snippets_for_current_user + SnippetsFinder.new.execute(current_user, filter: :by_user, user: current_user) + end + + def public_snippets + SnippetsFinder.new.execute(current_user, filter: :public) + end + end + + desc 'Get a snippets list for authenticated user' do + detail 'This feature was introduced in GitLab 8.15.' + success Entities::PersonalSnippet + end + params do + use :pagination + end + get do + present paginate(snippets_for_current_user), with: Entities::PersonalSnippet + end + + desc 'List all public snippets current_user has access to' do + detail 'This feature was introduced in GitLab 8.15.' + success Entities::PersonalSnippet + end + params do + use :pagination + end + get 'public' do + present paginate(public_snippets), with: Entities::PersonalSnippet + end + + desc 'Get a single snippet' do + detail 'This feature was introduced in GitLab 8.15.' + success Entities::PersonalSnippet + end + params do + requires :id, type: Integer, desc: 'The ID of a snippet' + end + get ':id' do + snippet = snippets_for_current_user.find(params[:id]) + present snippet, with: Entities::PersonalSnippet + end + + desc 'Create new snippet' do + detail 'This feature was introduced in GitLab 8.15.' + success Entities::PersonalSnippet + end + params do + requires :title, type: String, desc: 'The title of a snippet' + requires :file_name, type: String, desc: 'The name of a snippet file' + requires :content, type: String, desc: 'The content of a snippet' + optional :visibility_level, type: Integer, + values: Gitlab::VisibilityLevel.values, + default: Gitlab::VisibilityLevel::INTERNAL, + desc: 'The visibility level of the snippet' + end + post do + attrs = declared_params(include_missing: false).merge(request: request, api: true) + snippet = CreateSnippetService.new(nil, current_user, attrs).execute + + render_spam_error! if snippet.spam? + + if snippet.persisted? + present snippet, with: Entities::PersonalSnippet + else + render_validation_error!(snippet) + end + end + + desc 'Update an existing snippet' do + detail 'This feature was introduced in GitLab 8.15.' + success Entities::PersonalSnippet + end + params do + requires :id, type: Integer, desc: 'The ID of a snippet' + optional :title, type: String, desc: 'The title of a snippet' + optional :file_name, type: String, desc: 'The name of a snippet file' + optional :content, type: String, desc: 'The content of a snippet' + optional :visibility_level, type: Integer, + values: Gitlab::VisibilityLevel.values, + desc: 'The visibility level of the snippet' + at_least_one_of :title, :file_name, :content, :visibility_level + end + put ':id' do + snippet = snippets_for_current_user.find_by(id: params.delete(:id)) + return not_found!('Snippet') unless snippet + authorize! :update_personal_snippet, snippet + + attrs = declared_params(include_missing: false).merge(request: request, api: true) + + UpdateSnippetService.new(nil, current_user, snippet, attrs).execute + + render_spam_error! if snippet.spam? + + if snippet.persisted? + present snippet, with: Entities::PersonalSnippet + else + render_validation_error!(snippet) + end + end + + desc 'Remove snippet' do + detail 'This feature was introduced in GitLab 8.15.' + success Entities::PersonalSnippet + end + params do + requires :id, type: Integer, desc: 'The ID of a snippet' + end + delete ':id' do + snippet = snippets_for_current_user.find_by(id: params.delete(:id)) + return not_found!('Snippet') unless snippet + authorize! :destroy_personal_snippet, snippet + snippet.destroy + no_content! + end + + desc 'Get a raw snippet' do + detail 'This feature was introduced in GitLab 8.15.' + end + params do + requires :id, type: Integer, desc: 'The ID of a snippet' + end + get ":id/raw" do + snippet = snippets_for_current_user.find_by(id: params.delete(:id)) + return not_found!('Snippet') unless snippet + + env['api.format'] = :txt + content_type 'text/plain' + present snippet.content + end + end + end +end diff --git a/lib/api/subscriptions.rb b/lib/api/subscriptions.rb index c49e2a21b82..772b5cca017 100644 --- a/lib/api/subscriptions.rb +++ b/lib/api/subscriptions.rb @@ -3,55 +3,45 @@ module API before { authenticate! } subscribable_types = { - 'merge_request' => proc { |id| user_project.merge_requests.find(id) }, - 'merge_requests' => proc { |id| user_project.merge_requests.find(id) }, + 'merge_requests' => proc { |id| find_merge_request_with_access(id, :update_merge_request) }, 'issues' => proc { |id| find_project_issue(id) }, 'labels' => proc { |id| find_project_label(id) }, } + params do + requires :id, type: String, desc: 'The ID of a project' + requires :subscribable_id, type: String, desc: 'The ID of a resource' + end resource :projects do subscribable_types.each do |type, finder| type_singularized = type.singularize - type_id_str = :"#{type_singularized}_id" entity_class = Entities.const_get(type_singularized.camelcase) - # Subscribe to a resource - # - # Parameters: - # id (required) - The ID of a project - # subscribable_id (required) - The ID of a resource - # Example Request: - # POST /projects/:id/labels/:subscribable_id/subscription - # POST /projects/:id/issues/:subscribable_id/subscription - # POST /projects/:id/merge_requests/:subscribable_id/subscription - post ":id/#{type}/:#{type_id_str}/subscription" do - resource = instance_exec(params[type_id_str], &finder) + desc 'Subscribe to a resource' do + success entity_class + end + post ":id/#{type}/:subscribable_id/subscribe" do + resource = instance_exec(params[:subscribable_id], &finder) - if resource.subscribed?(current_user) + if resource.subscribed?(current_user, user_project) not_modified! else - resource.subscribe(current_user) - present resource, with: entity_class, current_user: current_user + resource.subscribe(current_user, user_project) + present resource, with: entity_class, current_user: current_user, project: user_project end end - # Unsubscribe from a resource - # - # Parameters: - # id (required) - The ID of a project - # subscribable_id (required) - The ID of a resource - # Example Request: - # DELETE /projects/:id/labels/:subscribable_id/subscription - # DELETE /projects/:id/issues/:subscribable_id/subscription - # DELETE /projects/:id/merge_requests/:subscribable_id/subscription - delete ":id/#{type}/:#{type_id_str}/subscription" do - resource = instance_exec(params[type_id_str], &finder) + desc 'Unsubscribe from a resource' do + success entity_class + end + post ":id/#{type}/:subscribable_id/unsubscribe" do + resource = instance_exec(params[:subscribable_id], &finder) - if !resource.subscribed?(current_user) + if !resource.subscribed?(current_user, user_project) not_modified! else - resource.unsubscribe(current_user) - present resource, with: entity_class, current_user: current_user + resource.unsubscribe(current_user, user_project) + present resource, with: entity_class, current_user: current_user, project: user_project end end end diff --git a/lib/api/system_hooks.rb b/lib/api/system_hooks.rb index 22b8f90dc5c..d038a3fa828 100644 --- a/lib/api/system_hooks.rb +++ b/lib/api/system_hooks.rb @@ -1,44 +1,49 @@ module API - # Hooks API class SystemHooks < Grape::API + include PaginationParams + before do authenticate! authenticated_as_admin! end resource :hooks do - # Get the list of system hooks - # - # Example Request: - # GET /hooks + desc 'Get the list of system hooks' do + success Entities::Hook + end + params do + use :pagination + end get do - @hooks = SystemHook.all - present @hooks, with: Entities::Hook + present paginate(SystemHook.all), with: Entities::Hook end - # Create new system hook - # - # Parameters: - # url (required) - url for system hook - # Example Request - # POST /hooks + desc 'Create a new system hook' do + success Entities::Hook + end + params do + requires :url, type: String, desc: "The URL to send the request to" + optional :token, type: String, desc: 'The token used to validate payloads' + optional :push_events, type: Boolean, desc: "Trigger hook on push events" + optional :tag_push_events, type: Boolean, desc: "Trigger hook on tag push events" + optional :enable_ssl_verification, type: Boolean, desc: "Do SSL verification when triggering the hook" + end post do - attrs = attributes_for_keys [:url] - required_attributes! [:url] - @hook = SystemHook.new attrs - if @hook.save - present @hook, with: Entities::Hook + hook = SystemHook.new(declared_params(include_missing: false)) + + if hook.save + present hook, with: Entities::Hook else - not_found! + render_validation_error!(hook) end end - # Test a hook - # - # Example Request - # GET /hooks/:id + desc 'Test a hook' + params do + requires :id, type: Integer, desc: 'The ID of the system hook' + end get ":id" do - @hook = SystemHook.find(params[:id]) + hook = SystemHook.find(params[:id]) data = { event_name: "project_create", name: "Ruby", @@ -47,23 +52,21 @@ module API owner_name: "Someone", owner_email: "example@gitlabhq.com" } - @hook.execute(data, 'system_hooks') + hook.execute(data, 'system_hooks') data end - # Delete a hook. This is an idempotent function. - # - # Parameters: - # id (required) - ID of the hook - # Example Request: - # DELETE /hooks/:id + desc 'Delete a hook' do + success Entities::Hook + end + params do + requires :id, type: Integer, desc: 'The ID of the system hook' + end delete ":id" do - begin - @hook = SystemHook.find(params[:id]) - @hook.destroy - rescue - # SystemHook raises an Error if no hook with id found - end + hook = SystemHook.find_by(id: params[:id]) + not_found!('System hook') unless hook + + present hook.destroy, with: Entities::Hook end end end diff --git a/lib/api/tags.rb b/lib/api/tags.rb index 7b675e05fbb..86759ab882f 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -1,28 +1,30 @@ module API - # Git Tags API class Tags < Grape::API - before { authenticate! } + include PaginationParams + before { authorize! :download_code, user_project } + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do - # Get a project repository tags - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # GET /projects/:id/repository/tags + desc 'Get a project repository tags' do + success Entities::RepoTag + end + params do + use :pagination + end get ":id/repository/tags" do - present user_project.repository.tags.sort_by(&:name).reverse, - with: Entities::RepoTag, project: user_project + tags = ::Kaminari.paginate_array(user_project.repository.tags.sort_by(&:name).reverse) + present paginate(tags), with: Entities::RepoTag, project: user_project end - # Get a single repository tag - # - # Parameters: - # id (required) - The ID of a project - # tag_name (required) - The name of the tag - # Example Request: - # GET /projects/:id/repository/tags/:tag_name + desc 'Get a single repository tag' do + success Entities::RepoTag + end + params do + requires :tag_name, type: String, desc: 'The name of the tag' + end get ":id/repository/tags/:tag_name", requirements: { tag_name: /.+/ } do tag = user_project.repository.find_tag(params[:tag_name]) not_found!('Tag') unless tag @@ -30,20 +32,20 @@ module API present tag, with: Entities::RepoTag, project: user_project end - # Create tag - # - # Parameters: - # id (required) - The ID of a project - # tag_name (required) - The name of the tag - # ref (required) - Create tag from commit sha or branch - # message (optional) - Specifying a message creates an annotated tag. - # Example Request: - # POST /projects/:id/repository/tags + desc 'Create a new repository tag' do + success Entities::RepoTag + end + params do + requires :tag_name, type: String, desc: 'The name of the tag' + requires :ref, type: String, desc: 'The commit sha or branch name' + optional :message, type: String, desc: 'Specifying a message creates an annotated tag' + optional :release_description, type: String, desc: 'Specifying release notes stored in the GitLab database' + end post ':id/repository/tags' do authorize_push_project - message = params[:message] || nil - result = CreateTagService.new(user_project, current_user). - execute(params[:tag_name], params[:ref], message, params[:release_description]) + + result = ::Tags::CreateService.new(user_project, current_user). + execute(params[:tag_name], params[:ref], params[:message], params[:release_description]) if result[:status] == :success present result[:tag], @@ -54,16 +56,14 @@ module API end end - # Delete tag - # - # Parameters: - # id (required) - The ID of a project - # tag_name (required) - The name of the tag - # Example Request: - # DELETE /projects/:id/repository/tags/:tag + desc 'Delete a repository tag' + params do + requires :tag_name, type: String, desc: 'The name of the tag' + end delete ":id/repository/tags/:tag_name", requirements: { tag_name: /.+/ } do authorize_push_project - result = DeleteTagService.new(user_project, current_user). + + result = ::Tags::DestroyService.new(user_project, current_user). execute(params[:tag_name]) if result[:status] == :success @@ -75,17 +75,16 @@ module API end end - # Add release notes to tag - # - # Parameters: - # id (required) - The ID of a project - # tag_name (required) - The name of the tag - # description (required) - Release notes with markdown support - # Example Request: - # POST /projects/:id/repository/tags/:tag_name/release + desc 'Add a release note to a tag' do + success Entities::Release + end + params do + requires :tag_name, type: String, desc: 'The name of the tag' + requires :description, type: String, desc: 'Release notes with markdown support' + end post ':id/repository/tags/:tag_name/release', requirements: { tag_name: /.+/ } do authorize_push_project - required_attributes! [:description] + result = CreateReleaseService.new(user_project, current_user). execute(params[:tag_name], params[:description]) @@ -96,17 +95,16 @@ module API end end - # Updates a release notes of a tag - # - # Parameters: - # id (required) - The ID of a project - # tag_name (required) - The name of the tag - # description (required) - Release notes with markdown support - # Example Request: - # PUT /projects/:id/repository/tags/:tag_name/release + desc "Update a tag's release note" do + success Entities::Release + end + params do + requires :tag_name, type: String, desc: 'The name of the tag' + requires :description, type: String, desc: 'Release notes with markdown support' + end put ':id/repository/tags/:tag_name/release', requirements: { tag_name: /.+/ } do authorize_push_project - required_attributes! [:description] + result = UpdateReleaseService.new(user_project, current_user). execute(params[:tag_name], params[:description]) diff --git a/lib/api/templates.rb b/lib/api/templates.rb index 18408797756..0fc13b35d5b 100644 --- a/lib/api/templates.rb +++ b/lib/api/templates.rb @@ -1,35 +1,110 @@ module API class Templates < Grape::API - TEMPLATE_TYPES = { - gitignores: Gitlab::Template::Gitignore, - gitlab_ci_ymls: Gitlab::Template::GitlabCiYml + include PaginationParams + + GLOBAL_TEMPLATE_TYPES = { + gitignores: { + klass: Gitlab::Template::GitignoreTemplate, + gitlab_version: 8.8 + }, + gitlab_ci_ymls: { + klass: Gitlab::Template::GitlabCiYmlTemplate, + gitlab_version: 8.9 + }, + dockerfiles: { + klass: Gitlab::Template::DockerfileTemplate, + gitlab_version: 8.15 + } }.freeze + PROJECT_TEMPLATE_REGEX = + /[\<\{\[] + (project|description| + one\sline\s.+\swhat\sit\sdoes\.) # matching the start and end is enough here + [\>\}\]]/xi.freeze + YEAR_TEMPLATE_REGEX = /[<{\[](year|yyyy)[>}\]]/i.freeze + FULLNAME_TEMPLATE_REGEX = + /[\<\{\[] + (fullname|name\sof\s(author|copyright\sowner)) + [\>\}\]]/xi.freeze + + helpers do + def parsed_license_template + # We create a fresh Licensee::License object since we'll modify its + # content in place below. + template = Licensee::License.new(params[:name]) + + template.content.gsub!(YEAR_TEMPLATE_REGEX, Time.now.year.to_s) + template.content.gsub!(PROJECT_TEMPLATE_REGEX, params[:project]) if params[:project].present? + + fullname = params[:fullname].presence || current_user.try(:name) + template.content.gsub!(FULLNAME_TEMPLATE_REGEX, fullname) if fullname + template + end + + def render_response(template_type, template) + not_found!(template_type.to_s.singularize) unless template + present template, with: Entities::Template + end + end + + desc 'Get the list of the available license template' do + detail 'This feature was introduced in GitLab 8.7.' + success ::API::Entities::RepoLicense + end + params do + optional :popular, type: Boolean, desc: 'If passed, returns only popular licenses' + use :pagination + end + get "templates/licenses" do + options = { + featured: declared(params).popular.present? ? true : nil + } + licences = ::Kaminari.paginate_array(Licensee::License.all(options)) + present paginate(licences), with: Entities::RepoLicense + end + + desc 'Get the text for a specific license' do + detail 'This feature was introduced in GitLab 8.7.' + success ::API::Entities::RepoLicense + end + params do + requires :name, type: String, desc: 'The name of the template' + end + get "templates/licenses/:name", requirements: { name: /[\w\.-]+/ } do + not_found!('License') unless Licensee::License.find(declared(params).name) + + template = parsed_license_template + + present template, with: ::API::Entities::RepoLicense + end + + GLOBAL_TEMPLATE_TYPES.each do |template_type, properties| + klass = properties[:klass] + gitlab_version = properties[:gitlab_version] + + desc 'Get the list of the available template' do + detail "This feature was introduced in GitLab #{gitlab_version}." + success Entities::TemplatesList + end + params do + use :pagination + end + get "templates/#{template_type}" do + templates = ::Kaminari.paginate_array(klass.all) + present paginate(templates), with: Entities::TemplatesList + end + + desc 'Get the text for a specific template present in local filesystem' do + detail "This feature was introduced in GitLab #{gitlab_version}." + success Entities::Template + end + params do + requires :name, type: String, desc: 'The name of the template' + end + get "templates/#{template_type}/:name" do + new_template = klass.find(declared(params).name) - TEMPLATE_TYPES.each do |template, klass| - # Get the list of the available template - # - # Example Request: - # GET /gitignores - # GET /gitlab_ci_ymls - get template.to_s do - present klass.all, with: Entities::TemplatesList - end - - # Get the text for a specific template - # - # Parameters: - # name (required) - The name of a template - # - # Example Request: - # GET /gitignores/Elixir - # GET /gitlab_ci_ymls/Ruby - get "#{template}/:name" do - required_attributes! [:name] - - new_template = klass.find(params[:name]) - not_found!(template.to_s.singularize) unless new_template - - present new_template, with: Entities::Template + render_response(template_type, new_template) end end end diff --git a/lib/api/time_tracking_endpoints.rb b/lib/api/time_tracking_endpoints.rb new file mode 100644 index 00000000000..85b5f7d98b8 --- /dev/null +++ b/lib/api/time_tracking_endpoints.rb @@ -0,0 +1,114 @@ +module API + module TimeTrackingEndpoints + extend ActiveSupport::Concern + + included do + helpers do + def issuable_name + declared_params.has_key?(:issue_id) ? 'issue' : 'merge_request' + end + + def issuable_key + "#{issuable_name}_id".to_sym + end + + def update_issuable_key + "update_#{issuable_name}".to_sym + end + + def read_issuable_key + "read_#{issuable_name}".to_sym + end + + def load_issuable + @issuable ||= begin + case issuable_name + when 'issue' + find_project_issue(params.delete(issuable_key)) + when 'merge_request' + find_project_merge_request(params.delete(issuable_key)) + end + end + end + + def update_issuable(attrs) + custom_params = declared_params(include_missing: false) + custom_params.merge!(attrs) + + issuable = update_service.new(user_project, current_user, custom_params).execute(load_issuable) + if issuable.valid? + present issuable, with: Entities::IssuableTimeStats + else + render_validation_error!(issuable) + end + end + + def update_service + issuable_name == 'issue' ? ::Issues::UpdateService : ::MergeRequests::UpdateService + end + end + + issuable_name = name.end_with?('Issues') ? 'issue' : 'merge_request' + issuable_collection_name = issuable_name.pluralize + issuable_key = "#{issuable_name}_id".to_sym + + desc "Set a time estimate for a project #{issuable_name}" + params do + requires issuable_key, type: Integer, desc: "The ID of a project #{issuable_name}" + requires :duration, type: String, desc: 'The duration to be parsed' + end + post ":id/#{issuable_collection_name}/:#{issuable_key}/time_estimate" do + authorize! update_issuable_key, load_issuable + + status :ok + update_issuable(time_estimate: Gitlab::TimeTrackingFormatter.parse(params.delete(:duration))) + end + + desc "Reset the time estimate for a project #{issuable_name}" + params do + requires issuable_key, type: Integer, desc: "The ID of a project #{issuable_name}" + end + post ":id/#{issuable_collection_name}/:#{issuable_key}/reset_time_estimate" do + authorize! update_issuable_key, load_issuable + + status :ok + update_issuable(time_estimate: 0) + end + + desc "Add spent time for a project #{issuable_name}" + params do + requires issuable_key, type: Integer, desc: "The ID of a project #{issuable_name}" + requires :duration, type: String, desc: 'The duration to be parsed' + end + post ":id/#{issuable_collection_name}/:#{issuable_key}/add_spent_time" do + authorize! update_issuable_key, load_issuable + + update_issuable(spend_time: { + duration: Gitlab::TimeTrackingFormatter.parse(params.delete(:duration)), + user: current_user + }) + end + + desc "Reset spent time for a project #{issuable_name}" + params do + requires issuable_key, type: Integer, desc: "The ID of a project #{issuable_name}" + end + post ":id/#{issuable_collection_name}/:#{issuable_key}/reset_spent_time" do + authorize! update_issuable_key, load_issuable + + status :ok + update_issuable(spend_time: { duration: :reset, user: current_user }) + end + + desc "Show time stats for a project #{issuable_name}" + params do + requires issuable_key, type: Integer, desc: "The ID of a project #{issuable_name}" + end + get ":id/#{issuable_collection_name}/:#{issuable_key}/time_stats" do + authorize! read_issuable_key, load_issuable + + present load_issuable, with: Entities::IssuableTimeStats + end + end + end +end diff --git a/lib/api/todos.rb b/lib/api/todos.rb index 26c24c3baff..e59030428da 100644 --- a/lib/api/todos.rb +++ b/lib/api/todos.rb @@ -1,25 +1,27 @@ module API - # Todos API class Todos < Grape::API + include PaginationParams + before { authenticate! } ISSUABLE_TYPES = { - 'merge_requests' => ->(id) { user_project.merge_requests.find(id) }, + 'merge_requests' => ->(id) { find_merge_request_with_access(id) }, 'issues' => ->(id) { find_project_issue(id) } - } + }.freeze + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do ISSUABLE_TYPES.each do |type, finder| type_id_str = "#{type.singularize}_id".to_sym - # Create a todo on an issuable - # - # Parameters: - # id (required) - The ID of a project - # issuable_id (required) - The ID of an issuable - # Example Request: - # POST /projects/:id/issues/:issuable_id/todo - # POST /projects/:id/merge_requests/:issuable_id/todo + desc 'Create a todo on an issuable' do + success Entities::Todo + end + params do + requires type_id_str, type: Integer, desc: 'The ID of an issuable' + end post ":id/#{type}/:#{type_id_str}/todo" do issuable = instance_exec(params[type_id_str], &finder) todo = TodoService.new.mark_todo(issuable, current_user).first @@ -40,42 +42,35 @@ module API end end - # Get a todo list - # - # Example Request: - # GET /todos - # + desc 'Get a todo list' do + success Entities::Todo + end + params do + use :pagination + end get do - todos = find_todos - - present paginate(todos), with: Entities::Todo, current_user: current_user + present paginate(find_todos), with: Entities::Todo, current_user: current_user end - # Mark a todo as done - # - # Parameters: - # id: (required) - The ID of the todo being marked as done - # - # Example Request: - # DELETE /todos/:id - # - delete ':id' do + desc 'Mark a todo as done' do + success Entities::Todo + end + params do + requires :id, type: Integer, desc: 'The ID of the todo being marked as done' + end + post ':id/mark_as_done' do todo = current_user.todos.find(params[:id]) - todo.done + TodoService.new.mark_todos_as_done([todo], current_user) - present todo, with: Entities::Todo, current_user: current_user + present todo.reload, with: Entities::Todo, current_user: current_user end - # Mark all todos as done - # - # Example Request: - # DELETE /todos - # - delete do + desc 'Mark all todos as done' + post '/mark_as_done' do todos = find_todos - todos.each(&:done) + TodoService.new.mark_todos_as_done(todos, current_user) - todos.length + no_content! end end end diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb index d1d07394e92..ea0ad852633 100644 --- a/lib/api/triggers.rb +++ b/lib/api/triggers.rb @@ -1,37 +1,29 @@ module API - # Triggers API class Triggers < Grape::API + include PaginationParams + + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do - # Trigger a GitLab project build - # - # Parameters: - # id (required) - The ID of a CI project - # ref (required) - The name of project's branch or tag - # token (required) - The uniq token of trigger - # variables (optional) - The list of variables to be injected into build - # Example Request: - # POST /projects/:id/trigger/builds - post ":id/trigger/builds" do - required_attributes! [:ref, :token] - - project = Project.find_with_namespace(params[:id]) || Project.find_by(id: params[:id]) + desc 'Trigger a GitLab project build' do + success Entities::TriggerRequest + end + params do + requires :ref, type: String, desc: 'The commit sha or name of a branch or tag' + requires :token, type: String, desc: 'The unique token of trigger' + optional :variables, type: Hash, desc: 'The list of variables to be injected into build' + end + post ":id/(ref/:ref/)trigger/builds" do + project = find_project(params[:id]) trigger = Ci::Trigger.find_by_token(params[:token].to_s) not_found! unless project && trigger unauthorized! unless trigger.project == project # validate variables - variables = params[:variables] - if variables - unless variables.is_a?(Hash) - render_api_error!('variables needs to be a hash', 400) - end - - unless variables.all? { |key, value| key.is_a?(String) && value.is_a?(String) } - render_api_error!('variables needs to be a map of key-valued strings', 400) - end - - # convert variables from Mash to Hash - variables = variables.to_h + variables = params[:variables].to_h + unless variables.all? { |key, value| key.is_a?(String) && value.is_a?(String) } + render_api_error!('variables needs to be a map of key-valued strings', 400) end # create request and trigger builds @@ -44,31 +36,27 @@ module API end end - # Get triggers list - # - # Parameters: - # id (required) - The ID of a project - # page (optional) - The page number for pagination - # per_page (optional) - The value of items per page to show - # Example Request: - # GET /projects/:id/triggers + desc 'Get triggers list' do + success Entities::Trigger + end + params do + use :pagination + end get ':id/triggers' do authenticate! authorize! :admin_build, user_project triggers = user_project.triggers.includes(:trigger_requests) - triggers = paginate(triggers) - present triggers, with: Entities::Trigger + present paginate(triggers), with: Entities::Trigger end - # Get specific trigger of a project - # - # Parameters: - # id (required) - The ID of a project - # token (required) - The `token` of a trigger - # Example Request: - # GET /projects/:id/triggers/:token + desc 'Get specific trigger of a project' do + success Entities::Trigger + end + params do + requires :token, type: String, desc: 'The unique token of trigger' + end get ':id/triggers/:token' do authenticate! authorize! :admin_build, user_project @@ -79,12 +67,9 @@ module API present trigger, with: Entities::Trigger end - # Create trigger - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # POST /projects/:id/triggers + desc 'Create a trigger' do + success Entities::Trigger + end post ':id/triggers' do authenticate! authorize! :admin_build, user_project @@ -94,13 +79,12 @@ module API present trigger, with: Entities::Trigger end - # Delete trigger - # - # Parameters: - # id (required) - The ID of a project - # token (required) - The `token` of a trigger - # Example Request: - # DELETE /projects/:id/triggers/:token + desc 'Delete a trigger' do + success Entities::Trigger + end + params do + requires :token, type: String, desc: 'The unique token of trigger' + end delete ':id/triggers/:token' do authenticate! authorize! :admin_build, user_project diff --git a/lib/api/users.rb b/lib/api/users.rb index 8a376d3c2a3..94b2b6653d2 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -1,93 +1,121 @@ module API - # Users API class Users < Grape::API - before { authenticate! } + include PaginationParams + + before do + allow_access_with_scope :read_user if request.get? + authenticate! + end resource :users, requirements: { uid: /[0-9]*/, id: /[0-9]*/ } do - # Get a users list - # - # Example Request: - # GET /users - # GET /users?search=Admin - # GET /users?username=root + helpers do + params :optional_attributes do + optional :skype, type: String, desc: 'The Skype username' + optional :linkedin, type: String, desc: 'The LinkedIn username' + optional :twitter, type: String, desc: 'The Twitter username' + optional :website_url, type: String, desc: 'The website of the user' + optional :organization, type: String, desc: 'The organization of the user' + optional :projects_limit, type: Integer, desc: 'The number of projects a user can create' + optional :extern_uid, type: String, desc: 'The external authentication provider UID' + optional :provider, type: String, desc: 'The external provider' + optional :bio, type: String, desc: 'The biography of the user' + optional :location, type: String, desc: 'The location of the user' + optional :admin, type: Boolean, desc: 'Flag indicating the user is an administrator' + optional :can_create_group, type: Boolean, desc: 'Flag indicating the user can create groups' + optional :confirm, type: Boolean, desc: 'Flag indicating the account needs to be confirmed' + optional :external, type: Boolean, desc: 'Flag indicating the user is an external user' + all_or_none_of :extern_uid, :provider + end + end + + desc 'Get the list of users' do + success Entities::UserBasic + end + params do + optional :username, type: String, desc: 'Get a single user with a specific username' + optional :search, type: String, desc: 'Search for a username' + optional :active, type: Boolean, default: false, desc: 'Filters only active users' + optional :external, type: Boolean, default: false, desc: 'Filters only external users' + optional :blocked, type: Boolean, default: false, desc: 'Filters only blocked users' + use :pagination + end get do unless can?(current_user, :read_users_list, nil) render_api_error!("Not authorized.", 403) end if params[:username].present? - @users = User.where(username: params[:username]) + users = User.where(username: params[:username]) else - @users = User.all - @users = @users.active if params[:active].present? - @users = @users.search(params[:search]) if params[:search].present? - @users = paginate @users + users = User.all + users = users.active if params[:active] + users = users.search(params[:search]) if params[:search].present? + users = users.blocked if params[:blocked] + users = users.external if params[:external] && current_user.is_admin? end - if current_user.is_admin? - present @users, with: Entities::UserFull - else - present @users, with: Entities::UserBasic - end + entity = current_user.is_admin? ? Entities::UserPublic : Entities::UserBasic + present paginate(users), with: entity end - # Get a single user - # - # Parameters: - # id (required) - The ID of a user - # Example Request: - # GET /users/:id + desc 'Get a single user' do + success Entities::UserBasic + end + params do + requires :id, type: Integer, desc: 'The ID of the user' + end get ":id" do - @user = User.find(params[:id]) + user = User.find_by(id: params[:id]) + not_found!('User') unless user if current_user && current_user.is_admin? - present @user, with: Entities::UserFull - elsif can?(current_user, :read_user, @user) - present @user, with: Entities::User + present user, with: Entities::UserPublic + elsif can?(current_user, :read_user, user) + present user, with: Entities::User else render_api_error!("User not found.", 404) end end - # Create user. Available only for admin - # - # Parameters: - # email (required) - Email - # password (required) - Password - # name (required) - Name - # username (required) - Name - # skype - Skype ID - # linkedin - Linkedin - # twitter - Twitter account - # website_url - Website url - # projects_limit - Number of projects user can create - # extern_uid - External authentication provider UID - # provider - External provider - # bio - Bio - # location - Location of the user - # admin - User is admin - true or false (default) - # can_create_group - User can create groups - true or false - # confirm - Require user confirmation - true (default) or false - # external - Flags the user as external - true or false(default) - # Example Request: - # POST /users + desc 'Create a user. Available only for admins.' do + success Entities::UserPublic + end + params do + requires :email, type: String, desc: 'The email of the user' + optional :password, type: String, desc: 'The password of the new user' + optional :reset_password, type: Boolean, desc: 'Flag indicating the user will be sent a password reset token' + at_least_one_of :password, :reset_password + requires :name, type: String, desc: 'The name of the user' + requires :username, type: String, desc: 'The username of the user' + use :optional_attributes + end post do authenticated_as_admin! - required_attributes! [:email, :password, :name, :username] - attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :projects_limit, :username, :bio, :location, :can_create_group, :admin, :confirm, :external] - admin = attrs.delete(:admin) - confirm = !(attrs.delete(:confirm) =~ /(false|f|no|0)$/i) - user = User.build_user(attrs) - user.admin = admin unless admin.nil? + + # Filter out params which are used later + user_params = declared_params(include_missing: false) + identity_attrs = user_params.slice(:provider, :extern_uid) + confirm = user_params.delete(:confirm) + user = User.new(user_params.except(:extern_uid, :provider, :reset_password)) + + if user_params.delete(:reset_password) + user.attributes = { + force_random_password: true, + password_expires_at: nil, + created_by_id: current_user.id + } + user.generate_password + user.generate_reset_token + end + user.skip_confirmation! unless confirm - identity_attrs = attributes_for_keys [:provider, :extern_uid] if identity_attrs.any? user.identities.build(identity_attrs) end if user.save - present user, with: Entities::UserFull + present user, with: Entities::UserPublic else conflict!('Email has already been taken') if User. where(email: user.email). @@ -101,45 +129,41 @@ module API end end - # Update user. Available only for admin - # - # Parameters: - # email - Email - # name - Name - # password - Password - # skype - Skype ID - # linkedin - Linkedin - # twitter - Twitter account - # website_url - Website url - # projects_limit - Limit projects each user can create - # bio - Bio - # location - Location of the user - # admin - User is admin - true or false (default) - # can_create_group - User can create groups - true or false - # external - Flags the user as external - true or false(default) - # Example Request: - # PUT /users/:id + desc 'Update a user. Available only for admins.' do + success Entities::UserPublic + end + params do + requires :id, type: Integer, desc: 'The ID of the user' + optional :email, type: String, desc: 'The email of the user' + optional :password, type: String, desc: 'The password of the new user' + optional :name, type: String, desc: 'The name of the user' + optional :username, type: String, desc: 'The username of the user' + use :optional_attributes + at_least_one_of :email, :password, :name, :username, :skype, :linkedin, + :twitter, :website_url, :organization, :projects_limit, + :extern_uid, :provider, :bio, :location, :admin, + :can_create_group, :confirm, :external + end put ":id" do authenticated_as_admin! - attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :website_url, :projects_limit, :username, :bio, :location, :can_create_group, :admin, :external] - user = User.find(params[:id]) + user = User.find_by(id: params.delete(:id)) not_found!('User') unless user - admin = attrs.delete(:admin) - user.admin = admin unless admin.nil? - - conflict!('Email has already been taken') if attrs[:email] && - User.where(email: attrs[:email]). + conflict!('Email has already been taken') if params[:email] && + User.where(email: params[:email]). where.not(id: user.id).count > 0 - conflict!('Username has already been taken') if attrs[:username] && - User.where(username: attrs[:username]). + conflict!('Username has already been taken') if params[:username] && + User.where(username: params[:username]). where.not(id: user.id).count > 0 - identity_attrs = attributes_for_keys [:provider, :extern_uid] + user_params = declared_params(include_missing: false) + identity_attrs = user_params.slice(:provider, :extern_uid) + if identity_attrs.any? identity = user.identities.find_by(provider: identity_attrs[:provider]) + if identity identity.update_attributes(identity_attrs) else @@ -148,28 +172,31 @@ module API end end - if user.update_attributes(attrs) - present user, with: Entities::UserFull + user_params[:password_expires_at] = Time.now if user_params[:password].present? + + if user.update_attributes(user_params.except(:extern_uid, :provider)) + present user, with: Entities::UserPublic else render_validation_error!(user) end end - # Add ssh key to a specified user. Only available to admin users. - # - # Parameters: - # id (required) - The ID of a user - # key (required) - New SSH Key - # title (required) - New SSH Key's title - # Example Request: - # POST /users/:id/keys + desc 'Add an SSH key to a specified user. Available only for admins.' do + success Entities::SSHKey + end + params do + requires :id, type: Integer, desc: 'The ID of the user' + requires :key, type: String, desc: 'The new SSH key' + requires :title, type: String, desc: 'The title of the new SSH key' + end post ":id/keys" do authenticated_as_admin! - required_attributes! [:title, :key] - user = User.find(params[:id]) - attrs = attributes_for_keys [:title, :key] - key = user.keys.new attrs + user = User.find_by(id: params.delete(:id)) + not_found!('User') unless user + + key = user.keys.new(declared_params(include_missing: false)) + if key.save present key, with: Entities::SSHKey else @@ -177,55 +204,56 @@ module API end end - # Get ssh keys of a specified user. Only available to admin users. - # - # Parameters: - # uid (required) - The ID of a user - # Example Request: - # GET /users/:uid/keys - get ':uid/keys' do + desc 'Get the SSH keys of a specified user. Available only for admins.' do + success Entities::SSHKey + end + params do + requires :id, type: Integer, desc: 'The ID of the user' + use :pagination + end + get ':id/keys' do authenticated_as_admin! - user = User.find_by(id: params[:uid]) + + user = User.find_by(id: params[:id]) not_found!('User') unless user - present user.keys, with: Entities::SSHKey + present paginate(user.keys), with: Entities::SSHKey end - # Delete existing ssh key of a specified user. Only available to admin - # users. - # - # Parameters: - # uid (required) - The ID of a user - # id (required) - SSH Key ID - # Example Request: - # DELETE /users/:uid/keys/:id - delete ':uid/keys/:id' do + desc 'Delete an existing SSH key from a specified user. Available only for admins.' do + success Entities::SSHKey + end + params do + requires :id, type: Integer, desc: 'The ID of the user' + requires :key_id, type: Integer, desc: 'The ID of the SSH key' + end + delete ':id/keys/:key_id' do authenticated_as_admin! - user = User.find_by(id: params[:uid]) + + user = User.find_by(id: params[:id]) not_found!('User') unless user - begin - key = user.keys.find params[:id] - key.destroy - rescue ActiveRecord::RecordNotFound - not_found!('Key') - end + key = user.keys.find_by(id: params[:key_id]) + not_found!('Key') unless key + + present key.destroy, with: Entities::SSHKey end - # Add email to a specified user. Only available to admin users. - # - # Parameters: - # id (required) - The ID of a user - # email (required) - Email address - # Example Request: - # POST /users/:id/emails + desc 'Add an email address to a specified user. Available only for admins.' do + success Entities::Email + end + params do + requires :id, type: Integer, desc: 'The ID of the user' + requires :email, type: String, desc: 'The email of the user' + end post ":id/emails" do authenticated_as_admin! - required_attributes! [:email] - user = User.find(params[:id]) - attrs = attributes_for_keys [:email] - email = user.emails.new attrs + user = User.find_by(id: params.delete(:id)) + not_found!('User') unless user + + email = user.emails.new(declared_params(include_missing: false)) + if email.save NotificationService.new.new_email(email) present email, with: Entities::Email @@ -234,131 +262,149 @@ module API end end - # Get emails of a specified user. Only available to admin users. - # - # Parameters: - # uid (required) - The ID of a user - # Example Request: - # GET /users/:uid/emails - get ':uid/emails' do + desc 'Get the emails addresses of a specified user. Available only for admins.' do + success Entities::Email + end + params do + requires :id, type: Integer, desc: 'The ID of the user' + use :pagination + end + get ':id/emails' do authenticated_as_admin! - user = User.find_by(id: params[:uid]) + user = User.find_by(id: params[:id]) not_found!('User') unless user - present user.emails, with: Entities::Email + present paginate(user.emails), with: Entities::Email end - # Delete existing email of a specified user. Only available to admin - # users. - # - # Parameters: - # uid (required) - The ID of a user - # id (required) - Email ID - # Example Request: - # DELETE /users/:uid/emails/:id - delete ':uid/emails/:id' do + desc 'Delete an email address of a specified user. Available only for admins.' do + success Entities::Email + end + params do + requires :id, type: Integer, desc: 'The ID of the user' + requires :email_id, type: Integer, desc: 'The ID of the email' + end + delete ':id/emails/:email_id' do authenticated_as_admin! - user = User.find_by(id: params[:uid]) + user = User.find_by(id: params[:id]) not_found!('User') unless user - begin - email = user.emails.find params[:id] - email.destroy + email = user.emails.find_by(id: params[:email_id]) + not_found!('Email') unless email - user.update_secondary_emails! - rescue ActiveRecord::RecordNotFound - not_found!('Email') - end + email.destroy + user.update_secondary_emails! end - # Delete user. Available only for admin - # - # Example Request: - # DELETE /users/:id + desc 'Delete a user. Available only for admins.' do + success Entities::Email + end + params do + requires :id, type: Integer, desc: 'The ID of the user' + end delete ":id" do authenticated_as_admin! user = User.find_by(id: params[:id]) + not_found!('User') unless user - if user - DeleteUserService.new(current_user).execute(user) - else - not_found!('User') - end + ::Users::DestroyService.new(current_user).execute(user) end - # Block user. Available only for admin - # - # Example Request: - # PUT /users/:id/block - put ':id/block' do + desc 'Block a user. Available only for admins.' + params do + requires :id, type: Integer, desc: 'The ID of the user' + end + post ':id/block' do authenticated_as_admin! user = User.find_by(id: params[:id]) + not_found!('User') unless user - if !user - not_found!('User') - elsif !user.ldap_blocked? + if !user.ldap_blocked? user.block else forbidden!('LDAP blocked users cannot be modified by the API') end end - # Unblock user. Available only for admin - # - # Example Request: - # PUT /users/:id/unblock - put ':id/unblock' do + desc 'Unblock a user. Available only for admins.' + params do + requires :id, type: Integer, desc: 'The ID of the user' + end + post ':id/unblock' do authenticated_as_admin! user = User.find_by(id: params[:id]) + not_found!('User') unless user - if !user - not_found!('User') - elsif user.ldap_blocked? + if user.ldap_blocked? forbidden!('LDAP blocked users cannot be unblocked by the API') else user.activate end end + + desc 'Get the contribution events of a specified user' do + detail 'This feature was introduced in GitLab 8.13.' + success Entities::Event + end + params do + requires :id, type: Integer, desc: 'The ID of the user' + use :pagination + end + get ':id/events' do + user = User.find_by(id: params[:id]) + not_found!('User') unless user + + events = user.events. + merge(ProjectsFinder.new.execute(current_user)). + references(:project). + with_associations. + recent + + present paginate(events), with: Entities::Event + end end resource :user do - # Get currently authenticated user - # - # Example Request: - # GET /user + desc 'Get the currently authenticated user' do + success Entities::UserPublic + end get do - present @current_user, with: Entities::UserLogin + present current_user, with: sudo? ? Entities::UserWithPrivateToken : Entities::UserPublic end - # Get currently authenticated user's keys - # - # Example Request: - # GET /user/keys + desc "Get the currently authenticated user's SSH keys" do + success Entities::SSHKey + end + params do + use :pagination + end get "keys" do - present current_user.keys, with: Entities::SSHKey + present paginate(current_user.keys), with: Entities::SSHKey end - # Get single key owned by currently authenticated user - # - # Example Request: - # GET /user/keys/:id - get "keys/:id" do - key = current_user.keys.find params[:id] + desc 'Get a single key owned by currently authenticated user' do + success Entities::SSHKey + end + params do + requires :key_id, type: Integer, desc: 'The ID of the SSH key' + end + get "keys/:key_id" do + key = current_user.keys.find_by(id: params[:key_id]) + not_found!('Key') unless key + present key, with: Entities::SSHKey end - # Add new ssh key to currently authenticated user - # - # Parameters: - # key (required) - New SSH Key - # title (required) - New SSH Key's title - # Example Request: - # POST /user/keys + desc 'Add a new SSH key to the currently authenticated user' do + success Entities::SSHKey + end + params do + requires :key, type: String, desc: 'The new SSH key' + requires :title, type: String, desc: 'The title of the new SSH key' + end post "keys" do - required_attributes! [:title, :key] + key = current_user.keys.new(declared_params) - attrs = attributes_for_keys [:title, :key] - key = current_user.keys.new attrs if key.save present key, with: Entities::SSHKey else @@ -366,48 +412,51 @@ module API end end - # Delete existing ssh key of currently authenticated user - # - # Parameters: - # id (required) - SSH Key ID - # Example Request: - # DELETE /user/keys/:id - delete "keys/:id" do - begin - key = current_user.keys.find params[:id] - key.destroy - rescue - end + desc 'Delete an SSH key from the currently authenticated user' do + success Entities::SSHKey + end + params do + requires :key_id, type: Integer, desc: 'The ID of the SSH key' end + delete "keys/:key_id" do + key = current_user.keys.find_by(id: params[:key_id]) + not_found!('Key') unless key - # Get currently authenticated user's emails - # - # Example Request: - # GET /user/emails + present key.destroy, with: Entities::SSHKey + end + + desc "Get the currently authenticated user's email addresses" do + success Entities::Email + end + params do + use :pagination + end get "emails" do - present current_user.emails, with: Entities::Email + present paginate(current_user.emails), with: Entities::Email end - # Get single email owned by currently authenticated user - # - # Example Request: - # GET /user/emails/:id - get "emails/:id" do - email = current_user.emails.find params[:id] + desc 'Get a single email address owned by the currently authenticated user' do + success Entities::Email + end + params do + requires :email_id, type: Integer, desc: 'The ID of the email' + end + get "emails/:email_id" do + email = current_user.emails.find_by(id: params[:email_id]) + not_found!('Email') unless email + present email, with: Entities::Email end - # Add new email to currently authenticated user - # - # Parameters: - # email (required) - Email address - # Example Request: - # POST /user/emails + desc 'Add new email address to the currently authenticated user' do + success Entities::Email + end + params do + requires :email, type: String, desc: 'The new email' + end post "emails" do - required_attributes! [:email] + email = current_user.emails.new(declared_params) - attrs = attributes_for_keys [:email] - email = current_user.emails.new attrs if email.save NotificationService.new.new_email(email) present email, with: Entities::Email @@ -416,20 +465,16 @@ module API end end - # Delete existing email of currently authenticated user - # - # Parameters: - # id (required) - EMail ID - # Example Request: - # DELETE /user/emails/:id - delete "emails/:id" do - begin - email = current_user.emails.find params[:id] - email.destroy + desc 'Delete an email address from the currently authenticated user' + params do + requires :email_id, type: Integer, desc: 'The ID of the email' + end + delete "emails/:email_id" do + email = current_user.emails.find_by(id: params[:email_id]) + not_found!('Email') unless email - current_user.update_secondary_emails! - rescue - end + email.destroy + current_user.update_secondary_emails! end end end diff --git a/lib/api/v3/boards.rb b/lib/api/v3/boards.rb new file mode 100644 index 00000000000..31d708bc2c8 --- /dev/null +++ b/lib/api/v3/boards.rb @@ -0,0 +1,51 @@ +module API + module V3 + class Boards < Grape::API + before { authenticate! } + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects do + desc 'Get all project boards' do + detail 'This feature was introduced in 8.13' + success ::API::Entities::Board + end + get ':id/boards' do + authorize!(:read_board, user_project) + present user_project.boards, with: ::API::Entities::Board + end + + params do + requires :board_id, type: Integer, desc: 'The ID of a board' + end + segment ':id/boards/:board_id' do + helpers do + def project_board + board = user_project.boards.first + + if params[:board_id] == board.id + board + else + not_found!('Board') + end + end + + def board_lists + project_board.lists.destroyable + end + end + + desc 'Get the lists of a project board' do + detail 'Does not include `done` list. This feature was introduced in 8.13' + success ::API::Entities::List + end + get '/lists' do + authorize!(:read_board, user_project) + present board_lists, with: ::API::Entities::List + end + end + end + end + end +end diff --git a/lib/api/v3/branches.rb b/lib/api/v3/branches.rb new file mode 100644 index 00000000000..51eb566cf7d --- /dev/null +++ b/lib/api/v3/branches.rb @@ -0,0 +1,31 @@ +require 'mime/types' + +module API + module V3 + class Branches < Grape::API + before { authenticate! } + before { authorize! :download_code, user_project } + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects do + desc 'Get a project repository branches' do + success ::API::Entities::RepoBranch + end + get ":id/repository/branches" do + branches = user_project.repository.branches.sort_by(&:name) + + present branches, with: ::API::Entities::RepoBranch, project: user_project + end + + desc 'Delete all merged branches' + delete ":id/repository/merged_branches" do + DeleteMergedBranchesService.new(user_project, current_user).async_execute + + status(200) + end + end + end + end +end diff --git a/lib/api/v3/commits.rb b/lib/api/v3/commits.rb new file mode 100644 index 00000000000..506204b3517 --- /dev/null +++ b/lib/api/v3/commits.rb @@ -0,0 +1,198 @@ +require 'mime/types' + +module API + module V3 + class Commits < Grape::API + include PaginationParams + + before { authenticate! } + before { authorize! :download_code, user_project } + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects do + desc 'Get a project repository commits' do + success ::API::Entities::RepoCommit + end + params do + optional :ref_name, type: String, desc: 'The name of a repository branch or tag, if not given the default branch is used' + optional :since, type: DateTime, desc: 'Only commits after or in this date will be returned' + optional :until, type: DateTime, desc: 'Only commits before or in this date will be returned' + optional :page, type: Integer, default: 0, desc: 'The page for pagination' + optional :per_page, type: Integer, default: 20, desc: 'The number of results per page' + optional :path, type: String, desc: 'The file path' + end + get ":id/repository/commits" do + ref = params[:ref_name] || user_project.try(:default_branch) || 'master' + offset = params[:page] * params[:per_page] + + commits = user_project.repository.commits(ref, + path: params[:path], + limit: params[:per_page], + offset: offset, + after: params[:since], + before: params[:until]) + + present commits, with: ::API::Entities::RepoCommit + end + + desc 'Commit multiple file changes as one commit' do + success ::API::Entities::RepoCommitDetail + detail 'This feature was introduced in GitLab 8.13' + end + params do + requires :branch_name, type: String, desc: 'The name of branch' + requires :commit_message, type: String, desc: 'Commit message' + requires :actions, type: Array[Hash], desc: 'Actions to perform in commit' + optional :author_email, type: String, desc: 'Author email for commit' + optional :author_name, type: String, desc: 'Author name for commit' + end + post ":id/repository/commits" do + authorize! :push_code, user_project + + attrs = declared_params.dup + branch = attrs.delete(:branch_name) + attrs.merge!(branch: branch, start_branch: branch, target_branch: branch) + + result = ::Files::MultiService.new(user_project, current_user, attrs).execute + + if result[:status] == :success + commit_detail = user_project.repository.commits(result[:result], limit: 1).first + present commit_detail, with: ::API::Entities::RepoCommitDetail + else + render_api_error!(result[:message], 400) + end + end + + desc 'Get a specific commit of a project' do + success ::API::Entities::RepoCommitDetail + failure [[404, 'Not Found']] + end + params do + requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' + end + get ":id/repository/commits/:sha" do + commit = user_project.commit(params[:sha]) + + not_found! "Commit" unless commit + + present commit, with: ::API::Entities::RepoCommitDetail + end + + desc 'Get the diff for a specific commit of a project' do + failure [[404, 'Not Found']] + end + params do + requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' + end + get ":id/repository/commits/:sha/diff" do + commit = user_project.commit(params[:sha]) + + not_found! "Commit" unless commit + + commit.raw_diffs.to_a + end + + desc "Get a commit's comments" do + success ::API::Entities::CommitNote + failure [[404, 'Not Found']] + end + params do + use :pagination + requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' + end + get ':id/repository/commits/:sha/comments' do + commit = user_project.commit(params[:sha]) + + not_found! 'Commit' unless commit + notes = Note.where(commit_id: commit.id).order(:created_at) + + present paginate(notes), with: ::API::Entities::CommitNote + end + + desc 'Cherry pick commit into a branch' do + detail 'This feature was introduced in GitLab 8.15' + success ::API::Entities::RepoCommit + end + params do + requires :sha, type: String, desc: 'A commit sha to be cherry picked' + requires :branch, type: String, desc: 'The name of the branch' + end + post ':id/repository/commits/:sha/cherry_pick' do + authorize! :push_code, user_project + + commit = user_project.commit(params[:sha]) + not_found!('Commit') unless commit + + branch = user_project.repository.find_branch(params[:branch]) + not_found!('Branch') unless branch + + commit_params = { + commit: commit, + create_merge_request: false, + source_project: user_project, + source_branch: commit.cherry_pick_branch_name, + target_branch: params[:branch] + } + + result = ::Commits::CherryPickService.new(user_project, current_user, commit_params).execute + + if result[:status] == :success + branch = user_project.repository.find_branch(params[:branch]) + present user_project.repository.commit(branch.dereferenced_target), with: ::API::Entities::RepoCommit + else + render_api_error!(result[:message], 400) + end + end + + desc 'Post comment to commit' do + success ::API::Entities::CommitNote + end + params do + requires :sha, type: String, regexp: /\A\h{6,40}\z/, desc: "The commit's SHA" + requires :note, type: String, desc: 'The text of the comment' + optional :path, type: String, desc: 'The file path' + given :path do + requires :line, type: Integer, desc: 'The line number' + requires :line_type, type: String, values: %w(new old), default: 'new', desc: 'The type of the line' + end + end + post ':id/repository/commits/:sha/comments' do + commit = user_project.commit(params[:sha]) + not_found! 'Commit' unless commit + + opts = { + note: params[:note], + noteable_type: 'Commit', + commit_id: commit.id + } + + if params[:path] + commit.raw_diffs(all_diffs: true).each do |diff| + next unless diff.new_path == params[:path] + lines = Gitlab::Diff::Parser.new.parse(diff.diff.each_line) + + lines.each do |line| + next unless line.new_pos == params[:line] && line.type == params[:line_type] + break opts[:line_code] = Gitlab::Diff::LineCode.generate(diff.new_path, line.new_pos, line.old_pos) + end + + break if opts[:line_code] + end + + opts[:type] = LegacyDiffNote.name if opts[:line_code] + end + + note = ::Notes::CreateService.new(user_project, current_user, opts).execute + + if note.save + present note, with: ::API::Entities::CommitNote + else + render_api_error!("Failed to save note #{note.errors.messages}", 400) + end + end + end + end + end +end diff --git a/lib/api/v3/deploy_keys.rb b/lib/api/v3/deploy_keys.rb new file mode 100644 index 00000000000..5bbb167755c --- /dev/null +++ b/lib/api/v3/deploy_keys.rb @@ -0,0 +1,122 @@ +module API + module V3 + class DeployKeys < Grape::API + before { authenticate! } + + get "deploy_keys" do + authenticated_as_admin! + + keys = DeployKey.all + present keys, with: ::API::Entities::SSHKey + end + + params do + requires :id, type: String, desc: 'The ID of the project' + end + resource :projects do + before { authorize_admin_project } + + %w(keys deploy_keys).each do |path| + desc "Get a specific project's deploy keys" do + success ::API::Entities::SSHKey + end + get ":id/#{path}" do + present user_project.deploy_keys, with: ::API::Entities::SSHKey + end + + desc 'Get single deploy key' do + success ::API::Entities::SSHKey + end + params do + requires :key_id, type: Integer, desc: 'The ID of the deploy key' + end + get ":id/#{path}/:key_id" do + key = user_project.deploy_keys.find params[:key_id] + present key, with: ::API::Entities::SSHKey + end + + desc 'Add new deploy key to currently authenticated user' do + success ::API::Entities::SSHKey + end + params do + requires :key, type: String, desc: 'The new deploy key' + requires :title, type: String, desc: 'The name of the deploy key' + end + post ":id/#{path}" do + params[:key].strip! + + # Check for an existing key joined to this project + key = user_project.deploy_keys.find_by(key: params[:key]) + if key + present key, with: ::API::Entities::SSHKey + break + end + + # Check for available deploy keys in other projects + key = current_user.accessible_deploy_keys.find_by(key: params[:key]) + if key + user_project.deploy_keys << key + present key, with: ::API::Entities::SSHKey + break + end + + # Create a new deploy key + key = DeployKey.new(declared_params(include_missing: false)) + if key.valid? && user_project.deploy_keys << key + present key, with: ::API::Entities::SSHKey + else + render_validation_error!(key) + end + end + + desc 'Enable a deploy key for a project' do + detail 'This feature was added in GitLab 8.11' + success ::API::Entities::SSHKey + end + params do + requires :key_id, type: Integer, desc: 'The ID of the deploy key' + end + post ":id/#{path}/:key_id/enable" do + key = ::Projects::EnableDeployKeyService.new(user_project, + current_user, declared_params).execute + + if key + present key, with: ::API::Entities::SSHKey + else + not_found!('Deploy Key') + end + end + + desc 'Disable a deploy key for a project' do + detail 'This feature was added in GitLab 8.11' + success ::API::Entities::SSHKey + end + params do + requires :key_id, type: Integer, desc: 'The ID of the deploy key' + end + delete ":id/#{path}/:key_id/disable" do + key = user_project.deploy_keys_projects.find_by(deploy_key_id: params[:key_id]) + key.destroy + + present key.deploy_key, with: ::API::Entities::SSHKey + end + + desc 'Delete deploy key for a project' do + success Key + end + params do + requires :key_id, type: Integer, desc: 'The ID of the deploy key' + end + delete ":id/#{path}/:key_id" do + key = user_project.deploy_keys_projects.find_by(deploy_key_id: params[:key_id]) + if key + key.destroy + else + not_found!('Deploy Key') + end + end + end + end + end + end +end diff --git a/lib/api/v3/entities.rb b/lib/api/v3/entities.rb new file mode 100644 index 00000000000..11d0e6dbf71 --- /dev/null +++ b/lib/api/v3/entities.rb @@ -0,0 +1,50 @@ +module API + module V3 + module Entities + class ProjectSnippet < Grape::Entity + expose :id, :title, :file_name + expose :author, using: ::API::Entities::UserBasic + expose :updated_at, :created_at + expose(:expires_at) { |snippet| nil } + + expose :web_url do |snippet, options| + Gitlab::UrlBuilder.build(snippet) + end + end + + class Note < Grape::Entity + expose :id + expose :note, as: :body + expose :attachment_identifier, as: :attachment + expose :author, using: ::API::Entities::UserBasic + expose :created_at, :updated_at + expose :system?, as: :system + expose :noteable_id, :noteable_type + # upvote? and downvote? are deprecated, always return false + expose(:upvote?) { |note| false } + expose(:downvote?) { |note| false } + end + + class Event < Grape::Entity + expose :title, :project_id, :action_name + expose :target_id, :target_type, :author_id + expose :data, :target_title + expose :created_at + expose :note, using: Entities::Note, if: ->(event, options) { event.note? } + expose :author, using: ::API::Entities::UserBasic, if: ->(event, options) { event.author } + + expose :author_username do |event, options| + event.author&.username + end + end + + class AwardEmoji < Grape::Entity + expose :id + expose :name + expose :user, using: ::API::Entities::UserBasic + expose :created_at, :updated_at + expose :awardable_id, :awardable_type + end + end + end +end diff --git a/lib/api/v3/files.rb b/lib/api/v3/files.rb new file mode 100644 index 00000000000..4f8d58d37c8 --- /dev/null +++ b/lib/api/v3/files.rb @@ -0,0 +1,138 @@ +module API + module V3 + class Files < Grape::API + helpers do + def commit_params(attrs) + { + file_path: attrs[:file_path], + start_branch: attrs[:branch], + target_branch: attrs[:branch], + commit_message: attrs[:commit_message], + file_content: attrs[:content], + file_content_encoding: attrs[:encoding], + author_email: attrs[:author_email], + author_name: attrs[:author_name] + } + end + + def commit_response(attrs) + { + file_path: attrs[:file_path], + branch: attrs[:branch] + } + end + + params :simple_file_params do + requires :file_path, type: String, desc: 'The path to new file. Ex. lib/class.rb' + requires :branch_name, type: String, desc: 'The name of branch' + requires :commit_message, type: String, desc: 'Commit Message' + optional :author_email, type: String, desc: 'The email of the author' + optional :author_name, type: String, desc: 'The name of the author' + end + + params :extended_file_params do + use :simple_file_params + requires :content, type: String, desc: 'File content' + optional :encoding, type: String, values: %w[base64], desc: 'File encoding' + end + end + + params do + requires :id, type: String, desc: 'The project ID' + end + resource :projects do + desc 'Get a file from repository' + params do + requires :file_path, type: String, desc: 'The path to the file. Ex. lib/class.rb' + requires :ref, type: String, desc: 'The name of branch, tag, or commit' + end + get ":id/repository/files" do + authorize! :download_code, user_project + + commit = user_project.commit(params[:ref]) + not_found!('Commit') unless commit + + repo = user_project.repository + blob = repo.blob_at(commit.sha, params[:file_path]) + not_found!('File') unless blob + + blob.load_all_data!(repo) + status(200) + + { + file_name: blob.name, + file_path: blob.path, + size: blob.size, + encoding: "base64", + content: Base64.strict_encode64(blob.data), + ref: params[:ref], + blob_id: blob.id, + commit_id: commit.id, + last_commit_id: repo.last_commit_id_for_path(commit.sha, params[:file_path]) + } + end + + desc 'Create new file in repository' + params do + use :extended_file_params + end + post ":id/repository/files" do + authorize! :push_code, user_project + + file_params = declared_params(include_missing: false) + file_params[:branch] = file_params.delete(:branch_name) + + result = ::Files::CreateService.new(user_project, current_user, commit_params(file_params)).execute + + if result[:status] == :success + status(201) + commit_response(file_params) + else + render_api_error!(result[:message], 400) + end + end + + desc 'Update existing file in repository' + params do + use :extended_file_params + end + put ":id/repository/files" do + authorize! :push_code, user_project + + file_params = declared_params(include_missing: false) + file_params[:branch] = file_params.delete(:branch_name) + + result = ::Files::UpdateService.new(user_project, current_user, commit_params(file_params)).execute + + if result[:status] == :success + status(200) + commit_response(file_params) + else + http_status = result[:http_status] || 400 + render_api_error!(result[:message], http_status) + end + end + + desc 'Delete an existing file in repository' + params do + use :simple_file_params + end + delete ":id/repository/files" do + authorize! :push_code, user_project + + file_params = declared_params(include_missing: false) + file_params[:branch] = file_params.delete(:branch_name) + + result = ::Files::DestroyService.new(user_project, current_user, commit_params(file_params)).execute + + if result[:status] == :success + status(200) + commit_response(file_params) + else + render_api_error!(result[:message], 400) + end + end + end + end + end +end diff --git a/lib/api/v3/groups.rb b/lib/api/v3/groups.rb new file mode 100644 index 00000000000..c826bc4fe0b --- /dev/null +++ b/lib/api/v3/groups.rb @@ -0,0 +1,38 @@ +module API + module V3 + class Groups < Grape::API + include PaginationParams + + before { authenticate! } + + helpers do + params :statistics_params do + optional :statistics, type: Boolean, default: false, desc: 'Include project statistics' + end + + def present_groups(groups, options = {}) + options = options.reverse_merge( + with: ::API::Entities::Group, + current_user: current_user, + ) + + groups = groups.with_statistics if options[:statistics] + present paginate(groups), options + end + end + + resource :groups do + desc 'Get list of owned groups for authenticated user' do + success ::API::Entities::Group + end + params do + use :pagination + use :statistics_params + end + get '/owned' do + present_groups current_user.owned_groups, statistics: params[:statistics] + end + end + end + end +end diff --git a/lib/api/v3/issues.rb b/lib/api/v3/issues.rb new file mode 100644 index 00000000000..d0af09f0e1e --- /dev/null +++ b/lib/api/v3/issues.rb @@ -0,0 +1,234 @@ +module API + module V3 + class Issues < Grape::API + include PaginationParams + + before { authenticate! } + + helpers do + def find_issues(args = {}) + args = params.merge(args) + + args.delete(:id) + args[:milestone_title] = args.delete(:milestone) + + match_all_labels = args.delete(:match_all_labels) + labels = args.delete(:labels) + args[:label_name] = labels if match_all_labels + + # IssuesFinder expects iids + args[:iids] = args.delete(:iid) if args.key?(:iid) + + issues = IssuesFinder.new(current_user, args).execute.inc_notes_with_associations + + if !match_all_labels && labels.present? + issues = issues.includes(:labels).where('labels.title' => labels.split(',')) + end + + issues.reorder(args[:order_by] => args[:sort]) + end + + params :issues_params do + optional :labels, type: String, desc: 'Comma-separated list of label names' + optional :milestone, type: String, desc: 'Milestone title' + optional :order_by, type: String, values: %w[created_at updated_at], default: 'created_at', + desc: 'Return issues ordered by `created_at` or `updated_at` fields.' + optional :sort, type: String, values: %w[asc desc], default: 'desc', + desc: 'Return issues sorted in `asc` or `desc` order.' + optional :milestone, type: String, desc: 'Return issues for a specific milestone' + use :pagination + end + + params :issue_params do + optional :description, type: String, desc: 'The description of an issue' + optional :assignee_id, type: Integer, desc: 'The ID of a user to assign issue' + optional :milestone_id, type: Integer, desc: 'The ID of a milestone to assign issue' + optional :labels, type: String, desc: 'Comma-separated list of label names' + optional :due_date, type: String, desc: 'Date time string in the format YEAR-MONTH-DAY' + optional :confidential, type: Boolean, desc: 'Boolean parameter if the issue should be confidential' + end + end + + resource :issues do + desc "Get currently authenticated user's issues" do + success ::API::Entities::Issue + end + params do + optional :state, type: String, values: %w[opened closed all], default: 'all', + desc: 'Return opened, closed, or all issues' + use :issues_params + end + get do + issues = find_issues(scope: 'authored') + + present paginate(issues), with: ::API::Entities::Issue, current_user: current_user + end + end + + params do + requires :id, type: String, desc: 'The ID of a group' + end + resource :groups do + desc 'Get a list of group issues' do + success ::API::Entities::Issue + end + params do + optional :state, type: String, values: %w[opened closed all], default: 'opened', + desc: 'Return opened, closed, or all issues' + use :issues_params + end + get ":id/issues" do + group = find_group!(params[:id]) + + issues = find_issues(group_id: group.id, state: params[:state] || 'opened', match_all_labels: true) + + present paginate(issues), with: ::API::Entities::Issue, current_user: current_user + end + end + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects do + include TimeTrackingEndpoints + + desc 'Get a list of project issues' do + detail 'iid filter is deprecated have been removed on V4' + success ::API::Entities::Issue + end + params do + optional :state, type: String, values: %w[opened closed all], default: 'all', + desc: 'Return opened, closed, or all issues' + optional :iid, type: Integer, desc: 'Return the issue having the given `iid`' + use :issues_params + end + get ":id/issues" do + project = find_project(params[:id]) + + issues = find_issues(project_id: project.id) + + present paginate(issues), with: ::API::Entities::Issue, current_user: current_user, project: user_project + end + + desc 'Get a single project issue' do + success ::API::Entities::Issue + end + params do + requires :issue_id, type: Integer, desc: 'The ID of a project issue' + end + get ":id/issues/:issue_id" do + issue = find_project_issue(params[:issue_id]) + present issue, with: ::API::Entities::Issue, current_user: current_user, project: user_project + end + + desc 'Create a new project issue' do + success ::API::Entities::Issue + end + params do + requires :title, type: String, desc: 'The title of an issue' + optional :created_at, type: DateTime, + desc: 'Date time when the issue was created. Available only for admins and project owners.' + optional :merge_request_for_resolving_discussions, type: Integer, + desc: 'The IID of a merge request for which to resolve discussions' + use :issue_params + end + post ':id/issues' do + # Setting created_at time only allowed for admins and project owners + unless current_user.admin? || user_project.owner == current_user + params.delete(:created_at) + end + + issue_params = declared_params(include_missing: false) + + if merge_request_iid = params[:merge_request_for_resolving_discussions] + issue_params[:merge_request_for_resolving_discussions] = MergeRequestsFinder.new(current_user, project_id: user_project.id). + execute. + find_by(iid: merge_request_iid) + end + + issue = ::Issues::CreateService.new(user_project, + current_user, + issue_params.merge(request: request, api: true)).execute + render_spam_error! if issue.spam? + + if issue.valid? + present issue, with: ::API::Entities::Issue, current_user: current_user, project: user_project + else + render_validation_error!(issue) + end + end + + desc 'Update an existing issue' do + success ::API::Entities::Issue + end + params do + requires :issue_id, type: Integer, desc: 'The ID of a project issue' + optional :title, type: String, desc: 'The title of an issue' + optional :updated_at, type: DateTime, + desc: 'Date time when the issue was updated. Available only for admins and project owners.' + optional :state_event, type: String, values: %w[reopen close], desc: 'State of the issue' + use :issue_params + at_least_one_of :title, :description, :assignee_id, :milestone_id, + :labels, :created_at, :due_date, :confidential, :state_event + end + put ':id/issues/:issue_id' do + issue = user_project.issues.find(params.delete(:issue_id)) + authorize! :update_issue, issue + + # Setting created_at time only allowed for admins and project owners + unless current_user.admin? || user_project.owner == current_user + params.delete(:updated_at) + end + + update_params = declared_params(include_missing: false).merge(request: request, api: true) + + issue = ::Issues::UpdateService.new(user_project, + current_user, + update_params).execute(issue) + + render_spam_error! if issue.spam? + + if issue.valid? + present issue, with: ::API::Entities::Issue, current_user: current_user, project: user_project + else + render_validation_error!(issue) + end + end + + desc 'Move an existing issue' do + success ::API::Entities::Issue + end + params do + requires :issue_id, type: Integer, desc: 'The ID of a project issue' + requires :to_project_id, type: Integer, desc: 'The ID of the new project' + end + post ':id/issues/:issue_id/move' do + issue = user_project.issues.find_by(id: params[:issue_id]) + not_found!('Issue') unless issue + + new_project = Project.find_by(id: params[:to_project_id]) + not_found!('Project') unless new_project + + begin + issue = ::Issues::MoveService.new(user_project, current_user).execute(issue, new_project) + present issue, with: ::API::Entities::Issue, current_user: current_user, project: user_project + rescue ::Issues::MoveService::MoveError => error + render_api_error!(error.message, 400) + end + end + + desc 'Delete a project issue' + params do + requires :issue_id, type: Integer, desc: 'The ID of a project issue' + end + delete ":id/issues/:issue_id" do + issue = user_project.issues.find_by(id: params[:issue_id]) + not_found!('Issue') unless issue + + authorize!(:destroy_issue, issue) + issue.destroy + end + end + end + end +end diff --git a/lib/api/v3/labels.rb b/lib/api/v3/labels.rb new file mode 100644 index 00000000000..5c3261311bf --- /dev/null +++ b/lib/api/v3/labels.rb @@ -0,0 +1,19 @@ +module API + module V3 + class Labels < Grape::API + before { authenticate! } + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects do + desc 'Get all labels of the project' do + success ::API::Entities::Label + end + get ':id/labels' do + present available_labels, with: ::API::Entities::Label, current_user: current_user, project: user_project + end + end + end + end +end diff --git a/lib/api/v3/members.rb b/lib/api/v3/members.rb new file mode 100644 index 00000000000..19f276d5484 --- /dev/null +++ b/lib/api/v3/members.rb @@ -0,0 +1,133 @@ +module API + module V3 + class Members < Grape::API + include PaginationParams + + before { authenticate! } + + helpers ::API::Helpers::MembersHelpers + + %w[group project].each do |source_type| + params do + requires :id, type: String, desc: "The #{source_type} ID" + end + resource source_type.pluralize do + desc 'Gets a list of group or project members viewable by the authenticated user.' do + success ::API::Entities::Member + end + params do + optional :query, type: String, desc: 'A query string to search for members' + use :pagination + end + get ":id/members" do + source = find_source(source_type, params[:id]) + + users = source.users + users = users.merge(User.search(params[:query])) if params[:query] + + present paginate(users), with: ::API::Entities::Member, source: source + end + + desc 'Gets a member of a group or project.' do + success ::API::Entities::Member + end + params do + requires :user_id, type: Integer, desc: 'The user ID of the member' + end + get ":id/members/:user_id" do + source = find_source(source_type, params[:id]) + + members = source.members + member = members.find_by!(user_id: params[:user_id]) + + present member.user, with: ::API::Entities::Member, member: member + end + + desc 'Adds a member to a group or project.' do + success ::API::Entities::Member + end + params do + requires :user_id, type: Integer, desc: 'The user ID of the new member' + requires :access_level, type: Integer, desc: 'A valid access level (defaults: `30`, developer access level)' + optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY' + end + post ":id/members" do + source = find_source(source_type, params[:id]) + authorize_admin_source!(source_type, source) + + member = source.members.find_by(user_id: params[:user_id]) + + # We need this explicit check because `source.add_user` doesn't + # currently return the member created so it would return 201 even if + # the member already existed... + # The `source_type == 'group'` check is to ensure back-compatibility + # but 409 behavior should be used for both project and group members in 9.0! + conflict!('Member already exists') if source_type == 'group' && member + + unless member + member = source.add_user(params[:user_id], params[:access_level], current_user: current_user, expires_at: params[:expires_at]) + end + if member.persisted? && member.valid? + present member.user, with: ::API::Entities::Member, member: member + else + # This is to ensure back-compatibility but 400 behavior should be used + # for all validation errors in 9.0! + render_api_error!('Access level is not known', 422) if member.errors.key?(:access_level) + render_validation_error!(member) + end + end + + desc 'Updates a member of a group or project.' do + success ::API::Entities::Member + end + params do + requires :user_id, type: Integer, desc: 'The user ID of the new member' + requires :access_level, type: Integer, desc: 'A valid access level' + optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY' + end + put ":id/members/:user_id" do + source = find_source(source_type, params.delete(:id)) + authorize_admin_source!(source_type, source) + + member = source.members.find_by!(user_id: params.delete(:user_id)) + + if member.update_attributes(declared_params(include_missing: false)) + present member.user, with: ::API::Entities::Member, member: member + else + # This is to ensure back-compatibility but 400 behavior should be used + # for all validation errors in 9.0! + render_api_error!('Access level is not known', 422) if member.errors.key?(:access_level) + render_validation_error!(member) + end + end + + desc 'Removes a user from a group or project.' + params do + requires :user_id, type: Integer, desc: 'The user ID of the member' + end + delete ":id/members/:user_id" do + source = find_source(source_type, params[:id]) + + # This is to ensure back-compatibility but find_by! should be used + # in that casse in 9.0! + member = source.members.find_by(user_id: params[:user_id]) + + # This is to ensure back-compatibility but this should be removed in + # favor of find_by! in 9.0! + not_found!("Member: user_id:#{params[:user_id]}") if source_type == 'group' && member.nil? + + # This is to ensure back-compatibility but 204 behavior should be used + # for all DELETE endpoints in 9.0! + if member.nil? + { message: "Access revoked", id: params[:user_id].to_i } + else + ::Members::DestroyService.new(source, current_user, declared_params).execute + + present member.user, with: ::API::Entities::Member, member: member + end + end + end + end + end + end +end diff --git a/lib/api/v3/merge_requests.rb b/lib/api/v3/merge_requests.rb new file mode 100644 index 00000000000..129f9d850e9 --- /dev/null +++ b/lib/api/v3/merge_requests.rb @@ -0,0 +1,280 @@ +module API + module V3 + class MergeRequests < Grape::API + include PaginationParams + + DEPRECATION_MESSAGE = 'This endpoint is deprecated and has been removed on V4'.freeze + + before { authenticate! } + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects do + include TimeTrackingEndpoints + + helpers do + def handle_merge_request_errors!(errors) + if errors[:project_access].any? + error!(errors[:project_access], 422) + elsif errors[:branch_conflict].any? + error!(errors[:branch_conflict], 422) + elsif errors[:validate_fork].any? + error!(errors[:validate_fork], 422) + elsif errors[:validate_branches].any? + conflict!(errors[:validate_branches]) + end + + render_api_error!(errors, 400) + end + + params :optional_params do + optional :description, type: String, desc: 'The description of the merge request' + optional :assignee_id, type: Integer, desc: 'The ID of a user to assign the merge request' + optional :milestone_id, type: Integer, desc: 'The ID of a milestone to assign the merge request' + optional :labels, type: String, desc: 'Comma-separated list of label names' + optional :remove_source_branch, type: Boolean, desc: 'Remove source branch when merging' + end + end + + desc 'List merge requests' do + detail 'iid filter is deprecated have been removed on V4' + success ::API::Entities::MergeRequest + end + params do + optional :state, type: String, values: %w[opened closed merged all], default: 'all', + desc: 'Return opened, closed, merged, or all merge requests' + optional :order_by, type: String, values: %w[created_at updated_at], default: 'created_at', + desc: 'Return merge requests ordered by `created_at` or `updated_at` fields.' + optional :sort, type: String, values: %w[asc desc], default: 'desc', + desc: 'Return merge requests sorted in `asc` or `desc` order.' + optional :iid, type: Array[Integer], desc: 'The IID of the merge requests' + use :pagination + end + get ":id/merge_requests" do + authorize! :read_merge_request, user_project + + merge_requests = user_project.merge_requests.inc_notes_with_associations + merge_requests = filter_by_iid(merge_requests, params[:iid]) if params[:iid].present? + + merge_requests = + case params[:state] + when 'opened' then merge_requests.opened + when 'closed' then merge_requests.closed + when 'merged' then merge_requests.merged + else merge_requests + end + + merge_requests = merge_requests.reorder(params[:order_by] => params[:sort]) + present paginate(merge_requests), with: ::API::Entities::MergeRequest, current_user: current_user, project: user_project + end + + desc 'Create a merge request' do + success ::API::Entities::MergeRequest + end + params do + requires :title, type: String, desc: 'The title of the merge request' + requires :source_branch, type: String, desc: 'The source branch' + requires :target_branch, type: String, desc: 'The target branch' + optional :target_project_id, type: Integer, + desc: 'The target project of the merge request defaults to the :id of the project' + use :optional_params + end + post ":id/merge_requests" do + authorize! :create_merge_request, user_project + + mr_params = declared_params(include_missing: false) + mr_params[:force_remove_source_branch] = mr_params.delete(:remove_source_branch) if mr_params[:remove_source_branch].present? + + merge_request = ::MergeRequests::CreateService.new(user_project, current_user, mr_params).execute + + if merge_request.valid? + present merge_request, with: ::API::Entities::MergeRequest, current_user: current_user, project: user_project + else + handle_merge_request_errors! merge_request.errors + end + end + + desc 'Delete a merge request' + params do + requires :merge_request_id, type: Integer, desc: 'The ID of a merge request' + end + delete ":id/merge_requests/:merge_request_id" do + merge_request = find_project_merge_request(params[:merge_request_id]) + + authorize!(:destroy_merge_request, merge_request) + merge_request.destroy + end + + params do + requires :merge_request_id, type: Integer, desc: 'The ID of a merge request' + end + { ":id/merge_request/:merge_request_id" => :deprecated, ":id/merge_requests/:merge_request_id" => :ok }.each do |path, status| + desc 'Get a single merge request' do + if status == :deprecated + detail DEPRECATION_MESSAGE + end + success ::API::Entities::MergeRequest + end + get path do + merge_request = find_merge_request_with_access(params[:merge_request_id]) + + present merge_request, with: ::API::Entities::MergeRequest, current_user: current_user, project: user_project + end + + desc 'Get the commits of a merge request' do + success ::API::Entities::RepoCommit + end + get "#{path}/commits" do + merge_request = find_merge_request_with_access(params[:merge_request_id]) + + present merge_request.commits, with: ::API::Entities::RepoCommit + end + + desc 'Show the merge request changes' do + success ::API::Entities::MergeRequestChanges + end + get "#{path}/changes" do + merge_request = find_merge_request_with_access(params[:merge_request_id]) + + present merge_request, with: ::API::Entities::MergeRequestChanges, current_user: current_user + end + + desc 'Update a merge request' do + success ::API::Entities::MergeRequest + end + params do + optional :title, type: String, allow_blank: false, desc: 'The title of the merge request' + optional :target_branch, type: String, allow_blank: false, desc: 'The target branch' + optional :state_event, type: String, values: %w[close reopen merge], + desc: 'Status of the merge request' + use :optional_params + at_least_one_of :title, :target_branch, :description, :assignee_id, + :milestone_id, :labels, :state_event, + :remove_source_branch + end + put path do + merge_request = find_merge_request_with_access(params.delete(:merge_request_id), :update_merge_request) + + mr_params = declared_params(include_missing: false) + mr_params[:force_remove_source_branch] = mr_params.delete(:remove_source_branch) if mr_params[:remove_source_branch].present? + + merge_request = ::MergeRequests::UpdateService.new(user_project, current_user, mr_params).execute(merge_request) + + if merge_request.valid? + present merge_request, with: ::API::Entities::MergeRequest, current_user: current_user, project: user_project + else + handle_merge_request_errors! merge_request.errors + end + end + + desc 'Merge a merge request' do + success ::API::Entities::MergeRequest + end + params do + optional :merge_commit_message, type: String, desc: 'Custom merge commit message' + optional :should_remove_source_branch, type: Boolean, + desc: 'When true, the source branch will be deleted if possible' + optional :merge_when_build_succeeds, type: Boolean, + desc: 'When true, this merge request will be merged when the pipeline succeeds' + optional :sha, type: String, desc: 'When present, must have the HEAD SHA of the source branch' + end + put "#{path}/merge" do + merge_request = find_project_merge_request(params[:merge_request_id]) + + # Merge request can not be merged + # because user dont have permissions to push into target branch + unauthorized! unless merge_request.can_be_merged_by?(current_user) + + not_allowed! unless merge_request.mergeable_state? + + render_api_error!('Branch cannot be merged', 406) unless merge_request.mergeable? + + if params[:sha] && merge_request.diff_head_sha != params[:sha] + render_api_error!("SHA does not match HEAD of source branch: #{merge_request.diff_head_sha}", 409) + end + + merge_params = { + commit_message: params[:merge_commit_message], + should_remove_source_branch: params[:should_remove_source_branch] + } + + if params[:merge_when_build_succeeds] && merge_request.head_pipeline && merge_request.head_pipeline.active? + ::MergeRequests::MergeWhenPipelineSucceedsService + .new(merge_request.target_project, current_user, merge_params) + .execute(merge_request) + else + ::MergeRequests::MergeService + .new(merge_request.target_project, current_user, merge_params) + .execute(merge_request) + end + + present merge_request, with: ::API::Entities::MergeRequest, current_user: current_user, project: user_project + end + + desc 'Cancel merge if "Merge When Pipeline Succeeds" is enabled' do + success ::API::Entities::MergeRequest + end + post "#{path}/cancel_merge_when_build_succeeds" do + merge_request = find_project_merge_request(params[:merge_request_id]) + + unauthorized! unless merge_request.can_cancel_merge_when_build_succeeds?(current_user) + + ::MergeRequest::MergeWhenPipelineSucceedsService + .new(merge_request.target_project, current_user) + .cancel(merge_request) + end + + desc 'Get the comments of a merge request' do + detail 'Duplicate. DEPRECATED and HAS BEEN REMOVED in V4' + success ::API::Entities::MRNote + end + params do + use :pagination + end + get "#{path}/comments" do + merge_request = find_merge_request_with_access(params[:merge_request_id]) + present paginate(merge_request.notes.fresh), with: ::API::Entities::MRNote + end + + desc 'Post a comment to a merge request' do + detail 'Duplicate. DEPRECATED and HAS BEEN REMOVED in V4' + success ::API::Entities::MRNote + end + params do + requires :note, type: String, desc: 'The text of the comment' + end + post "#{path}/comments" do + merge_request = find_merge_request_with_access(params[:merge_request_id], :create_note) + + opts = { + note: params[:note], + noteable_type: 'MergeRequest', + noteable_id: merge_request.id + } + + note = ::Notes::CreateService.new(user_project, current_user, opts).execute + + if note.save + present note, with: ::API::Entities::MRNote + else + render_api_error!("Failed to save note #{note.errors.messages}", 400) + end + end + + desc 'List issues that will be closed on merge' do + success ::API::Entities::MRNote + end + params do + use :pagination + end + get "#{path}/closes_issues" do + merge_request = find_merge_request_with_access(params[:merge_request_id]) + issues = ::Kaminari.paginate_array(merge_request.closes_issues(current_user)) + present paginate(issues), with: issue_entity(user_project), current_user: current_user + end + end + end + end + end +end diff --git a/lib/api/v3/notes.rb b/lib/api/v3/notes.rb new file mode 100644 index 00000000000..0796bb62e68 --- /dev/null +++ b/lib/api/v3/notes.rb @@ -0,0 +1,148 @@ +module API + module V3 + class Notes < Grape::API + include PaginationParams + + before { authenticate! } + + NOTEABLE_TYPES = [Issue, MergeRequest, Snippet].freeze + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects do + NOTEABLE_TYPES.each do |noteable_type| + noteables_str = noteable_type.to_s.underscore.pluralize + + desc 'Get a list of project +noteable+ notes' do + success ::API::V3::Entities::Note + end + params do + requires :noteable_id, type: Integer, desc: 'The ID of the noteable' + use :pagination + end + get ":id/#{noteables_str}/:noteable_id/notes" do + noteable = user_project.send(noteables_str.to_sym).find(params[:noteable_id]) + + if can?(current_user, noteable_read_ability_name(noteable), noteable) + # We exclude notes that are cross-references and that cannot be viewed + # by the current user. By doing this exclusion at this level and not + # at the DB query level (which we cannot in that case), the current + # page can have less elements than :per_page even if + # there's more than one page. + notes = + # paginate() only works with a relation. This could lead to a + # mismatch between the pagination headers info and the actual notes + # array returned, but this is really a edge-case. + paginate(noteable.notes). + reject { |n| n.cross_reference_not_visible_for?(current_user) } + present notes, with: ::API::V3::Entities::Note + else + not_found!("Notes") + end + end + + desc 'Get a single +noteable+ note' do + success ::API::V3::Entities::Note + end + params do + requires :note_id, type: Integer, desc: 'The ID of a note' + requires :noteable_id, type: Integer, desc: 'The ID of the noteable' + end + get ":id/#{noteables_str}/:noteable_id/notes/:note_id" do + noteable = user_project.send(noteables_str.to_sym).find(params[:noteable_id]) + note = noteable.notes.find(params[:note_id]) + can_read_note = can?(current_user, noteable_read_ability_name(noteable), noteable) && !note.cross_reference_not_visible_for?(current_user) + + if can_read_note + present note, with: ::API::V3::Entities::Note + else + not_found!("Note") + end + end + + desc 'Create a new +noteable+ note' do + success ::API::V3::Entities::Note + end + params do + requires :noteable_id, type: Integer, desc: 'The ID of the noteable' + requires :body, type: String, desc: 'The content of a note' + optional :created_at, type: String, desc: 'The creation date of the note' + end + post ":id/#{noteables_str}/:noteable_id/notes" do + opts = { + note: params[:body], + noteable_type: noteables_str.classify, + noteable_id: params[:noteable_id] + } + + noteable = user_project.send(noteables_str.to_sym).find(params[:noteable_id]) + + if can?(current_user, noteable_read_ability_name(noteable), noteable) + if params[:created_at] && (current_user.is_admin? || user_project.owner == current_user) + opts[:created_at] = params[:created_at] + end + + note = ::Notes::CreateService.new(user_project, current_user, opts).execute + if note.valid? + present note, with: ::API::V3::Entities.const_get(note.class.name) + else + not_found!("Note #{note.errors.messages}") + end + else + not_found!("Note") + end + end + + desc 'Update an existing +noteable+ note' do + success ::API::V3::Entities::Note + end + params do + requires :noteable_id, type: Integer, desc: 'The ID of the noteable' + requires :note_id, type: Integer, desc: 'The ID of a note' + requires :body, type: String, desc: 'The content of a note' + end + put ":id/#{noteables_str}/:noteable_id/notes/:note_id" do + note = user_project.notes.find(params[:note_id]) + + authorize! :admin_note, note + + opts = { + note: params[:body] + } + + note = ::Notes::UpdateService.new(user_project, current_user, opts).execute(note) + + if note.valid? + present note, with: ::API::V3::Entities::Note + else + render_api_error!("Failed to save note #{note.errors.messages}", 400) + end + end + + desc 'Delete a +noteable+ note' do + success ::API::V3::Entities::Note + end + params do + requires :noteable_id, type: Integer, desc: 'The ID of the noteable' + requires :note_id, type: Integer, desc: 'The ID of a note' + end + delete ":id/#{noteables_str}/:noteable_id/notes/:note_id" do + note = user_project.notes.find(params[:note_id]) + authorize! :admin_note, note + + ::Notes::DestroyService.new(user_project, current_user).execute(note) + + present note, with: ::API::V3::Entities::Note + end + end + end + + helpers do + def noteable_read_ability_name(noteable) + "read_#{noteable.class.to_s.underscore}".to_sym + end + end + end + end +end diff --git a/lib/api/v3/project_snippets.rb b/lib/api/v3/project_snippets.rb new file mode 100644 index 00000000000..e03e941d30b --- /dev/null +++ b/lib/api/v3/project_snippets.rb @@ -0,0 +1,141 @@ +module API + module V3 + class ProjectSnippets < Grape::API + include PaginationParams + + before { authenticate! } + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects do + helpers do + def handle_project_member_errors(errors) + if errors[:project_access].any? + error!(errors[:project_access], 422) + end + not_found! + end + + def snippets_for_current_user + finder_params = { filter: :by_project, project: user_project } + SnippetsFinder.new.execute(current_user, finder_params) + end + end + + desc 'Get all project snippets' do + success ::API::V3::Entities::ProjectSnippet + end + params do + use :pagination + end + get ":id/snippets" do + present paginate(snippets_for_current_user), with: ::API::V3::Entities::ProjectSnippet + end + + desc 'Get a single project snippet' do + success ::API::V3::Entities::ProjectSnippet + end + params do + requires :snippet_id, type: Integer, desc: 'The ID of a project snippet' + end + get ":id/snippets/:snippet_id" do + snippet = snippets_for_current_user.find(params[:snippet_id]) + present snippet, with: ::API::V3::Entities::ProjectSnippet + end + + desc 'Create a new project snippet' do + success ::API::V3::Entities::ProjectSnippet + end + params do + requires :title, type: String, desc: 'The title of the snippet' + requires :file_name, type: String, desc: 'The file name of the snippet' + requires :code, type: String, desc: 'The content of the snippet' + requires :visibility_level, type: Integer, + values: [Gitlab::VisibilityLevel::PRIVATE, + Gitlab::VisibilityLevel::INTERNAL, + Gitlab::VisibilityLevel::PUBLIC], + desc: 'The visibility level of the snippet' + end + post ":id/snippets" do + authorize! :create_project_snippet, user_project + snippet_params = declared_params.merge(request: request, api: true) + snippet_params[:content] = snippet_params.delete(:code) + + snippet = CreateSnippetService.new(user_project, current_user, snippet_params).execute + + render_spam_error! if snippet.spam? + + if snippet.persisted? + present snippet, with: ::API::V3::Entities::ProjectSnippet + else + render_validation_error!(snippet) + end + end + + desc 'Update an existing project snippet' do + success ::API::V3::Entities::ProjectSnippet + end + params do + requires :snippet_id, type: Integer, desc: 'The ID of a project snippet' + optional :title, type: String, desc: 'The title of the snippet' + optional :file_name, type: String, desc: 'The file name of the snippet' + optional :code, type: String, desc: 'The content of the snippet' + optional :visibility_level, type: Integer, + values: [Gitlab::VisibilityLevel::PRIVATE, + Gitlab::VisibilityLevel::INTERNAL, + Gitlab::VisibilityLevel::PUBLIC], + desc: 'The visibility level of the snippet' + at_least_one_of :title, :file_name, :code, :visibility_level + end + put ":id/snippets/:snippet_id" do + snippet = snippets_for_current_user.find_by(id: params.delete(:snippet_id)) + not_found!('Snippet') unless snippet + + authorize! :update_project_snippet, snippet + + snippet_params = declared_params(include_missing: false) + .merge(request: request, api: true) + + snippet_params[:content] = snippet_params.delete(:code) if snippet_params[:code].present? + + UpdateSnippetService.new(user_project, current_user, snippet, + snippet_params).execute + + render_spam_error! if snippet.spam? + + if snippet.valid? + present snippet, with: ::API::V3::Entities::ProjectSnippet + else + render_validation_error!(snippet) + end + end + + desc 'Delete a project snippet' + params do + requires :snippet_id, type: Integer, desc: 'The ID of a project snippet' + end + delete ":id/snippets/:snippet_id" do + snippet = snippets_for_current_user.find_by(id: params[:snippet_id]) + not_found!('Snippet') unless snippet + + authorize! :admin_project_snippet, snippet + snippet.destroy + end + + desc 'Get a raw project snippet' + params do + requires :snippet_id, type: Integer, desc: 'The ID of a project snippet' + end + get ":id/snippets/:snippet_id/raw" do + snippet = snippets_for_current_user.find_by(id: params[:snippet_id]) + not_found!('Snippet') unless snippet + + env['api.format'] = :txt + content_type 'text/plain' + present snippet.content + end + end + end + end +end diff --git a/lib/api/v3/projects.rb b/lib/api/v3/projects.rb new file mode 100644 index 00000000000..c3821555452 --- /dev/null +++ b/lib/api/v3/projects.rb @@ -0,0 +1,459 @@ +module API + module V3 + class Projects < Grape::API + include PaginationParams + + before { authenticate_non_get! } + + helpers do + params :optional_params do + optional :description, type: String, desc: 'The description of the project' + optional :issues_enabled, type: Boolean, desc: 'Flag indication if the issue tracker is enabled' + optional :merge_requests_enabled, type: Boolean, desc: 'Flag indication if merge requests are enabled' + optional :wiki_enabled, type: Boolean, desc: 'Flag indication if the wiki is enabled' + optional :builds_enabled, type: Boolean, desc: 'Flag indication if builds are enabled' + optional :snippets_enabled, type: Boolean, desc: 'Flag indication if snippets are enabled' + optional :shared_runners_enabled, type: Boolean, desc: 'Flag indication if shared runners are enabled for that project' + optional :container_registry_enabled, type: Boolean, desc: 'Flag indication if the container registry is enabled for that project' + optional :lfs_enabled, type: Boolean, desc: 'Flag indication if Git LFS is enabled for that project' + optional :public, type: Boolean, desc: 'Create a public project. The same as visibility_level = 20.' + optional :visibility_level, type: Integer, values: [ + Gitlab::VisibilityLevel::PRIVATE, + Gitlab::VisibilityLevel::INTERNAL, + Gitlab::VisibilityLevel::PUBLIC + ], desc: 'Create a public project. The same as visibility_level = 20.' + optional :public_builds, type: Boolean, desc: 'Perform public builds' + optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access' + optional :only_allow_merge_if_build_succeeds, type: Boolean, desc: 'Only allow to merge if builds succeed' + optional :only_allow_merge_if_all_discussions_are_resolved, type: Boolean, desc: 'Only allow to merge if all discussions are resolved' + end + + def map_public_to_visibility_level(attrs) + publik = attrs.delete(:public) + if !publik.nil? && !attrs[:visibility_level].present? + # Since setting the public attribute to private could mean either + # private or internal, use the more conservative option, private. + attrs[:visibility_level] = (publik == true) ? Gitlab::VisibilityLevel::PUBLIC : Gitlab::VisibilityLevel::PRIVATE + end + attrs + end + end + + resource :projects do + helpers do + params :collection_params do + use :sort_params + use :filter_params + use :pagination + + optional :simple, type: Boolean, default: false, + desc: 'Return only the ID, URL, name, and path of each project' + end + + params :sort_params do + optional :order_by, type: String, values: %w[id name path created_at updated_at last_activity_at], + default: 'created_at', desc: 'Return projects ordered by field' + optional :sort, type: String, values: %w[asc desc], default: 'desc', + desc: 'Return projects sorted in ascending and descending order' + end + + params :filter_params do + optional :archived, type: Boolean, default: false, desc: 'Limit by archived status' + optional :visibility, type: String, values: %w[public internal private], + desc: 'Limit by visibility' + optional :search, type: String, desc: 'Return list of authorized projects matching the search criteria' + end + + params :statistics_params do + optional :statistics, type: Boolean, default: false, desc: 'Include project statistics' + end + + params :create_params do + optional :namespace_id, type: Integer, desc: 'Namespace ID for the new project. Default to the user namespace.' + optional :import_url, type: String, desc: 'URL from which the project is imported' + end + + def present_projects(projects, options = {}) + options = options.reverse_merge( + with: ::API::Entities::Project, + current_user: current_user, + simple: params[:simple], + ) + + projects = filter_projects(projects) + projects = projects.with_statistics if options[:statistics] + options[:with] = ::API::Entities::BasicProjectDetails if options[:simple] + + present paginate(projects), options + end + end + + desc 'Get a list of visible projects for authenticated user' do + success ::API::Entities::BasicProjectDetails + end + params do + use :collection_params + end + get '/visible' do + entity = current_user ? ::API::Entities::ProjectWithAccess : ::API::Entities::BasicProjectDetails + present_projects ProjectsFinder.new.execute(current_user), with: entity + end + + desc 'Get a projects list for authenticated user' do + success ::API::Entities::BasicProjectDetails + end + params do + use :collection_params + end + get do + authenticate! + + present_projects current_user.authorized_projects, + with: ::API::Entities::ProjectWithAccess + end + + desc 'Get an owned projects list for authenticated user' do + success ::API::Entities::BasicProjectDetails + end + params do + use :collection_params + use :statistics_params + end + get '/owned' do + authenticate! + + present_projects current_user.owned_projects, + with: ::API::Entities::ProjectWithAccess, + statistics: params[:statistics] + end + + desc 'Gets starred project for the authenticated user' do + success ::API::Entities::BasicProjectDetails + end + params do + use :collection_params + end + get '/starred' do + authenticate! + + present_projects current_user.viewable_starred_projects + end + + desc 'Get all projects for admin user' do + success ::API::Entities::BasicProjectDetails + end + params do + use :collection_params + use :statistics_params + end + get '/all' do + authenticated_as_admin! + + present_projects Project.all, with: ::API::Entities::ProjectWithAccess, statistics: params[:statistics] + end + + desc 'Search for projects the current user has access to' do + success ::API::Entities::Project + end + params do + requires :query, type: String, desc: 'The project name to be searched' + use :sort_params + use :pagination + end + get "/search/:query", requirements: { query: /[^\/]+/ } do + search_service = Search::GlobalService.new(current_user, search: params[:query]).execute + projects = search_service.objects('projects', params[:page]) + projects = projects.reorder(params[:order_by] => params[:sort]) + + present paginate(projects), with: ::API::Entities::Project + end + + desc 'Create new project' do + success ::API::Entities::Project + end + params do + requires :name, type: String, desc: 'The name of the project' + optional :path, type: String, desc: 'The path of the repository' + use :optional_params + use :create_params + end + post do + attrs = map_public_to_visibility_level(declared_params(include_missing: false)) + project = ::Projects::CreateService.new(current_user, attrs).execute + + if project.saved? + present project, with: ::API::Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, project) + else + if project.errors[:limit_reached].present? + error!(project.errors[:limit_reached], 403) + end + render_validation_error!(project) + end + end + + desc 'Create new project for a specified user. Only available to admin users.' do + success ::API::Entities::Project + end + params do + requires :name, type: String, desc: 'The name of the project' + requires :user_id, type: Integer, desc: 'The ID of a user' + optional :default_branch, type: String, desc: 'The default branch of the project' + use :optional_params + use :create_params + end + post "user/:user_id" do + authenticated_as_admin! + user = User.find_by(id: params.delete(:user_id)) + not_found!('User') unless user + + attrs = map_public_to_visibility_level(declared_params(include_missing: false)) + project = ::Projects::CreateService.new(user, attrs).execute + + if project.saved? + present project, with: ::API::Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, project) + else + render_validation_error!(project) + end + end + end + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: { id: /[^\/]+/ } do + desc 'Get a single project' do + success ::API::Entities::ProjectWithAccess + end + get ":id" do + entity = current_user ? ::API::Entities::ProjectWithAccess : ::API::Entities::BasicProjectDetails + present user_project, with: entity, current_user: current_user, + user_can_admin_project: can?(current_user, :admin_project, user_project) + end + + desc 'Get events for a single project' do + success ::API::V3::Entities::Event + end + params do + use :pagination + end + get ":id/events" do + present paginate(user_project.events.recent), with: ::API::V3::Entities::Event + end + + desc 'Fork new project for the current user or provided namespace.' do + success ::API::Entities::Project + end + params do + optional :namespace, type: String, desc: 'The ID or name of the namespace that the project will be forked into' + end + post 'fork/:id' do + fork_params = declared_params(include_missing: false) + namespace_id = fork_params[:namespace] + + if namespace_id.present? + fork_params[:namespace] = if namespace_id =~ /^\d+$/ + Namespace.find_by(id: namespace_id) + else + Namespace.find_by_path_or_name(namespace_id) + end + + unless fork_params[:namespace] && can?(current_user, :create_projects, fork_params[:namespace]) + not_found!('Target Namespace') + end + end + + forked_project = ::Projects::ForkService.new(user_project, current_user, fork_params).execute + + if forked_project.errors.any? + conflict!(forked_project.errors.messages) + else + present forked_project, with: ::API::Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, forked_project) + end + end + + desc 'Update an existing project' do + success ::API::Entities::Project + end + params do + optional :name, type: String, desc: 'The name of the project' + optional :default_branch, type: String, desc: 'The default branch of the project' + optional :path, type: String, desc: 'The path of the repository' + use :optional_params + at_least_one_of :name, :description, :issues_enabled, :merge_requests_enabled, + :wiki_enabled, :builds_enabled, :snippets_enabled, + :shared_runners_enabled, :container_registry_enabled, + :lfs_enabled, :public, :visibility_level, :public_builds, + :request_access_enabled, :only_allow_merge_if_build_succeeds, + :only_allow_merge_if_all_discussions_are_resolved, :path, + :default_branch + end + put ':id' do + authorize_admin_project + attrs = map_public_to_visibility_level(declared_params(include_missing: false)) + authorize! :rename_project, user_project if attrs[:name].present? + authorize! :change_visibility_level, user_project if attrs[:visibility_level].present? + + result = ::Projects::UpdateService.new(user_project, current_user, attrs).execute + + if result[:status] == :success + present user_project, with: ::API::Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, user_project) + else + render_validation_error!(user_project) + end + end + + desc 'Archive a project' do + success ::API::Entities::Project + end + post ':id/archive' do + authorize!(:archive_project, user_project) + + user_project.archive! + + present user_project, with: ::API::Entities::Project + end + + desc 'Unarchive a project' do + success ::API::Entities::Project + end + post ':id/unarchive' do + authorize!(:archive_project, user_project) + + user_project.unarchive! + + present user_project, with: ::API::Entities::Project + end + + desc 'Star a project' do + success ::API::Entities::Project + end + post ':id/star' do + if current_user.starred?(user_project) + not_modified! + else + current_user.toggle_star(user_project) + user_project.reload + + present user_project, with: ::API::Entities::Project + end + end + + desc 'Unstar a project' do + success ::API::Entities::Project + end + delete ':id/star' do + if current_user.starred?(user_project) + current_user.toggle_star(user_project) + user_project.reload + + present user_project, with: ::API::Entities::Project + else + not_modified! + end + end + + desc 'Remove a project' + delete ":id" do + authorize! :remove_project, user_project + ::Projects::DestroyService.new(user_project, current_user, {}).async_execute + end + + desc 'Mark this project as forked from another' + params do + requires :forked_from_id, type: String, desc: 'The ID of the project it was forked from' + end + post ":id/fork/:forked_from_id" do + authenticated_as_admin! + + forked_from_project = find_project!(params[:forked_from_id]) + not_found!("Source Project") unless forked_from_project + + if user_project.forked_from_project.nil? + user_project.create_forked_project_link(forked_to_project_id: user_project.id, forked_from_project_id: forked_from_project.id) + else + render_api_error!("Project already forked", 409) + end + end + + desc 'Remove a forked_from relationship' + delete ":id/fork" do + authorize! :remove_fork_project, user_project + + if user_project.forked? + user_project.forked_project_link.destroy + else + not_modified! + end + end + + desc 'Share the project with a group' do + success ::API::Entities::ProjectGroupLink + end + params do + requires :group_id, type: Integer, desc: 'The ID of a group' + requires :group_access, type: Integer, values: Gitlab::Access.values, desc: 'The group access level' + optional :expires_at, type: Date, desc: 'Share expiration date' + end + post ":id/share" do + authorize! :admin_project, user_project + group = Group.find_by_id(params[:group_id]) + + unless group && can?(current_user, :read_group, group) + not_found!('Group') + end + + unless user_project.allowed_to_share_with_group? + return render_api_error!("The project sharing with group is disabled", 400) + end + + link = user_project.project_group_links.new(declared_params(include_missing: false)) + + if link.save + present link, with: ::API::Entities::ProjectGroupLink + else + render_api_error!(link.errors.full_messages.first, 409) + end + end + + params do + requires :group_id, type: Integer, desc: 'The ID of the group' + end + delete ":id/share/:group_id" do + authorize! :admin_project, user_project + + link = user_project.project_group_links.find_by(group_id: params[:group_id]) + not_found!('Group Link') unless link + + link.destroy + no_content! + end + + desc 'Upload a file' + params do + requires :file, type: File, desc: 'The file to be uploaded' + end + post ":id/uploads" do + ::Projects::UploadService.new(user_project, params[:file]).execute + end + + desc 'Get the users list of a project' do + success ::API::Entities::UserBasic + end + params do + optional :search, type: String, desc: 'Return list of users matching the search criteria' + use :pagination + end + get ':id/users' do + users = user_project.team.users + users = users.search(params[:search]) if params[:search].present? + + present paginate(users), with: ::API::Entities::UserBasic + end + end + end + end +end diff --git a/lib/api/v3/repositories.rb b/lib/api/v3/repositories.rb new file mode 100644 index 00000000000..3549ea225ef --- /dev/null +++ b/lib/api/v3/repositories.rb @@ -0,0 +1,55 @@ +require 'mime/types' + +module API + module V3 + class Repositories < Grape::API + before { authorize! :download_code, user_project } + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects do + helpers do + def handle_project_member_errors(errors) + if errors[:project_access].any? + error!(errors[:project_access], 422) + end + not_found! + end + end + + desc 'Get a project repository tree' do + success ::API::Entities::RepoTreeObject + end + params do + optional :ref_name, type: String, desc: 'The name of a repository branch or tag, if not given the default branch is used' + optional :path, type: String, desc: 'The path of the tree' + optional :recursive, type: Boolean, default: false, desc: 'Used to get a recursive tree' + end + get ':id/repository/tree' do + ref = params[:ref_name] || user_project.try(:default_branch) || 'master' + path = params[:path] || nil + + commit = user_project.commit(ref) + not_found!('Tree') unless commit + + tree = user_project.repository.tree(commit.id, path, recursive: params[:recursive]) + + present tree.sorted_entries, with: ::API::Entities::RepoTreeObject + end + + desc 'Get repository contributors' do + success ::API::Entities::Contributor + end + get ':id/repository/contributors' do + begin + present user_project.repository.contributors, + with: ::API::Entities::Contributor + rescue + not_found! + end + end + end + end + end +end diff --git a/lib/api/v3/subscriptions.rb b/lib/api/v3/subscriptions.rb new file mode 100644 index 00000000000..02a4157c26e --- /dev/null +++ b/lib/api/v3/subscriptions.rb @@ -0,0 +1,53 @@ +module API + module V3 + class Subscriptions < Grape::API + before { authenticate! } + + subscribable_types = { + 'merge_request' => proc { |id| find_merge_request_with_access(id, :update_merge_request) }, + 'merge_requests' => proc { |id| find_merge_request_with_access(id, :update_merge_request) }, + 'issues' => proc { |id| find_project_issue(id) }, + 'labels' => proc { |id| find_project_label(id) }, + } + + params do + requires :id, type: String, desc: 'The ID of a project' + requires :subscribable_id, type: String, desc: 'The ID of a resource' + end + resource :projects do + subscribable_types.each do |type, finder| + type_singularized = type.singularize + entity_class = ::API::Entities.const_get(type_singularized.camelcase) + + desc 'Subscribe to a resource' do + success entity_class + end + post ":id/#{type}/:subscribable_id/subscription" do + resource = instance_exec(params[:subscribable_id], &finder) + + if resource.subscribed?(current_user, user_project) + not_modified! + else + resource.subscribe(current_user, user_project) + present resource, with: entity_class, current_user: current_user, project: user_project + end + end + + desc 'Unsubscribe from a resource' do + success entity_class + end + delete ":id/#{type}/:subscribable_id/subscription" do + resource = instance_exec(params[:subscribable_id], &finder) + + if !resource.subscribed?(current_user, user_project) + not_modified! + else + resource.unsubscribe(current_user, user_project) + present resource, with: entity_class, current_user: current_user, project: user_project + end + end + end + end + end + end +end diff --git a/lib/api/v3/system_hooks.rb b/lib/api/v3/system_hooks.rb new file mode 100644 index 00000000000..391510b9ee0 --- /dev/null +++ b/lib/api/v3/system_hooks.rb @@ -0,0 +1,19 @@ +module API + module V3 + class SystemHooks < Grape::API + before do + authenticate! + authenticated_as_admin! + end + + resource :hooks do + desc 'Get the list of system hooks' do + success ::API::Entities::Hook + end + get do + present SystemHook.all, with: ::API::Entities::Hook + end + end + end + end +end diff --git a/lib/api/v3/tags.rb b/lib/api/v3/tags.rb new file mode 100644 index 00000000000..016e3d86932 --- /dev/null +++ b/lib/api/v3/tags.rb @@ -0,0 +1,20 @@ +module API + module V3 + class Tags < Grape::API + before { authorize! :download_code, user_project } + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects do + desc 'Get a project repository tags' do + success ::API::Entities::RepoTag + end + get ":id/repository/tags" do + tags = user_project.repository.tags.sort_by(&:name).reverse + present tags, with: ::API::Entities::RepoTag, project: user_project + end + end + end + end +end diff --git a/lib/api/v3/templates.rb b/lib/api/v3/templates.rb new file mode 100644 index 00000000000..4c577a8d2b7 --- /dev/null +++ b/lib/api/v3/templates.rb @@ -0,0 +1,122 @@ +module API + module V3 + class Templates < Grape::API + GLOBAL_TEMPLATE_TYPES = { + gitignores: { + klass: Gitlab::Template::GitignoreTemplate, + gitlab_version: 8.8 + }, + gitlab_ci_ymls: { + klass: Gitlab::Template::GitlabCiYmlTemplate, + gitlab_version: 8.9 + }, + dockerfiles: { + klass: Gitlab::Template::DockerfileTemplate, + gitlab_version: 8.15 + } + }.freeze + PROJECT_TEMPLATE_REGEX = + /[\<\{\[] + (project|description| + one\sline\s.+\swhat\sit\sdoes\.) # matching the start and end is enough here + [\>\}\]]/xi.freeze + YEAR_TEMPLATE_REGEX = /[<{\[](year|yyyy)[>}\]]/i.freeze + FULLNAME_TEMPLATE_REGEX = + /[\<\{\[] + (fullname|name\sof\s(author|copyright\sowner)) + [\>\}\]]/xi.freeze + DEPRECATION_MESSAGE = ' This endpoint is deprecated and has been removed in V4.'.freeze + + helpers do + def parsed_license_template + # We create a fresh Licensee::License object since we'll modify its + # content in place below. + template = Licensee::License.new(params[:name]) + + template.content.gsub!(YEAR_TEMPLATE_REGEX, Time.now.year.to_s) + template.content.gsub!(PROJECT_TEMPLATE_REGEX, params[:project]) if params[:project].present? + + fullname = params[:fullname].presence || current_user.try(:name) + template.content.gsub!(FULLNAME_TEMPLATE_REGEX, fullname) if fullname + template + end + + def render_response(template_type, template) + not_found!(template_type.to_s.singularize) unless template + present template, with: ::API::Entities::Template + end + end + + { "licenses" => :deprecated, "templates/licenses" => :ok }.each do |route, status| + desc 'Get the list of the available license template' do + detailed_desc = 'This feature was introduced in GitLab 8.7.' + detailed_desc << DEPRECATION_MESSAGE unless status == :ok + detail detailed_desc + success ::API::Entities::RepoLicense + end + params do + optional :popular, type: Boolean, desc: 'If passed, returns only popular licenses' + end + get route do + options = { + featured: declared(params).popular.present? ? true : nil + } + present Licensee::License.all(options), with: ::API::Entities::RepoLicense + end + end + + { "licenses/:name" => :deprecated, "templates/licenses/:name" => :ok }.each do |route, status| + desc 'Get the text for a specific license' do + detailed_desc = 'This feature was introduced in GitLab 8.7.' + detailed_desc << DEPRECATION_MESSAGE unless status == :ok + detail detailed_desc + success ::API::Entities::RepoLicense + end + params do + requires :name, type: String, desc: 'The name of the template' + end + get route, requirements: { name: /[\w\.-]+/ } do + not_found!('License') unless Licensee::License.find(declared(params).name) + + template = parsed_license_template + + present template, with: ::API::Entities::RepoLicense + end + end + + GLOBAL_TEMPLATE_TYPES.each do |template_type, properties| + klass = properties[:klass] + gitlab_version = properties[:gitlab_version] + + { template_type => :deprecated, "templates/#{template_type}" => :ok }.each do |route, status| + desc 'Get the list of the available template' do + detailed_desc = "This feature was introduced in GitLab #{gitlab_version}." + detailed_desc << DEPRECATION_MESSAGE unless status == :ok + detail detailed_desc + success ::API::Entities::TemplatesList + end + get route do + present klass.all, with: ::API::Entities::TemplatesList + end + end + + { "#{template_type}/:name" => :deprecated, "templates/#{template_type}/:name" => :ok }.each do |route, status| + desc 'Get the text for a specific template present in local filesystem' do + detailed_desc = "This feature was introduced in GitLab #{gitlab_version}." + detailed_desc << DEPRECATION_MESSAGE unless status == :ok + detail detailed_desc + success ::API::Entities::Template + end + params do + requires :name, type: String, desc: 'The name of the template' + end + get route do + new_template = klass.find(declared(params).name) + + render_response(template_type, new_template) + end + end + end + end + end +end diff --git a/lib/api/v3/todos.rb b/lib/api/v3/todos.rb new file mode 100644 index 00000000000..4f9b5fe72a6 --- /dev/null +++ b/lib/api/v3/todos.rb @@ -0,0 +1,28 @@ +module API + module V3 + class Todos < Grape::API + before { authenticate! } + + resource :todos do + desc 'Mark a todo as done' do + success ::API::Entities::Todo + end + params do + requires :id, type: Integer, desc: 'The ID of the todo being marked as done' + end + delete ':id' do + todo = current_user.todos.find(params[:id]) + TodoService.new.mark_todos_as_done([todo], current_user) + + present todo.reload, with: ::API::Entities::Todo, current_user: current_user + end + + desc 'Mark all todos as done' + delete do + todos = TodosFinder.new(current_user, params).execute + TodoService.new.mark_todos_as_done(todos, current_user) + end + end + end + end +end diff --git a/lib/api/v3/users.rb b/lib/api/v3/users.rb new file mode 100644 index 00000000000..7838cdc46a7 --- /dev/null +++ b/lib/api/v3/users.rb @@ -0,0 +1,117 @@ +module API + module V3 + class Users < Grape::API + include PaginationParams + + before do + allow_access_with_scope :read_user if request.get? + authenticate! + end + + resource :users, requirements: { uid: /[0-9]*/, id: /[0-9]*/ } do + desc 'Get the SSH keys of a specified user. Available only for admins.' do + success ::API::Entities::SSHKey + end + params do + requires :id, type: Integer, desc: 'The ID of the user' + use :pagination + end + get ':id/keys' do + authenticated_as_admin! + + user = User.find_by(id: params[:id]) + not_found!('User') unless user + + present paginate(user.keys), with: ::API::Entities::SSHKey + end + + desc 'Get the emails addresses of a specified user. Available only for admins.' do + success ::API::Entities::Email + end + params do + requires :id, type: Integer, desc: 'The ID of the user' + use :pagination + end + get ':id/emails' do + authenticated_as_admin! + user = User.find_by(id: params[:id]) + not_found!('User') unless user + + present user.emails, with: ::API::Entities::Email + end + + desc 'Block a user. Available only for admins.' + params do + requires :id, type: Integer, desc: 'The ID of the user' + end + put ':id/block' do + authenticated_as_admin! + user = User.find_by(id: params[:id]) + not_found!('User') unless user + + if !user.ldap_blocked? + user.block + else + forbidden!('LDAP blocked users cannot be modified by the API') + end + end + + desc 'Unblock a user. Available only for admins.' + params do + requires :id, type: Integer, desc: 'The ID of the user' + end + put ':id/unblock' do + authenticated_as_admin! + user = User.find_by(id: params[:id]) + not_found!('User') unless user + + if user.ldap_blocked? + forbidden!('LDAP blocked users cannot be unblocked by the API') + else + user.activate + end + end + + desc 'Get the contribution events of a specified user' do + detail 'This feature was introduced in GitLab 8.13.' + success ::API::V3::Entities::Event + end + params do + requires :id, type: Integer, desc: 'The ID of the user' + use :pagination + end + get ':id/events' do + user = User.find_by(id: params[:id]) + not_found!('User') unless user + + events = user.events. + merge(ProjectsFinder.new.execute(current_user)). + references(:project). + with_associations. + recent + + present paginate(events), with: ::API::V3::Entities::Event + end + end + + resource :user do + desc "Get the currently authenticated user's SSH keys" do + success ::API::Entities::SSHKey + end + params do + use :pagination + end + get "keys" do + present current_user.keys, with: ::API::Entities::SSHKey + end + + desc "Get the currently authenticated user's email addresses" do + success ::API::Entities::Email + end + get "emails" do + present current_user.emails, with: ::API::Entities::Email + end + end + end + end +end diff --git a/lib/api/variables.rb b/lib/api/variables.rb index f6495071a11..f623b1dfe9f 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -1,51 +1,51 @@ module API # Projects variables API class Variables < Grape::API + include PaginationParams + before { authenticate! } before { authorize! :admin_build, user_project } + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects do - # Get project variables - # - # Parameters: - # id (required) - The ID of a project - # page (optional) - The page number for pagination - # per_page (optional) - The value of items per page to show - # Example Request: - # GET /projects/:id/variables + desc 'Get project variables' do + success Entities::Variable + end + params do + use :pagination + end get ':id/variables' do variables = user_project.variables present paginate(variables), with: Entities::Variable end - # Get specific variable of a project - # - # Parameters: - # id (required) - The ID of a project - # key (required) - The `key` of variable - # Example Request: - # GET /projects/:id/variables/:key + desc 'Get a specific variable from a project' do + success Entities::Variable + end + params do + requires :key, type: String, desc: 'The key of the variable' + end get ':id/variables/:key' do key = params[:key] - variable = user_project.variables.find_by(key: key.to_s) + variable = user_project.variables.find_by(key: key) return not_found!('Variable') unless variable present variable, with: Entities::Variable end - # Create a new variable in project - # - # Parameters: - # id (required) - The ID of a project - # key (required) - The key of variable - # value (required) - The value of variable - # Example Request: - # POST /projects/:id/variables + desc 'Create a new variable in a project' do + success Entities::Variable + end + params do + requires :key, type: String, desc: 'The key of the variable' + requires :value, type: String, desc: 'The value of the variable' + end post ':id/variables' do - required_attributes! [:key, :value] - - variable = user_project.variables.create(key: params[:key], value: params[:value]) + variable = user_project.variables.create(declared(params, include_parent_namespaces: false).to_h) if variable.valid? present variable, with: Entities::Variable @@ -54,41 +54,37 @@ module API end end - # Update existing variable of a project - # - # Parameters: - # id (required) - The ID of a project - # key (optional) - The `key` of variable - # value (optional) - New value for `value` field of variable - # Example Request: - # PUT /projects/:id/variables/:key + desc 'Update an existing variable from a project' do + success Entities::Variable + end + params do + optional :key, type: String, desc: 'The key of the variable' + optional :value, type: String, desc: 'The value of the variable' + end put ':id/variables/:key' do - variable = user_project.variables.find_by(key: params[:key].to_s) + variable = user_project.variables.find_by(key: params[:key]) return not_found!('Variable') unless variable - attrs = attributes_for_keys [:value] - if variable.update(attrs) + if variable.update(value: params[:value]) present variable, with: Entities::Variable else render_validation_error!(variable) end end - # Delete existing variable of a project - # - # Parameters: - # id (required) - The ID of a project - # key (required) - The ID of a variable - # Example Request: - # DELETE /projects/:id/variables/:key + desc 'Delete an existing variable from a project' do + success Entities::Variable + end + params do + requires :key, type: String, desc: 'The key of the variable' + end delete ':id/variables/:key' do - variable = user_project.variables.find_by(key: params[:key].to_s) + variable = user_project.variables.find_by(key: params[:key]) return not_found!('Variable') unless variable - variable.destroy - present variable, with: Entities::Variable + present variable.destroy, with: Entities::Variable end end end diff --git a/lib/api/version.rb b/lib/api/version.rb new file mode 100644 index 00000000000..9ba576bd828 --- /dev/null +++ b/lib/api/version.rb @@ -0,0 +1,12 @@ +module API + class Version < Grape::API + before { authenticate! } + + desc 'Get the version information of the GitLab instance.' do + detail 'This feature was introduced in GitLab 8.13.' + end + get '/version' do + { version: Gitlab::VERSION, revision: Gitlab::REVISION } + end + end +end diff --git a/lib/backup/database.rb b/lib/backup/database.rb index 22319ec6623..4016ac76348 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -5,7 +5,7 @@ module Backup attr_reader :config, :db_file_name def initialize - @config = YAML.load_file(File.join(Rails.root,'config','database.yml'))[Rails.env] + @config = YAML.load_file(File.join(Rails.root, 'config', 'database.yml'))[Rails.env] @db_file_name = File.join(Gitlab.config.backup.path, 'db', 'database.sql.gz') end @@ -13,28 +13,32 @@ module Backup FileUtils.mkdir_p(File.dirname(db_file_name)) FileUtils.rm_f(db_file_name) compress_rd, compress_wr = IO.pipe - compress_pid = spawn(*%W(gzip -1 -c), in: compress_rd, out: [db_file_name, 'w', 0600]) + compress_pid = spawn(*%w(gzip -1 -c), in: compress_rd, out: [db_file_name, 'w', 0600]) compress_rd.close - dump_pid = case config["adapter"] - when /^mysql/ then - $progress.print "Dumping MySQL database #{config['database']} ... " - # Workaround warnings from MySQL 5.6 about passwords on cmd line - ENV['MYSQL_PWD'] = config["password"].to_s if config["password"] - spawn('mysqldump', *mysql_args, config['database'], out: compress_wr) - when "postgresql" then - $progress.print "Dumping PostgreSQL database #{config['database']} ... " - pg_env - pgsql_args = ["--clean"] # Pass '--clean' to include 'DROP TABLE' statements in the DB dump. - if Gitlab.config.backup.pg_schema - pgsql_args << "-n" - pgsql_args << Gitlab.config.backup.pg_schema + dump_pid = + case config["adapter"] + when /^mysql/ then + $progress.print "Dumping MySQL database #{config['database']} ... " + # Workaround warnings from MySQL 5.6 about passwords on cmd line + ENV['MYSQL_PWD'] = config["password"].to_s if config["password"] + spawn('mysqldump', *mysql_args, config['database'], out: compress_wr) + when "postgresql" then + $progress.print "Dumping PostgreSQL database #{config['database']} ... " + pg_env + pgsql_args = ["--clean"] # Pass '--clean' to include 'DROP TABLE' statements in the DB dump. + if Gitlab.config.backup.pg_schema + pgsql_args << "-n" + pgsql_args << Gitlab.config.backup.pg_schema + end + spawn('pg_dump', *pgsql_args, config['database'], out: compress_wr) end - spawn('pg_dump', *pgsql_args, config['database'], out: compress_wr) - end compress_wr.close - success = [compress_pid, dump_pid].all? { |pid| Process.waitpid(pid); $?.success? } + success = [compress_pid, dump_pid].all? do |pid| + Process.waitpid(pid) + $?.success? + end report_success(success) abort 'Backup failed' unless success @@ -42,23 +46,27 @@ module Backup def restore decompress_rd, decompress_wr = IO.pipe - decompress_pid = spawn(*%W(gzip -cd), out: decompress_wr, in: db_file_name) + decompress_pid = spawn(*%w(gzip -cd), out: decompress_wr, in: db_file_name) decompress_wr.close - restore_pid = case config["adapter"] - when /^mysql/ then - $progress.print "Restoring MySQL database #{config['database']} ... " - # Workaround warnings from MySQL 5.6 about passwords on cmd line - ENV['MYSQL_PWD'] = config["password"].to_s if config["password"] - spawn('mysql', *mysql_args, config['database'], in: decompress_rd) - when "postgresql" then - $progress.print "Restoring PostgreSQL database #{config['database']} ... " - pg_env - spawn('psql', config['database'], in: decompress_rd) - end + restore_pid = + case config["adapter"] + when /^mysql/ then + $progress.print "Restoring MySQL database #{config['database']} ... " + # Workaround warnings from MySQL 5.6 about passwords on cmd line + ENV['MYSQL_PWD'] = config["password"].to_s if config["password"] + spawn('mysql', *mysql_args, config['database'], in: decompress_rd) + when "postgresql" then + $progress.print "Restoring PostgreSQL database #{config['database']} ... " + pg_env + spawn('psql', config['database'], in: decompress_rd) + end decompress_rd.close - success = [decompress_pid, restore_pid].all? { |pid| Process.waitpid(pid); $?.success? } + success = [decompress_pid, restore_pid].all? do |pid| + Process.waitpid(pid) + $?.success? + end report_success(success) abort 'Restore failed' unless success diff --git a/lib/backup/files.rb b/lib/backup/files.rb index cedbb289f6a..30a91647b77 100644 --- a/lib/backup/files.rb +++ b/lib/backup/files.rb @@ -8,6 +8,7 @@ module Backup @name = name @app_files_dir = File.realpath(app_files_dir) @files_parent_dir = File.realpath(File.join(@app_files_dir, '..')) + @backup_files_dir = File.join(Gitlab.config.backup.path, File.basename(@app_files_dir) ) @backup_tarball = File.join(Gitlab.config.backup.path, name + '.tar.gz') end @@ -15,14 +16,28 @@ module Backup def dump FileUtils.mkdir_p(Gitlab.config.backup.path) FileUtils.rm_f(backup_tarball) - run_pipeline!([%W(tar -C #{app_files_dir} -cf - .), %W(gzip -c -1)], out: [backup_tarball, 'w', 0600]) + + if ENV['STRATEGY'] == 'copy' + cmd = %W(cp -a #{app_files_dir} #{Gitlab.config.backup.path}) + output, status = Gitlab::Popen.popen(cmd) + + unless status.zero? + puts output + abort 'Backup failed' + end + + run_pipeline!([%W(tar -C #{@backup_files_dir} -cf - .), %w(gzip -c -1)], out: [backup_tarball, 'w', 0600]) + FileUtils.rm_rf(@backup_files_dir) + else + run_pipeline!([%W(tar -C #{app_files_dir} -cf - .), %w(gzip -c -1)], out: [backup_tarball, 'w', 0600]) + end end def restore backup_existing_files_dir create_files_dir - run_pipeline!([%W(gzip -cd), %W(tar -C #{app_files_dir} -xf -)], in: backup_tarball) + run_pipeline!([%w(gzip -cd), %W(tar -C #{app_files_dir} -xf -)], in: backup_tarball) end def backup_existing_files_dir @@ -32,7 +47,7 @@ module Backup end end - def run_pipeline!(cmd_list, options={}) + def run_pipeline!(cmd_list, options = {}) status_list = Open3.pipeline(*cmd_list, options) abort 'Backup failed' unless status_list.compact.all?(&:success?) end diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 0dfffaf0bc6..5cc164a6325 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -1,7 +1,8 @@ module Backup class Manager - ARCHIVES_TO_BACKUP = %w[uploads builds artifacts lfs registry] - FOLDERS_TO_BACKUP = %w[repositories db] + ARCHIVES_TO_BACKUP = %w[uploads builds artifacts pages lfs registry].freeze + FOLDERS_TO_BACKUP = %w[repositories db].freeze + FILE_NAME_SUFFIX = '_gitlab_backup.tar'.freeze def pack # Make sure there is a connection @@ -14,18 +15,18 @@ module Backup s[:gitlab_version] = Gitlab::VERSION s[:tar_version] = tar_version s[:skipped] = ENV["SKIP"] - tar_file = "#{s[:backup_created_at].to_i}_gitlab_backup.tar" + tar_file = "#{s[:backup_created_at].strftime('%s_%Y_%m_%d')}#{FILE_NAME_SUFFIX}" Dir.chdir(Gitlab.config.backup.path) do File.open("#{Gitlab.config.backup.path}/backup_information.yml", "w+") do |file| - file << s.to_yaml.gsub(/^---\n/,'') + file << s.to_yaml.gsub(/^---\n/, '') end # create archive $progress.print "Creating backup archive: #{tar_file} ... " # Set file permissions on open to prevent chmod races. - tar_system_options = {out: [tar_file, 'w', Gitlab.config.backup.archive_permissions]} + tar_system_options = { out: [tar_file, 'w', Gitlab.config.backup.archive_permissions] } if Kernel.system('tar', '-cf', '-', *backup_contents, tar_system_options) $progress.puts "done".color(:green) else @@ -49,8 +50,8 @@ module Backup directory = connect_to_remote_directory(connection_settings) if directory.files.create(key: tar_file, body: File.open(tar_file), public: false, - multipart_chunk_size: Gitlab.config.backup.upload.multipart_chunk_size, - encryption: Gitlab.config.backup.upload.encryption) + multipart_chunk_size: Gitlab.config.backup.upload.multipart_chunk_size, + encryption: Gitlab.config.backup.upload.encryption) $progress.puts "done".color(:green) else puts "uploading backup to #{remote_directory} failed".color(:red) @@ -82,12 +83,17 @@ module Backup removed = 0 Dir.chdir(Gitlab.config.backup.path) do - file_list = Dir.glob('*_gitlab_backup.tar') - file_list.map! { |f| $1.to_i if f =~ /(\d+)_gitlab_backup.tar/ } - file_list.sort.each do |timestamp| + Dir.glob("*#{FILE_NAME_SUFFIX}").each do |file| + next unless file =~ /(\d+)(?:_\d{4}_\d{2}_\d{2})?_gitlab_backup\.tar/ + + timestamp = $1.to_i + if Time.at(timestamp) < (Time.now - keep_time) - if Kernel.system(*%W(rm #{timestamp}_gitlab_backup.tar)) + begin + FileUtils.rm(file) removed += 1 + rescue => e + $progress.puts "Deleting #{file} failed: #{e.message}".color(:red) end end end @@ -103,47 +109,56 @@ module Backup Dir.chdir(Gitlab.config.backup.path) # check for existing backups in the backup dir - file_list = Dir.glob("*_gitlab_backup.tar").each.map { |f| f.split(/_/).first.to_i } - puts "no backups found" if file_list.count == 0 + file_list = Dir.glob("*#{FILE_NAME_SUFFIX}") + + if file_list.count == 0 + $progress.puts "No backups found in #{Gitlab.config.backup.path}" + $progress.puts "Please make sure that file name ends with #{FILE_NAME_SUFFIX}" + exit 1 + end if file_list.count > 1 && ENV["BACKUP"].nil? - puts "Found more than one backup, please specify which one you want to restore:" - puts "rake gitlab:backup:restore BACKUP=timestamp_of_backup" + $progress.puts 'Found more than one backup, please specify which one you want to restore:' + $progress.puts 'rake gitlab:backup:restore BACKUP=timestamp_of_backup' exit 1 end - tar_file = ENV["BACKUP"].nil? ? File.join("#{file_list.first}_gitlab_backup.tar") : File.join(ENV["BACKUP"] + "_gitlab_backup.tar") + tar_file = if ENV['BACKUP'].present? + "#{ENV['BACKUP']}#{FILE_NAME_SUFFIX}" + else + file_list.first + end unless File.exist?(tar_file) - puts "The specified backup doesn't exist!" + $progress.puts "The backup file #{tar_file} does not exist!" exit 1 end - $progress.print "Unpacking backup ... " + $progress.print 'Unpacking backup ... ' unless Kernel.system(*%W(tar -xf #{tar_file})) - puts "unpacking backup failed".color(:red) + $progress.puts 'unpacking backup failed'.color(:red) exit 1 else - $progress.puts "done".color(:green) + $progress.puts 'done'.color(:green) end ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0 # restoring mismatching backups can lead to unexpected problems if settings[:gitlab_version] != Gitlab::VERSION - puts "GitLab version mismatch:".color(:red) - puts " Your current GitLab version (#{Gitlab::VERSION}) differs from the GitLab version in the backup!".color(:red) - puts " Please switch to the following version and try again:".color(:red) - puts " version: #{settings[:gitlab_version]}".color(:red) - puts - puts "Hint: git checkout v#{settings[:gitlab_version]}" + $progress.puts 'GitLab version mismatch:'.color(:red) + $progress.puts " Your current GitLab version (#{Gitlab::VERSION}) differs from the GitLab version in the backup!".color(:red) + $progress.puts ' Please switch to the following version and try again:'.color(:red) + $progress.puts " version: #{settings[:gitlab_version]}".color(:red) + $progress.puts + $progress.puts "Hint: git checkout v#{settings[:gitlab_version]}" exit 1 end end def tar_version - tar_version, _ = Gitlab::Popen.popen(%W(tar --version)) + tar_version, _ = Gitlab::Popen.popen(%w(tar --version)) tar_version.force_encoding('locale').split("\n").first end diff --git a/lib/backup/pages.rb b/lib/backup/pages.rb new file mode 100644 index 00000000000..215ded93bfe --- /dev/null +++ b/lib/backup/pages.rb @@ -0,0 +1,13 @@ +require 'backup/files' + +module Backup + class Pages < Files + def initialize + super('pages', Gitlab.config.pages.path) + end + + def create_files_dir + Dir.mkdir(app_files_dir, 0700) + end + end +end diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index f117fc3d37d..d16d5ba4960 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -2,20 +2,37 @@ require 'yaml' module Backup class Repository + # rubocop:disable Metrics/AbcSize def dump prepare Project.find_each(batch_size: 1000) do |project| $progress.print " * #{project.path_with_namespace} ... " + path_to_project_repo = path_to_repo(project) + path_to_project_bundle = path_to_bundle(project) # Create namespace dir if missing - FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.path)) if project.namespace + FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.full_path)) if project.namespace if project.empty_repo? $progress.puts "[SKIPPED]".color(:cyan) else - cmd = %W(tar -cf #{path_to_bundle(project)} -C #{path_to_repo(project)} .) + in_path(path_to_project_repo) do |dir| + FileUtils.mkdir_p(path_to_tars(project)) + cmd = %W(tar -cf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir}) + output, status = Gitlab::Popen.popen(cmd) + + unless status.zero? + puts "[FAILED]".color(:red) + puts "failed: #{cmd.join(' ')}" + puts output + abort 'Backup failed' + end + end + + cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_project_repo} bundle create #{path_to_project_bundle} --all) output, status = Gitlab::Popen.popen(cmd) + if status.zero? $progress.puts "[DONE]".color(:green) else @@ -27,19 +44,22 @@ module Backup end wiki = ProjectWiki.new(project) + path_to_wiki_repo = path_to_repo(wiki) + path_to_wiki_bundle = path_to_bundle(wiki) - if File.exist?(path_to_repo(wiki)) + if File.exist?(path_to_wiki_repo) $progress.print " * #{wiki.path_with_namespace} ... " if wiki.repository.empty? $progress.puts " [SKIPPED]".color(:cyan) else - cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_repo(wiki)} bundle create #{path_to_bundle(wiki)} --all) + cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_wiki_repo} bundle create #{path_to_wiki_bundle} --all) output, status = Gitlab::Popen.popen(cmd) if status.zero? $progress.puts " [DONE]".color(:green) else puts " [FAILED]".color(:red) puts "failed: #{cmd.join(' ')}" + puts output abort 'Backup failed' end end @@ -55,45 +75,64 @@ module Backup bk_repos_path = File.join(path, '..', 'repositories.old.' + Time.now.to_i.to_s) FileUtils.mv(path, bk_repos_path) # This is expected from gitlab:check - FileUtils.mkdir_p(path, mode: 2770) + FileUtils.mkdir_p(path, mode: 02770) end Project.find_each(batch_size: 1000) do |project| $progress.print " * #{project.path_with_namespace} ... " + path_to_project_repo = path_to_repo(project) + path_to_project_bundle = path_to_bundle(project) project.ensure_dir_exist - if File.exist?(path_to_bundle(project)) - FileUtils.mkdir_p(path_to_repo(project)) - cmd = %W(tar -xf #{path_to_bundle(project)} -C #{path_to_repo(project)}) - else - cmd = %W(#{Gitlab.config.git.bin_path} init --bare #{path_to_repo(project)}) - end + cmd = if File.exist?(path_to_project_bundle) + %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_project_bundle} #{path_to_project_repo}) + else + %W(#{Gitlab.config.git.bin_path} init --bare #{path_to_project_repo}) + end - if system(*cmd, silent) + output, status = Gitlab::Popen.popen(cmd) + if status.zero? $progress.puts "[DONE]".color(:green) else puts "[FAILED]".color(:red) puts "failed: #{cmd.join(' ')}" + puts output abort 'Restore failed' end + in_path(path_to_tars(project)) do |dir| + cmd = %W(tar -xf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir}) + + output, status = Gitlab::Popen.popen(cmd) + unless status.zero? + puts "[FAILED]".color(:red) + puts "failed: #{cmd.join(' ')}" + puts output + abort 'Restore failed' + end + end + wiki = ProjectWiki.new(project) + path_to_wiki_repo = path_to_repo(wiki) + path_to_wiki_bundle = path_to_bundle(wiki) - if File.exist?(path_to_bundle(wiki)) + if File.exist?(path_to_wiki_bundle) $progress.print " * #{wiki.path_with_namespace} ... " # If a wiki bundle exists, first remove the empty repo # that was initialized with ProjectWiki.new() and then # try to restore with 'git clone --bare'. - FileUtils.rm_rf(path_to_repo(wiki)) - cmd = %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_bundle(wiki)} #{path_to_repo(wiki)}) + FileUtils.rm_rf(path_to_wiki_repo) + cmd = %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_wiki_bundle} #{path_to_wiki_repo}) - if system(*cmd, silent) + output, status = Gitlab::Popen.popen(cmd) + if status.zero? $progress.puts " [DONE]".color(:green) else puts " [FAILED]".color(:red) puts "failed: #{cmd.join(' ')}" + puts output abort 'Restore failed' end end @@ -101,14 +140,17 @@ module Backup $progress.print 'Put GitLab hooks in repositories dirs'.color(:yellow) cmd = %W(#{Gitlab.config.gitlab_shell.path}/bin/create-hooks) + repository_storage_paths_args - if system(*cmd) + + output, status = Gitlab::Popen.popen(cmd) + if status.zero? $progress.puts " [DONE]".color(:green) else puts " [FAILED]".color(:red) puts "failed: #{cmd}" + puts output end - end + # rubocop:enable Metrics/AbcSize protected @@ -117,11 +159,30 @@ module Backup end def path_to_bundle(project) - File.join(backup_repos_path, project.path_with_namespace + ".bundle") + File.join(backup_repos_path, project.path_with_namespace + '.bundle') + end + + def path_to_tars(project, dir = nil) + path = File.join(backup_repos_path, project.path_with_namespace) + + if dir + File.join(path, "#{dir}.tar") + else + path + end end def backup_repos_path - File.join(Gitlab.config.backup.path, "repositories") + File.join(Gitlab.config.backup.path, 'repositories') + end + + def in_path(path) + return unless Dir.exist?(path) + + dir_entries = Dir.entries(path) + %w[annex custom_hooks].each do |entry| + yield(entry) if dir_entries.include?(entry) + end end def prepare @@ -133,7 +194,7 @@ module Backup end def silent - {err: '/dev/null', out: '/dev/null'} + { err: '/dev/null', out: '/dev/null' } end private diff --git a/lib/backup/uploads.rb b/lib/backup/uploads.rb index 9261f77f3c9..35118375499 100644 --- a/lib/backup/uploads.rb +++ b/lib/backup/uploads.rb @@ -2,7 +2,6 @@ require 'backup/files' module Backup class Uploads < Files - def initialize super('uploads', Rails.root.join('public/uploads')) end diff --git a/lib/banzai.rb b/lib/banzai.rb index 9ebe379f454..35ca234c1ba 100644 --- a/lib/banzai.rb +++ b/lib/banzai.rb @@ -3,6 +3,10 @@ module Banzai Renderer.render(text, context) end + def self.render_field(object, field) + Renderer.render_field(object, field) + end + def self.cache_collection_render(texts_and_contexts) Renderer.cache_collection_render(texts_and_contexts) end diff --git a/lib/banzai/cross_project_reference.rb b/lib/banzai/cross_project_reference.rb index 0257848b6bc..e2b57adf611 100644 --- a/lib/banzai/cross_project_reference.rb +++ b/lib/banzai/cross_project_reference.rb @@ -14,7 +14,7 @@ module Banzai def project_from_ref(ref) return context[:project] unless ref - Project.find_with_namespace(ref) + Project.find_by_full_path(ref) end end end diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index d77a5e3ff09..02d5ad70fa7 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -18,10 +18,6 @@ module Banzai @object_sym ||= object_name.to_sym end - def self.object_class_title - @object_title ||= object_class.name.titleize - end - # Public: Find references in text (like `!123` for merge requests) # # AnyReferenceFilter.references_in(text) do |match, id, project_ref, matches| @@ -37,7 +33,12 @@ module Banzai # Returns a String replaced with the return of the block. def self.references_in(text, pattern = object_class.reference_pattern) text.gsub(pattern) do |match| - yield match, $~[object_sym].to_i, $~[:project], $~ + symbol = $~[object_sym] + if object_class.reference_valid?(symbol) + yield match, symbol.to_i, $~[:project], $~[:namespace], $~ + else + match + end end end @@ -49,10 +50,6 @@ module Banzai self.class.object_sym end - def object_class_title - self.class.object_class_title - end - def references_in(*args, &block) self.class.references_in(*args, &block) end @@ -72,7 +69,7 @@ module Banzai end end - def project_from_ref_cache(ref) + def project_from_ref_cached(ref) if RequestStore.active? cache = project_refs_cache @@ -110,10 +107,10 @@ module Banzai end elsif element_node?(node) - yield_valid_link(node) do |link, text| + yield_valid_link(node) do |link, inner_html| if ref_pattern && link =~ /\A#{ref_pattern}\z/ replace_link_node_with_href(node, link) do - object_link_filter(link, ref_pattern, link_text: text) + object_link_filter(link, ref_pattern, link_content: inner_html) end next @@ -121,9 +118,9 @@ module Banzai next unless link_pattern - if link == text && text =~ /\A#{link_pattern}/ + if link == inner_html && inner_html =~ /\A#{link_pattern}/ replace_link_node_with_text(node, link) do - object_link_filter(text, link_pattern) + object_link_filter(inner_html, link_pattern) end next @@ -131,7 +128,7 @@ module Banzai if link =~ /\A#{link_pattern}\z/ replace_link_node_with_href(node, link) do - object_link_filter(link, link_pattern, link_text: text) + object_link_filter(link, link_pattern, link_content: inner_html) end next @@ -148,40 +145,43 @@ module Banzai # # text - String text to replace references in. # pattern - Reference pattern to match against. - # link_text - Original content of the link being replaced. + # link_content - Original content of the link being replaced. # # Returns a String with references replaced with links. All links # have `gfm` and `gfm-OBJECT_NAME` class names attached for styling. - def object_link_filter(text, pattern, link_text: nil) - references_in(text, pattern) do |match, id, project_ref, matches| - project = project_from_ref_cache(project_ref) + def object_link_filter(text, pattern, link_content: nil) + references_in(text, pattern) do |match, id, project_ref, namespace_ref, matches| + project_path = full_project_path(namespace_ref, project_ref) + project = project_from_ref_cached(project_path) if project && object = find_object_cached(project, id) title = object_link_title(object) klass = reference_class(object_sym) - data = data_attributes_for(link_text || match, project, object) + data = data_attributes_for(link_content || match, project, object, link: !!link_content) - if matches.names.include?("url") && matches[:url] - url = matches[:url] - else - url = url_for_object_cached(object, project) - end + url = + if matches.names.include?("url") && matches[:url] + matches[:url] + else + url_for_object_cached(object, project) + end - text = link_text || object_link_text(object, matches) + content = link_content || object_link_text(object, matches) %(<a href="#{url}" #{data} title="#{escape_once(title)}" - class="#{klass}">#{escape_once(text)}</a>) + class="#{klass}">#{content}</a>) else match end end end - def data_attributes_for(text, project, object) + def data_attributes_for(text, project, object, link: false) data_attribute( original: text, + link: link, project: project.id, object_sym => object.id ) @@ -198,7 +198,7 @@ module Banzai end def object_link_title(object) - "#{object_class_title}: #{object.title}" + object.title end def object_link_text(object, matches) @@ -216,15 +216,18 @@ module Banzai @references_per_project ||= begin refs = Hash.new { |hash, key| hash[key] = Set.new } - regex = Regexp.union(object_class.reference_pattern, - object_class.link_reference_pattern) + regex = + if uses_reference_pattern? + Regexp.union(object_class.reference_pattern, object_class.link_reference_pattern) + else + object_class.link_reference_pattern + end nodes.each do |node| node.to_html.scan(regex) do - project = $~[:project] || current_project_path + project_path = full_project_path($~[:namespace], $~[:project]) symbol = $~[object_sym] - - refs[project] << symbol if object_class.reference_valid?(symbol) + refs[project_path] << symbol if object_class.reference_valid?(symbol) end end @@ -251,17 +254,55 @@ module Banzai end end - # Returns the projects for the given paths. + def projects_relation_for_paths(paths) + Project.where_full_path_in(paths).includes(:namespace) + end + + # Returns projects for the given paths. def find_projects_for_paths(paths) - Project.where_paths_in(paths).includes(:namespace) + if RequestStore.active? + cache = project_refs_cache + to_query = paths - cache.keys + + unless to_query.empty? + projects = projects_relation_for_paths(to_query) + + found = [] + projects.each do |project| + ref = project.path_with_namespace + get_or_set_cache(cache, ref) { project } + found << ref + end + + not_found = to_query - found + not_found.each do |ref| + get_or_set_cache(cache, ref) { nil } + end + end + + cache.slice(*paths).values.compact + else + projects_relation_for_paths(paths) + end end def current_project_path @current_project_path ||= project.path_with_namespace end + def current_project_namespace_path + @current_project_namespace_path ||= project.namespace.full_path + end + private + def full_project_path(namespace, project_ref) + return current_project_path unless project_ref + + namespace_ref = namespace || current_project_namespace_path + "#{namespace_ref}/#{project_ref}" + end + def project_refs_cache RequestStore[:banzai_project_refs] ||= {} end @@ -287,6 +328,14 @@ module Banzai value end end + + # There might be special cases like filters + # that should ignore reference pattern + # eg: IssueReferenceFilter when using a external issues tracker + # In those cases this method should be overridden on the filter subclass + def uses_reference_pattern? + true + end end end end diff --git a/lib/banzai/filter/autolink_filter.rb b/lib/banzai/filter/autolink_filter.rb index 799b83b1069..b8d2673c1a6 100644 --- a/lib/banzai/filter/autolink_filter.rb +++ b/lib/banzai/filter/autolink_filter.rb @@ -37,7 +37,7 @@ module Banzai and contains(., '://') and not(starts-with(., 'http')) and not(starts-with(., 'ftp')) - ]) + ]).freeze def call return doc if context[:autolink] == false @@ -71,6 +71,14 @@ module Banzai @doc = parse_html(rinku) end + # Return true if any of the UNSAFE_PROTOCOLS strings are included in the URI scheme + def contains_unsafe?(scheme) + return false unless scheme + + scheme = scheme.strip.downcase + Banzai::Filter::SanitizationFilter::UNSAFE_PROTOCOLS.any? { |protocol| scheme.include?(protocol) } + end + # Autolinks any text matching LINK_PATTERN that Rinku didn't already # replace def text_parse @@ -89,17 +97,27 @@ module Banzai doc end - def autolink_filter(text) - text.gsub(LINK_PATTERN) do |match| - # Remove any trailing HTML entities and store them for appending - # outside the link element. The entity must be marked HTML safe in - # order to be output literally rather than escaped. - match.gsub!(/((?:&[\w#]+;)+)\z/, '') - dropped = ($1 || '').html_safe - - options = link_options.merge(href: match) - content_tag(:a, match, options) + dropped + def autolink_match(match) + # start by stripping out dangerous links + begin + uri = Addressable::URI.parse(match) + return match if contains_unsafe?(uri.scheme) + rescue Addressable::URI::InvalidURIError + return match end + + # Remove any trailing HTML entities and store them for appending + # outside the link element. The entity must be marked HTML safe in + # order to be output literally rather than escaped. + match.gsub!(/((?:&[\w#]+;)+)\z/, '') + dropped = ($1 || '').html_safe + + options = link_options.merge(href: match) + content_tag(:a, match, options) + dropped + end + + def autolink_filter(text) + text.gsub(LINK_PATTERN) { |match| autolink_match(match) } end def link_options diff --git a/lib/banzai/filter/commit_range_reference_filter.rb b/lib/banzai/filter/commit_range_reference_filter.rb index bbb88c979cc..eaacb9591b1 100644 --- a/lib/banzai/filter/commit_range_reference_filter.rb +++ b/lib/banzai/filter/commit_range_reference_filter.rb @@ -12,7 +12,7 @@ module Banzai def self.references_in(text, pattern = CommitRange.reference_pattern) text.gsub(pattern) do |match| - yield match, $~[:commit_range], $~[:project], $~ + yield match, $~[:commit_range], $~[:project], $~[:namespace], $~ end end @@ -35,7 +35,7 @@ module Banzai end def object_link_title(range) - range.reference_title + nil end end end diff --git a/lib/banzai/filter/commit_reference_filter.rb b/lib/banzai/filter/commit_reference_filter.rb index 2ce1816672b..69c06117eda 100644 --- a/lib/banzai/filter/commit_reference_filter.rb +++ b/lib/banzai/filter/commit_reference_filter.rb @@ -12,7 +12,7 @@ module Banzai def self.references_in(text, pattern = Commit.reference_pattern) text.gsub(pattern) do |match| - yield match, $~[:commit], $~[:project], $~ + yield match, $~[:commit], $~[:project], $~[:namespace], $~ end end @@ -28,10 +28,6 @@ module Banzai only_path: context[:only_path]) end - def object_link_title(commit) - commit.link_title - end - def object_link_text_extras(object, matches) extras = super diff --git a/lib/banzai/filter/emoji_filter.rb b/lib/banzai/filter/emoji_filter.rb index 2492b5213ac..a8c1ca0c60a 100644 --- a/lib/banzai/filter/emoji_filter.rb +++ b/lib/banzai/filter/emoji_filter.rb @@ -1,6 +1,6 @@ module Banzai module Filter - # HTML filter that replaces :emoji: with images. + # HTML filter that replaces :emoji: and unicode with images. # # Based on HTML::Pipeline::EmojiFilter # @@ -13,16 +13,17 @@ module Banzai def call search_text_nodes(doc).each do |node| content = node.to_html - next unless content.include?(':') next if has_ancestor?(node, IGNORED_ANCESTOR_TAGS) - html = emoji_image_filter(content) + next unless content.include?(':') || node.text.match(emoji_unicode_pattern) + + html = emoji_name_image_filter(content) + html = emoji_unicode_image_filter(html) next if html == content node.replace(html) end - doc end @@ -31,18 +32,38 @@ module Banzai # text - String text to replace :emoji: in. # # Returns a String with :emoji: replaced with images. - def emoji_image_filter(text) + def emoji_name_image_filter(text) text.gsub(emoji_pattern) do |match| name = $1 - "<img class='emoji' title=':#{name}:' alt=':#{name}:' src='#{emoji_url(name)}' height='20' width='20' align='absmiddle' />" + emoji_image_tag(name, emoji_url(name)) end end + # Replace unicode emoji with corresponding images if they exist. + # + # text - String text to replace unicode emoji in. + # + # Returns a String with unicode emoji replaced with images. + def emoji_unicode_image_filter(text) + text.gsub(emoji_unicode_pattern) do |moji| + emoji_image_tag(Gitlab::Emoji.emojis_by_moji[moji]['name'], emoji_unicode_url(moji)) + end + end + + def emoji_image_tag(emoji_name, emoji_url) + "<img class='emoji' title=':#{emoji_name}:' alt=':#{emoji_name}:' src='#{emoji_url}' height='20' width='20' align='absmiddle' />" + end + # Build a regexp that matches all valid :emoji: names. def self.emoji_pattern @emoji_pattern ||= /:(#{Gitlab::Emoji.emojis_names.map { |name| Regexp.escape(name) }.join('|')}):/ end + # Build a regexp that matches all valid unicode emojis names. + def self.emoji_unicode_pattern + @emoji_unicode_pattern ||= /(#{Gitlab::Emoji.emojis_unicodes.map { |moji| Regexp.escape(moji) }.join('|')})/ + end + private def emoji_url(name) @@ -60,6 +81,18 @@ module Banzai end end + def emoji_unicode_url(moji) + emoji_unicode_path = emoji_unicode_filename(moji) + + if context[:asset_host] + url_to_image(emoji_unicode_path) + elsif context[:asset_root] + File.join(context[:asset_root], url_to_image(emoji_unicode_path)) + else + url_to_image(emoji_unicode_path) + end + end + def url_to_image(image) ActionController::Base.helpers.url_to_image(image) end @@ -71,6 +104,14 @@ module Banzai def emoji_filename(name) "#{Gitlab::Emoji.emoji_filename(name)}.png" end + + def emoji_unicode_pattern + self.class.emoji_unicode_pattern + end + + def emoji_unicode_filename(name) + "#{Gitlab::Emoji.emoji_unicode_filename(name)}.png" + end end end end diff --git a/lib/banzai/filter/external_issue_reference_filter.rb b/lib/banzai/filter/external_issue_reference_filter.rb index eaa702952cc..dce4de3ceaf 100644 --- a/lib/banzai/filter/external_issue_reference_filter.rb +++ b/lib/banzai/filter/external_issue_reference_filter.rb @@ -8,7 +8,7 @@ module Banzai # Public: Find `JIRA-123` issue references in text # - # ExternalIssueReferenceFilter.references_in(text) do |match, issue| + # ExternalIssueReferenceFilter.references_in(text, pattern) do |match, issue| # "<a href=...>##{issue}</a>" # end # @@ -17,8 +17,8 @@ module Banzai # Yields the String match and the String issue reference. # # Returns a String replaced with the return of the block. - def self.references_in(text) - text.gsub(ExternalIssue.reference_pattern) do |match| + def self.references_in(text, pattern) + text.gsub(pattern) do |match| yield match, $~[:issue] end end @@ -27,7 +27,7 @@ module Banzai # Early return if the project isn't using an external tracker return doc if project.nil? || default_issues_tracker? - ref_pattern = ExternalIssue.reference_pattern + ref_pattern = issue_reference_pattern ref_start_pattern = /\A#{ref_pattern}\z/ each_node do |node| @@ -37,10 +37,10 @@ module Banzai end elsif element_node?(node) - yield_valid_link(node) do |link, text| + yield_valid_link(node) do |link, inner_html| if link =~ ref_start_pattern replace_link_node_with_href(node, link) do - issue_link_filter(link, link_text: text) + issue_link_filter(link, link_content: inner_html) end end end @@ -54,13 +54,14 @@ module Banzai # issue's details page. # # text - String text to replace references in. + # link_content - Original content of the link being replaced. # # Returns a String with `JIRA-123` references replaced with links. All # links have `gfm` and `gfm-issue` class names attached for styling. - def issue_link_filter(text, link_text: nil) + def issue_link_filter(text, link_content: nil) project = context[:project] - self.class.references_in(text) do |match, id| + self.class.references_in(text, issue_reference_pattern) do |match, id| ExternalIssue.new(id, project) url = url_for_issue(id, project, only_path: context[:only_path]) @@ -69,11 +70,11 @@ module Banzai klass = reference_class(:issue) data = data_attribute(project: project.id, external_issue: id) - text = link_text || match + content = link_content || match %(<a href="#{url}" #{data} title="#{escape_once(title)}" - class="#{klass}">#{escape_once(text)}</a>) + class="#{klass}">#{content}</a>) end end @@ -82,18 +83,21 @@ module Banzai end def default_issues_tracker? - if RequestStore.active? - default_issues_tracker_cache[project.id] ||= - project.default_issues_tracker? - else - project.default_issues_tracker? - end + external_issues_cached(:default_issues_tracker?) + end + + def issue_reference_pattern + external_issues_cached(:issue_reference_pattern) end private - def default_issues_tracker_cache - RequestStore[:banzai_default_issues_tracker_cache] ||= {} + def external_issues_cached(attribute) + return project.public_send(attribute) unless RequestStore.active? + + cached_attributes = RequestStore[:banzai_external_issues_tracker_attributes] ||= Hash.new { |h, k| h[k] = {} } + cached_attributes[project.id][attribute] = project.public_send(attribute) if cached_attributes[project.id][attribute].nil? + cached_attributes[project.id][attribute] end end end diff --git a/lib/banzai/filter/external_link_filter.rb b/lib/banzai/filter/external_link_filter.rb index 0a29c547a4d..d67d466bce8 100644 --- a/lib/banzai/filter/external_link_filter.rb +++ b/lib/banzai/filter/external_link_filter.rb @@ -3,10 +3,17 @@ module Banzai # HTML Filter to modify the attributes of external links class ExternalLinkFilter < HTML::Pipeline::Filter def call - # Skip non-HTTP(S) links and internal links - doc.xpath("descendant-or-self::a[starts-with(@href, 'http') and not(starts-with(@href, '#{internal_url}'))]").each do |node| - node.set_attribute('rel', 'nofollow noreferrer') - node.set_attribute('target', '_blank') + links.each do |node| + href = href_to_lowercase_scheme(node["href"].to_s) + + unless node["href"].to_s == href + node.set_attribute('href', href) + end + + if href =~ %r{\A(https?:)?//[^/]} && external_url?(href) + node.set_attribute('rel', 'nofollow noreferrer') + node.set_attribute('target', '_blank') + end end doc @@ -14,6 +21,25 @@ module Banzai private + def links + query = 'descendant-or-self::a[@href and not(@href = "")]' + doc.xpath(query) + end + + def href_to_lowercase_scheme(href) + scheme_match = href.match(/\A(\w+):\/\//) + + if scheme_match + scheme_match.to_s.downcase + scheme_match.post_match + else + href + end + end + + def external_url?(url) + !url.start_with?(internal_url) + end + def internal_url @internal_url ||= Gitlab.config.gitlab.url end diff --git a/lib/banzai/filter/gollum_tags_filter.rb b/lib/banzai/filter/gollum_tags_filter.rb index d08267a9d6c..0ea4eeaed5b 100644 --- a/lib/banzai/filter/gollum_tags_filter.rb +++ b/lib/banzai/filter/gollum_tags_filter.rb @@ -149,11 +149,12 @@ module Banzai name, reference = *parts.compact.map(&:strip) end - if url?(reference) - href = reference - else - href = ::File.join(project_wiki_base_path, reference) - end + href = + if url?(reference) + reference + else + ::File.join(project_wiki_base_path, reference) + end content_tag(:a, name || reference, href: href, class: 'gfm') end diff --git a/lib/banzai/filter/html_entity_filter.rb b/lib/banzai/filter/html_entity_filter.rb new file mode 100644 index 00000000000..f3bd587c28b --- /dev/null +++ b/lib/banzai/filter/html_entity_filter.rb @@ -0,0 +1,12 @@ +require 'erb' + +module Banzai + module Filter + # Text filter that escapes these HTML entities: & " < > + class HtmlEntityFilter < HTML::Pipeline::TextFilter + def call + ERB::Util.html_escape_once(text) + end + end + end +end diff --git a/lib/banzai/filter/issue_reference_filter.rb b/lib/banzai/filter/issue_reference_filter.rb index 4042e9a4c25..044d18ff824 100644 --- a/lib/banzai/filter/issue_reference_filter.rb +++ b/lib/banzai/filter/issue_reference_filter.rb @@ -4,6 +4,10 @@ module Banzai # issues that do not exist are ignored. # # This filter supports cross-project references. + # + # When external issues tracker like Jira is activated we should not + # use issue reference pattern, but we should still be able + # to reference issues from other GitLab projects. class IssueReferenceFilter < AbstractReferenceFilter self.reference_type = :issue @@ -11,6 +15,10 @@ module Banzai Issue end + def uses_reference_pattern? + context[:project].default_issues_tracker? + end + def find_object(project, iid) issues_per_project[project][iid] end @@ -31,11 +39,12 @@ module Banzai projects_per_reference.each do |path, project| issue_ids = references_per_project[path] - if project.default_issues_tracker? - issues = project.issues.where(iid: issue_ids.to_a) - else - issues = issue_ids.map { |id| ExternalIssue.new(id, project) } - end + issues = + if project.default_issues_tracker? + project.issues.where(iid: issue_ids.to_a) + else + issue_ids.map { |id| ExternalIssue.new(id, project) } + end issues.each do |issue| hash[project][issue.iid.to_i] = issue @@ -54,7 +63,7 @@ module Banzai end end - def data_attributes_for(text, project, object) + def data_attributes_for(text, project, object, link: false) if object.is_a?(ExternalIssue) data_attribute( project: project.id, @@ -66,7 +75,7 @@ module Banzai end end - def find_projects_for_paths(paths) + def projects_relation_for_paths(paths) super(paths).includes(:gitlab_issue_tracker_service) end end diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb index e258dc8e2bf..a605dea149e 100644 --- a/lib/banzai/filter/label_reference_filter.rb +++ b/lib/banzai/filter/label_reference_filter.rb @@ -9,21 +9,23 @@ module Banzai end def find_object(project, id) - project.labels.find(id) + find_labels(project).find(id) end def self.references_in(text, pattern = Label.reference_pattern) unescape_html_entities(text).gsub(pattern) do |match| - yield match, $~[:label_id].to_i, $~[:label_name], $~[:project], $~ + yield match, $~[:label_id].to_i, $~[:label_name], $~[:project], $~[:namespace], $~ end end def references_in(text, pattern = Label.reference_pattern) unescape_html_entities(text).gsub(pattern) do |match| - label = find_label($~[:project], $~[:label_id], $~[:label_name]) + namespace, project = $~[:namespace], $~[:project] + project_path = full_project_path(namespace, project) + label = find_label(project_path, $~[:label_id], $~[:label_name]) if label - yield match, label.id, $~[:project], $~ + yield match, label.id, project, namespace, $~ else match end @@ -35,7 +37,11 @@ module Banzai return unless project label_params = label_params(label_id, label_name) - project.labels.find_by(label_params) + find_labels(project).find_by(label_params) + end + + def find_labels(project) + LabelsFinder.new(nil, project_id: project.id).execute(skip_authorization: true) end # Parameters to pass to `Label.find_by` based on the given arguments @@ -60,16 +66,22 @@ module Banzai end def object_link_text(object, matches) - if context[:project] == object.project - LabelsHelper.render_colored_label(object) - else - LabelsHelper.render_colored_cross_project_label(object) - end + project_path = full_project_path(matches[:namespace], matches[:project]) + project_from_ref = project_from_ref_cached(project_path) + reference = project_from_ref.to_human_reference(project) + label_suffix = " <i>in #{reference}</i>" if reference.present? + + LabelsHelper.render_colored_label(object, label_suffix) end def unescape_html_entities(text) CGI.unescapeHTML(text.to_s) end + + def object_link_title(object) + # use title of wrapped element instead + nil + end end end end diff --git a/lib/banzai/filter/math_filter.rb b/lib/banzai/filter/math_filter.rb new file mode 100644 index 00000000000..b6e784c886b --- /dev/null +++ b/lib/banzai/filter/math_filter.rb @@ -0,0 +1,46 @@ +require 'uri' + +module Banzai + module Filter + # HTML filter that adds class="code math" and removes the dollar sign in $`2+2`$. + # + class MathFilter < HTML::Pipeline::Filter + # Attribute indicating inline or display math. + STYLE_ATTRIBUTE = 'data-math-style'.freeze + + # Class used for tagging elements that should be rendered + TAG_CLASS = 'js-render-math'.freeze + + INLINE_CLASSES = "code math #{TAG_CLASS}".freeze + + DOLLAR_SIGN = '$'.freeze + + def call + doc.css('code').each do |code| + closing = code.next + opening = code.previous + + # We need a sibling before and after. + # They should end and start with $ respectively. + if closing && opening && + closing.text? && opening.text? && + closing.content.first == DOLLAR_SIGN && + opening.content.last == DOLLAR_SIGN + + code[:class] = INLINE_CLASSES + code[STYLE_ATTRIBUTE] = 'inline' + closing.content = closing.content[1..-1] + opening.content = opening.content[0..-2] + end + end + + doc.css('pre.code.math').each do |el| + el[STYLE_ATTRIBUTE] = 'display' + el[:class] += " #{TAG_CLASS}" + end + + doc + end + end + end +end diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb index ca686c87d97..f12014e191f 100644 --- a/lib/banzai/filter/milestone_reference_filter.rb +++ b/lib/banzai/filter/milestone_reference_filter.rb @@ -19,18 +19,20 @@ module Banzai return super(text, pattern) if pattern != Milestone.reference_pattern text.gsub(pattern) do |match| - milestone = find_milestone($~[:project], $~[:milestone_iid], $~[:milestone_name]) + milestone = find_milestone($~[:project], $~[:namespace], $~[:milestone_iid], $~[:milestone_name]) if milestone - yield match, milestone.iid, $~[:project], $~ + yield match, milestone.iid, $~[:project], $~[:namespace], $~ else match end end end - def find_milestone(project_ref, milestone_id, milestone_name) - project = project_from_ref(project_ref) + def find_milestone(project_ref, namespace_ref, milestone_id, milestone_name) + project_path = full_project_path(namespace_ref, project_ref) + project = project_from_ref(project_path) + return unless project milestone_params = milestone_params(milestone_id, milestone_name) @@ -52,13 +54,19 @@ module Banzai end def object_link_text(object, matches) - if context[:project] == object.project - super + milestone_link = escape_once(super) + reference = object.project.to_reference(project) + + if reference.present? + "#{milestone_link} <i>in #{reference}</i>".html_safe else - "#{escape_once(super)} <i>in #{escape_once(object.project.path_with_namespace)}</i>". - html_safe + milestone_link end end + + def object_link_title(object) + nil + end end end end diff --git a/lib/banzai/filter/plantuml_filter.rb b/lib/banzai/filter/plantuml_filter.rb new file mode 100644 index 00000000000..b2537117558 --- /dev/null +++ b/lib/banzai/filter/plantuml_filter.rb @@ -0,0 +1,39 @@ +require "nokogiri" +require "asciidoctor-plantuml/plantuml" + +module Banzai + module Filter + # HTML that replaces all `code plantuml` tags with PlantUML img tags. + # + class PlantumlFilter < HTML::Pipeline::Filter + def call + return doc unless doc.at('pre.plantuml') && settings.plantuml_enabled + + plantuml_setup + + doc.css('pre.plantuml').each do |el| + img_tag = Nokogiri::HTML::DocumentFragment.parse( + Asciidoctor::PlantUml::Processor.plantuml_content(el.content, {})) + el.replace img_tag + end + + doc + end + + private + + def settings + ApplicationSetting.current || ApplicationSetting.create_from_defaults + end + + def plantuml_setup + Asciidoctor::PlantUml.configure do |conf| + conf.url = settings.plantuml_url + conf.png_enable = settings.plantuml_enabled + conf.svg_enable = false + conf.txt_enable = false + end + end + end + end +end diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index bf058241cda..6640168bfa2 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -20,10 +20,10 @@ module Banzai # Examples: # # data_attribute(project: 1, issue: 2) - # # => "data-reference-filter=\"SomeReferenceFilter\" data-project=\"1\" data-issue=\"2\"" + # # => "data-reference-type=\"SomeReferenceFilter\" data-project=\"1\" data-issue=\"2\"" # # data_attribute(project: 3, merge_request: 4) - # # => "data-reference-filter=\"SomeReferenceFilter\" data-project=\"3\" data-merge-request=\"4\"" + # # => "data-reference-type=\"SomeReferenceFilter\" data-project=\"3\" data-merge-request=\"4\"" # # Returns a String def data_attribute(attributes = {}) @@ -31,7 +31,9 @@ module Banzai attributes[:reference_type] ||= self.class.reference_type attributes.delete(:original) if context[:no_original_data] - attributes.map { |key, value| %Q(data-#{key.to_s.dasherize}="#{escape_once(value)}") }.join(" ") + attributes.map do |key, value| + %Q(data-#{key.to_s.dasherize}="#{escape_once(value)}") + end.join(' ') end def escape_once(html) @@ -51,8 +53,12 @@ module Banzai context[:project] end + def skip_project_check? + context[:skip_project_check] + end + def reference_class(type) - "gfm gfm-#{type}" + "gfm gfm-#{type} has-tooltip" end # Ensure that a :project key exists in context @@ -85,14 +91,14 @@ module Banzai @nodes ||= each_node.to_a end - # Yields the link's URL and text whenever the node is a valid <a> tag. + # Yields the link's URL and inner HTML whenever the node is a valid <a> tag. def yield_valid_link(node) link = CGI.unescape(node.attr('href').to_s) - text = node.text + inner_html = node.inner_html return unless link.force_encoding('UTF-8').valid_encoding? - yield link, text + yield link, inner_html end def replace_text_when_pattern_matches(node, pattern) diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb index 4fa8d05481f..9e23c8f8c55 100644 --- a/lib/banzai/filter/relative_link_filter.rb +++ b/lib/banzai/filter/relative_link_filter.rb @@ -46,21 +46,23 @@ module Banzai end def rebuild_relative_uri(uri) - file_path = relative_file_path(uri.path) + file_path = relative_file_path(uri) uri.path = [ relative_url_root, context[:project].path_with_namespace, uri_type(file_path), - ref, - file_path + Addressable::URI.escape(ref), + Addressable::URI.escape(file_path) ].compact.join('/').squeeze('/').chomp('/') uri end - def relative_file_path(path) - nested_path = build_relative_path(path, context[:requested_path]) + def relative_file_path(uri) + path = Addressable::URI.unescape(uri.path) + request_path = Addressable::URI.unescape(context[:requested_path]) + nested_path = build_relative_path(path, request_path) file_exists?(nested_path) ? nested_path : path end @@ -108,11 +110,7 @@ module Banzai end def uri_type(path) - @uri_types[path] ||= begin - unescaped_path = Addressable::URI.unescape(path) - - current_commit.uri_type(unescaped_path) - end + @uri_types[path] ||= current_commit.uri_type(path) end def current_commit diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index ca80aac5a08..af1e575fc89 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -25,7 +25,7 @@ module Banzai return if customized?(whitelist[:transformers]) # Allow code highlighting - whitelist[:attributes]['pre'] = %w(class) + whitelist[:attributes]['pre'] = %w(class v-pre) whitelist[:attributes]['span'] = %w(class) # Allow table alignment @@ -43,55 +43,57 @@ module Banzai whitelist[:protocols].delete('a') # ...but then remove links with unsafe protocols - whitelist[:transformers].push(remove_unsafe_links) + whitelist[:transformers].push(self.class.remove_unsafe_links) # Remove `rel` attribute from `a` elements - whitelist[:transformers].push(remove_rel) + whitelist[:transformers].push(self.class.remove_rel) # Remove `class` attribute from non-highlight spans - whitelist[:transformers].push(clean_spans) + whitelist[:transformers].push(self.class.clean_spans) whitelist end - def remove_unsafe_links - lambda do |env| - node = env[:node] + class << self + def remove_unsafe_links + lambda do |env| + node = env[:node] - return unless node.name == 'a' - return unless node.has_attribute?('href') + return unless node.name == 'a' + return unless node.has_attribute?('href') - begin - uri = Addressable::URI.parse(node['href']) - uri.scheme = uri.scheme.strip.downcase if uri.scheme + begin + uri = Addressable::URI.parse(node['href']) + uri.scheme = uri.scheme.strip.downcase if uri.scheme - node.remove_attribute('href') if UNSAFE_PROTOCOLS.include?(uri.scheme) - rescue Addressable::URI::InvalidURIError - node.remove_attribute('href') + node.remove_attribute('href') if UNSAFE_PROTOCOLS.include?(uri.scheme) + rescue Addressable::URI::InvalidURIError + node.remove_attribute('href') + end end end - end - def remove_rel - lambda do |env| - if env[:node_name] == 'a' - env[:node].remove_attribute('rel') + def remove_rel + lambda do |env| + if env[:node_name] == 'a' + env[:node].remove_attribute('rel') + end end end - end - def clean_spans - lambda do |env| - node = env[:node] + def clean_spans + lambda do |env| + node = env[:node] - return unless node.name == 'span' - return unless node.has_attribute?('class') + return unless node.name == 'span' + return unless node.has_attribute?('class') - unless has_ancestor?(node, 'pre') - node.remove_attribute('class') - end + unless node.ancestors.any? { |n| n.name.casecmp('pre').zero? } + node.remove_attribute('class') + end - { node_whitelist: [node] } + { node_whitelist: [node] } + end end end end diff --git a/lib/banzai/filter/set_direction_filter.rb b/lib/banzai/filter/set_direction_filter.rb new file mode 100644 index 00000000000..c2976aeb7c6 --- /dev/null +++ b/lib/banzai/filter/set_direction_filter.rb @@ -0,0 +1,15 @@ +module Banzai + module Filter + # HTML filter that sets dir="auto" for RTL languages support + class SetDirectionFilter < HTML::Pipeline::Filter + def call + # select these elements just on top level of the document + doc.xpath('p|h1|h2|h3|h4|h5|h6|ol|ul[not(@class="section-nav")]|blockquote|table').each do |el| + el['dir'] = 'auto' + end + + doc + end + end + end +end diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index fcdb496aed2..a447e2b8bff 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -20,17 +20,19 @@ module Banzai code = node.text css_classes = "code highlight" lexer = lexer_for(language) + lang = lexer.tag begin code = format(lex(lexer, code)) - css_classes << " js-syntax-highlight #{lexer.tag}" + css_classes << " js-syntax-highlight #{lang}" rescue + lang = nil # Gracefully handle syntax highlighter bugs/errors to ensure # users can still access an issue/comment/etc. end - highlighted = %(<pre class="#{css_classes}"><code>#{code}</code></pre>) + highlighted = %(<pre class="#{css_classes}" lang="#{lang}" v-pre="true"><code>#{code}</code></pre>) # Extracted to a method to measure it replace_parent_pre_element(node, highlighted) diff --git a/lib/banzai/filter/table_of_contents_filter.rb b/lib/banzai/filter/table_of_contents_filter.rb index a4eda6fdf76..8e7084f2543 100644 --- a/lib/banzai/filter/table_of_contents_filter.rb +++ b/lib/banzai/filter/table_of_contents_filter.rb @@ -35,9 +35,11 @@ module Banzai headers[id] += 1 if header_content = node.children.first + # namespace detection will be automatically handled via javascript (see issue #22781) + namespace = "user-content-" href = "#{id}#{uniq}" push_toc(href, text) - header_content.add_previous_sibling(anchor_tag(href)) + header_content.add_previous_sibling(anchor_tag("#{namespace}#{href}", href)) end end @@ -48,8 +50,8 @@ module Banzai private - def anchor_tag(href) - %Q{<a id="#{href}" class="anchor" href="##{href}" aria-hidden="true"></a>} + def anchor_tag(id, href) + %Q{<a id="#{id}" class="anchor" href="##{href}" aria-hidden="true"></a>} end def push_toc(href, text) diff --git a/lib/banzai/filter/task_list_filter.rb b/lib/banzai/filter/task_list_filter.rb index 66608c9859c..9fa5f589f3e 100644 --- a/lib/banzai/filter/task_list_filter.rb +++ b/lib/banzai/filter/task_list_filter.rb @@ -2,27 +2,7 @@ require 'task_list/filter' module Banzai module Filter - # Work around a bug in the default TaskList::Filter that adds a `task-list` - # class to every list element, regardless of whether or not it contains a - # task list. - # - # This is a (hopefully) temporary fix, pending a new release of the - # task_list gem. - # - # See https://github.com/github/task_list/pull/60 class TaskListFilter < TaskList::Filter - def add_css_class_with_fix(node, *new_class_names) - if new_class_names.include?('task-list') - # Don't add class to all lists - return - elsif new_class_names.include?('task-list-item') - add_css_class_without_fix(node.parent, 'task-list') - end - - add_css_class_without_fix(node, *new_class_names) - end - - alias_method_chain :add_css_class, :fix end end end diff --git a/lib/banzai/filter/user_reference_filter.rb b/lib/banzai/filter/user_reference_filter.rb index e1ca7f4d24b..c973897f420 100644 --- a/lib/banzai/filter/user_reference_filter.rb +++ b/lib/banzai/filter/user_reference_filter.rb @@ -24,7 +24,7 @@ module Banzai end def call - return doc if project.nil? + return doc if project.nil? && !skip_project_check? ref_pattern = User.reference_pattern ref_pattern_start = /\A#{ref_pattern}\z/ @@ -35,10 +35,10 @@ module Banzai user_link_filter(content) end elsif element_node?(node) - yield_valid_link(node) do |link, text| + yield_valid_link(node) do |link, inner_html| if link =~ ref_pattern_start replace_link_node_with_href(node, link) do - user_link_filter(link, link_text: text) + user_link_filter(link, link_content: inner_html) end end end @@ -52,15 +52,16 @@ module Banzai # user's profile page. # # text - String text to replace references in. + # link_content - Original content of the link being replaced. # # Returns a String with `@user` references replaced with links. All links # have `gfm` and `gfm-project_member` class names attached for styling. - def user_link_filter(text, link_text: nil) + def user_link_filter(text, link_content: nil) self.class.references_in(text) do |match, username| - if username == 'all' - link_to_all(link_text: link_text) + if username == 'all' && !skip_project_check? + link_to_all(link_content: link_content) elsif namespace = namespaces[username] - link_to_namespace(namespace, link_text: link_text) || match + link_to_namespace(namespace, link_content: link_content) || match else match end @@ -74,8 +75,8 @@ module Banzai # corresponding Namespace objects. def namespaces @namespaces ||= - Namespace.where(path: usernames).each_with_object({}) do |row, hash| - hash[row.path] = row + Namespace.where_full_path_in(usernames).each_with_object({}) do |row, hash| + hash[row.full_path] = row end end @@ -102,45 +103,49 @@ module Banzai reference_class(:project_member) end - def link_to_all(link_text: nil) + def link_to_all(link_content: nil) project = context[:project] author = context[:author] - url = urls.namespace_project_url(project.namespace, project, - only_path: context[:only_path]) + if author && !project.team.member?(author) + link_content + else + url = urls.namespace_project_url(project.namespace, project, + only_path: context[:only_path]) - data = data_attribute(project: project.id, author: author.try(:id)) - text = link_text || User.reference_prefix + 'all' + data = data_attribute(project: project.id, author: author.try(:id)) + content = link_content || User.reference_prefix + 'all' - link_tag(url, data, text, 'All Project and Group Members') + link_tag(url, data, content, 'All Project and Group Members') + end end - def link_to_namespace(namespace, link_text: nil) + def link_to_namespace(namespace, link_content: nil) if namespace.is_a?(Group) - link_to_group(namespace.path, namespace, link_text: link_text) + link_to_group(namespace.full_path, namespace, link_content: link_content) else - link_to_user(namespace.path, namespace, link_text: link_text) + link_to_user(namespace.path, namespace, link_content: link_content) end end - def link_to_group(group, namespace, link_text: nil) + def link_to_group(group, namespace, link_content: nil) url = urls.group_url(group, only_path: context[:only_path]) data = data_attribute(group: namespace.id) - text = link_text || Group.reference_prefix + group + content = link_content || Group.reference_prefix + group - link_tag(url, data, text, namespace.name) + link_tag(url, data, content, namespace.name) end - def link_to_user(user, namespace, link_text: nil) + def link_to_user(user, namespace, link_content: nil) url = urls.user_url(user, only_path: context[:only_path]) data = data_attribute(user: namespace.owner_id) - text = link_text || User.reference_prefix + user + content = link_content || User.reference_prefix + user - link_tag(url, data, text, namespace.owner_name) + link_tag(url, data, content, namespace.owner_name) end - def link_tag(url, data, text, title) - %(<a href="#{url}" #{data} class="#{link_class}" title="#{escape_once(title)}">#{escape_once(text)}</a>) + def link_tag(url, data, link_content, title) + %(<a href="#{url}" #{data} class="#{link_class}" title="#{escape_once(title)}">#{link_content}</a>) end end end diff --git a/lib/banzai/filter/video_link_filter.rb b/lib/banzai/filter/video_link_filter.rb index ac7bbcb0d10..b64a1287d4d 100644 --- a/lib/banzai/filter/video_link_filter.rb +++ b/lib/banzai/filter/video_link_filter.rb @@ -35,7 +35,8 @@ module Banzai src: element['src'], width: '400', controls: true, - 'data-setup' => '{}') + 'data-setup' => '{}', + 'data-title' => element['title'] || element['alt']) link = doc.document.create_element( 'a', diff --git a/lib/banzai/filter/wiki_link_filter/rewriter.rb b/lib/banzai/filter/wiki_link_filter/rewriter.rb index 2e2c8da311e..e7a1ec8457d 100644 --- a/lib/banzai/filter/wiki_link_filter/rewriter.rb +++ b/lib/banzai/filter/wiki_link_filter/rewriter.rb @@ -31,6 +31,7 @@ module Banzai def apply_relative_link_rules! if @uri.relative? && @uri.path.present? link = ::File.join(@wiki_base_path, @uri.path) + link = "#{link}##{@uri.fragment}" if @uri.fragment @uri = Addressable::URI.parse(link) end end diff --git a/lib/banzai/note_renderer.rb b/lib/banzai/note_renderer.rb index bab6a9934d1..2b7c10f1a0e 100644 --- a/lib/banzai/note_renderer.rb +++ b/lib/banzai/note_renderer.rb @@ -3,7 +3,7 @@ module Banzai # Renders a collection of Note instances. # # notes - The notes to render. - # project - The project to use for rendering/redacting. + # project - The project to use for redacting. # user - The user viewing the notes. # path - The request path. # wiki - The project's wiki. @@ -13,8 +13,7 @@ module Banzai user, requested_path: path, project_wiki: wiki, - ref: git_ref, - pipeline: :note) + ref: git_ref) renderer.render(notes, :note) end diff --git a/lib/banzai/object_renderer.rb b/lib/banzai/object_renderer.rb index 9aef807c152..9f8eb0931b8 100644 --- a/lib/banzai/object_renderer.rb +++ b/lib/banzai/object_renderer.rb @@ -1,28 +1,32 @@ module Banzai - # Class for rendering multiple objects (e.g. Note instances) in a single pass. + # Class for rendering multiple objects (e.g. Note instances) in a single pass, + # using +render_field+ to benefit from caching in the database. Rendering and + # redaction are both performed. # - # Rendered Markdown is stored in an attribute in every object based on the - # name of the attribute containing the Markdown. For example, when the - # attribute `note` is rendered the HTML is stored in `note_html`. + # The unredacted HTML is generated according to the usual +render_field+ + # policy, so specify the pipeline and any other context options on the model. + # + # The *redacted* (i.e., suitable for use) HTML is placed in an attribute + # named "redacted_<foo>", where <foo> is the name of the cache field for the + # chosen attribute. + # + # As an example, rendering the attribute `note` would place the unredacted + # HTML into `note_html` and the redacted HTML into `redacted_note_html`. class ObjectRenderer attr_reader :project, :user - # Make sure to set the appropriate pipeline in the `raw_context` attribute - # (e.g. `:note` for Note instances). - # - # project - A Project to use for rendering and redacting Markdown. + # project - A Project to use for redacting Markdown. # user - The user viewing the Markdown/HTML documents, if any. - # context - A Hash containing extra attributes to use in the rendering - # pipeline. - def initialize(project, user = nil, raw_context = {}) + # context - A Hash containing extra attributes to use during redaction + def initialize(project, user = nil, redaction_context = {}) @project = project @user = user - @raw_context = raw_context + @redaction_context = redaction_context end # Renders and redacts an Array of objects. # - # objects - The objects to render + # objects - The objects to render. # attribute - The attribute containing the raw Markdown to render. # # Returns the same input objects. @@ -32,7 +36,7 @@ module Banzai objects.each_with_index do |object, index| redacted_data = redacted[index] - object.__send__("#{attribute}_html=", redacted_data[:document].to_html.html_safe) + object.__send__("redacted_#{attribute}_html=", redacted_data[:document].to_html.html_safe) object.user_visible_reference_count = redacted_data[:visible_reference_count] end end @@ -53,12 +57,8 @@ module Banzai # Returns a Banzai context for the given object and attribute. def context_for(object, attribute) - context = base_context.merge(cache_key: [object, attribute]) - - if object.respond_to?(:author) - context[:author] = object.author - end - + context = base_context.dup + context = context.merge(object.banzai_render_context(attribute)) context end @@ -66,21 +66,16 @@ module Banzai # # Returns an Array of `Nokogiri::HTML::Document`. def render_attributes(objects, attribute) - strings_and_contexts = objects.map do |object| + objects.map do |object| + string = Banzai.render_field(object, attribute) context = context_for(object, attribute) - string = object.__send__(attribute) - - { text: string, context: context } - end - - Banzai.cache_collection_render(strings_and_contexts).each_with_index.map do |html, index| - Banzai::Pipeline[:relative_link].to_document(html, strings_and_contexts[index][:context]) + Banzai::Pipeline[:relative_link].to_document(string, context) end end def base_context - @base_context ||= @raw_context.merge(current_user: user, project: project) + @base_context ||= @redaction_context.merge(current_user: user, project: project) end end end diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index 8d94b199c66..b25d6f18d59 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -1,11 +1,19 @@ module Banzai module Pipeline class GfmPipeline < BasePipeline + # These filters convert GitLab Flavored Markdown (GFM) to HTML. + # The handlers defined in app/assets/javascripts/copy_as_gfm.js.es6 + # consequently convert that same HTML to GFM to be copied to the clipboard. + # Every filter that generates HTML from GFM should have a handler in + # app/assets/javascripts/copy_as_gfm.js.es6, in reverse order. + # The GFM-to-HTML-to-GFM cycle is tested in spec/features/copy_as_gfm_spec.rb. def self.filters @filters ||= FilterArray[ Filter::SyntaxHighlightFilter, + Filter::PlantumlFilter, Filter::SanitizationFilter, + Filter::MathFilter, Filter::UploadLinkFilter, Filter::VideoLinkFilter, Filter::ImageLinkFilter, @@ -25,7 +33,9 @@ module Banzai Filter::MilestoneReferenceFilter, Filter::TaskListFilter, - Filter::InlineDiffFilter + Filter::InlineDiffFilter, + + Filter::SetDirectionFilter ] end diff --git a/lib/banzai/pipeline/single_line_pipeline.rb b/lib/banzai/pipeline/single_line_pipeline.rb index ba2555df98d..1929099931b 100644 --- a/lib/banzai/pipeline/single_line_pipeline.rb +++ b/lib/banzai/pipeline/single_line_pipeline.rb @@ -3,6 +3,7 @@ module Banzai class SingleLinePipeline < GfmPipeline def self.filters @filters ||= FilterArray[ + Filter::HtmlEntityFilter, Filter::SanitizationFilter, Filter::EmojiFilter, diff --git a/lib/banzai/querying.rb b/lib/banzai/querying.rb index 1e1b51e683e..fb2faae02bc 100644 --- a/lib/banzai/querying.rb +++ b/lib/banzai/querying.rb @@ -1,18 +1,64 @@ module Banzai module Querying + module_function + # Searches a Nokogiri document using a CSS query, optionally optimizing it # whenever possible. # - # document - A document/element to search. - # query - The CSS query to use. + # document - A document/element to search. + # query - The CSS query to use. + # reference_options - A hash with nodes filter options # - # Returns a Nokogiri::XML::NodeSet. - def self.css(document, query) + # Returns an array of Nokogiri::XML::Element objects if location is specified + # in reference_options. Otherwise it would a Nokogiri::XML::NodeSet. + def css(document, query, reference_options = {}) # When using "a.foo" Nokogiri compiles this to "//a[...]" but # "descendant::a[...]" is quite a bit faster and achieves the same result. xpath = Nokogiri::CSS.xpath_for(query)[0].gsub(%r{^//}, 'descendant::') + xpath = restrict_to_p_nodes_at_root(xpath) if filter_nodes_at_beginning?(reference_options) + nodes = document.xpath(xpath) + + filter_nodes(nodes, reference_options) + end + + def restrict_to_p_nodes_at_root(xpath) + xpath.gsub('descendant::', './p/') + end + + def filter_nodes(nodes, reference_options) + if filter_nodes_at_beginning?(reference_options) + filter_nodes_at_beginning(nodes) + else + nodes + end + end + + def filter_nodes_at_beginning?(reference_options) + reference_options && reference_options[:location] == :beginning + end + + # Selects child nodes if they are present in the beginning among other siblings. + # + # nodes - A Nokogiri::XML::NodeSet. + # + # Returns an array of Nokogiri::XML::Element objects. + def filter_nodes_at_beginning(nodes) + parents_and_nodes = nodes.group_by(&:parent) + filtered_nodes = [] + + parents_and_nodes.each do |parent, nodes| + children = parent.children + nodes = nodes.to_a + + children.each do |child| + next if child.text.blank? + node = nodes.shift + break unless node == child + filtered_nodes << node + end + end - document.xpath(xpath) + filtered_nodes end end end diff --git a/lib/banzai/redactor.rb b/lib/banzai/redactor.rb index 0df3a72d1c4..de3ebe72720 100644 --- a/lib/banzai/redactor.rb +++ b/lib/banzai/redactor.rb @@ -41,10 +41,10 @@ module Banzai next if visible.include?(node) doc_data[:visible_reference_count] -= 1 - # The reference should be replaced by the original text, - # which is not always the same as the rendered text. - text = node.attr('data-original') || node.text - node.replace(text) + # The reference should be replaced by the original link's content, + # which is not always the same as the rendered one. + content = node.attr('data-original') || node.inner_html + node.replace(content) end end diff --git a/lib/banzai/reference_extractor.rb b/lib/banzai/reference_extractor.rb index b26a41a1f3b..8e3b0c4db79 100644 --- a/lib/banzai/reference_extractor.rb +++ b/lib/banzai/reference_extractor.rb @@ -16,6 +16,11 @@ module Banzai processor.process(html_documents) end + def reset_memoized_values + @html_documents = nil + @texts_and_contexts = [] + end + private def html_documents diff --git a/lib/banzai/reference_parser/base_parser.rb b/lib/banzai/reference_parser/base_parser.rb index 6cf218aaa0d..2058a58d0ae 100644 --- a/lib/banzai/reference_parser/base_parser.rb +++ b/lib/banzai/reference_parser/base_parser.rb @@ -33,7 +33,7 @@ module Banzai # they have access to. class BaseParser class << self - attr_accessor :reference_type + attr_accessor :reference_type, :reference_options end # Returns the attribute name containing the value for every object to be @@ -63,12 +63,7 @@ module Banzai nodes.select do |node| if node.has_attribute?(project_attr) node_id = node.attr(project_attr).to_i - - if project && project.id == node_id - true - else - can?(user, :read_project, projects[node_id]) - end + can_read_reference?(user, projects[node_id]) else true end @@ -79,7 +74,11 @@ module Banzai def referenced_by(nodes) ids = unique_attribute_values(nodes, self.class.data_attribute) - references_relation.where(id: ids) + if ids.empty? + references_relation.none + else + references_relation.where(id: ids) + end end # Returns the ActiveRecord::Relation to use for querying references in the @@ -183,9 +182,10 @@ module Banzai # the references. def process(documents) type = self.class.reference_type + reference_options = self.class.reference_options nodes = documents.flat_map do |document| - Querying.css(document, "a[data-reference-type='#{type}'].gfm").to_a + Querying.css(document, "a[data-reference-type='#{type}'].gfm", reference_options).to_a end gather_references(nodes) @@ -211,7 +211,7 @@ module Banzai end def can?(user, permission, subject) - Ability.abilities.allowed?(user, permission, subject) + Ability.allowed?(user, permission, subject) end def find_projects_for_hash_keys(hash) @@ -222,6 +222,15 @@ module Banzai attr_reader :current_user, :project + # When a feature is disabled or visible only for + # team members we should not allow team members + # see reference comments. + # Override this method on subclasses + # to check if user can read resource + def can_read_reference?(user, ref_project) + raise NotImplementedError + end + def lazy(&block) Gitlab::Lazy.new(&block) end diff --git a/lib/banzai/reference_parser/commit_parser.rb b/lib/banzai/reference_parser/commit_parser.rb index 0fee9d267de..8c54a041cb8 100644 --- a/lib/banzai/reference_parser/commit_parser.rb +++ b/lib/banzai/reference_parser/commit_parser.rb @@ -29,6 +29,12 @@ module Banzai commits end + + private + + def can_read_reference?(user, ref_project) + can?(user, :download_code, ref_project) + end end end end diff --git a/lib/banzai/reference_parser/commit_range_parser.rb b/lib/banzai/reference_parser/commit_range_parser.rb index 69d01f8db15..0878b6afba3 100644 --- a/lib/banzai/reference_parser/commit_range_parser.rb +++ b/lib/banzai/reference_parser/commit_range_parser.rb @@ -33,6 +33,12 @@ module Banzai range.valid_commits? ? range : nil end + + private + + def can_read_reference?(user, ref_project) + can?(user, :download_code, ref_project) + end end end end diff --git a/lib/banzai/reference_parser/directly_addressed_user_parser.rb b/lib/banzai/reference_parser/directly_addressed_user_parser.rb new file mode 100644 index 00000000000..77df9bbd024 --- /dev/null +++ b/lib/banzai/reference_parser/directly_addressed_user_parser.rb @@ -0,0 +1,8 @@ +module Banzai + module ReferenceParser + class DirectlyAddressedUserParser < UserParser + self.reference_type = :user + self.reference_options = { location: :beginning } + end + end +end diff --git a/lib/banzai/reference_parser/external_issue_parser.rb b/lib/banzai/reference_parser/external_issue_parser.rb index a1264db2111..6e7b7669578 100644 --- a/lib/banzai/reference_parser/external_issue_parser.rb +++ b/lib/banzai/reference_parser/external_issue_parser.rb @@ -20,6 +20,12 @@ module Banzai def issue_ids_per_project(nodes) gather_attributes_per_project(nodes, self.class.data_attribute) end + + private + + def can_read_reference?(user, ref_project) + can?(user, :read_issue, ref_project) + end end end end diff --git a/lib/banzai/reference_parser/label_parser.rb b/lib/banzai/reference_parser/label_parser.rb index e5d1eb11d7f..aa76c64ac5f 100644 --- a/lib/banzai/reference_parser/label_parser.rb +++ b/lib/banzai/reference_parser/label_parser.rb @@ -6,6 +6,12 @@ module Banzai def references_relation Label end + + private + + def can_read_reference?(user, ref_project) + can?(user, :read_label, ref_project) + end end end end diff --git a/lib/banzai/reference_parser/merge_request_parser.rb b/lib/banzai/reference_parser/merge_request_parser.rb index c9a9ca79c09..40451947e6c 100644 --- a/lib/banzai/reference_parser/merge_request_parser.rb +++ b/lib/banzai/reference_parser/merge_request_parser.rb @@ -6,6 +6,12 @@ module Banzai def references_relation MergeRequest.includes(:author, :assignee, :target_project) end + + private + + def can_read_reference?(user, ref_project) + can?(user, :read_merge_request, ref_project) + end end end end diff --git a/lib/banzai/reference_parser/milestone_parser.rb b/lib/banzai/reference_parser/milestone_parser.rb index a000ac61e5c..d3968d6b229 100644 --- a/lib/banzai/reference_parser/milestone_parser.rb +++ b/lib/banzai/reference_parser/milestone_parser.rb @@ -6,6 +6,12 @@ module Banzai def references_relation Milestone end + + private + + def can_read_reference?(user, ref_project) + can?(user, :read_milestone, ref_project) + end end end end diff --git a/lib/banzai/reference_parser/snippet_parser.rb b/lib/banzai/reference_parser/snippet_parser.rb index fa71b3c952a..63b592137bb 100644 --- a/lib/banzai/reference_parser/snippet_parser.rb +++ b/lib/banzai/reference_parser/snippet_parser.rb @@ -6,6 +6,12 @@ module Banzai def references_relation Snippet end + + private + + def can_read_reference?(user, ref_project) + can?(user, :read_project_snippet, ref_project) + end end end end diff --git a/lib/banzai/reference_parser/user_parser.rb b/lib/banzai/reference_parser/user_parser.rb index 863f5725d3b..7adaffa19c1 100644 --- a/lib/banzai/reference_parser/user_parser.rb +++ b/lib/banzai/reference_parser/user_parser.rb @@ -30,22 +30,36 @@ module Banzai nodes.each do |node| if node.has_attribute?(group_attr) - node_group = groups[node.attr(group_attr).to_i] - - if node_group && - can?(user, :read_group, node_group) - visible << node - end - # Remaining nodes will be processed by the parent class' - # implementation of this method. + next unless can_read_group_reference?(node, user, groups) + visible << node + elsif can_read_project_reference?(node) + visible << node else remaining << node end end + # If project does not belong to a group + # and does not have the same project id as the current project + # base class will check if user can read the project that contains + # the user reference. visible + super(current_user, remaining) end + # Check if project belongs to a group which + # user can read. + def can_read_group_reference?(node, user, groups) + node_group = groups[node.attr('data-group').to_i] + + node_group && can?(user, :read_group, node_group) + end + + def can_read_project_reference?(node) + node_id = node.attr('data-project').to_i + + project && project.id == node_id + end + def nodes_user_can_reference(current_user, nodes) project_attr = 'data-project' author_attr = 'data-author' @@ -88,6 +102,10 @@ module Banzai collection_objects_for_ids(Project, ids). flat_map { |p| p.team.members.to_a } end + + def can_read_reference?(user, ref_project) + can?(user, :read_project, ref_project) + end end end end diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index a4ae27eefd8..74663556cbb 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -1,6 +1,6 @@ module Banzai module Renderer - extend self + module_function # Convert a Markdown String into an HTML-safe String of HTML # @@ -31,6 +31,34 @@ module Banzai end end + # Convert a Markdown-containing field on an object into an HTML-safe String + # of HTML. This method is analogous to calling render(object.field), but it + # can cache the rendered HTML in the object, rather than Redis. + # + # The context to use is learned from the passed-in object by calling + # #banzai_render_context(field), and cannot be changed. Use #render, passing + # it the field text, if a custom rendering is needed. The generated context + # is returned along with the HTML. + def render_field(object, field) + html_field = object.markdown_cache_field_for(field) + + html = object.__send__(html_field) + return html if html.present? + + html = cacheless_render_field(object, field) + update_object(object, html_field, html) unless object.new_record? || object.destroyed? + + html + end + + # Same as +render_field+, but without consulting or updating the cache field + def cacheless_render_field(object, field, options = {}) + text = object.__send__(field) + context = object.banzai_render_context(field).merge(options) + + cacheless_render(text, context) + end + # Perform multiple render from an Array of Markdown String into an # Array of HTML-safe String of HTML. # @@ -113,8 +141,6 @@ module Banzai end.html_safe end - private - def cacheless_render(text, context = {}) Gitlab::Metrics.measure(:banzai_cacheless_render) do result = render_result(text, context) @@ -140,5 +166,9 @@ module Banzai return unless cache_key Rails.cache.send(:expanded_key, full_cache_key(cache_key, pipeline_name)) end + + def update_object(object, html_field, html) + object.update_column(html_field, html) + end end end diff --git a/lib/bitbucket/client.rb b/lib/bitbucket/client.rb new file mode 100644 index 00000000000..f8ee7e0f9ae --- /dev/null +++ b/lib/bitbucket/client.rb @@ -0,0 +1,58 @@ +module Bitbucket + class Client + attr_reader :connection + + def initialize(options = {}) + @connection = Connection.new(options) + end + + def issues(repo) + path = "/repositories/#{repo}/issues" + get_collection(path, :issue) + end + + def issue_comments(repo, issue_id) + path = "/repositories/#{repo}/issues/#{issue_id}/comments" + get_collection(path, :comment) + end + + def pull_requests(repo) + path = "/repositories/#{repo}/pullrequests?state=ALL" + get_collection(path, :pull_request) + end + + def pull_request_comments(repo, pull_request) + path = "/repositories/#{repo}/pullrequests/#{pull_request}/comments" + get_collection(path, :pull_request_comment) + end + + def pull_request_diff(repo, pull_request) + path = "/repositories/#{repo}/pullrequests/#{pull_request}/diff" + connection.get(path) + end + + def repo(name) + parsed_response = connection.get("/repositories/#{name}") + Representation::Repo.new(parsed_response) + end + + def repos + path = "/repositories?role=member" + get_collection(path, :repo) + end + + def user + @user ||= begin + parsed_response = connection.get('/user') + Representation::User.new(parsed_response) + end + end + + private + + def get_collection(path, type) + paginator = Paginator.new(connection, path, type) + Collection.new(paginator) + end + end +end diff --git a/lib/bitbucket/collection.rb b/lib/bitbucket/collection.rb new file mode 100644 index 00000000000..3a9379ff680 --- /dev/null +++ b/lib/bitbucket/collection.rb @@ -0,0 +1,21 @@ +module Bitbucket + class Collection < Enumerator + def initialize(paginator) + super() do |yielder| + loop do + paginator.items.each { |item| yielder << item } + end + end + + lazy + end + + def method_missing(method, *args) + return super unless self.respond_to?(method) + + self.send(method, *args) do |item| + block_given? ? yield(item) : item + end + end + end +end diff --git a/lib/bitbucket/connection.rb b/lib/bitbucket/connection.rb new file mode 100644 index 00000000000..b9279c33f5b --- /dev/null +++ b/lib/bitbucket/connection.rb @@ -0,0 +1,67 @@ +module Bitbucket + class Connection + DEFAULT_API_VERSION = '2.0'.freeze + DEFAULT_BASE_URI = 'https://api.bitbucket.org/'.freeze + DEFAULT_QUERY = {}.freeze + + attr_reader :expires_at, :expires_in, :refresh_token, :token + + def initialize(options = {}) + @api_version = options.fetch(:api_version, DEFAULT_API_VERSION) + @base_uri = options.fetch(:base_uri, DEFAULT_BASE_URI) + @default_query = options.fetch(:query, DEFAULT_QUERY) + + @token = options[:token] + @expires_at = options[:expires_at] + @expires_in = options[:expires_in] + @refresh_token = options[:refresh_token] + end + + def get(path, extra_query = {}) + refresh! if expired? + + response = connection.get(build_url(path), params: @default_query.merge(extra_query)) + response.parsed + end + + delegate :expired?, to: :connection + + def refresh! + response = connection.refresh! + + @token = response.token + @expires_at = response.expires_at + @expires_in = response.expires_in + @refresh_token = response.refresh_token + @connection = nil + end + + private + + def client + @client ||= OAuth2::Client.new(provider.app_id, provider.app_secret, options) + end + + def connection + @connection ||= OAuth2::AccessToken.new(client, @token, refresh_token: @refresh_token, expires_at: @expires_at, expires_in: @expires_in) + end + + def build_url(path) + return path if path.starts_with?(root_url) + + "#{root_url}#{path}" + end + + def root_url + @root_url ||= "#{@base_uri}#{@api_version}" + end + + def provider + Gitlab::OAuth::Provider.config_for('bitbucket') + end + + def options + OmniAuth::Strategies::Bitbucket.default_options[:client_options].deep_symbolize_keys + end + end +end diff --git a/lib/bitbucket/error/unauthorized.rb b/lib/bitbucket/error/unauthorized.rb new file mode 100644 index 00000000000..5e2eb57bb0e --- /dev/null +++ b/lib/bitbucket/error/unauthorized.rb @@ -0,0 +1,6 @@ +module Bitbucket + module Error + class Unauthorized < StandardError + end + end +end diff --git a/lib/bitbucket/page.rb b/lib/bitbucket/page.rb new file mode 100644 index 00000000000..2b0a3fe7b1a --- /dev/null +++ b/lib/bitbucket/page.rb @@ -0,0 +1,34 @@ +module Bitbucket + class Page + attr_reader :attrs, :items + + def initialize(raw, type) + @attrs = parse_attrs(raw) + @items = parse_values(raw, representation_class(type)) + end + + def next? + attrs.fetch(:next, false) + end + + def next + attrs.fetch(:next) + end + + private + + def parse_attrs(raw) + raw.slice(*%w(size page pagelen next previous)).symbolize_keys + end + + def parse_values(raw, bitbucket_rep_class) + return [] unless raw['values'] && raw['values'].is_a?(Array) + + bitbucket_rep_class.decorate(raw['values']) + end + + def representation_class(type) + Bitbucket::Representation.const_get(type.to_s.camelize) + end + end +end diff --git a/lib/bitbucket/paginator.rb b/lib/bitbucket/paginator.rb new file mode 100644 index 00000000000..135d0d55674 --- /dev/null +++ b/lib/bitbucket/paginator.rb @@ -0,0 +1,36 @@ +module Bitbucket + class Paginator + PAGE_LENGTH = 50 # The minimum length is 10 and the maximum is 100. + + def initialize(connection, url, type) + @connection = connection + @type = type + @url = url + @page = nil + end + + def items + raise StopIteration unless has_next_page? + + @page = fetch_next_page + @page.items + end + + private + + attr_reader :connection, :page, :url, :type + + def has_next_page? + page.nil? || page.next? + end + + def next_url + page.nil? ? url : page.next + end + + def fetch_next_page + parsed_response = connection.get(next_url, pagelen: PAGE_LENGTH, sort: :created_on) + Page.new(parsed_response, type) + end + end +end diff --git a/lib/bitbucket/representation/base.rb b/lib/bitbucket/representation/base.rb new file mode 100644 index 00000000000..94adaacc9b5 --- /dev/null +++ b/lib/bitbucket/representation/base.rb @@ -0,0 +1,17 @@ +module Bitbucket + module Representation + class Base + def initialize(raw) + @raw = raw + end + + def self.decorate(entries) + entries.map { |entry| new(entry)} + end + + private + + attr_reader :raw + end + end +end diff --git a/lib/bitbucket/representation/comment.rb b/lib/bitbucket/representation/comment.rb new file mode 100644 index 00000000000..4937aa9728f --- /dev/null +++ b/lib/bitbucket/representation/comment.rb @@ -0,0 +1,27 @@ +module Bitbucket + module Representation + class Comment < Representation::Base + def author + user['username'] + end + + def note + raw.fetch('content', {}).fetch('raw', nil) + end + + def created_at + raw['created_on'] + end + + def updated_at + raw['updated_on'] || raw['created_on'] + end + + private + + def user + raw.fetch('user', {}) + end + end + end +end diff --git a/lib/bitbucket/representation/issue.rb b/lib/bitbucket/representation/issue.rb new file mode 100644 index 00000000000..054064395c3 --- /dev/null +++ b/lib/bitbucket/representation/issue.rb @@ -0,0 +1,53 @@ +module Bitbucket + module Representation + class Issue < Representation::Base + CLOSED_STATUS = %w(resolved invalid duplicate wontfix closed).freeze + + def iid + raw['id'] + end + + def kind + raw['kind'] + end + + def author + raw.fetch('reporter', {}).fetch('username', nil) + end + + def description + raw.fetch('content', {}).fetch('raw', nil) + end + + def state + closed? ? 'closed' : 'opened' + end + + def title + raw['title'] + end + + def milestone + raw['milestone']['name'] if raw['milestone'].present? + end + + def created_at + raw['created_on'] + end + + def updated_at + raw['edited_on'] + end + + def to_s + iid + end + + private + + def closed? + CLOSED_STATUS.include?(raw['state']) + end + end + end +end diff --git a/lib/bitbucket/representation/pull_request.rb b/lib/bitbucket/representation/pull_request.rb new file mode 100644 index 00000000000..eebf8093380 --- /dev/null +++ b/lib/bitbucket/representation/pull_request.rb @@ -0,0 +1,65 @@ +module Bitbucket + module Representation + class PullRequest < Representation::Base + def author + raw.fetch('author', {}).fetch('username', nil) + end + + def description + raw['description'] + end + + def iid + raw['id'] + end + + def state + if raw['state'] == 'MERGED' + 'merged' + elsif raw['state'] == 'DECLINED' + 'closed' + else + 'opened' + end + end + + def created_at + raw['created_on'] + end + + def updated_at + raw['updated_on'] + end + + def title + raw['title'] + end + + def source_branch_name + source_branch.fetch('branch', {}).fetch('name', nil) + end + + def source_branch_sha + source_branch.fetch('commit', {}).fetch('hash', nil) + end + + def target_branch_name + target_branch.fetch('branch', {}).fetch('name', nil) + end + + def target_branch_sha + target_branch.fetch('commit', {}).fetch('hash', nil) + end + + private + + def source_branch + raw['source'] + end + + def target_branch + raw['destination'] + end + end + end +end diff --git a/lib/bitbucket/representation/pull_request_comment.rb b/lib/bitbucket/representation/pull_request_comment.rb new file mode 100644 index 00000000000..4f8efe03bae --- /dev/null +++ b/lib/bitbucket/representation/pull_request_comment.rb @@ -0,0 +1,39 @@ +module Bitbucket + module Representation + class PullRequestComment < Comment + def iid + raw['id'] + end + + def file_path + inline.fetch('path') + end + + def old_pos + inline.fetch('from') + end + + def new_pos + inline.fetch('to') + end + + def parent_id + raw.fetch('parent', {}).fetch('id', nil) + end + + def inline? + raw.has_key?('inline') + end + + def has_parent? + raw.has_key?('parent') + end + + private + + def inline + raw.fetch('inline', {}) + end + end + end +end diff --git a/lib/bitbucket/representation/repo.rb b/lib/bitbucket/representation/repo.rb new file mode 100644 index 00000000000..59b0fda8e14 --- /dev/null +++ b/lib/bitbucket/representation/repo.rb @@ -0,0 +1,71 @@ +module Bitbucket + module Representation + class Repo < Representation::Base + attr_reader :owner, :slug + + def initialize(raw) + super(raw) + end + + def owner_and_slug + @owner_and_slug ||= full_name.split('/', 2) + end + + def owner + owner_and_slug.first + end + + def slug + owner_and_slug.last + end + + def clone_url(token = nil) + url = raw['links']['clone'].find { |link| link['name'] == 'https' }.fetch('href') + + if token.present? + clone_url = URI.parse(url) + clone_url.user = "x-token-auth:#{token}" + clone_url.to_s + else + url + end + end + + def description + raw['description'] + end + + def full_name + raw['full_name'] + end + + def issues_enabled? + raw['has_issues'] + end + + def name + raw['name'] + end + + def valid? + raw['scm'] == 'git' + end + + def has_wiki? + raw['has_wiki'] + end + + def visibility_level + if raw['is_private'] + Gitlab::VisibilityLevel::PRIVATE + else + Gitlab::VisibilityLevel::PUBLIC + end + end + + def to_s + full_name + end + end + end +end diff --git a/lib/bitbucket/representation/user.rb b/lib/bitbucket/representation/user.rb new file mode 100644 index 00000000000..ba6b7667b49 --- /dev/null +++ b/lib/bitbucket/representation/user.rb @@ -0,0 +1,9 @@ +module Bitbucket + module Representation + class User < Representation::Base + def username + raw['username'] + end + end + end +end diff --git a/lib/ci/ansi2html.rb b/lib/ci/ansi2html.rb index 229050151d3..b3ccad7b28d 100644 --- a/lib/ci/ansi2html.rb +++ b/lib/ci/ansi2html.rb @@ -13,7 +13,7 @@ module Ci 5 => 'magenta', 6 => 'cyan', 7 => 'white', # not that this is gray in the dark (aka default) color table - } + }.freeze STYLE_SWITCHES = { bold: 0x01, @@ -21,7 +21,7 @@ module Ci underline: 0x04, conceal: 0x08, cross: 0x10, - } + }.freeze def self.convert(ansi, state = nil) Converter.new.convert(ansi, state) @@ -29,64 +29,108 @@ module Ci class Converter def on_0(s) reset() end + def on_1(s) enable(STYLE_SWITCHES[:bold]) end + def on_3(s) enable(STYLE_SWITCHES[:italic]) end + def on_4(s) enable(STYLE_SWITCHES[:underline]) end + def on_8(s) enable(STYLE_SWITCHES[:conceal]) end + def on_9(s) enable(STYLE_SWITCHES[:cross]) end def on_21(s) disable(STYLE_SWITCHES[:bold]) end + def on_22(s) disable(STYLE_SWITCHES[:bold]) end + def on_23(s) disable(STYLE_SWITCHES[:italic]) end + def on_24(s) disable(STYLE_SWITCHES[:underline]) end + def on_28(s) disable(STYLE_SWITCHES[:conceal]) end + def on_29(s) disable(STYLE_SWITCHES[:cross]) end def on_30(s) set_fg_color(0) end + def on_31(s) set_fg_color(1) end + def on_32(s) set_fg_color(2) end + def on_33(s) set_fg_color(3) end + def on_34(s) set_fg_color(4) end + def on_35(s) set_fg_color(5) end + def on_36(s) set_fg_color(6) end + def on_37(s) set_fg_color(7) end + def on_38(s) set_fg_color_256(s) end + def on_39(s) set_fg_color(9) end def on_40(s) set_bg_color(0) end + def on_41(s) set_bg_color(1) end + def on_42(s) set_bg_color(2) end + def on_43(s) set_bg_color(3) end + def on_44(s) set_bg_color(4) end + def on_45(s) set_bg_color(5) end + def on_46(s) set_bg_color(6) end + def on_47(s) set_bg_color(7) end + def on_48(s) set_bg_color_256(s) end + def on_49(s) set_bg_color(9) end def on_90(s) set_fg_color(0, 'l') end + def on_91(s) set_fg_color(1, 'l') end + def on_92(s) set_fg_color(2, 'l') end + def on_93(s) set_fg_color(3, 'l') end + def on_94(s) set_fg_color(4, 'l') end + def on_95(s) set_fg_color(5, 'l') end + def on_96(s) set_fg_color(6, 'l') end + def on_97(s) set_fg_color(7, 'l') end + def on_99(s) set_fg_color(9, 'l') end def on_100(s) set_bg_color(0, 'l') end + def on_101(s) set_bg_color(1, 'l') end + def on_102(s) set_bg_color(2, 'l') end + def on_103(s) set_bg_color(3, 'l') end + def on_104(s) set_bg_color(4, 'l') end + def on_105(s) set_bg_color(5, 'l') end + def on_106(s) set_bg_color(6, 'l') end + def on_107(s) set_bg_color(7, 'l') end + def on_109(s) set_bg_color(9, 'l') end attr_accessor :offset, :n_open_tags, :fg_color, :bg_color, :style_mask - STATE_PARAMS = [:offset, :n_open_tags, :fg_color, :bg_color, :style_mask] + STATE_PARAMS = [:offset, :n_open_tags, :fg_color, :bg_color, :style_mask].freeze def convert(raw, new_state) reset_state @@ -105,7 +149,7 @@ module Ci break elsif s.scan(/</) @out << '<' - elsif s.scan(/\n/) + elsif s.scan(/\r?\n/) @out << '<br>' else @out << s.scan(/./m) @@ -126,7 +170,7 @@ module Ci # We are only interested in color and text style changes - triggered by # sequences starting with '\e[' and ending with 'm'. Any other control # sequence gets stripped (including stuff like "delete last line") - return unless indicator == '[' and terminator == 'm' + return unless indicator == '[' && terminator == 'm' close_open_tags() diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb index 17bb99a2ae5..24bb3649a76 100644 --- a/lib/ci/api/api.rb +++ b/lib/ci/api/api.rb @@ -8,23 +8,25 @@ module Ci rack_response({ 'message' => '404 Not found' }.to_json, 404) end - rescue_from :all do |exception| - # lifted from https://github.com/rails/rails/blob/master/actionpack/lib/action_dispatch/middleware/debug_exceptions.rb#L60 - # why is this not wrapped in something reusable? - trace = exception.backtrace + # Retain 405 error rather than a 500 error for Grape 0.15.0+. + # https://github.com/ruby-grape/grape/blob/a3a28f5b5dfbb2797442e006dbffd750b27f2a76/UPGRADING.md#changes-to-method-not-allowed-routes + rescue_from Grape::Exceptions::MethodNotAllowed do |e| + error! e.message, e.status, e.headers + end - message = "\n#{exception.class} (#{exception.message}):\n" - message << exception.annoted_source_code.to_s if exception.respond_to?(:annoted_source_code) - message << " " << trace.join("\n ") + rescue_from Grape::Exceptions::Base do |e| + error! e.message, e.status, e.headers + end - API.logger.add Logger::FATAL, message - rack_response({ 'message' => '500 Internal Server Error' }, 500) + rescue_from :all do |exception| + handle_api_exception(exception) end content_type :txt, 'text/plain' content_type :json, 'application/json' format :json + helpers ::SentryHelper helpers ::Ci::API::Helpers helpers ::API::Helpers helpers Gitlab::CurrentSettings diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 260ac81f5fa..0e17ac24d5a 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -12,17 +12,37 @@ module Ci # POST /builds/register post "register" do authenticate_runner! - update_runner_last_contact - update_runner_info required_attributes! [:token] not_found! unless current_runner.active? + update_runner_info + + if current_runner.is_runner_queue_value_latest?(params[:last_update]) + header 'X-GitLab-Last-Update', params[:last_update] + Gitlab::Metrics.add_event(:build_not_found_cached) + return build_not_found! + end + + new_update = current_runner.ensure_runner_queue_value + + result = Ci::RegisterBuildService.new(current_runner).execute - build = Ci::RegisterBuildService.new.execute(current_runner) + if result.valid? + if result.build + Gitlab::Metrics.add_event(:build_found, + project: result.build.project.path_with_namespace) - if build - present build, with: Entities::BuildDetails + present result.build, with: Entities::BuildDetails + else + Gitlab::Metrics.add_event(:build_not_found) + + header 'X-GitLab-Last-Update', new_update + + build_not_found! + end else - not_found! + # We received build that is invalid due to concurrency conflict + Gitlab::Metrics.add_event(:build_invalid) + conflict! end end @@ -36,12 +56,16 @@ module Ci # PUT /builds/:id put ":id" do authenticate_runner! - update_runner_last_contact build = Ci::Build.where(runner_id: current_runner.id).running.find(params[:id]) - forbidden!('Build has been erased!') if build.erased? + validate_build!(build) + + update_runner_info build.update_attributes(trace: params[:trace]) if params[:trace] + Gitlab::Metrics.add_event(:update_build, + project: build.project.path_with_namespace) + case params[:state].to_s when 'success' build.success @@ -63,9 +87,7 @@ module Ci # PATCH /builds/:id/trace.txt patch ":id/trace.txt" do build = Ci::Build.find_by_id(params[:id]) - not_found! unless build - authenticate_build_token!(build) - forbidden!('Build has been erased!') if build.erased? + authenticate_build!(build) error!('400 Missing header Content-Range', 400) unless request.headers.has_key?('Content-Range') content_range = request.headers['Content-Range'] @@ -93,10 +115,10 @@ module Ci # POST /builds/:id/artifacts/authorize post ":id/artifacts/authorize" do require_gitlab_workhorse! + Gitlab::Workhorse.verify_api_request!(headers) not_allowed! unless Gitlab.config.artifacts.enabled build = Ci::Build.find_by_id(params[:id]) - not_found! unless build - authenticate_build_token!(build) + authenticate_build!(build) forbidden!('build is not running') unless build.running? if params[:filesize] @@ -105,7 +127,8 @@ module Ci end status 200 - { TempPath: ArtifactUploader.artifacts_upload_path } + content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE + Gitlab::Workhorse.artifact_upload_ok end # Upload artifacts to build - Runners only @@ -132,10 +155,8 @@ module Ci require_gitlab_workhorse! not_allowed! unless Gitlab.config.artifacts.enabled build = Ci::Build.find_by_id(params[:id]) - not_found! unless build - authenticate_build_token!(build) + authenticate_build!(build) forbidden!('Build is not running!') unless build.running? - forbidden!('Build has been erased!') if build.erased? artifacts_upload_path = ArtifactUploader.artifacts_upload_path artifacts = uploaded_file(:file, artifacts_upload_path) @@ -146,7 +167,10 @@ module Ci build.artifacts_file = artifacts build.artifacts_metadata = metadata - build.artifacts_expire_in = params['expire_in'] + build.artifacts_expire_in = + params['expire_in'] || + Gitlab::CurrentSettings.current_application_settings + .default_artifacts_expire_in if build.save present(build, with: Entities::BuildDetails) @@ -166,8 +190,7 @@ module Ci # GET /builds/:id/artifacts get ":id/artifacts" do build = Ci::Build.find_by_id(params[:id]) - not_found! unless build - authenticate_build_token!(build) + authenticate_build!(build) artifacts_file = build.artifacts_file unless artifacts_file.file_storage? @@ -192,8 +215,7 @@ module Ci # DELETE /builds/:id/artifacts delete ":id/artifacts" do build = Ci::Build.find_by_id(params[:id]) - not_found! unless build - authenticate_build_token!(build) + authenticate_build!(build) build.erase_artifacts! end diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb index 3f5bdaba3f5..792ff628b09 100644 --- a/lib/ci/api/entities.rb +++ b/lib/ci/api/entities.rb @@ -15,6 +15,15 @@ module Ci expose :filename, :size end + class BuildOptions < Grape::Entity + expose :image + expose :services + expose :artifacts + expose :cache + expose :dependencies + expose :after_script + end + class Build < Grape::Entity expose :id, :ref, :tag, :sha, :status expose :name, :token, :stage @@ -23,6 +32,10 @@ module Ci expose :artifacts_file, using: ArtifactFile, if: ->(build, _) { build.artifacts? } end + class BuildCredentials < Grape::Entity + expose :type, :url, :username, :password + end + class BuildDetails < Build expose :commands expose :repo_url @@ -41,6 +54,8 @@ module Ci expose :variables expose :depends_on_builds, using: Build + + expose :credentials, using: BuildCredentials end class Runner < Grape::Entity diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb index 199d62d9b8a..996990b464f 100644 --- a/lib/ci/api/helpers.rb +++ b/lib/ci/api/helpers.rb @@ -1,9 +1,9 @@ module Ci module API module Helpers - BUILD_TOKEN_HEADER = "HTTP_BUILD_TOKEN" + BUILD_TOKEN_HEADER = "HTTP_BUILD_TOKEN".freeze BUILD_TOKEN_PARAM = :token - UPDATE_RUNNER_EVERY = 60 + UPDATE_RUNNER_EVERY = 10 * 60 def authenticate_runners! forbidden! unless runner_registration_token_valid? @@ -13,20 +13,57 @@ module Ci forbidden! unless current_runner end - def authenticate_build_token!(build) - token = (params[BUILD_TOKEN_PARAM] || env[BUILD_TOKEN_HEADER]).to_s - forbidden! unless token && build.valid_token?(token) + def authenticate_build!(build) + validate_build!(build) do + forbidden! unless build_token_valid?(build) + end + end + + def validate_build!(build) + not_found! unless build + + yield if block_given? + + forbidden!('Project has been deleted!') unless build.project + forbidden!('Build has been erased!') if build.erased? end def runner_registration_token_valid? - params[:token] == current_application_settings.runners_registration_token + ActiveSupport::SecurityUtils.variable_size_secure_compare( + params[:token], + current_application_settings.runners_registration_token) + end + + def build_token_valid?(build) + token = (params[BUILD_TOKEN_PARAM] || env[BUILD_TOKEN_HEADER]).to_s + + # We require to also check `runners_token` to maintain compatibility with old version of runners + token && (build.valid_token?(token) || build.project.valid_runners_token?(token)) + end + + def update_runner_info + return unless update_runner? + + current_runner.contacted_at = Time.now + current_runner.assign_attributes(get_runner_version_from_params) + current_runner.save if current_runner.changed? end - def update_runner_last_contact - # Use a random threshold to prevent beating DB updates + def update_runner? + # Use a random threshold to prevent beating DB updates. + # It generates a distribution between [40m, 80m]. + # contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY) - if current_runner.contacted_at.nil? || Time.now - current_runner.contacted_at >= contacted_at_max_age - current_runner.update_attributes(contacted_at: Time.now) + + current_runner.contacted_at.nil? || + (Time.now - current_runner.contacted_at) >= contacted_at_max_age + end + + def build_not_found! + if headers['User-Agent'].to_s =~ /gitlab-ci-multi-runner \d+\.\d+\.\d+(~beta\.\d+\.g[0-9a-f]+)? / + no_content! + else + not_found! end end @@ -36,12 +73,7 @@ module Ci def get_runner_version_from_params return unless params["info"].present? - attributes_for_keys(["name", "version", "revision", "platform", "architecture"], params["info"]) - end - - def update_runner_info - current_runner.assign_attributes(get_runner_version_from_params) - current_runner.save if current_runner.changed? + attributes_for_keys(%w(name version revision platform architecture), params["info"]) end def max_artifacts_size diff --git a/lib/ci/api/runners.rb b/lib/ci/api/runners.rb index bcc82969eb3..2a611a67eaf 100644 --- a/lib/ci/api/runners.rb +++ b/lib/ci/api/runners.rb @@ -1,44 +1,36 @@ module Ci module API - # Runners API class Runners < Grape::API resource :runners do - # Delete runner - # Parameters: - # token (required) - The unique token of runner - # - # Example Request: - # GET /runners/delete + desc 'Delete a runner' + params do + requires :token, type: String, desc: 'The unique token of the runner' + end delete "delete" do - required_attributes! [:token] authenticate_runner! Ci::Runner.find_by_token(params[:token]).destroy end - # Register a new runner - # - # Note: This is an "internal" API called when setting up - # runners, so it is authenticated differently. - # - # Parameters: - # token (required) - The unique token of runner - # - # Example Request: - # POST /runners/register + desc 'Register a new runner' do + success Entities::Runner + end + params do + requires :token, type: String, desc: 'The unique token of the runner' + optional :description, type: String, desc: 'The description of the runner' + optional :tag_list, type: Array[String], desc: 'A list of tags the runner should run for' + optional :run_untagged, type: Boolean, desc: 'Flag if the runner should execute untagged jobs' + optional :locked, type: Boolean, desc: 'Lock this runner for this specific project' + end post "register" do - required_attributes! [:token] - - attributes = attributes_for_keys( - [:description, :tag_list, :run_untagged, :locked] - ) + runner_params = declared(params, include_missing: false) runner = if runner_registration_token_valid? # Create shared runner. Requires admin access - Ci::Runner.create(attributes.merge(is_shared: true)) - elsif project = Project.find_by(runners_token: params[:token]) + Ci::Runner.create(runner_params.merge(is_shared: true)) + elsif project = Project.find_by(runners_token: runner_params[:token]) # Create a specific runner for project. - project.runners.create(attributes) + project.runners.create(runner_params) end return forbidden! unless runner diff --git a/lib/ci/api/triggers.rb b/lib/ci/api/triggers.rb index 63b42113513..6e622601680 100644 --- a/lib/ci/api/triggers.rb +++ b/lib/ci/api/triggers.rb @@ -1,41 +1,30 @@ module Ci module API - # Build Trigger API class Triggers < Grape::API resource :projects do - # Trigger a GitLab CI project build - # - # Parameters: - # id (required) - The ID of a CI project - # ref (required) - The name of project's branch or tag - # token (required) - The uniq token of trigger - # Example Request: - # POST /projects/:id/ref/:ref/trigger + desc 'Trigger a GitLab CI project build' do + success Entities::TriggerRequest + end + params do + requires :id, type: Integer, desc: 'The ID of a CI project' + requires :ref, type: String, desc: "The name of project's branch or tag" + requires :token, type: String, desc: 'The unique token of the trigger' + optional :variables, type: Hash, desc: 'Optional build variables' + end post ":id/refs/:ref/trigger" do - required_attributes! [:token] - - project = Project.find_by(ci_id: params[:id].to_i) - trigger = Ci::Trigger.find_by_token(params[:token].to_s) + project = Project.find_by(ci_id: params[:id]) + trigger = Ci::Trigger.find_by_token(params[:token]) not_found! unless project && trigger unauthorized! unless trigger.project == project - # validate variables - variables = params[:variables] - if variables - unless variables.is_a?(Hash) - render_api_error!('variables needs to be a hash', 400) - end - - unless variables.all? { |key, value| key.is_a?(String) && value.is_a?(String) } - render_api_error!('variables needs to be a map of key-valued strings', 400) - end - - # convert variables from Mash to Hash - variables = variables.to_h + # Validate variables + variables = params[:variables].to_h + unless variables.all? { |key, value| key.is_a?(String) && value.is_a?(String) } + render_api_error!('variables needs to be a map of key-valued strings', 400) end # create request and trigger builds - trigger_request = Ci::CreateTriggerRequestService.new.execute(project, trigger, params[:ref].to_s, variables) + trigger_request = Ci::CreateTriggerRequestService.new.execute(project, trigger, params[:ref], variables) if trigger_request present trigger_request, with: Entities::TriggerRequest else diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 47efd5bd9f2..649ee4d018b 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -2,9 +2,9 @@ module Ci class GitlabCiYamlProcessor class ValidationError < StandardError; end - include Gitlab::Ci::Config::Node::LegacyValidationHelpers + include Gitlab::Ci::Config::Entry::LegacyValidationHelpers - attr_reader :path, :cache, :stages + attr_reader :path, :cache, :stages, :jobs def initialize(config, path = nil) @ci_config = Gitlab::Ci::Config.new(config) @@ -55,29 +55,37 @@ module Ci { stage_idx: @stages.index(job[:stage]), stage: job[:stage], - ## - # Refactoring note: - # - before script behaves differently than after script - # - after script returns an array of commands - # - before script should be a concatenated command - commands: [job[:before_script] || @before_script, job[:script]].flatten.compact.join("\n"), + commands: job[:commands], tag_list: job[:tags] || [], name: job[:name].to_s, allow_failure: job[:allow_failure] || false, when: job[:when] || 'on_success', - environment: job[:environment], + environment: job[:environment_name], + coverage_regex: job[:coverage], yaml_variables: yaml_variables(name), options: { - image: job[:image] || @image, - services: job[:services] || @services, + image: job[:image], + services: job[:services], artifacts: job[:artifacts], - cache: job[:cache] || @cache, + cache: job[:cache], dependencies: job[:dependencies], - after_script: job[:after_script] || @after_script, + after_script: job[:after_script], + environment: job[:environment], }.compact } end + def self.validation_message(content) + return 'Please provide content of .gitlab-ci.yml' if content.blank? + + begin + Ci::GitlabCiYamlProcessor.new(content) + nil + rescue ValidationError, Psych::SyntaxError => e + e.message + end + end + private def initial_parsing @@ -102,6 +110,7 @@ module Ci validate_job_stage!(name, job) validate_job_dependencies!(name, job) + validate_job_environment!(name, job) end end @@ -110,7 +119,7 @@ module Ci .merge(job_variables(name)) variables.map do |key, value| - { key: key, value: value, public: true } + { key: key.to_s, value: value, public: true } end end @@ -143,6 +152,35 @@ module Ci end end + def validate_job_environment!(name, job) + return unless job[:environment] + return unless job[:environment].is_a?(Hash) + + environment = job[:environment] + validate_on_stop_job!(name, environment, environment[:on_stop]) + end + + def validate_on_stop_job!(name, environment, on_stop) + return unless on_stop + + on_stop_job = @jobs[on_stop.to_sym] + unless on_stop_job + raise ValidationError, "#{name} job: on_stop job #{on_stop} is not defined" + end + + unless on_stop_job[:environment] + raise ValidationError, "#{name} job: on_stop job #{on_stop} does not have environment defined" + end + + unless on_stop_job[:environment][:name] == environment[:name] + raise ValidationError, "#{name} job: on_stop job #{on_stop} have different environment name" + end + + unless on_stop_job[:environment][:action] == 'stop' + raise ValidationError, "#{name} job: on_stop job #{on_stop} needs to have action stop defined" + end + end + def process?(only_params, except_params, ref, tag, trigger_request) if only_params.present? return false unless matching?(only_params, ref, tag, trigger_request) diff --git a/lib/ci/mask_secret.rb b/lib/ci/mask_secret.rb new file mode 100644 index 00000000000..997377abc55 --- /dev/null +++ b/lib/ci/mask_secret.rb @@ -0,0 +1,10 @@ +module Ci::MaskSecret + class << self + def mask!(value, token) + return value unless value.present? && token.present? + + value.gsub!(token, 'x' * token.length) + value + end + end +end diff --git a/lib/ci/version_info.rb b/lib/ci/version_info.rb deleted file mode 100644 index 2a87c91db5e..00000000000 --- a/lib/ci/version_info.rb +++ /dev/null @@ -1,52 +0,0 @@ -class VersionInfo - include Comparable - - attr_reader :major, :minor, :patch - - def self.parse(str) - if str && m = str.match(/(\d+)\.(\d+)\.(\d+)/) - VersionInfo.new(m[1].to_i, m[2].to_i, m[3].to_i) - else - VersionInfo.new - end - end - - def initialize(major = 0, minor = 0, patch = 0) - @major = major - @minor = minor - @patch = patch - end - - def <=>(other) - return unless other.is_a? VersionInfo - return unless valid? && other.valid? - - if other.major < @major - 1 - elsif @major < other.major - -1 - elsif other.minor < @minor - 1 - elsif @minor < other.minor - -1 - elsif other.patch < @patch - 1 - elsif @patch < other.patch - -1 - else - 0 - end - end - - def to_s - if valid? - "%d.%d.%d" % [@major, @minor, @patch] - else - "Unknown" - end - end - - def valid? - @major >= 0 && @minor >= 0 && @patch >= 0 && @major + @minor + @patch > 0 - end -end diff --git a/lib/constraints/group_url_constrainer.rb b/lib/constraints/group_url_constrainer.rb new file mode 100644 index 00000000000..bae4db1ca4d --- /dev/null +++ b/lib/constraints/group_url_constrainer.rb @@ -0,0 +1,17 @@ +class GroupUrlConstrainer + def matches?(request) + id = request.params[:id] + + return false unless valid?(id) + + Group.find_by_full_path(id).present? + end + + private + + def valid?(id) + id.split('/').all? do |namespace| + NamespaceValidator.valid?(namespace) + end + end +end diff --git a/lib/constraints/project_url_constrainer.rb b/lib/constraints/project_url_constrainer.rb new file mode 100644 index 00000000000..a10b4657d7d --- /dev/null +++ b/lib/constraints/project_url_constrainer.rb @@ -0,0 +1,13 @@ +class ProjectUrlConstrainer + def matches?(request) + namespace_path = request.params[:namespace_id] + project_path = request.params[:project_id] || request.params[:id] + full_path = namespace_path + '/' + project_path + + unless ProjectPathValidator.valid?(project_path) + return false + end + + Project.find_by_full_path(full_path).present? + end +end diff --git a/lib/constraints/user_url_constrainer.rb b/lib/constraints/user_url_constrainer.rb new file mode 100644 index 00000000000..9ab5bcb12ff --- /dev/null +++ b/lib/constraints/user_url_constrainer.rb @@ -0,0 +1,5 @@ +class UserUrlConstrainer + def matches?(request) + User.find_by_username(request.params[:username]).present? + end +end diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb index 2edddb84fc3..7f5f6d9ddb6 100644 --- a/lib/container_registry/client.rb +++ b/lib/container_registry/client.rb @@ -5,7 +5,7 @@ module ContainerRegistry class Client attr_accessor :uri - MANIFEST_VERSION = 'application/vnd.docker.distribution.manifest.v2+json' + MANIFEST_VERSION = 'application/vnd.docker.distribution.manifest.v2+json'.freeze # Taken from: FaradayMiddleware::FollowRedirects REDIRECT_CODES = Set.new [301, 302, 303, 307] diff --git a/lib/email_template_interceptor.rb b/lib/email_template_interceptor.rb new file mode 100644 index 00000000000..63f9f8d7a5a --- /dev/null +++ b/lib/email_template_interceptor.rb @@ -0,0 +1,13 @@ +# Read about interceptors in http://guides.rubyonrails.org/action_mailer_basics.html#intercepting-emails +class EmailTemplateInterceptor + include Gitlab::CurrentSettings + + def self.delivering_email(message) + # Remove HTML part if HTML emails are disabled. + unless current_application_settings.html_emails_enabled + message.parts.delete_if do |part| + part.content_type.start_with?('text/html') + end + end + end +end diff --git a/lib/event_filter.rb b/lib/event_filter.rb index 668d2fa41b3..515095af1c2 100644 --- a/lib/event_filter.rb +++ b/lib/event_filter.rb @@ -2,8 +2,8 @@ class EventFilter attr_accessor :params class << self - def default_filter - %w{ push issues merge_requests team} + def all + 'all' end def push @@ -14,6 +14,10 @@ class EventFilter 'merged' end + def issue + 'issue' + end + def comments 'comments' end @@ -32,22 +36,20 @@ class EventFilter end def apply_filter(events) - return events unless params.present? - - filter = params.dup - - actions = [] - actions << Event::PUSHED if filter.include? 'push' - actions << Event::MERGED if filter.include? 'merged' - - if filter.include? 'team' - actions << Event::JOINED - actions << Event::LEFT + return events if params.blank? || params == EventFilter.all + + case params + when EventFilter.push + events.where(action: Event::PUSHED) + when EventFilter.merged + events.where(action: Event::MERGED) + when EventFilter.comments + events.where(action: Event::COMMENTED) + when EventFilter.team + events.where(action: [Event::JOINED, Event::LEFT, Event::EXPIRED]) + when EventFilter.issue + events.where(action: [Event::CREATED, Event::UPDATED, Event::CLOSED, Event::REOPENED]) end - - actions << Event::COMMENTED if filter.include? 'comments' - - events.where(action: actions) end def options(key) @@ -63,6 +65,10 @@ class EventFilter end def active?(key) - params.include? key + if params.present? + params.include? key + else + key == EventFilter.all + end end end diff --git a/lib/expand_variables.rb b/lib/expand_variables.rb new file mode 100644 index 00000000000..7b1533d0d32 --- /dev/null +++ b/lib/expand_variables.rb @@ -0,0 +1,17 @@ +module ExpandVariables + class << self + def expand(value, variables) + # Convert hash array to variables + if variables.is_a?(Array) + variables = variables.reduce({}) do |hash, variable| + hash[variable[:key]] = variable[:value] + hash + end + end + + value.gsub(/\$([a-zA-Z_][a-zA-Z0-9_]*)|\${\g<1>}|%\g<1>%/) do + variables[$1 || $2] + end + end + end +end diff --git a/lib/extracts_path.rb b/lib/extracts_path.rb index 84688f6646e..9ece84cc469 100644 --- a/lib/extracts_path.rb +++ b/lib/extracts_path.rb @@ -42,7 +42,7 @@ module ExtractsPath return pair unless @project - if id.match(/^([[:alnum:]]{40})(.+)/) + if id =~ /^(\h{40})(.+)/ # If the ref appears to be a SHA, we're done, just split the string pair = $~.captures else @@ -52,8 +52,7 @@ module ExtractsPath # Append a trailing slash if we only get a ref and no file path id += '/' unless id.ends_with?('/') - valid_refs = @project.repository.ref_names - valid_refs.select! { |v| id.start_with?("#{v}/") } + valid_refs = ref_names.select { |v| id.start_with?("#{v}/") } if valid_refs.length == 0 # No exact ref match, so just try our best @@ -74,6 +73,19 @@ module ExtractsPath pair end + # If we have an ID of 'foo.atom', and the controller provides Atom and HTML + # formats, then we have to check if the request was for the Atom version of + # the ID without the '.atom' suffix, or the HTML version of the ID including + # the suffix. We only check this if the version including the suffix doesn't + # match, so it is possible to create a branch which has an unroutable Atom + # feed. + def extract_ref_without_atom(id) + id_without_atom = id.sub(/\.atom$/, '') + valid_refs = ref_names.select { |v| "#{id_without_atom}/".start_with?("#{v}/") } + + valid_refs.max_by(&:length) + end + # Assigns common instance variables for views working with Git tree-ish objects # # Assignments are: @@ -86,21 +98,29 @@ module ExtractsPath # If the :id parameter appears to be requesting a specific response format, # that will be handled as well. # + # If there is no path and the ref doesn't exist in the repo, try to resolve + # the ref without an '.atom' suffix. If _that_ ref is found, set the request's + # format to Atom manually. + # # Automatically renders `not_found!` if a valid tree path could not be # resolved (e.g., when a user inserts an invalid path or ref). def assign_ref_vars # assign allowed options - allowed_options = ["filter_ref", "extended_sha1"] + allowed_options = ["filter_ref"] @options = params.select {|key, value| allowed_options.include?(key) && !value.blank? } @options = HashWithIndifferentAccess.new(@options) - @id = Addressable::URI.normalize_component(get_id) + @id = get_id @ref, @path = extract_ref(@id) @repo = @project.repository - if @options[:extended_sha1].blank? + + @commit = @repo.commit(@ref) + + if @path.empty? && !@commit && @id.ends_with?('.atom') + @id = @ref = extract_ref_without_atom(@id) @commit = @repo.commit(@ref) - else - @commit = @repo.commit(@options[:extended_sha1]) + + request.format = :atom if @commit end raise InvalidPathError unless @commit @@ -119,9 +139,16 @@ module ExtractsPath private + # overriden in subclasses, do not remove def get_id id = params[:id] || params[:ref] id += "/" + params[:path] unless params[:path].blank? id end + + def ref_names + return [] unless @project + + @ref_names ||= @project.repository.ref_names + end end diff --git a/lib/file_size_validator.rb b/lib/file_size_validator.rb index 440dd44ece7..eb19ab45ac3 100644 --- a/lib/file_size_validator.rb +++ b/lib/file_size_validator.rb @@ -32,9 +32,9 @@ class FileSizeValidator < ActiveModel::EachValidator end def validate_each(record, attribute, value) - raise(ArgumentError, "A CarrierWave::Uploader::Base object was expected") unless value.kind_of? CarrierWave::Uploader::Base + raise(ArgumentError, "A CarrierWave::Uploader::Base object was expected") unless value.is_a? CarrierWave::Uploader::Base - value = (options[:tokenizer] || DEFAULT_TOKENIZER).call(value) if value.kind_of?(String) + value = (options[:tokenizer] || DEFAULT_TOKENIZER).call(value) if value.is_a?(String) CHECKS.each do |key, validity_check| next unless check_value = options[key] diff --git a/lib/generators/rails/post_deployment_migration/post_deployment_migration_generator.rb b/lib/generators/rails/post_deployment_migration/post_deployment_migration_generator.rb new file mode 100644 index 00000000000..7cb4bccb23c --- /dev/null +++ b/lib/generators/rails/post_deployment_migration/post_deployment_migration_generator.rb @@ -0,0 +1,15 @@ +require 'rails/generators' + +module Rails + class PostDeploymentMigrationGenerator < Rails::Generators::NamedBase + def create_migration_file + timestamp = Time.now.strftime('%Y%m%d%H%I%S') + + template "migration.rb", "db/post_migrate/#{timestamp}_#{file_name}.rb" + end + + def migration_class_name + file_name.camelize + end + end +end diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb index a533bac2692..3b210eeda9d 100644 --- a/lib/gitlab/access.rb +++ b/lib/gitlab/access.rb @@ -21,9 +21,7 @@ module Gitlab PROTECTION_DEV_CAN_MERGE = 3 class << self - def values - options.values - end + delegate :values, to: :options def all_values options_with_owner.values @@ -53,6 +51,10 @@ module Gitlab } end + def sym_options_with_owner + sym_options.merge(owner: OWNER) + end + def protection_options { "Not protected: Both developers and masters can push new commits, force push, or delete the branch." => PROTECTION_NONE, diff --git a/lib/gitlab/akismet_helper.rb b/lib/gitlab/akismet_helper.rb deleted file mode 100644 index 207736b59db..00000000000 --- a/lib/gitlab/akismet_helper.rb +++ /dev/null @@ -1,47 +0,0 @@ -module Gitlab - module AkismetHelper - def akismet_enabled? - current_application_settings.akismet_enabled - end - - def akismet_client - @akismet_client ||= ::Akismet::Client.new(current_application_settings.akismet_api_key, - Gitlab.config.gitlab.url) - end - - def client_ip(env) - env['action_dispatch.remote_ip'].to_s - end - - def user_agent(env) - env['HTTP_USER_AGENT'] - end - - def check_for_spam?(project) - akismet_enabled? && project.public? - end - - def is_spam?(environment, user, text) - client = akismet_client - ip_address = client_ip(environment) - user_agent = user_agent(environment) - - params = { - type: 'comment', - text: text, - created_at: DateTime.now, - author: user.name, - author_email: user.email, - referrer: environment['HTTP_REFERER'], - } - - begin - is_spam, is_blatant = client.check(ip_address, user_agent, params) - is_spam || is_blatant - rescue => e - Rails.logger.error("Unable to connect to Akismet: #{e}, skipping check") - false - end - end - end -end diff --git a/lib/gitlab/allowable.rb b/lib/gitlab/allowable.rb new file mode 100644 index 00000000000..f48abcc86d5 --- /dev/null +++ b/lib/gitlab/allowable.rb @@ -0,0 +1,7 @@ +module Gitlab + module Allowable + def can?(user, action, subject) + Ability.allowed?(user, action, subject) + end + end +end diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb index 1a22ad9acf5..d575367d81a 100644 --- a/lib/gitlab/asciidoc.rb +++ b/lib/gitlab/asciidoc.rb @@ -1,4 +1,6 @@ require 'asciidoctor' +require 'asciidoctor/converter/html5' +require "asciidoctor-plantuml" module Gitlab # Parser/renderer for the AsciiDoc format that uses Asciidoctor and filters @@ -6,7 +8,7 @@ module Gitlab module Asciidoc DEFAULT_ADOC_ATTRS = [ 'showtitle', 'idprefix=user-content-', 'idseparator=-', 'env=gitlab', - 'env-gitlab', 'source-highlighter=html-pipeline' + 'env-gitlab', 'source-highlighter=html-pipeline', 'icons=font' ].freeze # Public: Converts the provided Asciidoc markup into HTML. @@ -23,16 +25,54 @@ module Gitlab def self.render(input, context, asciidoc_opts = {}) asciidoc_opts.reverse_merge!( safe: :secure, - backend: :html5, + backend: :gitlab_html5, attributes: [] ) asciidoc_opts[:attributes].unshift(*DEFAULT_ADOC_ATTRS) + plantuml_setup + html = ::Asciidoctor.convert(input, asciidoc_opts) html = Banzai.post_process(html, context) + filter = Banzai::Filter::SanitizationFilter.new(html) + html = filter.call.to_s + html.html_safe end + + def self.plantuml_setup + Asciidoctor::PlantUml.configure do |conf| + conf.url = ApplicationSetting.current.plantuml_url + conf.svg_enable = ApplicationSetting.current.plantuml_enabled + conf.png_enable = ApplicationSetting.current.plantuml_enabled + conf.txt_enable = false + end + end + + class Html5Converter < Asciidoctor::Converter::Html5Converter + extend Asciidoctor::Converter::Config + + register_for 'gitlab_html5' + + def stem(node) + return super unless node.style.to_sym == :latexmath + + %(<pre#{id_attribute(node)} class="code math js-render-math #{node.role}" data-math-style="display"><code>#{node.content}</code></pre>) + end + + def inline_quoted(node) + return super unless node.type.to_sym == :latexmath + + %(<code#{id_attribute(node)} class="code math js-render-math #{node.role}" data-math-style="inline">#{node.text}</code>) + end + + private + + def id_attribute(node) + node.id ? %( id="#{node.id}") : nil + end + end end end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index db1704af75e..89db6c3da46 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -1,22 +1,29 @@ module Gitlab module Auth - Result = Struct.new(:user, :type) + class MissingPersonalTokenError < StandardError; end + + SCOPES = [:api, :read_user].freeze + DEFAULT_SCOPES = [:api].freeze + OPTIONAL_SCOPES = SCOPES - DEFAULT_SCOPES class << self def find_for_git_client(login, password, project:, ip:) raise "Must provide an IP for rate limiting" if ip.nil? - result = Result.new + # `user_with_password_for_git` should be the last check + # because it's the most expensive, especially when LDAP + # is enabled. + result = + service_request_check(login, password, project) || + build_access_token_check(login, password) || + lfs_token_check(login, password) || + oauth_access_token_check(login, password) || + personal_access_token_check(login, password) || + user_with_password_for_git(login, password) || + Gitlab::Auth::Result.new - if valid_ci_request?(login, password, project) - result.type = :ci - elsif result.user = find_with_user_password(login, password) - result.type = :gitlab_or_ldap - elsif result.user = oauth_access_token_check(login, password) - result.type = :oauth - end + rate_limit!(ip, success: result.success?, login: login) - rate_limit!(ip, success: !!result.user || (result.type == :ci), login: login) result end @@ -58,30 +65,134 @@ module Gitlab private - def valid_ci_request?(login, password, project) + def service_request_check(login, password, project) matched_login = /(?<service>^[a-zA-Z]*-ci)-token$/.match(login) - return false unless project && matched_login.present? + return unless project && matched_login.present? underscored_service = matched_login['service'].underscore - if underscored_service == 'gitlab_ci' - project && project.valid_build_token?(password) - elsif Service.available_services_names.include?(underscored_service) + if Service.available_services_names.include?(underscored_service) # We treat underscored_service as a trusted input because it is included # in the Service.available_services_names whitelist. service = project.public_send("#{underscored_service}_service") - service && service.activated? && service.valid_token?(password) + if service && service.activated? && service.valid_token?(password) + Gitlab::Auth::Result.new(nil, project, :ci, build_authentication_abilities) + end end end + def user_with_password_for_git(login, password) + user = find_with_user_password(login, password) + return unless user + + raise Gitlab::Auth::MissingPersonalTokenError if user.two_factor_enabled? + + Gitlab::Auth::Result.new(user, nil, :gitlab_or_ldap, full_authentication_abilities) + end + def oauth_access_token_check(login, password) if login == "oauth2" && password.present? token = Doorkeeper::AccessToken.by_token(password) - token && token.accessible? && User.find_by(id: token.resource_owner_id) + if valid_oauth_token?(token) + user = User.find_by(id: token.resource_owner_id) + Gitlab::Auth::Result.new(user, nil, :oauth, read_authentication_abilities) + end + end + end + + def personal_access_token_check(login, password) + if login && password + token = PersonalAccessToken.active.find_by_token(password) + validation = User.by_login(login) + + if valid_personal_access_token?(token, validation) + Gitlab::Auth::Result.new(validation, nil, :personal_token, full_authentication_abilities) + end + end + end + + def valid_oauth_token?(token) + token && token.accessible? && valid_api_token?(token) + end + + def valid_personal_access_token?(token, user) + token && token.user == user && valid_api_token?(token) + end + + def valid_api_token?(token) + AccessTokenValidationService.new(token).include_any_scope?(['api']) + end + + def lfs_token_check(login, password) + deploy_key_matches = login.match(/\Alfs\+deploy-key-(\d+)\z/) + + actor = + if deploy_key_matches + DeployKey.find(deploy_key_matches[1]) + else + User.by_login(login) + end + + return unless actor + + token_handler = Gitlab::LfsToken.new(actor) + + authentication_abilities = + if token_handler.user? + full_authentication_abilities + else + read_authentication_abilities + end + + if Devise.secure_compare(token_handler.token, password) + Gitlab::Auth::Result.new(actor, nil, token_handler.type, authentication_abilities) end end + + def build_access_token_check(login, password) + return unless login == 'gitlab-ci-token' + return unless password + + build = ::Ci::Build.running.find_by_token(password) + return unless build + return unless build.project.builds_enabled? + + if build.user + # If user is assigned to build, use restricted credentials of user + Gitlab::Auth::Result.new(build.user, build.project, :build, build_authentication_abilities) + else + # Otherwise use generic CI credentials (backward compatibility) + Gitlab::Auth::Result.new(nil, build.project, :ci, build_authentication_abilities) + end + end + + public + + def build_authentication_abilities + [ + :read_project, + :build_download_code, + :build_read_container_image, + :build_create_container_image + ] + end + + def read_authentication_abilities + [ + :read_project, + :download_code, + :read_container_image + ] + end + + def full_authentication_abilities + read_authentication_abilities + [ + :push_code, + :create_container_image + ] + end end end end diff --git a/lib/gitlab/auth/result.rb b/lib/gitlab/auth/result.rb new file mode 100644 index 00000000000..39b86c61a18 --- /dev/null +++ b/lib/gitlab/auth/result.rb @@ -0,0 +1,20 @@ +module Gitlab + module Auth + Result = Struct.new(:actor, :project, :type, :authentication_abilities) do + def ci?(for_project) + type == :ci && + project && + project == for_project + end + + def lfs_deploy_token?(for_project) + type == :lfs_deploy_token && + actor.try(:has_access_to?, for_project) + end + + def success? + actor.present? || type == :ci + end + end + end +end diff --git a/lib/gitlab/award_emoji.rb b/lib/gitlab/award_emoji.rb index 39b43ab5489..7555326d384 100644 --- a/lib/gitlab/award_emoji.rb +++ b/lib/gitlab/award_emoji.rb @@ -69,11 +69,12 @@ module Gitlab end JSON.parse(File.read(path)).map do |hash| - if digest - fname = "#{hash['unicode']}-#{hash['digest']}" - else - fname = hash['unicode'] - end + fname = + if digest + "#{hash['unicode']}-#{hash['digest']}" + else + hash['unicode'] + end { name: hash['name'], path: File.join(base, prefix, "#{fname}.png") } end diff --git a/lib/gitlab/badge/base.rb b/lib/gitlab/badge/base.rb new file mode 100644 index 00000000000..909fa24fa90 --- /dev/null +++ b/lib/gitlab/badge/base.rb @@ -0,0 +1,21 @@ +module Gitlab + module Badge + class Base + def entity + raise NotImplementedError + end + + def status + raise NotImplementedError + end + + def metadata + raise NotImplementedError + end + + def template + raise NotImplementedError + end + end + end +end diff --git a/lib/gitlab/badge/build.rb b/lib/gitlab/badge/build.rb deleted file mode 100644 index 1de721a2269..00000000000 --- a/lib/gitlab/badge/build.rb +++ /dev/null @@ -1,30 +0,0 @@ -module Gitlab - module Badge - ## - # Build badge - # - class Build - delegate :key_text, :value_text, to: :template - - def initialize(project, ref) - @project = project - @ref = ref - @sha = @project.commit(@ref).try(:sha) - end - - def status - @project.pipelines - .where(sha: @sha, ref: @ref) - .status || 'unknown' - end - - def metadata - @metadata ||= Build::Metadata.new(@project, @ref) - end - - def template - @template ||= Build::Template.new(status) - end - end - end -end diff --git a/lib/gitlab/badge/build/metadata.rb b/lib/gitlab/badge/build/metadata.rb index 553ef8d7b16..f87a7b7942e 100644 --- a/lib/gitlab/badge/build/metadata.rb +++ b/lib/gitlab/badge/build/metadata.rb @@ -1,25 +1,17 @@ module Gitlab module Badge - class Build + module Build ## # Class that describes build badge metadata # - class Metadata - include Gitlab::Application.routes.url_helpers - include ActionView::Helpers::AssetTagHelper - include ActionView::Helpers::UrlHelper - - def initialize(project, ref) - @project = project - @ref = ref - end - - def to_html - link_to(image_tag(image_url, alt: 'build status'), link_url) + class Metadata < Badge::Metadata + def initialize(badge) + @project = badge.project + @ref = badge.ref end - def to_markdown - "[](#{link_url})" + def title + 'build status' end def image_url diff --git a/lib/gitlab/badge/build/status.rb b/lib/gitlab/badge/build/status.rb new file mode 100644 index 00000000000..b762d85b6e5 --- /dev/null +++ b/lib/gitlab/badge/build/status.rb @@ -0,0 +1,37 @@ +module Gitlab + module Badge + module Build + ## + # Build status badge + # + class Status < Badge::Base + attr_reader :project, :ref + + def initialize(project, ref) + @project = project + @ref = ref + + @sha = @project.commit(@ref).try(:sha) + end + + def entity + 'build' + end + + def status + @project.pipelines + .where(sha: @sha) + .latest_status(@ref) || 'unknown' + end + + def metadata + @metadata ||= Build::Metadata.new(self) + end + + def template + @template ||= Build::Template.new(self) + end + end + end + end +end diff --git a/lib/gitlab/badge/build/template.rb b/lib/gitlab/badge/build/template.rb index deba3b669b3..bc0e0cd441d 100644 --- a/lib/gitlab/badge/build/template.rb +++ b/lib/gitlab/badge/build/template.rb @@ -1,12 +1,12 @@ module Gitlab module Badge - class Build + module Build ## # Class that represents a build badge template. # # Template object will be passed to badge.svg.erb template. # - class Template + class Template < Badge::Template STATUS_COLOR = { success: '#4c1', failed: '#e05d44', @@ -15,18 +15,19 @@ module Gitlab canceled: '#9f9f9f', skipped: '#9f9f9f', unknown: '#9f9f9f' - } + }.freeze - def initialize(status) - @status = status + def initialize(badge) + @entity = badge.entity + @status = badge.status end def key_text - 'build' + @entity.to_s end def value_text - @status + @status.to_s end def key_width @@ -37,25 +38,8 @@ module Gitlab 54 end - def key_color - '#555' - end - def value_color - STATUS_COLOR[@status.to_sym] || - STATUS_COLOR[:unknown] - end - - def key_text_anchor - key_width / 2 - end - - def value_text_anchor - key_width + (value_width / 2) - end - - def width - key_width + value_width + STATUS_COLOR[@status.to_sym] || STATUS_COLOR[:unknown] end end end diff --git a/lib/gitlab/badge/coverage/metadata.rb b/lib/gitlab/badge/coverage/metadata.rb new file mode 100644 index 00000000000..53588185622 --- /dev/null +++ b/lib/gitlab/badge/coverage/metadata.rb @@ -0,0 +1,30 @@ +module Gitlab + module Badge + module Coverage + ## + # Class that describes coverage badge metadata + # + class Metadata < Badge::Metadata + def initialize(badge) + @project = badge.project + @ref = badge.ref + @job = badge.job + end + + def title + 'coverage report' + end + + def image_url + coverage_namespace_project_badges_url(@project.namespace, + @project, @ref, + format: :svg) + end + + def link_url + namespace_project_commits_url(@project.namespace, @project, id: @ref) + end + end + end + end +end diff --git a/lib/gitlab/badge/coverage/report.rb b/lib/gitlab/badge/coverage/report.rb new file mode 100644 index 00000000000..9a0482306b7 --- /dev/null +++ b/lib/gitlab/badge/coverage/report.rb @@ -0,0 +1,53 @@ +module Gitlab + module Badge + module Coverage + ## + # Test coverage report badge + # + class Report < Badge::Base + attr_reader :project, :ref, :job + + def initialize(project, ref, job = nil) + @project = project + @ref = ref + @job = job + + @pipeline = @project.pipelines.latest_successful_for(@ref) + end + + def entity + 'coverage' + end + + def status + @coverage ||= raw_coverage + return unless @coverage + + @coverage.to_i + end + + def metadata + @metadata ||= Coverage::Metadata.new(self) + end + + def template + @template ||= Coverage::Template.new(self) + end + + private + + def raw_coverage + return unless @pipeline + + if @job.blank? + @pipeline.coverage + else + @pipeline.builds + .find_by(name: @job) + .try(:coverage) + end + end + end + end + end +end diff --git a/lib/gitlab/badge/coverage/template.rb b/lib/gitlab/badge/coverage/template.rb new file mode 100644 index 00000000000..fcecb1d9665 --- /dev/null +++ b/lib/gitlab/badge/coverage/template.rb @@ -0,0 +1,52 @@ +module Gitlab + module Badge + module Coverage + ## + # Class that represents a coverage badge template. + # + # Template object will be passed to badge.svg.erb template. + # + class Template < Badge::Template + STATUS_COLOR = { + good: '#4c1', + acceptable: '#a3c51c', + medium: '#dfb317', + low: '#e05d44', + unknown: '#9f9f9f' + }.freeze + + def initialize(badge) + @entity = badge.entity + @status = badge.status + end + + def key_text + @entity.to_s + end + + def value_text + @status ? "#{@status}%" : 'unknown' + end + + def key_width + 62 + end + + def value_width + @status ? 36 : 58 + end + + def value_color + case @status + when 95..100 then STATUS_COLOR[:good] + when 90..95 then STATUS_COLOR[:acceptable] + when 75..90 then STATUS_COLOR[:medium] + when 0..75 then STATUS_COLOR[:low] + else + STATUS_COLOR[:unknown] + end + end + end + end + end +end diff --git a/lib/gitlab/badge/metadata.rb b/lib/gitlab/badge/metadata.rb new file mode 100644 index 00000000000..4a049ef758d --- /dev/null +++ b/lib/gitlab/badge/metadata.rb @@ -0,0 +1,40 @@ +module Gitlab + module Badge + ## + # Abstract class for badge metadata + # + class Metadata + include Gitlab::Application.routes.url_helpers + include ActionView::Helpers::AssetTagHelper + include ActionView::Helpers::UrlHelper + + def initialize(badge) + @badge = badge + end + + def to_html + link_to(image_tag(image_url, alt: title), link_url) + end + + def to_markdown + "[](#{link_url})" + end + + def to_asciidoc + "image:#{image_url}[link=\"#{link_url}\",title=\"#{title}\"]" + end + + def title + raise NotImplementedError + end + + def image_url + raise NotImplementedError + end + + def link_url + raise NotImplementedError + end + end + end +end diff --git a/lib/gitlab/badge/template.rb b/lib/gitlab/badge/template.rb new file mode 100644 index 00000000000..bfeb0052642 --- /dev/null +++ b/lib/gitlab/badge/template.rb @@ -0,0 +1,49 @@ +module Gitlab + module Badge + ## + # Abstract template class for badges + # + class Template + def initialize(badge) + @entity = badge.entity + @status = badge.status + end + + def key_text + raise NotImplementedError + end + + def value_text + raise NotImplementedError + end + + def key_width + raise NotImplementedError + end + + def value_width + raise NotImplementedError + end + + def value_color + raise NotImplementedError + end + + def key_color + '#555' + end + + def key_text_anchor + key_width / 2 + end + + def value_text_anchor + key_width + (value_width / 2) + end + + def width + key_width + value_width + end + end + end +end diff --git a/lib/gitlab/bitbucket_import.rb b/lib/gitlab/bitbucket_import.rb deleted file mode 100644 index 7298152e7e9..00000000000 --- a/lib/gitlab/bitbucket_import.rb +++ /dev/null @@ -1,6 +0,0 @@ -module Gitlab - module BitbucketImport - mattr_accessor :public_key - @public_key = nil - end -end diff --git a/lib/gitlab/bitbucket_import/client.rb b/lib/gitlab/bitbucket_import/client.rb deleted file mode 100644 index 8d1ad62fae0..00000000000 --- a/lib/gitlab/bitbucket_import/client.rb +++ /dev/null @@ -1,142 +0,0 @@ -module Gitlab - module BitbucketImport - class Client - class Unauthorized < StandardError; end - - attr_reader :consumer, :api - - def self.from_project(project) - import_data_credentials = project.import_data.credentials if project.import_data - if import_data_credentials && import_data_credentials[:bb_session] - token = import_data_credentials[:bb_session][:bitbucket_access_token] - token_secret = import_data_credentials[:bb_session][:bitbucket_access_token_secret] - new(token, token_secret) - else - raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{project.id}" - end - end - - def initialize(access_token = nil, access_token_secret = nil) - @consumer = ::OAuth::Consumer.new( - config.app_id, - config.app_secret, - bitbucket_options - ) - - if access_token && access_token_secret - @api = ::OAuth::AccessToken.new(@consumer, access_token, access_token_secret) - end - end - - def request_token(redirect_uri) - request_token = consumer.get_request_token(oauth_callback: redirect_uri) - - { - oauth_token: request_token.token, - oauth_token_secret: request_token.secret, - oauth_callback_confirmed: request_token.callback_confirmed?.to_s - } - end - - def authorize_url(request_token, redirect_uri) - request_token = ::OAuth::RequestToken.from_hash(consumer, request_token) if request_token.is_a?(Hash) - - if request_token.callback_confirmed? - request_token.authorize_url - else - request_token.authorize_url(oauth_callback: redirect_uri) - end - end - - def get_token(request_token, oauth_verifier, redirect_uri) - request_token = ::OAuth::RequestToken.from_hash(consumer, request_token) if request_token.is_a?(Hash) - - if request_token.callback_confirmed? - request_token.get_access_token(oauth_verifier: oauth_verifier) - else - request_token.get_access_token(oauth_callback: redirect_uri) - end - end - - def user - JSON.parse(get("/api/1.0/user").body) - end - - def issues(project_identifier) - all_issues = [] - offset = 0 - per_page = 50 # Maximum number allowed by Bitbucket - index = 0 - - begin - issues = JSON.parse(get(issue_api_endpoint(project_identifier, per_page, offset)).body) - # Find out how many total issues are present - total = issues["count"] if index == 0 - all_issues.concat(issues["issues"]) - offset += issues["issues"].count - index += 1 - end while all_issues.count < total - - all_issues - end - - def issue_comments(project_identifier, issue_id) - comments = JSON.parse(get("/api/1.0/repositories/#{project_identifier}/issues/#{issue_id}/comments").body) - comments.sort_by { |comment| comment["utc_created_on"] } - end - - def project(project_identifier) - JSON.parse(get("/api/1.0/repositories/#{project_identifier}").body) - end - - def find_deploy_key(project_identifier, key) - JSON.parse(get("/api/1.0/repositories/#{project_identifier}/deploy-keys").body).find do |deploy_key| - deploy_key["key"].chomp == key.chomp - end - end - - def add_deploy_key(project_identifier, key) - deploy_key = find_deploy_key(project_identifier, key) - return if deploy_key - - JSON.parse(api.post("/api/1.0/repositories/#{project_identifier}/deploy-keys", key: key, label: "GitLab import key").body) - end - - def delete_deploy_key(project_identifier, key) - deploy_key = find_deploy_key(project_identifier, key) - return unless deploy_key - - api.delete("/api/1.0/repositories/#{project_identifier}/deploy-keys/#{deploy_key["pk"]}").code == "204" - end - - def projects - JSON.parse(get("/api/1.0/user/repositories").body).select { |repo| repo["scm"] == "git" } - end - - def incompatible_projects - JSON.parse(get("/api/1.0/user/repositories").body).reject { |repo| repo["scm"] == "git" } - end - - private - - def get(url) - response = api.get(url) - raise Unauthorized if (400..499).cover?(response.code.to_i) - - response - end - - def issue_api_endpoint(project_identifier, per_page, offset) - "/api/1.0/repositories/#{project_identifier}/issues?sort=utc_created_on&limit=#{per_page}&start=#{offset}" - end - - def config - Gitlab.config.omniauth.providers.find { |provider| provider.name == "bitbucket" } - end - - def bitbucket_options - OmniAuth::Strategies::Bitbucket.default_options[:client_options].symbolize_keys - end - end - end -end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 7beaecd1cf0..44323b47dca 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -1,84 +1,247 @@ module Gitlab module BitbucketImport class Importer - attr_reader :project, :client + include Gitlab::ShellAdapter + + LABELS = [{ title: 'bug', color: '#FF0000' }, + { title: 'enhancement', color: '#428BCA' }, + { title: 'proposal', color: '#69D100' }, + { title: 'task', color: '#7F8C8D' }].freeze + + attr_reader :project, :client, :errors, :users def initialize(project) @project = project - @client = Client.from_project(@project) + @client = Bitbucket::Client.new(project.import_data.credentials) @formatter = Gitlab::ImportFormatter.new + @labels = {} + @errors = [] + @users = {} end def execute - import_issues if has_issues? + import_wiki + import_issues + import_pull_requests + handle_errors true - rescue ActiveRecord::RecordInvalid => e - raise Projects::ImportService::Error.new, e.message - ensure - Gitlab::BitbucketImport::KeyDeleter.new(project).execute end private - def gl_user_id(project, bitbucket_id) - if bitbucket_id - user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", bitbucket_id.to_s) - (user && user.id) || project.creator_id - else - project.creator_id - end + def handle_errors + return unless errors.any? + + project.update_column(:import_error, { + message: 'The remote data could not be fully imported.', + errors: errors + }.to_json) + end + + def gitlab_user_id(project, username) + find_user_id(username) || project.creator_id end - def identifier - project.import_source + def find_user_id(username) + return nil unless username + + return users[username] if users.key?(username) + + users[username] = User.select(:id) + .joins(:identities) + .find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", username) + .try(:id) end - def has_issues? - client.project(identifier)["has_issues"] + def repo + @repo ||= client.repo(project.import_source) end - def import_issues - issues = client.issues(identifier) + def import_wiki + return if project.wiki.repository_exists? - issues.each do |issue| - body = '' - reporter = nil - author = 'Anonymous' + path_with_namespace = "#{project.path_with_namespace}.wiki" + import_url = project.import_url.sub(/\.git\z/, ".git/wiki") + gitlab_shell.import_repository(project.repository_storage_path, path_with_namespace, import_url) + rescue StandardError => e + errors << { type: :wiki, errors: e.message } + end - if issue["reported_by"] && issue["reported_by"]["username"] - reporter = issue["reported_by"]["username"] - author = reporter + def import_issues + return unless repo.issues_enabled? + + create_labels + + client.issues(repo).each do |issue| + begin + description = '' + description += @formatter.author_line(issue.author) unless find_user_id(issue.author) + description += issue.description + + label_name = issue.kind + milestone = issue.milestone ? project.milestones.find_or_create_by(title: issue.milestone) : nil + + gitlab_issue = project.issues.create!( + iid: issue.iid, + title: issue.title, + description: description, + state: issue.state, + author_id: gitlab_user_id(project, issue.author), + milestone: milestone, + created_at: issue.created_at, + updated_at: issue.updated_at + ) + + gitlab_issue.labels << @labels[label_name] + + import_issue_comments(issue, gitlab_issue) if gitlab_issue.persisted? + rescue StandardError => e + errors << { type: :issue, iid: issue.iid, errors: e.message } end + end + end - body = @formatter.author_line(author) - body += issue["content"] + def import_issue_comments(issue, gitlab_issue) + client.issue_comments(repo, issue.iid).each do |comment| + # The note can be blank for issue service messages like "Changed title: ..." + # We would like to import those comments as well but there is no any + # specific parameter that would allow to process them, it's just an empty comment. + # To prevent our importer from just crashing or from creating useless empty comments + # we do this check. + next unless comment.note.present? + + note = '' + note += @formatter.author_line(comment.author) unless find_user_id(comment.author) + note += comment.note + + begin + gitlab_issue.notes.create!( + project: project, + note: note, + author_id: gitlab_user_id(project, comment.author), + created_at: comment.created_at, + updated_at: comment.updated_at + ) + rescue StandardError => e + errors << { type: :issue_comment, iid: issue.iid, errors: e.message } + end + end + end - comments = client.issue_comments(identifier, issue["local_id"]) + def create_labels + LABELS.each do |label| + @labels[label[:title]] = project.labels.create!(label) + end + end - if comments.any? - body += @formatter.comments_header + def import_pull_requests + pull_requests = client.pull_requests(repo) + + pull_requests.each do |pull_request| + begin + description = '' + description += @formatter.author_line(pull_request.author) unless find_user_id(pull_request.author) + description += pull_request.description + + merge_request = project.merge_requests.create( + iid: pull_request.iid, + title: pull_request.title, + description: description, + source_project: project, + source_branch: pull_request.source_branch_name, + source_branch_sha: pull_request.source_branch_sha, + target_project: project, + target_branch: pull_request.target_branch_name, + target_branch_sha: pull_request.target_branch_sha, + state: pull_request.state, + author_id: gitlab_user_id(project, pull_request.author), + assignee_id: nil, + created_at: pull_request.created_at, + updated_at: pull_request.updated_at + ) + + import_pull_request_comments(pull_request, merge_request) if merge_request.persisted? + rescue StandardError => e + errors << { type: :pull_request, iid: pull_request.iid, errors: e.message } end + end + end + + def import_pull_request_comments(pull_request, merge_request) + comments = client.pull_request_comments(repo, pull_request.iid) + + inline_comments, pr_comments = comments.partition(&:inline?) + + import_inline_comments(inline_comments, pull_request, merge_request) + import_standalone_pr_comments(pr_comments, merge_request) + end - comments.each do |comment| - author = 'Anonymous' + def import_inline_comments(inline_comments, pull_request, merge_request) + line_code_map = {} - if comment["author_info"] && comment["author_info"]["username"] - author = comment["author_info"]["username"] - end + children, parents = inline_comments.partition(&:has_parent?) - body += @formatter.comment(author, comment["utc_created_on"], comment["content"]) + # The Bitbucket API returns threaded replies as parent-child + # relationships. We assume that the child can appear in any order in + # the JSON. + parents.each do |comment| + line_code_map[comment.iid] = generate_line_code(comment) + end + + children.each do |comment| + line_code_map[comment.iid] = line_code_map.fetch(comment.parent_id, nil) + end + + inline_comments.each do |comment| + begin + attributes = pull_request_comment_attributes(comment) + attributes.merge!( + position: build_position(merge_request, comment), + line_code: line_code_map.fetch(comment.iid), + type: 'DiffNote') + + merge_request.notes.create!(attributes) + rescue StandardError => e + errors << { type: :pull_request, iid: comment.iid, errors: e.message } end + end + end + + def build_position(merge_request, pr_comment) + params = { + diff_refs: merge_request.diff_refs, + old_path: pr_comment.file_path, + new_path: pr_comment.file_path, + old_line: pr_comment.old_pos, + new_line: pr_comment.new_pos + } - project.issues.create!( - description: body, - title: issue["title"], - state: %w(resolved invalid duplicate wontfix closed).include?(issue["status"]) ? 'closed' : 'opened', - author_id: gl_user_id(project, reporter) - ) + Gitlab::Diff::Position.new(params) + end + + def import_standalone_pr_comments(pr_comments, merge_request) + pr_comments.each do |comment| + begin + merge_request.notes.create!(pull_request_comment_attributes(comment)) + rescue StandardError => e + errors << { type: :pull_request, iid: comment.iid, errors: e.message } + end end - rescue ActiveRecord::RecordInvalid => e - raise Projects::ImportService::Error, e.message + end + + def generate_line_code(pr_comment) + Gitlab::Diff::LineCode.generate(pr_comment.file_path, pr_comment.new_pos, pr_comment.old_pos) + end + + def pull_request_comment_attributes(comment) + { + project: project, + note: comment.note, + author_id: gitlab_user_id(project, comment.author), + created_at: comment.created_at, + updated_at: comment.updated_at + } end end end diff --git a/lib/gitlab/bitbucket_import/key_adder.rb b/lib/gitlab/bitbucket_import/key_adder.rb deleted file mode 100644 index 0b63f025d0a..00000000000 --- a/lib/gitlab/bitbucket_import/key_adder.rb +++ /dev/null @@ -1,24 +0,0 @@ -module Gitlab - module BitbucketImport - class KeyAdder - attr_reader :repo, :current_user, :client - - def initialize(repo, current_user, access_params) - @repo, @current_user = repo, current_user - @client = Client.new(access_params[:bitbucket_access_token], - access_params[:bitbucket_access_token_secret]) - end - - def execute - return false unless BitbucketImport.public_key.present? - - project_identifier = "#{repo["owner"]}/#{repo["slug"]}" - client.add_deploy_key(project_identifier, BitbucketImport.public_key) - - true - rescue - false - end - end - end -end diff --git a/lib/gitlab/bitbucket_import/key_deleter.rb b/lib/gitlab/bitbucket_import/key_deleter.rb deleted file mode 100644 index e03c3155b3e..00000000000 --- a/lib/gitlab/bitbucket_import/key_deleter.rb +++ /dev/null @@ -1,23 +0,0 @@ -module Gitlab - module BitbucketImport - class KeyDeleter - attr_reader :project, :current_user, :client - - def initialize(project) - @project = project - @current_user = project.creator - @client = Client.from_project(@project) - end - - def execute - return false unless BitbucketImport.public_key.present? - - client.delete_deploy_key(project.import_source, BitbucketImport.public_key) - - true - rescue - false - end - end - end -end diff --git a/lib/gitlab/bitbucket_import/project_creator.rb b/lib/gitlab/bitbucket_import/project_creator.rb index b90ef0b0fba..d94f70fd1fb 100644 --- a/lib/gitlab/bitbucket_import/project_creator.rb +++ b/lib/gitlab/bitbucket_import/project_creator.rb @@ -1,10 +1,11 @@ module Gitlab module BitbucketImport class ProjectCreator - attr_reader :repo, :namespace, :current_user, :session_data + attr_reader :repo, :name, :namespace, :current_user, :session_data - def initialize(repo, namespace, current_user, session_data) + def initialize(repo, name, namespace, current_user, session_data) @repo = repo + @name = name @namespace = namespace @current_user = current_user @session_data = session_data @@ -13,17 +14,24 @@ module Gitlab def execute ::Projects::CreateService.new( current_user, - name: repo["name"], - path: repo["slug"], - description: repo["description"], + name: name, + path: name, + description: repo.description, namespace_id: namespace.id, - visibility_level: repo["is_private"] ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::VisibilityLevel::PUBLIC, - import_type: "bitbucket", - import_source: "#{repo["owner"]}/#{repo["slug"]}", - import_url: "ssh://git@bitbucket.org/#{repo["owner"]}/#{repo["slug"]}.git", - import_data: { credentials: { bb_session: session_data } } + visibility_level: repo.visibility_level, + import_type: 'bitbucket', + import_source: repo.full_name, + import_url: repo.clone_url(session_data[:token]), + import_data: { credentials: session_data }, + skip_wiki: skip_wiki ).execute end + + private + + def skip_wiki + repo.has_wiki? + end end end end diff --git a/lib/gitlab/changes_list.rb b/lib/gitlab/changes_list.rb index 95308aca95f..5b32fca00a4 100644 --- a/lib/gitlab/changes_list.rb +++ b/lib/gitlab/changes_list.rb @@ -5,7 +5,7 @@ module Gitlab attr_reader :raw_changes def initialize(changes) - @raw_changes = changes.kind_of?(String) ? changes.lines : changes + @raw_changes = changes.is_a?(String) ? changes.lines : changes end def each(&block) diff --git a/lib/gitlab/chat_commands/base_command.rb b/lib/gitlab/chat_commands/base_command.rb new file mode 100644 index 00000000000..25da8474e95 --- /dev/null +++ b/lib/gitlab/chat_commands/base_command.rb @@ -0,0 +1,47 @@ +module Gitlab + module ChatCommands + class BaseCommand + QUERY_LIMIT = 5 + + def self.match(_text) + raise NotImplementedError + end + + def self.help_message + raise NotImplementedError + end + + def self.available?(_project) + raise NotImplementedError + end + + def self.allowed?(_user, _ability) + true + end + + def self.can?(object, action, subject) + Ability.allowed?(object, action, subject) + end + + def execute(_) + raise NotImplementedError + end + + def collection + raise NotImplementedError + end + + attr_accessor :project, :current_user, :params + + def initialize(project, user, params = {}) + @project, @current_user, @params = project, user, params.dup + end + + private + + def find_by_iid(iid) + collection.find_by(iid: iid) + end + end + end +end diff --git a/lib/gitlab/chat_commands/command.rb b/lib/gitlab/chat_commands/command.rb new file mode 100644 index 00000000000..f34ed0f4cf2 --- /dev/null +++ b/lib/gitlab/chat_commands/command.rb @@ -0,0 +1,44 @@ +module Gitlab + module ChatCommands + class Command < BaseCommand + COMMANDS = [ + Gitlab::ChatCommands::IssueShow, + Gitlab::ChatCommands::IssueNew, + Gitlab::ChatCommands::IssueSearch, + Gitlab::ChatCommands::Deploy, + ].freeze + + def execute + command, match = match_command + + if command + if command.allowed?(project, current_user) + command.new(project, current_user, params).execute(match) + else + Gitlab::ChatCommands::Presenters::Access.new.access_denied + end + else + Gitlab::ChatCommands::Help.new(project, current_user, params).execute(available_commands, params[:text]) + end + end + + def match_command + match = nil + service = + available_commands.find do |klass| + match = klass.match(params[:text]) + end + + [service, match] + end + + private + + def available_commands + COMMANDS.select do |klass| + klass.available?(project) + end + end + end + end +end diff --git a/lib/gitlab/chat_commands/deploy.rb b/lib/gitlab/chat_commands/deploy.rb new file mode 100644 index 00000000000..458d90f84e8 --- /dev/null +++ b/lib/gitlab/chat_commands/deploy.rb @@ -0,0 +1,50 @@ +module Gitlab + module ChatCommands + class Deploy < BaseCommand + def self.match(text) + /\Adeploy\s+(?<from>\S+.*)\s+to+\s+(?<to>\S+.*)\z/.match(text) + end + + def self.help_message + 'deploy <environment> to <target-environment>' + end + + def self.available?(project) + project.builds_enabled? + end + + def self.allowed?(project, user) + can?(user, :create_deployment, project) + end + + def execute(match) + from = match[:from] + to = match[:to] + + actions = find_actions(from, to) + + if actions.none? + Gitlab::ChatCommands::Presenters::Deploy.new(nil).no_actions + elsif actions.one? + action = play!(from, to, actions.first) + Gitlab::ChatCommands::Presenters::Deploy.new(action).present(from, to) + else + Gitlab::ChatCommands::Presenters::Deploy.new(actions).too_many_actions + end + end + + private + + def play!(from, to, action) + action.play(current_user) + end + + def find_actions(from, to) + environment = project.environments.find_by(name: from) + return [] unless environment + + environment.actions_for(to).select(&:starts_environment?) + end + end + end +end diff --git a/lib/gitlab/chat_commands/help.rb b/lib/gitlab/chat_commands/help.rb new file mode 100644 index 00000000000..6c0e4d304a4 --- /dev/null +++ b/lib/gitlab/chat_commands/help.rb @@ -0,0 +1,28 @@ +module Gitlab + module ChatCommands + class Help < BaseCommand + # This class has to be used last, as it always matches. It has to match + # because other commands were not triggered and we want to show the help + # command + def self.match(_text) + true + end + + def self.help_message + 'help' + end + + def self.allowed?(_project, _user) + true + end + + def execute(commands, text) + Gitlab::ChatCommands::Presenters::Help.new(commands).present(trigger, text) + end + + def trigger + params[:command] + end + end + end +end diff --git a/lib/gitlab/chat_commands/issue_command.rb b/lib/gitlab/chat_commands/issue_command.rb new file mode 100644 index 00000000000..84de3e44c70 --- /dev/null +++ b/lib/gitlab/chat_commands/issue_command.rb @@ -0,0 +1,13 @@ +module Gitlab + module ChatCommands + class IssueCommand < BaseCommand + def self.available?(project) + project.issues_enabled? && project.default_issues_tracker? + end + + def collection + IssuesFinder.new(current_user, project_id: project.id).execute + end + end + end +end diff --git a/lib/gitlab/chat_commands/issue_new.rb b/lib/gitlab/chat_commands/issue_new.rb new file mode 100644 index 00000000000..016054ecd46 --- /dev/null +++ b/lib/gitlab/chat_commands/issue_new.rb @@ -0,0 +1,42 @@ +module Gitlab + module ChatCommands + class IssueNew < IssueCommand + def self.match(text) + # we can not match \n with the dot by passing the m modifier as than + # the title and description are not seperated + /\Aissue\s+(new|create)\s+(?<title>[^\n]*)\n*(?<description>(.|\n)*)/.match(text) + end + + def self.help_message + 'issue new <title> *`⇧ Shift`*+*`↵ Enter`* <description>' + end + + def self.allowed?(project, user) + can?(user, :create_issue, project) + end + + def execute(match) + title = match[:title] + description = match[:description].to_s.rstrip + + issue = create_issue(title: title, description: description) + + if issue.persisted? + presenter(issue).present + else + presenter(issue).display_errors + end + end + + private + + def create_issue(title:, description:) + Issues::CreateService.new(project, current_user, title: title, description: description).execute + end + + def presenter(issue) + Gitlab::ChatCommands::Presenters::IssueNew.new(issue) + end + end + end +end diff --git a/lib/gitlab/chat_commands/issue_search.rb b/lib/gitlab/chat_commands/issue_search.rb new file mode 100644 index 00000000000..3491b53093e --- /dev/null +++ b/lib/gitlab/chat_commands/issue_search.rb @@ -0,0 +1,23 @@ +module Gitlab + module ChatCommands + class IssueSearch < IssueCommand + def self.match(text) + /\Aissue\s+search\s+(?<query>.*)/.match(text) + end + + def self.help_message + "issue search <your query>" + end + + def execute(match) + issues = collection.search(match[:query]).limit(QUERY_LIMIT) + + if issues.present? + Presenters::IssueSearch.new(issues).present + else + Presenters::Access.new(issues).not_found + end + end + end + end +end diff --git a/lib/gitlab/chat_commands/issue_show.rb b/lib/gitlab/chat_commands/issue_show.rb new file mode 100644 index 00000000000..d6013f4d10c --- /dev/null +++ b/lib/gitlab/chat_commands/issue_show.rb @@ -0,0 +1,23 @@ +module Gitlab + module ChatCommands + class IssueShow < IssueCommand + def self.match(text) + /\Aissue\s+show\s+#{Issue.reference_prefix}?(?<iid>\d+)/.match(text) + end + + def self.help_message + "issue show <id>" + end + + def execute(match) + issue = find_by_iid(match[:iid]) + + if issue + Gitlab::ChatCommands::Presenters::IssueShow.new(issue).present + else + Gitlab::ChatCommands::Presenters::Access.new.not_found + end + end + end + end +end diff --git a/lib/gitlab/chat_commands/presenters/access.rb b/lib/gitlab/chat_commands/presenters/access.rb new file mode 100644 index 00000000000..92f4fa17f78 --- /dev/null +++ b/lib/gitlab/chat_commands/presenters/access.rb @@ -0,0 +1,40 @@ +module Gitlab + module ChatCommands + module Presenters + class Access < Presenters::Base + def access_denied + ephemeral_response(text: "Whoops! This action is not allowed. This incident will be [reported](https://xkcd.com/838/).") + end + + def not_found + ephemeral_response(text: "404 not found! GitLab couldn't find what you were looking for! :boom:") + end + + def authorize + message = + if @resource + ":wave: Hi there! Before I do anything for you, please [connect your GitLab account](#{@resource})." + else + ":sweat_smile: Couldn't identify you, nor can I autorize you!" + end + + ephemeral_response(text: message) + end + + def unknown_command(commands) + ephemeral_response(text: help_message(trigger)) + end + + private + + def help_message(trigger) + header_with_list("Command not found, these are the commands you can use", full_commands(trigger)) + end + + def full_commands(trigger) + @resource.map { |command| "#{trigger} #{command.help_message}" } + end + end + end + end +end diff --git a/lib/gitlab/chat_commands/presenters/base.rb b/lib/gitlab/chat_commands/presenters/base.rb new file mode 100644 index 00000000000..2700a5a2ad5 --- /dev/null +++ b/lib/gitlab/chat_commands/presenters/base.rb @@ -0,0 +1,77 @@ +module Gitlab + module ChatCommands + module Presenters + class Base + include Gitlab::Routing.url_helpers + + def initialize(resource = nil) + @resource = resource + end + + def display_errors + message = header_with_list("The action was not successful, because:", @resource.errors.full_messages) + + ephemeral_response(text: message) + end + + private + + def header_with_list(header, items) + message = [header] + + items.each do |item| + message << "- #{item}" + end + + message.join("\n") + end + + def ephemeral_response(message) + response = { + response_type: :ephemeral, + status: 200 + }.merge(message) + + format_response(response) + end + + def in_channel_response(message) + response = { + response_type: :in_channel, + status: 200 + }.merge(message) + + format_response(response) + end + + def format_response(response) + response[:text] = format(response[:text]) if response.has_key?(:text) + + if response.has_key?(:attachments) + response[:attachments].each do |attachment| + attachment[:pretext] = format(attachment[:pretext]) if attachment[:pretext] + attachment[:text] = format(attachment[:text]) if attachment[:text] + end + end + + response + end + + # Convert Markdown to slacks format + def format(string) + Slack::Notifier::LinkFormatter.format(string) + end + + def resource_url + url_for( + [ + @resource.project.namespace.becomes(Namespace), + @resource.project, + @resource + ] + ) + end + end + end + end +end diff --git a/lib/gitlab/chat_commands/presenters/deploy.rb b/lib/gitlab/chat_commands/presenters/deploy.rb new file mode 100644 index 00000000000..863d0bf99ca --- /dev/null +++ b/lib/gitlab/chat_commands/presenters/deploy.rb @@ -0,0 +1,21 @@ +module Gitlab + module ChatCommands + module Presenters + class Deploy < Presenters::Base + def present(from, to) + message = "Deployment started from #{from} to #{to}. [Follow its progress](#{resource_url})." + + in_channel_response(text: message) + end + + def no_actions + ephemeral_response(text: "No action found to be executed") + end + + def too_many_actions + ephemeral_response(text: "Too many actions defined") + end + end + end + end +end diff --git a/lib/gitlab/chat_commands/presenters/help.rb b/lib/gitlab/chat_commands/presenters/help.rb new file mode 100644 index 00000000000..cd47b7f4c6a --- /dev/null +++ b/lib/gitlab/chat_commands/presenters/help.rb @@ -0,0 +1,27 @@ +module Gitlab + module ChatCommands + module Presenters + class Help < Presenters::Base + def present(trigger, text) + ephemeral_response(text: help_message(trigger, text)) + end + + private + + def help_message(trigger, text) + return "No commands available :thinking_face:" unless @resource.present? + + if text.start_with?('help') + header_with_list("Available commands", full_commands(trigger)) + else + header_with_list("Unknown command, these commands are available", full_commands(trigger)) + end + end + + def full_commands(trigger) + @resource.map { |command| "#{trigger} #{command.help_message}" } + end + end + end + end +end diff --git a/lib/gitlab/chat_commands/presenters/issue_base.rb b/lib/gitlab/chat_commands/presenters/issue_base.rb new file mode 100644 index 00000000000..054f7f4be0c --- /dev/null +++ b/lib/gitlab/chat_commands/presenters/issue_base.rb @@ -0,0 +1,43 @@ +module Gitlab + module ChatCommands + module Presenters + module IssueBase + def color(issuable) + issuable.open? ? '#38ae67' : '#d22852' + end + + def status_text(issuable) + issuable.open? ? 'Open' : 'Closed' + end + + def project + @resource.project + end + + def author + @resource.author + end + + def fields + [ + { + title: "Assignee", + value: @resource.assignee ? @resource.assignee.name : "_None_", + short: true + }, + { + title: "Milestone", + value: @resource.milestone ? @resource.milestone.title : "_None_", + short: true + }, + { + title: "Labels", + value: @resource.labels.any? ? @resource.label_names.join(', ') : "_None_", + short: true + } + ] + end + end + end + end +end diff --git a/lib/gitlab/chat_commands/presenters/issue_new.rb b/lib/gitlab/chat_commands/presenters/issue_new.rb new file mode 100644 index 00000000000..3674ba25641 --- /dev/null +++ b/lib/gitlab/chat_commands/presenters/issue_new.rb @@ -0,0 +1,50 @@ +module Gitlab + module ChatCommands + module Presenters + class IssueNew < Presenters::Base + include Presenters::IssueBase + + def present + in_channel_response(new_issue) + end + + private + + def new_issue + { + attachments: [ + { + title: "#{@resource.title} · #{@resource.to_reference}", + title_link: resource_url, + author_name: author.name, + author_icon: author.avatar_url, + fallback: "New issue #{@resource.to_reference}: #{@resource.title}", + pretext: pretext, + color: color(@resource), + fields: fields, + mrkdwn_in: [ + :title, + :pretext, + :text, + :fields + ] + } + ] + } + end + + def pretext + "I created an issue on #{author_profile_link}'s behalf: **#{@resource.to_reference}** in #{project_link}" + end + + def project_link + "[#{project.name_with_namespace}](#{project.web_url})" + end + + def author_profile_link + "[#{author.to_reference}](#{url_for(author)})" + end + end + end + end +end diff --git a/lib/gitlab/chat_commands/presenters/issue_search.rb b/lib/gitlab/chat_commands/presenters/issue_search.rb new file mode 100644 index 00000000000..73788cf9662 --- /dev/null +++ b/lib/gitlab/chat_commands/presenters/issue_search.rb @@ -0,0 +1,47 @@ +module Gitlab + module ChatCommands + module Presenters + class IssueSearch < Presenters::Base + include Presenters::IssueBase + + def present + text = if @resource.count >= 5 + "Here are the first 5 issues I found:" + elsif @resource.one? + "Here is the only issue I found:" + else + "Here are the #{@resource.count} issues I found:" + end + + ephemeral_response(text: text, attachments: attachments) + end + + private + + def attachments + @resource.map do |issue| + url = "[#{issue.to_reference}](#{url_for([namespace, project, issue])})" + + { + color: color(issue), + fallback: "#{issue.to_reference} #{issue.title}", + text: "#{url} · #{issue.title} (#{status_text(issue)})", + + mrkdwn_in: [ + :text + ] + } + end + end + + def project + @project ||= @resource.first.project + end + + def namespace + @namespace ||= project.namespace.becomes(Namespace) + end + end + end + end +end diff --git a/lib/gitlab/chat_commands/presenters/issue_show.rb b/lib/gitlab/chat_commands/presenters/issue_show.rb new file mode 100644 index 00000000000..bd784ad241e --- /dev/null +++ b/lib/gitlab/chat_commands/presenters/issue_show.rb @@ -0,0 +1,61 @@ +module Gitlab + module ChatCommands + module Presenters + class IssueShow < Presenters::Base + include Presenters::IssueBase + + def present + if @resource.confidential? + ephemeral_response(show_issue) + else + in_channel_response(show_issue) + end + end + + private + + def show_issue + { + attachments: [ + { + title: "#{@resource.title} · #{@resource.to_reference}", + title_link: resource_url, + author_name: author.name, + author_icon: author.avatar_url, + fallback: "Issue #{@resource.to_reference}: #{@resource.title}", + pretext: pretext, + text: text, + color: color(@resource), + fields: fields, + mrkdwn_in: [ + :pretext, + :text, + :fields + ] + } + ] + } + end + + def text + message = "**#{status_text(@resource)}**" + + if @resource.upvotes.zero? && @resource.downvotes.zero? && @resource.user_notes_count.zero? + return message + end + + message << " · " + message << ":+1: #{@resource.upvotes} " unless @resource.upvotes.zero? + message << ":-1: #{@resource.downvotes} " unless @resource.downvotes.zero? + message << ":speech_balloon: #{@resource.user_notes_count}" unless @resource.user_notes_count.zero? + + message + end + + def pretext + "Issue *#{@resource.to_reference}* from #{project.name_with_namespace}" + end + end + end + end +end diff --git a/lib/gitlab/chat_commands/result.rb b/lib/gitlab/chat_commands/result.rb new file mode 100644 index 00000000000..324d7ef43a3 --- /dev/null +++ b/lib/gitlab/chat_commands/result.rb @@ -0,0 +1,5 @@ +module Gitlab + module ChatCommands + Result = Struct.new(:type, :message) + end +end diff --git a/lib/gitlab/chat_name_token.rb b/lib/gitlab/chat_name_token.rb new file mode 100644 index 00000000000..1b081aa9b1d --- /dev/null +++ b/lib/gitlab/chat_name_token.rb @@ -0,0 +1,45 @@ +require 'json' + +module Gitlab + class ChatNameToken + attr_reader :token + + TOKEN_LENGTH = 50 + EXPIRY_TIME = 10.minutes + + def initialize(token = new_token) + @token = token + end + + def get + Gitlab::Redis.with do |redis| + data = redis.get(redis_key) + JSON.parse(data, symbolize_names: true) if data + end + end + + def store!(params) + Gitlab::Redis.with do |redis| + params = params.to_json + redis.set(redis_key, params, ex: EXPIRY_TIME) + token + end + end + + def delete + Gitlab::Redis.with do |redis| + redis.del(redis_key) + end + end + + private + + def new_token + Devise.friendly_token(TOKEN_LENGTH) + end + + def redis_key + "gitlab:chat_names:#{token}" + end + end +end diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb index 52f117e963b..273118135a9 100644 --- a/lib/gitlab/checks/change_access.rb +++ b/lib/gitlab/checks/change_access.rb @@ -1,17 +1,20 @@ module Gitlab module Checks class ChangeAccess - attr_reader :user_access, :project + attr_reader :user_access, :project, :skip_authorization - def initialize(change, user_access:, project:) + def initialize( + change, user_access:, project:, env: {}, skip_authorization: false) @oldrev, @newrev, @ref = change.values_at(:oldrev, :newrev, :ref) @branch_name = Gitlab::Git.branch_name(@ref) @user_access = user_access @project = project + @env = env + @skip_authorization = skip_authorization end def exec - error = protected_branch_checks || tag_checks || push_checks + error = push_checks || tag_checks || protected_branch_checks if error GitAccessStatus.new(false, error) @@ -23,11 +26,13 @@ module Gitlab protected def protected_branch_checks + return if skip_authorization + return unless @branch_name return unless project.protected_branch?(@branch_name) - if forced_push? && user_access.cannot_do_action?(:force_push_code_to_protected_branches) + if forced_push? return "You are not allowed to force push code to a protected branch on this project." - elsif Gitlab::Git.blank_ref?(@newrev) && user_access.cannot_do_action?(:remove_protected_branches) + elsif Gitlab::Git.blank_ref?(@newrev) return "You are not allowed to delete protected branches from this project." end @@ -47,6 +52,8 @@ module Gitlab end def tag_checks + return if skip_authorization + tag_ref = Gitlab::Git.tag_name(@ref) if tag_ref && protected_tag?(tag_ref) && user_access.cannot_do_action?(:admin_project) @@ -55,6 +62,8 @@ module Gitlab end def push_checks + return if skip_authorization + if user_access.cannot_do_action?(:push_code) "You are not allowed to push code to this project." end @@ -67,7 +76,7 @@ module Gitlab end def forced_push? - Gitlab::Checks::ForcePush.force_push?(@project, @oldrev, @newrev) + Gitlab::Checks::ForcePush.force_push?(@project, @oldrev, @newrev, env: @env) end def matching_merge_request? diff --git a/lib/gitlab/checks/force_push.rb b/lib/gitlab/checks/force_push.rb index 5fe86553bd0..de0c9049ebf 100644 --- a/lib/gitlab/checks/force_push.rb +++ b/lib/gitlab/checks/force_push.rb @@ -1,15 +1,20 @@ module Gitlab module Checks class ForcePush - def self.force_push?(project, oldrev, newrev) + def self.force_push?(project, oldrev, newrev, env: {}) return false if project.empty_repo? # Created or deleted branch if Gitlab::Git.blank_ref?(oldrev) || Gitlab::Git.blank_ref?(newrev) false else - missed_ref, _ = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} --git-dir=#{project.repository.path_to_repo} rev-list --max-count=1 #{oldrev} ^#{newrev})) - missed_ref.present? + missed_ref, exit_status = Gitlab::Git::RevList.new(oldrev, newrev, project: project, env: env).execute + + if exit_status == 0 + missed_ref.present? + else + raise "Got a non-zero exit code while calling out to `git rev-list` in the force-push check." + end end end end diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb index 7f4c750b6fd..6f799c2f031 100644 --- a/lib/gitlab/ci/build/artifacts/metadata/entry.rb +++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb @@ -27,6 +27,8 @@ module Gitlab end end + delegate :empty?, to: :children + def directory? blank_node? || @path.end_with?('/') end @@ -91,10 +93,6 @@ module Gitlab blank_node? || @entries.include?(@path) end - def empty? - children.empty? - end - def total_size descendant_pattern = %r{^#{Regexp.escape(@path)}} entries.sum do |path, entry| diff --git a/lib/gitlab/ci/build/credentials/base.rb b/lib/gitlab/ci/build/credentials/base.rb new file mode 100644 index 00000000000..29a7a27c963 --- /dev/null +++ b/lib/gitlab/ci/build/credentials/base.rb @@ -0,0 +1,13 @@ +module Gitlab + module Ci + module Build + module Credentials + class Base + def type + self.class.name.demodulize.underscore + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/credentials/factory.rb b/lib/gitlab/ci/build/credentials/factory.rb new file mode 100644 index 00000000000..2423aa8857d --- /dev/null +++ b/lib/gitlab/ci/build/credentials/factory.rb @@ -0,0 +1,27 @@ +module Gitlab + module Ci + module Build + module Credentials + class Factory + def initialize(build) + @build = build + end + + def create! + credentials.select(&:valid?) + end + + private + + def credentials + providers.map { |provider| provider.new(@build) } + end + + def providers + [Registry] + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/credentials/registry.rb b/lib/gitlab/ci/build/credentials/registry.rb new file mode 100644 index 00000000000..55eafcaed10 --- /dev/null +++ b/lib/gitlab/ci/build/credentials/registry.rb @@ -0,0 +1,24 @@ +module Gitlab + module Ci + module Build + module Credentials + class Registry < Base + attr_reader :username, :password + + def initialize(build) + @username = 'gitlab-ci-token' + @password = build.token + end + + def url + Gitlab.config.registry.host_port + end + + def valid? + Gitlab.config.registry.enabled + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index ae82c0db3f1..f7ff7ea212e 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -4,17 +4,11 @@ module Gitlab # Base GitLab CI Configuration facade # class Config - ## - # Temporary delegations that should be removed after refactoring - # - delegate :before_script, :image, :services, :after_script, :variables, - :stages, :cache, :jobs, to: :@global - def initialize(config) @config = Loader.new(config).load! - @global = Node::Global.new(@config) - @global.process! + @global = Entry::Global.new(@config) + @global.compose! end def valid? @@ -28,6 +22,41 @@ module Gitlab def to_hash @config end + + ## + # Temporary method that should be removed after refactoring + # + def before_script + @global.before_script_value + end + + def image + @global.image_value + end + + def services + @global.services_value + end + + def after_script + @global.after_script_value + end + + def variables + @global.variables_value + end + + def stages + @global.stages_value + end + + def cache + @global.cache_value + end + + def jobs + @global.jobs_value + end end end end diff --git a/lib/gitlab/ci/config/node/artifacts.rb b/lib/gitlab/ci/config/entry/artifacts.rb index 844bd2fe998..8275aacee9b 100644 --- a/lib/gitlab/ci/config/node/artifacts.rb +++ b/lib/gitlab/ci/config/entry/artifacts.rb @@ -1,15 +1,15 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Entry that represents a configuration of job artifacts. # - class Artifacts < Entry + class Artifacts < Node include Validatable include Attributable - ALLOWED_KEYS = %i[name untracked paths when expire_in] + ALLOWED_KEYS = %i[name untracked paths when expire_in].freeze attributes ALLOWED_KEYS diff --git a/lib/gitlab/ci/config/node/attributable.rb b/lib/gitlab/ci/config/entry/attributable.rb index 221b666f9f6..1c8b55ee4c4 100644 --- a/lib/gitlab/ci/config/node/attributable.rb +++ b/lib/gitlab/ci/config/entry/attributable.rb @@ -1,7 +1,7 @@ module Gitlab module Ci class Config - module Node + module Entry module Attributable extend ActiveSupport::Concern diff --git a/lib/gitlab/ci/config/node/boolean.rb b/lib/gitlab/ci/config/entry/boolean.rb index 84b03ee7832..f3357f85b99 100644 --- a/lib/gitlab/ci/config/node/boolean.rb +++ b/lib/gitlab/ci/config/entry/boolean.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Entry that represents a boolean value. # - class Boolean < Entry + class Boolean < Node include Validatable validations do diff --git a/lib/gitlab/ci/config/node/cache.rb b/lib/gitlab/ci/config/entry/cache.rb index b4bda2841ac..066643ccfcc 100644 --- a/lib/gitlab/ci/config/node/cache.rb +++ b/lib/gitlab/ci/config/entry/cache.rb @@ -1,26 +1,26 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Entry that represents a cache configuration # - class Cache < Entry + class Cache < Node include Configurable - ALLOWED_KEYS = %i[key untracked paths] + ALLOWED_KEYS = %i[key untracked paths].freeze validations do validates :config, allowed_keys: ALLOWED_KEYS end - node :key, Node::Key, + entry :key, Entry::Key, description: 'Cache key used to define a cache affinity.' - node :untracked, Node::Boolean, + entry :untracked, Entry::Boolean, description: 'Cache all untracked files.' - node :paths, Node::Paths, + entry :paths, Entry::Paths, description: 'Specify which paths should be cached across builds.' end end diff --git a/lib/gitlab/ci/config/node/commands.rb b/lib/gitlab/ci/config/entry/commands.rb index d7657ae314b..65d19db249c 100644 --- a/lib/gitlab/ci/config/node/commands.rb +++ b/lib/gitlab/ci/config/entry/commands.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Entry that represents a job script. # - class Commands < Entry + class Commands < Node include Validatable validations do diff --git a/lib/gitlab/ci/config/node/configurable.rb b/lib/gitlab/ci/config/entry/configurable.rb index 2de82d40c9d..e05aca9881b 100644 --- a/lib/gitlab/ci/config/node/configurable.rb +++ b/lib/gitlab/ci/config/entry/configurable.rb @@ -1,7 +1,7 @@ module Gitlab module Ci class Config - module Node + module Entry ## # This mixin is responsible for adding DSL, which purpose is to # simplifly process of adding child nodes. @@ -23,9 +23,9 @@ module Gitlab end end - private + def compose!(deps = nil) + return unless valid? - def compose! self.class.nodes.each do |key, factory| factory .value(@config[key]) @@ -33,6 +33,12 @@ module Gitlab @entries[key] = factory.create! end + + yield if block_given? + + @entries.each_value do |entry| + entry.compose!(deps) + end end class_methods do @@ -42,8 +48,8 @@ module Gitlab private # rubocop:disable Lint/UselessAccessModifier - def node(key, node, metadata) - factory = Node::Factory.new(node) + def entry(key, entry, metadata) + factory = Entry::Factory.new(entry) .with(description: metadata[:description]) (@nodes ||= {}).merge!(key.to_sym => factory) @@ -52,7 +58,7 @@ module Gitlab def helpers(*nodes) nodes.each do |symbol| define_method("#{symbol}_defined?") do - @entries[symbol].specified? if @entries[symbol] + @entries[symbol]&.specified? end define_method("#{symbol}_value") do @@ -60,8 +66,6 @@ module Gitlab @entries[symbol].value end - - alias_method symbol.to_sym, "#{symbol}_value".to_sym end end end diff --git a/lib/gitlab/ci/config/entry/coverage.rb b/lib/gitlab/ci/config/entry/coverage.rb new file mode 100644 index 00000000000..12a063059cb --- /dev/null +++ b/lib/gitlab/ci/config/entry/coverage.rb @@ -0,0 +1,22 @@ +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents Coverage settings. + # + class Coverage < Node + include Validatable + + validations do + validates :config, regexp: true + end + + def value + @config[1...-1] + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/environment.rb b/lib/gitlab/ci/config/entry/environment.rb new file mode 100644 index 00000000000..0c1f9eb7cbf --- /dev/null +++ b/lib/gitlab/ci/config/entry/environment.rb @@ -0,0 +1,83 @@ +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents an environment. + # + class Environment < Node + include Validatable + + ALLOWED_KEYS = %i[name url action on_stop].freeze + + validations do + validate do + unless hash? || string? + errors.add(:config, 'should be a hash or a string') + end + end + + validates :name, presence: true + validates :name, + type: { + with: String, + message: Gitlab::Regex.environment_name_regex_message + } + + validates :name, + format: { + with: Gitlab::Regex.environment_name_regex, + message: Gitlab::Regex.environment_name_regex_message + } + + with_options if: :hash? do + validates :config, allowed_keys: ALLOWED_KEYS + + validates :url, + length: { maximum: 255 }, + allow_nil: true + + validates :action, + inclusion: { in: %w[start stop], message: 'should be start or stop' }, + allow_nil: true + + validates :on_stop, type: String, allow_nil: true + end + end + + def hash? + @config.is_a?(Hash) + end + + def string? + @config.is_a?(String) + end + + def name + value[:name] + end + + def url + value[:url] + end + + def action + value[:action] || 'start' + end + + def on_stop + value[:on_stop] + end + + def value + case @config + when String then { name: @config, action: 'start' } + when Hash then @config + else {} + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/factory.rb b/lib/gitlab/ci/config/entry/factory.rb index 707b052e6a8..9f5e393d191 100644 --- a/lib/gitlab/ci/config/node/factory.rb +++ b/lib/gitlab/ci/config/entry/factory.rb @@ -1,15 +1,15 @@ module Gitlab module Ci class Config - module Node + module Entry ## - # Factory class responsible for fabricating node entry objects. + # Factory class responsible for fabricating entry objects. # class Factory class InvalidFactory < StandardError; end - def initialize(node) - @node = node + def initialize(entry) + @entry = entry @metadata = {} @attributes = {} end @@ -37,33 +37,33 @@ module Gitlab # See issue #18775. # if @value.nil? - Node::Undefined.new( - fabricate_undefined + Entry::Unspecified.new( + fabricate_unspecified ) else - fabricate(@node, @value) + fabricate(@entry, @value) end end private - def fabricate_undefined + def fabricate_unspecified ## - # If node has a default value we fabricate concrete node + # If entry has a default value we fabricate concrete node # with default value. # - if @node.default.nil? - fabricate(Node::Null) + if @entry.default.nil? + fabricate(Entry::Undefined) else - fabricate(@node, @node.default) + fabricate(@entry, @entry.default) end end - def fabricate(node, value = nil) - node.new(value, @metadata).tap do |entry| - entry.key = @attributes[:key] - entry.parent = @attributes[:parent] - entry.description = @attributes[:description] + def fabricate(entry, value = nil) + entry.new(value, @metadata).tap do |node| + node.key = @attributes[:key] + node.parent = @attributes[:parent] + node.description = @attributes[:description] end end end diff --git a/lib/gitlab/ci/config/node/global.rb b/lib/gitlab/ci/config/entry/global.rb index ccd539fb003..a4ec8f0ff2f 100644 --- a/lib/gitlab/ci/config/node/global.rb +++ b/lib/gitlab/ci/config/entry/global.rb @@ -1,52 +1,52 @@ module Gitlab module Ci class Config - module Node + module Entry ## - # This class represents a global entry - root node for entire + # This class represents a global entry - root Entry for entire # GitLab CI Configuration file. # - class Global < Entry + class Global < Node include Configurable - node :before_script, Node::Script, + entry :before_script, Entry::Script, description: 'Script that will be executed before each job.' - node :image, Node::Image, + entry :image, Entry::Image, description: 'Docker image that will be used to execute jobs.' - node :services, Node::Services, + entry :services, Entry::Services, description: 'Docker images that will be linked to the container.' - node :after_script, Node::Script, + entry :after_script, Entry::Script, description: 'Script that will be executed after each job.' - node :variables, Node::Variables, + entry :variables, Entry::Variables, description: 'Environment variables that will be used.' - node :stages, Node::Stages, + entry :stages, Entry::Stages, description: 'Configuration of stages for this pipeline.' - node :types, Node::Stages, + entry :types, Entry::Stages, description: 'Deprecated: stages for this pipeline.' - node :cache, Node::Cache, + entry :cache, Entry::Cache, description: 'Configure caching between build jobs.' helpers :before_script, :image, :services, :after_script, :variables, :stages, :types, :cache, :jobs - private - - def compose! - super - - compose_jobs! - compose_deprecated_entries! + def compose!(_deps = nil) + super(self) do + compose_jobs! + compose_deprecated_entries! + end end + private + def compose_jobs! - factory = Node::Factory.new(Node::Jobs) + factory = Entry::Factory.new(Entry::Jobs) .value(@config.except(*self.class.nodes.keys)) .with(key: :jobs, parent: self, description: 'Jobs definition for this pipeline') diff --git a/lib/gitlab/ci/config/node/hidden_job.rb b/lib/gitlab/ci/config/entry/hidden.rb index 073044b66f8..6fc3aa385bc 100644 --- a/lib/gitlab/ci/config/node/hidden_job.rb +++ b/lib/gitlab/ci/config/entry/hidden.rb @@ -1,15 +1,14 @@ module Gitlab module Ci class Config - module Node + module Entry ## - # Entry that represents a hidden CI/CD job. + # Entry that represents a hidden CI/CD key. # - class HiddenJob < Entry + class Hidden < Node include Validatable validations do - validates :config, type: Hash validates :config, presence: true end diff --git a/lib/gitlab/ci/config/node/image.rb b/lib/gitlab/ci/config/entry/image.rb index 5d3c7c5eab0..b5050257688 100644 --- a/lib/gitlab/ci/config/node/image.rb +++ b/lib/gitlab/ci/config/entry/image.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Entry that represents a Docker image. # - class Image < Entry + class Image < Node include Validatable validations do diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb new file mode 100644 index 00000000000..7f7662f2776 --- /dev/null +++ b/lib/gitlab/ci/config/entry/job.rb @@ -0,0 +1,144 @@ +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents a concrete CI/CD job. + # + class Job < Node + include Configurable + include Attributable + + ALLOWED_KEYS = %i[tags script only except type image services allow_failure + type stage when artifacts cache dependencies before_script + after_script variables environment coverage].freeze + + validations do + validates :config, allowed_keys: ALLOWED_KEYS + validates :config, presence: true + validates :script, presence: true + validates :name, presence: true + validates :name, type: Symbol + + with_options allow_nil: true do + validates :tags, array_of_strings: true + validates :allow_failure, boolean: true + validates :when, + inclusion: { in: %w[on_success on_failure always manual], + message: 'should be on_success, on_failure, ' \ + 'always or manual' } + + validates :dependencies, array_of_strings: true + end + end + + entry :before_script, Entry::Script, + description: 'Global before script overridden in this job.' + + entry :script, Entry::Commands, + description: 'Commands that will be executed in this job.' + + entry :stage, Entry::Stage, + description: 'Pipeline stage this job will be executed into.' + + entry :type, Entry::Stage, + description: 'Deprecated: stage this job will be executed into.' + + entry :after_script, Entry::Script, + description: 'Commands that will be executed when finishing job.' + + entry :cache, Entry::Cache, + description: 'Cache definition for this job.' + + entry :image, Entry::Image, + description: 'Image that will be used to execute this job.' + + entry :services, Entry::Services, + description: 'Services that will be used to execute this job.' + + entry :only, Entry::Trigger, + description: 'Refs policy this job will be executed for.' + + entry :except, Entry::Trigger, + description: 'Refs policy this job will be executed for.' + + entry :variables, Entry::Variables, + description: 'Environment variables available for this job.' + + entry :artifacts, Entry::Artifacts, + description: 'Artifacts configuration for this job.' + + entry :environment, Entry::Environment, + description: 'Environment configuration for this job.' + + entry :coverage, Entry::Coverage, + description: 'Coverage configuration for this job.' + + helpers :before_script, :script, :stage, :type, :after_script, + :cache, :image, :services, :only, :except, :variables, + :artifacts, :commands, :environment, :coverage + + attributes :script, :tags, :allow_failure, :when, :dependencies + + def compose!(deps = nil) + super do + if type_defined? && !stage_defined? + @entries[:stage] = @entries[:type] + end + + @entries.delete(:type) + end + + inherit!(deps) + end + + def name + @metadata[:name] + end + + def value + @config.merge(to_hash.compact) + end + + def commands + (before_script_value.to_a + script_value.to_a).join("\n") + end + + private + + def inherit!(deps) + return unless deps + + self.class.nodes.each_key do |key| + global_entry = deps[key] + job_entry = self[key] + + if global_entry.specified? && !job_entry.specified? + @entries[key] = global_entry + end + end + end + + def to_hash + { name: name, + before_script: before_script_value, + script: script_value, + commands: commands, + image: image_value, + services: services_value, + stage: stage_value, + cache: cache_value, + only: only_value, + except: except_value, + variables: variables_defined? ? variables_value : nil, + environment: environment_defined? ? environment_value : nil, + environment_name: environment_defined? ? environment_value[:name] : nil, + coverage: coverage_defined? ? coverage_value : nil, + artifacts: artifacts_value, + after_script: after_script_value } + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/jobs.rb b/lib/gitlab/ci/config/entry/jobs.rb index 51683c82ceb..5671a09480b 100644 --- a/lib/gitlab/ci/config/node/jobs.rb +++ b/lib/gitlab/ci/config/entry/jobs.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Entry that represents a set of jobs. # - class Jobs < Entry + class Jobs < Node include Validatable validations do @@ -26,19 +26,23 @@ module Gitlab name.to_s.start_with?('.') end - private + def compose!(deps = nil) + super do + @config.each do |name, config| + node = hidden?(name) ? Entry::Hidden : Entry::Job - def compose! - @config.each do |name, config| - node = hidden?(name) ? Node::HiddenJob : Node::Job + factory = Entry::Factory.new(node) + .value(config || {}) + .metadata(name: name) + .with(key: name, parent: self, + description: "#{name} job definition.") - factory = Node::Factory.new(node) - .value(config || {}) - .metadata(name: name) - .with(key: name, parent: self, - description: "#{name} job definition.") + @entries[name] = factory.create! + end - @entries[name] = factory.create! + @entries.each_value do |entry| + entry.compose!(deps) + end end end end diff --git a/lib/gitlab/ci/config/node/key.rb b/lib/gitlab/ci/config/entry/key.rb index f8b461ca098..0e4c9fe6edc 100644 --- a/lib/gitlab/ci/config/node/key.rb +++ b/lib/gitlab/ci/config/entry/key.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Entry that represents a key. # - class Key < Entry + class Key < Node include Validatable validations do diff --git a/lib/gitlab/ci/config/node/legacy_validation_helpers.rb b/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb index 0c291efe6a5..9b9a0a8125a 100644 --- a/lib/gitlab/ci/config/node/legacy_validation_helpers.rb +++ b/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb @@ -1,7 +1,7 @@ module Gitlab module Ci class Config - module Node + module Entry module LegacyValidationHelpers private @@ -28,17 +28,21 @@ module Gitlab value.is_a?(String) || value.is_a?(Symbol) end + def validate_regexp(value) + !value.nil? && Regexp.new(value.to_s) && true + rescue RegexpError, TypeError + false + end + def validate_string_or_regexp(value) return true if value.is_a?(Symbol) return false unless value.is_a?(String) if value.first == '/' && value.last == '/' - Regexp.new(value[1...-1]) + validate_regexp(value[1...-1]) else true end - rescue RegexpError - false end def validate_boolean(value) diff --git a/lib/gitlab/ci/config/node/entry.rb b/lib/gitlab/ci/config/entry/node.rb index 0c782c422b5..5eef2868cd6 100644 --- a/lib/gitlab/ci/config/node/entry.rb +++ b/lib/gitlab/ci/config/entry/node.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Base abstract class for each configuration entry node. # - class Entry + class Node class InvalidError < StandardError; end attr_reader :config, :metadata @@ -20,11 +20,14 @@ module Gitlab @validator.validate(:new) end - def process! + def [](key) + @entries[key] || Entry::Undefined.new + end + + def compose!(deps = nil) return unless valid? - compose! - descendants.each(&:process!) + yield if block_given? end def leaf? @@ -73,11 +76,6 @@ module Gitlab def self.validator Validator end - - private - - def compose! - end end end end diff --git a/lib/gitlab/ci/config/node/paths.rb b/lib/gitlab/ci/config/entry/paths.rb index 3c6d3a52966..68dad161149 100644 --- a/lib/gitlab/ci/config/node/paths.rb +++ b/lib/gitlab/ci/config/entry/paths.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Entry that represents an array of paths. # - class Paths < Entry + class Paths < Node include Validatable validations do diff --git a/lib/gitlab/ci/config/node/script.rb b/lib/gitlab/ci/config/entry/script.rb index 39328f0fade..29ecd9995ca 100644 --- a/lib/gitlab/ci/config/node/script.rb +++ b/lib/gitlab/ci/config/entry/script.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Entry that represents a script. # - class Script < Entry + class Script < Node include Validatable validations do diff --git a/lib/gitlab/ci/config/node/services.rb b/lib/gitlab/ci/config/entry/services.rb index 481e2b66adc..84f8ab780f5 100644 --- a/lib/gitlab/ci/config/node/services.rb +++ b/lib/gitlab/ci/config/entry/services.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Entry that represents a configuration of Docker services. # - class Services < Entry + class Services < Node include Validatable validations do diff --git a/lib/gitlab/ci/config/node/stage.rb b/lib/gitlab/ci/config/entry/stage.rb index cbc97641f5a..b7afaba1de8 100644 --- a/lib/gitlab/ci/config/node/stage.rb +++ b/lib/gitlab/ci/config/entry/stage.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Entry that represents a stage for a job. # - class Stage < Entry + class Stage < Node include Validatable validations do diff --git a/lib/gitlab/ci/config/node/stages.rb b/lib/gitlab/ci/config/entry/stages.rb index b1fe45357ff..ec187bd3732 100644 --- a/lib/gitlab/ci/config/node/stages.rb +++ b/lib/gitlab/ci/config/entry/stages.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Entry that represents a configuration for pipeline stages. # - class Stages < Entry + class Stages < Node include Validatable validations do diff --git a/lib/gitlab/ci/config/entry/trigger.rb b/lib/gitlab/ci/config/entry/trigger.rb new file mode 100644 index 00000000000..16b234e6c59 --- /dev/null +++ b/lib/gitlab/ci/config/entry/trigger.rb @@ -0,0 +1,18 @@ +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents a trigger policy for the job. + # + class Trigger < Node + include Validatable + + validations do + validates :config, array_of_strings_or_regexps: true + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/null.rb b/lib/gitlab/ci/config/entry/undefined.rb index 88a5f53f13c..b33b8238230 100644 --- a/lib/gitlab/ci/config/node/null.rb +++ b/lib/gitlab/ci/config/entry/undefined.rb @@ -1,13 +1,15 @@ module Gitlab module Ci class Config - module Node + module Entry ## - # This class represents an undefined node. + # This class represents an undefined entry. # - # Implements the Null Object pattern. - # - class Null < Entry + class Undefined < Node + def initialize(*) + super(nil) + end + def value nil end diff --git a/lib/gitlab/ci/config/node/undefined.rb b/lib/gitlab/ci/config/entry/unspecified.rb index 45fef8c3ae5..fbb2551e870 100644 --- a/lib/gitlab/ci/config/node/undefined.rb +++ b/lib/gitlab/ci/config/entry/unspecified.rb @@ -1,14 +1,14 @@ module Gitlab module Ci class Config - module Node + module Entry ## - # This class represents an unspecified entry node. + # This class represents an unspecified entry. # # It decorates original entry adding method that indicates it is # unspecified. # - class Undefined < SimpleDelegator + class Unspecified < SimpleDelegator def specified? false end diff --git a/lib/gitlab/ci/config/node/validatable.rb b/lib/gitlab/ci/config/entry/validatable.rb index 085e6e988d1..f7f1b111571 100644 --- a/lib/gitlab/ci/config/node/validatable.rb +++ b/lib/gitlab/ci/config/entry/validatable.rb @@ -1,13 +1,13 @@ module Gitlab module Ci class Config - module Node + module Entry module Validatable extend ActiveSupport::Concern class_methods do def validator - @validator ||= Class.new(Node::Validator).tap do |validator| + @validator ||= Class.new(Entry::Validator).tap do |validator| if defined?(@validations) @validations.each { |rules| validator.class_eval(&rules) } end diff --git a/lib/gitlab/ci/config/node/validator.rb b/lib/gitlab/ci/config/entry/validator.rb index 43c7e102b50..55343005fe3 100644 --- a/lib/gitlab/ci/config/node/validator.rb +++ b/lib/gitlab/ci/config/entry/validator.rb @@ -1,14 +1,14 @@ module Gitlab module Ci class Config - module Node + module Entry class Validator < SimpleDelegator include ActiveModel::Validations - include Node::Validators + include Entry::Validators - def initialize(node) - super(node) - @node = node + def initialize(entry) + super(entry) + @entry = entry end def messages @@ -30,7 +30,7 @@ module Gitlab def key_name if key.blank? - @node.class.name.demodulize.underscore.humanize + @entry.class.name.demodulize.underscore.humanize else key end diff --git a/lib/gitlab/ci/config/node/validators.rb b/lib/gitlab/ci/config/entry/validators.rb index e20908ad3cb..bd7428b1272 100644 --- a/lib/gitlab/ci/config/node/validators.rb +++ b/lib/gitlab/ci/config/entry/validators.rb @@ -1,7 +1,7 @@ module Gitlab module Ci class Config - module Node + module Entry module Validators class AllowedKeysValidator < ActiveModel::EachValidator def validate_each(record, attribute, value) @@ -54,6 +54,51 @@ module Gitlab end end + class RegexpValidator < ActiveModel::EachValidator + include LegacyValidationHelpers + + def validate_each(record, attribute, value) + unless validate_regexp(value) + record.errors.add(attribute, 'must be a regular expression') + end + end + + private + + def look_like_regexp?(value) + value.is_a?(String) && value.start_with?('/') && + value.end_with?('/') + end + + def validate_regexp(value) + look_like_regexp?(value) && + Regexp.new(value.to_s[1...-1]) && + true + rescue RegexpError + false + end + end + + class ArrayOfStringsOrRegexpsValidator < RegexpValidator + def validate_each(record, attribute, value) + unless validate_array_of_strings_or_regexps(value) + record.errors.add(attribute, 'should be an array of strings or regexps') + end + end + + private + + def validate_array_of_strings_or_regexps(values) + values.is_a?(Array) && values.all?(&method(:validate_string_or_regexp)) + end + + def validate_string_or_regexp(value) + return false unless value.is_a?(String) + return validate_regexp(value) if look_like_regexp?(value) + true + end + end + class TypeValidator < ActiveModel::EachValidator def validate_each(record, attribute, value) type = options[:with] diff --git a/lib/gitlab/ci/config/node/variables.rb b/lib/gitlab/ci/config/entry/variables.rb index 5f813f81f55..c3b0e651c3a 100644 --- a/lib/gitlab/ci/config/node/variables.rb +++ b/lib/gitlab/ci/config/entry/variables.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Entry that represents environment variables. # - class Variables < Entry + class Variables < Node include Validatable validations do diff --git a/lib/gitlab/ci/config/node/job.rb b/lib/gitlab/ci/config/node/job.rb deleted file mode 100644 index e84737acbb9..00000000000 --- a/lib/gitlab/ci/config/node/job.rb +++ /dev/null @@ -1,123 +0,0 @@ -module Gitlab - module Ci - class Config - module Node - ## - # Entry that represents a concrete CI/CD job. - # - class Job < Entry - include Configurable - include Attributable - - ALLOWED_KEYS = %i[tags script only except type image services allow_failure - type stage when artifacts cache dependencies before_script - after_script variables environment] - - attributes :tags, :allow_failure, :when, :environment, :dependencies - - validations do - validates :config, allowed_keys: ALLOWED_KEYS - - validates :config, presence: true - validates :name, presence: true - validates :name, type: Symbol - - with_options allow_nil: true do - validates :tags, array_of_strings: true - validates :allow_failure, boolean: true - validates :when, - inclusion: { in: %w[on_success on_failure always manual], - message: 'should be on_success, on_failure, ' \ - 'always or manual' } - validates :environment, - type: { - with: String, - message: Gitlab::Regex.environment_name_regex_message } - validates :environment, - format: { - with: Gitlab::Regex.environment_name_regex, - message: Gitlab::Regex.environment_name_regex_message } - - validates :dependencies, array_of_strings: true - end - end - - node :before_script, Script, - description: 'Global before script overridden in this job.' - - node :script, Commands, - description: 'Commands that will be executed in this job.' - - node :stage, Stage, - description: 'Pipeline stage this job will be executed into.' - - node :type, Stage, - description: 'Deprecated: stage this job will be executed into.' - - node :after_script, Script, - description: 'Commands that will be executed when finishing job.' - - node :cache, Cache, - description: 'Cache definition for this job.' - - node :image, Image, - description: 'Image that will be used to execute this job.' - - node :services, Services, - description: 'Services that will be used to execute this job.' - - node :only, Trigger, - description: 'Refs policy this job will be executed for.' - - node :except, Trigger, - description: 'Refs policy this job will be executed for.' - - node :variables, Variables, - description: 'Environment variables available for this job.' - - node :artifacts, Artifacts, - description: 'Artifacts configuration for this job.' - - helpers :before_script, :script, :stage, :type, :after_script, - :cache, :image, :services, :only, :except, :variables, - :artifacts - - def name - @metadata[:name] - end - - def value - @config.merge(to_hash.compact) - end - - private - - def to_hash - { name: name, - before_script: before_script, - script: script, - image: image, - services: services, - stage: stage, - cache: cache, - only: only, - except: except, - variables: variables_defined? ? variables : nil, - artifacts: artifacts, - after_script: after_script } - end - - def compose! - super - - if type_defined? && !stage_defined? - @entries[:stage] = @entries[:type] - end - - @entries.delete(:type) - end - end - end - end - end -end diff --git a/lib/gitlab/ci/config/node/trigger.rb b/lib/gitlab/ci/config/node/trigger.rb deleted file mode 100644 index d8b31975088..00000000000 --- a/lib/gitlab/ci/config/node/trigger.rb +++ /dev/null @@ -1,26 +0,0 @@ -module Gitlab - module Ci - class Config - module Node - ## - # Entry that represents a trigger policy for the job. - # - class Trigger < Entry - include Validatable - - validations do - include LegacyValidationHelpers - - validate :array_of_strings_or_regexps - - def array_of_strings_or_regexps - unless validate_array_of_strings_or_regexps(config) - errors.add(:config, 'should be an array of strings or regexps') - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/pipeline_duration.rb b/lib/gitlab/ci/pipeline_duration.rb new file mode 100644 index 00000000000..a210e76acaa --- /dev/null +++ b/lib/gitlab/ci/pipeline_duration.rb @@ -0,0 +1,141 @@ +module Gitlab + module Ci + # # Introduction - total running time + # + # The problem this module is trying to solve is finding the total running + # time amongst all the jobs, excluding retries and pending (queue) time. + # We could reduce this problem down to finding the union of periods. + # + # So each job would be represented as a `Period`, which consists of + # `Period#first` as when the job started and `Period#last` as when the + # job was finished. A simple example here would be: + # + # * A (1, 3) + # * B (2, 4) + # * C (6, 7) + # + # Here A begins from 1, and ends to 3. B begins from 2, and ends to 4. + # C begins from 6, and ends to 7. Visually it could be viewed as: + # + # 0 1 2 3 4 5 6 7 + # AAAAAAA + # BBBBBBB + # CCCC + # + # The union of A, B, and C would be (1, 4) and (6, 7), therefore the + # total running time should be: + # + # (4 - 1) + (7 - 6) => 4 + # + # # The Algorithm + # + # The algorithm used here for union would be described as follow. + # First we make sure that all periods are sorted by `Period#first`. + # Then we try to merge periods by iterating through the first period + # to the last period. The goal would be merging all overlapped periods + # so that in the end all the periods are discrete. When all periods + # are discrete, we're free to just sum all the periods to get real + # running time. + # + # Here we begin from A, and compare it to B. We could find that + # before A ends, B already started. That is `B.first <= A.last` + # that is `2 <= 3` which means A and B are overlapping! + # + # When we found that two periods are overlapping, we would need to merge + # them into a new period and disregard the old periods. To make a new + # period, we take `A.first` as the new first because remember? we sorted + # them, so `A.first` must be smaller or equal to `B.first`. And we take + # `[A.last, B.last].max` as the new last because we want whoever ended + # later. This could be broken into two cases: + # + # 0 1 2 3 4 + # AAAAAAA + # BBBBBBB + # + # Or: + # + # 0 1 2 3 4 + # AAAAAAAAAA + # BBBB + # + # So that we need to take whoever ends later. Back to our example, + # after merging and discard A and B it could be visually viewed as: + # + # 0 1 2 3 4 5 6 7 + # DDDDDDDDDD + # CCCC + # + # Now we could go on and compare the newly created D and the old C. + # We could figure out that D and C are not overlapping by checking + # `C.first <= D.last` is `false`. Therefore we need to keep both C + # and D. The example would end here because there are no more jobs. + # + # After having the union of all periods, we just need to sum the length + # of all periods to get total time. + # + # (4 - 1) + (7 - 6) => 4 + # + # That is 4 is the answer in the example. + module PipelineDuration + extend self + + Period = Struct.new(:first, :last) do + def duration + last - first + end + end + + def from_pipeline(pipeline) + status = %w[success failed running canceled] + builds = pipeline.builds.latest. + where(status: status).where.not(started_at: nil).order(:started_at) + + from_builds(builds) + end + + def from_builds(builds) + now = Time.now + + periods = builds.map do |b| + Period.new(b.started_at, b.finished_at || now) + end + + from_periods(periods) + end + + # periods should be sorted by `first` + def from_periods(periods) + process_duration(process_periods(periods)) + end + + private + + def process_periods(periods) + return periods if periods.empty? + + periods.drop(1).inject([periods.first]) do |result, current| + previous = result.last + + if overlap?(previous, current) + result[-1] = merge(previous, current) + result + else + result << current + end + end + end + + def overlap?(previous, current) + current.first <= previous.last + end + + def merge(previous, current) + Period.new(previous.first, [previous.last, current.last].max) + end + + def process_duration(periods) + periods.sum(&:duration) + end + end + end +end diff --git a/lib/gitlab/ci/status/build/cancelable.rb b/lib/gitlab/ci/status/build/cancelable.rb new file mode 100644 index 00000000000..67bbc3c4849 --- /dev/null +++ b/lib/gitlab/ci/status/build/cancelable.rb @@ -0,0 +1,37 @@ +module Gitlab + module Ci + module Status + module Build + class Cancelable < SimpleDelegator + include Status::Extended + + def has_action? + can?(user, :update_build, subject) + end + + def action_icon + 'icon_action_cancel' + end + + def action_path + cancel_namespace_project_build_path(subject.project.namespace, + subject.project, + subject) + end + + def action_method + :post + end + + def action_title + 'Cancel' + end + + def self.matches?(build, user) + build.cancelable? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/common.rb b/lib/gitlab/ci/status/build/common.rb new file mode 100644 index 00000000000..3fec2c5d4db --- /dev/null +++ b/lib/gitlab/ci/status/build/common.rb @@ -0,0 +1,19 @@ +module Gitlab + module Ci + module Status + module Build + module Common + def has_details? + can?(user, :read_build, subject) + end + + def details_path + namespace_project_build_path(subject.project.namespace, + subject.project, + subject) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/factory.rb b/lib/gitlab/ci/status/build/factory.rb new file mode 100644 index 00000000000..38ac6edc9f1 --- /dev/null +++ b/lib/gitlab/ci/status/build/factory.rb @@ -0,0 +1,21 @@ +module Gitlab + module Ci + module Status + module Build + class Factory < Status::Factory + def self.extended_statuses + [[Status::Build::Cancelable, + Status::Build::Retryable], + [Status::Build::FailedAllowed, + Status::Build::Play, + Status::Build::Stop]] + end + + def self.common_helpers + Status::Build::Common + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/failed_allowed.rb b/lib/gitlab/ci/status/build/failed_allowed.rb new file mode 100644 index 00000000000..807afe24bd5 --- /dev/null +++ b/lib/gitlab/ci/status/build/failed_allowed.rb @@ -0,0 +1,27 @@ +module Gitlab + module Ci + module Status + module Build + class FailedAllowed < SimpleDelegator + include Status::Extended + + def label + 'failed (allowed to fail)' + end + + def icon + 'icon_status_warning' + end + + def group + 'failed_with_warnings' + end + + def self.matches?(build, user) + build.failed? && build.allow_failure? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/play.rb b/lib/gitlab/ci/status/build/play.rb new file mode 100644 index 00000000000..0f4b7b24cef --- /dev/null +++ b/lib/gitlab/ci/status/build/play.rb @@ -0,0 +1,53 @@ +module Gitlab + module Ci + module Status + module Build + class Play < SimpleDelegator + include Status::Extended + + def text + 'manual' + end + + def label + 'manual play action' + end + + def icon + 'icon_status_manual' + end + + def group + 'manual' + end + + def has_action? + can?(user, :update_build, subject) + end + + def action_icon + 'icon_action_play' + end + + def action_title + 'Play' + end + + def action_path + play_namespace_project_build_path(subject.project.namespace, + subject.project, + subject) + end + + def action_method + :post + end + + def self.matches?(build, user) + build.playable? && !build.stops_environment? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/retryable.rb b/lib/gitlab/ci/status/build/retryable.rb new file mode 100644 index 00000000000..6b362af7634 --- /dev/null +++ b/lib/gitlab/ci/status/build/retryable.rb @@ -0,0 +1,37 @@ +module Gitlab + module Ci + module Status + module Build + class Retryable < SimpleDelegator + include Status::Extended + + def has_action? + can?(user, :update_build, subject) + end + + def action_icon + 'icon_action_retry' + end + + def action_title + 'Retry' + end + + def action_path + retry_namespace_project_build_path(subject.project.namespace, + subject.project, + subject) + end + + def action_method + :post + end + + def self.matches?(build, user) + build.retryable? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/stop.rb b/lib/gitlab/ci/status/build/stop.rb new file mode 100644 index 00000000000..90401cad0d2 --- /dev/null +++ b/lib/gitlab/ci/status/build/stop.rb @@ -0,0 +1,53 @@ +module Gitlab + module Ci + module Status + module Build + class Stop < SimpleDelegator + include Status::Extended + + def text + 'manual' + end + + def label + 'manual stop action' + end + + def icon + 'icon_status_manual' + end + + def group + 'manual' + end + + def has_action? + can?(user, :update_build, subject) + end + + def action_icon + 'icon_action_stop' + end + + def action_title + 'Stop' + end + + def action_path + play_namespace_project_build_path(subject.project.namespace, + subject.project, + subject) + end + + def action_method + :post + end + + def self.matches?(build, user) + build.playable? && build.stops_environment? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/canceled.rb b/lib/gitlab/ci/status/canceled.rb new file mode 100644 index 00000000000..dd6d99e9075 --- /dev/null +++ b/lib/gitlab/ci/status/canceled.rb @@ -0,0 +1,19 @@ +module Gitlab + module Ci + module Status + class Canceled < Status::Core + def text + 'canceled' + end + + def label + 'canceled' + end + + def icon + 'icon_status_canceled' + end + end + end + end +end diff --git a/lib/gitlab/ci/status/core.rb b/lib/gitlab/ci/status/core.rb new file mode 100644 index 00000000000..3dd2b9e01f6 --- /dev/null +++ b/lib/gitlab/ci/status/core.rb @@ -0,0 +1,59 @@ +module Gitlab + module Ci + module Status + # Base abstract class fore core status + # + class Core + include Gitlab::Routing + include Gitlab::Allowable + + attr_reader :subject, :user + + def initialize(subject, user) + @subject = subject + @user = user + end + + def icon + raise NotImplementedError + end + + def label + raise NotImplementedError + end + + def group + self.class.name.demodulize.underscore + end + + def has_details? + false + end + + def details_path + raise NotImplementedError + end + + def has_action? + false + end + + def action_icon + raise NotImplementedError + end + + def action_path + raise NotImplementedError + end + + def action_method + raise NotImplementedError + end + + def action_title + raise NotImplementedError + end + end + end + end +end diff --git a/lib/gitlab/ci/status/created.rb b/lib/gitlab/ci/status/created.rb new file mode 100644 index 00000000000..6596d7e01ca --- /dev/null +++ b/lib/gitlab/ci/status/created.rb @@ -0,0 +1,19 @@ +module Gitlab + module Ci + module Status + class Created < Status::Core + def text + 'created' + end + + def label + 'created' + end + + def icon + 'icon_status_created' + end + end + end + end +end diff --git a/lib/gitlab/ci/status/extended.rb b/lib/gitlab/ci/status/extended.rb new file mode 100644 index 00000000000..d367c9bda69 --- /dev/null +++ b/lib/gitlab/ci/status/extended.rb @@ -0,0 +1,15 @@ +module Gitlab + module Ci + module Status + module Extended + extend ActiveSupport::Concern + + class_methods do + def matches?(_subject, _user) + raise NotImplementedError + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/external/common.rb b/lib/gitlab/ci/status/external/common.rb new file mode 100644 index 00000000000..4969a350862 --- /dev/null +++ b/lib/gitlab/ci/status/external/common.rb @@ -0,0 +1,22 @@ +module Gitlab + module Ci + module Status + module External + module Common + def has_details? + subject.target_url.present? && + can?(user, :read_commit_status, subject) + end + + def details_path + subject.target_url + end + + def has_action? + false + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/external/factory.rb b/lib/gitlab/ci/status/external/factory.rb new file mode 100644 index 00000000000..07b15bd8d97 --- /dev/null +++ b/lib/gitlab/ci/status/external/factory.rb @@ -0,0 +1,13 @@ +module Gitlab + module Ci + module Status + module External + class Factory < Status::Factory + def self.common_helpers + Status::External::Common + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/factory.rb b/lib/gitlab/ci/status/factory.rb new file mode 100644 index 00000000000..15836c699c7 --- /dev/null +++ b/lib/gitlab/ci/status/factory.rb @@ -0,0 +1,52 @@ +module Gitlab + module Ci + module Status + class Factory + def initialize(subject, user) + @subject = subject + @user = user + @status = subject.status || HasStatus::DEFAULT_STATUS + end + + def fabricate! + if extended_statuses.none? + core_status + else + compound_extended_status + end + end + + def core_status + Gitlab::Ci::Status + .const_get(@status.capitalize) + .new(@subject, @user) + .extend(self.class.common_helpers) + end + + def compound_extended_status + extended_statuses.inject(core_status) do |status, extended| + extended.new(status) + end + end + + def extended_statuses + return @extended_statuses if defined?(@extended_statuses) + + groups = self.class.extended_statuses.map do |group| + Array(group).find { |status| status.matches?(@subject, @user) } + end + + @extended_statuses = groups.flatten.compact + end + + def self.extended_statuses + [] + end + + def self.common_helpers + Module.new + end + end + end + end +end diff --git a/lib/gitlab/ci/status/failed.rb b/lib/gitlab/ci/status/failed.rb new file mode 100644 index 00000000000..c5b5e3203ad --- /dev/null +++ b/lib/gitlab/ci/status/failed.rb @@ -0,0 +1,19 @@ +module Gitlab + module Ci + module Status + class Failed < Status::Core + def text + 'failed' + end + + def label + 'failed' + end + + def icon + 'icon_status_failed' + end + end + end + end +end diff --git a/lib/gitlab/ci/status/pending.rb b/lib/gitlab/ci/status/pending.rb new file mode 100644 index 00000000000..d30f35a59a2 --- /dev/null +++ b/lib/gitlab/ci/status/pending.rb @@ -0,0 +1,19 @@ +module Gitlab + module Ci + module Status + class Pending < Status::Core + def text + 'pending' + end + + def label + 'pending' + end + + def icon + 'icon_status_pending' + end + end + end + end +end diff --git a/lib/gitlab/ci/status/pipeline/common.rb b/lib/gitlab/ci/status/pipeline/common.rb new file mode 100644 index 00000000000..76bfd18bf40 --- /dev/null +++ b/lib/gitlab/ci/status/pipeline/common.rb @@ -0,0 +1,23 @@ +module Gitlab + module Ci + module Status + module Pipeline + module Common + def has_details? + can?(user, :read_pipeline, subject) + end + + def details_path + namespace_project_pipeline_path(subject.project.namespace, + subject.project, + subject) + end + + def has_action? + false + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/pipeline/factory.rb b/lib/gitlab/ci/status/pipeline/factory.rb new file mode 100644 index 00000000000..13c8343b12a --- /dev/null +++ b/lib/gitlab/ci/status/pipeline/factory.rb @@ -0,0 +1,17 @@ +module Gitlab + module Ci + module Status + module Pipeline + class Factory < Status::Factory + def self.extended_statuses + [Status::SuccessWarning] + end + + def self.common_helpers + Status::Pipeline::Common + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/running.rb b/lib/gitlab/ci/status/running.rb new file mode 100644 index 00000000000..2aba3c373c7 --- /dev/null +++ b/lib/gitlab/ci/status/running.rb @@ -0,0 +1,19 @@ +module Gitlab + module Ci + module Status + class Running < Status::Core + def text + 'running' + end + + def label + 'running' + end + + def icon + 'icon_status_running' + end + end + end + end +end diff --git a/lib/gitlab/ci/status/skipped.rb b/lib/gitlab/ci/status/skipped.rb new file mode 100644 index 00000000000..16282aefd03 --- /dev/null +++ b/lib/gitlab/ci/status/skipped.rb @@ -0,0 +1,19 @@ +module Gitlab + module Ci + module Status + class Skipped < Status::Core + def text + 'skipped' + end + + def label + 'skipped' + end + + def icon + 'icon_status_skipped' + end + end + end + end +end diff --git a/lib/gitlab/ci/status/stage/common.rb b/lib/gitlab/ci/status/stage/common.rb new file mode 100644 index 00000000000..7852f492e1d --- /dev/null +++ b/lib/gitlab/ci/status/stage/common.rb @@ -0,0 +1,24 @@ +module Gitlab + module Ci + module Status + module Stage + module Common + def has_details? + can?(user, :read_pipeline, subject.pipeline) + end + + def details_path + namespace_project_pipeline_path(subject.project.namespace, + subject.project, + subject.pipeline, + anchor: subject.name) + end + + def has_action? + false + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/stage/factory.rb b/lib/gitlab/ci/status/stage/factory.rb new file mode 100644 index 00000000000..4c37f084d07 --- /dev/null +++ b/lib/gitlab/ci/status/stage/factory.rb @@ -0,0 +1,17 @@ +module Gitlab + module Ci + module Status + module Stage + class Factory < Status::Factory + def self.extended_statuses + [Status::SuccessWarning] + end + + def self.common_helpers + Status::Stage::Common + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/success.rb b/lib/gitlab/ci/status/success.rb new file mode 100644 index 00000000000..c09c5f006e3 --- /dev/null +++ b/lib/gitlab/ci/status/success.rb @@ -0,0 +1,19 @@ +module Gitlab + module Ci + module Status + class Success < Status::Core + def text + 'passed' + end + + def label + 'passed' + end + + def icon + 'icon_status_success' + end + end + end + end +end diff --git a/lib/gitlab/ci/status/success_warning.rb b/lib/gitlab/ci/status/success_warning.rb new file mode 100644 index 00000000000..d4cdab6957a --- /dev/null +++ b/lib/gitlab/ci/status/success_warning.rb @@ -0,0 +1,33 @@ +module Gitlab + module Ci + module Status + ## + # Extended status used when pipeline or stage passed conditionally. + # This means that failed jobs that are allowed to fail were present. + # + class SuccessWarning < SimpleDelegator + include Status::Extended + + def text + 'passed' + end + + def label + 'passed with warnings' + end + + def icon + 'icon_status_warning' + end + + def group + 'success_with_warnings' + end + + def self.matches?(subject, user) + subject.success? && subject.has_warnings? + end + end + end + end +end diff --git a/lib/gitlab/ci/trace_reader.rb b/lib/gitlab/ci/trace_reader.rb new file mode 100644 index 00000000000..1d7ddeb3e0f --- /dev/null +++ b/lib/gitlab/ci/trace_reader.rb @@ -0,0 +1,50 @@ +module Gitlab + module Ci + # This was inspired from: http://stackoverflow.com/a/10219411/1520132 + class TraceReader + BUFFER_SIZE = 4096 + + attr_accessor :path, :buffer_size + + def initialize(new_path, buffer_size: BUFFER_SIZE) + self.path = new_path + self.buffer_size = Integer(buffer_size) + end + + def read(last_lines: nil) + if last_lines + read_last_lines(last_lines) + else + File.read(path) + end + end + + def read_last_lines(max_lines) + File.open(path) do |file| + chunks = [] + pos = lines = 0 + max = file.size + + # We want an extra line to make sure fist line has full contents + while lines <= max_lines && pos < max + pos += buffer_size + + buf = if pos <= max + file.seek(-pos, IO::SEEK_END) + file.read(buffer_size) + else # Reached the head, read only left + file.seek(0) + file.read(buffer_size - (pos - max)) + end + + lines += buf.count("\n") + chunks.unshift(buf) + end + + chunks.join.lines.last(max_lines).join + .force_encoding(Encoding.default_external) + end + end + end + end +end diff --git a/lib/gitlab/conflict/file.rb b/lib/gitlab/conflict/file.rb new file mode 100644 index 00000000000..d80bc748209 --- /dev/null +++ b/lib/gitlab/conflict/file.rb @@ -0,0 +1,246 @@ +module Gitlab + module Conflict + class File + include Gitlab::Routing.url_helpers + include IconsHelper + + class MissingResolution < ResolutionError + end + + CONTEXT_LINES = 3 + + attr_reader :merge_file_result, :their_path, :our_path, :our_mode, :merge_request, :repository + + def initialize(merge_file_result, conflict, merge_request:) + @merge_file_result = merge_file_result + @their_path = conflict[:theirs][:path] + @our_path = conflict[:ours][:path] + @our_mode = conflict[:ours][:mode] + @merge_request = merge_request + @repository = merge_request.project.repository + @match_line_headers = {} + end + + def content + merge_file_result[:data] + end + + def our_blob + @our_blob ||= repository.blob_at(merge_request.diff_refs.head_sha, our_path) + end + + def type + lines unless @type + + @type.inquiry + end + + # Array of Gitlab::Diff::Line objects + def lines + return @lines if defined?(@lines) + + begin + @type = 'text' + @lines = Gitlab::Conflict::Parser.new.parse(content, + our_path: our_path, + their_path: their_path, + parent_file: self) + rescue Gitlab::Conflict::Parser::ParserError + @type = 'text-editor' + @lines = nil + end + end + + def resolve_lines(resolution) + section_id = nil + + lines.map do |line| + unless line.type + section_id = nil + next line + end + + section_id ||= line_code(line) + + case resolution[section_id] + when 'head' + next unless line.type == 'new' + when 'origin' + next unless line.type == 'old' + else + raise MissingResolution, "Missing resolution for section ID: #{section_id}" + end + + line + end.compact + end + + def resolve_content(resolution) + if resolution == content + raise MissingResolution, "Resolved content has no changes for file #{our_path}" + end + + resolution + end + + def highlight_lines! + their_file = lines.reject { |line| line.type == 'new' }.map(&:text).join("\n") + our_file = lines.reject { |line| line.type == 'old' }.map(&:text).join("\n") + + their_highlight = Gitlab::Highlight.highlight(their_path, their_file, repository: repository).lines + our_highlight = Gitlab::Highlight.highlight(our_path, our_file, repository: repository).lines + + lines.each do |line| + line.rich_text = + if line.type == 'old' + their_highlight[line.old_line - 1].try(:html_safe) + else + our_highlight[line.new_line - 1].try(:html_safe) + end + end + end + + def sections + return @sections if @sections + + chunked_lines = lines.chunk { |line| line.type.nil? }.to_a + match_line = nil + + sections_count = chunked_lines.size + + @sections = chunked_lines.flat_map.with_index do |(no_conflict, lines), i| + section = nil + + # We need to reduce context sections to CONTEXT_LINES. Conflict sections are + # always shown in full. + if no_conflict + conflict_before = i > 0 + conflict_after = (sections_count - i) > 1 + + if conflict_before && conflict_after + # Create a gap in a long context section. + if lines.length > CONTEXT_LINES * 2 + head_lines = lines.first(CONTEXT_LINES) + tail_lines = lines.last(CONTEXT_LINES) + + # Ensure any existing match line has text for all lines up to the last + # line of its context. + update_match_line_text(match_line, head_lines.last) + + # Insert a new match line after the created gap. + match_line = create_match_line(tail_lines.first) + + section = [ + { conflict: false, lines: head_lines }, + { conflict: false, lines: tail_lines.unshift(match_line) } + ] + end + elsif conflict_after + tail_lines = lines.last(CONTEXT_LINES) + + # Create a gap and insert a match line at the start. + if lines.length > tail_lines.length + match_line = create_match_line(tail_lines.first) + + tail_lines.unshift(match_line) + end + + lines = tail_lines + elsif conflict_before + # We're at the end of the file (no conflicts after), so just remove extra + # trailing lines. + lines = lines.first(CONTEXT_LINES) + end + end + + # We want to update the match line's text every time unless we've already + # created a gap and its corresponding match line. + update_match_line_text(match_line, lines.last) unless section + + section ||= { conflict: !no_conflict, lines: lines } + section[:id] = line_code(lines.first) unless no_conflict + section + end + end + + def line_code(line) + Gitlab::Diff::LineCode.generate(our_path, line.new_pos, line.old_pos) + end + + def create_match_line(line) + Gitlab::Diff::Line.new('', 'match', line.index, line.old_pos, line.new_pos) + end + + # Any line beginning with a letter, an underscore, or a dollar can be used in a + # match line header. Only context sections can contain match lines, as match lines + # have to exist in both versions of the file. + def find_match_line_header(index) + return @match_line_headers[index] if @match_line_headers.key?(index) + + @match_line_headers[index] = begin + if index >= 0 + line = lines[index] + + if line.type.nil? && line.text.match(/\A[A-Za-z$_]/) + " #{line.text}" + else + find_match_line_header(index - 1) + end + end + end + end + + # Set the match line's text for the current line. A match line takes its start + # position and context header (where present) from itself, and its end position from + # the line passed in. + def update_match_line_text(match_line, line) + return unless match_line + + header = find_match_line_header(match_line.index - 1) + + match_line.text = "@@ -#{match_line.old_pos},#{line.old_pos} +#{match_line.new_pos},#{line.new_pos} @@#{header}" + end + + def as_json(opts = {}) + json_hash = { + old_path: their_path, + new_path: our_path, + blob_icon: file_type_icon_class('file', our_mode, our_path), + blob_path: namespace_project_blob_path(merge_request.project.namespace, + merge_request.project, + ::File.join(merge_request.diff_refs.head_sha, our_path)) + } + + json_hash.tap do |json_hash| + if opts[:full_content] + json_hash[:content] = content + json_hash[:blob_ace_mode] = our_blob && our_blob.language.try(:ace_mode) + else + json_hash[:sections] = sections if type.text? + json_hash[:type] = type + json_hash[:content_path] = content_path + end + end + end + + def content_path + conflict_for_path_namespace_project_merge_request_path(merge_request.project.namespace, + merge_request.project, + merge_request, + old_path: their_path, + new_path: our_path) + end + + # Don't try to print merge_request or repository. + def inspect + instance_variables = [:merge_file_result, :their_path, :our_path, :our_mode, :type].map do |instance_variable| + value = instance_variable_get("@#{instance_variable}") + + "#{instance_variable}=\"#{value}\"" + end + + "#<#{self.class} #{instance_variables.join(' ')}>" + end + end + end +end diff --git a/lib/gitlab/conflict/file_collection.rb b/lib/gitlab/conflict/file_collection.rb new file mode 100644 index 00000000000..fa5bd4649d4 --- /dev/null +++ b/lib/gitlab/conflict/file_collection.rb @@ -0,0 +1,61 @@ +module Gitlab + module Conflict + class FileCollection + class ConflictSideMissing < StandardError + end + + attr_reader :merge_request, :our_commit, :their_commit + + def initialize(merge_request) + @merge_request = merge_request + @our_commit = merge_request.source_branch_head.raw.raw_commit + @their_commit = merge_request.target_branch_head.raw.raw_commit + end + + def repository + merge_request.project.repository + end + + def merge_index + @merge_index ||= repository.rugged.merge_commits(our_commit, their_commit) + end + + def files + @files ||= merge_index.conflicts.map do |conflict| + raise ConflictSideMissing unless conflict[:theirs] && conflict[:ours] + + Gitlab::Conflict::File.new(merge_index.merge_file(conflict[:ours][:path]), + conflict, + merge_request: merge_request) + end + end + + def file_for_path(old_path, new_path) + files.find { |file| file.their_path == old_path && file.our_path == new_path } + end + + def as_json(opts = nil) + { + target_branch: merge_request.target_branch, + source_branch: merge_request.source_branch, + commit_sha: merge_request.diff_head_sha, + commit_message: default_commit_message, + files: files + } + end + + def default_commit_message + conflict_filenames = merge_index.conflicts.map do |conflict| + "# #{conflict[:ours][:path]}" + end + + <<EOM.chomp +Merge branch '#{merge_request.target_branch}' into '#{merge_request.source_branch}' + +# Conflicts: +#{conflict_filenames.join("\n")} +EOM + end + end + end +end diff --git a/lib/gitlab/conflict/parser.rb b/lib/gitlab/conflict/parser.rb new file mode 100644 index 00000000000..ddd657903fb --- /dev/null +++ b/lib/gitlab/conflict/parser.rb @@ -0,0 +1,76 @@ +module Gitlab + module Conflict + class Parser + class UnresolvableError < StandardError + end + + class UnmergeableFile < UnresolvableError + end + + class UnsupportedEncoding < UnresolvableError + end + + # Recoverable errors - the conflict can be resolved in an editor, but not with + # sections. + class ParserError < StandardError + end + + class UnexpectedDelimiter < ParserError + end + + class MissingEndDelimiter < ParserError + end + + def parse(text, our_path:, their_path:, parent_file: nil) + raise UnmergeableFile if text.blank? # Typically a binary file + raise UnmergeableFile if text.length > 200.kilobytes + + begin + text.to_json + rescue Encoding::UndefinedConversionError + raise UnsupportedEncoding + end + + line_obj_index = 0 + line_old = 1 + line_new = 1 + type = nil + lines = [] + conflict_start = "<<<<<<< #{our_path}" + conflict_middle = '=======' + conflict_end = ">>>>>>> #{their_path}" + + text.each_line.map do |line| + full_line = line.delete("\n") + + if full_line == conflict_start + raise UnexpectedDelimiter unless type.nil? + + type = 'new' + elsif full_line == conflict_middle + raise UnexpectedDelimiter unless type == 'new' + + type = 'old' + elsif full_line == conflict_end + raise UnexpectedDelimiter unless type == 'old' + + type = nil + elsif line[0] == '\\' + type = 'nonewline' + lines << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new, parent_file: parent_file) + else + lines << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new, parent_file: parent_file) + line_old += 1 if type != 'new' + line_new += 1 if type != 'old' + + line_obj_index += 1 + end + end + + raise MissingEndDelimiter unless type.nil? + + lines + end + end + end +end diff --git a/lib/gitlab/conflict/resolution_error.rb b/lib/gitlab/conflict/resolution_error.rb new file mode 100644 index 00000000000..a0f2006bc24 --- /dev/null +++ b/lib/gitlab/conflict/resolution_error.rb @@ -0,0 +1,6 @@ +module Gitlab + module Conflict + class ResolutionError < StandardError + end + end +end diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index 9dc2602867e..15992b77680 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -1,47 +1,46 @@ module Gitlab class ContributionsCalendar - attr_reader :timestamps, :projects, :user + attr_reader :contributor + attr_reader :current_user + attr_reader :projects - def initialize(projects, user) - @projects = projects - @user = user + def initialize(contributor, current_user = nil) + @contributor = contributor + @current_user = current_user + @projects = ContributedProjectsFinder.new(contributor).execute(current_user) end - def timestamps - return @timestamps if @timestamps.present? + def activity_dates + return @activity_dates if @activity_dates.present? - @timestamps = {} + # Can't use Event.contributions here because we need to check 3 different + # project_features for the (currently) 3 different contribution types date_from = 1.year.ago + repo_events = event_counts(date_from, :repository). + having(action: Event::PUSHED) + issue_events = event_counts(date_from, :issues). + having(action: [Event::CREATED, Event::CLOSED], target_type: "Issue") + mr_events = event_counts(date_from, :merge_requests). + having(action: [Event::MERGED, Event::CREATED, Event::CLOSED], target_type: "MergeRequest") + note_events = event_counts(date_from, :merge_requests). + having(action: [Event::COMMENTED], target_type: "Note") - events = Event.reorder(nil).contributions.where(author_id: user.id). - where("created_at > ?", date_from).where(project_id: projects). - group('date(created_at)'). - select('date(created_at) as date, count(id) as total_amount'). - map(&:attributes) + union = Gitlab::SQL::Union.new([repo_events, issue_events, mr_events, note_events]) + events = Event.find_by_sql(union.to_sql).map(&:attributes) - dates = (1.year.ago.to_date..Date.today).to_a - - dates.each do |date| - date_id = date.to_time.to_i.to_s - @timestamps[date_id] = 0 - day_events = events.find { |day_events| day_events["date"] == date } - - if day_events - @timestamps[date_id] = day_events["total_amount"] - end + @activity_events = events.each_with_object(Hash.new {|h, k| h[k] = 0 }) do |event, activities| + activities[event["date"]] += event["total_amount"] end - - @timestamps end def events_by_date(date) - events = Event.contributions.where(author_id: user.id). - where("created_at > ? AND created_at < ?", date.beginning_of_day, date.end_of_day). + events = Event.contributions.where(author_id: contributor.id). + where(created_at: date.beginning_of_day..date.end_of_day). where(project_id: projects) - events.select do |event| - event.push? || event.issue? || event.merge_request? - end + # Use visible_to_user? instead of the complicated logic in activity_dates + # because we're only viewing the events for a single day. + events.select { |event| event.visible_to_user?(current_user) } end def starting_year @@ -51,5 +50,30 @@ module Gitlab def starting_month Date.today.month end + + private + + def event_counts(date_from, feature) + t = Event.arel_table + + # re-running the contributed projects query in each union is expensive, so + # use IN(project_ids...) instead. It's the intersection of two users so + # the list will be (relatively) short + @contributed_project_ids ||= projects.uniq.pluck(:id) + authed_projects = Project.where(id: @contributed_project_ids). + with_feature_available_for_user(feature, current_user). + reorder(nil). + select(:id) + + conditions = t[:created_at].gteq(date_from.beginning_of_day). + and(t[:created_at].lteq(Date.today.end_of_day)). + and(t[:author_id].eq(contributor.id)) + + Event.reorder(nil). + select(t[:project_id], t[:target_type], t[:action], 'date(created_at) AS date', 'count(id) as total_amount'). + group(t[:project_id], t[:target_type], t[:action], 'date(created_at)'). + where(conditions). + having(t[:project_id].in(Arel::Nodes::SqlLiteral.new(authed_projects.to_sql))) + end end end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 735331df66c..82576d197fe 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -9,7 +9,9 @@ module Gitlab end def ensure_application_settings! - if connect_to_db? + return fake_application_settings unless connect_to_db? + + unless ENV['IN_MEMORY_APPLICATION_SETTINGS'] == 'true' begin settings = ::ApplicationSetting.current # In case Redis isn't running or the Redis UNIX socket file is not available @@ -20,36 +22,21 @@ module Gitlab settings ||= ::ApplicationSetting.create_from_defaults unless ActiveRecord::Migrator.needs_migration? end - settings || fake_application_settings + settings || in_memory_application_settings + end + + delegate :sidekiq_throttling_enabled?, to: :current_application_settings + + def in_memory_application_settings + @in_memory_application_settings ||= ::ApplicationSetting.new(::ApplicationSetting.defaults) + # In case migrations the application_settings table is not created yet, + # we fallback to a simple OpenStruct + rescue ActiveRecord::StatementInvalid, ActiveRecord::UnknownAttributeError + fake_application_settings end def fake_application_settings - OpenStruct.new( - default_projects_limit: Settings.gitlab['default_projects_limit'], - default_branch_protection: Settings.gitlab['default_branch_protection'], - signup_enabled: Settings.gitlab['signup_enabled'], - signin_enabled: Settings.gitlab['signin_enabled'], - gravatar_enabled: Settings.gravatar['enabled'], - sign_in_text: nil, - after_sign_up_text: nil, - help_page_text: nil, - shared_runners_text: nil, - restricted_visibility_levels: Settings.gitlab['restricted_visibility_levels'], - max_attachment_size: Settings.gitlab['max_attachment_size'], - session_expire_delay: Settings.gitlab['session_expire_delay'], - default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'], - default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'], - domain_whitelist: Settings.gitlab['domain_whitelist'], - import_sources: %w[github bitbucket gitlab gitorious google_code fogbugz git gitlab_project], - shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], - max_artifacts_size: Settings.artifacts['max_size'], - require_two_factor_authentication: false, - two_factor_grace_period: 48, - akismet_enabled: false, - repository_checks_enabled: true, - container_registry_token_expire_delay: 5, - user_default_external: false, - ) + OpenStruct.new(::ApplicationSetting.defaults) end private @@ -58,10 +45,8 @@ module Gitlab # When the DBMS is not available, an exception (e.g. PG::ConnectionBad) is raised active_db_connection = ActiveRecord::Base.connection.active? rescue false - ENV['USE_DB'] != 'false' && active_db_connection && - ActiveRecord::Base.connection.table_exists?('application_settings') - + ActiveRecord::Base.connection.table_exists?('application_settings') rescue ActiveRecord::NoDatabaseError false end diff --git a/lib/gitlab/cycle_analytics/base_event_fetcher.rb b/lib/gitlab/cycle_analytics/base_event_fetcher.rb new file mode 100644 index 00000000000..ab115afcaa5 --- /dev/null +++ b/lib/gitlab/cycle_analytics/base_event_fetcher.rb @@ -0,0 +1,67 @@ +module Gitlab + module CycleAnalytics + class BaseEventFetcher + include BaseQuery + + attr_reader :projections, :query, :stage, :order + + MAX_EVENTS = 50 + + def initialize(project:, stage:, options:) + @project = project + @stage = stage + @options = options + end + + def fetch + update_author! + + event_result.map do |event| + serialize(event) if has_permission?(event['id']) + end.compact + end + + def order + @order || default_order + end + + private + + def update_author! + return unless event_result.any? && event_result.first['author_id'] + + Updater.update!(event_result, from: 'author_id', to: 'author', klass: User) + end + + def event_result + @event_result ||= ActiveRecord::Base.connection.exec_query(events_query.to_sql).to_a + end + + def events_query + diff_fn = subtract_datetimes_diff(base_query, @options[:start_time_attrs], @options[:end_time_attrs]) + + base_query.project(extract_diff_epoch(diff_fn).as('total_time'), *projections).order(order.desc).take(MAX_EVENTS) + end + + def default_order + [@options[:start_time_attrs]].flatten.first + end + + def serialize(_event) + raise NotImplementedError.new("Expected #{self.name} to implement serialize(event)") + end + + def has_permission?(id) + allowed_ids.nil? || allowed_ids.include?(id.to_i) + end + + def allowed_ids + nil + end + + def event_result_ids + event_result.map { |event| event['id'] } + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/base_query.rb b/lib/gitlab/cycle_analytics/base_query.rb new file mode 100644 index 00000000000..d560dca45c8 --- /dev/null +++ b/lib/gitlab/cycle_analytics/base_query.rb @@ -0,0 +1,31 @@ +module Gitlab + module CycleAnalytics + module BaseQuery + include MetricsTables + include Gitlab::Database::Median + include Gitlab::Database::DateTime + + private + + def base_query + @base_query ||= stage_query + end + + def stage_query + query = mr_closing_issues_table.join(issue_table).on(issue_table[:id].eq(mr_closing_issues_table[:issue_id])). + join(issue_metrics_table).on(issue_table[:id].eq(issue_metrics_table[:issue_id])). + where(issue_table[:project_id].eq(@project.id)). + where(issue_table[:deleted_at].eq(nil)). + where(issue_table[:created_at].gteq(@options[:from])) + + # Load merge_requests + query = query.join(mr_table, Arel::Nodes::OuterJoin). + on(mr_table[:id].eq(mr_closing_issues_table[:merge_request_id])). + join(mr_metrics_table). + on(mr_table[:id].eq(mr_metrics_table[:merge_request_id])) + + query + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/base_stage.rb b/lib/gitlab/cycle_analytics/base_stage.rb new file mode 100644 index 00000000000..559e3939da6 --- /dev/null +++ b/lib/gitlab/cycle_analytics/base_stage.rb @@ -0,0 +1,54 @@ +module Gitlab + module CycleAnalytics + class BaseStage + include BaseQuery + + def initialize(project:, options:) + @project = project + @options = options + end + + def events + event_fetcher.fetch + end + + def as_json + AnalyticsStageSerializer.new.represent(self) + end + + def title + name.to_s.capitalize + end + + def median + cte_table = Arel::Table.new("cte_table_for_#{name}") + + # Build a `SELECT` query. We find the first of the `end_time_attrs` that isn't `NULL` (call this end_time). + # Next, we find the first of the start_time_attrs that isn't `NULL` (call this start_time). + # We compute the (end_time - start_time) interval, and give it an alias based on the current + # cycle analytics stage. + interval_query = Arel::Nodes::As.new( + cte_table, + subtract_datetimes(base_query.dup, start_time_attrs, end_time_attrs, name.to_s)) + + median_datetime(cte_table, interval_query, name) + end + + def name + raise NotImplementedError.new("Expected #{self.name} to implement name") + end + + private + + def event_fetcher + @event_fetcher ||= Gitlab::CycleAnalytics::EventFetcher[name].new(project: @project, + stage: name, + options: event_options) + end + + def event_options + @options.merge(start_time_attrs: start_time_attrs, end_time_attrs: end_time_attrs) + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/code_event_fetcher.rb b/lib/gitlab/cycle_analytics/code_event_fetcher.rb new file mode 100644 index 00000000000..d5bf6149749 --- /dev/null +++ b/lib/gitlab/cycle_analytics/code_event_fetcher.rb @@ -0,0 +1,25 @@ +module Gitlab + module CycleAnalytics + class CodeEventFetcher < BaseEventFetcher + include MergeRequestAllowed + + def initialize(*args) + @projections = [mr_table[:title], + mr_table[:iid], + mr_table[:id], + mr_table[:created_at], + mr_table[:state], + mr_table[:author_id]] + @order = mr_table[:created_at] + + super(*args) + end + + private + + def serialize(event) + AnalyticsMergeRequestSerializer.new(project: @project).represent(event) + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/code_stage.rb b/lib/gitlab/cycle_analytics/code_stage.rb new file mode 100644 index 00000000000..1e52b6614a1 --- /dev/null +++ b/lib/gitlab/cycle_analytics/code_stage.rb @@ -0,0 +1,25 @@ +module Gitlab + module CycleAnalytics + class CodeStage < BaseStage + def start_time_attrs + @start_time_attrs ||= issue_metrics_table[:first_mentioned_in_commit_at] + end + + def end_time_attrs + @end_time_attrs ||= mr_table[:created_at] + end + + def name + :code + end + + def legend + "Related Merge Requests" + end + + def description + "Time until first merge request" + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/event_fetcher.rb b/lib/gitlab/cycle_analytics/event_fetcher.rb new file mode 100644 index 00000000000..50e126cf00b --- /dev/null +++ b/lib/gitlab/cycle_analytics/event_fetcher.rb @@ -0,0 +1,9 @@ +module Gitlab + module CycleAnalytics + module EventFetcher + def self.[](stage_name) + CycleAnalytics.const_get("#{stage_name.to_s.camelize}EventFetcher") + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/issue_allowed.rb b/lib/gitlab/cycle_analytics/issue_allowed.rb new file mode 100644 index 00000000000..a7652a70641 --- /dev/null +++ b/lib/gitlab/cycle_analytics/issue_allowed.rb @@ -0,0 +1,9 @@ +module Gitlab + module CycleAnalytics + module IssueAllowed + def allowed_ids + @allowed_ids ||= IssuesFinder.new(@options[:current_user], project_id: @project.id).execute.where(id: event_result_ids).pluck(:id) + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/issue_event_fetcher.rb b/lib/gitlab/cycle_analytics/issue_event_fetcher.rb new file mode 100644 index 00000000000..3df9cbdcfce --- /dev/null +++ b/lib/gitlab/cycle_analytics/issue_event_fetcher.rb @@ -0,0 +1,23 @@ +module Gitlab + module CycleAnalytics + class IssueEventFetcher < BaseEventFetcher + include IssueAllowed + + def initialize(*args) + @projections = [issue_table[:title], + issue_table[:iid], + issue_table[:id], + issue_table[:created_at], + issue_table[:author_id]] + + super(*args) + end + + private + + def serialize(event) + AnalyticsIssueSerializer.new(project: @project).represent(event) + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/issue_stage.rb b/lib/gitlab/cycle_analytics/issue_stage.rb new file mode 100644 index 00000000000..213994988a5 --- /dev/null +++ b/lib/gitlab/cycle_analytics/issue_stage.rb @@ -0,0 +1,26 @@ +module Gitlab + module CycleAnalytics + class IssueStage < BaseStage + def start_time_attrs + @start_time_attrs ||= issue_table[:created_at] + end + + def end_time_attrs + @end_time_attrs ||= [issue_metrics_table[:first_associated_with_milestone_at], + issue_metrics_table[:first_added_to_board_at]] + end + + def name + :issue + end + + def legend + "Related Issues" + end + + def description + "Time before an issue gets scheduled" + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/merge_request_allowed.rb b/lib/gitlab/cycle_analytics/merge_request_allowed.rb new file mode 100644 index 00000000000..28f6db44759 --- /dev/null +++ b/lib/gitlab/cycle_analytics/merge_request_allowed.rb @@ -0,0 +1,9 @@ +module Gitlab + module CycleAnalytics + module MergeRequestAllowed + def allowed_ids + @allowed_ids ||= MergeRequestsFinder.new(@options[:current_user], project_id: @project.id).execute.where(id: event_result_ids).pluck(:id) + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/metrics_tables.rb b/lib/gitlab/cycle_analytics/metrics_tables.rb new file mode 100644 index 00000000000..9d25ef078e8 --- /dev/null +++ b/lib/gitlab/cycle_analytics/metrics_tables.rb @@ -0,0 +1,37 @@ +module Gitlab + module CycleAnalytics + module MetricsTables + def mr_metrics_table + MergeRequest::Metrics.arel_table + end + + def mr_table + MergeRequest.arel_table + end + + def mr_diff_table + MergeRequestDiff.arel_table + end + + def mr_closing_issues_table + MergeRequestsClosingIssues.arel_table + end + + def issue_table + Issue.arel_table + end + + def issue_metrics_table + Issue::Metrics.arel_table + end + + def user_table + User.arel_table + end + + def build_table + ::CommitStatus.arel_table + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/permissions.rb b/lib/gitlab/cycle_analytics/permissions.rb new file mode 100644 index 00000000000..bef3b95ff1b --- /dev/null +++ b/lib/gitlab/cycle_analytics/permissions.rb @@ -0,0 +1,44 @@ +module Gitlab + module CycleAnalytics + class Permissions + STAGE_PERMISSIONS = { + issue: :read_issue, + code: :read_merge_request, + test: :read_build, + review: :read_merge_request, + staging: :read_build, + production: :read_issue, + }.freeze + + def self.get(*args) + new(*args).get + end + + def initialize(user:, project:) + @user = user + @project = project + @stage_permission_hash = {} + end + + def get + ::CycleAnalytics::STAGES.each do |stage| + @stage_permission_hash[stage] = authorized_stage?(stage) + end + + @stage_permission_hash + end + + private + + def authorized_stage?(stage) + return false unless authorize_project(:read_cycle_analytics) + + STAGE_PERMISSIONS[stage] ? authorize_project(STAGE_PERMISSIONS[stage]) : true + end + + def authorize_project(permission) + Ability.allowed?(@user, permission, @project) + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb new file mode 100644 index 00000000000..7d342a2d2cb --- /dev/null +++ b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb @@ -0,0 +1,44 @@ +module Gitlab + module CycleAnalytics + class PlanEventFetcher < BaseEventFetcher + def initialize(*args) + @projections = [mr_diff_table[:st_commits].as('commits'), + issue_metrics_table[:first_mentioned_in_commit_at]] + + super(*args) + end + + def events_query + base_query.join(mr_diff_table).on(mr_diff_table[:merge_request_id].eq(mr_table[:id])) + + super + end + + private + + def serialize(event) + st_commit = first_time_reference_commit(event.delete('commits'), event) + + return unless st_commit + + serialize_commit(event, st_commit, query) + end + + def first_time_reference_commit(commits, event) + return nil if commits.blank? + + YAML.load(commits).find do |commit| + next unless commit[:committed_date] && event['first_mentioned_in_commit_at'] + + commit[:committed_date].to_i == DateTime.parse(event['first_mentioned_in_commit_at'].to_s).to_i + end + end + + def serialize_commit(event, st_commit, query) + commit = Commit.new(Gitlab::Git::Commit.new(st_commit), @project) + + AnalyticsCommitSerializer.new(project: @project, total_time: event['total_time']).represent(commit) + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/plan_stage.rb b/lib/gitlab/cycle_analytics/plan_stage.rb new file mode 100644 index 00000000000..45d51d30ccc --- /dev/null +++ b/lib/gitlab/cycle_analytics/plan_stage.rb @@ -0,0 +1,26 @@ +module Gitlab + module CycleAnalytics + class PlanStage < BaseStage + def start_time_attrs + @start_time_attrs ||= [issue_metrics_table[:first_associated_with_milestone_at], + issue_metrics_table[:first_added_to_board_at]] + end + + def end_time_attrs + @end_time_attrs ||= issue_metrics_table[:first_mentioned_in_commit_at] + end + + def name + :plan + end + + def legend + "Related Commits" + end + + def description + "Time before an issue starts implementation" + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/production_event_fetcher.rb b/lib/gitlab/cycle_analytics/production_event_fetcher.rb new file mode 100644 index 00000000000..0fa2e87f673 --- /dev/null +++ b/lib/gitlab/cycle_analytics/production_event_fetcher.rb @@ -0,0 +1,6 @@ +module Gitlab + module CycleAnalytics + class ProductionEventFetcher < IssueEventFetcher + end + end +end diff --git a/lib/gitlab/cycle_analytics/production_helper.rb b/lib/gitlab/cycle_analytics/production_helper.rb new file mode 100644 index 00000000000..d693443bfa4 --- /dev/null +++ b/lib/gitlab/cycle_analytics/production_helper.rb @@ -0,0 +1,9 @@ +module Gitlab + module CycleAnalytics + module ProductionHelper + def stage_query + super.where(mr_metrics_table[:first_deployed_to_production_at].gteq(@options[:from])) + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/production_stage.rb b/lib/gitlab/cycle_analytics/production_stage.rb new file mode 100644 index 00000000000..9f387a02945 --- /dev/null +++ b/lib/gitlab/cycle_analytics/production_stage.rb @@ -0,0 +1,32 @@ +module Gitlab + module CycleAnalytics + class ProductionStage < BaseStage + include ProductionHelper + + def start_time_attrs + @start_time_attrs ||= issue_table[:created_at] + end + + def end_time_attrs + @end_time_attrs ||= mr_metrics_table[:first_deployed_to_production_at] + end + + def name + :production + end + + def legend + "Related Issues" + end + + def description + "From issue creation until deploy to production" + end + + def query + # Limit to merge requests that have been deployed to production after `@from` + query.where(mr_metrics_table[:first_deployed_to_production_at].gteq(@from)) + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/review_event_fetcher.rb b/lib/gitlab/cycle_analytics/review_event_fetcher.rb new file mode 100644 index 00000000000..4c7b3f4467f --- /dev/null +++ b/lib/gitlab/cycle_analytics/review_event_fetcher.rb @@ -0,0 +1,22 @@ +module Gitlab + module CycleAnalytics + class ReviewEventFetcher < BaseEventFetcher + include MergeRequestAllowed + + def initialize(*args) + @projections = [mr_table[:title], + mr_table[:iid], + mr_table[:id], + mr_table[:created_at], + mr_table[:state], + mr_table[:author_id]] + + super(*args) + end + + def serialize(event) + AnalyticsMergeRequestSerializer.new(project: @project).represent(event) + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/review_stage.rb b/lib/gitlab/cycle_analytics/review_stage.rb new file mode 100644 index 00000000000..4744be834de --- /dev/null +++ b/lib/gitlab/cycle_analytics/review_stage.rb @@ -0,0 +1,25 @@ +module Gitlab + module CycleAnalytics + class ReviewStage < BaseStage + def start_time_attrs + @start_time_attrs ||= mr_table[:created_at] + end + + def end_time_attrs + @end_time_attrs ||= mr_metrics_table[:merged_at] + end + + def name + :review + end + + def legend + "Relative Merged Requests" + end + + def description + "Time between merge request creation and merge/close" + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/stage.rb b/lib/gitlab/cycle_analytics/stage.rb new file mode 100644 index 00000000000..28e0455df59 --- /dev/null +++ b/lib/gitlab/cycle_analytics/stage.rb @@ -0,0 +1,9 @@ +module Gitlab + module CycleAnalytics + module Stage + def self.[](stage_name) + CycleAnalytics.const_get("#{stage_name.to_s.camelize}Stage") + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/stage_summary.rb b/lib/gitlab/cycle_analytics/stage_summary.rb new file mode 100644 index 00000000000..fc77bd86097 --- /dev/null +++ b/lib/gitlab/cycle_analytics/stage_summary.rb @@ -0,0 +1,23 @@ +module Gitlab + module CycleAnalytics + class StageSummary + def initialize(project, from:, current_user:) + @project = project + @from = from + @current_user = current_user + end + + def data + [serialize(Summary::Issue.new(project: @project, from: @from, current_user: @current_user)), + serialize(Summary::Commit.new(project: @project, from: @from)), + serialize(Summary::Deploy.new(project: @project, from: @from))] + end + + private + + def serialize(summary_object) + AnalyticsSummarySerializer.new.represent(summary_object) + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/staging_event_fetcher.rb b/lib/gitlab/cycle_analytics/staging_event_fetcher.rb new file mode 100644 index 00000000000..36c0260dbfe --- /dev/null +++ b/lib/gitlab/cycle_analytics/staging_event_fetcher.rb @@ -0,0 +1,30 @@ +module Gitlab + module CycleAnalytics + class StagingEventFetcher < BaseEventFetcher + def initialize(*args) + @projections = [build_table[:id]] + @order = build_table[:created_at] + + super(*args) + end + + def fetch + Updater.update!(event_result, from: 'id', to: 'build', klass: ::Ci::Build) + + super + end + + def events_query + base_query.join(build_table).on(mr_metrics_table[:pipeline_id].eq(build_table[:commit_id])) + + super + end + + private + + def serialize(event) + AnalyticsBuildSerializer.new.represent(event['build']) + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/staging_stage.rb b/lib/gitlab/cycle_analytics/staging_stage.rb new file mode 100644 index 00000000000..3cdbe04fbaf --- /dev/null +++ b/lib/gitlab/cycle_analytics/staging_stage.rb @@ -0,0 +1,26 @@ +module Gitlab + module CycleAnalytics + class StagingStage < BaseStage + include ProductionHelper + def start_time_attrs + @start_time_attrs ||= mr_metrics_table[:merged_at] + end + + def end_time_attrs + @end_time_attrs ||= mr_metrics_table[:first_deployed_to_production_at] + end + + def name + :staging + end + + def legend + "Relative Deployed Builds" + end + + def description + "From merge request merge until deploy to production" + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/summary/base.rb b/lib/gitlab/cycle_analytics/summary/base.rb new file mode 100644 index 00000000000..43fa3795e5c --- /dev/null +++ b/lib/gitlab/cycle_analytics/summary/base.rb @@ -0,0 +1,20 @@ +module Gitlab + module CycleAnalytics + module Summary + class Base + def initialize(project:, from:) + @project = project + @from = from + end + + def title + self.class.name.demodulize + end + + def value + raise NotImplementedError.new("Expected #{self.name} to implement value") + end + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/summary/commit.rb b/lib/gitlab/cycle_analytics/summary/commit.rb new file mode 100644 index 00000000000..7b8faa4d854 --- /dev/null +++ b/lib/gitlab/cycle_analytics/summary/commit.rb @@ -0,0 +1,39 @@ +module Gitlab + module CycleAnalytics + module Summary + class Commit < Base + def value + @value ||= count_commits + end + + private + + # Don't use the `Gitlab::Git::Repository#log` method, because it enforces + # a limit. Since we need a commit count, we _can't_ enforce a limit, so + # the easiest way forward is to replicate the relevant portions of the + # `log` function here. + def count_commits + return unless ref + + repository = @project.repository.raw_repository + sha = @project.repository.commit(ref).sha + + cmd = %W(git --git-dir=#{repository.path} log) + cmd << '--format=%H' + cmd << "--after=#{@from.iso8601}" + cmd << sha + + output, status = Gitlab::Popen.popen(cmd) + + raise IOError, output unless status.zero? + + output.lines.count + end + + def ref + @ref ||= @project.default_branch.presence + end + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/summary/deploy.rb b/lib/gitlab/cycle_analytics/summary/deploy.rb new file mode 100644 index 00000000000..06032e9200e --- /dev/null +++ b/lib/gitlab/cycle_analytics/summary/deploy.rb @@ -0,0 +1,11 @@ +module Gitlab + module CycleAnalytics + module Summary + class Deploy < Base + def value + @value ||= @project.deployments.where("created_at > ?", @from).count + end + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/summary/issue.rb b/lib/gitlab/cycle_analytics/summary/issue.rb new file mode 100644 index 00000000000..008468f24b9 --- /dev/null +++ b/lib/gitlab/cycle_analytics/summary/issue.rb @@ -0,0 +1,21 @@ +module Gitlab + module CycleAnalytics + module Summary + class Issue < Base + def initialize(project:, from:, current_user:) + @project = project + @from = from + @current_user = current_user + end + + def title + 'New Issue' + end + + def value + @value ||= IssuesFinder.new(@current_user, project_id: @project.id).execute.created_after(@from).count + end + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/test_event_fetcher.rb b/lib/gitlab/cycle_analytics/test_event_fetcher.rb new file mode 100644 index 00000000000..a2589c6601a --- /dev/null +++ b/lib/gitlab/cycle_analytics/test_event_fetcher.rb @@ -0,0 +1,6 @@ +module Gitlab + module CycleAnalytics + class TestEventFetcher < StagingEventFetcher + end + end +end diff --git a/lib/gitlab/cycle_analytics/test_stage.rb b/lib/gitlab/cycle_analytics/test_stage.rb new file mode 100644 index 00000000000..e96943833bc --- /dev/null +++ b/lib/gitlab/cycle_analytics/test_stage.rb @@ -0,0 +1,33 @@ +module Gitlab + module CycleAnalytics + class TestStage < BaseStage + def start_time_attrs + @start_time_attrs ||= mr_metrics_table[:latest_build_started_at] + end + + def end_time_attrs + @end_time_attrs ||= mr_metrics_table[:latest_build_finished_at] + end + + def name + :test + end + + def legend + "Relative Builds Trigger by Commits" + end + + def description + "Total test time for all commits/merges" + end + + def stage_query + if @options[:branch] + super.where(build_table[:ref].eq(@options[:branch])) + else + super + end + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/updater.rb b/lib/gitlab/cycle_analytics/updater.rb new file mode 100644 index 00000000000..953268ebd46 --- /dev/null +++ b/lib/gitlab/cycle_analytics/updater.rb @@ -0,0 +1,30 @@ +module Gitlab + module CycleAnalytics + class Updater + def self.update!(*args) + new(*args).update! + end + + def initialize(event_result, from:, to:, klass:) + @event_result = event_result + @klass = klass + @from = from + @to = to + end + + def update! + @event_result.each do |event| + event[@to] = items[event.delete(@from).to_i].first + end + end + + def result_ids + @event_result.map { |event| event[@from] } + end + + def items + @items ||= @klass.find(result_ids).group_by { |item| item['id'] } + end + end + end +end diff --git a/lib/gitlab/build_data_builder.rb b/lib/gitlab/data_builder/build.rb index 9f45aefda0f..f78106f5b10 100644 --- a/lib/gitlab/build_data_builder.rb +++ b/lib/gitlab/data_builder/build.rb @@ -1,11 +1,15 @@ module Gitlab - class BuildDataBuilder - class << self + module DataBuilder + module Build + extend self + def build(build) project = build.project commit = build.pipeline user = build.user + author_url = build_author_url(build.commit, commit) + data = { object_kind: 'build', @@ -41,6 +45,7 @@ module Gitlab message: commit.git_commit_message, author_name: commit.git_author_name, author_email: commit.git_author_email, + author_url: author_url, status: commit.status, duration: commit.duration, started_at: commit.started_at, @@ -60,6 +65,13 @@ module Gitlab data end + + private + + def build_author_url(commit, pipeline) + author = commit.try(:author) + author ? Gitlab::Routing.url_helpers.user_url(author) : "mailto:#{pipeline.git_author_email}" + end end end end diff --git a/lib/gitlab/note_data_builder.rb b/lib/gitlab/data_builder/note.rb index 8bdc89a7751..50fea1232af 100644 --- a/lib/gitlab/note_data_builder.rb +++ b/lib/gitlab/data_builder/note.rb @@ -1,6 +1,8 @@ module Gitlab - class NoteDataBuilder - class << self + module DataBuilder + module Note + extend self + # Produce a hash of post-receive data # # For all notes: diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb new file mode 100644 index 00000000000..e50e54b6e99 --- /dev/null +++ b/lib/gitlab/data_builder/pipeline.rb @@ -0,0 +1,62 @@ +module Gitlab + module DataBuilder + module Pipeline + extend self + + def build(pipeline) + { + object_kind: 'pipeline', + object_attributes: hook_attrs(pipeline), + user: pipeline.user.try(:hook_attrs), + project: pipeline.project.hook_attrs(backward: false), + commit: pipeline.commit.try(:hook_attrs), + builds: pipeline.builds.map(&method(:build_hook_attrs)) + } + end + + def hook_attrs(pipeline) + { + id: pipeline.id, + ref: pipeline.ref, + tag: pipeline.tag, + sha: pipeline.sha, + before_sha: pipeline.before_sha, + status: pipeline.status, + stages: pipeline.stages_name, + created_at: pipeline.created_at, + finished_at: pipeline.finished_at, + duration: pipeline.duration + } + end + + def build_hook_attrs(build) + { + id: build.id, + stage: build.stage, + name: build.name, + status: build.status, + created_at: build.created_at, + started_at: build.started_at, + finished_at: build.finished_at, + when: build.when, + manual: build.manual?, + user: build.user.try(:hook_attrs), + runner: build.runner && runner_hook_attrs(build.runner), + artifacts_file: { + filename: build.artifacts_file.filename, + size: build.artifacts_size + } + } + end + + def runner_hook_attrs(runner) + { + id: runner.id, + description: runner.description, + active: runner.active?, + is_shared: runner.is_shared? + } + end + end + end +end diff --git a/lib/gitlab/push_data_builder.rb b/lib/gitlab/data_builder/push.rb index c8f12577112..d76aa38f741 100644 --- a/lib/gitlab/push_data_builder.rb +++ b/lib/gitlab/data_builder/push.rb @@ -1,6 +1,8 @@ module Gitlab - class PushDataBuilder - class << self + module DataBuilder + module Push + extend self + # Produce a hash of post-receive data # # data = { @@ -81,7 +83,7 @@ module Gitlab tag = repository.find_tag(tag_name) if tag - commit = repository.commit(tag.target) + commit = repository.commit(tag.dereferenced_target) commit.try(:sha) end else diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 55b8f888d53..d160cadc2d0 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -6,7 +6,7 @@ module Gitlab MAX_INT_VALUE = 2147483647 def self.adapter_name - connection.adapter_name + ActiveRecord::Base.configurations[Rails.env]['adapter'] end def self.mysql? @@ -35,6 +35,20 @@ module Gitlab order end + def self.nulls_first_order(field, direction = 'ASC') + order = "#{field} #{direction}" + + if Gitlab::Database.postgresql? + order << ' NULLS FIRST' + else + # `field IS NULL` will be `0` for non-NULL columns and `1` for NULL + # columns. In the (default) ascending order, `0` comes first. + order.prepend("#{field} IS NULL, ") if direction == 'DESC' + end + + order + end + def self.random Gitlab::Database.postgresql? ? "RANDOM()" : "RAND()" end @@ -55,6 +69,36 @@ module Gitlab end end + def self.with_connection_pool(pool_size) + pool = create_connection_pool(pool_size) + + begin + yield(pool) + ensure + pool.disconnect! + end + end + + # pool_size - The size of the DB pool. + # host - An optional host name to use instead of the default one. + def self.create_connection_pool(pool_size, host = nil) + # See activerecord-4.2.7.1/lib/active_record/connection_adapters/connection_specification.rb + env = Rails.env + original_config = ActiveRecord::Base.configurations + + env_config = original_config[env].merge('pool' => pool_size) + env_config['host'] = host if host + + config = original_config.merge(env => env_config) + + spec = + ActiveRecord:: + ConnectionAdapters:: + ConnectionSpecification::Resolver.new(config).spec(env.to_sym) + + ActiveRecord::ConnectionAdapters::ConnectionPool.new(spec) + end + def self.connection ActiveRecord::Base.connection end diff --git a/lib/gitlab/database/date_time.rb b/lib/gitlab/database/date_time.rb new file mode 100644 index 00000000000..25e56998038 --- /dev/null +++ b/lib/gitlab/database/date_time.rb @@ -0,0 +1,31 @@ +module Gitlab + module Database + module DateTime + # Find the first of the `end_time_attrs` that isn't `NULL`. Subtract from it + # the first of the `start_time_attrs` that isn't NULL. `SELECT` the resulting interval + # along with an alias specified by the `as` parameter. + # + # Note: For MySQL, the interval is returned in seconds. + # For PostgreSQL, the interval is returned as an INTERVAL type. + def subtract_datetimes(query_so_far, start_time_attrs, end_time_attrs, as) + diff_fn = subtract_datetimes_diff(query_so_far, start_time_attrs, end_time_attrs) + + query_so_far.project(diff_fn.as(as)) + end + + def subtract_datetimes_diff(query_so_far, start_time_attrs, end_time_attrs) + if Gitlab::Database.postgresql? + Arel::Nodes::Subtraction.new( + Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(end_time_attrs)), + Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(start_time_attrs))) + elsif Gitlab::Database.mysql? + Arel::Nodes::NamedFunction.new( + "TIMESTAMPDIFF", + [Arel.sql('second'), + Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(start_time_attrs)), + Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(end_time_attrs))]) + end + end + end + end +end diff --git a/lib/gitlab/database/median.rb b/lib/gitlab/database/median.rb new file mode 100644 index 00000000000..23890e5f493 --- /dev/null +++ b/lib/gitlab/database/median.rb @@ -0,0 +1,118 @@ +# https://www.periscopedata.com/blog/medians-in-sql.html +module Gitlab + module Database + module Median + def median_datetime(arel_table, query_so_far, column_sym) + median_queries = + if Gitlab::Database.postgresql? + pg_median_datetime_sql(arel_table, query_so_far, column_sym) + elsif Gitlab::Database.mysql? + mysql_median_datetime_sql(arel_table, query_so_far, column_sym) + end + + results = Array.wrap(median_queries).map do |query| + ActiveRecord::Base.connection.execute(query) + end + extract_median(results).presence + end + + def extract_median(results) + result = results.compact.first + + if Gitlab::Database.postgresql? + result = result.first.presence + median = result['median'] if result + median.to_f if median + elsif Gitlab::Database.mysql? + result.to_a.flatten.first + end + end + + def mysql_median_datetime_sql(arel_table, query_so_far, column_sym) + query = arel_table. + from(arel_table.project(Arel.sql('*')).order(arel_table[column_sym]).as(arel_table.table_name)). + project(average([arel_table[column_sym]], 'median')). + where( + Arel::Nodes::Between.new( + Arel.sql("(select @row_id := @row_id + 1)"), + Arel::Nodes::And.new( + [Arel.sql('@ct/2.0'), + Arel.sql('@ct/2.0 + 1')] + ) + ) + ). + # Disallow negative values + where(arel_table[column_sym].gteq(0)) + + [ + Arel.sql("CREATE TEMPORARY TABLE IF NOT EXISTS #{query_so_far.to_sql}"), + Arel.sql("set @ct := (select count(1) from #{arel_table.table_name});"), + Arel.sql("set @row_id := 0;"), + query.to_sql, + Arel.sql("DROP TEMPORARY TABLE IF EXISTS #{arel_table.table_name};") + ] + end + + def pg_median_datetime_sql(arel_table, query_so_far, column_sym) + # Create a CTE with the column we're operating on, row number (after sorting by the column + # we're operating on), and count of the table we're operating on (duplicated across) all rows + # of the CTE. For example, if we're looking to find the median of the `projects.star_count` + # column, the CTE might look like this: + # + # star_count | row_id | ct + # ------------+--------+---- + # 5 | 1 | 3 + # 9 | 2 | 3 + # 15 | 3 | 3 + cte_table = Arel::Table.new("ordered_records") + cte = Arel::Nodes::As.new( + cte_table, + arel_table. + project( + arel_table[column_sym].as(column_sym.to_s), + Arel::Nodes::Over.new(Arel::Nodes::NamedFunction.new("row_number", []), + Arel::Nodes::Window.new.order(arel_table[column_sym])).as('row_id'), + arel_table.project("COUNT(1)").as('ct')). + # Disallow negative values + where(arel_table[column_sym].gteq(zero_interval))) + + # From the CTE, select either the middle row or the middle two rows (this is accomplished + # by 'where cte.row_id between cte.ct / 2.0 AND cte.ct / 2.0 + 1'). Find the average of the + # selected rows, and this is the median value. + cte_table.project(average([extract_epoch(cte_table[column_sym])], "median")). + where( + Arel::Nodes::Between.new( + cte_table[:row_id], + Arel::Nodes::And.new( + [(cte_table[:ct] / Arel.sql('2.0')), + (cte_table[:ct] / Arel.sql('2.0') + 1)] + ) + ) + ). + with(query_so_far, cte). + to_sql + end + + private + + def average(args, as) + Arel::Nodes::NamedFunction.new("AVG", args, as) + end + + def extract_epoch(arel_attribute) + Arel.sql(%Q{EXTRACT(EPOCH FROM "#{arel_attribute.relation.name}"."#{arel_attribute.name}")}) + end + + def extract_diff_epoch(diff) + return diff unless Gitlab::Database.postgresql? + + Arel.sql(%Q{EXTRACT(EPOCH FROM (#{diff.to_sql}))}) + end + + # Need to cast '0' to an INTERVAL before we can check if the interval is positive + def zero_interval + Arel::Nodes::NamedFunction.new("CAST", [Arel.sql("'0' AS INTERVAL")]) + end + end + end +end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 927f9dad20b..fc445ab9483 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -26,11 +26,68 @@ module Gitlab add_index(table_name, column_name, options) end + # Adds a foreign key with only minimal locking on the tables involved. + # + # This method only requires minimal locking when using PostgreSQL. When + # using MySQL this method will use Rails' default `add_foreign_key`. + # + # source - The source table containing the foreign key. + # target - The target table the key points to. + # column - The name of the column to create the foreign key on. + # on_delete - The action to perform when associated data is removed, + # defaults to "CASCADE". + def add_concurrent_foreign_key(source, target, column:, on_delete: :cascade) + # Transactions would result in ALTER TABLE locks being held for the + # duration of the transaction, defeating the purpose of this method. + if transaction_open? + raise 'add_concurrent_foreign_key can not be run inside a transaction' + end + + # While MySQL does allow disabling of foreign keys it has no equivalent + # of PostgreSQL's "VALIDATE CONSTRAINT". As a result we'll just fall + # back to the normal foreign key procedure. + if Database.mysql? + return add_foreign_key(source, target, + column: column, + on_delete: on_delete) + end + + disable_statement_timeout + + key_name = concurrent_foreign_key_name(source, column) + + # Using NOT VALID allows us to create a key without immediately + # validating it. This means we keep the ALTER TABLE lock only for a + # short period of time. The key _is_ enforced for any newly created + # data. + execute <<-EOF.strip_heredoc + ALTER TABLE #{source} + ADD CONSTRAINT #{key_name} + FOREIGN KEY (#{column}) + REFERENCES #{target} (id) + ON DELETE #{on_delete} NOT VALID; + EOF + + # Validate the existing constraint. This can potentially take a very + # long time to complete, but fortunately does not lock the source table + # while running. + execute("ALTER TABLE #{source} VALIDATE CONSTRAINT #{key_name};") + end + + # Returns the name for a concurrent foreign key. + # + # PostgreSQL constraint names have a limit of 63 bytes. The logic used + # here is based on Rails' foreign_key_name() method, which unfortunately + # is private so we can't rely on it directly. + def concurrent_foreign_key_name(table, column) + "fk_#{Digest::SHA256.hexdigest("#{table}_#{column}_fk").first(10)}" + end + # Long-running migrations may take more than the timeout allowed by # the database. Disable the session's statement timeout to ensure # migrations don't get killed prematurely. (PostgreSQL only) def disable_statement_timeout - ActiveRecord::Base.connection.execute('SET statement_timeout TO 0') if Database.postgresql? + execute('SET statement_timeout TO 0') if Database.postgresql? end # Updates the value of a column in batches. @@ -129,12 +186,14 @@ module Gitlab # column - The name of the column to add. # type - The column type (e.g. `:integer`). # default - The default value for the column. + # limit - Sets a column limit. For example, for :integer, the default is + # 4-bytes. Set `limit: 8` to allow 8-byte integers. # allow_null - When set to `true` the column will allow NULL values, the # default is to not allow NULL values. # # This method can also take a block which is passed directly to the # `update_column_in_batches` method. - def add_column_with_default(table, column, type, default:, allow_null: false, &block) + def add_column_with_default(table, column, type, default:, limit: nil, allow_null: false, &block) if transaction_open? raise 'add_column_with_default can not be run inside a transaction, ' \ 'you can disable transactions by calling disable_ddl_transaction! ' \ @@ -144,7 +203,11 @@ module Gitlab disable_statement_timeout transaction do - add_column(table, column, type, default: nil) + if limit + add_column(table, column, type, default: nil, limit: limit) + else + add_column(table, column, type, default: nil) + end # Changing the default before the update ensures any newly inserted # rows already use the proper default value. diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index e47df508ca2..c6bf25b5874 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -55,6 +55,12 @@ module Gitlab repository.commit(deleted_file ? old_ref : new_ref) end + def old_content_commit + return unless diff_refs + + repository.commit(old_ref) + end + def old_ref diff_refs.try(:base_sha) end @@ -111,13 +117,10 @@ module Gitlab diff_lines.count(&:removed?) end - def old_blob(commit = content_commit) + def old_blob(commit = old_content_commit) return unless commit - parent_id = commit.parent_id - return unless parent_id - - repository.blob_at(parent_id, old_path) + repository.blob_at(commit.id, old_path) end def blob(commit = content_commit) @@ -125,6 +128,10 @@ module Gitlab repository.blob_at(commit.id, file_path) end + + def file_identifier + "#{file_path}-#{new_file}-#{deleted_file}-#{renamed_file}" + end end end end diff --git a/lib/gitlab/diff/file_collection/merge_request.rb b/lib/gitlab/diff/file_collection/merge_request_diff.rb index 4f946908e2f..329d12f13d1 100644 --- a/lib/gitlab/diff/file_collection/merge_request.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff.rb @@ -1,14 +1,14 @@ module Gitlab module Diff module FileCollection - class MergeRequest < Base - def initialize(merge_request, diff_options:) - @merge_request = merge_request + class MergeRequestDiff < Base + def initialize(merge_request_diff, diff_options:) + @merge_request_diff = merge_request_diff - super(merge_request, - project: merge_request.project, + super(merge_request_diff, + project: merge_request_diff.project, diff_options: diff_options, - diff_refs: merge_request.diff_refs) + diff_refs: merge_request_diff.diff_refs) end def diff_files @@ -20,7 +20,7 @@ module Gitlab # Extracted method to highlight in the same iteration to the diff_collection. def decorate_diff!(diff) diff_file = super - cache_highlight!(diff_file) if cacheable? + cache_highlight!(diff_file) if cacheable?(diff_file) diff_file end @@ -35,16 +35,16 @@ module Gitlab # for the highlighted ones, so we just skip their execution. # If the highlighted diff files lines are not cached we calculate and cache them. # - # The content of the cache is a Hash where the key correspond to the file_path and the values are Arrays of + # The content of the cache is a Hash where the key identifies the file and the values are Arrays of # hashes that represent serialized diff lines. # def cache_highlight!(diff_file) - file_path = diff_file.file_path + item_key = diff_file.file_identifier - if highlight_cache[file_path] - highlight_diff_file_from_cache!(diff_file, highlight_cache[file_path]) + if highlight_cache[item_key] + highlight_diff_file_from_cache!(diff_file, highlight_cache[item_key]) else - highlight_cache[file_path] = diff_file.highlighted_diff_lines.map(&:to_hash) + highlight_cache[item_key] = diff_file.highlighted_diff_lines.map(&:to_hash) end end @@ -60,12 +60,15 @@ module Gitlab Rails.cache.write(cache_key, highlight_cache) if @highlight_cache_was_empty end - def cacheable? - @merge_request.merge_request_diff.present? + def cacheable?(diff_file) + @merge_request_diff.present? && + diff_file.blob && + diff_file.blob.text? && + @project.repository.diffable?(diff_file.blob) end def cache_key - [@merge_request.merge_request_diff, 'highlighted-diff-files', diff_options] + [@merge_request_diff, 'highlighted-diff-files', diff_options] end end end diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index 9ea976e18fa..7db896522a9 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -50,7 +50,7 @@ module Gitlab # Only update text if line is found. This will prevent # issues with submodules given the line only exists in diff content. if rich_line - line_prefix = diff_line.text.match(/\A(.)/) ? $1 : ' ' + line_prefix = diff_line.text =~ /\A(.)/ ? $1 : ' ' "#{line_prefix}#{rich_line}".html_safe end end diff --git a/lib/gitlab/diff/inline_diff_marker.rb b/lib/gitlab/diff/inline_diff_marker.rb index 87a9b1e23ac..736933b1c4b 100644 --- a/lib/gitlab/diff/inline_diff_marker.rb +++ b/lib/gitlab/diff/inline_diff_marker.rb @@ -4,7 +4,7 @@ module Gitlab MARKDOWN_SYMBOLS = { addition: "+", deletion: "-" - } + }.freeze attr_accessor :raw_line, :rich_line diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index cf097e0d0de..80a146b4a5a 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -2,11 +2,13 @@ module Gitlab module Diff class Line attr_reader :type, :index, :old_pos, :new_pos + attr_writer :rich_text attr_accessor :text - def initialize(text, type, index, old_pos, new_pos) + def initialize(text, type, index, old_pos, new_pos, parent_file: nil) @text, @type, @index = text, type, index @old_pos, @new_pos = old_pos, new_pos + @parent_file = parent_file end def self.init_from_hash(hash) @@ -43,9 +45,25 @@ module Gitlab type == 'old' end + def rich_text + @parent_file.highlight_lines! if @parent_file && !@rich_text + + @rich_text + end + def meta? type == 'match' || type == 'nonewline' end + + def as_json(opts = nil) + { + type: type, + old_line: old_line, + new_line: new_line, + text: text, + rich_text: rich_text || text + } + end end end end diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb index 59a2367b65d..8f844224a7a 100644 --- a/lib/gitlab/diff/parser.rb +++ b/lib/gitlab/diff/parser.rb @@ -20,7 +20,7 @@ module Gitlab full_line = line.delete("\n") - if line.match(/^@@ -/) + if line =~ /^@@ -/ type = "match" line_old = line.match(/\-[0-9]*/)[0].to_i.abs rescue 0 @@ -45,7 +45,7 @@ module Gitlab line_new += 1 when "-" line_old += 1 - when "\\" + when "\\" # rubocop:disable Lint/EmptyWhen # No increment else line_new += 1 diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb index 2fdcf8d7838..fc728123c97 100644 --- a/lib/gitlab/diff/position.rb +++ b/lib/gitlab/diff/position.rb @@ -139,13 +139,20 @@ module Gitlab private def find_diff_file(repository) - diffs = Gitlab::Git::Compare.new( - repository.raw_repository, - start_sha, - head_sha - ).diffs(paths: paths) + # We're at the initial commit, so just get that as we can't compare to anything. + compare = + if Gitlab::Git.blank_ref?(start_sha) + Gitlab::Git::Commit.find(repository.raw_repository, head_sha) + else + Gitlab::Git::Compare.new( + repository.raw_repository, + start_sha, + head_sha + ) + end + + diff = compare.diffs(paths: paths).first - diff = diffs.first return unless diff Gitlab::Diff::File.new(diff, repository: repository, diff_refs: diff_refs) diff --git a/lib/gitlab/downtime_check/message.rb b/lib/gitlab/downtime_check/message.rb index 4446e921e0d..543e62794c5 100644 --- a/lib/gitlab/downtime_check/message.rb +++ b/lib/gitlab/downtime_check/message.rb @@ -1,10 +1,10 @@ module Gitlab class DowntimeCheck class Message - attr_reader :path, :offline, :reason + attr_reader :path, :offline - OFFLINE = "\e[32moffline\e[0m" - ONLINE = "\e[31monline\e[0m" + OFFLINE = "\e[31moffline\e[0m".freeze + ONLINE = "\e[32monline\e[0m".freeze # path - The file path of the migration. # offline - When set to `true` the migration will require downtime. @@ -19,10 +19,21 @@ module Gitlab label = offline ? OFFLINE : ONLINE message = "[#{label}]: #{path}" - message += ": #{reason}" if reason + + if reason? + message += ":\n\n#{reason}\n\n" + end message end + + def reason? + @reason.present? + end + + def reason + @reason.strip.lines.map(&:strip).join("\n") + end end end end diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb new file mode 100644 index 00000000000..e0fdf3f3d64 --- /dev/null +++ b/lib/gitlab/ee_compat_check.rb @@ -0,0 +1,275 @@ +# rubocop: disable Rails/Output +module Gitlab + # Checks if a set of migrations requires downtime or not. + class EeCompatCheck + CE_REPO = 'https://gitlab.com/gitlab-org/gitlab-ce.git'.freeze + EE_REPO = 'https://gitlab.com/gitlab-org/gitlab-ee.git'.freeze + CHECK_DIR = Rails.root.join('ee_compat_check') + MAX_FETCH_DEPTH = 500 + IGNORED_FILES_REGEX = /(VERSION|CHANGELOG\.md:\d+)/.freeze + + attr_reader :repo_dir, :patches_dir, :ce_repo, :ce_branch + + def initialize(branch:, ce_repo: CE_REPO) + @repo_dir = CHECK_DIR.join('repo') + @patches_dir = CHECK_DIR.join('patches') + @ce_branch = branch + @ce_repo = ce_repo + end + + def check + ensure_ee_repo + ensure_patches_dir + + generate_patch(ce_branch, ce_patch_full_path) + + Dir.chdir(repo_dir) do + step("In the #{repo_dir} directory") + + status = catch(:halt_check) do + ce_branch_compat_check! + delete_ee_branch_locally! + ee_branch_presence_check! + ee_branch_compat_check! + end + + delete_ee_branch_locally! + + if status.nil? + true + else + false + end + end + end + + private + + def ensure_ee_repo + if Dir.exist?(repo_dir) + step("#{repo_dir} already exists") + else + cmd = %W[git clone --branch master --single-branch --depth 200 #{EE_REPO} #{repo_dir}] + step("Cloning #{EE_REPO} into #{repo_dir}", cmd) + end + end + + def ensure_patches_dir + FileUtils.mkdir_p(patches_dir) + end + + def generate_patch(branch, patch_path) + FileUtils.rm(patch_path, force: true) + + depth = 0 + loop do + depth += 50 + cmd = %W[git fetch --depth #{depth} origin --prune +refs/heads/master:refs/remotes/origin/master] + Gitlab::Popen.popen(cmd) + _, status = Gitlab::Popen.popen(%w[git merge-base FETCH_HEAD HEAD]) + + raise "#{branch} is too far behind master, please rebase it!" if depth >= MAX_FETCH_DEPTH + break if status.zero? + end + + step("Generating the patch against master in #{patch_path}") + output, status = Gitlab::Popen.popen(%w[git format-patch FETCH_HEAD --stdout]) + throw(:halt_check, :ko) unless status.zero? + + File.write(patch_path, output) + throw(:halt_check, :ko) unless File.exist?(patch_path) + end + + def ce_branch_compat_check! + if check_patch(ce_patch_full_path).zero? + puts applies_cleanly_msg(ce_branch) + throw(:halt_check) + end + end + + def ee_branch_presence_check! + status = step("Fetching origin/#{ee_branch}", %W[git fetch origin #{ee_branch}]) + + unless status.zero? + puts + puts ce_branch_doesnt_apply_cleanly_and_no_ee_branch_msg + + throw(:halt_check, :ko) + end + end + + def ee_branch_compat_check! + step("Checking out origin/#{ee_branch}", %W[git checkout -b #{ee_branch} FETCH_HEAD]) + + generate_patch(ee_branch, ee_patch_full_path) + + unless check_patch(ee_patch_full_path).zero? + puts + puts ee_branch_doesnt_apply_cleanly_msg + + throw(:halt_check, :ko) + end + + puts + puts applies_cleanly_msg(ee_branch) + end + + def check_patch(patch_path) + step("Checking out master", %w[git checkout master]) + step("Reseting to latest master", %w[git reset --hard origin/master]) + + step("Checking if #{patch_path} applies cleanly to EE/master") + output, status = Gitlab::Popen.popen(%W[git apply --check --3way #{patch_path}]) + + unless status.zero? + failed_files = output.lines.reduce([]) do |memo, line| + if line.start_with?('error: patch failed:') + file = line.sub(/\Aerror: patch failed: /, '') + memo << file unless file =~ IGNORED_FILES_REGEX + end + memo + end + + if failed_files.empty? + status = 0 + else + puts "\nConflicting files:" + failed_files.each do |file| + puts " - #{file}" + end + end + end + + status + end + + def delete_ee_branch_locally! + command(%w[git checkout master]) + step("Deleting the local #{ee_branch} branch", %W[git branch -D #{ee_branch}]) + end + + def ce_patch_name + @ce_patch_name ||= patch_name_from_branch(ce_branch) + end + + def ce_patch_full_path + @ce_patch_full_path ||= patches_dir.join(ce_patch_name) + end + + def ee_branch + @ee_branch ||= "#{ce_branch}-ee" + end + + def ee_patch_name + @ee_patch_name ||= patch_name_from_branch(ee_branch) + end + + def ee_patch_full_path + @ee_patch_full_path ||= patches_dir.join(ee_patch_name) + end + + def patch_name_from_branch(branch_name) + branch_name.parameterize << '.patch' + end + + def step(desc, cmd = nil) + puts "\n=> #{desc}\n" + + if cmd + start = Time.now + puts "\n$ #{cmd.join(' ')}" + status = command(cmd) + puts "\nFinished in #{Time.now - start} seconds" + status + end + end + + def command(cmd) + output, status = Gitlab::Popen.popen(cmd) + puts output + + status + end + + def applies_cleanly_msg(branch) + <<-MSG.strip_heredoc + ================================================================= + 🎉 Congratulations!! 🎉 + + The #{branch} branch applies cleanly to EE/master! + + Much ❤️!! + =================================================================\n + MSG + end + + def ce_branch_doesnt_apply_cleanly_and_no_ee_branch_msg + <<-MSG.strip_heredoc + ================================================================= + 💥 Oh no! 💥 + + The #{ce_branch} branch does not apply cleanly to the current + EE/master, and no #{ee_branch} branch was found in the EE repository. + + Please create a #{ee_branch} branch that includes changes from + #{ce_branch} but also specific changes than can be applied cleanly + to EE/master. + + There are different ways to create such branch: + + 1. Create a new branch based on the CE branch and rebase it on top of EE/master + + # In the EE repo + $ git fetch #{ce_repo} #{ce_branch} + $ git checkout -b #{ee_branch} FETCH_HEAD + + # You can squash the #{ce_branch} commits into a single "Port of #{ce_branch} to EE" commit + # before rebasing to limit the conflicts-resolving steps during the rebase + $ git fetch origin + $ git rebase origin/master + + At this point you will likely have conflicts. + Solve them, and continue/finish the rebase. + + You can squash the #{ce_branch} commits into a single "Port of #{ce_branch} to EE". + + 2. Create a new branch from master and cherry-pick your CE commits + + # In the EE repo + $ git fetch origin + $ git checkout -b #{ee_branch} origin/master + $ git fetch #{ce_repo} #{ce_branch} + $ git cherry-pick SHA # Repeat for all the commits you want to pick + + You can squash the #{ce_branch} commits into a single "Port of #{ce_branch} to EE" commit. + + Don't forget to push your branch to #{EE_REPO}: + + # In the EE repo + $ git push origin #{ee_branch} + + You can then retry this failed build, and hopefully it should pass. + + Stay 💪 ! + =================================================================\n + MSG + end + + def ee_branch_doesnt_apply_cleanly_msg + <<-MSG.strip_heredoc + ================================================================= + 💥 Oh no! 💥 + + The #{ce_branch} does not apply cleanly to the current + EE/master, and even though a #{ee_branch} branch exists in the EE + repository, it does not apply cleanly either to EE/master! + + Please update the #{ee_branch}, push it again to #{EE_REPO}, and + retry this build. + + Stay 💪 ! + =================================================================\n + MSG + end + end +end diff --git a/lib/gitlab/email/handler.rb b/lib/gitlab/email/handler.rb index bd3267e2a80..35ea2e0ef59 100644 --- a/lib/gitlab/email/handler.rb +++ b/lib/gitlab/email/handler.rb @@ -1,10 +1,11 @@ require 'gitlab/email/handler/create_note_handler' require 'gitlab/email/handler/create_issue_handler' +require 'gitlab/email/handler/unsubscribe_handler' module Gitlab module Email module Handler - HANDLERS = [CreateNoteHandler, CreateIssueHandler] + HANDLERS = [UnsubscribeHandler, CreateNoteHandler, CreateIssueHandler].freeze def self.for(mail, mail_key) HANDLERS.find do |klass| diff --git a/lib/gitlab/email/handler/base_handler.rb b/lib/gitlab/email/handler/base_handler.rb index b7ed11cb638..3f6ace0311a 100644 --- a/lib/gitlab/email/handler/base_handler.rb +++ b/lib/gitlab/email/handler/base_handler.rb @@ -9,51 +9,13 @@ module Gitlab @mail_key = mail_key end - def message - @message ||= process_message - end - - def author + def can_execute? raise NotImplementedError end - def project + def execute raise NotImplementedError end - - private - - def validate_permission!(permission) - raise UserNotFoundError unless author - raise UserBlockedError if author.blocked? - raise ProjectNotFound unless author.can?(:read_project, project) - raise UserNotAuthorizedError unless author.can?(permission, project) - end - - def process_message - message = ReplyParser.new(mail).execute.strip - add_attachments(message) - end - - def add_attachments(reply) - attachments = Email::AttachmentUploader.new(mail).execute(project) - - reply + attachments.map do |link| - "\n\n#{link[:markdown]}" - end.join - end - - def verify_record!(record:, invalid_exception:, record_name:) - return if record.persisted? - - error_title = "The #{record_name} could not be created for the following reasons:" - - msg = error_title + record.errors.full_messages.map do |error| - "\n\n- #{error}" - end.join - - raise invalid_exception, msg - end end end end diff --git a/lib/gitlab/email/handler/create_issue_handler.rb b/lib/gitlab/email/handler/create_issue_handler.rb index 4e6566af8ab..b8ec9138c10 100644 --- a/lib/gitlab/email/handler/create_issue_handler.rb +++ b/lib/gitlab/email/handler/create_issue_handler.rb @@ -5,16 +5,17 @@ module Gitlab module Email module Handler class CreateIssueHandler < BaseHandler - attr_reader :project_path, :authentication_token + include ReplyProcessing + attr_reader :project_path, :incoming_email_token def initialize(mail, mail_key) super(mail, mail_key) - @project_path, @authentication_token = + @project_path, @incoming_email_token = mail_key && mail_key.split('+', 2) end def can_handle? - !authentication_token.nil? + !incoming_email_token.nil? end def execute @@ -29,11 +30,11 @@ module Gitlab end def author - @author ||= User.find_by(authentication_token: authentication_token) + @author ||= User.find_by(incoming_email_token: incoming_email_token) end def project - @project ||= Project.find_with_namespace(project_path) + @project ||= Project.find_by_full_path(project_path) end private diff --git a/lib/gitlab/email/handler/create_note_handler.rb b/lib/gitlab/email/handler/create_note_handler.rb index 06dae31cc27..d87ba427f4b 100644 --- a/lib/gitlab/email/handler/create_note_handler.rb +++ b/lib/gitlab/email/handler/create_note_handler.rb @@ -1,10 +1,13 @@ require 'gitlab/email/handler/base_handler' +require 'gitlab/email/handler/reply_processing' module Gitlab module Email module Handler class CreateNoteHandler < BaseHandler + include ReplyProcessing + def can_handle? mail_key =~ /\A\w+\z/ end @@ -24,6 +27,8 @@ module Gitlab record_name: 'comment') end + private + def author sent_notification.recipient end @@ -36,8 +41,6 @@ module Gitlab @sent_notification ||= SentNotification.for(mail_key) end - private - def create_note Notes::CreateService.new( project, @@ -46,7 +49,9 @@ module Gitlab noteable_type: sent_notification.noteable_type, noteable_id: sent_notification.noteable_id, commit_id: sent_notification.commit_id, - line_code: sent_notification.line_code + line_code: sent_notification.line_code, + position: sent_notification.position, + type: sent_notification.note_type ).execute end end diff --git a/lib/gitlab/email/handler/reply_processing.rb b/lib/gitlab/email/handler/reply_processing.rb new file mode 100644 index 00000000000..32c5caf93e8 --- /dev/null +++ b/lib/gitlab/email/handler/reply_processing.rb @@ -0,0 +1,54 @@ +module Gitlab + module Email + module Handler + module ReplyProcessing + private + + def author + raise NotImplementedError + end + + def project + raise NotImplementedError + end + + def message + @message ||= process_message + end + + def process_message + message = ReplyParser.new(mail).execute.strip + add_attachments(message) + end + + def add_attachments(reply) + attachments = Email::AttachmentUploader.new(mail).execute(project) + + reply + attachments.map do |link| + "\n\n#{link[:markdown]}" + end.join + end + + def validate_permission!(permission) + raise UserNotFoundError unless author + raise UserBlockedError if author.blocked? + raise ProjectNotFound unless author.can?(:read_project, project) + raise UserNotAuthorizedError unless author.can?(permission, project) + end + + def verify_record!(record:, invalid_exception:, record_name:) + return if record.persisted? + return if record.errors.key?(:commands_only) + + error_title = "The #{record_name} could not be created for the following reasons:" + + msg = error_title + record.errors.full_messages.map do |error| + "\n\n- #{error}" + end.join + + raise invalid_exception, msg + end + end + end + end +end diff --git a/lib/gitlab/email/handler/unsubscribe_handler.rb b/lib/gitlab/email/handler/unsubscribe_handler.rb new file mode 100644 index 00000000000..97d7a8d65ff --- /dev/null +++ b/lib/gitlab/email/handler/unsubscribe_handler.rb @@ -0,0 +1,32 @@ +require 'gitlab/email/handler/base_handler' + +module Gitlab + module Email + module Handler + class UnsubscribeHandler < BaseHandler + def can_handle? + mail_key =~ /\A\w+#{Regexp.escape(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX)}\z/ + end + + def execute + raise SentNotificationNotFoundError unless sent_notification + return unless sent_notification.unsubscribable? + + noteable = sent_notification.noteable + raise NoteableNotFoundError unless noteable + noteable.unsubscribe(sent_notification.recipient) + end + + private + + def sent_notification + @sent_notification ||= SentNotification.for(reply_key) + end + + def reply_key + mail_key.sub(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX, '') + end + end + end + end +end diff --git a/lib/gitlab/email/html_parser.rb b/lib/gitlab/email/html_parser.rb new file mode 100644 index 00000000000..a4ca62bfc41 --- /dev/null +++ b/lib/gitlab/email/html_parser.rb @@ -0,0 +1,34 @@ +module Gitlab + module Email + class HTMLParser + def self.parse_reply(raw_body) + new(raw_body).filtered_text + end + + attr_reader :raw_body + def initialize(raw_body) + @raw_body = raw_body + end + + def document + @document ||= Nokogiri::HTML.parse(raw_body) + end + + def filter_replies! + document.xpath('//blockquote').each(&:remove) + document.xpath('//table').each(&:remove) + end + + def filtered_html + @filtered_html ||= begin + filter_replies! + document.inner_html + end + end + + def filtered_text + @filtered_text ||= Html2Text.convert(filtered_html) + end + end + end +end diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb index 0e3b65fceb4..6c69cd9e6a9 100644 --- a/lib/gitlab/email/message/repository_push.rb +++ b/lib/gitlab/email/message/repository_push.rb @@ -46,7 +46,7 @@ module Gitlab end def diffs_count - diffs.size if diffs + diffs&.size end def compare @@ -58,7 +58,7 @@ module Gitlab end def compare_timeout - diffs.overflow? if diffs + diffs&.overflow? end def reverse_compare? diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index a40c44eb1bc..b64db5d01ae 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -35,6 +35,8 @@ module Gitlab handler.execute end + private + def build_mail Mail::Message.new(@raw) rescue Encoding::UndefinedConversionError, @@ -54,7 +56,24 @@ module Gitlab end def key_from_additional_headers(mail) - Array(mail.references).find do |mail_id| + references = ensure_references_array(mail.references) + + find_key_from_references(references) + end + + def ensure_references_array(references) + case references + when Array + references + when String + # Handle emails from clients which append with commas, + # example clients are Microsoft exchange and iOS app + Gitlab::IncomingEmail.scan_fallback_references(references) + end + end + + def find_key_from_references(references) + references.find do |mail_id| key = Gitlab::IncomingEmail.key_from_fallback_message_id(mail_id) break key if key end diff --git a/lib/gitlab/email/reply_parser.rb b/lib/gitlab/email/reply_parser.rb index 3411eb1d9ce..558df87f36d 100644 --- a/lib/gitlab/email/reply_parser.rb +++ b/lib/gitlab/email/reply_parser.rb @@ -13,9 +13,17 @@ module Gitlab encoding = body.encoding - body = discourse_email_trimmer(body) + body = EmailReplyTrimmer.trim(body) - body = EmailReplyParser.parse_reply(body) + return '' unless body + + # not using /\s+$/ here because that deletes empty lines + body = body.gsub(/[ \t]$/, '') + + # NOTE: We currently don't support empty quotes. + # EmailReplyTrimmer allows this as a special case, + # so we detect it manually here. + return "" if body.lines.all? { |l| l.strip.empty? || l.start_with?('>') } body.force_encoding(encoding).encode("UTF-8") end @@ -23,19 +31,27 @@ module Gitlab private def select_body(message) - text = message.text_part if message.multipart? - text ||= message if message.content_type !~ /text\/html/ + part = + if message.multipart? + message.text_part || message.html_part || message + else + message + end - return "" unless text + decoded = fix_charset(part) - text = fix_charset(text) + return "" unless decoded # Certain trigger phrases that means we didn't parse correctly - if text =~ /(Content\-Type\:|multipart\/alternative|text\/plain)/ + if decoded =~ /(Content\-Type\:|multipart\/alternative|text\/plain)/ return "" end - text + if (part.content_type || '').include? 'text/html' + HTMLParser.parse_reply(decoded) + else + decoded + end end # Force encoding to UTF-8 on a Mail::Message or Mail::Part @@ -50,30 +66,6 @@ module Gitlab rescue nil end - - REPLYING_HEADER_LABELS = %w(From Sent To Subject Reply To Cc Bcc Date) - REPLYING_HEADER_REGEX = Regexp.union(REPLYING_HEADER_LABELS.map { |label| "#{label}:" }) - - def discourse_email_trimmer(body) - lines = body.scrub.lines.to_a - range_end = 0 - - lines.each_with_index do |l, idx| - # This one might be controversial but so many reply lines have years, times and end with a colon. - # Let's try it and see how well it works. - break if (l =~ /\d{4}/ && l =~ /\d:\d\d/ && l =~ /\:$/) || - (l =~ /On \w+ \d+,? \d+,?.*wrote:/) - - # Headers on subsequent lines - break if (0..2).all? { |off| lines[idx + off] =~ REPLYING_HEADER_REGEX } - # Headers on the same line - break if REPLYING_HEADER_LABELS.count { |label| l.include?(label) } >= 3 - - range_end = idx - end - - lines[0..range_end].join.strip - end end end end diff --git a/lib/gitlab/emoji.rb b/lib/gitlab/emoji.rb index b63213ae208..bbbca8acc40 100644 --- a/lib/gitlab/emoji.rb +++ b/lib/gitlab/emoji.rb @@ -10,12 +10,20 @@ module Gitlab Gemojione.index.instance_variable_get(:@emoji_by_moji) end + def emojis_unicodes + emojis_by_moji.keys + end + def emojis_names - emojis.keys.sort + emojis.keys end def emoji_filename(name) emojis[name]["unicode"] end + + def emoji_unicode_filename(moji) + emojis_by_moji[moji]["unicode"] + end end end diff --git a/lib/gitlab/production_logger.rb b/lib/gitlab/environment_logger.rb index 89ce7144b1b..407cc572656 100644 --- a/lib/gitlab/production_logger.rb +++ b/lib/gitlab/environment_logger.rb @@ -1,7 +1,7 @@ module Gitlab - class ProductionLogger < Gitlab::Logger + class EnvironmentLogger < Gitlab::Logger def self.file_name_noext - 'production' + Rails.env end end end diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index ffe49364379..62ddd45785d 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -1,59 +1,52 @@ +require 'securerandom' + module Gitlab # This class implements an 'exclusive lease'. We call it a 'lease' # because it has a set expiry time. We call it 'exclusive' because only # one caller may obtain a lease for a given key at a time. The # implementation is intended to work across GitLab processes and across - # servers. It is a 'cheap' alternative to using SQL queries and updates: + # servers. It is a cheap alternative to using SQL queries and updates: # you do not need to change the SQL schema to start using # ExclusiveLease. # - # It is important to choose the timeout wisely. If the timeout is very - # high (1 hour) then the throughput of your operation gets very low (at - # most once an hour). If the timeout is lower than how long your - # operation may take then you cannot count on exclusivity. For example, - # if the timeout is 10 seconds and you do an operation which may take 20 - # seconds then two overlapping operations may hold a lease for the same - # key at the same time. - # - # This class has no 'cancel' method. I originally decided against adding - # it because it would add complexity and a false sense of security. The - # complexity: instead of setting '1' we would have to set a UUID, and to - # delete it we would have to execute Lua on the Redis server to only - # delete the key if the value was our own UUID. Otherwise there is a - # chance that when you intend to cancel your lease you actually delete - # someone else's. The false sense of security: you cannot design your - # system to rely too much on the lease being cancelled after use because - # the calling (Ruby) process may crash or be killed. You _cannot_ count - # on begin/ensure blocks to cancel a lease, because the 'ensure' does - # not always run. Think of 'kill -9' from the Unicorn master for - # instance. - # - # If you find that leases are getting in your way, ask yourself: would - # it be enough to lower the lease timeout? Another thing that might be - # appropriate is to only use a lease for bulk/automated operations, and - # to ignore the lease when you get a single 'manual' user request (a - # button click). - # class ExclusiveLease + LUA_CANCEL_SCRIPT = <<-EOS.freeze + local key, uuid = KEYS[1], ARGV[1] + if redis.call("get", key) == uuid then + redis.call("del", key) + end + EOS + + def self.cancel(key, uuid) + Gitlab::Redis.with do |redis| + redis.eval(LUA_CANCEL_SCRIPT, keys: [redis_key(key)], argv: [uuid]) + end + end + + def self.redis_key(key) + "gitlab:exclusive_lease:#{key}" + end + def initialize(key, timeout:) - @key, @timeout = key, timeout + @redis_key = self.class.redis_key(key) + @timeout = timeout + @uuid = SecureRandom.uuid end - # Try to obtain the lease. Return true on success, + # Try to obtain the lease. Return lease UUID on success, # false if the lease is already taken. def try_obtain # Performing a single SET is atomic Gitlab::Redis.with do |redis| - !!redis.set(redis_key, '1', nx: true, ex: @timeout) + redis.set(@redis_key, @uuid, nx: true, ex: @timeout) && @uuid end end - # No #cancel method. See comments above! - - private - - def redis_key - "gitlab:exclusive_lease:#{@key}" + # Returns true if the key for this lease is set. + def exists? + Gitlab::Redis.with do |redis| + redis.exists(@redis_key) + end end end end diff --git a/lib/gitlab/file_detector.rb b/lib/gitlab/file_detector.rb new file mode 100644 index 00000000000..c9ca4cadd1c --- /dev/null +++ b/lib/gitlab/file_detector.rb @@ -0,0 +1,63 @@ +require 'set' + +module Gitlab + # Module that can be used to detect if a path points to a special file such as + # a README or a CONTRIBUTING file. + module FileDetector + PATTERNS = { + readme: /\Areadme/i, + changelog: /\A(changelog|history|changes|news)/i, + license: /\A(licen[sc]e|copying)(\..+|\z)/i, + contributing: /\Acontributing/i, + version: 'version', + gitignore: '.gitignore', + koding: '.koding.yml', + gitlab_ci: '.gitlab-ci.yml', + avatar: /\Alogo\.(png|jpg|gif)\z/ + }.freeze + + # Returns an Array of file types based on the given paths. + # + # This method can be used to check if a list of file paths (e.g. of changed + # files) involve any special files such as a README or a LICENSE file. + # + # Example: + # + # types_in_paths(%w{README.md foo/bar.txt}) # => [:readme] + def self.types_in_paths(paths) + types = Set.new + + paths.each do |path| + type = type_of(path) + + types << type if type + end + + types.to_a + end + + # Returns the type of a file path, or nil if none could be detected. + # + # Returned types are Symbols such as `:readme`, `:version`, etc. + # + # Example: + # + # type_of('README.md') # => :readme + # type_of('VERSION') # => :version + def self.type_of(path) + name = File.basename(path) + + PATTERNS.each do |type, search| + did_match = if search.is_a?(Regexp) + name =~ search + else + name.casecmp(search) == 0 + end + + return type if did_match + end + + nil + end + end +end diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb index 501d5a95547..222bcdcbf9c 100644 --- a/lib/gitlab/fogbugz_import/importer.rb +++ b/lib/gitlab/fogbugz_import/importer.rb @@ -74,8 +74,8 @@ module Gitlab end def create_label(name) - color = nice_label_color(name) - Label.create!(project_id: project.id, title: name, color: color) + params = { title: name, color: nice_label_color(name) } + ::Labels::FindOrCreateService.new(nil, project, params).execute(skip_authorization: true) end def user_info(person_id) @@ -122,25 +122,21 @@ module Gitlab author_id = user_info(bug['ixPersonOpenedBy'])[:gitlab_id] || project.creator_id issue = Issue.create!( - project_id: project.id, - title: bug['sTitle'], - description: body, - author_id: author_id, - assignee_id: assignee_id, - state: bug['fOpen'] == 'true' ? 'opened' : 'closed' + iid: bug['ixBug'], + project_id: project.id, + title: bug['sTitle'], + description: body, + author_id: author_id, + assignee_id: assignee_id, + state: bug['fOpen'] == 'true' ? 'opened' : 'closed', + created_at: date, + updated_at: DateTime.parse(bug['dtLastUpdated']) ) - issue.add_labels_by_names(labels) - if issue.iid != bug['ixBug'] - issue.update_attribute(:iid, bug['ixBug']) - end + issue_labels = ::LabelsFinder.new(nil, project_id: project.id, title: labels).execute(skip_authorization: true) + issue.update_attribute(:label_ids, issue_labels.pluck(:id)) import_issue_comments(issue, comments) - - issue.update_attribute(:created_at, date) - - last_update = DateTime.parse(bug['dtLastUpdated']) - issue.update_attribute(:updated_at, last_update) end end diff --git a/lib/gitlab/gfm/reference_rewriter.rb b/lib/gitlab/gfm/reference_rewriter.rb index 78d7a4f27cf..b984492d369 100644 --- a/lib/gitlab/gfm/reference_rewriter.rb +++ b/lib/gitlab/gfm/reference_rewriter.rb @@ -58,7 +58,7 @@ module Gitlab referable = find_referable(reference) return reference unless referable - cross_reference = referable.to_reference(target_project) + cross_reference = build_cross_reference(referable, target_project) return reference if reference == cross_reference new_text = before + cross_reference + after @@ -72,6 +72,14 @@ module Gitlab extractor.all.first end + def build_cross_reference(referable, target_project) + if referable.respond_to?(:project) + referable.to_reference(target_project) + else + referable.to_reference(@source_project, target_project: target_project) + end + end + def substitution_valid?(substituted) @original_html == markdown(substituted) end diff --git a/lib/gitlab/gfm/uploads_rewriter.rb b/lib/gitlab/gfm/uploads_rewriter.rb index abc8c8c55e6..8fab5489616 100644 --- a/lib/gitlab/gfm/uploads_rewriter.rb +++ b/lib/gitlab/gfm/uploads_rewriter.rb @@ -1,3 +1,5 @@ +require 'fileutils' + module Gitlab module Gfm ## @@ -22,7 +24,9 @@ module Gitlab return markdown unless file.try(:exists?) new_uploader = FileUploader.new(target_project) - new_uploader.store!(file) + with_link_in_tmp_dir(file.file) do |open_tmp_file| + new_uploader.store!(open_tmp_file) + end new_uploader.to_markdown end end @@ -46,6 +50,19 @@ module Gitlab uploader.retrieve_from_store!(file) uploader.file end + + # Because the uploaders use 'move_to_store' we must have a temporary + # file that is allowed to be (re)moved. + def with_link_in_tmp_dir(file) + dir = Dir.mktmpdir('UploadsRewriter', File.dirname(file)) + # The filename matters to Carrierwave so we make sure to preserve it + tmp_file = File.join(dir, File.basename(file)) + File.link(file, tmp_file) + # Open the file to placate Carrierwave + File.open(tmp_file) { |open_file| yield open_file } + ensure + FileUtils.rm_rf(dir) + end end end end diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 7584efe4fa8..d3df3f1bca1 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -6,7 +6,7 @@ module Gitlab class << self def ref_name(ref) - ref.gsub(/\Arefs\/(tags|heads)\//, '') + ref.sub(/\Arefs\/(tags|heads)\//, '') end def branch_name(ref) @@ -18,6 +18,16 @@ module Gitlab end end + def committer_hash(email:, name:) + return if email.nil? || name.nil? + + { + email: email, + name: name, + time: Time.now + } + end + def tag_name(ref) ref = ref.to_s if self.tag_ref?(ref) diff --git a/lib/gitlab/git/attributes.rb b/lib/gitlab/git/attributes.rb new file mode 100644 index 00000000000..42140ecc993 --- /dev/null +++ b/lib/gitlab/git/attributes.rb @@ -0,0 +1,131 @@ +module Gitlab + module Git + # Class for parsing Git attribute files and extracting the attributes for + # file patterns. + # + # Unlike Rugged this parser only needs a single IO call (a call to `open`), + # vastly reducing the time spent in extracting attributes. + # + # This class _only_ supports parsing the attributes file located at + # `$GIT_DIR/info/attributes` as GitLab doesn't use any other files + # (`.gitattributes` is copied to this particular path). + # + # Basic usage: + # + # attributes = Gitlab::Git::Attributes.new(some_repo.path) + # + # attributes.attributes('README.md') # => { "eol" => "lf } + class Attributes + # path - The path to the Git repository. + def initialize(path) + @path = File.expand_path(path) + @patterns = nil + end + + # Returns all the Git attributes for the given path. + # + # path - A path to a file for which to get the attributes. + # + # Returns a Hash. + def attributes(path) + full_path = File.join(@path, path) + + patterns.each do |pattern, attrs| + return attrs if File.fnmatch?(pattern, full_path) + end + + {} + end + + # Returns a Hash containing the file patterns and their attributes. + def patterns + @patterns ||= parse_file + end + + # Parses an attribute string. + # + # These strings can be in the following formats: + # + # text # => { "text" => true } + # -text # => { "text" => false } + # key=value # => { "key" => "value" } + # + # string - The string to parse. + # + # Returns a Hash containing the attributes and their values. + def parse_attributes(string) + values = {} + dash = '-' + equal = '=' + binary = 'binary' + + string.split(/\s+/).each do |chunk| + # Data such as "foo = bar" should be treated as "foo" and "bar" being + # separate boolean attributes. + next if chunk == equal + + key = chunk + + # Input: "-foo" + if chunk.start_with?(dash) + key = chunk.byteslice(1, chunk.length - 1) + value = false + + # Input: "foo=bar" + elsif chunk.include?(equal) + key, value = chunk.split(equal, 2) + + # Input: "foo" + else + value = true + end + + values[key] = value + + # When the "binary" option is set the "diff" option should be set to + # the inverse. If "diff" is later set it should overwrite the + # automatically set value. + values['diff'] = false if key == binary && value + end + + values + end + + # Iterates over every line in the attributes file. + def each_line + full_path = File.join(@path, 'info/attributes') + + return unless File.exist?(full_path) + + File.open(full_path, 'r') do |handle| + handle.each_line do |line| + break unless line.valid_encoding? + + yield line.strip + end + end + end + + private + + # Parses the Git attributes file. + def parse_file + pairs = [] + comment = '#' + + each_line do |line| + next if line.start_with?(comment) || line.empty? + + pattern, attrs = line.split(/\s+/, 2) + + parsed = attrs ? parse_attributes(attrs) : {} + + pairs << [File.join(@path, pattern), parsed] + end + + # Newer entries take precedence over older entries. + pairs.reverse.to_h + end + end + end +end diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb new file mode 100644 index 00000000000..58193391926 --- /dev/null +++ b/lib/gitlab/git/blame.rb @@ -0,0 +1,75 @@ +module Gitlab + module Git + class Blame + include Gitlab::Git::EncodingHelper + + attr_reader :lines, :blames + + def initialize(repository, sha, path) + @repo = repository + @sha = sha + @path = path + @lines = [] + @blames = load_blame + end + + def each + @blames.each do |blame| + yield( + Gitlab::Git::Commit.new(blame.commit), + blame.line + ) + end + end + + private + + def load_blame + cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{@repo.path} blame -p #{@sha} -- #{@path}) + # Read in binary mode to ensure ASCII-8BIT + raw_output = IO.popen(cmd, 'rb') {|io| io.read } + output = encode_utf8(raw_output) + process_raw_blame output + end + + def process_raw_blame(output) + lines, final = [], [] + info, commits = {}, {} + + # process the output + output.split("\n").each do |line| + if line[0, 1] == "\t" + lines << line[1, line.size] + elsif m = /^(\w{40}) (\d+) (\d+)/.match(line) + commit_id, old_lineno, lineno = m[1], m[2].to_i, m[3].to_i + commits[commit_id] = nil unless commits.key?(commit_id) + info[lineno] = [commit_id, old_lineno] + end + end + + # load all commits in single call + commits.keys.each do |key| + commits[key] = @repo.lookup(key) + end + + # get it together + info.sort.each do |lineno, (commit_id, old_lineno)| + commit = commits[commit_id] + final << BlameLine.new(lineno, old_lineno, commit, lines[lineno - 1]) + end + + @lines = final + end + end + + class BlameLine + attr_accessor :lineno, :oldlineno, :commit, :line + def initialize(lineno, oldlineno, commit, line) + @lineno = lineno + @oldlineno = oldlineno + @commit = commit + @line = line + end + end + end +end diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb new file mode 100644 index 00000000000..e56eb0d3beb --- /dev/null +++ b/lib/gitlab/git/blob.rb @@ -0,0 +1,173 @@ +module Gitlab + module Git + class Blob + include Linguist::BlobHelper + include Gitlab::Git::EncodingHelper + + # This number is the maximum amount of data that we want to display to + # the user. We load as much as we can for encoding detection + # (Linguist) and LFS pointer parsing. All other cases where we need full + # blob data should use load_all_data!. + MAX_DATA_DISPLAY_SIZE = 10485760 + + attr_accessor :name, :path, :size, :data, :mode, :id, :commit_id, :loaded_size, :binary + + class << self + def find(repository, sha, path) + commit = repository.lookup(sha) + root_tree = commit.tree + + blob_entry = find_entry_by_path(repository, root_tree.oid, path) + + return nil unless blob_entry + + if blob_entry[:type] == :commit + submodule_blob(blob_entry, path, sha) + else + blob = repository.lookup(blob_entry[:oid]) + + if blob + new( + id: blob.oid, + name: blob_entry[:name], + size: blob.size, + data: blob.content(MAX_DATA_DISPLAY_SIZE), + mode: blob_entry[:filemode].to_s(8), + path: path, + commit_id: sha, + binary: blob.binary? + ) + end + end + end + + def raw(repository, sha) + blob = repository.lookup(sha) + + new( + id: blob.oid, + size: blob.size, + data: blob.content(MAX_DATA_DISPLAY_SIZE), + binary: blob.binary? + ) + end + + # Recursive search of blob id by path + # + # Ex. + # blog/ # oid: 1a + # app/ # oid: 2a + # models/ # oid: 3a + # file.rb # oid: 4a + # + # + # Blob.find_entry_by_path(repo, '1a', 'app/file.rb') # => '4a' + # + def find_entry_by_path(repository, root_id, path) + root_tree = repository.lookup(root_id) + # Strip leading slashes + path[/^\/*/] = '' + path_arr = path.split('/') + + entry = root_tree.find do |entry| + entry[:name] == path_arr[0] + end + + return nil unless entry + + if path_arr.size > 1 + return nil unless entry[:type] == :tree + path_arr.shift + find_entry_by_path(repository, entry[:oid], path_arr.join('/')) + else + [:blob, :commit].include?(entry[:type]) ? entry : nil + end + end + + def submodule_blob(blob_entry, path, sha) + new( + id: blob_entry[:oid], + name: blob_entry[:name], + data: '', + path: path, + commit_id: sha, + ) + end + end + + def initialize(options) + %w(id name path size data mode commit_id binary).each do |key| + self.send("#{key}=", options[key.to_sym]) + end + + @loaded_all_data = false + # Retain the actual size before it is encoded + @loaded_size = @data.bytesize if @data + end + + def binary? + @binary.nil? ? super : @binary == true + end + + def empty? + !data || data == '' + end + + def data + encode! @data + end + + # Load all blob data (not just the first MAX_DATA_DISPLAY_SIZE bytes) into + # memory as a Ruby string. + def load_all_data!(repository) + return if @data == '' # don't mess with submodule blobs + return @data if @loaded_all_data + + @loaded_all_data = true + @data = repository.lookup(id).content + @loaded_size = @data.bytesize + end + + def name + encode! @name + end + + # Valid LFS object pointer is a text file consisting of + # version + # oid + # size + # see https://github.com/github/git-lfs/blob/v1.1.0/docs/spec.md#the-pointer + def lfs_pointer? + has_lfs_version_key? && lfs_oid.present? && lfs_size.present? + end + + def lfs_oid + if has_lfs_version_key? + oid = data.match(/(?<=sha256:)([0-9a-f]{64})/) + return oid[1] if oid + end + + nil + end + + def lfs_size + if has_lfs_version_key? + size = data.match(/(?<=size )([0-9]+)/) + return size[1] if size + end + + nil + end + + def truncated? + size && (size > loaded_size) + end + + private + + def has_lfs_version_key? + !empty? && text? && data.start_with?("version https://git-lfs.github.com/spec") + end + end + end +end diff --git a/lib/gitlab/git/blob_snippet.rb b/lib/gitlab/git/blob_snippet.rb new file mode 100644 index 00000000000..d7975f88aaa --- /dev/null +++ b/lib/gitlab/git/blob_snippet.rb @@ -0,0 +1,32 @@ +module Gitlab + module Git + class BlobSnippet + include Linguist::BlobHelper + + attr_accessor :ref + attr_accessor :lines + attr_accessor :filename + attr_accessor :startline + + def initialize(ref, lines, startline, filename) + @ref, @lines, @startline, @filename = ref, lines, startline, filename + end + + def data + lines&.join("\n") + end + + def name + filename + end + + def size + data.length + end + + def mode + nil + end + end + end +end diff --git a/lib/gitlab/git/branch.rb b/lib/gitlab/git/branch.rb new file mode 100644 index 00000000000..586380da94a --- /dev/null +++ b/lib/gitlab/git/branch.rb @@ -0,0 +1,6 @@ +module Gitlab + module Git + class Branch < Ref + end + end +end diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb new file mode 100644 index 00000000000..3a73697dc5d --- /dev/null +++ b/lib/gitlab/git/commit.rb @@ -0,0 +1,308 @@ +# Gitlab::Git::Commit is a wrapper around native Rugged::Commit object +module Gitlab + module Git + class Commit + include Gitlab::Git::EncodingHelper + + attr_accessor :raw_commit, :head, :refs + + SERIALIZE_KEYS = [ + :id, :message, :parent_ids, + :authored_date, :author_name, :author_email, + :committed_date, :committer_name, :committer_email + ].freeze + + attr_accessor *SERIALIZE_KEYS # rubocop:disable Lint/AmbiguousOperator + + delegate :tree, to: :raw_commit + + def ==(other) + return false unless other.is_a?(Gitlab::Git::Commit) + + methods = [:message, :parent_ids, :authored_date, :author_name, + :author_email, :committed_date, :committer_name, + :committer_email] + + methods.all? do |method| + send(method) == other.send(method) + end + end + + class << self + # Get commits collection + # + # Ex. + # Commit.where( + # repo: repo, + # ref: 'master', + # path: 'app/models', + # limit: 10, + # offset: 5, + # ) + # + def where(options) + repo = options.delete(:repo) + raise 'Gitlab::Git::Repository is required' unless repo.respond_to?(:log) + + repo.log(options).map { |c| decorate(c) } + end + + # Get single commit + # + # Ex. + # Commit.find(repo, '29eda46b') + # + # Commit.find(repo, 'master') + # + def find(repo, commit_id = "HEAD") + return decorate(commit_id) if commit_id.is_a?(Rugged::Commit) + + obj = if commit_id.is_a?(String) + repo.rev_parse_target(commit_id) + else + Gitlab::Git::Ref.dereference_object(commit_id) + end + + return nil unless obj.is_a?(Rugged::Commit) + + decorate(obj) + rescue Rugged::ReferenceError, Rugged::InvalidError, Rugged::ObjectError, Gitlab::Git::Repository::NoRepository + nil + end + + # Get last commit for HEAD + # + # Ex. + # Commit.last(repo) + # + def last(repo) + find(repo) + end + + # Get last commit for specified path and ref + # + # Ex. + # Commit.last_for_path(repo, '29eda46b', 'app/models') + # + # Commit.last_for_path(repo, 'master', 'Gemfile') + # + def last_for_path(repo, ref, path = nil) + where( + repo: repo, + ref: ref, + path: path, + limit: 1 + ).first + end + + # Get commits between two revspecs + # See also #repository.commits_between + # + # Ex. + # Commit.between(repo, '29eda46b', 'master') + # + def between(repo, base, head) + repo.commits_between(base, head).map do |commit| + decorate(commit) + end + rescue Rugged::ReferenceError + [] + end + + # Delegate Repository#find_commits + def find_all(repo, options = {}) + repo.find_commits(options) + end + + def decorate(commit, ref = nil) + Gitlab::Git::Commit.new(commit, ref) + end + + # Returns a diff object for the changes introduced by +rugged_commit+. + # If +rugged_commit+ doesn't have a parent, then the diff is between + # this commit and an empty repo. See Repository#diff for the keys + # allowed in the +options+ hash. + def diff_from_parent(rugged_commit, options = {}) + options ||= {} + break_rewrites = options[:break_rewrites] + actual_options = Gitlab::Git::Diff.filter_diff_options(options) + + diff = if rugged_commit.parents.empty? + rugged_commit.diff(actual_options.merge(reverse: true)) + else + rugged_commit.parents[0].diff(rugged_commit, actual_options) + end + + diff.find_similar!(break_rewrites: break_rewrites) + diff + end + end + + def initialize(raw_commit, head = nil) + raise "Nil as raw commit passed" unless raw_commit + + if raw_commit.is_a?(Hash) + init_from_hash(raw_commit) + elsif raw_commit.is_a?(Rugged::Commit) + init_from_rugged(raw_commit) + else + raise "Invalid raw commit type: #{raw_commit.class}" + end + + @head = head + end + + def sha + id + end + + def short_id(length = 10) + id.to_s[0..length] + end + + def safe_message + @safe_message ||= message + end + + def created_at + committed_date + end + + # Was this commit committed by a different person than the original author? + def different_committer? + author_name != committer_name || author_email != committer_email + end + + def parent_id + parent_ids.first + end + + # Shows the diff between the commit's parent and the commit. + # + # Cuts out the header and stats from #to_patch and returns only the diff. + def to_diff(options = {}) + diff_from_parent(options).patch + end + + # Returns a diff object for the changes from this commit's first parent. + # If there is no parent, then the diff is between this commit and an + # empty repo. See Repository#diff for keys allowed in the +options+ + # hash. + def diff_from_parent(options = {}) + Commit.diff_from_parent(raw_commit, options) + end + + def has_zero_stats? + stats.total.zero? + rescue + true + end + + def no_commit_message + "--no commit message" + end + + def to_hash + serialize_keys.map.with_object({}) do |key, hash| + hash[key] = send(key) + end + end + + def date + committed_date + end + + def diffs(options = {}) + Gitlab::Git::DiffCollection.new(diff_from_parent(options), options) + end + + def parents + raw_commit.parents.map { |c| Gitlab::Git::Commit.new(c) } + end + + def stats + Gitlab::Git::CommitStats.new(self) + end + + def to_patch(options = {}) + begin + raw_commit.to_mbox(options) + rescue Rugged::InvalidError => ex + if ex.message =~ /Commit \w+ is a merge commit/ + 'Patch format is not currently supported for merge commits.' + end + end + end + + # Get a collection of Rugged::Reference objects for this commit. + # + # Ex. + # commit.ref(repo) + # + def refs(repo) + repo.refs_hash[id] + end + + # Get ref names collection + # + # Ex. + # commit.ref_names(repo) + # + def ref_names(repo) + refs(repo).map do |ref| + ref.name.sub(%r{^refs/(heads|remotes|tags)/}, "") + end + end + + def message + encode! @message + end + + def author_name + encode! @author_name + end + + def author_email + encode! @author_email + end + + def committer_name + encode! @committer_name + end + + def committer_email + encode! @committer_email + end + + private + + def init_from_hash(hash) + raw_commit = hash.symbolize_keys + + serialize_keys.each do |key| + send("#{key}=", raw_commit[key]) + end + end + + def init_from_rugged(commit) + author = commit.author + committer = commit.committer + + @raw_commit = commit + @id = commit.oid + @message = commit.message + @authored_date = author[:time] + @committed_date = committer[:time] + @author_name = author[:name] + @author_email = author[:email] + @committer_name = committer[:name] + @committer_email = committer[:email] + @parent_ids = commit.parents.map(&:oid) + end + + def serialize_keys + SERIALIZE_KEYS + end + end + end +end diff --git a/lib/gitlab/git/commit_stats.rb b/lib/gitlab/git/commit_stats.rb new file mode 100644 index 00000000000..e9118bbed0e --- /dev/null +++ b/lib/gitlab/git/commit_stats.rb @@ -0,0 +1,26 @@ +# Gitlab::Git::CommitStats counts the additions, deletions, and total changes +# in a commit. +module Gitlab + module Git + class CommitStats + attr_reader :id, :additions, :deletions, :total + + # Instantiate a CommitStats object + def initialize(commit) + @id = commit.id + @additions = 0 + @deletions = 0 + @total = 0 + + diff = commit.diff_from_parent + + diff.each_patch do |p| + # TODO: Use the new Rugged convenience methods when they're released + @additions += p.stat[0] + @deletions += p.stat[1] + @total += p.changes + end + end + end + end +end diff --git a/lib/gitlab/git/compare.rb b/lib/gitlab/git/compare.rb new file mode 100644 index 00000000000..696a2acd5e3 --- /dev/null +++ b/lib/gitlab/git/compare.rb @@ -0,0 +1,43 @@ +module Gitlab + module Git + class Compare + attr_reader :head, :base, :straight + + def initialize(repository, base, head, straight = false) + @repository = repository + @straight = straight + + unless base && head + @commits = [] + return + end + + @base = Gitlab::Git::Commit.find(repository, base.try(:strip)) + @head = Gitlab::Git::Commit.find(repository, head.try(:strip)) + + @commits = [] unless @base && @head + @commits = [] if same + end + + def same + @base && @head && @base.id == @head.id + end + + def commits + return @commits if defined?(@commits) + + @commits = Gitlab::Git::Commit.between(@repository, @base.id, @head.id) + end + + def diffs(options = {}) + unless @head && @base + return Gitlab::Git::DiffCollection.new([]) + end + + paths = options.delete(:paths) || [] + options[:straight] = @straight + Gitlab::Git::Diff.between(@repository, @head.id, @base.id, options, *paths) + end + end + end +end diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb new file mode 100644 index 00000000000..d6b3b5705a9 --- /dev/null +++ b/lib/gitlab/git/diff.rb @@ -0,0 +1,322 @@ +# Gitlab::Git::Diff is a wrapper around native Rugged::Diff object +module Gitlab + module Git + class Diff + class TimeoutError < StandardError; end + include Gitlab::Git::EncodingHelper + + # Diff properties + attr_accessor :old_path, :new_path, :a_mode, :b_mode, :diff + + # Stats properties + attr_accessor :new_file, :renamed_file, :deleted_file + + attr_accessor :too_large + + # The maximum size of a diff to display. + DIFF_SIZE_LIMIT = 102400 # 100 KB + + # The maximum size before a diff is collapsed. + DIFF_COLLAPSE_LIMIT = 10240 # 10 KB + + class << self + def between(repo, head, base, options = {}, *paths) + straight = options.delete(:straight) || false + + common_commit = if straight + base + else + # Only show what is new in the source branch + # compared to the target branch, not the other way + # around. The linex below with merge_base is + # equivalent to diff with three dots (git diff + # branch1...branch2) From the git documentation: + # "git diff A...B" is equivalent to "git diff + # $(git-merge-base A B) B" + repo.merge_base_commit(head, base) + end + + options ||= {} + actual_options = filter_diff_options(options) + repo.diff(common_commit, head, actual_options, *paths) + end + + # Return a copy of the +options+ hash containing only keys that can be + # passed to Rugged. Allowed options are: + # + # :max_size :: + # An integer specifying the maximum byte size of a file before a it + # will be treated as binary. The default value is 512MB. + # + # :context_lines :: + # The number of unchanged lines that define the boundary of a hunk + # (and to display before and after the actual changes). The default is + # 3. + # + # :interhunk_lines :: + # The maximum number of unchanged lines between hunk boundaries before + # the hunks will be merged into a one. The default is 0. + # + # :old_prefix :: + # The virtual "directory" to prefix to old filenames in hunk headers. + # The default is "a". + # + # :new_prefix :: + # The virtual "directory" to prefix to new filenames in hunk headers. + # The default is "b". + # + # :reverse :: + # If true, the sides of the diff will be reversed. + # + # :force_text :: + # If true, all files will be treated as text, disabling binary + # attributes & detection. + # + # :ignore_whitespace :: + # If true, all whitespace will be ignored. + # + # :ignore_whitespace_change :: + # If true, changes in amount of whitespace will be ignored. + # + # :ignore_whitespace_eol :: + # If true, whitespace at end of line will be ignored. + # + # :ignore_submodules :: + # if true, submodules will be excluded from the diff completely. + # + # :patience :: + # If true, the "patience diff" algorithm will be used (currenlty + # unimplemented). + # + # :include_ignored :: + # If true, ignored files will be included in the diff. + # + # :include_untracked :: + # If true, untracked files will be included in the diff. + # + # :include_unmodified :: + # If true, unmodified files will be included in the diff. + # + # :recurse_untracked_dirs :: + # Even if +:include_untracked+ is true, untracked directories will + # only be marked with a single entry in the diff. If this flag is set + # to true, all files under ignored directories will be included in the + # diff, too. + # + # :disable_pathspec_match :: + # If true, the given +*paths+ will be applied as exact matches, + # instead of as fnmatch patterns. + # + # :deltas_are_icase :: + # If true, filename comparisons will be made with case-insensitivity. + # + # :include_untracked_content :: + # if true, untracked content will be contained in the the diff patch + # text. + # + # :skip_binary_check :: + # If true, diff deltas will be generated without spending time on + # binary detection. This is useful to improve performance in cases + # where the actual file content difference is not needed. + # + # :include_typechange :: + # If true, type changes for files will not be interpreted as deletion + # of the "old file" and addition of the "new file", but will generate + # typechange records. + # + # :include_typechange_trees :: + # Even if +:include_typechange+ is true, blob -> tree changes will + # still usually be handled as a deletion of the blob. If this flag is + # set to true, blob -> tree changes will be marked as typechanges. + # + # :ignore_filemode :: + # If true, file mode changes will be ignored. + # + # :recurse_ignored_dirs :: + # Even if +:include_ignored+ is true, ignored directories will only be + # marked with a single entry in the diff. If this flag is set to true, + # all files under ignored directories will be included in the diff, + # too. + def filter_diff_options(options, default_options = {}) + allowed_options = [:max_size, :context_lines, :interhunk_lines, + :old_prefix, :new_prefix, :reverse, :force_text, + :ignore_whitespace, :ignore_whitespace_change, + :ignore_whitespace_eol, :ignore_submodules, + :patience, :include_ignored, :include_untracked, + :include_unmodified, :recurse_untracked_dirs, + :disable_pathspec_match, :deltas_are_icase, + :include_untracked_content, :skip_binary_check, + :include_typechange, :include_typechange_trees, + :ignore_filemode, :recurse_ignored_dirs, :paths, + :max_files, :max_lines, :all_diffs, :no_collapse] + + if default_options + actual_defaults = default_options.dup + actual_defaults.keep_if do |key| + allowed_options.include?(key) + end + else + actual_defaults = {} + end + + if options + filtered_opts = options.dup + filtered_opts.keep_if do |key| + allowed_options.include?(key) + end + filtered_opts = actual_defaults.merge(filtered_opts) + else + filtered_opts = actual_defaults + end + + filtered_opts + end + end + + def initialize(raw_diff, collapse: false) + case raw_diff + when Hash + init_from_hash(raw_diff, collapse: collapse) + when Rugged::Patch, Rugged::Diff::Delta + init_from_rugged(raw_diff, collapse: collapse) + when nil + raise "Nil as raw diff passed" + else + raise "Invalid raw diff type: #{raw_diff.class}" + end + end + + def serialize_keys + @serialize_keys ||= %i(diff new_path old_path a_mode b_mode new_file renamed_file deleted_file too_large) + end + + def to_hash + hash = {} + + keys = serialize_keys + + keys.each do |key| + hash[key] = send(key) + end + + hash + end + + def submodule? + a_mode == '160000' || b_mode == '160000' + end + + def line_count + @line_count ||= Util.count_lines(@diff) + end + + def too_large? + if @too_large.nil? + @too_large = @diff.bytesize >= DIFF_SIZE_LIMIT + else + @too_large + end + end + + def collapsible? + @diff.bytesize >= DIFF_COLLAPSE_LIMIT + end + + def prune_large_diff! + @diff = '' + @line_count = 0 + @too_large = true + end + + def collapsed? + return @collapsed if defined?(@collapsed) + false + end + + def prune_collapsed_diff! + @diff = '' + @line_count = 0 + @collapsed = true + end + + private + + def init_from_rugged(rugged, collapse: false) + if rugged.is_a?(Rugged::Patch) + init_from_rugged_patch(rugged, collapse: collapse) + d = rugged.delta + else + d = rugged + end + + @new_path = encode!(d.new_file[:path]) + @old_path = encode!(d.old_file[:path]) + @a_mode = d.old_file[:mode].to_s(8) + @b_mode = d.new_file[:mode].to_s(8) + @new_file = d.added? + @renamed_file = d.renamed? + @deleted_file = d.deleted? + end + + def init_from_rugged_patch(patch, collapse: false) + # Don't bother initializing diffs that are too large. If a diff is + # binary we're not going to display anything so we skip the size check. + return if !patch.delta.binary? && prune_large_patch(patch, collapse) + + @diff = encode!(strip_diff_headers(patch.to_s)) + end + + def init_from_hash(hash, collapse: false) + raw_diff = hash.symbolize_keys + + serialize_keys.each do |key| + send(:"#{key}=", raw_diff[key.to_sym]) + end + + prune_large_diff! if too_large? + prune_collapsed_diff! if collapse && collapsible? + end + + # If the patch surpasses any of the diff limits it calls the appropiate + # prune method and returns true. Otherwise returns false. + def prune_large_patch(patch, collapse) + size = 0 + + patch.each_hunk do |hunk| + hunk.each_line do |line| + size += line.content.bytesize + + if size >= DIFF_SIZE_LIMIT + prune_large_diff! + return true + end + end + end + + if collapse && size >= DIFF_COLLAPSE_LIMIT + prune_collapsed_diff! + return true + end + + false + end + + # Strip out the information at the beginning of the patch's text to match + # Grit's output + def strip_diff_headers(diff_text) + # Delete everything up to the first line that starts with '---' or + # 'Binary' + diff_text.sub!(/\A.*?^(---|Binary)/m, '\1') + + if diff_text.start_with?('---', 'Binary') + diff_text + else + # If the diff_text did not contain a line starting with '---' or + # 'Binary', return the empty string. No idea why; we are just + # preserving behavior from before the refactor. + '' + end + end + end + end +end diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb new file mode 100644 index 00000000000..65e06f5065d --- /dev/null +++ b/lib/gitlab/git/diff_collection.rb @@ -0,0 +1,129 @@ +module Gitlab + module Git + class DiffCollection + include Enumerable + + DEFAULT_LIMITS = { max_files: 100, max_lines: 5000 }.freeze + + def initialize(iterator, options = {}) + @iterator = iterator + @max_files = options.fetch(:max_files, DEFAULT_LIMITS[:max_files]) + @max_lines = options.fetch(:max_lines, DEFAULT_LIMITS[:max_lines]) + @max_bytes = @max_files * 5120 # Average 5 KB per file + @safe_max_files = [@max_files, DEFAULT_LIMITS[:max_files]].min + @safe_max_lines = [@max_lines, DEFAULT_LIMITS[:max_lines]].min + @safe_max_bytes = @safe_max_files * 5120 # Average 5 KB per file + @all_diffs = !!options.fetch(:all_diffs, false) + @no_collapse = !!options.fetch(:no_collapse, true) + @deltas_only = !!options.fetch(:deltas_only, false) + + @line_count = 0 + @byte_count = 0 + @overflow = false + @array = Array.new + end + + def each(&block) + if @populated + # @iterator.each is slower than just iterating the array in place + @array.each(&block) + elsif @deltas_only + each_delta(&block) + else + each_patch(&block) + end + end + + def empty? + !@iterator.any? + end + + def overflow? + populate! + !!@overflow + end + + def size + @size ||= count # forces a loop using each method + end + + def real_size + populate! + + if @overflow + "#{size}+" + else + size.to_s + end + end + + def decorate! + collection = each_with_index do |element, i| + @array[i] = yield(element) + end + @populated = true + collection + end + + private + + def populate! + return if @populated + + each { nil } # force a loop through all diffs + @populated = true + nil + end + + def over_safe_limits?(files) + files >= @safe_max_files || @line_count > @safe_max_lines || @byte_count >= @safe_max_bytes + end + + def each_delta + @iterator.each_delta.with_index do |delta, i| + diff = Gitlab::Git::Diff.new(delta) + + yield @array[i] = diff + end + end + + def each_patch + @iterator.each_with_index do |raw, i| + # First yield cached Diff instances from @array + if @array[i] + yield @array[i] + next + end + + # We have exhausted @array, time to create new Diff instances or stop. + break if @overflow + + if !@all_diffs && i >= @max_files + @overflow = true + break + end + + collapse = !@all_diffs && !@no_collapse + + diff = Gitlab::Git::Diff.new(raw, collapse: collapse) + + if collapse && over_safe_limits?(i) + diff.prune_collapsed_diff! + end + + @line_count += diff.line_count + @byte_count += diff.diff.bytesize + + if !@all_diffs && (@line_count >= @max_lines || @byte_count >= @max_bytes) + # This last Diff instance pushes us over the lines limit. We stop and + # discard it. + @overflow = true + break + end + + yield @array[i] = diff + end + end + end + end +end diff --git a/lib/gitlab/git/encoding_helper.rb b/lib/gitlab/git/encoding_helper.rb new file mode 100644 index 00000000000..e57d228e688 --- /dev/null +++ b/lib/gitlab/git/encoding_helper.rb @@ -0,0 +1,58 @@ +module Gitlab + module Git + module EncodingHelper + extend self + + # This threshold is carefully tweaked to prevent usage of encodings detected + # by CharlockHolmes with low confidence. If CharlockHolmes confidence is low, + # we're better off sticking with utf8 encoding. + # Reason: git diff can return strings with invalid utf8 byte sequences if it + # truncates a diff in the middle of a multibyte character. In this case + # CharlockHolmes will try to guess the encoding and will likely suggest an + # obscure encoding with low confidence. + # There is a lot more info with this merge request: + # https://gitlab.com/gitlab-org/gitlab_git/merge_requests/77#note_4754193 + ENCODING_CONFIDENCE_THRESHOLD = 40 + + def encode!(message) + return nil unless message.respond_to? :force_encoding + + # if message is utf-8 encoding, just return it + message.force_encoding("UTF-8") + return message if message.valid_encoding? + + # return message if message type is binary + detect = CharlockHolmes::EncodingDetector.detect(message) + return message.force_encoding("BINARY") if detect && detect[:type] == :binary + + # force detected encoding if we have sufficient confidence. + if detect && detect[:encoding] && detect[:confidence] > ENCODING_CONFIDENCE_THRESHOLD + message.force_encoding(detect[:encoding]) + end + + # encode and clean the bad chars + message.replace clean(message) + rescue + encoding = detect ? detect[:encoding] : "unknown" + "--broken encoding: #{encoding}" + end + + def encode_utf8(message) + detect = CharlockHolmes::EncodingDetector.detect(message) + if detect + CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8') + else + clean(message) + end + end + + private + + def clean(message) + message.encode("UTF-16BE", undef: :replace, invalid: :replace, replace: "") + .encode("UTF-8") + .gsub("\0".encode("UTF-8"), "") + end + end + end +end diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb index 9b681e636c7..bd90d24a2ec 100644 --- a/lib/gitlab/git/hook.rb +++ b/lib/gitlab/git/hook.rb @@ -17,11 +17,13 @@ module Gitlab def trigger(gl_id, oldrev, newrev, ref) return [true, nil] unless exists? - case name - when "pre-receive", "post-receive" - call_receive_hook(gl_id, oldrev, newrev, ref) - when "update" - call_update_hook(gl_id, oldrev, newrev, ref) + Bundler.with_clean_env do + case name + when "pre-receive", "post-receive" + call_receive_hook(gl_id, oldrev, newrev, ref) + when "update" + call_update_hook(gl_id, oldrev, newrev, ref) + end end end diff --git a/lib/gitlab/git/index.rb b/lib/gitlab/git/index.rb new file mode 100644 index 00000000000..af1744c9c46 --- /dev/null +++ b/lib/gitlab/git/index.rb @@ -0,0 +1,126 @@ +module Gitlab + module Git + class Index + DEFAULT_MODE = 0o100644 + + attr_reader :repository, :raw_index + + def initialize(repository) + @repository = repository + @raw_index = repository.rugged.index + end + + delegate :read_tree, :get, to: :raw_index + + def write_tree + raw_index.write_tree(repository.rugged) + end + + def dir_exists?(path) + raw_index.find { |entry| entry[:path].start_with?("#{path}/") } + end + + def create(options) + options = normalize_options(options) + + file_entry = get(options[:file_path]) + if file_entry + raise Gitlab::Git::Repository::InvalidBlobName.new("Filename already exists") + end + + add_blob(options) + end + + def create_dir(options) + options = normalize_options(options) + + file_entry = get(options[:file_path]) + if file_entry + raise Gitlab::Git::Repository::InvalidBlobName.new("Directory already exists as a file") + end + + if dir_exists?(options[:file_path]) + raise Gitlab::Git::Repository::InvalidBlobName.new("Directory already exists") + end + + options = options.dup + options[:file_path] += '/.gitkeep' + options[:content] = '' + + add_blob(options) + end + + def update(options) + options = normalize_options(options) + + file_entry = get(options[:file_path]) + unless file_entry + raise Gitlab::Git::Repository::InvalidBlobName.new("File doesn't exist") + end + + add_blob(options, mode: file_entry[:mode]) + end + + def move(options) + options = normalize_options(options) + + file_entry = get(options[:previous_path]) + unless file_entry + raise Gitlab::Git::Repository::InvalidBlobName.new("File doesn't exist") + end + + raw_index.remove(options[:previous_path]) + + add_blob(options, mode: file_entry[:mode]) + end + + def delete(options) + options = normalize_options(options) + + file_entry = get(options[:file_path]) + unless file_entry + raise Gitlab::Git::Repository::InvalidBlobName.new("File doesn't exist") + end + + raw_index.remove(options[:file_path]) + end + + private + + def normalize_options(options) + options = options.dup + options[:file_path] = normalize_path(options[:file_path]) if options[:file_path] + options[:previous_path] = normalize_path(options[:previous_path]) if options[:previous_path] + options + end + + def normalize_path(path) + pathname = Gitlab::Git::PathHelper.normalize_path(path.dup) + + if pathname.each_filename.include?('..') + raise Gitlab::Git::Repository::InvalidBlobName.new('Invalid path') + end + + pathname.to_s + end + + def add_blob(options, mode: nil) + content = options[:content] + content = Base64.decode64(content) if options[:encoding] == 'base64' + + detect = CharlockHolmes::EncodingDetector.new.detect(content) + unless detect && detect[:type] == :binary + # When writing to the repo directly as we are doing here, + # the `core.autocrlf` config isn't taken into account. + content.gsub!("\r\n", "\n") if repository.autocrlf + end + + oid = repository.rugged.write(content, :blob) + + raw_index.add(path: options[:file_path], oid: oid, mode: mode || DEFAULT_MODE) + rescue Rugged::IndexError => e + raise Gitlab::Git::Repository::InvalidBlobName.new(e.message) + end + end + end +end diff --git a/lib/gitlab/git/path_helper.rb b/lib/gitlab/git/path_helper.rb new file mode 100644 index 00000000000..0148cd8df05 --- /dev/null +++ b/lib/gitlab/git/path_helper.rb @@ -0,0 +1,16 @@ +module Gitlab + module Git + class PathHelper + class << self + def normalize_path(filename) + # Strip all leading slashes so that //foo -> foo + filename[/^\/*/] = '' + + # Expand relative paths (e.g. foo/../bar) + filename = Pathname.new(filename) + filename.relative_path_from(Pathname.new('')) + end + end + end + end +end diff --git a/lib/gitlab/git/popen.rb b/lib/gitlab/git/popen.rb new file mode 100644 index 00000000000..df9ca3ee5ac --- /dev/null +++ b/lib/gitlab/git/popen.rb @@ -0,0 +1,26 @@ +require 'open3' + +module Gitlab + module Git + module Popen + def popen(cmd, path) + unless cmd.is_a?(Array) + raise "System commands must be given as an array of strings" + end + + vars = { "PWD" => path } + options = { chdir: path } + + @cmd_output = "" + @cmd_status = 0 + Open3.popen3(vars, *cmd, options) do |stdin, stdout, stderr, wait_thr| + @cmd_output << stdout.read + @cmd_output << stderr.read + @cmd_status = wait_thr.value.exitstatus + end + + [@cmd_output, @cmd_status] + end + end + end +end diff --git a/lib/gitlab/git/ref.rb b/lib/gitlab/git/ref.rb new file mode 100644 index 00000000000..37ef6836742 --- /dev/null +++ b/lib/gitlab/git/ref.rb @@ -0,0 +1,49 @@ +module Gitlab + module Git + class Ref + include Gitlab::Git::EncodingHelper + + # Branch or tag name + # without "refs/tags|heads" prefix + attr_reader :name + + # Target sha. + # Usually it is commit sha but in case + # when tag reference on other tag it can be tag sha + attr_reader :target + + # Dereferenced target + # Commit object to which the Ref points to + attr_reader :dereferenced_target + + # Extract branch name from full ref path + # + # Ex. + # Ref.extract_branch_name('refs/heads/master') #=> 'master' + def self.extract_branch_name(str) + str.gsub(/\Arefs\/heads\//, '') + end + + def self.dereference_object(object) + object = object.target while object.is_a?(Rugged::Tag::Annotation) + + object + end + + def initialize(repository, name, target) + encode! name + @name = name.gsub(/\Arefs\/(tags|heads)\//, '') + @dereferenced_target = Gitlab::Git::Commit.find(repository, target) + @target = if target.respond_to?(:oid) + target.oid + elsif target.respond_to?(:name) + target.name + elsif target.is_a? String + target + else + nil + end + end + end + end +end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb new file mode 100644 index 00000000000..8ec90885231 --- /dev/null +++ b/lib/gitlab/git/repository.rb @@ -0,0 +1,1198 @@ +# Gitlab::Git::Repository is a wrapper around native Rugged::Repository object +require 'tempfile' +require 'forwardable' +require "rubygems/package" + +module Gitlab + module Git + class Repository + include Gitlab::Git::Popen + + SEARCH_CONTEXT_LINES = 3 + + class NoRepository < StandardError; end + class InvalidBlobName < StandardError; end + class InvalidRef < StandardError; end + + # Full path to repo + attr_reader :path + + # Directory name of repo + attr_reader :name + + # Rugged repo object + attr_reader :rugged + + # 'path' must be the path to a _bare_ git repository, e.g. + # /path/to/my-repo.git + def initialize(path) + @path = path + @name = path.split("/").last + @attributes = Gitlab::Git::Attributes.new(path) + end + + delegate :empty?, + :bare?, + to: :rugged + + # Default branch in the repository + def root_ref + @root_ref ||= discover_default_branch + end + + # Alias to old method for compatibility + def raw + rugged + end + + def rugged + @rugged ||= Rugged::Repository.new(path) + rescue Rugged::RepositoryError, Rugged::OSError + raise NoRepository.new('no repository for such path') + end + + # Returns an Array of branch names + # sorted by name ASC + def branch_names + branches.map(&:name) + end + + # Returns an Array of Branches + def branches + rugged.branches.map do |rugged_ref| + begin + Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target) + rescue Rugged::ReferenceError + # Omit invalid branch + end + end.compact.sort_by(&:name) + end + + def reload_rugged + @rugged = nil + end + + # Directly find a branch with a simple name (e.g. master) + # + # force_reload causes a new Rugged repository to be instantiated + # + # This is to work around a bug in libgit2 that causes in-memory refs to + # be stale/invalid when packed-refs is changed. + # See https://gitlab.com/gitlab-org/gitlab-ce/issues/15392#note_14538333 + def find_branch(name, force_reload = false) + reload_rugged if force_reload + + rugged_ref = rugged.branches[name] + Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target) if rugged_ref + end + + def local_branches + rugged.branches.each(:local).map do |branch| + Gitlab::Git::Branch.new(self, branch.name, branch.target) + end + end + + # Returns the number of valid branches + def branch_count + rugged.branches.count do |ref| + begin + ref.name && ref.target # ensures the branch is valid + + true + rescue Rugged::ReferenceError + false + end + end + end + + # Returns an Array of tag names + def tag_names + rugged.tags.map { |t| t.name } + end + + # Returns an Array of Tags + def tags + rugged.references.each("refs/tags/*").map do |ref| + message = nil + + if ref.target.is_a?(Rugged::Tag::Annotation) + tag_message = ref.target.message + + if tag_message.respond_to?(:chomp) + message = tag_message.chomp + end + end + + Gitlab::Git::Tag.new(self, ref.name, ref.target, message) + end.sort_by(&:name) + end + + # Returns true if the given tag exists + # + # name - The name of the tag as a String. + def tag_exists?(name) + !!rugged.tags[name] + end + + # Returns true if the given branch exists + # + # name - The name of the branch as a String. + def branch_exists?(name) + rugged.branches.exists?(name) + + # If the branch name is invalid (e.g. ".foo") Rugged will raise an error. + # Whatever code calls this method shouldn't have to deal with that so + # instead we just return `false` (which is true since a branch doesn't + # exist when it has an invalid name). + rescue Rugged::ReferenceError + false + end + + # Returns an Array of branch and tag names + def ref_names + branch_names + tag_names + end + + # Deprecated. Will be removed in 5.2 + def heads + rugged.references.each("refs/heads/*").map do |head| + Gitlab::Git::Ref.new(self, head.name, head.target) + end.sort_by(&:name) + end + + def has_commits? + !empty? + end + + def repo_exists? + !!rugged + end + + # Discovers the default branch based on the repository's available branches + # + # - If no branches are present, returns nil + # - If one branch is present, returns its name + # - If two or more branches are present, returns current HEAD or master or first branch + def discover_default_branch + names = branch_names + + return if names.empty? + + return names[0] if names.length == 1 + + if rugged_head + extracted_name = Ref.extract_branch_name(rugged_head.name) + + return extracted_name if names.include?(extracted_name) + end + + if names.include?('master') + 'master' + else + names[0] + end + end + + def rugged_head + rugged.head + rescue Rugged::ReferenceError + nil + end + + def archive_metadata(ref, storage_path, format = "tar.gz") + ref ||= root_ref + commit = Gitlab::Git::Commit.find(self, ref) + return {} if commit.nil? + + project_name = self.name.chomp('.git') + prefix = "#{project_name}-#{ref}-#{commit.id}" + + { + 'RepoPath' => path, + 'ArchivePrefix' => prefix, + 'ArchivePath' => archive_file_path(prefix, storage_path, format), + 'CommitId' => commit.id, + } + end + + def archive_file_path(name, storage_path, format = "tar.gz") + # Build file path + return nil unless name + + extension = + case format + when "tar.bz2", "tbz", "tbz2", "tb2", "bz2" + "tar.bz2" + when "tar" + "tar" + when "zip" + "zip" + else + # everything else should fall back to tar.gz + "tar.gz" + end + + file_name = "#{name}.#{extension}" + File.join(storage_path, self.name, file_name) + end + + # Return repo size in megabytes + def size + size = popen(%w(du -sk), path).first.strip.to_i + (size.to_f / 1024).round(2) + end + + # Returns an array of BlobSnippets for files at the specified +ref+ that + # contain the +query+ string. + def search_files(query, ref = nil) + greps = [] + ref ||= root_ref + + populated_index(ref).each do |entry| + # Discard submodules + next if submodule?(entry) + + blob = Gitlab::Git::Blob.raw(self, entry[:oid]) + + # Skip binary files + next if blob.data.encoding == Encoding::ASCII_8BIT + + blob.load_all_data!(self) + greps += build_greps(blob.data, query, ref, entry[:path]) + end + + greps + end + + # Use the Rugged Walker API to build an array of commits. + # + # Usage. + # repo.log( + # ref: 'master', + # path: 'app/models', + # limit: 10, + # offset: 5, + # after: Time.new(2016, 4, 21, 14, 32, 10) + # ) + # + def log(options) + default_options = { + limit: 10, + offset: 0, + path: nil, + follow: false, + skip_merges: false, + disable_walk: false, + after: nil, + before: nil + } + + options = default_options.merge(options) + options[:limit] ||= 0 + options[:offset] ||= 0 + actual_ref = options[:ref] || root_ref + begin + sha = sha_from_ref(actual_ref) + rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError + # Return an empty array if the ref wasn't found + return [] + end + + if log_using_shell?(options) + log_by_shell(sha, options) + else + log_by_walk(sha, options) + end + end + + def log_using_shell?(options) + options[:path].present? || + options[:disable_walk] || + options[:skip_merges] || + options[:after] || + options[:before] + end + + def log_by_walk(sha, options) + walk_options = { + show: sha, + sort: Rugged::SORT_DATE, + limit: options[:limit], + offset: options[:offset] + } + Rugged::Walker.walk(rugged, walk_options).to_a + end + + def log_by_shell(sha, options) + limit = options[:limit].to_i + offset = options[:offset].to_i + use_follow_flag = options[:follow] && options[:path].present? + + # We will perform the offset in Ruby because --follow doesn't play well with --skip. + # See: https://gitlab.com/gitlab-org/gitlab-ce/issues/3574#note_3040520 + offset_in_ruby = use_follow_flag && options[:offset].present? + limit += offset if offset_in_ruby + + cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} log] + cmd << "--max-count=#{limit}" + cmd << '--format=%H' + cmd << "--skip=#{offset}" unless offset_in_ruby + cmd << '--follow' if use_follow_flag + cmd << '--no-merges' if options[:skip_merges] + cmd << "--after=#{options[:after].iso8601}" if options[:after] + cmd << "--before=#{options[:before].iso8601}" if options[:before] + cmd << sha + cmd += %W[-- #{options[:path]}] if options[:path].present? + + raw_output = IO.popen(cmd) { |io| io.read } + lines = offset_in_ruby ? raw_output.lines.drop(offset) : raw_output.lines + + lines.map! { |c| Rugged::Commit.new(rugged, c.strip) } + end + + def sha_from_ref(ref) + rev_parse_target(ref).oid + end + + # Return the object that +revspec+ points to. If +revspec+ is an + # annotated tag, then return the tag's target instead. + def rev_parse_target(revspec) + obj = rugged.rev_parse(revspec) + Ref.dereference_object(obj) + end + + # Return a collection of Rugged::Commits between the two revspec arguments. + # See http://git-scm.com/docs/git-rev-parse.html#_specifying_revisions for + # a detailed list of valid arguments. + def commits_between(from, to) + walker = Rugged::Walker.new(rugged) + walker.sorting(Rugged::SORT_DATE | Rugged::SORT_REVERSE) + + sha_from = sha_from_ref(from) + sha_to = sha_from_ref(to) + + walker.push(sha_to) + walker.hide(sha_from) + + commits = walker.to_a + walker.reset + + commits + end + + # Counts the amount of commits between `from` and `to`. + def count_commits_between(from, to) + commits_between(from, to).size + end + + # Returns the SHA of the most recent common ancestor of +from+ and +to+ + def merge_base_commit(from, to) + rugged.merge_base(from, to) + end + + # Return an array of Diff objects that represent the diff + # between +from+ and +to+. See Diff::filter_diff_options for the allowed + # diff options. The +options+ hash can also include :break_rewrites to + # split larger rewrites into delete/add pairs. + def diff(from, to, options = {}, *paths) + Gitlab::Git::DiffCollection.new(diff_patches(from, to, options, *paths), options) + end + + # Returns commits collection + # + # Ex. + # repo.find_commits( + # ref: 'master', + # max_count: 10, + # skip: 5, + # order: :date + # ) + # + # +options+ is a Hash of optional arguments to git + # :ref is the ref from which to begin (SHA1 or name) + # :contains is the commit contained by the refs from which to begin (SHA1 or name) + # :max_count is the maximum number of commits to fetch + # :skip is the number of commits to skip + # :order is the commits order and allowed value is :date(default) or :topo + # + def find_commits(options = {}) + actual_options = options.dup + + allowed_options = [:ref, :max_count, :skip, :contains, :order] + + actual_options.keep_if do |key| + allowed_options.include?(key) + end + + default_options = { skip: 0 } + actual_options = default_options.merge(actual_options) + + walker = Rugged::Walker.new(rugged) + + if actual_options[:ref] + walker.push(rugged.rev_parse_oid(actual_options[:ref])) + elsif actual_options[:contains] + branches_contains(actual_options[:contains]).each do |branch| + walker.push(branch.target_id) + end + else + rugged.references.each("refs/heads/*") do |ref| + walker.push(ref.target_id) + end + end + + if actual_options[:order] == :topo + walker.sorting(Rugged::SORT_TOPO) + else + walker.sorting(Rugged::SORT_DATE) + end + + commits = [] + offset = actual_options[:skip] + limit = actual_options[:max_count] + walker.each(offset: offset, limit: limit) do |commit| + gitlab_commit = Gitlab::Git::Commit.decorate(commit) + commits.push(gitlab_commit) + end + + walker.reset + + commits + rescue Rugged::OdbError + [] + end + + # Returns branch names collection that contains the special commit(SHA1 + # or name) + # + # Ex. + # repo.branch_names_contains('master') + # + def branch_names_contains(commit) + branches_contains(commit).map { |c| c.name } + end + + # Returns branch collection that contains the special commit(SHA1 or name) + # + # Ex. + # repo.branch_names_contains('master') + # + def branches_contains(commit) + commit_obj = rugged.rev_parse(commit) + parent = commit_obj.parents.first unless commit_obj.parents.empty? + + walker = Rugged::Walker.new(rugged) + + rugged.branches.select do |branch| + walker.push(branch.target_id) + walker.hide(parent) if parent + result = walker.any? { |c| c.oid == commit_obj.oid } + walker.reset + + result + end + end + + # Get refs hash which key is SHA1 + # and value is a Rugged::Reference + def refs_hash + # Initialize only when first call + if @refs_hash.nil? + @refs_hash = Hash.new { |h, k| h[k] = [] } + + rugged.references.each do |r| + # Symbolic/remote references may not have an OID; skip over them + target_oid = r.target.try(:oid) + if target_oid + sha = rev_parse_target(target_oid).oid + @refs_hash[sha] << r + end + end + end + @refs_hash + end + + # Lookup for rugged object by oid or ref name + def lookup(oid_or_ref_name) + rugged.rev_parse(oid_or_ref_name) + end + + # Return hash with submodules info for this repository + # + # Ex. + # { + # "rack" => { + # "id" => "c67be4624545b4263184c4a0e8f887efd0a66320", + # "path" => "rack", + # "url" => "git://github.com/chneukirchen/rack.git" + # }, + # "encoding" => { + # "id" => .... + # } + # } + # + def submodules(ref) + commit = rev_parse_target(ref) + return {} unless commit + + begin + content = blob_content(commit, ".gitmodules") + rescue InvalidBlobName + return {} + end + + parse_gitmodules(commit, content) + end + + # Return total commits count accessible from passed ref + def commit_count(ref) + walker = Rugged::Walker.new(rugged) + walker.sorting(Rugged::SORT_TOPO | Rugged::SORT_REVERSE) + oid = rugged.rev_parse_oid(ref) + walker.push(oid) + walker.count + end + + # Sets HEAD to the commit specified by +ref+; +ref+ can be a branch or + # tag name or a commit SHA. Valid +reset_type+ values are: + # + # [:soft] + # the head will be moved to the commit. + # [:mixed] + # will trigger a +:soft+ reset, plus the index will be replaced + # with the content of the commit tree. + # [:hard] + # will trigger a +:mixed+ reset and the working directory will be + # replaced with the content of the index. (Untracked and ignored files + # will be left alone) + delegate :reset, to: :rugged + + # Mimic the `git clean` command and recursively delete untracked files. + # Valid keys that can be passed in the +options+ hash are: + # + # :d - Remove untracked directories + # :f - Remove untracked directories that are managed by a different + # repository + # :x - Remove ignored files + # + # The value in +options+ must evaluate to true for an option to take + # effect. + # + # Examples: + # + # repo.clean(d: true, f: true) # Enable the -d and -f options + # + # repo.clean(d: false, x: true) # -x is enabled, -d is not + def clean(options = {}) + strategies = [:remove_untracked] + strategies.push(:force) if options[:f] + strategies.push(:remove_ignored) if options[:x] + + # TODO: implement this method + end + + # Check out the specified ref. Valid options are: + # + # :b - Create a new branch at +start_point+ and set HEAD to the new + # branch. + # + # * These options are passed to the Rugged::Repository#checkout method: + # + # :progress :: + # A callback that will be executed for checkout progress notifications. + # Up to 3 parameters are passed on each execution: + # + # - The path to the last updated file (or +nil+ on the very first + # invocation). + # - The number of completed checkout steps. + # - The number of total checkout steps to be performed. + # + # :notify :: + # A callback that will be executed for each checkout notification + # types specified with +:notify_flags+. Up to 5 parameters are passed + # on each execution: + # + # - An array containing the +:notify_flags+ that caused the callback + # execution. + # - The path of the current file. + # - A hash describing the baseline blob (or +nil+ if it does not + # exist). + # - A hash describing the target blob (or +nil+ if it does not exist). + # - A hash describing the workdir blob (or +nil+ if it does not + # exist). + # + # :strategy :: + # A single symbol or an array of symbols representing the strategies + # to use when performing the checkout. Possible values are: + # + # :none :: + # Perform a dry run (default). + # + # :safe :: + # Allow safe updates that cannot overwrite uncommitted data. + # + # :safe_create :: + # Allow safe updates plus creation of missing files. + # + # :force :: + # Allow all updates to force working directory to look like index. + # + # :allow_conflicts :: + # Allow checkout to make safe updates even if conflicts are found. + # + # :remove_untracked :: + # Remove untracked files not in index (that are not ignored). + # + # :remove_ignored :: + # Remove ignored files not in index. + # + # :update_only :: + # Only update existing files, don't create new ones. + # + # :dont_update_index :: + # Normally checkout updates index entries as it goes; this stops + # that. + # + # :no_refresh :: + # Don't refresh index/config/etc before doing checkout. + # + # :disable_pathspec_match :: + # Treat pathspec as simple list of exact match file paths. + # + # :skip_locked_directories :: + # Ignore directories in use, they will be left empty. + # + # :skip_unmerged :: + # Allow checkout to skip unmerged files (NOT IMPLEMENTED). + # + # :use_ours :: + # For unmerged files, checkout stage 2 from index (NOT IMPLEMENTED). + # + # :use_theirs :: + # For unmerged files, checkout stage 3 from index (NOT IMPLEMENTED). + # + # :update_submodules :: + # Recursively checkout submodules with same options (NOT + # IMPLEMENTED). + # + # :update_submodules_if_changed :: + # Recursively checkout submodules if HEAD moved in super repo (NOT + # IMPLEMENTED). + # + # :disable_filters :: + # If +true+, filters like CRLF line conversion will be disabled. + # + # :dir_mode :: + # Mode for newly created directories. Default: +0755+. + # + # :file_mode :: + # Mode for newly created files. Default: +0755+ or +0644+. + # + # :file_open_flags :: + # Mode for opening files. Default: + # <code>IO::CREAT | IO::TRUNC | IO::WRONLY</code>. + # + # :notify_flags :: + # A single symbol or an array of symbols representing the cases in + # which the +:notify+ callback should be invoked. Possible values are: + # + # :none :: + # Do not invoke the +:notify+ callback (default). + # + # :conflict :: + # Invoke the callback for conflicting paths. + # + # :dirty :: + # Invoke the callback for "dirty" files, i.e. those that do not need + # an update but no longer match the baseline. + # + # :updated :: + # Invoke the callback for any file that was changed. + # + # :untracked :: + # Invoke the callback for untracked files. + # + # :ignored :: + # Invoke the callback for ignored files. + # + # :all :: + # Invoke the callback for all these cases. + # + # :paths :: + # A glob string or an array of glob strings specifying which paths + # should be taken into account for the checkout operation. +nil+ will + # match all files. Default: +nil+. + # + # :baseline :: + # A Rugged::Tree that represents the current, expected contents of the + # workdir. Default: +HEAD+. + # + # :target_directory :: + # A path to an alternative workdir directory in which the checkout + # should be performed. + def checkout(ref, options = {}, start_point = "HEAD") + if options[:b] + rugged.branches.create(ref, start_point) + options.delete(:b) + end + default_options = { strategy: [:recreate_missing, :safe] } + rugged.checkout(ref, default_options.merge(options)) + end + + # Delete the specified branch from the repository + def delete_branch(branch_name) + rugged.branches.delete(branch_name) + end + + # Create a new branch named **ref+ based on **stat_point+, HEAD by default + # + # Examples: + # create_branch("feature") + # create_branch("other-feature", "master") + def create_branch(ref, start_point = "HEAD") + rugged_ref = rugged.branches.create(ref, start_point) + Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target) + rescue Rugged::ReferenceError => e + raise InvalidRef.new("Branch #{ref} already exists") if e.to_s =~ /'refs\/heads\/#{ref}'/ + raise InvalidRef.new("Invalid reference #{start_point}") + end + + # Return an array of this repository's remote names + def remote_names + rugged.remotes.each_name.to_a + end + + # Delete the specified remote from this repository. + def remote_delete(remote_name) + rugged.remotes.delete(remote_name) + end + + # Add a new remote to this repository. Returns a Rugged::Remote object + def remote_add(remote_name, url) + rugged.remotes.create(remote_name, url) + end + + # Update the specified remote using the values in the +options+ hash + # + # Example + # repo.update_remote("origin", url: "path/to/repo") + def remote_update(remote_name, options = {}) + # TODO: Implement other remote options + rugged.remotes.set_url(remote_name, options[:url]) if options[:url] + end + + # Fetch the specified remote + def fetch(remote_name) + rugged.remotes[remote_name].fetch + end + + # Push +*refspecs+ to the remote identified by +remote_name+. + def push(remote_name, *refspecs) + rugged.remotes[remote_name].push(refspecs) + end + + # Merge the +source_name+ branch into the +target_name+ branch. This is + # equivalent to `git merge --no_ff +source_name+`, since a merge commit + # is always created. + def merge(source_name, target_name, options = {}) + our_commit = rugged.branches[target_name].target + their_commit = rugged.branches[source_name].target + + raise "Invalid merge target" if our_commit.nil? + raise "Invalid merge source" if their_commit.nil? + + merge_index = rugged.merge_commits(our_commit, their_commit) + return false if merge_index.conflicts? + + actual_options = options.merge( + parents: [our_commit, their_commit], + tree: merge_index.write_tree(rugged), + update_ref: "refs/heads/#{target_name}" + ) + Rugged::Commit.create(rugged, actual_options) + end + + def commits_since(from_date) + walker = Rugged::Walker.new(rugged) + walker.sorting(Rugged::SORT_DATE | Rugged::SORT_REVERSE) + + rugged.references.each("refs/heads/*") do |ref| + walker.push(ref.target_id) + end + + commits = [] + walker.each do |commit| + break if commit.author[:time].to_date < from_date + commits.push(commit) + end + + commits + end + + AUTOCRLF_VALUES = { + "true" => true, + "false" => false, + "input" => :input + }.freeze + + def autocrlf + AUTOCRLF_VALUES[rugged.config['core.autocrlf']] + end + + def autocrlf=(value) + rugged.config['core.autocrlf'] = AUTOCRLF_VALUES.invert[value] + end + + # Returns result like "git ls-files" , recursive and full file path + # + # Ex. + # repo.ls_files('master') + # + def ls_files(ref) + actual_ref = ref || root_ref + + begin + sha_from_ref(actual_ref) + rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError + # Return an empty array if the ref wasn't found + return [] + end + + cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path} ls-tree) + cmd += %w(-r) + cmd += %w(--full-tree) + cmd += %w(--full-name) + cmd += %W(-- #{actual_ref}) + + raw_output = IO.popen(cmd, &:read).split("\n").map do |f| + stuff, path = f.split("\t") + _mode, type, _sha = stuff.split(" ") + path if type == "blob" + # Contain only blob type + end + + raw_output.compact + end + + def copy_gitattributes(ref) + begin + commit = lookup(ref) + rescue Rugged::ReferenceError + raise InvalidRef.new("Ref #{ref} is invalid") + end + + # Create the paths + info_dir_path = File.join(path, 'info') + info_attributes_path = File.join(info_dir_path, 'attributes') + + begin + # Retrieve the contents of the blob + gitattributes_content = blob_content(commit, '.gitattributes') + rescue InvalidBlobName + # No .gitattributes found. Should now remove any info/attributes and return + File.delete(info_attributes_path) if File.exist?(info_attributes_path) + return + end + + # Create the info directory if needed + Dir.mkdir(info_dir_path) unless File.directory?(info_dir_path) + + # Write the contents of the .gitattributes file to info/attributes + # Use binary mode to prevent Rails from converting ASCII-8BIT to UTF-8 + File.open(info_attributes_path, "wb") do |file| + file.write(gitattributes_content) + end + end + + # Checks if the blob should be diffable according to its attributes + def diffable?(blob) + attributes(blob.path).fetch('diff') { blob.text? } + end + + # Returns the Git attributes for the given file path. + # + # See `Gitlab::Git::Attributes` for more information. + def attributes(path) + @attributes.attributes(path) + end + + private + + # Get the content of a blob for a given commit. If the blob is a commit + # (for submodules) then return the blob's OID. + def blob_content(commit, blob_name) + blob_entry = tree_entry(commit, blob_name) + + unless blob_entry + raise InvalidBlobName.new("Invalid blob name: #{blob_name}") + end + + case blob_entry[:type] + when :commit + blob_entry[:oid] + when :tree + raise InvalidBlobName.new("#{blob_name} is a tree, not a blob") + when :blob + rugged.lookup(blob_entry[:oid]).content + end + end + + # Parses the contents of a .gitmodules file and returns a hash of + # submodule information. + def parse_gitmodules(commit, content) + results = {} + + current = "" + content.split("\n").each do |txt| + if txt =~ /^\s*\[/ + current = txt.match(/(?<=").*(?=")/)[0] + results[current] = {} + else + next unless results[current] + match_data = txt.match(/(\w+)\s*=\s*(.*)/) + next unless match_data + target = match_data[2].chomp + results[current][match_data[1]] = target + + if match_data[1] == "path" + begin + results[current]["id"] = blob_content(commit, target) + rescue InvalidBlobName + results.delete(current) + end + end + end + end + + results + end + + # Returns true if +commit+ introduced changes to +path+, using commit + # trees to make that determination. Uses the history simplification + # rules that `git log` uses by default, where a commit is omitted if it + # is TREESAME to any parent. + # + # If the +follow+ option is true and the file specified by +path+ was + # renamed, then the path value is set to the old path. + def commit_touches_path?(commit, path, follow, walker) + entry = tree_entry(commit, path) + + if commit.parents.empty? + # This is the root commit, return true if it has +path+ in its tree + return !entry.nil? + end + + num_treesame = 0 + commit.parents.each do |parent| + parent_entry = tree_entry(parent, path) + + # Only follow the first TREESAME parent for merge commits + if num_treesame > 0 + walker.hide(parent) + next + end + + if entry.nil? && parent_entry.nil? + num_treesame += 1 + elsif entry && parent_entry && entry[:oid] == parent_entry[:oid] + num_treesame += 1 + end + end + + case num_treesame + when 0 + detect_rename(commit, commit.parents.first, path) if follow + true + else false + end + end + + # Find the entry for +path+ in the tree for +commit+ + def tree_entry(commit, path) + pathname = Pathname.new(path) + first = true + tmp_entry = nil + + pathname.each_filename do |dir| + if first + tmp_entry = commit.tree[dir] + first = false + elsif tmp_entry.nil? + return nil + else + tmp_entry = rugged.lookup(tmp_entry[:oid]) + return nil unless tmp_entry.type == :tree + tmp_entry = tmp_entry[dir] + end + end + + tmp_entry + end + + # Compare +commit+ and +parent+ for +path+. If +path+ is a file and was + # renamed in +commit+, then set +path+ to the old filename. + def detect_rename(commit, parent, path) + diff = parent.diff(commit, paths: [path], disable_pathspec_match: true) + + # If +path+ is a filename, not a directory, then we should only have + # one delta. We don't need to follow renames for directories. + return nil if diff.each_delta.count > 1 + + delta = diff.each_delta.first + if delta.added? + full_diff = parent.diff(commit) + full_diff.find_similar! + + full_diff.each_delta do |full_delta| + if full_delta.renamed? && path == full_delta.new_file[:path] + # Look for the old path in ancestors + path.replace(full_delta.old_file[:path]) + end + end + end + end + + def archive_to_file(treeish = 'master', filename = 'archive.tar.gz', format = nil, compress_cmd = %w(gzip -n)) + git_archive_cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path} archive) + + # Put files into a directory before archiving + prefix = "#{archive_name(treeish)}/" + git_archive_cmd << "--prefix=#{prefix}" + + # Format defaults to tar + git_archive_cmd << "--format=#{format}" if format + + git_archive_cmd += %W(-- #{treeish}) + + open(filename, 'w') do |file| + # Create a pipe to act as the '|' in 'git archive ... | gzip' + pipe_rd, pipe_wr = IO.pipe + + # Get the compression process ready to accept data from the read end + # of the pipe + compress_pid = spawn(*nice(compress_cmd), in: pipe_rd, out: file) + # The read end belongs to the compression process now; we should + # close our file descriptor for it. + pipe_rd.close + + # Start 'git archive' and tell it to write into the write end of the + # pipe. + git_archive_pid = spawn(*nice(git_archive_cmd), out: pipe_wr) + # The write end belongs to 'git archive' now; close it. + pipe_wr.close + + # When 'git archive' and the compression process are finished, we are + # done. + Process.waitpid(git_archive_pid) + raise "#{git_archive_cmd.join(' ')} failed" unless $?.success? + Process.waitpid(compress_pid) + raise "#{compress_cmd.join(' ')} failed" unless $?.success? + end + end + + def nice(cmd) + nice_cmd = %w(nice -n 20) + unless unsupported_platform? + nice_cmd += %w(ionice -c 2 -n 7) + end + nice_cmd + cmd + end + + def unsupported_platform? + %w[darwin freebsd solaris].map { |platform| RUBY_PLATFORM.include?(platform) }.any? + end + + # Returns true if the index entry has the special file mode that denotes + # a submodule. + def submodule?(index_entry) + index_entry[:mode] == 57344 + end + + # Return a Rugged::Index that has read from the tree at +ref_name+ + def populated_index(ref_name) + commit = rev_parse_target(ref_name) + index = rugged.index + index.read_tree(commit.tree) + index + end + + # Return an array of BlobSnippets for lines in +file_contents+ that match + # +query+ + def build_greps(file_contents, query, ref, filename) + # The file_contents string is potentially huge so we make sure to loop + # through it one line at a time. This gives Ruby the chance to GC lines + # we are not interested in. + # + # We need to do a little extra work because we are not looking for just + # the lines that matches the query, but also for the context + # (surrounding lines). We will use Enumerable#each_cons to efficiently + # loop through the lines while keeping surrounding lines on hand. + # + # First, we turn "foo\nbar\nbaz" into + # [ + # [nil, -3], [nil, -2], [nil, -1], + # ['foo', 0], ['bar', 1], ['baz', 3], + # [nil, 4], [nil, 5], [nil, 6] + # ] + lines_with_index = Enumerator.new do |yielder| + # Yield fake 'before' lines for the first line of file_contents + (-SEARCH_CONTEXT_LINES..-1).each do |i| + yielder.yield [nil, i] + end + + # Yield the actual file contents + count = 0 + file_contents.each_line do |line| + line.chomp! + yielder.yield [line, count] + count += 1 + end + + # Yield fake 'after' lines for the last line of file_contents + (count + 1..count + SEARCH_CONTEXT_LINES).each do |i| + yielder.yield [nil, i] + end + end + + greps = [] + + # Loop through consecutive blocks of lines with indexes + lines_with_index.each_cons(2 * SEARCH_CONTEXT_LINES + 1) do |line_block| + # Get the 'middle' line and index from the block + line, _ = line_block[SEARCH_CONTEXT_LINES] + + next unless line && line.match(/#{Regexp.escape(query)}/i) + + # Yay, 'line' contains a match! + # Get an array with just the context lines (no indexes) + match_with_context = line_block.map(&:first) + # Remove 'nil' lines in case we are close to the first or last line + match_with_context.compact! + + # Get the line number (1-indexed) of the first context line + first_context_line_number = line_block[0][1] + 1 + + greps << Gitlab::Git::BlobSnippet.new( + ref, + match_with_context, + first_context_line_number, + filename + ) + end + + greps + end + + # Return the Rugged patches for the diff between +from+ and +to+. + def diff_patches(from, to, options = {}, *paths) + options ||= {} + break_rewrites = options[:break_rewrites] + actual_options = Gitlab::Git::Diff.filter_diff_options(options.merge(paths: paths)) + + diff = rugged.diff(from, to, actual_options) + diff.find_similar!(break_rewrites: break_rewrites) + diff.each_patch + end + end + end +end diff --git a/lib/gitlab/git/rev_list.rb b/lib/gitlab/git/rev_list.rb new file mode 100644 index 00000000000..79dd0cf7df2 --- /dev/null +++ b/lib/gitlab/git/rev_list.rb @@ -0,0 +1,42 @@ +module Gitlab + module Git + class RevList + attr_reader :project, :env + + ALLOWED_VARIABLES = %w[GIT_OBJECT_DIRECTORY GIT_ALTERNATE_OBJECT_DIRECTORIES].freeze + + def initialize(oldrev, newrev, project:, env: nil) + @project = project + @env = env.presence || {} + @args = [Gitlab.config.git.bin_path, + "--git-dir=#{project.repository.path_to_repo}", + "rev-list", + "--max-count=1", + oldrev, + "^#{newrev}"] + end + + def execute + Gitlab::Popen.popen(@args, nil, parse_environment_variables) + end + + def valid? + environment_variables.all? do |(name, value)| + value.to_s.start_with?(project.repository.path_to_repo) + end + end + + private + + def parse_environment_variables + return {} unless valid? + + environment_variables + end + + def environment_variables + @environment_variables ||= env.slice(*ALLOWED_VARIABLES).compact + end + end + end +end diff --git a/lib/gitlab/git/tag.rb b/lib/gitlab/git/tag.rb new file mode 100644 index 00000000000..b5342c3d310 --- /dev/null +++ b/lib/gitlab/git/tag.rb @@ -0,0 +1,17 @@ +module Gitlab + module Git + class Tag < Ref + attr_reader :object_sha + + def initialize(repository, name, target, message = nil) + super(repository, name, target) + + @message = message + end + + def message + encode! @message + end + end + end +end diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb new file mode 100644 index 00000000000..f7450e8b58f --- /dev/null +++ b/lib/gitlab/git/tree.rb @@ -0,0 +1,104 @@ +module Gitlab + module Git + class Tree + include Gitlab::Git::EncodingHelper + + attr_accessor :id, :root_id, :name, :path, :type, + :mode, :commit_id, :submodule_url + + class << self + # Get list of tree objects + # for repository based on commit sha and path + # Uses rugged for raw objects + def where(repository, sha, path = nil) + path = nil if path == '' || path == '/' + + commit = repository.lookup(sha) + root_tree = commit.tree + + tree = if path + id = find_id_by_path(repository, root_tree.oid, path) + if id + repository.lookup(id) + else + [] + end + else + root_tree + end + + tree.map do |entry| + new( + id: entry[:oid], + root_id: root_tree.oid, + name: entry[:name], + type: entry[:type], + mode: entry[:filemode], + path: path ? File.join(path, entry[:name]) : entry[:name], + commit_id: sha, + ) + end + end + + # Recursive search of tree id for path + # + # Ex. + # blog/ # oid: 1a + # app/ # oid: 2a + # models/ # oid: 3a + # views/ # oid: 4a + # + # + # Tree.find_id_by_path(repo, '1a', 'app/models') # => '3a' + # + def find_id_by_path(repository, root_id, path) + root_tree = repository.lookup(root_id) + path_arr = path.split('/') + + entry = root_tree.find do |entry| + entry[:name] == path_arr[0] && entry[:type] == :tree + end + + return nil unless entry + + if path_arr.size > 1 + path_arr.shift + find_id_by_path(repository, entry[:oid], path_arr.join('/')) + else + entry[:oid] + end + end + end + + def initialize(options) + %w(id root_id name path type mode commit_id).each do |key| + self.send("#{key}=", options[key.to_sym]) + end + end + + def name + encode! @name + end + + def dir? + type == :tree + end + + def file? + type == :blob + end + + def submodule? + type == :commit + end + + def readme? + name =~ /^readme/i + end + + def contributing? + name =~ /^contributing/i + end + end + end +end diff --git a/lib/gitlab/git/util.rb b/lib/gitlab/git/util.rb new file mode 100644 index 00000000000..7973da2e8f8 --- /dev/null +++ b/lib/gitlab/git/util.rb @@ -0,0 +1,18 @@ +module Gitlab + module Git + module Util + LINE_SEP = "\n".freeze + + def self.count_lines(string) + case string[-1] + when nil + 0 + when LINE_SEP + string.count(LINE_SEP) + else + string.count(LINE_SEP) + 1 + end + end + end + end +end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 1882eb8d050..ffb178334bc 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -2,119 +2,178 @@ # class return an instance of `GitlabAccessStatus` module Gitlab class GitAccess - DOWNLOAD_COMMANDS = %w{ git-upload-pack git-upload-archive } - PUSH_COMMANDS = %w{ git-receive-pack } + UnauthorizedError = Class.new(StandardError) - attr_reader :actor, :project, :protocol, :user_access + ERROR_MESSAGES = { + upload: 'You are not allowed to upload code for this project.', + download: 'You are not allowed to download code from this project.', + deploy_key_upload: + 'This deploy key does not have write access to this project.', + no_repo: 'A repository for this project does not exist yet.' + }.freeze - def initialize(actor, project, protocol) + DOWNLOAD_COMMANDS = %w{ git-upload-pack git-upload-archive }.freeze + PUSH_COMMANDS = %w{ git-receive-pack }.freeze + ALL_COMMANDS = DOWNLOAD_COMMANDS + PUSH_COMMANDS + + attr_reader :actor, :project, :protocol, :user_access, :authentication_abilities + + def initialize(actor, project, protocol, authentication_abilities:, env: {}) @actor = actor @project = project @protocol = protocol + @authentication_abilities = authentication_abilities @user_access = UserAccess.new(user, project: project) + @env = env end def check(cmd, changes) - return build_status_object(false, "Git access over #{protocol.upcase} is not allowed") unless protocol_allowed? + check_protocol! + check_active_user! + check_project_accessibility! + check_command_existence!(cmd) + check_repository_existence! - unless actor - return build_status_object(false, "No user or key was provided.") + case cmd + when *DOWNLOAD_COMMANDS + check_download_access! + when *PUSH_COMMANDS + check_push_access!(changes) end + build_status_object(true) + rescue UnauthorizedError => ex + build_status_object(false, ex.message) + end + + def guest_can_download_code? + Guest.can?(:download_code, project) + end + + def user_can_download_code? + authentication_abilities.include?(:download_code) && user_access.can_do_action?(:download_code) + end + + def build_can_download_code? + authentication_abilities.include?(:build_download_code) && user_access.can_do_action?(:build_download_code) + end + + def protocol_allowed? + Gitlab::ProtocolAccess.allowed?(protocol) + end + + private + + def check_protocol! + unless protocol_allowed? + raise UnauthorizedError, "Git access over #{protocol.upcase} is not allowed" + end + end + + def check_active_user! + return if deploy_key? + if user && !user_access.allowed? - return build_status_object(false, "Your account has been blocked.") + raise UnauthorizedError, "Your account has been blocked." end + end - unless project && (user_access.can_read_project? || deploy_key_can_read_project?) - return build_status_object(false, 'The project you were looking for could not be found.') + def check_project_accessibility! + if project.blank? || !can_read_project? + raise UnauthorizedError, 'The project you were looking for could not be found.' end + end - case cmd - when *DOWNLOAD_COMMANDS - download_access_check - when *PUSH_COMMANDS - push_access_check(changes) - else - build_status_object(false, "The command you're trying to execute is not allowed.") + def check_command_existence!(cmd) + unless ALL_COMMANDS.include?(cmd) + raise UnauthorizedError, "The command you're trying to execute is not allowed." end end - def download_access_check - if user - user_download_access_check - elsif deploy_key - build_status_object(true) - else - raise 'Wrong actor' + def check_repository_existence! + unless project.repository.exists? + raise UnauthorizedError, ERROR_MESSAGES[:no_repo] end end - def push_access_check(changes) - if user - user_push_access_check(changes) - elsif deploy_key - build_status_object(false, "Deploy keys are not allowed to push code.") - else - raise 'Wrong actor' + def check_download_access! + return if deploy_key? + + passed = user_can_download_code? || + build_can_download_code? || + guest_can_download_code? + + unless passed + raise UnauthorizedError, ERROR_MESSAGES[:download] end end - def user_download_access_check - unless user_access.can_do_action?(:download_code) - return build_status_object(false, "You are not allowed to download code from this project.") + def check_push_access!(changes) + if deploy_key + check_deploy_key_push_access! + elsif user + check_user_push_access! + else + raise UnauthorizedError, ERROR_MESSAGES[:upload] end - build_status_object(true) + return if changes.blank? # Allow access. + + check_change_access!(changes) end - def user_push_access_check(changes) - if changes.blank? - return build_status_object(true) + def check_user_push_access! + unless authentication_abilities.include?(:push_code) + raise UnauthorizedError, ERROR_MESSAGES[:upload] end + end - unless project.repository.exists? - return build_status_object(false, "A repository for this project does not exist yet.") + def check_deploy_key_push_access! + unless deploy_key.can_push_to?(project) + raise UnauthorizedError, ERROR_MESSAGES[:deploy_key_upload] end + end + def check_change_access!(changes) changes_list = Gitlab::ChangesList.new(changes) # Iterate over all changes to find if user allowed all of them to be applied changes_list.each do |change| - status = change_access_check(change) + status = check_single_change_access(change) unless status.allowed? # If user does not have access to make at least one change - cancel all push - return status + raise UnauthorizedError, status.message end end - - build_status_object(true) end - def change_access_check(change) - Checks::ChangeAccess.new(change, user_access: user_access, project: project).exec - end - - def protocol_allowed? - Gitlab::ProtocolAccess.allowed?(protocol) + def check_single_change_access(change) + Checks::ChangeAccess.new( + change, + user_access: user_access, + project: project, + env: @env, + skip_authorization: deploy_key?).exec end - private - def matching_merge_request?(newrev, branch_name) Checks::MatchingMergeRequest.new(newrev, branch_name, project).match? end def deploy_key - actor if actor.is_a?(DeployKey) + actor if deploy_key? + end + + def deploy_key? + actor.is_a?(DeployKey) end - def deploy_key_can_read_project? + def can_read_project? if deploy_key - return true if project.public? - deploy_key.projects.include?(project) - else - false - end + deploy_key.has_access_to?(project) + elsif user + user.can?(:read_project, project) + end || Guest.can?(:read_project, project) end protected diff --git a/lib/gitlab/git_access_wiki.rb b/lib/gitlab/git_access_wiki.rb index f71d3575909..67eaa5e088d 100644 --- a/lib/gitlab/git_access_wiki.rb +++ b/lib/gitlab/git_access_wiki.rb @@ -1,6 +1,14 @@ module Gitlab class GitAccessWiki < GitAccess - def change_access_check(change) + def guest_can_download_code? + Guest.can?(:download_wiki_code, project) + end + + def user_can_download_code? + authentication_abilities.include?(:download_code) && user_access.can_do_action?(:download_wiki_code) + end + + def check_single_change_access(change) if user_access.can_do_action?(:create_wiki) build_status_object(true) else diff --git a/lib/gitlab/git_post_receive.rb b/lib/gitlab/git_post_receive.rb index d32bdd86427..6babea144c7 100644 --- a/lib/gitlab/git_post_receive.rb +++ b/lib/gitlab/git_post_receive.rb @@ -30,11 +30,11 @@ module Gitlab def retrieve_project_and_type @type = :project - @project = Project.find_with_namespace(@repo_path) + @project = Project.find_by_full_path(@repo_path) if @repo_path.end_with?('.wiki') && !@project @type = :wiki - @project = Project.find_with_namespace(@repo_path.gsub(/\.wiki\z/, '')) + @project = Project.find_by_full_path(@repo_path.gsub(/\.wiki\z/, '')) end end diff --git a/lib/gitlab/github_import/base_formatter.rb b/lib/gitlab/github_import/base_formatter.rb index 72992baffd4..8c80791e7c9 100644 --- a/lib/gitlab/github_import/base_formatter.rb +++ b/lib/gitlab/github_import/base_formatter.rb @@ -1,24 +1,23 @@ module Gitlab module GithubImport class BaseFormatter - attr_reader :formatter, :project, :raw_data + attr_reader :client, :formatter, :project, :raw_data - def initialize(project, raw_data) + def initialize(project, raw_data, client = nil) @project = project @raw_data = raw_data + @client = client @formatter = Gitlab::ImportFormatter.new end def create! - self.klass.create!(self.attributes) + project.public_send(project_association).find_or_create_by!(find_condition) do |record| + record.attributes = attributes + end end - private - - def gl_user_id(github_id) - User.joins(:identities). - find_by("identities.extern_uid = ? AND identities.provider = 'github'", github_id.to_s). - try(:id) + def url + raw_data.url || '' end end end diff --git a/lib/gitlab/github_import/branch_formatter.rb b/lib/gitlab/github_import/branch_formatter.rb index 4750675ae9d..0a8d05b5fe1 100644 --- a/lib/gitlab/github_import/branch_formatter.rb +++ b/lib/gitlab/github_import/branch_formatter.rb @@ -8,7 +8,7 @@ module Gitlab end def valid? - repo.present? + sha.present? && ref.present? end private diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 084e514492c..7dbeec5b010 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -4,10 +4,13 @@ module Gitlab GITHUB_SAFE_REMAINING_REQUESTS = 100 GITHUB_SAFE_SLEEP_TIME = 500 - attr_reader :access_token + attr_reader :access_token, :host, :api_version - def initialize(access_token) + def initialize(access_token, host: nil, api_version: 'v3') @access_token = access_token + @host = host.to_s.sub(%r{/+\z}, '') + @api_version = api_version + @users = {} if access_token ::Octokit.auto_paginate = false @@ -17,7 +20,7 @@ module Gitlab def api @api ||= ::Octokit::Client.new( access_token: access_token, - api_endpoint: github_options[:site], + api_endpoint: api_endpoint, # If there is no config, we're connecting to github.com and we # should verify ssl. connection_options: { @@ -52,7 +55,7 @@ module Gitlab def method_missing(method, *args, &block) if api.respond_to?(method) - request { api.send(method, *args, &block) } + request(method, *args, &block) else super(method, *args, &block) end @@ -62,8 +65,23 @@ module Gitlab api.respond_to?(method) || super end + def user(login) + return nil unless login.present? + return @users[login] if @users.key?(login) + + @users[login] = api.user(login) + end + private + def api_endpoint + if host.present? && api_version.present? + "#{host}/api/#{api_version}" + else + github_options[:site] + end + end + def config Gitlab.config.omniauth.providers.find { |provider| provider.name == "github" } end @@ -99,20 +117,31 @@ module Gitlab rate_limit.resets_in + GITHUB_SAFE_SLEEP_TIME end - def request + def request(method, *args, &block) sleep rate_limit_sleep_time if rate_limit_exceed? - data = yield + data = api.send(method, *args) + return data unless data.is_a?(Array) last_response = api.last_response + if block_given? + yield data + # api.last_response could change while we're yielding (e.g. fetching labels for each PR) + # so we cache our own last response + each_response_page(last_response, &block) + else + each_response_page(last_response) { |page| data.concat(page) } + data + end + end + + def each_response_page(last_response) while last_response.rels[:next] sleep rate_limit_sleep_time if rate_limit_exceed? last_response = last_response.rels[:next].get - data.concat(last_response.data) if last_response.data.is_a?(Array) + yield last_response.data if last_response.data.is_a?(Array) end - - data end end end diff --git a/lib/gitlab/github_import/comment_formatter.rb b/lib/gitlab/github_import/comment_formatter.rb index 2c1b94ef2cd..e21922070c1 100644 --- a/lib/gitlab/github_import/comment_formatter.rb +++ b/lib/gitlab/github_import/comment_formatter.rb @@ -1,6 +1,8 @@ module Gitlab module GithubImport class CommentFormatter < BaseFormatter + attr_writer :author_id + def attributes { project: project, @@ -17,11 +19,11 @@ module Gitlab private def author - raw_data.user.login + @author ||= UserFormatter.new(client, raw_data.user) end def author_id - gl_user_id(raw_data.user.id) || project.creator_id + author.gitlab_id || project.creator_id end def body @@ -52,7 +54,11 @@ module Gitlab end def note - formatter.author_line(author) + body + if author.gitlab_id + body + else + formatter.author_line(author.login) + body + end end def type diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 9ddc8905bd6..dc73cad93a5 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -3,88 +3,157 @@ module Gitlab class Importer include Gitlab::ShellAdapter - attr_reader :client, :project, :repo, :repo_url + attr_reader :errors, :project, :repo, :repo_url def initialize(project) @project = project @repo = project.import_source @repo_url = project.import_url + @errors = [] + @labels = {} + end + + def client + return @client if defined?(@client) + unless credentials + raise Projects::ImportService::Error, + "Unable to find project import data credentials for project ID: #{@project.id}" + end - if credentials - @client = Client.new(credentials[:user]) - else - raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{@project.id}" + opts = {} + # Gitea plan to be GitHub compliant + if project.gitea_import? + uri = URI.parse(project.import_url) + host = "#{uri.scheme}://#{uri.host}:#{uri.port}#{uri.path}".sub(%r{/?[\w-]+/[\w-]+\.git\z}, '') + opts = { + host: host, + api_version: 'v1' + } end + + @client = Client.new(credentials[:user], opts) end def execute - import_labels && import_milestones && import_issues && - import_pull_requests && import_wiki + # The ordering of importing is important here due to the way GitHub structures their data + # 1. Labels are required by other items while not having a dependency on anything else + # so need to be first + # 2. Pull requests must come before issues. Every pull request is also an issue but not + # all issues are pull requests. Only the issue entity has labels defined in GitHub. GitLab + # doesn't structure data like this so we need to make sure that we've created the MRs + # before we attempt to add the labels defined in the GitHub issue for the related, already + # imported, pull request + import_labels + import_milestones + import_pull_requests + import_issues + import_comments(:issues) + import_comments(:pull_requests) + import_wiki + + # Gitea doesn't have a Release API yet + # See https://github.com/go-gitea/gitea/issues/330 + unless project.gitea_import? + import_releases + end + + handle_errors + + true end private def credentials - @credentials ||= project.import_data.credentials if project.import_data + return @credentials if defined?(@credentials) + + @credentials = project.import_data ? project.import_data.credentials : nil + end + + def handle_errors + return unless errors.any? + + project.update_column(:import_error, { + message: 'The remote data could not be fully imported.', + errors: errors + }.to_json) end def import_labels - labels = client.labels(repo, per_page: 100) - labels.each { |raw| LabelFormatter.new(project, raw).create! } + fetch_resources(:labels, repo, per_page: 100) do |labels| + labels.each do |raw| + begin + gh_label = LabelFormatter.new(project, raw) + gh_label.create! + rescue => e + errors << { type: :label, url: Gitlab::UrlSanitizer.sanitize(gh_label.url), errors: e.message } + end + end + end - true - rescue ActiveRecord::RecordInvalid => e - raise Projects::ImportService::Error, e.message + cache_labels! end def import_milestones - milestones = client.milestones(repo, state: :all, per_page: 100) - milestones.each { |raw| MilestoneFormatter.new(project, raw).create! } - - true - rescue ActiveRecord::RecordInvalid => e - raise Projects::ImportService::Error, e.message + fetch_resources(:milestones, repo, state: :all, per_page: 100) do |milestones| + milestones.each do |raw| + begin + gh_milestone = MilestoneFormatter.new(project, raw) + gh_milestone.create! + rescue => e + errors << { type: :milestone, url: Gitlab::UrlSanitizer.sanitize(gh_milestone.url), errors: e.message } + end + end + end end def import_issues - issues = client.issues(repo, state: :all, sort: :created, direction: :asc, per_page: 100) + fetch_resources(:issues, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |issues| + issues.each do |raw| + gh_issue = IssueFormatter.new(project, raw, client) - issues.each do |raw| - gh_issue = IssueFormatter.new(project, raw) + begin + issuable = + if gh_issue.pull_request? + MergeRequest.find_by(target_project_id: project.id, iid: gh_issue.number) + else + gh_issue.create! + end - if gh_issue.valid? - issue = gh_issue.create! - apply_labels(issue) - import_comments(issue) if gh_issue.has_comments? + apply_labels(issuable, raw) + rescue => e + errors << { type: :issue, url: Gitlab::UrlSanitizer.sanitize(gh_issue.url), errors: e.message } + end end end - - true - rescue ActiveRecord::RecordInvalid => e - raise Projects::ImportService::Error, e.message end def import_pull_requests - pull_requests = client.pull_requests(repo, state: :all, sort: :created, direction: :asc, per_page: 100) - pull_requests = pull_requests.map { |raw| PullRequestFormatter.new(project, raw) }.select(&:valid?) - - pull_requests.each do |pull_request| - begin - restore_source_branch(pull_request) unless pull_request.source_branch_exists? - restore_target_branch(pull_request) unless pull_request.target_branch_exists? - - merge_request = pull_request.create! - apply_labels(merge_request) - import_comments(merge_request) - import_comments_on_diff(merge_request) - rescue ActiveRecord::RecordInvalid => e - raise Projects::ImportService::Error, e.message - ensure - clean_up_restored_branches(pull_request) + fetch_resources(:pull_requests, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |pull_requests| + pull_requests.each do |raw| + gh_pull_request = PullRequestFormatter.new(project, raw, client) + + next unless gh_pull_request.valid? + + begin + restore_source_branch(gh_pull_request) unless gh_pull_request.source_branch_exists? + restore_target_branch(gh_pull_request) unless gh_pull_request.target_branch_exists? + + merge_request = gh_pull_request.create! + + # Gitea doesn't return PR in the Issue API endpoint, so labels must be assigned at this stage + if project.gitea_import? + apply_labels(merge_request, raw) + end + rescue => e + errors << { type: :pull_request, url: Gitlab::UrlSanitizer.sanitize(gh_pull_request.url), errors: e.message } + ensure + clean_up_restored_branches(gh_pull_request) + end end end - true + project.repository.after_remove_branch end def restore_source_branch(pull_request) @@ -98,63 +167,161 @@ module Gitlab def remove_branch(name) project.repository.delete_branch(name) rescue Rugged::ReferenceError - nil + errors << { type: :remove_branch, name: name } end def clean_up_restored_branches(pull_request) remove_branch(pull_request.source_branch_name) unless pull_request.source_branch_exists? remove_branch(pull_request.target_branch_name) unless pull_request.target_branch_exists? + end - project.repository.after_remove_branch + def apply_labels(issuable, raw) + return unless raw.labels.count > 0 + + label_ids = raw.labels + .map { |attrs| @labels[attrs.name] } + .compact + + issuable.update_attribute(:label_ids, label_ids) end - def apply_labels(issuable) - issue = client.issue(repo, issuable.iid) + def import_comments(issuable_type) + resource_type = "#{issuable_type}_comments".to_sym + + # Two notes here: + # 1. We don't have a distinctive attribute for comments (unlike issues iid), so we fetch the last inserted note, + # compare it against every comment in the current imported page until we find match, and that's where start importing + # 2. GH returns comments for _both_ issues and PRs through issues_comments API, while pull_requests_comments returns + # only comments on diffs, so select last note not based on noteable_type but on line_code + line_code_is = issuable_type == :pull_requests ? 'NOT NULL' : 'NULL' + last_note = project.notes.where("line_code IS #{line_code_is}").last - if issue.labels.count > 0 - label_ids = issue.labels.map do |raw| - Label.find_by(LabelFormatter.new(project, raw).attributes).try(:id) + fetch_resources(resource_type, repo, per_page: 100) do |comments| + if last_note + discard_inserted_comments(comments, last_note) + last_note = nil end - issuable.update_attribute(:label_ids, label_ids) + create_comments(comments) end end - def import_comments(issuable) - comments = client.issue_comments(repo, issuable.iid, per_page: 100) - create_comments(issuable, comments) - end + def create_comments(comments) + ActiveRecord::Base.no_touching do + comments.each do |raw| + begin + comment = CommentFormatter.new(project, raw, client) + + # GH does not return info about comment's parent, so we guess it by checking its URL! + *_, parent, iid = URI(raw.html_url).path.split('/') - def import_comments_on_diff(merge_request) - comments = client.pull_request_comments(repo, merge_request.iid, per_page: 100) - create_comments(merge_request, comments) + issuable = if parent == 'issues' + Issue.find_by(project_id: project.id, iid: iid) + else + MergeRequest.find_by(target_project_id: project.id, iid: iid) + end + + next unless issuable + + issuable.notes.create!(comment.attributes) + rescue => e + errors << { type: :comment, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + end + end + end end - def create_comments(issuable, comments) - comments.each do |raw| - comment = CommentFormatter.new(project, raw) - issuable.notes.create!(comment.attributes) + def discard_inserted_comments(comments, last_note) + last_note_attrs = nil + + cut_off_index = comments.find_index do |raw| + comment = CommentFormatter.new(project, raw) + comment_attrs = comment.attributes + last_note_attrs ||= last_note.slice(*comment_attrs.keys) + + comment_attrs.with_indifferent_access == last_note_attrs end + + # No matching resource in the collection, which means we got halted right on the end of the last page, so all good + return unless cut_off_index + + # Otherwise, remove the resources we've already inserted + comments.shift(cut_off_index + 1) end def import_wiki - unless project.wiki_enabled? + unless project.wiki.repository_exists? wiki = WikiFormatter.new(project) gitlab_shell.import_repository(project.repository_storage_path, wiki.path_with_namespace, wiki.import_url) - project.update_attribute(:wiki_enabled, true) end - - true rescue Gitlab::Shell::Error => e # GitHub error message when the wiki repo has not been created, # this means that repo has wiki enabled, but have no pages. So, # we can skip the import. if e.message !~ /repository not exported/ - raise Projects::ImportService::Error, e.message - else - true + errors << { type: :wiki, errors: e.message } + end + end + + def import_releases + fetch_resources(:releases, repo, per_page: 100) do |releases| + releases.each do |raw| + begin + gh_release = ReleaseFormatter.new(project, raw) + gh_release.create! if gh_release.valid? + rescue => e + errors << { type: :release, url: Gitlab::UrlSanitizer.sanitize(gh_release.url), errors: e.message } + end + end end end + + def cache_labels! + project.labels.select(:id, :title).find_each do |label| + @labels[label.title] = label.id + end + end + + def fetch_resources(resource_type, *opts) + return if imported?(resource_type) + + opts.last[:page] = current_page(resource_type) + + client.public_send(resource_type, *opts) do |resources| + yield resources + increment_page(resource_type) + end + + imported!(resource_type) + end + + def imported?(resource_type) + Rails.cache.read("#{cache_key_prefix}:#{resource_type}:imported") + end + + def imported!(resource_type) + Rails.cache.write("#{cache_key_prefix}:#{resource_type}:imported", true, ex: 1.day) + end + + def increment_page(resource_type) + key = "#{cache_key_prefix}:#{resource_type}:current-page" + + # Rails.cache.increment calls INCRBY directly on the value stored under the key, which is + # a serialized ActiveSupport::Cache::Entry, so it will return an error by Redis, hence this ugly work-around + page = Rails.cache.read(key) + page += 1 + Rails.cache.write(key, page) + + page + end + + def current_page(resource_type) + Rails.cache.fetch("#{cache_key_prefix}:#{resource_type}:current-page", ex: 1.day) { 1 } + end + + def cache_key_prefix + @cache_key_prefix ||= "github-import:#{project.id}" + end end end end diff --git a/lib/gitlab/github_import/issuable_formatter.rb b/lib/gitlab/github_import/issuable_formatter.rb new file mode 100644 index 00000000000..27b171d6ddb --- /dev/null +++ b/lib/gitlab/github_import/issuable_formatter.rb @@ -0,0 +1,66 @@ +module Gitlab + module GithubImport + class IssuableFormatter < BaseFormatter + attr_writer :assignee_id, :author_id + + def project_association + raise NotImplementedError + end + + delegate :number, to: :raw_data + + def find_condition + { iid: number } + end + + private + + def state + raw_data.state == 'closed' ? 'closed' : 'opened' + end + + def assigned? + raw_data.assignee.present? + end + + def author + @author ||= UserFormatter.new(client, raw_data.user) + end + + def author_id + @author_id ||= author.gitlab_id || project.creator_id + end + + def assignee + if assigned? + @assignee ||= UserFormatter.new(client, raw_data.assignee) + end + end + + def assignee_id + return @assignee_id if defined?(@assignee_id) + + @assignee_id = assignee.try(:gitlab_id) + end + + def body + raw_data.body || "" + end + + def description + if author.gitlab_id + body + else + formatter.author_line(author.login) + body + end + end + + def milestone + if raw_data.milestone.present? + milestone = MilestoneFormatter.new(project, raw_data.milestone) + project.milestones.find_by(milestone.find_condition) + end + end + end + end +end diff --git a/lib/gitlab/github_import/issue_formatter.rb b/lib/gitlab/github_import/issue_formatter.rb index 835ec858b35..6f5ac4dac0d 100644 --- a/lib/gitlab/github_import/issue_formatter.rb +++ b/lib/gitlab/github_import/issue_formatter.rb @@ -1,6 +1,6 @@ module Gitlab module GithubImport - class IssueFormatter < BaseFormatter + class IssueFormatter < IssuableFormatter def attributes { iid: number, @@ -12,7 +12,7 @@ module Gitlab author_id: author_id, assignee_id: assignee_id, created_at: raw_data.created_at, - updated_at: updated_at + updated_at: raw_data.updated_at } end @@ -20,58 +20,12 @@ module Gitlab raw_data.comments > 0 end - def klass - Issue + def project_association + :issues end - def number - raw_data.number - end - - def valid? - raw_data.pull_request.nil? - end - - private - - def assigned? - raw_data.assignee.present? - end - - def assignee_id - if assigned? - gl_user_id(raw_data.assignee.id) - end - end - - def author - raw_data.user.login - end - - def author_id - gl_user_id(raw_data.user.id) || project.creator_id - end - - def body - raw_data.body || "" - end - - def description - @formatter.author_line(author) + body - end - - def milestone - if raw_data.milestone.present? - project.milestones.find_by(iid: raw_data.milestone.number) - end - end - - def state - raw_data.state == 'closed' ? 'closed' : 'opened' - end - - def updated_at - state == 'closed' ? raw_data.closed_at : raw_data.updated_at + def pull_request? + raw_data.pull_request.present? end end end diff --git a/lib/gitlab/github_import/label_formatter.rb b/lib/gitlab/github_import/label_formatter.rb index 9f18244e7d7..211ccdc51bb 100644 --- a/lib/gitlab/github_import/label_formatter.rb +++ b/lib/gitlab/github_import/label_formatter.rb @@ -9,8 +9,18 @@ module Gitlab } end - def klass - Label + def project_association + :labels + end + + def create! + params = attributes.except(:project) + service = ::Labels::FindOrCreateService.new(nil, project, params) + label = service.execute(skip_authorization: true) + + raise ActiveRecord::RecordInvalid.new(label) unless label.persisted? + + label end private diff --git a/lib/gitlab/github_import/milestone_formatter.rb b/lib/gitlab/github_import/milestone_formatter.rb index 53d4b3102d1..dd782eff059 100644 --- a/lib/gitlab/github_import/milestone_formatter.rb +++ b/lib/gitlab/github_import/milestone_formatter.rb @@ -5,48 +5,36 @@ module Gitlab { iid: number, project: project, - title: title, - description: description, - due_date: due_date, + title: raw_data.title, + description: raw_data.description, + due_date: raw_data.due_on, state: state, - created_at: created_at, - updated_at: updated_at + created_at: raw_data.created_at, + updated_at: raw_data.updated_at } end - def klass - Milestone + def project_association + :milestones end - private - - def number - raw_data.number + def find_condition + { iid: number } end - def title - raw_data.title - end - - def description - raw_data.description + def number + if project.gitea_import? + raw_data.id + else + raw_data.number + end end - def due_date - raw_data.due_on - end + private def state raw_data.state == 'closed' ? 'closed' : 'active' end - - def created_at - raw_data.created_at - end - - def updated_at - state == 'closed' ? raw_data.closed_at : raw_data.updated_at - end end end end diff --git a/lib/gitlab/github_import/project_creator.rb b/lib/gitlab/github_import/project_creator.rb index f4221003db5..a55adc9b1c8 100644 --- a/lib/gitlab/github_import/project_creator.rb +++ b/lib/gitlab/github_import/project_creator.rb @@ -1,29 +1,52 @@ module Gitlab module GithubImport class ProjectCreator - attr_reader :repo, :namespace, :current_user, :session_data + include Gitlab::CurrentSettings - def initialize(repo, namespace, current_user, session_data) + attr_reader :repo, :name, :namespace, :current_user, :session_data, :type + + def initialize(repo, name, namespace, current_user, session_data, type: 'github') @repo = repo + @name = name @namespace = namespace @current_user = current_user @session_data = session_data + @type = type end def execute ::Projects::CreateService.new( current_user, - name: repo.name, - path: repo.name, + name: name, + path: name, description: repo.description, namespace_id: namespace.id, - visibility_level: repo.private ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::VisibilityLevel::PUBLIC, - import_type: "github", + visibility_level: visibility_level, + import_type: type, import_source: repo.full_name, - import_url: repo.clone_url.sub("https://", "https://#{@session_data[:github_access_token]}@"), - wiki_enabled: !repo.has_wiki? # If repo has wiki we'll import it later + import_url: import_url, + skip_wiki: skip_wiki ).execute end + + private + + def import_url + repo.clone_url.sub('://', "://#{session_data[:github_access_token]}@") + end + + def visibility_level + repo.private ? Gitlab::VisibilityLevel::PRIVATE : current_application_settings.default_project_visibility + end + + # + # If the GitHub project repository has wiki, we should not create the + # default wiki. Otherwise the GitHub importer will fail because the wiki + # repository already exist. + # + def skip_wiki + repo.has_wiki? + end end end end diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb index b84538a090a..4ea0200e89b 100644 --- a/lib/gitlab/github_import/pull_request_formatter.rb +++ b/lib/gitlab/github_import/pull_request_formatter.rb @@ -1,6 +1,6 @@ module Gitlab module GithubImport - class PullRequestFormatter < BaseFormatter + class PullRequestFormatter < IssuableFormatter delegate :exists?, :project, :ref, :repo, :sha, to: :source_branch, prefix: true delegate :exists?, :project, :ref, :repo, :sha, to: :target_branch, prefix: true @@ -20,16 +20,12 @@ module Gitlab author_id: author_id, assignee_id: assignee_id, created_at: raw_data.created_at, - updated_at: updated_at + updated_at: raw_data.updated_at } end - def klass - MergeRequest - end - - def number - raw_data.number + def project_association + :merge_requests end def valid? @@ -58,54 +54,11 @@ module Gitlab private - def assigned? - raw_data.assignee.present? - end - - def assignee_id - if assigned? - gl_user_id(raw_data.assignee.id) - end - end - - def author - raw_data.user.login - end - - def author_id - gl_user_id(raw_data.user.id) || project.creator_id - end - - def body - raw_data.body || "" - end - - def description - formatter.author_line(author) + body - end - - def milestone - if raw_data.milestone.present? - project.milestones.find_by(iid: raw_data.milestone.number) - end - end - def state - @state ||= if raw_data.state == 'closed' && raw_data.merged_at.present? - 'merged' - elsif raw_data.state == 'closed' - 'closed' - else - 'opened' - end - end - - def updated_at - case state - when 'merged' then raw_data.merged_at - when 'closed' then raw_data.closed_at + if raw_data.state == 'closed' && raw_data.merged_at.present? + 'merged' else - raw_data.updated_at + super end end end diff --git a/lib/gitlab/github_import/release_formatter.rb b/lib/gitlab/github_import/release_formatter.rb new file mode 100644 index 00000000000..1ad702a6058 --- /dev/null +++ b/lib/gitlab/github_import/release_formatter.rb @@ -0,0 +1,27 @@ +module Gitlab + module GithubImport + class ReleaseFormatter < BaseFormatter + def attributes + { + project: project, + tag: raw_data.tag_name, + description: raw_data.body, + created_at: raw_data.created_at, + updated_at: raw_data.created_at + } + end + + def project_association + :releases + end + + def find_condition + { tag: raw_data.tag_name } + end + + def valid? + !raw_data.draft + end + end + end +end diff --git a/lib/gitlab/github_import/user_formatter.rb b/lib/gitlab/github_import/user_formatter.rb new file mode 100644 index 00000000000..04c2964da20 --- /dev/null +++ b/lib/gitlab/github_import/user_formatter.rb @@ -0,0 +1,45 @@ +module Gitlab + module GithubImport + class UserFormatter + attr_reader :client, :raw + + delegate :id, :login, to: :raw, allow_nil: true + + def initialize(client, raw) + @client = client + @raw = raw + end + + def gitlab_id + return @gitlab_id if defined?(@gitlab_id) + + @gitlab_id = find_by_external_uid || find_by_email + end + + private + + def email + @email ||= client.user(raw.login).try(:email) + end + + def find_by_email + return nil unless email + + User.find_by_any_email(email) + .try(:id) + end + + def find_by_external_uid + return nil unless id + + identities = ::Identity.arel_table + + User.select(:id) + .joins(:identities).where(identities[:provider].eq(:github) + .and(identities[:extern_uid].eq(id))) + .first + .try(:id) + end + end + end +end diff --git a/lib/gitlab/gitlab_import/importer.rb b/lib/gitlab/gitlab_import/importer.rb index 46d40f75be6..e44d7934fda 100644 --- a/lib/gitlab/gitlab_import/importer.rb +++ b/lib/gitlab/gitlab_import/importer.rb @@ -41,7 +41,8 @@ module Gitlab title: issue["title"], state: issue["state"], updated_at: issue["updated_at"], - author_id: gl_user_id(project, issue["author"]["id"]) + author_id: gitlab_user_id(project, issue["author"]["id"]), + confidential: issue["confidential"] ) end end @@ -51,7 +52,7 @@ module Gitlab private - def gl_user_id(project, gitlab_id) + def gitlab_user_id(project, gitlab_id) user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'gitlab'", gitlab_id.to_s) (user && user.id) || project.creator_id end diff --git a/lib/gitlab/gitorious_import.rb b/lib/gitlab/gitorious_import.rb deleted file mode 100644 index 8d0132a744c..00000000000 --- a/lib/gitlab/gitorious_import.rb +++ /dev/null @@ -1,5 +0,0 @@ -module Gitlab - module GitoriousImport - GITORIOUS_HOST = "https://gitorious.org" - end -end diff --git a/lib/gitlab/gitorious_import/client.rb b/lib/gitlab/gitorious_import/client.rb deleted file mode 100644 index 99fe5bdebfc..00000000000 --- a/lib/gitlab/gitorious_import/client.rb +++ /dev/null @@ -1,29 +0,0 @@ -module Gitlab - module GitoriousImport - class Client - attr_reader :repo_list - - def initialize(repo_list) - @repo_list = repo_list - end - - def authorize_url(redirect_uri) - "#{GITORIOUS_HOST}/gitlab-import?callback_url=#{redirect_uri}" - end - - def repos - @repos ||= repo_names.map { |full_name| GitoriousImport::Repository.new(full_name) } - end - - def repo(id) - repos.find { |repo| repo.id == id } - end - - private - - def repo_names - repo_list.to_s.split(',').map(&:strip).reject(&:blank?) - end - end - end -end diff --git a/lib/gitlab/gitorious_import/project_creator.rb b/lib/gitlab/gitorious_import/project_creator.rb deleted file mode 100644 index 8e22aa9286d..00000000000 --- a/lib/gitlab/gitorious_import/project_creator.rb +++ /dev/null @@ -1,27 +0,0 @@ -module Gitlab - module GitoriousImport - class ProjectCreator - attr_reader :repo, :namespace, :current_user - - def initialize(repo, namespace, current_user) - @repo = repo - @namespace = namespace - @current_user = current_user - end - - def execute - ::Projects::CreateService.new( - current_user, - name: repo.name, - path: repo.path, - description: repo.description, - namespace_id: namespace.id, - visibility_level: Gitlab::VisibilityLevel::PUBLIC, - import_type: "gitorious", - import_source: repo.full_name, - import_url: repo.import_url - ).execute - end - end - end -end diff --git a/lib/gitlab/gitorious_import/repository.rb b/lib/gitlab/gitorious_import/repository.rb deleted file mode 100644 index c88f1ae358d..00000000000 --- a/lib/gitlab/gitorious_import/repository.rb +++ /dev/null @@ -1,35 +0,0 @@ -module Gitlab - module GitoriousImport - Repository = Struct.new(:full_name) do - def id - Digest::SHA1.hexdigest(full_name) - end - - def namespace - segments.first - end - - def path - segments.last - end - - def name - path.titleize - end - - def description - "" - end - - def import_url - "#{GITORIOUS_HOST}/#{full_name}.git" - end - - private - - def segments - full_name.split('/') - end - end - end -end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index c5a11148d33..101b1b80c1e 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -2,16 +2,18 @@ module Gitlab module GonHelper def add_gon_variables gon.api_version = API::API.version - gon.default_avatar_url = URI::join(Gitlab.config.gitlab.url, ActionController::Base.helpers.image_path('no_avatar.png')).to_s + gon.default_avatar_url = URI.join(Gitlab.config.gitlab.url, ActionController::Base.helpers.image_path('no_avatar.png')).to_s gon.max_file_size = current_application_settings.max_attachment_size gon.relative_url_root = Gitlab.config.gitlab.relative_url_root gon.shortcuts_path = help_page_path('shortcuts') gon.user_color_scheme = Gitlab::ColorSchemes.for_user(current_user).css_class gon.award_menu_url = emojis_path + gon.katex_css_url = ActionController::Base.helpers.asset_path('katex.css') + gon.katex_js_url = ActionController::Base.helpers.asset_path('katex.js') if current_user gon.current_user_id = current_user.id - gon.api_token = current_user.private_token + gon.current_username = current_user.username end end end diff --git a/lib/gitlab/google_code_import/importer.rb b/lib/gitlab/google_code_import/importer.rb index 62da327931f..b02b9737493 100644 --- a/lib/gitlab/google_code_import/importer.rb +++ b/lib/gitlab/google_code_import/importer.rb @@ -92,19 +92,17 @@ module Gitlab end issue = Issue.create!( - project_id: project.id, - title: raw_issue["title"], - description: body, - author_id: project.creator_id, - assignee_id: assignee_id, - state: raw_issue["state"] == "closed" ? "closed" : "opened" + iid: raw_issue['id'], + project_id: project.id, + title: raw_issue['title'], + description: body, + author_id: project.creator_id, + assignee_id: assignee_id, + state: raw_issue['state'] == 'closed' ? 'closed' : 'opened' ) - issue.add_labels_by_names(labels) - - if issue.iid != raw_issue["id"] - issue.update_attribute(:iid, raw_issue["id"]) - end + issue_labels = ::LabelsFinder.new(nil, project_id: project.id, title: labels).execute(skip_authorization: true) + issue.update_attribute(:label_ids, issue_labels.pluck(:id)) import_issue_comments(issue, comments) end @@ -236,8 +234,8 @@ module Gitlab end def create_label(name) - color = nice_label_color(name) - Label.create!(project_id: project.id, name: name, color: color) + params = { name: name, color: nice_label_color(name) } + ::Labels::FindOrCreateService.new(nil, project, params).execute(skip_authorization: true) end def format_content(raw_content) @@ -312,7 +310,7 @@ module Gitlab if name == project.import_source "##{id}" else - "#{project.namespace.path}/#{name}##{id}" + "#{project.namespace.full_path}/#{name}##{id}" end text = "~~#{text}~~" if deleted text diff --git a/lib/gitlab/identifier.rb b/lib/gitlab/identifier.rb index 3e5d728f3bc..94678b6ec40 100644 --- a/lib/gitlab/identifier.rb +++ b/lib/gitlab/identifier.rb @@ -5,19 +5,59 @@ module Gitlab def identify(identifier, project, newrev) if identifier.blank? # Local push from gitlab - email = project.commit(newrev).author_email rescue nil - User.find_by(email: email) if email - + identify_using_commit(project, newrev) elsif identifier =~ /\Auser-\d+\Z/ # git push over http - user_id = identifier.gsub("user-", "") - User.find_by(id: user_id) - + identify_using_user(identifier) elsif identifier =~ /\Akey-\d+\Z/ # git push over ssh - key_id = identifier.gsub("key-", "") - Key.find_by(id: key_id).try(:user) + identify_using_ssh_key(identifier) + end + end + + # Tries to identify a user based on a commit SHA. + def identify_using_commit(project, ref) + commit = project.commit(ref) + + return if !commit || !commit.author_email + + identify_with_cache(:email, commit.author_email) do + commit.author + end + end + + # Tries to identify a user based on a user identifier (e.g. "user-123"). + def identify_using_user(identifier) + user_id = identifier.gsub("user-", "") + + identify_with_cache(:user, user_id) do + User.find_by(id: user_id) + end + end + + # Tries to identify a user based on an SSH key identifier (e.g. "key-123"). + def identify_using_ssh_key(identifier) + key_id = identifier.gsub("key-", "") + + identify_with_cache(:ssh_key, key_id) do + User.find_by_ssh_key_id(key_id) end end + + def identify_with_cache(category, key) + if identification_cache[category].key?(key) + identification_cache[category][key] + else + identification_cache[category][key] = yield + end + end + + def identification_cache + @identification_cache ||= { + email: {}, + user: {}, + ssh_key: {} + } + end end end diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index bb562bdcd2c..8b327cfc226 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -2,7 +2,8 @@ module Gitlab module ImportExport extend self - VERSION = '0.1.3' + # For every version update, the version history in import_export.md has to be kept up to date. + VERSION = '0.1.6'.freeze FILENAME_LIMIT = 50 def export_path(relative_path:) @@ -34,7 +35,7 @@ module Gitlab end def export_filename(project:) - basename = "#{Time.now.strftime('%Y-%m-%d_%H-%M-%3N')}_#{project.namespace.path}_#{project.path}" + basename = "#{Time.now.strftime('%Y-%m-%d_%H-%M-%3N')}_#{project.full_path.tr('/', '_')}" "#{basename[0..FILENAME_LIMIT]}_export.tar.gz" end diff --git a/lib/gitlab/import_export/attribute_cleaner.rb b/lib/gitlab/import_export/attribute_cleaner.rb new file mode 100644 index 00000000000..34169319b26 --- /dev/null +++ b/lib/gitlab/import_export/attribute_cleaner.rb @@ -0,0 +1,28 @@ +module Gitlab + module ImportExport + class AttributeCleaner + ALLOWED_REFERENCES = RelationFactory::PROJECT_REFERENCES + RelationFactory::USER_REFERENCES + ['group_id'] + + def self.clean(*args) + new(*args).clean + end + + def initialize(relation_hash:, relation_class:) + @relation_hash = relation_hash + @relation_class = relation_class + end + + def clean + @relation_hash.reject do |key, _value| + prohibited_key?(key) || !@relation_class.attribute_method?(key) + end.except('id') + end + + private + + def prohibited_key?(key) + key.end_with?('_id') && !ALLOWED_REFERENCES.include?(key) + end + end + end +end diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index e522a0fc8f6..90942774a2e 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -1,6 +1,8 @@ module Gitlab module ImportExport module CommandLineUtil + DEFAULT_MODE = 0700 + def tar_czf(archive:, dir:) tar_with_options(archive: archive, dir: dir, options: 'czf') end @@ -13,12 +15,9 @@ module Gitlab execute(%W(#{git_bin_path} --git-dir=#{repo_path} bundle create #{bundle_path} --all)) end - def git_unbundle(repo_path:, bundle_path:) - execute(%W(#{git_bin_path} clone --bare #{bundle_path} #{repo_path})) - end - - def git_restore_hooks - execute(%W(#{Gitlab.config.gitlab_shell.path}/bin/create-hooks) + repository_storage_paths_args) + def mkdir_p(path) + FileUtils.mkdir_p(path, mode: DEFAULT_MODE) + FileUtils.chmod(DEFAULT_MODE, path) end private @@ -45,14 +44,10 @@ module Gitlab # if we are copying files, create the destination folder destination_folder = File.file?(source) ? File.dirname(destination) : destination - FileUtils.mkdir_p(destination_folder) + mkdir_p(destination_folder) FileUtils.copy_entry(source, destination) true end - - def repository_storage_paths_args - Gitlab.config.repositories.storages.values - end end end end diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index eca6e5b6d51..ffd17118c91 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -15,7 +15,7 @@ module Gitlab end def import - FileUtils.mkdir_p(@shared.export_path) + mkdir_p(@shared.export_path) wait_for_archived_file do decompress_archive @@ -43,6 +43,14 @@ module Gitlab raise Projects::ImportService::Error.new("Unable to decompress #{@archive_file} into #{@shared.export_path}") unless result + remove_symlinks! + end + + def remove_symlinks! + Dir["#{@shared.export_path}/**/*"].each do |path| + FileUtils.rm(path) if File.lstat(path).symlink? + end + true end end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 1da51043611..416194e57d7 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -1,15 +1,22 @@ # Model relationships to be included in the project import/export project_tree: + - labels: + :priorities + - milestones: + - :events - issues: - :events + - :timelogs - notes: - :author - :events - label_links: - - :label + - label: + :priorities - milestone: - :events - snippets: + - :award_emoji - notes: :author - :releases @@ -21,8 +28,10 @@ project_tree: - :events - :merge_request_diff - :events + - :timelogs - label_links: - - :label + - label: + :priorities - milestone: - :events - pipelines: @@ -30,24 +39,19 @@ project_tree: - :author - :events - :statuses - - :variables - :triggers - :deploy_keys - :services - :hooks - - :protected_branches - - :labels - - milestones: - - :events + - protected_branches: + - :merge_access_levels + - :push_access_levels + - :project_feature # Only include the following attributes for the models specified. included_attributes: project: - :description - - :issues_enabled - - :merge_requests_enabled - - :wiki_enabled - - :snippets_enabled - :visibility_level - :archived user: @@ -67,9 +71,17 @@ excluded_attributes: - :milestone_id merge_requests: - :milestone_id + award_emoji: + - :awardable_id methods: + labels: + - :type + label: + - :type statuses: - :type + services: + - :type merge_request_diff: - - :utf8_st_diffs
\ No newline at end of file + - :utf8_st_diffs diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index e9ee47fc090..063ce74ecad 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -56,7 +56,7 @@ module Gitlab end def path_with_namespace - File.join(@project.namespace.path, @project.path) + File.join(@project.namespace.full_path, @project.path) end def repo_path diff --git a/lib/gitlab/import_export/json_hash_builder.rb b/lib/gitlab/import_export/json_hash_builder.rb index 008300bde45..48c09dafcb6 100644 --- a/lib/gitlab/import_export/json_hash_builder.rb +++ b/lib/gitlab/import_export/json_hash_builder.rb @@ -57,22 +57,25 @@ module Gitlab # +value+ existing model to be included in the hash # +json_config_hash+ the original hash containing the root model def create_model_value(current_key, value, json_config_hash) - parsed_hash = { include: value } - parse_hash(value, parsed_hash) - - json_config_hash[current_key] = parsed_hash + json_config_hash[current_key] = parse_hash(value) || { include: value } end # Calls attributes finder to parse the hash and add any attributes to it # # +value+ existing model to be included in the hash # +parsed_hash+ the original hash - def parse_hash(value, parsed_hash) + def parse_hash(value) + return nil if already_contains_methods?(value) + @attributes_finder.parse(value) do |hash| - parsed_hash = { include: hash_or_merge(value, hash) } + { include: hash_or_merge(value, hash) } end end + def already_contains_methods?(value) + value.is_a?(Hash) && value.values.detect { |val| val[:methods]} + end + # Adds new model configuration to an existing hash with key +current_key+ # It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+ # diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index 36c4cf6efa0..8b8e48aac76 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -1,13 +1,10 @@ module Gitlab module ImportExport class MembersMapper - attr_reader :missing_author_ids - def initialize(exported_members:, user:, project:) - @exported_members = exported_members + @exported_members = user.admin? ? exported_members : [] @user = user @project = project - @missing_author_ids = [] # This needs to run first, as second call would be from #map # which means project members already exist. @@ -35,16 +32,21 @@ module Gitlab @user.id end + def include?(old_author_id) + map.keys.include?(old_author_id) && map[old_author_id] != default_user_id + end + private def missing_keys_tracking_hash Hash.new do |_, key| - @missing_author_ids << key default_user_id end end def ensure_default_member! + @project.project_members.destroy_all + ProjectMember.create!(user: @user, access_level: ProjectMember::MASTER, source_id: @project.id, importing: true) end @@ -55,11 +57,16 @@ module Gitlab end def member_hash(member) - member.except('id').merge(source_id: @project.id, importing: true) + parsed_hash(member).merge('source_id' => @project.id, 'importing' => true) + end + + def parsed_hash(member) + Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: member.deep_stringify_keys, + relation_class: ProjectMember) end def find_project_user_query(member) - user_arel[:username].eq(member['user']['username']).or(user_arel[:email].eq(member['user']['email'])) + user_arel[:email].eq(member['user']['email']).or(user_arel[:username].eq(member['user']['username'])) end def user_arel diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index c7b3551b84c..cda6ddf0443 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -9,8 +9,14 @@ module Gitlab end def restore - json = IO.read(@path) - @tree_hash = ActiveSupport::JSON.decode(json) + begin + json = IO.read(@path) + @tree_hash = ActiveSupport::JSON.decode(json) + rescue => e + Rails.logger.error("Import/Export error: #{e.message}") + raise Gitlab::ImportExport::Error.new('Incorrect JSON format') + end + @project_members = @tree_hash.delete('project_members') ActiveRecord::Base.no_touching do @@ -61,11 +67,17 @@ module Gitlab def restore_project return @project unless @tree_hash - project_params = @tree_hash.reject { |_key, value| value.is_a?(Array) } @project.update(project_params) @project end + def project_params + @tree_hash.reject do |key, value| + # return params that are not 1 to many or 1 to 1 relations + value.is_a?(Array) || key == key.singularize + end + end + # Given a relation hash containing one or more models and its relationships, # loops through each model and each object from a model type and # and assigns its correspondent attributes hash from +tree_hash+ @@ -104,13 +116,18 @@ module Gitlab def create_relation(relation, relation_hash_list) relation_array = [relation_hash_list].flatten.map do |relation_hash| Gitlab::ImportExport::RelationFactory.create(relation_sym: relation.to_sym, - relation_hash: relation_hash.merge('project_id' => restored_project.id), + relation_hash: parsed_relation_hash(relation_hash), members_mapper: members_mapper, - user: @user) - end + user: @user, + project_id: restored_project.id) + end.compact relation_hash_list.is_a?(Array) ? relation_array : relation_array.first end + + def parsed_relation_hash(relation_hash) + relation_hash.merge!('group_id' => restored_project.group.try(:id), 'project_id' => restored_project.id) + end end end end diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb index 9153088e966..b79be62245b 100644 --- a/lib/gitlab/import_export/project_tree_saver.rb +++ b/lib/gitlab/import_export/project_tree_saver.rb @@ -1,16 +1,19 @@ module Gitlab module ImportExport class ProjectTreeSaver + include Gitlab::ImportExport::CommandLineUtil + attr_reader :full_path - def initialize(project:, shared:) + def initialize(project:, current_user:, shared:) @project = project + @current_user = current_user @shared = shared @full_path = File.join(@shared.export_path, ImportExport.project_filename) end def save - FileUtils.mkdir_p(@shared.export_path) + mkdir_p(@shared.export_path) File.write(full_path, project_json_tree) true @@ -22,7 +25,29 @@ module Gitlab private def project_json_tree - @project.to_json(Gitlab::ImportExport::Reader.new(shared: @shared).project_tree) + project_json['project_members'] += group_members_json + + project_json.to_json + end + + def project_json + @project_json ||= @project.as_json(reader.project_tree) + end + + def reader + @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared) + end + + def group_members_json + group_members.as_json(reader.group_members_tree).each do |group_member| + group_member['source_type'] = 'Project' # Make group members project members of the future import + end + end + + def group_members + return [] unless @current_user.can?(:admin_group, @project.group) + + MembersFinder.new(@project.project_members, @project.group).execute(@current_user) end end end diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb index 5021a1a14ce..a1e7159fe42 100644 --- a/lib/gitlab/import_export/reader.rb +++ b/lib/gitlab/import_export/reader.rb @@ -21,6 +21,10 @@ module Gitlab false end + def group_members_tree + @attributes_finder.find_included(:project_members).merge(include: @attributes_finder.find(:user)) + end + private # Builds a hash in the format described here: http://api.rubyonrails.org/classes/ActiveModel/Serializers/JSON.html diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index b0726268ca6..fae792237d9 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -4,26 +4,34 @@ module Gitlab OVERRIDES = { snippets: :project_snippets, pipelines: 'Ci::Pipeline', statuses: 'commit_status', - variables: 'Ci::Variable', triggers: 'Ci::Trigger', builds: 'Ci::Build', - hooks: 'ProjectHook' }.freeze + hooks: 'ProjectHook', + merge_access_levels: 'ProtectedBranch::MergeAccessLevel', + push_access_levels: 'ProtectedBranch::PushAccessLevel', + labels: :project_labels, + priorities: :label_priorities, + label: :project_label }.freeze - USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id].freeze + USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id created_by_id merge_user_id resolved_by_id].freeze + + PROJECT_REFERENCES = %w[project_id source_project_id gl_project_id target_project_id].freeze BUILD_MODELS = %w[Ci::Build commit_status].freeze IMPORTED_OBJECT_MAX_RETRIES = 5.freeze - EXISTING_OBJECT_CHECK = %i[milestone milestones label labels].freeze + EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels group_label group_labels].freeze + + TOKEN_RESET_MODELS = %w[Ci::Trigger Ci::Build ProjectHook].freeze def self.create(*args) new(*args).create end - def initialize(relation_sym:, relation_hash:, members_mapper:, user:) + def initialize(relation_sym:, relation_hash:, members_mapper:, user:, project_id:) @relation_name = OVERRIDES[relation_sym] || relation_sym - @relation_hash = relation_hash.except('id', 'noteable_id') + @relation_hash = relation_hash.except('noteable_id').merge('project_id' => project_id) @members_mapper = members_mapper @user = user @imported_object_retries = 0 @@ -33,6 +41,8 @@ module Gitlab # the relation_hash, updating references with new object IDs, mapping users using # the "members_mapper" object, also updating notes if required. def create + return nil if unknown_service? + setup_models generate_imported_object @@ -50,7 +60,11 @@ module Gitlab update_user_references update_project_references - reset_ci_tokens if @relation_name == 'Ci::Trigger' + + handle_group_label if group_label? + reset_tokens! + remove_encrypted_attributes! + @relation_hash['data'].deep_symbolize_keys! if @relation_name == :events && @relation_hash['data'] set_st_diffs if @relation_name == :merge_request_diff end @@ -69,17 +83,13 @@ module Gitlab # is left. def set_note_author old_author_id = @relation_hash['author_id'] - - # Users with admin access can map users - @relation_hash['author_id'] = admin_user? ? @members_mapper.map[old_author_id] : @members_mapper.default_user_id - author = @relation_hash.delete('author') - update_note_for_missing_author(author['name']) if missing_author?(old_author_id) + update_note_for_missing_author(author['name']) unless has_author?(old_author_id) end - def missing_author?(old_author_id) - !admin_user? || @members_mapper.missing_author_ids.include?(old_author_id) + def has_author?(old_author_id) + admin_user? && @members_mapper.include?(old_author_id) end def missing_author_note(updated_at, author_name) @@ -90,6 +100,8 @@ module Gitlab def generate_imported_object if BUILD_MODELS.include?(@relation_name) # call #trace= method after assigning the other attributes trace = @relation_hash.delete('trace') + @relation_hash.delete('token') + imported_object do |object| object.trace = trace object.commit_id = nil @@ -117,11 +129,36 @@ module Gitlab @relation_hash['target_project_id'] && @relation_hash['target_project_id'] == @relation_hash['source_project_id'] end - def reset_ci_tokens - return unless Gitlab::ImportExport.reset_tokens? + def group_label? + @relation_hash['type'] == 'GroupLabel' + end + + def handle_group_label + # If there's no group, move the label to a project label + if @relation_hash['group_id'] + @relation_hash['project_id'] = nil + @relation_name = :group_label + else + @relation_hash['type'] = 'ProjectLabel' + end + end + + def reset_tokens! + return unless Gitlab::ImportExport.reset_tokens? && TOKEN_RESET_MODELS.include?(@relation_name.to_s) # If we import/export a project to the same instance, tokens will have to be reset. - @relation_hash['token'] = nil + # We also have to reset them to avoid issues when the gitlab secrets file cannot be copied across. + relation_class.attribute_names.select { |name| name.include?('token') }.each do |token| + @relation_hash[token] = nil + end + end + + def remove_encrypted_attributes! + return unless relation_class.respond_to?(:encrypted_attributes) && relation_class.encrypted_attributes.any? + + relation_class.encrypted_attributes.each_key do |key| + @relation_hash[key.to_s] = nil + end end def relation_class @@ -149,7 +186,8 @@ module Gitlab end def parsed_relation_hash - @relation_hash.reject { |k, _v| !relation_class.attribute_method?(k) } + @parsed_relation_hash ||= Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: @relation_hash, + relation_class: relation_class) end def set_st_diffs @@ -161,14 +199,59 @@ module Gitlab # Otherwise always create the record, skipping the extra SELECT clause. @existing_or_new_object ||= begin if EXISTING_OBJECT_CHECK.include?(@relation_name) - existing_object = relation_class.find_or_initialize_by(parsed_relation_hash.slice('title', 'project_id')) - existing_object.assign_attributes(parsed_relation_hash) + attribute_hash = attribute_hash_for(['events']) + + existing_object.assign_attributes(attribute_hash) if attribute_hash.any? + existing_object else relation_class.new(parsed_relation_hash) end end end + + def attribute_hash_for(attributes) + attributes.inject({}) do |hash, value| + hash[value] = parsed_relation_hash.delete(value) if parsed_relation_hash[value] + hash + end + end + + def existing_object + @existing_object ||= + begin + existing_object = find_or_create_object! + + # Done in two steps, as MySQL behaves differently than PostgreSQL using + # the +find_or_create_by+ method and does not return the ID the second time. + existing_object.update!(parsed_relation_hash) + existing_object + end + end + + def unknown_service? + @relation_name == :services && parsed_relation_hash['type'] && + !Object.const_defined?(parsed_relation_hash['type']) + end + + def find_or_create_object! + finder_attributes = @relation_name == :group_label ? %w[title group_id] : %w[title project_id] + finder_hash = parsed_relation_hash.slice(*finder_attributes) + + if label? + label = relation_class.find_or_initialize_by(finder_hash) + parsed_relation_hash.delete('priorities') if label.persisted? + + label.save! + label + else + relation_class.find_or_create_by(finder_hash) + end + end + + def label? + @relation_name.to_s.include?('label') + end end end end diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index 6d9379acf25..c824d3ea9fc 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -2,6 +2,7 @@ module Gitlab module ImportExport class RepoRestorer include Gitlab::ImportExport::CommandLineUtil + include Gitlab::ShellAdapter def initialize(project:, shared:, path_to_bundle:) @project = project @@ -12,33 +13,11 @@ module Gitlab def restore return true unless File.exist?(@path_to_bundle) - FileUtils.mkdir_p(path_to_repo) - - git_unbundle(repo_path: path_to_repo, bundle_path: @path_to_bundle) && repo_restore_hooks + gitlab_shell.import_repository(@project.repository_storage_path, @project.path_with_namespace, @path_to_bundle) rescue => e @shared.error(e) false end - - private - - def repos_path - Gitlab.config.gitlab_shell.repos_path - end - - def path_to_repo - @project.repository.path_to_repo - end - - def repo_restore_hooks - return true if wiki? - - git_restore_hooks - end - - def wiki? - @project.class.name == 'ProjectWiki' - end end end end diff --git a/lib/gitlab/import_export/repo_saver.rb b/lib/gitlab/import_export/repo_saver.rb index 331e14021e6..a7028a32570 100644 --- a/lib/gitlab/import_export/repo_saver.rb +++ b/lib/gitlab/import_export/repo_saver.rb @@ -20,7 +20,7 @@ module Gitlab private def bundle_to_disk - FileUtils.mkdir_p(@shared.export_path) + mkdir_p(@shared.export_path) git_bundle(repo_path: path_to_repo, bundle_path: @full_path) rescue => e @shared.error(e) diff --git a/lib/gitlab/import_export/version_checker.rb b/lib/gitlab/import_export/version_checker.rb index de3fe6d822e..bd3c3ee3b2f 100644 --- a/lib/gitlab/import_export/version_checker.rb +++ b/lib/gitlab/import_export/version_checker.rb @@ -24,12 +24,19 @@ module Gitlab end def verify_version!(version) - if Gem::Version.new(version) > Gem::Version.new(Gitlab::ImportExport.version) - raise Gitlab::ImportExport::Error.new("Import version mismatch: Required <= #{Gitlab::ImportExport.version} but was #{version}") + if different_version?(version) + raise Gitlab::ImportExport::Error.new("Import version mismatch: Required #{Gitlab::ImportExport.version} but was #{version}") else true end end + + def different_version?(version) + Gem::Version.new(version) != Gem::Version.new(Gitlab::ImportExport.version) + rescue => e + Rails.logger.error("Import/Export error: #{e.message}") + raise Gitlab::ImportExport::Error.new('Incorrect VERSION format') + end end end end diff --git a/lib/gitlab/import_export/version_saver.rb b/lib/gitlab/import_export/version_saver.rb index 9b642d740b7..7cf88298642 100644 --- a/lib/gitlab/import_export/version_saver.rb +++ b/lib/gitlab/import_export/version_saver.rb @@ -1,12 +1,14 @@ module Gitlab module ImportExport class VersionSaver + include Gitlab::ImportExport::CommandLineUtil + def initialize(shared:) @shared = shared end def save - FileUtils.mkdir_p(@shared.export_path) + mkdir_p(@shared.export_path) File.write(version_file, Gitlab::ImportExport.version, mode: 'w') rescue => e diff --git a/lib/gitlab/import_export/wiki_repo_saver.rb b/lib/gitlab/import_export/wiki_repo_saver.rb index 6107420e4dd..1e6722a7bba 100644 --- a/lib/gitlab/import_export/wiki_repo_saver.rb +++ b/lib/gitlab/import_export/wiki_repo_saver.rb @@ -9,7 +9,7 @@ module Gitlab end def bundle_to_disk(full_path) - FileUtils.mkdir_p(@shared.export_path) + mkdir_p(@shared.export_path) git_bundle(repo_path: path_to_repo, bundle_path: full_path) rescue => e @shared.error(e) diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index 59a05411fe9..52276cbcd9a 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -5,24 +5,38 @@ # module Gitlab module ImportSources - extend CurrentSettings + ImportSource = Struct.new(:name, :title, :importer) + + ImportTable = [ + ImportSource.new('github', 'GitHub', Gitlab::GithubImport::Importer), + ImportSource.new('bitbucket', 'Bitbucket', Gitlab::BitbucketImport::Importer), + ImportSource.new('gitlab', 'GitLab.com', Gitlab::GitlabImport::Importer), + ImportSource.new('google_code', 'Google Code', Gitlab::GoogleCodeImport::Importer), + ImportSource.new('fogbugz', 'FogBugz', Gitlab::FogbugzImport::Importer), + ImportSource.new('git', 'Repo by URL', nil), + ImportSource.new('gitlab_project', 'GitLab export', Gitlab::ImportExport::Importer), + ImportSource.new('gitea', 'Gitea', Gitlab::GithubImport::Importer) + ].freeze class << self + def options + @options ||= Hash[ImportTable.map { |importer| [importer.title, importer.name] }] + end + def values - options.values + @values ||= ImportTable.map(&:name) end - def options - { - 'GitHub' => 'github', - 'Bitbucket' => 'bitbucket', - 'GitLab.com' => 'gitlab', - 'Gitorious.org' => 'gitorious', - 'Google Code' => 'google_code', - 'FogBugz' => 'fogbugz', - 'Repo by URL' => 'git', - 'GitLab export' => 'gitlab_project' - } + def importer_names + @importer_names ||= ImportTable.select(&:importer).map(&:name) + end + + def importer(name) + ImportTable.find { |import_source| import_source.name == name }.importer + end + + def title(name) + options.key(name) end end end diff --git a/lib/gitlab/incoming_email.rb b/lib/gitlab/incoming_email.rb index d7be50bd437..c9122a23568 100644 --- a/lib/gitlab/incoming_email.rb +++ b/lib/gitlab/incoming_email.rb @@ -1,14 +1,27 @@ module Gitlab module IncomingEmail - class << self - FALLBACK_MESSAGE_ID_REGEX = /\Areply\-(.+)@#{Gitlab.config.gitlab.host}\Z/.freeze + UNSUBSCRIBE_SUFFIX = '+unsubscribe'.freeze + WILDCARD_PLACEHOLDER = '%{key}'.freeze + class << self def enabled? config.enabled && config.address end + def supports_wildcard? + config.address && config.address.include?(WILDCARD_PLACEHOLDER) + end + + def supports_issue_creation? + enabled? && supports_wildcard? + end + def reply_address(key) - config.address.gsub('%{key}', key) + config.address.sub(WILDCARD_PLACEHOLDER, key) + end + + def unsubscribe_address(key) + config.address.sub(WILDCARD_PLACEHOLDER, "#{key}#{UNSUBSCRIBE_SUFFIX}") end def key_from_address(address) @@ -22,10 +35,14 @@ module Gitlab end def key_from_fallback_message_id(mail_id) - match = mail_id.match(FALLBACK_MESSAGE_ID_REGEX) - return unless match + message_id_regexp = /\Areply\-(.+)@#{Gitlab.config.gitlab.host}\z/ - match[1] + mail_id[message_id_regexp, 1] + end + + def scan_fallback_references(references) + # It's looking for each <...> + references.scan(/(?!<)[^<>]+(?=>)/) end def config @@ -39,7 +56,7 @@ module Gitlab return nil unless wildcard_address regex = Regexp.escape(wildcard_address) - regex = regex.gsub(Regexp.escape('%{key}'), "(.+)") + regex = regex.sub(Regexp.escape(WILDCARD_PLACEHOLDER), '(.+)') Regexp.new(regex).freeze end end diff --git a/lib/gitlab/issues_labels.rb b/lib/gitlab/issues_labels.rb index 1bec6088292..b8ca7f2f55f 100644 --- a/lib/gitlab/issues_labels.rb +++ b/lib/gitlab/issues_labels.rb @@ -18,8 +18,8 @@ module Gitlab { title: "enhancement", color: green } ] - labels.each do |label| - project.labels.create(label) + labels.each do |params| + ::Labels::FindOrCreateService.new(nil, project, params).execute(skip_authorization: true) end end end diff --git a/lib/gitlab/job_waiter.rb b/lib/gitlab/job_waiter.rb new file mode 100644 index 00000000000..8db91d25a4b --- /dev/null +++ b/lib/gitlab/job_waiter.rb @@ -0,0 +1,27 @@ +module Gitlab + # JobWaiter can be used to wait for a number of Sidekiq jobs to complete. + class JobWaiter + # The sleep interval between checking keys, in seconds. + INTERVAL = 0.1 + + # jobs - The job IDs to wait for. + def initialize(jobs) + @jobs = jobs + end + + # Waits for all the jobs to be completed. + # + # timeout - The maximum amount of seconds to block the caller for. This + # ensures we don't indefinitely block a caller in case a job takes + # long to process, or is never processed. + def wait(timeout = 60) + start = Time.current + + while (Time.current - start) <= timeout + break if SidekiqStatus.all_completed?(@jobs) + + sleep(INTERVAL) # to not overload Redis too much. + end + end + end +end diff --git a/lib/gitlab/kubernetes.rb b/lib/gitlab/kubernetes.rb new file mode 100644 index 00000000000..3a7af363548 --- /dev/null +++ b/lib/gitlab/kubernetes.rb @@ -0,0 +1,80 @@ +module Gitlab + # Helper methods to do with Kubernetes network services & resources + module Kubernetes + # This is the comand that is run to start a terminal session. Kubernetes + # expects `command=foo&command=bar, not `command[]=foo&command[]=bar` + EXEC_COMMAND = URI.encode_www_form( + ['sh', '-c', 'bash || sh'].map { |value| ['command', value] } + ) + + # Filters an array of pods (as returned by the kubernetes API) by their labels + def filter_pods(pods, labels = {}) + pods.select do |pod| + metadata = pod.fetch("metadata", {}) + pod_labels = metadata.fetch("labels", nil) + next unless pod_labels + + labels.all? { |k, v| pod_labels[k.to_s] == v } + end + end + + # Converts a pod (as returned by the kubernetes API) into a terminal + def terminals_for_pod(api_url, namespace, pod) + metadata = pod.fetch("metadata", {}) + status = pod.fetch("status", {}) + spec = pod.fetch("spec", {}) + + containers = spec["containers"] + pod_name = metadata["name"] + phase = status["phase"] + + return unless containers.present? && pod_name.present? && phase == "Running" + + created_at = DateTime.parse(metadata["creationTimestamp"]) rescue nil + + containers.map do |container| + { + selectors: { pod: pod_name, container: container["name"] }, + url: container_exec_url(api_url, namespace, pod_name, container["name"]), + subprotocols: ['channel.k8s.io'], + headers: Hash.new { |h, k| h[k] = [] }, + created_at: created_at, + } + end + end + + def add_terminal_auth(terminal, token:, max_session_time:, ca_pem: nil) + terminal[:headers]['Authorization'] << "Bearer #{token}" + terminal[:max_session_time] = max_session_time + terminal[:ca_pem] = ca_pem if ca_pem.present? + end + + def container_exec_url(api_url, namespace, pod_name, container_name) + url = URI.parse(api_url) + url.path = [ + url.path.sub(%r{/+\z}, ''), + 'api', 'v1', + 'namespaces', ERB::Util.url_encode(namespace), + 'pods', ERB::Util.url_encode(pod_name), + 'exec' + ].join('/') + + url.query = { + container: container_name, + tty: true, + stdin: true, + stdout: true, + stderr: true, + }.to_query + '&' + EXEC_COMMAND + + case url.scheme + when 'http' + url.scheme = 'ws' + when 'https' + url.scheme = 'wss' + end + + url.to_s + end + end +end diff --git a/lib/gitlab/ldap/access.rb b/lib/gitlab/ldap/access.rb index 2f326d00a2f..54a5b1d31cd 100644 --- a/lib/gitlab/ldap/access.rb +++ b/lib/gitlab/ldap/access.rb @@ -34,25 +34,23 @@ module Gitlab def allowed? if ldap_user unless ldap_config.active_directory - user.activate if user.ldap_blocked? + unblock_user(user, 'is available again') if user.ldap_blocked? return true end # Block user in GitLab if he/she was blocked in AD if Gitlab::LDAP::Person.disabled_via_active_directory?(user.ldap_identity.extern_uid, adapter) - user.ldap_block + block_user(user, 'is disabled in Active Directory') false else - user.activate if user.ldap_blocked? + unblock_user(user, 'is not disabled anymore') if user.ldap_blocked? true end else # Block the user if they no longer exist in LDAP/AD - user.ldap_block + block_user(user, 'does not exist anymore') false end - rescue - false end def adapter @@ -66,6 +64,24 @@ module Gitlab def ldap_user @ldap_user ||= Gitlab::LDAP::Person.find_by_dn(user.ldap_identity.extern_uid, adapter) end + + def block_user(user, reason) + user.ldap_block + + Gitlab::AppLogger.info( + "LDAP account \"#{user.ldap_identity.extern_uid}\" #{reason}, " \ + "blocking Gitlab user \"#{user.name}\" (#{user.email})" + ) + end + + def unblock_user(user, reason) + user.activate + + Gitlab::AppLogger.info( + "LDAP account \"#{user.ldap_identity.extern_uid}\" #{reason}, " \ + "unblocking Gitlab user \"#{user.name}\" (#{user.email})" + ) + end end end end diff --git a/lib/gitlab/ldap/adapter.rb b/lib/gitlab/ldap/adapter.rb index 9a5bcfb5c9b..7b05290e5cc 100644 --- a/lib/gitlab/ldap/adapter.rb +++ b/lib/gitlab/ldap/adapter.rb @@ -23,31 +23,7 @@ module Gitlab end def users(field, value, limit = nil) - if field.to_sym == :dn - options = { - base: value, - scope: Net::LDAP::SearchScope_BaseObject - } - else - options = { - base: config.base, - filter: Net::LDAP::Filter.eq(field, value) - } - end - - if config.user_filter.present? - user_filter = Net::LDAP::Filter.construct(config.user_filter) - - options[:filter] = if options[:filter] - Net::LDAP::Filter.join(options[:filter], user_filter) - else - user_filter - end - end - - if limit.present? - options.merge!(size: limit) - end + options = user_options(field, value, limit) entries = ldap_search(options).select do |entry| entry.respond_to? config.uid @@ -86,10 +62,47 @@ module Gitlab results end end + rescue Net::LDAP::Error => error + Rails.logger.warn("LDAP search raised exception #{error.class}: #{error.message}") + [] rescue Timeout::Error Rails.logger.warn("LDAP search timed out after #{config.timeout} seconds") [] end + + private + + def user_options(field, value, limit) + options = { attributes: user_attributes } + options[:size] = limit if limit + + if field.to_sym == :dn + options[:base] = value + options[:scope] = Net::LDAP::SearchScope_BaseObject + options[:filter] = user_filter + else + options[:base] = config.base + options[:filter] = user_filter(Net::LDAP::Filter.eq(field, value)) + end + + options + end + + def user_filter(filter = nil) + user_filter = config.constructed_user_filter if config.user_filter.present? + + if user_filter && filter + Net::LDAP::Filter.join(filter, user_filter) + elsif user_filter + user_filter + else + filter + end + end + + def user_attributes + %W(#{config.uid} cn mail dn) + end end end end diff --git a/lib/gitlab/ldap/auth_hash.rb b/lib/gitlab/ldap/auth_hash.rb index bf4dd9542d5..95378e5a769 100644 --- a/lib/gitlab/ldap/auth_hash.rb +++ b/lib/gitlab/ldap/auth_hash.rb @@ -25,7 +25,7 @@ module Gitlab end def get_raw(key) - auth_hash.extra[:raw_info][key] + auth_hash.extra[:raw_info][key] if auth_hash.extra end def ldap_config diff --git a/lib/gitlab/ldap/authentication.rb b/lib/gitlab/ldap/authentication.rb index bad683c6511..4745311402c 100644 --- a/lib/gitlab/ldap/authentication.rb +++ b/lib/gitlab/ldap/authentication.rb @@ -54,11 +54,9 @@ module Gitlab # Apply LDAP user filter if present if config.user_filter.present? - filter = Net::LDAP::Filter.join( - filter, - Net::LDAP::Filter.construct(config.user_filter) - ) + filter = Net::LDAP::Filter.join(filter, config.constructed_user_filter) end + filter end diff --git a/lib/gitlab/ldap/config.rb b/lib/gitlab/ldap/config.rb index f9bb5775323..28129198438 100644 --- a/lib/gitlab/ldap/config.rb +++ b/lib/gitlab/ldap/config.rb @@ -13,7 +13,7 @@ module Gitlab end def self.providers - servers.map {|server| server['provider_name'] } + servers.map { |server| server['provider_name'] } end def self.valid_provider?(provider) @@ -38,13 +38,31 @@ module Gitlab end def adapter_options - { - host: options['host'], - port: options['port'], - encryption: encryption - }.tap do |options| - options.merge!(auth_options) if has_auth? + opts = base_options.merge( + encryption: encryption, + ) + + opts.merge!(auth_options) if has_auth? + + opts + end + + def omniauth_options + opts = base_options.merge( + base: base, + method: options['method'], + filter: omniauth_user_filter, + name_proc: name_proc + ) + + if has_auth? + opts.merge!( + bind_dn: options['bind_dn'], + password: options['password'] + ) end + + opts end def base @@ -68,6 +86,10 @@ module Gitlab options['user_filter'] end + def constructed_user_filter + @constructed_user_filter ||= Net::LDAP::Filter.construct(user_filter) + end + def group_base options['group_base'] end @@ -85,15 +107,48 @@ module Gitlab end def attributes - options['attributes'] + default_attributes.merge(options['attributes']) end def timeout options['timeout'].to_i end + def has_auth? + options['password'] || options['bind_dn'] + end + + def allow_username_or_email_login + options['allow_username_or_email_login'] + end + + def name_proc + if allow_username_or_email_login + Proc.new { |name| name.gsub(/@.*\z/, '') } + else + Proc.new { |name| name } + end + end + + def default_attributes + { + 'username' => %w(uid userid sAMAccountName), + 'email' => %w(mail email userPrincipalName), + 'name' => 'cn', + 'first_name' => 'givenName', + 'last_name' => 'sn' + } + end + protected + def base_options + { + host: options['host'], + port: options['port'] + } + end + def base_config Gitlab.config.ldap end @@ -123,8 +178,14 @@ module Gitlab } end - def has_auth? - options['password'] || options['bind_dn'] + def omniauth_user_filter + uid_filter = Net::LDAP::Filter.eq(uid, '%{username}') + + if user_filter.present? + Net::LDAP::Filter.join(uid_filter, constructed_user_filter).to_s + else + uid_filter.to_s + end end end end diff --git a/lib/gitlab/ldap/person.rb b/lib/gitlab/ldap/person.rb index b81f3e8e8f5..43eb73250b7 100644 --- a/lib/gitlab/ldap/person.rb +++ b/lib/gitlab/ldap/person.rb @@ -28,7 +28,7 @@ module Gitlab end def name - entry.cn.first + attribute_value(:name).first end def uid @@ -40,12 +40,10 @@ module Gitlab end def email - entry.try(:mail) + attribute_value(:email) end - def dn - entry.dn - end + delegate :dn, to: :entry private @@ -56,6 +54,19 @@ module Gitlab def config @config ||= Gitlab::LDAP::Config.new(provider) end + + # Using the LDAP attributes configuration, find and return the first + # attribute with a value. For example, by default, when given 'email', + # this method looks for 'mail', 'email' and 'userPrincipalName' and + # returns the first with a value. + def attribute_value(attribute) + attributes = Array(config.attributes[attribute.to_s]) + selected_attr = attributes.find { |attr| entry.respond_to?(attr) } + + return nil unless selected_attr + + entry.public_send(selected_attr) + end end end end diff --git a/lib/gitlab/lfs_token.rb b/lib/gitlab/lfs_token.rb new file mode 100644 index 00000000000..5f67e97fa2a --- /dev/null +++ b/lib/gitlab/lfs_token.rb @@ -0,0 +1,48 @@ +module Gitlab + class LfsToken + attr_accessor :actor + + TOKEN_LENGTH = 50 + EXPIRY_TIME = 1800 + + def initialize(actor) + @actor = + case actor + when DeployKey, User + actor + when Key + actor.user + else + raise 'Bad Actor' + end + end + + def token + Gitlab::Redis.with do |redis| + token = redis.get(redis_key) + token ||= Devise.friendly_token(TOKEN_LENGTH) + redis.set(redis_key, token, ex: EXPIRY_TIME) + + token + end + end + + def user? + actor.is_a?(User) + end + + def type + actor.is_a?(User) ? :lfs_token : :lfs_deploy_token + end + + def actor_name + actor.is_a?(User) ? actor.username : "lfs+deploy-key-#{actor.id}" + end + + private + + def redis_key + "gitlab:lfs_token:#{actor.class.name.underscore}_#{actor.id}" if actor + end + end +end diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb index 12999a90a29..3503fac40e8 100644 --- a/lib/gitlab/mail_room.rb +++ b/lib/gitlab/mail_room.rb @@ -31,9 +31,15 @@ module Gitlab config[:ssl] = false if config[:ssl].nil? config[:start_tls] = false if config[:start_tls].nil? config[:mailbox] = 'inbox' if config[:mailbox].nil? + config[:idle_timeout] = 60 if config[:idle_timeout].nil? if config[:enabled] && config[:address] - config[:redis_url] = Gitlab::Redis.new(rails_env).url + gitlab_redis = Gitlab::Redis.new(rails_env) + config[:redis_url] = gitlab_redis.url + + if gitlab_redis.sentinels? + config[:sentinels] = gitlab_redis.sentinels + end end config diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index 41fcd971c22..857e0abf710 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -112,7 +112,7 @@ module Gitlab def self.tag_transaction(name, value) trans = current_transaction - trans.add_tag(name, value) if trans + trans&.add_tag(name, value) end # Sets the action of the current transaction (if any) @@ -121,7 +121,16 @@ module Gitlab def self.action=(action) trans = current_transaction - trans.action = action if trans + trans&.action = action + end + + # Tracks an event. + # + # See `Gitlab::Metrics::Transaction#add_event` for more details. + def self.add_event(*args) + trans = current_transaction + + trans&.add_event(*args) end # Returns the prefix to use for the name of a series. diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb index 4b7a791e497..6aa38542cb4 100644 --- a/lib/gitlab/metrics/instrumentation.rb +++ b/lib/gitlab/metrics/instrumentation.rb @@ -143,11 +143,12 @@ module Gitlab # signature this would break things. As a result we'll make sure the # generated method _only_ accepts regular arguments if the underlying # method also accepts them. - if method.arity == 0 - args_signature = '' - else - args_signature = '*args' - end + args_signature = + if method.arity == 0 + '' + else + '*args' + end proxy_module.class_eval <<-EOF, __FILE__, __LINE__ + 1 def #{name}(#{args_signature}) diff --git a/lib/gitlab/metrics/metric.rb b/lib/gitlab/metrics/metric.rb index f23d67e1e38..bd0afe53c51 100644 --- a/lib/gitlab/metrics/metric.rb +++ b/lib/gitlab/metrics/metric.rb @@ -4,15 +4,20 @@ module Gitlab class Metric JITTER_RANGE = 0.000001..0.001 - attr_reader :series, :values, :tags + attr_reader :series, :values, :tags, :type # series - The name of the series (as a String) to store the metric in. # values - A Hash containing the values to store. # tags - A Hash containing extra tags to add to the metrics. - def initialize(series, values, tags = {}) + def initialize(series, values, tags = {}, type = :metric) @values = values @series = series @tags = tags + @type = type + end + + def event? + type == :event end # Returns a Hash in a format that can be directly written to InfluxDB. diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb index e61670f491c..adc0db1a874 100644 --- a/lib/gitlab/metrics/rack_middleware.rb +++ b/lib/gitlab/metrics/rack_middleware.rb @@ -2,8 +2,19 @@ module Gitlab module Metrics # Rack middleware for tracking Rails and Grape requests. class RackMiddleware - CONTROLLER_KEY = 'action_controller.instance' - ENDPOINT_KEY = 'api.endpoint' + CONTROLLER_KEY = 'action_controller.instance'.freeze + ENDPOINT_KEY = 'api.endpoint'.freeze + CONTENT_TYPES = { + 'text/html' => :html, + 'text/plain' => :txt, + 'application/json' => :json, + 'text/js' => :js, + 'application/atom+xml' => :atom, + 'image/png' => :png, + 'image/jpeg' => :jpeg, + 'image/gif' => :gif, + 'image/svg+xml' => :svg + }.freeze def initialize(app) @app = app @@ -17,6 +28,10 @@ module Gitlab begin retval = trans.run { @app.call(env) } + rescue Exception => error # rubocop: disable Lint/RescueException + trans.add_event(:rails_exception) + + raise error # Even in the event of an error we want to submit any metrics we # might've gathered up to this point. ensure @@ -42,14 +57,32 @@ module Gitlab end def tag_controller(trans, env) - controller = env[CONTROLLER_KEY] - trans.action = "#{controller.class.name}##{controller.action_name}" + controller = env[CONTROLLER_KEY] + action = "#{controller.class.name}##{controller.action_name}" + suffix = CONTENT_TYPES[controller.content_type] + + if suffix && suffix != :html + action += ".#{suffix}" + end + + trans.action = action end def tag_endpoint(trans, env) endpoint = env[ENDPOINT_KEY] - path = endpoint_paths_cache[endpoint.route.route_method][endpoint.route.route_path] - trans.action = "Grape##{endpoint.route.route_method} #{path}" + + begin + route = endpoint.route + rescue + # endpoint.route is calling env[Grape::Env::GRAPE_ROUTING_ARGS][:route_info] + # but env[Grape::Env::GRAPE_ROUTING_ARGS] is nil in the case of a 405 response + # so we're rescuing exceptions and bailing out + end + + if route + path = endpoint_paths_cache[route.request_method][route.path] + trans.action = "Grape##{route.request_method} #{path}" + end end private diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb index a1240fd33ee..f9dd8e41912 100644 --- a/lib/gitlab/metrics/sidekiq_middleware.rb +++ b/lib/gitlab/metrics/sidekiq_middleware.rb @@ -11,6 +11,10 @@ module Gitlab # Old gitlad-shell messages don't provide enqueued_at/created_at attributes trans.set(:sidekiq_queue_duration, Time.now.to_f - (message['enqueued_at'] || message['created_at'] || 0)) trans.run { yield } + rescue Exception => error # rubocop: disable Lint/RescueException + trans.add_event(:sidekiq_exception) + + raise error ensure trans.finish end diff --git a/lib/gitlab/metrics/subscribers/action_view.rb b/lib/gitlab/metrics/subscribers/action_view.rb index 2e9dd4645e3..d435a33e9c7 100644 --- a/lib/gitlab/metrics/subscribers/action_view.rb +++ b/lib/gitlab/metrics/subscribers/action_view.rb @@ -5,7 +5,7 @@ module Gitlab class ActionView < ActiveSupport::Subscriber attach_to :action_view - SERIES = 'views' + SERIES = 'views'.freeze def render_template(event) track(event) if current_transaction diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb index 287b7a83547..3aaebb3e9c3 100644 --- a/lib/gitlab/metrics/system.rb +++ b/lib/gitlab/metrics/system.rb @@ -11,7 +11,7 @@ module Gitlab mem = 0 match = File.read('/proc/self/status').match(/VmRSS:\s+(\d+)/) - if match and match[1] + if match && match[1] mem = match[1].to_f * 1024 end diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index 968f3218950..4f9fb1c7853 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -4,7 +4,10 @@ module Gitlab class Transaction THREAD_KEY = :_gitlab_metrics_transaction - attr_reader :tags, :values, :methods + # The series to store events (e.g. Git pushes) in. + EVENT_SERIES = 'events'.freeze + + attr_reader :tags, :values, :method, :metrics attr_accessor :action @@ -55,6 +58,20 @@ module Gitlab @metrics << Metric.new("#{Metrics.series_prefix}#{series}", values, tags) end + # Tracks a business level event + # + # Business level events including events such as Git pushes, Emails being + # sent, etc. + # + # event_name - The name of the event (e.g. "git_push"). + # tags - A set of tags to attach to the event. + def add_event(event_name, tags = {}) + @metrics << Metric.new(EVENT_SERIES, + { count: 1 }, + { event: event_name }.merge(tags), + :event) + end + # Returns a MethodCall object for the given name. def method_call_for(name) unless method = @methods[name] @@ -101,7 +118,7 @@ module Gitlab submit_hashes = submit.map do |metric| hash = metric.to_hash - hash[:tags][:action] ||= @action if @action + hash[:tags][:action] ||= @action if @action && !metric.event? hash end diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb new file mode 100644 index 00000000000..fee741b47be --- /dev/null +++ b/lib/gitlab/middleware/multipart.rb @@ -0,0 +1,103 @@ +# Gitlab::Middleware::Multipart - a Rack::Multipart replacement +# +# Rack::Multipart leaves behind tempfiles in /tmp and uses valuable Ruby +# process time to copy files around. This alternative solution uses +# gitlab-workhorse to clean up the tempfiles and puts the tempfiles in a +# location where copying should not be needed. +# +# When gitlab-workhorse finds files in a multipart MIME body it sends +# a signed message via a request header. This message lists the names of +# the multipart entries that gitlab-workhorse filtered out of the +# multipart structure and saved to tempfiles. Workhorse adds new entries +# in the multipart structure with paths to the tempfiles. +# +# The job of this Rack middleware is to detect and decode the message +# from workhorse. If present, it walks the Rack 'params' hash for the +# current request, opens the respective tempfiles, and inserts the open +# Ruby File objects in the params hash where Rack::Multipart would have +# put them. The goal is that application code deeper down can keep +# working the way it did with Rack::Multipart without changes. +# +# CAVEAT: the code that modifies the params hash is a bit complex. It is +# conceivable that certain Rack params structures will not be modified +# correctly. We are not aware of such bugs at this time though. +# + +module Gitlab + module Middleware + class Multipart + RACK_ENV_KEY = 'HTTP_GITLAB_WORKHORSE_MULTIPART_FIELDS'.freeze + + class Handler + def initialize(env, message) + @request = Rack::Request.new(env) + @rewritten_fields = message['rewritten_fields'] + @open_files = [] + end + + def with_open_files + @rewritten_fields.each do |field, tmp_path| + parsed_field = Rack::Utils.parse_nested_query(field) + raise "unexpected field: #{field.inspect}" unless parsed_field.count == 1 + + key, value = parsed_field.first + if value.nil? + value = open_file(tmp_path) + @open_files << value + else + value = decorate_params_value(value, @request.params[key], tmp_path) + end + @request.update_param(key, value) + end + + yield + ensure + @open_files.each(&:close) + end + + # This function calls itself recursively + def decorate_params_value(path_hash, value_hash, tmp_path) + unless path_hash.is_a?(Hash) && path_hash.count == 1 + raise "invalid path: #{path_hash.inspect}" + end + path_key, path_value = path_hash.first + + unless value_hash.is_a?(Hash) && value_hash[path_key] + raise "invalid value hash: #{value_hash.inspect}" + end + + case path_value + when nil + value_hash[path_key] = open_file(tmp_path) + @open_files << value_hash[path_key] + value_hash + when Hash + decorate_params_value(path_value, value_hash[path_key], tmp_path) + value_hash + else + raise "unexpected path value: #{path_value.inspect}" + end + end + + def open_file(path) + ::UploadedFile.new(path, File.basename(path), 'application/octet-stream') + end + end + + def initialize(app) + @app = app + end + + def call(env) + encoded_message = env.delete(RACK_ENV_KEY) + return @app.call(env) if encoded_message.blank? + + message = Gitlab::Workhorse.decode_jwt(encoded_message)[0] + + Handler.new(env, message).with_open_files do + @app.call(env) + end + end + end + end +end diff --git a/lib/gitlab/middleware/rails_queue_duration.rb b/lib/gitlab/middleware/rails_queue_duration.rb index 56608b1b276..5d2d7d0026c 100644 --- a/lib/gitlab/middleware/rails_queue_duration.rb +++ b/lib/gitlab/middleware/rails_queue_duration.rb @@ -11,7 +11,7 @@ module Gitlab def call(env) trans = Gitlab::Metrics.current_transaction - proxy_start = env['HTTP_GITLAB_WORHORSE_PROXY_START'].presence + proxy_start = env['HTTP_GITLAB_WORKHORSE_PROXY_START'].presence if trans && proxy_start # Time in milliseconds since gitlab-workhorse started the request trans.set(:rails_queue_duration, Time.now.to_f * 1_000 - proxy_start.to_f / 1_000_000) diff --git a/lib/gitlab/middleware/webpack_proxy.rb b/lib/gitlab/middleware/webpack_proxy.rb new file mode 100644 index 00000000000..6105d165810 --- /dev/null +++ b/lib/gitlab/middleware/webpack_proxy.rb @@ -0,0 +1,24 @@ +# This Rack middleware is intended to proxy the webpack assets directory to the +# webpack-dev-server. It is only intended for use in development. + +module Gitlab + module Middleware + class WebpackProxy < Rack::Proxy + def initialize(app = nil, opts = {}) + @proxy_host = opts.fetch(:proxy_host, 'localhost') + @proxy_port = opts.fetch(:proxy_port, 3808) + @proxy_path = opts[:proxy_path] if opts[:proxy_path] + + super(app, backend: "http://#{@proxy_host}:#{@proxy_port}", **opts) + end + + def perform_request(env) + if @proxy_path && env['PATH_INFO'].start_with?("/#{@proxy_path}") + super(env) + else + @app.call(env) + end + end + end + end +end diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index 0a91d3918d5..95d2f559588 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -29,17 +29,16 @@ module Gitlab def save(provider = 'OAuth') unauthorized_to_create unless gl_user - if needs_blocking? - gl_user.save! - gl_user.block - else - gl_user.save! - end + block_after_save = needs_blocking? + + gl_user.save! + + gl_user.block if block_after_save log.info "(#{provider}) saving user #{auth_hash.email} from login with extern_uid => #{auth_hash.uid}" gl_user rescue ActiveRecord::RecordInvalid => e - log.info "(#{provider}) Error saving user: #{gl_user.errors.full_messages}" + log.info "(#{provider}) Error saving user #{auth_hash.uid} (#{auth_hash.email}): #{gl_user.errors.full_messages}" return self, e.record.errors end @@ -102,6 +101,8 @@ module Gitlab Gitlab::LDAP::Config.providers.each do |provider| adapter = Gitlab::LDAP::Adapter.new(provider) @ldap_person = Gitlab::LDAP::Person.find_by_uid(auth_hash.uid, adapter) + # The `uid` might actually be a DN. Try it next. + @ldap_person ||= Gitlab::LDAP::Person.find_by_dn(auth_hash.uid, adapter) break if @ldap_person end @ldap_person diff --git a/lib/gitlab/optimistic_locking.rb b/lib/gitlab/optimistic_locking.rb new file mode 100644 index 00000000000..962ff4d3985 --- /dev/null +++ b/lib/gitlab/optimistic_locking.rb @@ -0,0 +1,21 @@ +module Gitlab + module OptimisticLocking + module_function + + def retry_lock(subject, retries = 100, &block) + loop do + begin + ActiveRecord::Base.transaction do + return yield(subject) + end + rescue ActiveRecord::StaleObjectError + retries -= 1 + raise unless retries >= 0 + subject.reload + end + end + end + + alias_method :retry_optimistic_lock, :retry_lock + end +end diff --git a/lib/gitlab/other_markup.rb b/lib/gitlab/other_markup.rb index 4e2f8ed5587..e67acf28c94 100644 --- a/lib/gitlab/other_markup.rb +++ b/lib/gitlab/other_markup.rb @@ -17,6 +17,9 @@ module Gitlab html = Banzai.post_process(html, context) + filter = Banzai::Filter::SanitizationFilter.new(html) + html = filter.call.to_s + html.html_safe end end diff --git a/lib/gitlab/pages_transfer.rb b/lib/gitlab/pages_transfer.rb new file mode 100644 index 00000000000..fb215f27cbd --- /dev/null +++ b/lib/gitlab/pages_transfer.rb @@ -0,0 +1,7 @@ +module Gitlab + class PagesTransfer < ProjectTransfer + def root_dir + Gitlab.config.pages.path + end + end +end diff --git a/lib/gitlab/popen.rb b/lib/gitlab/popen.rb index ca23ccef25b..4bc5cda8cb5 100644 --- a/lib/gitlab/popen.rb +++ b/lib/gitlab/popen.rb @@ -5,31 +5,31 @@ module Gitlab module Popen extend self - def popen(cmd, path = nil) + def popen(cmd, path = nil, vars = {}) unless cmd.is_a?(Array) raise "System commands must be given as an array of strings" end path ||= Dir.pwd - vars = { "PWD" => path } + vars['PWD'] = path options = { chdir: path } unless File.directory?(path) FileUtils.mkdir_p(path) end - @cmd_output = "" - @cmd_status = 0 + cmd_output = "" + cmd_status = 0 Open3.popen3(vars, *cmd, options) do |stdin, stdout, stderr, wait_thr| - # We are not using stdin so we should close it, in case the command we - # are running waits for input. + yield(stdin) if block_given? stdin.close - @cmd_output << stdout.read - @cmd_output << stderr.read - @cmd_status = wait_thr.value.exitstatus + + cmd_output << stdout.read + cmd_output << stderr.read + cmd_status = wait_thr.value.exitstatus end - [@cmd_output, @cmd_status] + [cmd_output, cmd_status] end end end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 183bd10d6a3..db325c00705 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -5,11 +5,7 @@ module Gitlab def initialize(current_user, project, query, repository_ref = nil) @current_user = current_user @project = project - @repository_ref = if repository_ref.present? - repository_ref - else - nil - end + @repository_ref = repository_ref.presence || project.default_branch @query = query end @@ -28,11 +24,6 @@ module Gitlab end end - def total_count - @total_count ||= issues_count + merge_requests_count + blobs_count + - notes_count + wiki_blobs_count + commits_count - end - def blobs_count @blobs_count ||= blobs.count end @@ -49,40 +40,107 @@ module Gitlab @commits_count ||= commits.count end + def self.parse_search_result(result) + ref = nil + filename = nil + basename = nil + startline = 0 + + result.each_line.each_with_index do |line, index| + if line =~ /^.*:.*:\d+:/ + ref, filename, startline = line.split(':') + startline = startline.to_i - index + extname = Regexp.escape(File.extname(filename)) + basename = filename.sub(/#{extname}$/, '') + break + end + end + + data = "" + + result.each_line do |line| + data << line.sub(ref, '').sub(filename, '').sub(/^:-\d+-/, '').sub(/^::\d+:/, '') + end + + OpenStruct.new( + filename: filename, + basename: basename, + ref: ref, + startline: startline, + data: data + ) + end + + def single_commit_result? + commits_count == 1 && total_result_count == 1 + end + + def total_result_count + issues_count + merge_requests_count + milestones_count + notes_count + blobs_count + wiki_blobs_count + commits_count + end + private def blobs - if project.empty_repo? || query.blank? - [] - else - project.repository.search_files(query, repository_ref) + @blobs ||= begin + blobs = project.repository.search_files_by_content(query, repository_ref).first(100) + found_file_names = Set.new + + results = blobs.map do |blob| + blob = self.class.parse_search_result(blob) + found_file_names << blob.filename + + [blob.filename, blob] + end + + project.repository.search_files_by_name(query, repository_ref).first(100).each do |filename| + results << [filename, nil] unless found_file_names.include?(filename) + end + + results.sort_by(&:first) end end def wiki_blobs - if project.wiki_enabled? && query.present? - project_wiki = ProjectWiki.new(project) + @wiki_blobs ||= begin + if project.wiki_enabled? && query.present? + project_wiki = ProjectWiki.new(project) - unless project_wiki.empty? - project_wiki.search_files(query) + unless project_wiki.empty? + project_wiki.search_files(query) + else + [] + end else [] end - else - [] end end def notes - project.notes.user.search(query, as_user: @current_user).order('updated_at DESC') + @notes ||= NotesFinder.new(project, @current_user, search: query).execute.user.order('updated_at DESC') end def commits - if project.empty_repo? || query.blank? - [] - else - project.repository.find_commits_by_message(query).compact - end + @commits ||= find_commits(query) + end + + def find_commits(query) + return [] unless Ability.allowed?(@current_user, :download_code, @project) + + commits = find_commits_by_message(query) + commit_by_sha = find_commit_by_sha(query) + commits |= [commit_by_sha] if commit_by_sha + commits + end + + def find_commits_by_message(query) + project.repository.find_commits_by_message(query) + end + + def find_commit_by_sha(query) + key = query.strip + project.repository.commit(key) if Commit.valid_hash?(key) end def project_ids_relation diff --git a/lib/gitlab/project_transfer.rb b/lib/gitlab/project_transfer.rb new file mode 100644 index 00000000000..1bba0b78e2f --- /dev/null +++ b/lib/gitlab/project_transfer.rb @@ -0,0 +1,35 @@ +module Gitlab + class ProjectTransfer + def move_project(project_path, namespace_path_was, namespace_path) + new_namespace_folder = File.join(root_dir, namespace_path) + FileUtils.mkdir_p(new_namespace_folder) unless Dir.exist?(new_namespace_folder) + from = File.join(root_dir, namespace_path_was, project_path) + to = File.join(root_dir, namespace_path, project_path) + move(from, to, "") + end + + def rename_project(path_was, path, namespace_path) + base_dir = File.join(root_dir, namespace_path) + move(path_was, path, base_dir) + end + + def rename_namespace(path_was, path) + move(path_was, path) + end + + def root_dir + raise NotImplementedError + end + + private + + def move(path_was, path, base_dir = nil) + base_dir = root_dir unless base_dir + from = File.join(base_dir, path_was) + to = File.join(base_dir, path) + FileUtils.mv(from, to) + rescue Errno::ENOENT + false + end + end +end diff --git a/lib/gitlab/recaptcha.rb b/lib/gitlab/recaptcha.rb index 70e7f25d518..4bc76ea033f 100644 --- a/lib/gitlab/recaptcha.rb +++ b/lib/gitlab/recaptcha.rb @@ -10,5 +10,9 @@ module Gitlab true end end + + def self.enabled? + current_application_settings.recaptcha_enabled + end end end diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb index 9376b54f43b..62dbd429156 100644 --- a/lib/gitlab/redis.rb +++ b/lib/gitlab/redis.rb @@ -4,40 +4,41 @@ require 'active_support/core_ext/hash/keys' module Gitlab class Redis - CACHE_NAMESPACE = 'cache:gitlab' - SESSION_NAMESPACE = 'session:gitlab' - SIDEKIQ_NAMESPACE = 'resque:gitlab' - MAILROOM_NAMESPACE = 'mail_room:gitlab' - DEFAULT_REDIS_URL = 'redis://localhost:6379' - - # To be thread-safe we must be careful when writing the class instance - # variables @url and @pool. Because @pool depends on @url we need two - # mutexes to prevent deadlock. - PARAMS_MUTEX = Mutex.new - POOL_MUTEX = Mutex.new - private_constant :PARAMS_MUTEX, :POOL_MUTEX + CACHE_NAMESPACE = 'cache:gitlab'.freeze + SESSION_NAMESPACE = 'session:gitlab'.freeze + SIDEKIQ_NAMESPACE = 'resque:gitlab'.freeze + MAILROOM_NAMESPACE = 'mail_room:gitlab'.freeze + DEFAULT_REDIS_URL = 'redis://localhost:6379'.freeze + CONFIG_FILE = File.expand_path('../../config/resque.yml', __dir__) class << self - def params - @params || PARAMS_MUTEX.synchronize { @params = new.params } - end + delegate :params, :url, to: :new - # @deprecated Use .params instead to get sentinel support - def url - new.url + def with + @pool ||= ConnectionPool.new(size: pool_size) { ::Redis.new(params) } + @pool.with { |redis| yield redis } end - def with - if @pool.nil? - POOL_MUTEX.synchronize do - @pool = ConnectionPool.new { ::Redis.new(params) } - end + def pool_size + if Sidekiq.server? + # the pool will be used in a multi-threaded context + Sidekiq.options[:concurrency] + 5 + else + # probably this is a Unicorn process, so single threaded + 5 end - @pool.with { |redis| yield redis } end - def reset_params! - @params = nil + def _raw_config + return @_raw_config if defined?(@_raw_config) + + begin + @_raw_config = ERB.new(File.read(CONFIG_FILE)).result.freeze + rescue Errno::ENOENT + @_raw_config = false + end + + @_raw_config end end @@ -53,6 +54,14 @@ module Gitlab raw_config_hash[:url] end + def sentinels + raw_config_hash[:sentinels] + end + + def sentinels? + sentinels && !sentinels.empty? + end + private def redis_store_options @@ -83,12 +92,7 @@ module Gitlab end def fetch_config - file = config_file - File.exist?(file) ? YAML.load_file(file)[@rails_env] : false - end - - def config_file - File.expand_path('../../../config/resque.yml', __FILE__) + self.class._raw_config ? YAML.load(self.class._raw_config)[@rails_env] : false end end end diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index 11c0b01f0dc..7668ecacc4b 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -1,13 +1,12 @@ module Gitlab # Extract possible GFM references from an arbitrary String for further processing. class ReferenceExtractor < Banzai::ReferenceExtractor - REFERABLES = %i(user issue label milestone merge_request snippet commit commit_range) + REFERABLES = %i(user issue label milestone merge_request snippet commit commit_range directly_addressed_user).freeze attr_accessor :project, :current_user, :author def initialize(project, current_user = nil) @project = project @current_user = current_user - @references = {} super() @@ -21,6 +20,11 @@ module Gitlab super(type, project, current_user) end + def reset_memoized_values + @references = {} + super() + end + REFERABLES.each do |type| define_method("#{type}s") do @references[type] ||= references(type) diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index ffad5e17c78..5e5f5ff1589 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -2,15 +2,33 @@ module Gitlab module Regex extend self - NAMESPACE_REGEX_STR = '(?:[a-zA-Z0-9_\.][a-zA-Z0-9_\-\.]*[a-zA-Z0-9_\-]|[a-zA-Z0-9_])'.freeze + # The namespace regex is used in Javascript to validate usernames in the "Register" form. However, Javascript + # does not support the negative lookbehind assertion (?<!) that disallows usernames ending in `.git` and `.atom`. + # Since this is a non-trivial problem to solve in Javascript (heavily complicate the regex, modify view code to + # allow non-regex validatiions, etc), `NAMESPACE_REGEX_STR_JS` serves as a Javascript-compatible version of + # `NAMESPACE_REGEX_STR`, with the negative lookbehind assertion removed. This means that the client-side validation + # will pass for usernames ending in `.atom` and `.git`, but will be caught by the server-side validation. + PATH_REGEX_STR = '[a-zA-Z0-9_\.][a-zA-Z0-9_\-\.]*'.freeze + NAMESPACE_REGEX_STR_JS = PATH_REGEX_STR + '[a-zA-Z0-9_\-]|[a-zA-Z0-9_]'.freeze + NO_SUFFIX_REGEX_STR = '(?<!\.git|\.atom)'.freeze + NAMESPACE_REGEX_STR = "(?:#{NAMESPACE_REGEX_STR_JS})#{NO_SUFFIX_REGEX_STR}".freeze + PROJECT_REGEX_STR = "(?:#{PATH_REGEX_STR})#{NO_SUFFIX_REGEX_STR}".freeze + + # Same as NAMESPACE_REGEX_STR but allows `/` in the path. + # So `group/subgroup` will match this regex but not NAMESPACE_REGEX_STR + FULL_NAMESPACE_REGEX_STR = "(?:#{NAMESPACE_REGEX_STR}/)*#{NAMESPACE_REGEX_STR}".freeze def namespace_regex @namespace_regex ||= /\A#{NAMESPACE_REGEX_STR}\z/.freeze end + def namespace_route_regex + @namespace_route_regex ||= /#{NAMESPACE_REGEX_STR}/.freeze + end + def namespace_regex_message "can contain only letters, digits, '_', '-' and '.'. " \ - "Cannot start with '-' or end in '.'." \ + "Cannot start with '-' or end in '.', '.git' or '.atom'." \ end def namespace_name_regex @@ -22,16 +40,24 @@ module Gitlab end def project_name_regex - @project_name_regex ||= /\A[\p{Alnum}_][\p{Alnum}\p{Pd}_\. ]*\z/.freeze + @project_name_regex ||= /\A[\p{Alnum}\u{00A9}-\u{1f9c0}_][\p{Alnum}\p{Pd}\u{00A9}-\u{1f9c0}_\. ]*\z/.freeze end def project_name_regex_message - "can contain only letters, digits, '_', '.', dash and space. " \ - "It must start with letter, digit or '_'." + "can contain only letters, digits, emojis, '_', '.', dash, space. " \ + "It must start with letter, digit, emoji or '_'." end def project_path_regex - @project_path_regex ||= /\A[a-zA-Z0-9_.][a-zA-Z0-9_\-\.]*(?<!\.git|\.atom)\z/.freeze + @project_path_regex ||= /\A#{PROJECT_REGEX_STR}\z/.freeze + end + + def project_route_regex + @project_route_regex ||= /#{PROJECT_REGEX_STR}/.freeze + end + + def project_git_route_regex + @project_route_git_regex ||= /#{PATH_REGEX_STR}\.git/.freeze end def project_path_regex_message @@ -40,19 +66,19 @@ module Gitlab end def file_name_regex - @file_name_regex ||= /\A[a-zA-Z0-9_\-\.\@]*\z/.freeze + @file_name_regex ||= /\A[[[:alnum:]]_\-\.\@\+]*\z/.freeze end def file_name_regex_message - "can contain only letters, digits, '_', '-', '@' and '.'. " + "can contain only letters, digits, '_', '-', '@', '+' and '.'." end def file_path_regex - @file_path_regex ||= /\A[a-zA-Z0-9_\-\.\/\@]*\z/.freeze + @file_path_regex ||= /\A[[[:alnum:]]_\-\.\/\@]*\z/.freeze end def file_path_regex_message - "can contain only letters, digits, '_', '-', '@' and '.'. Separate directories with a '/'. " + "can contain only letters, digits, '_', '-', '@' and '.'. Separate directories with a '/'." end def directory_traversal_regex @@ -60,7 +86,7 @@ module Gitlab end def directory_traversal_regex_message - "cannot include directory traversal. " + "cannot include directory traversal." end def archive_formats_regex @@ -96,11 +122,28 @@ module Gitlab end def environment_name_regex - @environment_name_regex ||= /\A[a-zA-Z0-9_-]+\z/.freeze + @environment_name_regex ||= /\A[a-zA-Z0-9_\\\/\${}. -]+\z/.freeze end def environment_name_regex_message - "can contain only letters, digits, '-' and '_'." + "can contain only letters, digits, '-', '_', '/', '$', '{', '}', '.' and spaces" + end + + def kubernetes_namespace_regex + /\A[a-z0-9]([-a-z0-9]*[a-z0-9])?\z/ + end + + def kubernetes_namespace_regex_message + "can contain only letters, digits or '-', and cannot start or end with '-'" + end + + def environment_slug_regex + @environment_slug_regex ||= /\A[a-z]([a-z0-9-]*[a-z0-9])?\z/.freeze + end + + def environment_slug_regex_message + "can contain only lowercase letters, digits, and '-'. " \ + "Must start with a letter, and cannot end with '-'" end end end diff --git a/lib/gitlab/request_profiler.rb b/lib/gitlab/request_profiler.rb index 8130e55351e..0c9ab759e81 100644 --- a/lib/gitlab/request_profiler.rb +++ b/lib/gitlab/request_profiler.rb @@ -2,7 +2,7 @@ require 'fileutils' module Gitlab module RequestProfiler - PROFILES_DIR = "#{Gitlab.config.shared.path}/tmp/requests_profiles" + PROFILES_DIR = "#{Gitlab.config.shared.path}/tmp/requests_profiles".freeze def profile_token Rails.cache.fetch('profile-token') do diff --git a/lib/gitlab/request_profiler/middleware.rb b/lib/gitlab/request_profiler/middleware.rb index 786e1d49f5e..ef42b0557e0 100644 --- a/lib/gitlab/request_profiler/middleware.rb +++ b/lib/gitlab/request_profiler/middleware.rb @@ -1,5 +1,4 @@ require 'ruby-prof' -require_dependency 'gitlab/request_profiler' module Gitlab module RequestProfiler @@ -20,7 +19,7 @@ module Gitlab header_token = env['HTTP_X_PROFILE_TOKEN'] return unless header_token.present? - profile_token = RequestProfiler.profile_token + profile_token = Gitlab::RequestProfiler.profile_token return unless profile_token.present? header_token == profile_token diff --git a/lib/gitlab/route_map.rb b/lib/gitlab/route_map.rb new file mode 100644 index 00000000000..72d00abfcc2 --- /dev/null +++ b/lib/gitlab/route_map.rb @@ -0,0 +1,50 @@ +module Gitlab + class RouteMap + class FormatError < StandardError; end + + def initialize(data) + begin + entries = YAML.safe_load(data) + rescue + raise FormatError, 'Route map is not valid YAML' + end + + raise FormatError, 'Route map is not an array' unless entries.is_a?(Array) + + @map = entries.map { |entry| parse_entry(entry) } + end + + def public_path_for_source_path(path) + mapping = @map.find { |mapping| mapping[:source] === path } + return unless mapping + + path.sub(mapping[:source], mapping[:public]) + end + + private + + def parse_entry(entry) + raise FormatError, 'Route map entry is not a hash' unless entry.is_a?(Hash) + raise FormatError, 'Route map entry does not have a source key' unless entry.has_key?('source') + raise FormatError, 'Route map entry does not have a public key' unless entry.has_key?('public') + + source_pattern = entry['source'] + public_path = entry['public'] + + if source_pattern.start_with?('/') && source_pattern.end_with?('/') + source_pattern = source_pattern[1...-1].gsub('\/', '/') + + begin + source_pattern = /\A#{source_pattern}\z/ + rescue RegexpError => e + raise FormatError, "Route map entry source is not a valid regular expression: #{e}" + end + end + + { + source: source_pattern, + public: public_path + } + end + end +end diff --git a/lib/gitlab/routing.rb b/lib/gitlab/routing.rb index 5132177de51..632e2d87500 100644 --- a/lib/gitlab/routing.rb +++ b/lib/gitlab/routing.rb @@ -1,5 +1,11 @@ module Gitlab module Routing + extend ActiveSupport::Concern + + included do + include Gitlab::Routing.url_helpers + end + # Returns the URL helpers Module. # # This method caches the output as Rails' "url_helpers" method creates an diff --git a/lib/gitlab/saml/user.rb b/lib/gitlab/saml/user.rb index f253dc7477e..8a7cc690046 100644 --- a/lib/gitlab/saml/user.rb +++ b/lib/gitlab/saml/user.rb @@ -28,11 +28,12 @@ module Gitlab if external_users_enabled? && @user # Check if there is overlap between the user's groups and the external groups # setting then set user as external or internal. - if (auth_hash.groups & Gitlab::Saml::Config.external_groups).empty? - @user.external = false - else - @user.external = true - end + @user.external = + if (auth_hash.groups & Gitlab::Saml::Config.external_groups).empty? + false + else + true + end end @user diff --git a/lib/gitlab/sanitizers/svg/whitelist.rb b/lib/gitlab/sanitizers/svg/whitelist.rb index 7b6b70d8dbc..d50f826f924 100644 --- a/lib/gitlab/sanitizers/svg/whitelist.rb +++ b/lib/gitlab/sanitizers/svg/whitelist.rb @@ -6,18 +6,19 @@ module Gitlab module SVG class Whitelist ALLOWED_ELEMENTS = %w[ - a altGlyph altGlyphDef altGlyphItem animate - animateColor animateMotion animateTransform circle clipPath color-profile - cursor defs desc ellipse feBlend feColorMatrix feComponentTransfer - feComposite feConvolveMatrix feDiffuseLighting feDisplacementMap - feDistantLight feFlood feFuncA feFuncB feFuncG feFuncR feGaussianBlur - feImage feMerge feMergeNode feMorphology feOffset fePointLight - feSpecularLighting feSpotLight feTile feTurbulence filter font font-face - font-face-format font-face-name font-face-src font-face-uri foreignObject - g glyph glyphRef hkern image line linearGradient marker mask metadata - missing-glyph mpath path pattern polygon polyline radialGradient rect - script set stop style svg switch symbol text textPath title tref tspan use - view vkern].freeze + a altGlyph altGlyphDef altGlyphItem animate + animateColor animateMotion animateTransform circle clipPath color-profile + cursor defs desc ellipse feBlend feColorMatrix feComponentTransfer + feComposite feConvolveMatrix feDiffuseLighting feDisplacementMap + feDistantLight feFlood feFuncA feFuncB feFuncG feFuncR feGaussianBlur + feImage feMerge feMergeNode feMorphology feOffset fePointLight + feSpecularLighting feSpotLight feTile feTurbulence filter font font-face + font-face-format font-face-name font-face-src font-face-uri foreignObject + g glyph glyphRef hkern image line linearGradient marker mask metadata + missing-glyph mpath path pattern polygon polyline radialGradient rect + script set stop style svg switch symbol text textPath title tref tspan use + view vkern + ].freeze ALLOWED_DATA_ATTRIBUTES_IN_ELEMENTS = %w[svg].freeze diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index f8ab2b1f09e..ccfa517e04b 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -27,11 +27,6 @@ module Gitlab end end - def total_count - @total_count ||= projects_count + issues_count + merge_requests_count + - milestones_count - end - def projects_count @projects_count ||= projects.count end @@ -48,8 +43,8 @@ module Gitlab @milestones_count ||= milestones.count end - def empty? - total_count.zero? + def single_commit_result? + false end private @@ -59,13 +54,14 @@ module Gitlab end def issues - issues = Issue.visible_to_user(current_user).where(project_id: project_ids_relation) + issues = IssuesFinder.new(current_user).execute.where(project_id: project_ids_relation) - if query =~ /#(\d+)\z/ - issues = issues.where(iid: $1) - else - issues = issues.full_search(query) - end + issues = + if query =~ /#(\d+)\z/ + issues.where(iid: $1) + else + issues.full_search(query) + end issues.order('updated_at DESC') end @@ -77,12 +73,13 @@ module Gitlab end def merge_requests - merge_requests = MergeRequest.in_projects(project_ids_relation) - if query =~ /[#!](\d+)\z/ - merge_requests = merge_requests.where(iid: $1) - else - merge_requests = merge_requests.full_search(query) - end + merge_requests = MergeRequestsFinder.new(current_user).execute.in_projects(project_ids_relation) + merge_requests = + if query =~ /[#!](\d+)\z/ + merge_requests.where(iid: $1) + else + merge_requests.full_search(query) + end merge_requests.order('updated_at DESC') end diff --git a/lib/gitlab/seeder.rb b/lib/gitlab/seeder.rb index 7cf506ebe64..b7f825e8284 100644 --- a/lib/gitlab/seeder.rb +++ b/lib/gitlab/seeder.rb @@ -18,7 +18,7 @@ def Notify.deliver_later self end eos - eval(code) + eval(code) # rubocop:disable Security/Eval end end end diff --git a/lib/gitlab/sentry.rb b/lib/gitlab/sentry.rb new file mode 100644 index 00000000000..117fc508135 --- /dev/null +++ b/lib/gitlab/sentry.rb @@ -0,0 +1,27 @@ +module Gitlab + module Sentry + def self.enabled? + Rails.env.production? && current_application_settings.sentry_enabled? + end + + def self.context(current_user = nil) + return unless self.enabled? + + if current_user + Raven.user_context( + id: current_user.id, + email: current_user.email, + username: current_user.username, + ) + end + end + + def self.program_context + if Sidekiq.server? + 'sidekiq' + else + 'rails' + end + end + end +end diff --git a/lib/gitlab/serializer/ci/variables.rb b/lib/gitlab/serializer/ci/variables.rb new file mode 100644 index 00000000000..c059c454eac --- /dev/null +++ b/lib/gitlab/serializer/ci/variables.rb @@ -0,0 +1,27 @@ +module Gitlab + module Serializer + module Ci + # This serializer could make sure our YAML variables' keys and values + # are always strings. This is more for legacy build data because + # from now on we convert them into strings before saving to database. + module Variables + extend self + + def load(string) + return unless string + + object = YAML.safe_load(string, [Symbol]) + + object.map do |variable| + variable[:key] = variable[:key].to_s + variable + end + end + + def dump(object) + YAML.dump(object) + end + end + end + end +end diff --git a/lib/gitlab/serializer/pagination.rb b/lib/gitlab/serializer/pagination.rb new file mode 100644 index 00000000000..bf2c0acc729 --- /dev/null +++ b/lib/gitlab/serializer/pagination.rb @@ -0,0 +1,36 @@ +module Gitlab + module Serializer + class Pagination + class InvalidResourceError < StandardError; end + include ::API::Helpers::Pagination + + def initialize(request, response) + @request = request + @response = response + end + + def paginate(resource) + if resource.respond_to?(:page) + super(resource) + else + raise InvalidResourceError + end + end + + private + + # Methods needed by `API::Helpers::Pagination` + # + + attr_reader :request + + def params + @request.query_parameters + end + + def header(header, value) + @response.headers[header] = value + end + end + end +end diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/shell.rb index 839a4fa30d5..7374d2bc8b8 100644 --- a/lib/gitlab/backend/shell.rb +++ b/lib/gitlab/shell.rb @@ -6,16 +6,56 @@ module Gitlab KeyAdder = Struct.new(:io) do def add_key(id, key) - key.gsub!(/[[:space:]]+/, ' ').strip! + key = Gitlab::Shell.strip_key(key) + # Newline and tab are part of the 'protocol' used to transmit id+key to the other end + if key.include?("\t") || key.include?("\n") + raise Error.new("Invalid key: #{key.inspect}") + end + io.puts("#{id}\t#{key}") end end class << self + def secret_token + @secret_token ||= begin + File.read(Gitlab.config.gitlab_shell.secret_file).chomp + end + end + + def ensure_secret_token! + return if File.exist?(File.join(Gitlab.config.gitlab_shell.path, '.gitlab_shell_secret')) + + generate_and_link_secret_token + end + def version_required @version_required ||= File.read(Rails.root. join('GITLAB_SHELL_VERSION')).strip end + + def strip_key(key) + key.split(/ /)[0, 2].join(' ') + end + + private + + # Create (if necessary) and link the secret token file + def generate_and_link_secret_token + secret_file = Gitlab.config.gitlab_shell.secret_file + shell_path = Gitlab.config.gitlab_shell.path + + unless File.size?(secret_file) + # Generate a new token of 16 random hexadecimal characters and store it in secret_file. + @secret_token = SecureRandom.hex(16) + File.write(secret_file, @secret_token) + end + + link_path = File.join(shell_path, '.gitlab_shell_secret') + if File.exist?(shell_path) && !File.exist?(link_path) + FileUtils.symlink(secret_file, link_path) + end + end end # Init new repository @@ -40,8 +80,10 @@ module Gitlab # import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://github.com/randx/six.git") # def import_repository(storage, name, url) - output, status = Popen::popen([gitlab_shell_projects_path, 'import-project', - storage, "#{name}.git", url, '900']) + # Timeout should be less than 900 ideally, to prevent the memory killer + # to silently kill the process without knowing we are timing out here. + output, status = Popen.popen([gitlab_shell_projects_path, 'import-project', + storage, "#{name}.git", url, '800']) raise Error, output unless status.zero? true end @@ -87,19 +129,6 @@ module Gitlab 'rm-project', storage, "#{name}.git"]) end - # Gc repository - # - # storage - project storage path - # path - project path with namespace - # - # Ex. - # gc("/path/to/storage", "gitlab/gitlab-ci") - # - def gc(storage, path) - Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'gc', - storage, "#{path}.git"]) - end - # Add new key to gitlab-shell # # Ex. @@ -107,7 +136,7 @@ module Gitlab # def add_key(key_id, key_content) Gitlab::Utils.system_silent([gitlab_shell_keys_path, - 'add-key', key_id, key_content]) + 'add-key', key_id, self.class.strip_key(key_content)]) end # Batch-add keys to authorized_keys @@ -116,7 +145,7 @@ module Gitlab # batch_add_keys { |adder| adder.add_key("key-42", "sha-rsa ...") } def batch_add_keys(&block) IO.popen(%W(#{gitlab_shell_path}/bin/gitlab-keys batch-add-keys), 'w') do |io| - block.call(KeyAdder.new(io)) + yield(KeyAdder.new(io)) end end @@ -145,7 +174,7 @@ module Gitlab # add_namespace("/path/to/storage", "gitlab") # def add_namespace(storage, name) - FileUtils.mkdir(full_path(storage, name), mode: 0770) unless exists?(storage, name) + FileUtils.mkdir_p(full_path(storage, name), mode: 0770) unless exists?(storage, name) end # Remove directory from repositories storage @@ -192,21 +221,6 @@ module Gitlab File.exist?(full_path(storage, dir_name)) end - # Create (if necessary) and link the secret token file - def generate_and_link_secret_token - secret_file = Gitlab.config.gitlab_shell.secret_file - unless File.exist? secret_file - # Generate a new token of 16 random hexadecimal characters and store it in secret_file. - token = SecureRandom.hex(16) - File.write(secret_file, token) - end - - link_path = File.join(gitlab_shell_path, '.gitlab_shell_secret') - if File.exist?(gitlab_shell_path) && !File.exist?(link_path) - FileUtils.symlink(secret_file, link_path) - end - end - protected def gitlab_shell_path diff --git a/lib/gitlab/backend/shell_adapter.rb b/lib/gitlab/shell_adapter.rb index fbe2a7a0d72..fbe2a7a0d72 100644 --- a/lib/gitlab/backend/shell_adapter.rb +++ b/lib/gitlab/shell_adapter.rb diff --git a/lib/gitlab/sherlock/query.rb b/lib/gitlab/sherlock/query.rb index 4917c4ae2ac..99e56e923eb 100644 --- a/lib/gitlab/sherlock/query.rb +++ b/lib/gitlab/sherlock/query.rb @@ -94,11 +94,12 @@ module Gitlab private def raw_explain(query) - if Gitlab::Database.postgresql? - explain = "EXPLAIN ANALYZE #{query};" - else - explain = "EXPLAIN #{query};" - end + explain = + if Gitlab::Database.postgresql? + "EXPLAIN ANALYZE #{query};" + else + "EXPLAIN #{query};" + end ActiveRecord::Base.connection.execute(explain) end diff --git a/lib/gitlab/sidekiq_middleware/arguments_logger.rb b/lib/gitlab/sidekiq_middleware/arguments_logger.rb index 7813091ec7b..82a59a7a87e 100644 --- a/lib/gitlab/sidekiq_middleware/arguments_logger.rb +++ b/lib/gitlab/sidekiq_middleware/arguments_logger.rb @@ -2,7 +2,7 @@ module Gitlab module SidekiqMiddleware class ArgumentsLogger def call(worker, job, queue) - Sidekiq.logger.info "arguments: #{job['args']}" + Sidekiq.logger.info "arguments: #{JSON.dump(job['args'])}" yield end end diff --git a/lib/gitlab/sidekiq_status.rb b/lib/gitlab/sidekiq_status.rb new file mode 100644 index 00000000000..aadc401ff8d --- /dev/null +++ b/lib/gitlab/sidekiq_status.rb @@ -0,0 +1,66 @@ +module Gitlab + # The SidekiqStatus module and its child classes can be used for checking if a + # Sidekiq job has been processed or not. + # + # To check if a job has been completed, simply pass the job ID to the + # `completed?` method: + # + # job_id = SomeWorker.perform_async(...) + # + # if Gitlab::SidekiqStatus.completed?(job_id) + # ... + # end + # + # For each job ID registered a separate key is stored in Redis, making lookups + # much faster than using Sidekiq's built-in job finding/status API. These keys + # expire after a certain period of time to prevent storing too many keys in + # Redis. + module SidekiqStatus + STATUS_KEY = 'gitlab-sidekiq-status:%s'.freeze + + # The default time (in seconds) after which a status key is expired + # automatically. The default of 30 minutes should be more than sufficient + # for most jobs. + DEFAULT_EXPIRATION = 30.minutes.to_i + + # Starts tracking of the given job. + # + # jid - The Sidekiq job ID + # expire - The expiration time of the Redis key. + def self.set(jid, expire = DEFAULT_EXPIRATION) + Sidekiq.redis do |redis| + redis.set(key_for(jid), 1, ex: expire) + end + end + + # Stops the tracking of the given job. + # + # jid - The Sidekiq job ID to remove. + def self.unset(jid) + Sidekiq.redis do |redis| + redis.del(key_for(jid)) + end + end + + # Returns true if all the given job have been completed. + # + # jids - The Sidekiq job IDs to check. + # + # Returns true or false. + def self.all_completed?(jids) + keys = jids.map { |jid| key_for(jid) } + + responses = Sidekiq.redis do |redis| + redis.pipelined do + keys.each { |key| redis.exists(key) } + end + end + + responses.all? { |value| !value } + end + + def self.key_for(jid) + STATUS_KEY % jid + end + end +end diff --git a/lib/gitlab/sidekiq_status/client_middleware.rb b/lib/gitlab/sidekiq_status/client_middleware.rb new file mode 100644 index 00000000000..d47609f490d --- /dev/null +++ b/lib/gitlab/sidekiq_status/client_middleware.rb @@ -0,0 +1,10 @@ +module Gitlab + module SidekiqStatus + class ClientMiddleware + def call(_, job, _, _) + Gitlab::SidekiqStatus.set(job['jid']) + yield + end + end + end +end diff --git a/lib/gitlab/sidekiq_status/server_middleware.rb b/lib/gitlab/sidekiq_status/server_middleware.rb new file mode 100644 index 00000000000..ceab10b8301 --- /dev/null +++ b/lib/gitlab/sidekiq_status/server_middleware.rb @@ -0,0 +1,13 @@ +module Gitlab + module SidekiqStatus + class ServerMiddleware + def call(worker, job, queue) + ret = yield + + Gitlab::SidekiqStatus.unset(job['jid']) + + ret + end + end + end +end diff --git a/lib/gitlab/sidekiq_throttler.rb b/lib/gitlab/sidekiq_throttler.rb new file mode 100644 index 00000000000..d4d39a888e7 --- /dev/null +++ b/lib/gitlab/sidekiq_throttler.rb @@ -0,0 +1,23 @@ +module Gitlab + class SidekiqThrottler + class << self + def execute! + if Gitlab::CurrentSettings.sidekiq_throttling_enabled? + Gitlab::CurrentSettings.current_application_settings.sidekiq_throttling_queues.each do |queue| + Sidekiq::Queue[queue].limit = queue_limit + end + end + end + + private + + def queue_limit + @queue_limit ||= + begin + factor = Gitlab::CurrentSettings.current_application_settings.sidekiq_throttling_factor + (factor * Sidekiq.options[:concurrency]).ceil + end + end + end + end +end diff --git a/lib/gitlab/slash_commands/command_definition.rb b/lib/gitlab/slash_commands/command_definition.rb new file mode 100644 index 00000000000..60d35be2599 --- /dev/null +++ b/lib/gitlab/slash_commands/command_definition.rb @@ -0,0 +1,57 @@ +module Gitlab + module SlashCommands + class CommandDefinition + attr_accessor :name, :aliases, :description, :params, :condition_block, :action_block + + def initialize(name, attributes = {}) + @name = name + + @aliases = attributes[:aliases] || [] + @description = attributes[:description] || '' + @params = attributes[:params] || [] + @condition_block = attributes[:condition_block] + @action_block = attributes[:action_block] + end + + def all_names + [name, *aliases] + end + + def noop? + action_block.nil? + end + + def available?(opts) + return true unless condition_block + + context = OpenStruct.new(opts) + context.instance_exec(&condition_block) + end + + def execute(context, opts, arg) + return if noop? || !available?(opts) + + if arg.present? + context.instance_exec(arg, &action_block) + elsif action_block.arity == 0 + context.instance_exec(&action_block) + end + end + + def to_h(opts) + desc = description + if desc.respond_to?(:call) + context = OpenStruct.new(opts) + desc = context.instance_exec(&desc) rescue '' + end + + { + name: name, + aliases: aliases, + description: desc, + params: params + } + end + end + end +end diff --git a/lib/gitlab/slash_commands/dsl.rb b/lib/gitlab/slash_commands/dsl.rb new file mode 100644 index 00000000000..50b0937d267 --- /dev/null +++ b/lib/gitlab/slash_commands/dsl.rb @@ -0,0 +1,98 @@ +module Gitlab + module SlashCommands + module Dsl + extend ActiveSupport::Concern + + included do + cattr_accessor :command_definitions, instance_accessor: false do + [] + end + + cattr_accessor :command_definitions_by_name, instance_accessor: false do + {} + end + end + + class_methods do + # Allows to give a description to the next slash command. + # This description is shown in the autocomplete menu. + # It accepts a block that will be evaluated with the context given to + # `CommandDefintion#to_h`. + # + # Example: + # + # desc do + # "This is a dynamic description for #{noteable.to_ability_name}" + # end + # command :command_key do |arguments| + # # Awesome code block + # end + def desc(text = '', &block) + @description = block_given? ? block : text + end + + # Allows to define params for the next slash command. + # These params are shown in the autocomplete menu. + # + # Example: + # + # params "~label ~label2" + # command :command_key do |arguments| + # # Awesome code block + # end + def params(*params) + @params = params + end + + # Allows to define conditions that must be met in order for the command + # to be returned by `.command_names` & `.command_definitions`. + # It accepts a block that will be evaluated with the context given to + # `CommandDefintion#to_h`. + # + # Example: + # + # condition do + # project.public? + # end + # command :command_key do |arguments| + # # Awesome code block + # end + def condition(&block) + @condition_block = block + end + + # Registers a new command which is recognizeable from body of email or + # comment. + # It accepts aliases and takes a block. + # + # Example: + # + # command :my_command, :alias_for_my_command do |arguments| + # # Awesome code block + # end + def command(*command_names, &block) + name, *aliases = command_names + + definition = CommandDefinition.new( + name, + aliases: aliases, + description: @description, + params: @params, + condition_block: @condition_block, + action_block: block + ) + + self.command_definitions << definition + + definition.all_names.each do |name| + self.command_definitions_by_name[name] = definition + end + + @description = nil + @params = nil + @condition_block = nil + end + end + end + end +end diff --git a/lib/gitlab/slash_commands/extractor.rb b/lib/gitlab/slash_commands/extractor.rb new file mode 100644 index 00000000000..6dbb467d70d --- /dev/null +++ b/lib/gitlab/slash_commands/extractor.rb @@ -0,0 +1,122 @@ +module Gitlab + module SlashCommands + # This class takes an array of commands that should be extracted from a + # given text. + # + # ``` + # extractor = Gitlab::SlashCommands::Extractor.new([:open, :assign, :labels]) + # ``` + class Extractor + attr_reader :command_definitions + + def initialize(command_definitions) + @command_definitions = command_definitions + end + + # Extracts commands from content and return an array of commands. + # The array looks like the following: + # [ + # ['command1'], + # ['command3', 'arg1 arg2'], + # ] + # The command and the arguments are stripped. + # The original command text is removed from the given `content`. + # + # Usage: + # ``` + # extractor = Gitlab::SlashCommands::Extractor.new([:open, :assign, :labels]) + # msg = %(hello\n/labels ~foo ~"bar baz"\nworld) + # commands = extractor.extract_commands(msg) #=> [['labels', '~foo ~"bar baz"']] + # msg #=> "hello\nworld" + # ``` + def extract_commands(content, opts = {}) + return [content, []] unless content + + content = content.dup + + commands = [] + + content.delete!("\r") + content.gsub!(commands_regex(opts)) do + if $~[:cmd] + commands << [$~[:cmd], $~[:arg]].reject(&:blank?) + '' + else + $~[0] + end + end + + [content.strip, commands] + end + + private + + # Builds a regular expression to match known commands. + # First match group captures the command name and + # second match group captures its arguments. + # + # It looks something like: + # + # /^\/(?<cmd>close|reopen|...)(?:( |$))(?<arg>[^\/\n]*)(?:\n|$)/ + def commands_regex(opts) + names = command_names(opts).map(&:to_s) + + @commands_regex ||= %r{ + (?<code> + # Code blocks: + # ``` + # Anything, including `/cmd arg` which are ignored by this filter + # ``` + + ^``` + .+? + \n```$ + ) + | + (?<html> + # HTML block: + # <tag> + # Anything, including `/cmd arg` which are ignored by this filter + # </tag> + + ^<[^>]+?>\n + .+? + \n<\/[^>]+?>$ + ) + | + (?<html> + # Quote block: + # >>> + # Anything, including `/cmd arg` which are ignored by this filter + # >>> + + ^>>> + .+? + \n>>>$ + ) + | + (?: + # Command not in a blockquote, blockcode, or HTML tag: + # /close + + ^\/ + (?<cmd>#{Regexp.union(names)}) + (?: + [ ] + (?<arg>[^\n]*) + )? + (?:\n|$) + ) + }mx + end + + def command_names(opts) + command_definitions.flat_map do |command| + next if command.noop? + + command.all_names + end.compact + end + end + end +end diff --git a/lib/gitlab/snippet_search_results.rb b/lib/gitlab/snippet_search_results.rb index e0e74ff8359..b85f70e450e 100644 --- a/lib/gitlab/snippet_search_results.rb +++ b/lib/gitlab/snippet_search_results.rb @@ -20,10 +20,6 @@ module Gitlab end end - def total_count - @total_count ||= snippet_titles_count + snippet_blobs_count - end - def snippet_titles_count @snippet_titles_count ||= snippet_titles.count end @@ -35,11 +31,11 @@ module Gitlab private def snippet_titles - limit_snippets.search(query).order('updated_at DESC') + limit_snippets.search(query).order('updated_at DESC').includes(:author) end def snippet_blobs - limit_snippets.search_code(query).order('updated_at DESC') + limit_snippets.search_code(query).order('updated_at DESC').includes(:author) end def default_scope diff --git a/lib/gitlab/sql/union.rb b/lib/gitlab/sql/union.rb index 1cd89b3a9c4..222021e8802 100644 --- a/lib/gitlab/sql/union.rb +++ b/lib/gitlab/sql/union.rb @@ -22,9 +22,7 @@ module Gitlab # By using "unprepared_statements" we remove the usage of placeholders # (thus fixing this problem), at a slight performance cost. fragments = ActiveRecord::Base.connection.unprepared_statement do - @relations.map do |rel| - rel.reorder(nil).to_sql - end + @relations.map { |rel| rel.reorder(nil).to_sql }.reject(&:blank?) end fragments.join("\nUNION\n") diff --git a/lib/gitlab/template/base_template.rb b/lib/gitlab/template/base_template.rb index 760ff3e614a..7ebec8e2cff 100644 --- a/lib/gitlab/template/base_template.rb +++ b/lib/gitlab/template/base_template.rb @@ -1,8 +1,9 @@ module Gitlab module Template class BaseTemplate - def initialize(path) + def initialize(path, project = nil) @path = path + @finder = self.class.finder(project) end def name @@ -10,23 +11,32 @@ module Gitlab end def content - File.read(@path) + @finder.read(@path) + end + + def to_json + { name: name, content: content } end class << self - def all - self.categories.keys.flat_map { |cat| by_category(cat) } + def all(project = nil) + if categories.any? + categories.keys.flat_map { |cat| by_category(cat, project) } + else + by_category("", project) + end end - def find(key) - file_name = "#{key}#{self.extension}" - - directory = select_directory(file_name) - directory ? new(File.join(category_directory(directory), file_name)) : nil + def find(key, project = nil) + path = self.finder(project).find(key) + path.present? ? new(path, project) : nil end + # Set categories as sub directories + # Example: { "category_name_1" => "directory_path_1", "category_name_2" => "directory_name_2" } + # Default is no category with all files in base dir of each class def categories - raise NotImplementedError + {} end def extension @@ -37,29 +47,40 @@ module Gitlab raise NotImplementedError end - def by_category(category) - templates_for_directory(category_directory(category)) + # Defines which strategy will be used to get templates files + # RepoTemplateFinder - Finds templates on project repository, templates are filtered perproject + # GlobalTemplateFinder - Finds templates on gitlab installation source, templates can be used in all projects + def finder(project = nil) + raise NotImplementedError end - def category_directory(category) - File.join(base_dir, categories[category]) + def by_category(category, project = nil) + directory = category_directory(category) + files = finder(project).list_files_for(directory) + + files.map { |f| new(f, project) } end - private + def category_directory(category) + return base_dir unless category.present? - def select_directory(file_name) - categories.keys.find do |category| - File.exist?(File.join(category_directory(category), file_name)) - end + File.join(base_dir, categories[category]) end - def templates_for_directory(dir) - dir << '/' unless dir.end_with?('/') - Dir.glob(File.join(dir, "*#{self.extension}")).select { |f| f =~ filter_regex }.map { |f| new(f) } - end + # If template is organized by category it returns { category_name: [{ name: template_name }, { name: template2_name }] } + # If no category is present returns [{ name: template_name }, { name: template2_name}] + def dropdown_names(project = nil) + return [] if project && !project.repository.exists? - def filter_regex - @filter_reges ||= /#{Regexp.escape(extension)}\z/ + if categories.any? + categories.keys.map do |category| + files = self.by_category(category, project) + [category, files.map { |t| { name: t.name } }] + end.to_h + else + files = self.all(project) + files.map { |t| { name: t.name } } + end end end end diff --git a/lib/gitlab/template/gitlab_ci_yml.rb b/lib/gitlab/template/dockerfile_template.rb index 7f480fe33c0..d5d3e045a42 100644 --- a/lib/gitlab/template/gitlab_ci_yml.rb +++ b/lib/gitlab/template/dockerfile_template.rb @@ -1,6 +1,6 @@ module Gitlab module Template - class GitlabCiYml < BaseTemplate + class DockerfileTemplate < BaseTemplate def content explanation = "# This file is a template, and might need editing before it works on your project." [explanation, super].join("\n") @@ -8,18 +8,21 @@ module Gitlab class << self def extension - '.gitlab-ci.yml' + 'Dockerfile' end def categories { - "General" => '', - "Pages" => 'Pages' + "General" => '' } end def base_dir - Rails.root.join('vendor/gitlab-ci-yml') + Rails.root.join('vendor/dockerfile') + end + + def finder(project = nil) + Gitlab::Template::Finders::GlobalTemplateFinder.new(self.base_dir, self.extension, self.categories) end end end diff --git a/lib/gitlab/template/finders/base_template_finder.rb b/lib/gitlab/template/finders/base_template_finder.rb new file mode 100644 index 00000000000..473b05257c6 --- /dev/null +++ b/lib/gitlab/template/finders/base_template_finder.rb @@ -0,0 +1,35 @@ +module Gitlab + module Template + module Finders + class BaseTemplateFinder + def initialize(base_dir) + @base_dir = base_dir + end + + def list_files_for + raise NotImplementedError + end + + def read + raise NotImplementedError + end + + def find + raise NotImplementedError + end + + def category_directory(category) + return @base_dir unless category.present? + + @base_dir + @categories[category] + end + + class << self + def filter_regex(extension) + /#{Regexp.escape(extension)}\z/ + end + end + end + end + end +end diff --git a/lib/gitlab/template/finders/global_template_finder.rb b/lib/gitlab/template/finders/global_template_finder.rb new file mode 100644 index 00000000000..831da45191f --- /dev/null +++ b/lib/gitlab/template/finders/global_template_finder.rb @@ -0,0 +1,38 @@ +# Searches and reads file present on Gitlab installation directory +module Gitlab + module Template + module Finders + class GlobalTemplateFinder < BaseTemplateFinder + def initialize(base_dir, extension, categories = {}) + @categories = categories + @extension = extension + super(base_dir) + end + + def read(path) + File.read(path) + end + + def find(key) + file_name = "#{key}#{@extension}" + + directory = select_directory(file_name) + directory ? File.join(category_directory(directory), file_name) : nil + end + + def list_files_for(dir) + dir << '/' unless dir.end_with?('/') + Dir.glob(File.join(dir, "*#{@extension}")).select { |f| f =~ self.class.filter_regex(@extension) } + end + + private + + def select_directory(file_name) + @categories.keys.find do |category| + File.exist?(File.join(category_directory(category), file_name)) + end + end + end + end + end +end diff --git a/lib/gitlab/template/finders/repo_template_finder.rb b/lib/gitlab/template/finders/repo_template_finder.rb new file mode 100644 index 00000000000..22c39436cb2 --- /dev/null +++ b/lib/gitlab/template/finders/repo_template_finder.rb @@ -0,0 +1,59 @@ +# Searches and reads files present on each Gitlab project repository +module Gitlab + module Template + module Finders + class RepoTemplateFinder < BaseTemplateFinder + # Raised when file is not found + class FileNotFoundError < StandardError; end + + def initialize(project, base_dir, extension, categories = {}) + @categories = categories + @extension = extension + @repository = project.repository + @commit = @repository.head_commit if @repository.exists? + + super(base_dir) + end + + def read(path) + blob = @repository.blob_at(@commit.id, path) if @commit + raise FileNotFoundError if blob.nil? + blob.data + end + + def find(key) + file_name = "#{key}#{@extension}" + directory = select_directory(file_name) + raise FileNotFoundError if directory.nil? + + category_directory(directory) + file_name + end + + def list_files_for(dir) + return [] unless @commit + + dir << '/' unless dir.end_with?('/') + + entries = @repository.tree(:head, dir).entries + + names = entries.map(&:name) + names.select { |f| f =~ self.class.filter_regex(@extension) } + end + + private + + def select_directory(file_name) + return [] unless @commit + + # Insert root as directory + directories = ["", @categories.keys] + + directories.find do |category| + path = category_directory(category) + file_name + @repository.blob_at(@commit.id, path) + end + end + end + end + end +end diff --git a/lib/gitlab/template/gitignore.rb b/lib/gitlab/template/gitignore_template.rb index 964fbfd4de3..8d2a9d2305c 100644 --- a/lib/gitlab/template/gitignore.rb +++ b/lib/gitlab/template/gitignore_template.rb @@ -1,6 +1,6 @@ module Gitlab module Template - class Gitignore < BaseTemplate + class GitignoreTemplate < BaseTemplate class << self def extension '.gitignore' @@ -16,6 +16,10 @@ module Gitlab def base_dir Rails.root.join('vendor/gitignore') end + + def finder(project = nil) + Gitlab::Template::Finders::GlobalTemplateFinder.new(self.base_dir, self.extension, self.categories) + end end end end diff --git a/lib/gitlab/template/gitlab_ci_yml_template.rb b/lib/gitlab/template/gitlab_ci_yml_template.rb new file mode 100644 index 00000000000..fd040148a1e --- /dev/null +++ b/lib/gitlab/template/gitlab_ci_yml_template.rb @@ -0,0 +1,37 @@ +module Gitlab + module Template + class GitlabCiYmlTemplate < BaseTemplate + def content + explanation = "# This file is a template, and might need editing before it works on your project." + [explanation, super].join("\n") + end + + class << self + def extension + '.gitlab-ci.yml' + end + + def categories + { + 'General' => '', + 'Pages' => 'Pages', + 'Auto deploy' => 'autodeploy' + } + end + + def base_dir + Rails.root.join('vendor/gitlab-ci-yml') + end + + def finder(project = nil) + Gitlab::Template::Finders::GlobalTemplateFinder.new(self.base_dir, self.extension, self.categories) + end + + def dropdown_names(context) + categories = context == 'autodeploy' ? ['Auto deploy'] : %w(General Pages) + super().slice(*categories) + end + end + end + end +end diff --git a/lib/gitlab/template/issue_template.rb b/lib/gitlab/template/issue_template.rb new file mode 100644 index 00000000000..c6fa8d3eafc --- /dev/null +++ b/lib/gitlab/template/issue_template.rb @@ -0,0 +1,19 @@ +module Gitlab + module Template + class IssueTemplate < BaseTemplate + class << self + def extension + '.md' + end + + def base_dir + '.gitlab/issue_templates/' + end + + def finder(project) + Gitlab::Template::Finders::RepoTemplateFinder.new(project, self.base_dir, self.extension, self.categories) + end + end + end + end +end diff --git a/lib/gitlab/template/merge_request_template.rb b/lib/gitlab/template/merge_request_template.rb new file mode 100644 index 00000000000..f826c02f3b5 --- /dev/null +++ b/lib/gitlab/template/merge_request_template.rb @@ -0,0 +1,19 @@ +module Gitlab + module Template + class MergeRequestTemplate < BaseTemplate + class << self + def extension + '.md' + end + + def base_dir + '.gitlab/merge_request_templates/' + end + + def finder(project) + Gitlab::Template::Finders::RepoTemplateFinder.new(project, self.base_dir, self.extension, self.categories) + end + end + end + end +end diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb deleted file mode 100644 index d4020af76f9..00000000000 --- a/lib/gitlab/themes.rb +++ /dev/null @@ -1,87 +0,0 @@ -module Gitlab - # Module containing GitLab's application theme definitions and helper methods - # for accessing them. - module Themes - extend self - - # Theme ID used when no `default_theme` configuration setting is provided. - APPLICATION_DEFAULT = 2 - - # Struct class representing a single Theme - Theme = Struct.new(:id, :name, :css_class) - - # All available Themes - THEMES = [ - Theme.new(1, 'Graphite', 'ui_graphite'), - Theme.new(2, 'Charcoal', 'ui_charcoal'), - Theme.new(3, 'Green', 'ui_green'), - Theme.new(4, 'Gray', 'ui_gray'), - Theme.new(5, 'Violet', 'ui_violet'), - Theme.new(6, 'Blue', 'ui_blue') - ].freeze - - # Convenience method to get a space-separated String of all the theme - # classes that might be applied to the `body` element - # - # Returns a String - def body_classes - THEMES.collect(&:css_class).uniq.join(' ') - end - - # Get a Theme by its ID - # - # If the ID is invalid, returns the default Theme. - # - # id - Integer ID - # - # Returns a Theme - def by_id(id) - THEMES.detect { |t| t.id == id } || default - end - - # Returns the number of defined Themes - def count - THEMES.size - end - - # Get the default Theme - # - # Returns a Theme - def default - by_id(default_id) - end - - # Iterate through each Theme - # - # Yields the Theme object - def each(&block) - THEMES.each(&block) - end - - # Get the Theme for the specified user, or the default - # - # user - User record - # - # Returns a Theme - def for_user(user) - if user - by_id(user.theme_id) - else - default - end - end - - private - - def default_id - id = Gitlab.config.gitlab.default_theme.to_i - - # Prevent an invalid configuration setting from causing an infinite loop - if id < THEMES.first.id || id > THEMES.last.id - APPLICATION_DEFAULT - else - id - end - end - end -end diff --git a/lib/gitlab/time_tracking_formatter.rb b/lib/gitlab/time_tracking_formatter.rb new file mode 100644 index 00000000000..d615c24149a --- /dev/null +++ b/lib/gitlab/time_tracking_formatter.rb @@ -0,0 +1,34 @@ +module Gitlab + module TimeTrackingFormatter + extend self + + def parse(string) + with_custom_config do + string.sub!(/\A-/, '') + + seconds = ChronicDuration.parse(string, default_unit: 'hours') rescue nil + seconds *= -1 if seconds && Regexp.last_match + seconds + end + end + + def output(seconds) + with_custom_config do + ChronicDuration.output(seconds, format: :short, limit_to_hours: false, weeks: true) rescue nil + end + end + + def with_custom_config + # We may want to configure it through project settings in a future version. + ChronicDuration.hours_per_day = 8 + ChronicDuration.days_per_week = 5 + + result = yield + + ChronicDuration.hours_per_day = 24 + ChronicDuration.days_per_week = 7 + + result + end + end +end diff --git a/lib/gitlab/update_path_error.rb b/lib/gitlab/update_path_error.rb new file mode 100644 index 00000000000..ce14cc887d0 --- /dev/null +++ b/lib/gitlab/update_path_error.rb @@ -0,0 +1,3 @@ +module Gitlab + class UpdatePathError < StandardError; end +end diff --git a/lib/gitlab/upgrader.rb b/lib/gitlab/upgrader.rb index f3567f3ef85..961df0468a4 100644 --- a/lib/gitlab/upgrader.rb +++ b/lib/gitlab/upgrader.rb @@ -46,7 +46,7 @@ module Gitlab git_tags = fetch_git_tags git_tags = git_tags.select { |version| version =~ /v\d+\.\d+\.\d+\Z/ } git_versions = git_tags.map { |tag| Gitlab::VersionInfo.parse(tag.match(/v\d+\.\d+\.\d+/).to_s) } - "v#{git_versions.sort.last.to_s}" + "v#{git_versions.sort.last}" end def fetch_git_tags @@ -59,15 +59,18 @@ module Gitlab "Stash changed files" => %W(#{Gitlab.config.git.bin_path} stash), "Get latest code" => %W(#{Gitlab.config.git.bin_path} fetch), "Switch to new version" => %W(#{Gitlab.config.git.bin_path} checkout v#{latest_version}), - "Install gems" => %W(bundle), - "Migrate DB" => %W(bundle exec rake db:migrate), - "Recompile assets" => %W(bundle exec rake assets:clean assets:precompile), - "Clear cache" => %W(bundle exec rake cache:clear) + "Install gems" => %w(bundle), + "Migrate DB" => %w(bundle exec rake db:migrate), + "Recompile assets" => %w(bundle exec rake yarn:install gitlab:assets:clean gitlab:assets:compile), + "Clear cache" => %w(bundle exec rake cache:clear) } end def env - { 'RAILS_ENV' => 'production' } + { + 'RAILS_ENV' => 'production', + 'NODE_ENV' => 'production' + } end def upgrade diff --git a/lib/gitlab/uploads_transfer.rb b/lib/gitlab/uploads_transfer.rb index be8fcc7b2d2..81701831a6a 100644 --- a/lib/gitlab/uploads_transfer.rb +++ b/lib/gitlab/uploads_transfer.rb @@ -1,33 +1,5 @@ module Gitlab - class UploadsTransfer - def move_project(project_path, namespace_path_was, namespace_path) - new_namespace_folder = File.join(root_dir, namespace_path) - FileUtils.mkdir_p(new_namespace_folder) unless Dir.exist?(new_namespace_folder) - from = File.join(root_dir, namespace_path_was, project_path) - to = File.join(root_dir, namespace_path, project_path) - move(from, to, "") - end - - def rename_project(path_was, path, namespace_path) - base_dir = File.join(root_dir, namespace_path) - move(path_was, path, base_dir) - end - - def rename_namespace(path_was, path) - move(path_was, path) - end - - private - - def move(path_was, path, base_dir = nil) - base_dir = root_dir unless base_dir - from = File.join(base_dir, path_was) - to = File.join(base_dir, path) - FileUtils.mv(from, to) - rescue Errno::ENOENT - false - end - + class UploadsTransfer < ProjectTransfer def root_dir File.join(Rails.root, "public", "uploads") end diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index fe65c246101..ccb456bcc94 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -22,6 +22,10 @@ module Gitlab note_url when WikiPage wiki_page_url + when ProjectSnippet + project_snippet_url(object) + when Snippet + personal_snippet_url(object) else raise NotImplementedError.new("No URL builder defined for #{object.class}") end diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb index 19dad699edf..1f0d96088cf 100644 --- a/lib/gitlab/url_sanitizer.rb +++ b/lib/gitlab/url_sanitizer.rb @@ -1,7 +1,7 @@ module Gitlab class UrlSanitizer def self.sanitize(content) - regexp = URI::Parser.new.make_regexp(['http', 'https', 'ssh', 'git']) + regexp = URI::Parser.new.make_regexp(%w(http https ssh git)) content.gsub(regexp) { |url| new(url).masked_url } rescue Addressable::URI::InvalidURIError diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb index c55a7fc4d3d..6ce9b229294 100644 --- a/lib/gitlab/user_access.rb +++ b/lib/gitlab/user_access.rb @@ -8,6 +8,8 @@ module Gitlab end def can_do_action?(action) + return false if no_user_or_blocked? + @permission_cache ||= {} @permission_cache[action] ||= user.can?(action, project) end @@ -17,7 +19,7 @@ module Gitlab end def allowed? - return false if user.blank? || user.blocked? + return false if no_user_or_blocked? if user.requires_ldap_check? && user.try_obtain_ldap_lease return false unless Gitlab::LDAP::Access.allowed?(user) @@ -27,23 +29,25 @@ module Gitlab end def can_push_to_branch?(ref) - return false unless user + return false if no_user_or_blocked? if project.protected_branch?(ref) return true if project.empty_repo? && project.user_can_push_to_empty_repo?(user) - access_levels = project.protected_branches.matching(ref).map(&:push_access_level) - access_levels.any? { |access_level| access_level.check_access(user) } + access_levels = project.protected_branches.matching(ref).map(&:push_access_levels).flatten + has_access = access_levels.any? { |access_level| access_level.check_access(user) } + + has_access || !project.repository.branch_exists?(ref) && can_merge_to_branch?(ref) else user.can?(:push_code, project) end end def can_merge_to_branch?(ref) - return false unless user + return false if no_user_or_blocked? if project.protected_branch?(ref) - access_levels = project.protected_branches.matching(ref).map(&:merge_access_level) + access_levels = project.protected_branches.matching(ref).map(&:merge_access_levels).flatten access_levels.any? { |access_level| access_level.check_access(user) } else user.can?(:push_code, project) @@ -51,9 +55,15 @@ module Gitlab end def can_read_project? - return false unless user + return false if no_user_or_blocked? user.can?(:read_project, project) end + + private + + def no_user_or_blocked? + user.nil? || user.blocked? + end end end diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index d13fe0ef8a9..4c395b4266e 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -7,11 +7,19 @@ module Gitlab # @param cmd [Array<String>] # @return [Boolean] def system_silent(cmd) - Popen::popen(cmd).last.zero? + Popen.popen(cmd).last.zero? end def force_utf8(str) str.force_encoding(Encoding::UTF_8) end + + def to_boolean(value) + return value if [true, false].include?(value) + return true if value =~ /^(true|t|yes|y|1|on)$/i + return false if value =~ /^(false|f|no|n|0|off)$/i + + nil + end end end diff --git a/lib/gitlab/view/presenter/base.rb b/lib/gitlab/view/presenter/base.rb new file mode 100644 index 00000000000..dbfe0941e4d --- /dev/null +++ b/lib/gitlab/view/presenter/base.rb @@ -0,0 +1,30 @@ +module Gitlab + module View + module Presenter + CannotOverrideMethodError = Class.new(StandardError) + + module Base + extend ActiveSupport::Concern + + include Gitlab::Routing + include Gitlab::Allowable + + attr_reader :subject + + def can?(user, action, overriden_subject = nil) + super(user, action, overriden_subject || subject) + end + + class_methods do + def presenter? + true + end + + def presents(name) + define_method(name) { subject } + end + end + end + end + end +end diff --git a/lib/gitlab/view/presenter/delegated.rb b/lib/gitlab/view/presenter/delegated.rb new file mode 100644 index 00000000000..387ff0f5d43 --- /dev/null +++ b/lib/gitlab/view/presenter/delegated.rb @@ -0,0 +1,23 @@ +module Gitlab + module View + module Presenter + class Delegated < SimpleDelegator + include Gitlab::View::Presenter::Base + + def initialize(subject, **attributes) + @subject = subject + + attributes.each do |key, value| + if subject.respond_to?(key) + raise CannotOverrideMethodError.new("#{subject} already respond to #{key}!") + end + + define_singleton_method(key) { value } + end + + super(subject) + end + end + end + end +end diff --git a/lib/gitlab/view/presenter/factory.rb b/lib/gitlab/view/presenter/factory.rb new file mode 100644 index 00000000000..d172d61e2c9 --- /dev/null +++ b/lib/gitlab/view/presenter/factory.rb @@ -0,0 +1,24 @@ +module Gitlab + module View + module Presenter + class Factory + def initialize(subject, **attributes) + @subject = subject + @attributes = attributes + end + + def fabricate! + presenter_class.new(subject, attributes) + end + + private + + attr_reader :subject, :attributes + + def presenter_class + "#{subject.class.name}Presenter".constantize + end + end + end + end +end diff --git a/lib/gitlab/view/presenter/simple.rb b/lib/gitlab/view/presenter/simple.rb new file mode 100644 index 00000000000..b7653a0f3cc --- /dev/null +++ b/lib/gitlab/view/presenter/simple.rb @@ -0,0 +1,17 @@ +module Gitlab + module View + module Presenter + class Simple + include Gitlab::View::Presenter::Base + + def initialize(subject, **attributes) + @subject = subject + + attributes.each do |key, value| + define_singleton_method(key) { value } + end + end + end + end + end +end diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index 9462f3368e6..b28708c34e1 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -11,8 +11,21 @@ module Gitlab included do scope :public_only, -> { where(visibility_level: PUBLIC) } scope :public_and_internal_only, -> { where(visibility_level: [PUBLIC, INTERNAL] ) } - - scope :public_to_user, -> (user) { user && !user.external ? public_and_internal_only : public_only } + scope :non_public_only, -> { where.not(visibility_level: PUBLIC) } + + scope :public_to_user, -> (user) do + if user + if user.admin? + all + elsif !user.external? + public_and_internal_only + else + public_only + end + else + public_only + end + end end PRIVATE = 0 unless const_defined?(:PRIVATE) @@ -20,9 +33,7 @@ module Gitlab PUBLIC = 20 unless const_defined?(:PUBLIC) class << self - def values - options.values - end + delegate :values, to: :options def options { diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index c6826a09bd2..3ff9f9eb5e7 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -1,19 +1,45 @@ require 'base64' require 'json' +require 'securerandom' module Gitlab class Workhorse - SEND_DATA_HEADER = 'Gitlab-Workhorse-Send-Data' - VERSION_FILE = 'GITLAB_WORKHORSE_VERSION' + SEND_DATA_HEADER = 'Gitlab-Workhorse-Send-Data'.freeze + VERSION_FILE = 'GITLAB_WORKHORSE_VERSION'.freeze + INTERNAL_API_CONTENT_TYPE = 'application/vnd.gitlab-workhorse+json'.freeze + INTERNAL_API_REQUEST_HEADER = 'Gitlab-Workhorse-Api-Request'.freeze + + # Supposedly the effective key size for HMAC-SHA256 is 256 bits, i.e. 32 + # bytes https://tools.ietf.org/html/rfc4868#section-2.6 + SECRET_LENGTH = 32 class << self def git_http_ok(repository, user) + params = { + GL_ID: Gitlab::GlId.gl_id(user), + RepoPath: repository.path_to_repo, + } + + params.merge!( + GitalySocketPath: Gitlab.config.gitaly.socket_path, + GitalyResourcePath: "/projects/#{repository.project.id}/git-http/info-refs", + ) if Gitlab.config.gitaly.socket_path.present? + + params + end + + def lfs_upload_ok(oid, size) { - 'GL_ID' => Gitlab::GlId.gl_id(user), - 'RepoPath' => repository.path_to_repo, + StoreLFSPath: "#{Gitlab.config.lfs.storage_path}/tmp/upload", + LfsOid: oid, + LfsSize: size, } end + def artifact_upload_ok + { TempPath: ArtifactUploader.artifacts_upload_path } + end + def send_git_blob(repository, blob) params = { 'RepoPath' => repository.path_to_repo, @@ -41,7 +67,7 @@ module Gitlab def send_git_diff(repository, diff_refs) params = { 'RepoPath' => repository.path_to_repo, - 'ShaFrom' => diff_refs.start_sha, + 'ShaFrom' => diff_refs.base_sha, 'ShaTo' => diff_refs.head_sha } @@ -54,7 +80,7 @@ module Gitlab def send_git_patch(repository, diff_refs) params = { 'RepoPath' => repository.path_to_repo, - 'ShaFrom' => diff_refs.start_sha, + 'ShaFrom' => diff_refs.base_sha, 'ShaTo' => diff_refs.head_sha } @@ -76,11 +102,58 @@ module Gitlab ] end + def terminal_websocket(terminal) + details = { + 'Terminal' => { + 'Subprotocols' => terminal[:subprotocols], + 'Url' => terminal[:url], + 'Header' => terminal[:headers], + 'MaxSessionTime' => terminal[:max_session_time], + } + } + details['Terminal']['CAPem'] = terminal[:ca_pem] if terminal.has_key?(:ca_pem) + + details + end + def version path = Rails.root.join(VERSION_FILE) path.readable? ? path.read.chomp : 'unknown' end + def secret + @secret ||= begin + bytes = Base64.strict_decode64(File.read(secret_path).chomp) + raise "#{secret_path} does not contain #{SECRET_LENGTH} bytes" if bytes.length != SECRET_LENGTH + bytes + end + end + + def write_secret + bytes = SecureRandom.random_bytes(SECRET_LENGTH) + File.open(secret_path, 'w:BINARY', 0600) do |f| + f.chmod(0600) # If the file already existed, the '0600' passed to 'open' above was a no-op. + f.write(Base64.strict_encode64(bytes)) + end + end + + def verify_api_request!(request_headers) + decode_jwt(request_headers[INTERNAL_API_REQUEST_HEADER]) + end + + def decode_jwt(encoded_message) + JWT.decode( + encoded_message, + secret, + true, + { iss: 'gitlab-workhorse', verify_iss: true, algorithm: 'HS256' }, + ) + end + + def secret_path + Rails.root.join('.gitlab_workhorse_secret') + end + protected def encode(hash) diff --git a/lib/mattermost/client.rb b/lib/mattermost/client.rb new file mode 100644 index 00000000000..e55c0d6ac49 --- /dev/null +++ b/lib/mattermost/client.rb @@ -0,0 +1,51 @@ +module Mattermost + class ClientError < Mattermost::Error; end + + class Client + attr_reader :user + + def initialize(user) + @user = user + end + + def with_session(&blk) + Mattermost::Session.new(user).with_session(&blk) + end + + private + + # Should be used in a session manually + def get(session, path, options = {}) + json_response session.get(path, options) + end + + # Should be used in a session manually + def post(session, path, options = {}) + json_response session.post(path, options) + end + + def session_get(path, options = {}) + with_session do |session| + get(session, path, options) + end + end + + def session_post(path, options = {}) + with_session do |session| + post(session, path, options) + end + end + + def json_response(response) + json_response = JSON.parse(response.body) + + unless response.success? + raise Mattermost::ClientError.new(json_response['message'] || 'Undefined error') + end + + json_response + rescue JSON::JSONError + raise Mattermost::ClientError.new('Cannot parse response') + end + end +end diff --git a/lib/mattermost/command.rb b/lib/mattermost/command.rb new file mode 100644 index 00000000000..33e450d7f0a --- /dev/null +++ b/lib/mattermost/command.rb @@ -0,0 +1,10 @@ +module Mattermost + class Command < Client + def create(params) + response = session_post("/api/v3/teams/#{params[:team_id]}/commands/create", + body: params.to_json) + + response['token'] + end + end +end diff --git a/lib/mattermost/error.rb b/lib/mattermost/error.rb new file mode 100644 index 00000000000..014df175be0 --- /dev/null +++ b/lib/mattermost/error.rb @@ -0,0 +1,3 @@ +module Mattermost + class Error < StandardError; end +end diff --git a/lib/mattermost/session.rb b/lib/mattermost/session.rb new file mode 100644 index 00000000000..377cb7b1021 --- /dev/null +++ b/lib/mattermost/session.rb @@ -0,0 +1,160 @@ +module Mattermost + class NoSessionError < Mattermost::Error + def message + 'No session could be set up, is Mattermost configured with Single Sign On?' + end + end + + class ConnectionError < Mattermost::Error; end + + # This class' prime objective is to obtain a session token on a Mattermost + # instance with SSO configured where this GitLab instance is the provider. + # + # The process depends on OAuth, but skips a step in the authentication cycle. + # For example, usually a user would click the 'login in GitLab' button on + # Mattermost, which would yield a 302 status code and redirects you to GitLab + # to approve the use of your account on Mattermost. Which would trigger a + # callback so Mattermost knows this request is approved and gets the required + # data to create the user account etc. + # + # This class however skips the button click, and also the approval phase to + # speed up the process and keep it without manual action and get a session + # going. + class Session + include Doorkeeper::Helpers::Controller + include HTTParty + + LEASE_TIMEOUT = 60 + + base_uri Settings.mattermost.host + + attr_accessor :current_resource_owner, :token + + def initialize(current_user) + @current_resource_owner = current_user + end + + def with_session + with_lease do + raise Mattermost::NoSessionError unless create + + begin + yield self + rescue Errno::ECONNREFUSED + raise Mattermost::NoSessionError + ensure + destroy + end + end + end + + # Next methods are needed for Doorkeeper + def pre_auth + @pre_auth ||= Doorkeeper::OAuth::PreAuthorization.new( + Doorkeeper.configuration, server.client_via_uid, params) + end + + def authorization + @authorization ||= strategy.request + end + + def strategy + @strategy ||= server.authorization_request(pre_auth.response_type) + end + + def request + @request ||= OpenStruct.new(parameters: params) + end + + def params + Rack::Utils.parse_query(oauth_uri.query).symbolize_keys + end + + def get(path, options = {}) + handle_exceptions do + self.class.get(path, options.merge(headers: @headers)) + end + end + + def post(path, options = {}) + handle_exceptions do + self.class.post(path, options.merge(headers: @headers)) + end + end + + private + + def create + return unless oauth_uri + return unless token_uri + + @token = request_token + @headers = { + Authorization: "Bearer #{@token}" + } + + @token + end + + def destroy + post('/api/v3/users/logout') + end + + def oauth_uri + return @oauth_uri if defined?(@oauth_uri) + + @oauth_uri = nil + + response = get("/api/v3/oauth/gitlab/login", follow_redirects: false) + return unless 300 <= response.code && response.code < 400 + + redirect_uri = response.headers['location'] + return unless redirect_uri + + @oauth_uri = URI.parse(redirect_uri) + end + + def token_uri + @token_uri ||= + if oauth_uri + authorization.authorize.redirect_uri if pre_auth.authorizable? + end + end + + def request_token + response = get(token_uri, follow_redirects: false) + + if 200 <= response.code && response.code < 400 + response.headers['token'] + end + end + + def with_lease + lease_uuid = lease_try_obtain + raise NoSessionError unless lease_uuid + + begin + yield + ensure + Gitlab::ExclusiveLease.cancel(lease_key, lease_uuid) + end + end + + def lease_key + "mattermost:session" + end + + def lease_try_obtain + lease = ::Gitlab::ExclusiveLease.new(lease_key, timeout: LEASE_TIMEOUT) + lease.try_obtain + end + + def handle_exceptions + yield + rescue HTTParty::Error => e + raise Mattermost::ConnectionError.new(e.message) + rescue Errno::ECONNREFUSED + raise Mattermost::ConnectionError.new(e.message) + end + end +end diff --git a/lib/mattermost/team.rb b/lib/mattermost/team.rb new file mode 100644 index 00000000000..09dfd082b3a --- /dev/null +++ b/lib/mattermost/team.rb @@ -0,0 +1,7 @@ +module Mattermost + class Team < Client + def all + session_get('/api/v3/teams/all') + end + end +end diff --git a/lib/omniauth/strategies/bitbucket.rb b/lib/omniauth/strategies/bitbucket.rb new file mode 100644 index 00000000000..5a7d67c2390 --- /dev/null +++ b/lib/omniauth/strategies/bitbucket.rb @@ -0,0 +1,41 @@ +require 'omniauth-oauth2' + +module OmniAuth + module Strategies + class Bitbucket < OmniAuth::Strategies::OAuth2 + option :name, 'bitbucket' + + option :client_options, { + site: 'https://bitbucket.org', + authorize_url: 'https://bitbucket.org/site/oauth2/authorize', + token_url: 'https://bitbucket.org/site/oauth2/access_token' + } + + uid do + raw_info['username'] + end + + info do + { + name: raw_info['display_name'], + avatar: raw_info['links']['avatar']['href'], + email: primary_email + } + end + + def raw_info + @raw_info ||= access_token.get('api/2.0/user').parsed + end + + def primary_email + primary = emails.find { |i| i['is_primary'] && i['is_confirmed'] } + primary && primary['email'] || nil + end + + def emails + email_response = access_token.get('api/2.0/user/emails').parsed + @emails ||= email_response && email_response['values'] || nil + end + end + end +end diff --git a/lib/rouge/lexers/math.rb b/lib/rouge/lexers/math.rb new file mode 100644 index 00000000000..80784adfd76 --- /dev/null +++ b/lib/rouge/lexers/math.rb @@ -0,0 +1,21 @@ +module Rouge + module Lexers + class Math < Lexer + title "A passthrough lexer used for LaTeX input" + desc "A boring lexer that doesn't highlight anything" + + tag 'math' + mimetypes 'text/plain' + + default_options token: 'Text' + + def token + @token ||= Token[option :token] + end + + def stream_tokens(string, &b) + yield self.token, string + end + end + end +end diff --git a/lib/rouge/lexers/plantuml.rb b/lib/rouge/lexers/plantuml.rb new file mode 100644 index 00000000000..7d5700b7f6d --- /dev/null +++ b/lib/rouge/lexers/plantuml.rb @@ -0,0 +1,21 @@ +module Rouge + module Lexers + class Plantuml < Lexer + title "A passthrough lexer used for PlantUML input" + desc "A boring lexer that doesn't highlight anything" + + tag 'plantuml' + mimetypes 'text/plain' + + default_options token: 'Text' + + def token + @token ||= Token[option :token] + end + + def stream_tokens(string, &b) + yield self.token, string + end + end + end +end diff --git a/lib/support/deploy/deploy.sh b/lib/support/deploy/deploy.sh index adea4c7a747..ab46c47d8f5 100755 --- a/lib/support/deploy/deploy.sh +++ b/lib/support/deploy/deploy.sh @@ -31,8 +31,8 @@ echo 'Deploy: Bundle and migrate' sudo -u git -H bundle --without aws development test mysql --deployment sudo -u git -H bundle exec rake db:migrate RAILS_ENV=production -sudo -u git -H bundle exec rake assets:clean RAILS_ENV=production -sudo -u git -H bundle exec rake assets:precompile RAILS_ENV=production +sudo -u git -H bundle exec rake gitlab:assets:clean RAILS_ENV=production +sudo -u git -H bundle exec rake gitlab:assets:compile RAILS_ENV=production sudo -u git -H bundle exec rake cache:clear RAILS_ENV=production # return stashed changes (if necessary) diff --git a/lib/support/init.d/gitlab b/lib/support/init.d/gitlab index 31b00ff128a..5fd7f0f98bd 100755 --- a/lib/support/init.d/gitlab +++ b/lib/support/init.d/gitlab @@ -42,6 +42,11 @@ gitlab_workhorse_dir=$(cd $app_root/../gitlab-workhorse 2> /dev/null && pwd) gitlab_workhorse_pid_path="$pid_path/gitlab-workhorse.pid" gitlab_workhorse_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-workhorse.socket -authBackend http://127.0.0.1:8080 -authSocket $rails_socket -documentRoot $app_root/public" gitlab_workhorse_log="$app_root/log/gitlab-workhorse.log" +gitlab_pages_enabled=false +gitlab_pages_dir=$(cd $app_root/../gitlab-pages 2> /dev/null && pwd) +gitlab_pages_pid_path="$pid_path/gitlab-pages.pid" +gitlab_pages_options="-pages-domain example.com -pages-root $app_root/shared/pages -listen-proxy 127.0.0.1:8090" +gitlab_pages_log="$app_root/log/gitlab-pages.log" shell_path="/bin/bash" # Read configuration variable file if it is present @@ -89,13 +94,20 @@ check_pids(){ mpid=0 fi fi + if [ "$gitlab_pages_enabled" = true ]; then + if [ -f "$gitlab_pages_pid_path" ]; then + gppid=$(cat "$gitlab_pages_pid_path") + else + gppid=0 + fi + fi } ## Called when we have started the two processes and are waiting for their pid files. wait_for_pids(){ # We are sleeping a bit here mostly because sidekiq is slow at writing its pid i=0; - while [ ! -f $web_server_pid_path ] || [ ! -f $sidekiq_pid_path ] || [ ! -f $gitlab_workhorse_pid_path ] || { [ "$mail_room_enabled" = true ] && [ ! -f $mail_room_pid_path ]; }; do + while [ ! -f $web_server_pid_path ] || [ ! -f $sidekiq_pid_path ] || [ ! -f $gitlab_workhorse_pid_path ] || { [ "$mail_room_enabled" = true ] && [ ! -f $mail_room_pid_path ]; } || { [ "$gitlab_pages_enabled" = true ] && [ ! -f $gitlab_pages_pid_path ]; }; do sleep 0.1; i=$((i+1)) if [ $((i%10)) = 0 ]; then @@ -144,7 +156,15 @@ check_status(){ mail_room_status="-1" fi fi - if [ $web_status = 0 ] && [ $sidekiq_status = 0 ] && [ $gitlab_workhorse_status = 0 ] && { [ "$mail_room_enabled" != true ] || [ $mail_room_status = 0 ]; }; then + if [ "$gitlab_pages_enabled" = true ]; then + if [ $gppid -ne 0 ]; then + kill -0 "$gppid" 2>/dev/null + gitlab_pages_status="$?" + else + gitlab_pages_status="-1" + fi + fi + if [ $web_status = 0 ] && [ $sidekiq_status = 0 ] && [ $gitlab_workhorse_status = 0 ] && { [ "$mail_room_enabled" != true ] || [ $mail_room_status = 0 ]; } && { [ "$gitlab_pages_enabled" != true ] || [ $gitlab_pages_status = 0 ]; }; then gitlab_status=0 else # http://refspecs.linuxbase.org/LSB_4.1.0/LSB-Core-generic/LSB-Core-generic/iniscrptact.html @@ -186,12 +206,19 @@ check_stale_pids(){ exit 1 fi fi + if [ "$gitlab_pages_enabled" = true ] && [ "$gppid" != "0" ] && [ "$gitlab_pages_status" != "0" ]; then + echo "Removing stale GitLab Pages job dispatcher pid. This is most likely caused by GitLab Pages crashing the last time it ran." + if ! rm "$gitlab_pages_pid_path"; then + echo "Unable to remove stale pid, exiting" + exit 1 + fi + fi } ## If no parts of the service is running, bail out. exit_if_not_running(){ check_stale_pids - if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then + if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; } && { [ "$gitlab_pages_enabled" != true ] || [ "$gitlab_pages_status" != "0" ]; }; then echo "GitLab is not running." exit fi @@ -213,6 +240,9 @@ start_gitlab() { if [ "$mail_room_enabled" = true ] && [ "$mail_room_status" != "0" ]; then echo "Starting GitLab MailRoom" fi + if [ "$gitlab_pages_enabled" = true ] && [ "$gitlab_pages_status" != "0" ]; then + echo "Starting GitLab Pages" + fi # Then check if the service is running. If it is: don't start again. if [ "$web_status" = "0" ]; then @@ -252,6 +282,16 @@ start_gitlab() { fi fi + if [ "$gitlab_pages_enabled" = true ]; then + if [ "$gitlab_pages_status" = "0" ]; then + echo "The GitLab Pages is already running with pid $spid, not restarting" + else + $app_root/bin/daemon_with_pidfile $gitlab_pages_pid_path \ + $gitlab_pages_dir/gitlab-pages $gitlab_pages_options \ + >> $gitlab_pages_log 2>&1 & + fi + fi + # Wait for the pids to be planted wait_for_pids # Finally check the status to tell wether or not GitLab is running @@ -278,13 +318,17 @@ stop_gitlab() { echo "Shutting down GitLab MailRoom" RAILS_ENV=$RAILS_ENV bin/mail_room stop fi + if [ "$gitlab_pages_status" = "0" ]; then + echo "Shutting down gitlab-pages" + kill -- $(cat $gitlab_pages_pid_path) + fi # If something needs to be stopped, lets wait for it to stop. Never use SIGKILL in a script. - while [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_workhorse_status" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; }; do + while [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_workhorse_status" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; } || { [ "$gitlab_pages_enabled" = true ] && [ "$gitlab_pages_status" = "0" ]; }; do sleep 1 check_status printf "." - if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then + if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; } && { [ "$gitlab_pages_enabled" != true ] || [ "$gitlab_pages_status" != "0" ]; }; then printf "\n" break fi @@ -298,6 +342,7 @@ stop_gitlab() { if [ "$mail_room_enabled" = true ]; then rm "$mail_room_pid_path" 2>/dev/null fi + rm -f "$gitlab_pages_pid_path" print_status } @@ -305,7 +350,7 @@ stop_gitlab() { ## Prints the status of GitLab and its components. print_status() { check_status - if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then + if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && [ "$gitlab_workhorse_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; } && { [ "$gitlab_pages_enabled" != true ] || [ "$gitlab_pages_status" != "0" ]; }; then echo "GitLab is not running." return fi @@ -331,7 +376,14 @@ print_status() { printf "The GitLab MailRoom email processor is \033[31mnot running\033[0m.\n" fi fi - if [ "$web_status" = "0" ] && [ "$sidekiq_status" = "0" ] && [ "$gitlab_workhorse_status" = "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" = "0" ]; }; then + if [ "$gitlab_pages_enabled" = true ]; then + if [ "$gitlab_pages_status" = "0" ]; then + echo "The GitLab Pages with pid $mpid is running." + else + printf "The GitLab Pages is \033[31mnot running\033[0m.\n" + fi + fi + if [ "$web_status" = "0" ] && [ "$sidekiq_status" = "0" ] && [ "$gitlab_workhorse_status" = "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" = "0" ]; } && { [ "$gitlab_pages_enabled" != true ] || [ "$gitlab_pages_status" = "0" ]; }; then printf "GitLab and all its components are \033[32mup and running\033[0m.\n" fi } @@ -362,7 +414,7 @@ reload_gitlab(){ ## Restarts Sidekiq and Unicorn. restart_gitlab(){ check_status - if [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_workhorse" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; }; then + if [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || [ "$gitlab_workhorse" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; } || { [ "$gitlab_pages_enabled" = true ] && [ "$gitlab_pages_status" = "0" ]; }; then stop_gitlab fi start_gitlab diff --git a/lib/support/init.d/gitlab.default.example b/lib/support/init.d/gitlab.default.example index cc8617b72ca..e5797d8fe3c 100755..100644 --- a/lib/support/init.d/gitlab.default.example +++ b/lib/support/init.d/gitlab.default.example @@ -47,6 +47,30 @@ gitlab_workhorse_pid_path="$pid_path/gitlab-workhorse.pid" gitlab_workhorse_options="-listenUmask 0 -listenNetwork unix -listenAddr $socket_path/gitlab-workhorse.socket -authBackend http://127.0.0.1:8080 -authSocket $socket_path/gitlab.socket -documentRoot $app_root/public" gitlab_workhorse_log="$app_root/log/gitlab-workhorse.log" +# The GitLab Pages Daemon needs either a separate IP address on which it will +# listen or use different ports than 80 or 443 that will be forwarded to GitLab +# Pages Daemon. +# +# To enable HTTP support for custom domains add the `-listen-http` directive +# in `gitlab_pages_options` below. +# The value of -listen-http must be set to `gitlab.yml > pages > external_http` +# as well. For example: +# +# -listen-http 1.1.1.1:80 +# +# To enable HTTPS support for custom domains add the `-listen-https`, +# `-root-cert` and `-root-key` directives in `gitlab_pages_options` below. +# The value of -listen-https must be set to `gitlab.yml > pages > external_https` +# as well. For example: +# +# -listen-https 1.1.1.1:443 -root-cert /path/to/example.com.crt -root-key /path/to/example.com.key +# +# The -pages-domain must be specified the same as in `gitlab.yml > pages > host`. +# Set `gitlab_pages_enabled=true` if you want to enable the Pages feature. +gitlab_pages_enabled=false +gitlab_pages_options="-pages-domain example.com -pages-root $app_root/shared/pages -listen-proxy 127.0.0.1:8090" +gitlab_pages_log="$app_root/log/gitlab-pages.log" + # mail_room_enabled specifies whether mail_room, which is used to process incoming email, is enabled. # This is required for the Reply by email feature. # The default is "false" diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab index d521de28e8a..2f7c34a3f31 100644 --- a/lib/support/nginx/gitlab +++ b/lib/support/nginx/gitlab @@ -20,6 +20,11 @@ upstream gitlab-workhorse { server unix:/home/git/gitlab/tmp/sockets/gitlab-workhorse.socket fail_timeout=0; } +map $http_upgrade $connection_upgrade_gitlab { + default upgrade; + '' close; +} + ## Normal HTTP host server { ## Either remove "default_server" from the listen line below, @@ -53,6 +58,8 @@ server { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade_gitlab; proxy_pass http://gitlab-workhorse; } diff --git a/lib/support/nginx/gitlab-pages b/lib/support/nginx/gitlab-pages new file mode 100644 index 00000000000..d9746c5c1aa --- /dev/null +++ b/lib/support/nginx/gitlab-pages @@ -0,0 +1,28 @@ +## GitLab +## + +## Pages serving host +server { + listen 0.0.0.0:80; + listen [::]:80 ipv6only=on; + + ## Replace this with something like pages.gitlab.com + server_name ~^.*\.YOUR_GITLAB_PAGES\.DOMAIN$; + + ## Individual nginx logs for GitLab pages + access_log /var/log/nginx/gitlab_pages_access.log; + error_log /var/log/nginx/gitlab_pages_error.log; + + location / { + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + # The same address as passed to GitLab Pages: `-listen-proxy` + proxy_pass http://localhost:8090/; + } + + # Define custom error pages + error_page 403 /403.html; + error_page 404 /404.html; +} diff --git a/lib/support/nginx/gitlab-pages-ssl b/lib/support/nginx/gitlab-pages-ssl new file mode 100644 index 00000000000..a1ccf266835 --- /dev/null +++ b/lib/support/nginx/gitlab-pages-ssl @@ -0,0 +1,77 @@ +## GitLab +## + +## Redirects all HTTP traffic to the HTTPS host +server { + ## Either remove "default_server" from the listen line below, + ## or delete the /etc/nginx/sites-enabled/default file. This will cause gitlab + ## to be served if you visit any address that your server responds to, eg. + ## the ip address of the server (http://x.x.x.x/) + listen 0.0.0.0:80; + listen [::]:80 ipv6only=on; + + ## Replace this with something like pages.gitlab.com + server_name ~^.*\.YOUR_GITLAB_PAGES\.DOMAIN$; + server_tokens off; ## Don't show the nginx version number, a security best practice + + return 301 https://$http_host$request_uri; + + access_log /var/log/nginx/gitlab_pages_access.log; + error_log /var/log/nginx/gitlab_pages_access.log; +} + +## Pages serving host +server { + listen 0.0.0.0:443 ssl; + listen [::]:443 ipv6only=on ssl http2; + + ## Replace this with something like pages.gitlab.com + server_name ~^.*\.YOUR_GITLAB_PAGES\.DOMAIN$; + server_tokens off; ## Don't show the nginx version number, a security best practice + + ## Strong SSL Security + ## https://raymii.org/s/tutorials/Strong_SSL_Security_On_nginx.html & https://cipherli.st/ + ssl on; + ssl_certificate /etc/nginx/ssl/gitlab-pages.crt; + ssl_certificate_key /etc/nginx/ssl/gitlab-pages.key; + + # GitLab needs backwards compatible ciphers to retain compatibility with Java IDEs + ssl_ciphers "ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA256:ECDHE-RSA-AES256-SHA:ECDHE-RSA-AES128-SHA:ECDHE-RSA-DES-CBC3-SHA:AES256-GCM-SHA384:AES128-GCM-SHA256:AES256-SHA256:AES128-SHA256:AES256-SHA:AES128-SHA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!MD5:!PSK:!RC4"; + ssl_protocols TLSv1 TLSv1.1 TLSv1.2; + ssl_prefer_server_ciphers on; + ssl_session_cache shared:SSL:10m; + ssl_session_timeout 5m; + + ## See app/controllers/application_controller.rb for headers set + + ## [Optional] If your certficate has OCSP, enable OCSP stapling to reduce the overhead and latency of running SSL. + ## Replace with your ssl_trusted_certificate. For more info see: + ## - https://medium.com/devops-programming/4445f4862461 + ## - https://www.ruby-forum.com/topic/4419319 + ## - https://www.digitalocean.com/community/tutorials/how-to-configure-ocsp-stapling-on-apache-and-nginx + # ssl_stapling on; + # ssl_stapling_verify on; + # ssl_trusted_certificate /etc/nginx/ssl/stapling.trusted.crt; + + ## [Optional] Generate a stronger DHE parameter: + ## sudo openssl dhparam -out /etc/ssl/certs/dhparam.pem 4096 + ## + # ssl_dhparam /etc/ssl/certs/dhparam.pem; + + ## Individual nginx logs for GitLab pages + access_log /var/log/nginx/gitlab_pages_access.log; + error_log /var/log/nginx/gitlab_pages_error.log; + + location / { + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + # The same address as passed to GitLab Pages: `-listen-proxy` + proxy_pass http://localhost:8090/; + } + + # Define custom error pages + error_page 403 /403.html; + error_page 404 /404.html; +} diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl index bf014b56cf6..5661394058d 100644 --- a/lib/support/nginx/gitlab-ssl +++ b/lib/support/nginx/gitlab-ssl @@ -24,6 +24,11 @@ upstream gitlab-workhorse { server unix:/home/git/gitlab/tmp/sockets/gitlab-workhorse.socket fail_timeout=0; } +map $http_upgrade $connection_upgrade_gitlab_ssl { + default upgrade; + '' close; +} + ## Redirects all HTTP traffic to the HTTPS host server { ## Either remove "default_server" from the listen line below, @@ -98,6 +103,9 @@ server { proxy_set_header X-Forwarded-Ssl on; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade_gitlab_ssl; + proxy_pass http://gitlab-workhorse; } diff --git a/lib/tasks/.gitkeep b/lib/tasks/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 --- a/lib/tasks/.gitkeep +++ /dev/null diff --git a/lib/tasks/brakeman.rake b/lib/tasks/brakeman.rake index d5a402907d8..2301ec9b228 100644 --- a/lib/tasks/brakeman.rake +++ b/lib/tasks/brakeman.rake @@ -2,7 +2,7 @@ desc 'Security check via brakeman' task :brakeman do # We get 0 warnings at level 'w3' but we would like to reach 'w2'. Merge # requests are welcome! - if system(*%W(brakeman --no-progress --skip-files lib/backup/repository.rb -w3 -z)) + if system(*%w(brakeman --no-progress --skip-files lib/backup/repository.rb -w3 -z)) puts 'Security check succeed' else puts 'Security check failed' diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake index 2214f855200..d55923673b1 100644 --- a/lib/tasks/cache.rake +++ b/lib/tasks/cache.rake @@ -1,22 +1,33 @@ namespace :cache do - CLEAR_BATCH_SIZE = 1000 # There seems to be no speedup when pushing beyond 1,000 - REDIS_SCAN_START_STOP = '0' # Magic value, see http://redis.io/commands/scan + namespace :clear do + REDIS_CLEAR_BATCH_SIZE = 1000 # There seems to be no speedup when pushing beyond 1,000 + REDIS_SCAN_START_STOP = '0'.freeze # Magic value, see http://redis.io/commands/scan - desc "GitLab | Clear redis cache" - task :clear => :environment do - Gitlab::Redis.with do |redis| - cursor = REDIS_SCAN_START_STOP - loop do - cursor, keys = redis.scan( - cursor, - match: "#{Gitlab::Redis::CACHE_NAMESPACE}*", - count: CLEAR_BATCH_SIZE - ) - - redis.del(*keys) if keys.any? - - break if cursor == REDIS_SCAN_START_STOP + desc "GitLab | Clear redis cache" + task redis: :environment do + Gitlab::Redis.with do |redis| + cursor = REDIS_SCAN_START_STOP + loop do + cursor, keys = redis.scan( + cursor, + match: "#{Gitlab::Redis::CACHE_NAMESPACE}*", + count: REDIS_CLEAR_BATCH_SIZE + ) + + redis.del(*keys) if keys.any? + + break if cursor == REDIS_SCAN_START_STOP + end end end + + desc "GitLab | Clear database cache (in the background)" + task db: :environment do + ClearDatabaseCacheWorker.perform_async + end + + task all: [:db, :redis] end + + task clear: 'cache:clear:redis' end diff --git a/lib/tasks/ci/.gitkeep b/lib/tasks/ci/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 --- a/lib/tasks/ci/.gitkeep +++ /dev/null diff --git a/lib/tasks/config_lint.rake b/lib/tasks/config_lint.rake new file mode 100644 index 00000000000..ddbcf1e1eb8 --- /dev/null +++ b/lib/tasks/config_lint.rake @@ -0,0 +1,25 @@ +module ConfigLint + def self.run(files) + failures = files.reject do |file| + yield(file) + end + + if failures.present? + puts failures + exit failures.count + end + end +end + +desc "Checks syntax for shell scripts and nginx config files in 'lib/support/'" +task :config_lint do + shell_scripts = [ + 'lib/support/init.d/gitlab', + 'lib/support/init.d/gitlab.default.example', + 'lib/support/deploy/deploy.sh' + ] + + ConfigLint.run(shell_scripts) do |file| + Kernel.system('bash', '-n', file) + end +end diff --git a/lib/tasks/dev.rake b/lib/tasks/dev.rake index 6f27972c4e4..e65609d7001 100644 --- a/lib/tasks/dev.rake +++ b/lib/tasks/dev.rake @@ -2,14 +2,9 @@ task dev: ["dev:setup"] namespace :dev do desc "GitLab | Setup developer environment (db, fixtures)" - task :setup => :environment do + task setup: :environment do ENV['force'] = 'yes' Rake::Task["gitlab:setup"].invoke Rake::Task["gitlab:shell:setup"].invoke end - - desc 'GitLab | Start/restart foreman and watch for changes' - task :foreman => :environment do - sh 'rerun --dir app,config,lib -- foreman start' - end end diff --git a/lib/tasks/downtime_check.rake b/lib/tasks/downtime_check.rake index afe5d42910c..557f4fef10b 100644 --- a/lib/tasks/downtime_check.rake +++ b/lib/tasks/downtime_check.rake @@ -1,10 +1,10 @@ desc 'Checks if migrations in a branch require downtime' task downtime_check: :environment do - if defined?(Gitlab::License) - repo = 'gitlab-ee' - else - repo = 'gitlab-ce' - end + repo = if defined?(Gitlab::License) + 'gitlab-ee' + else + 'gitlab-ce' + end `git fetch https://gitlab.com/gitlab-org/#{repo}.git --depth 1` diff --git a/lib/tasks/ee_compat_check.rake b/lib/tasks/ee_compat_check.rake new file mode 100644 index 00000000000..f494fa5c5c2 --- /dev/null +++ b/lib/tasks/ee_compat_check.rake @@ -0,0 +1,4 @@ +desc 'Checks if the branch would apply cleanly to EE' +task ee_compat_check: :environment do + Rake::Task['gitlab:dev:ee_compat_check'].invoke +end diff --git a/lib/tasks/eslint.rake b/lib/tasks/eslint.rake new file mode 100644 index 00000000000..51f5d768102 --- /dev/null +++ b/lib/tasks/eslint.rake @@ -0,0 +1,8 @@ +unless Rails.env.production? + desc "GitLab | Run ESLint" + task eslint: ['yarn:check'] do + unless system('yarn run eslint') + abort('rake eslint failed') + end + end +end diff --git a/lib/tasks/flay.rake b/lib/tasks/flay.rake index e9587595fef..7ad2b2e4d39 100644 --- a/lib/tasks/flay.rake +++ b/lib/tasks/flay.rake @@ -1,6 +1,6 @@ desc 'Code duplication analyze via flay' task :flay do - output = %x(bundle exec flay --mass 35 app/ lib/gitlab/) + output = `bundle exec flay --mass 35 app/ lib/gitlab/` if output.include? "Similar code found" puts output diff --git a/lib/tasks/flog.rake b/lib/tasks/flog.rake deleted file mode 100644 index 3bfe999ae74..00000000000 --- a/lib/tasks/flog.rake +++ /dev/null @@ -1,25 +0,0 @@ -desc 'Code complexity analyze via flog' -task :flog do - output = %x(bundle exec flog -m app/ lib/gitlab) - exit_code = 0 - minimum_score = 70 - output = output.lines - - # Skip total complexity score - output.shift - - # Skip some trash info - output.shift - - output.each do |line| - score, method = line.split(" ") - score = score.to_i - - if score > minimum_score - exit_code = 1 - puts "High complexity in #{method}. Score: #{score}" - end - end - - exit exit_code -end diff --git a/lib/tasks/gitlab/assets.rake b/lib/tasks/gitlab/assets.rake new file mode 100644 index 00000000000..098f9851b45 --- /dev/null +++ b/lib/tasks/gitlab/assets.rake @@ -0,0 +1,48 @@ +namespace :gitlab do + namespace :assets do + desc 'GitLab | Assets | Compile all frontend assets' + task compile: [ + 'yarn:check', + 'assets:precompile', + 'webpack:compile', + 'gitlab:assets:fix_urls' + ] + + desc 'GitLab | Assets | Clean up old compiled frontend assets' + task clean: ['assets:clean'] + + desc 'GitLab | Assets | Remove all compiled frontend assets' + task purge: ['assets:clobber'] + + desc 'GitLab | Assets | Uninstall frontend dependencies' + task purge_modules: ['yarn:clobber'] + + desc 'GitLab | Assets | Fix all absolute url references in CSS' + task :fix_urls do + css_files = Dir['public/assets/*.css'] + css_files.each do |file| + # replace url(/assets/*) with url(./*) + puts "Fixing #{file}" + system "sed", "-i", "-e", 's/url(\([\"\']\?\)\/assets\//url(\1.\//g', file + + # rewrite the corresponding gzip file (if it exists) + gzip = "#{file}.gz" + if File.exist?(gzip) + puts "Fixing #{gzip}" + + FileUtils.rm(gzip) + mtime = File.stat(file).mtime + + File.open(gzip, 'wb+') do |f| + gz = Zlib::GzipWriter.new(f, Zlib::BEST_COMPRESSION) + gz.mtime = mtime + gz.write IO.binread(file) + gz.close + + File.utime(mtime, mtime, f.path) + end + end + end + end + end +end diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index b43ee5b3383..1650263b98d 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -13,6 +13,7 @@ namespace :gitlab do Rake::Task["gitlab:backup:uploads:create"].invoke Rake::Task["gitlab:backup:builds:create"].invoke Rake::Task["gitlab:backup:artifacts:create"].invoke + Rake::Task["gitlab:backup:pages:create"].invoke Rake::Task["gitlab:backup:lfs:create"].invoke Rake::Task["gitlab:backup:registry:create"].invoke @@ -51,13 +52,16 @@ namespace :gitlab do $progress.puts 'done'.color(:green) Rake::Task['gitlab:backup:db:restore'].invoke end + Rake::Task['gitlab:backup:repo:restore'].invoke unless backup.skipped?('repositories') Rake::Task['gitlab:backup:uploads:restore'].invoke unless backup.skipped?('uploads') Rake::Task['gitlab:backup:builds:restore'].invoke unless backup.skipped?('builds') Rake::Task['gitlab:backup:artifacts:restore'].invoke unless backup.skipped?('artifacts') + Rake::Task["gitlab:backup:pages:restore"].invoke unless backup.skipped?('pages') Rake::Task['gitlab:backup:lfs:restore'].invoke unless backup.skipped?('lfs') Rake::Task['gitlab:backup:registry:restore'].invoke unless backup.skipped?('registry') Rake::Task['gitlab:shell:setup'].invoke + Rake::Task['cache:clear'].invoke backup.cleanup end @@ -157,6 +161,25 @@ namespace :gitlab do end end + namespace :pages do + task create: :environment do + $progress.puts "Dumping pages ... ".color(:blue) + + if ENV["SKIP"] && ENV["SKIP"].include?("pages") + $progress.puts "[SKIPPED]".color(:cyan) + else + Backup::Pages.new.dump + $progress.puts "done".color(:green) + end + end + + task restore: :environment do + $progress.puts "Restoring pages ... ".color(:blue) + Backup::Pages.new.restore + $progress.puts "done".color(:green) + end + end + namespace :lfs do task create: :environment do $progress.puts "Dumping lfs objects ... ".color(:blue) diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 5f4a6bbfa35..38edd49b6ed 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -6,8 +6,6 @@ namespace :gitlab do gitlab:ldap:check gitlab:app:check} - - namespace :app do desc "GitLab | Check the configuration of the GitLab Rails app" task check: :environment do @@ -34,7 +32,6 @@ namespace :gitlab do finished_checking "GitLab" end - # Checks ######################## @@ -194,7 +191,7 @@ namespace :gitlab do def check_migrations_are_up print "All migrations up? ... " - migration_status, _ = Gitlab::Popen.popen(%W(bundle exec rake db:migrate:status)) + migration_status, _ = Gitlab::Popen.popen(%w(bundle exec rake db:migrate:status)) unless migration_status =~ /down\s+\d{14}/ puts "yes".color(:green) @@ -279,7 +276,7 @@ namespace :gitlab do upload_path_tmp = File.join(upload_path, 'tmp') if File.stat(upload_path).mode == 040700 - unless Dir.exists?(upload_path_tmp) + unless Dir.exist?(upload_path_tmp) puts 'skipped (no tmp uploads folder yet)'.color(:magenta) return end @@ -316,7 +313,7 @@ namespace :gitlab do min_redis_version = "2.8.0" print "Redis version >= #{min_redis_version}? ... " - redis_version = run_command(%W(redis-cli --version)) + redis_version = run_command(%w(redis-cli --version)) redis_version = redis_version.try(:match, /redis-cli (\d+\.\d+\.\d+)/) if redis_version && (Gem::Version.new(redis_version[1]) > Gem::Version.new(min_redis_version)) @@ -351,7 +348,6 @@ namespace :gitlab do finished_checking "GitLab Shell" end - # Checks ######################## @@ -387,7 +383,7 @@ namespace :gitlab do unless File.exist?(repo_base_path) puts "can't check because of previous errors".color(:magenta) - return + break end unless File.symlink?(repo_base_path) @@ -410,7 +406,7 @@ namespace :gitlab do unless File.exist?(repo_base_path) puts "can't check because of previous errors".color(:magenta) - return + break end if File.stat(repo_base_path).mode.to_s(8).ends_with?("2770") @@ -440,7 +436,7 @@ namespace :gitlab do unless File.exist?(repo_base_path) puts "can't check because of previous errors".color(:magenta) - return + break end uid = uid_for(gitlab_shell_ssh_user) @@ -493,7 +489,6 @@ namespace :gitlab do ) fix_and_rerun end - end end @@ -565,8 +560,6 @@ namespace :gitlab do end end - - namespace :sidekiq do desc "GitLab | Check the configuration of Sidekiq" task check: :environment do @@ -579,7 +572,6 @@ namespace :gitlab do finished_checking "Sidekiq" end - # Checks ######################## @@ -621,12 +613,11 @@ namespace :gitlab do end def sidekiq_process_count - ps_ux, _ = Gitlab::Popen.popen(%W(ps ux)) + ps_ux, _ = Gitlab::Popen.popen(%w(ps ux)) ps_ux.scan(/sidekiq \d+\.\d+\.\d+/).count end end - namespace :incoming_email do desc "GitLab | Check the configuration of Reply by email" task check: :environment do @@ -649,7 +640,6 @@ namespace :gitlab do finished_checking "Reply by email" end - # Checks ######################## @@ -671,7 +661,7 @@ namespace :gitlab do "Enable mail_room in the init.d configuration." ) for_more_information( - "doc/incoming_email/README.md" + "doc/administration/reply_by_email.md" ) fix_and_rerun end @@ -690,7 +680,7 @@ namespace :gitlab do "Enable mail_room in your Procfile." ) for_more_information( - "doc/incoming_email/README.md" + "doc/administration/reply_by_email.md" ) fix_and_rerun end @@ -724,8 +714,11 @@ namespace :gitlab do def check_imap_authentication print "IMAP server credentials are correct? ... " - config_path = Rails.root.join('config', 'mail_room.yml') - config_file = YAML.load(ERB.new(File.read(config_path)).result) + config_path = Rails.root.join('config', 'mail_room.yml').to_s + erb = ERB.new(File.read(config_path)) + erb.filename = config_path + config_file = YAML.load(erb.result) + config = config_file[:mailboxes].first if config @@ -747,20 +740,20 @@ namespace :gitlab do "Check that the information in config/gitlab.yml is correct" ) for_more_information( - "doc/incoming_email/README.md" + "doc/administration/reply_by_email.md" ) fix_and_rerun end end def mail_room_running? - ps_ux, _ = Gitlab::Popen.popen(%W(ps ux)) + ps_ux, _ = Gitlab::Popen.popen(%w(ps ux)) ps_ux.include?("mail_room") end end namespace :ldap do - task :check, [:limit] => :environment do |t, args| + task :check, [:limit] => :environment do |_, args| # Only show up to 100 results because LDAP directories can be very big. # This setting only affects the `rake gitlab:check` script. args.with_defaults(limit: 100) @@ -768,7 +761,7 @@ namespace :gitlab do start_checking "LDAP" if Gitlab::LDAP::Config.enabled? - print_users(args.limit) + check_ldap(args.limit) else puts 'LDAP is disabled in config/gitlab.yml' end @@ -776,21 +769,42 @@ namespace :gitlab do finished_checking "LDAP" end - def print_users(limit) - puts "LDAP users with access to your GitLab server (only showing the first #{limit} results)" - + def check_ldap(limit) servers = Gitlab::LDAP::Config.providers servers.each do |server| puts "Server: #{server}" - Gitlab::LDAP::Adapter.open(server) do |adapter| - users = adapter.users(adapter.config.uid, '*', limit) - users.each do |user| - puts "\tDN: #{user.dn}\t #{adapter.config.uid}: #{user.uid}" + + begin + Gitlab::LDAP::Adapter.open(server) do |adapter| + check_ldap_auth(adapter) + + puts "LDAP users with access to your GitLab server (only showing the first #{limit} results)" + + users = adapter.users(adapter.config.uid, '*', limit) + users.each do |user| + puts "\tDN: #{user.dn}\t #{adapter.config.uid}: #{user.uid}" + end end + rescue Net::LDAP::ConnectionRefusedError, Errno::ECONNREFUSED => e + puts "Could not connect to the LDAP server: #{e.message}".color(:red) end end end + + def check_ldap_auth(adapter) + auth = adapter.config.has_auth? + + message = if auth && adapter.ldap.bind + 'Success'.color(:green) + elsif auth + 'Failed. Check `bind_dn` and `password` configuration values'.color(:red) + else + 'Anonymous. No `bind_dn` or `password` configured'.color(:yellow) + end + + puts "LDAP authentication... #{message}" + end end namespace :repo do @@ -814,11 +828,11 @@ namespace :gitlab do user = User.find_by(username: username) if user repo_dirs = user.authorized_projects.map do |p| - File.join( - p.repository_storage_path, - "#{p.path_with_namespace}.git" - ) - end + File.join( + p.repository_storage_path, + "#{p.path_with_namespace}.git" + ) + end repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) } else @@ -831,7 +845,7 @@ namespace :gitlab do ########################## def fix_and_rerun - puts " Please #{"fix the error above"} and rerun the checks.".color(:red) + puts " Please fix the error above and rerun the checks.".color(:red) end def for_more_information(*sources) @@ -893,7 +907,7 @@ namespace :gitlab do def check_ruby_version required_version = Gitlab::VersionInfo.new(2, 1, 0) - current_version = Gitlab::VersionInfo.parse(run_command(%W(ruby --version))) + current_version = Gitlab::VersionInfo.parse(run_command(%w(ruby --version))) print "Ruby version >= #{required_version} ? ... " @@ -964,13 +978,13 @@ namespace :gitlab do end def check_config_lock(repo_dir) - config_exists = File.exist?(File.join(repo_dir,'config.lock')) + config_exists = File.exist?(File.join(repo_dir, 'config.lock')) config_output = config_exists ? 'yes'.color(:red) : 'no'.color(:green) puts "'config.lock' file exists?".color(:yellow) + " ... #{config_output}" end def check_ref_locks(repo_dir) - lock_files = Dir.glob(File.join(repo_dir,'refs/heads/*.lock')) + lock_files = Dir.glob(File.join(repo_dir, 'refs/heads/*.lock')) if lock_files.present? puts "Ref lock files exist:".color(:red) lock_files.each do |lock_file| diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index b7cbdc6cd78..daf7382dd02 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -25,7 +25,6 @@ namespace :gitlab do end all_dirs.each do |dir_path| - if remove_flag if FileUtils.rm_rf dir_path puts "Removed...#{dir_path}".color(:red) @@ -53,12 +52,12 @@ namespace :gitlab do IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find| find.each_line do |path| path.chomp! - repo_with_namespace = path. - sub(repo_root, ''). - sub(%r{^/*}, ''). - chomp('.git'). - chomp('.wiki') - next if Project.find_with_namespace(repo_with_namespace) + repo_with_namespace = path + .sub(repo_root, '') + .sub(%r{^/*}, '') + .chomp('.git') + .chomp('.wiki') + next if Project.find_by_full_path(repo_with_namespace) new_path = path + move_suffix puts path.inspect + ' -> ' + new_path.inspect File.rename(path, new_path) @@ -91,5 +90,28 @@ namespace :gitlab do puts "To block these users run this command with BLOCK=true".color(:yellow) end end + + # This is a rake task which removes faulty refs. These refs where only + # created in the 8.13.RC cycle, and fixed in the stable builds which were + # released. So likely this should only be run once on gitlab.com + # Faulty refs are moved so they are kept around, else some features break. + desc 'GitLab | Cleanup | Remove faulty deployment refs' + task move_faulty_deployment_refs: :environment do + projects = Project.where(id: Deployment.select(:project_id).distinct) + + projects.find_each do |project| + rugged = project.repository.rugged + + max_iid = project.deployments.maximum(:iid) + + rugged.references.each('refs/environments/**/*') do |ref| + id = ref.name.split('/').last.to_i + next unless id > max_iid + + project.deployments.find(id).create_ref + rugged.references.delete(ref) + end + end + end end end diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 7c96bc864ce..ecf6b6e068b 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -23,7 +23,7 @@ namespace :gitlab do end desc 'Drop all tables' - task :drop_tables => :environment do + task drop_tables: :environment do connection = ActiveRecord::Base.connection # If MySQL, turn off foreign key checks @@ -62,9 +62,9 @@ namespace :gitlab do ref = Shellwords.escape(args[:ref]) - migrations = `git diff #{ref}.. --name-only -- db/migrate`.lines. - map { |file| Rails.root.join(file.strip).to_s }. - select { |file| File.file?(file) } + migrations = `git diff #{ref}.. --name-only -- db/migrate`.lines + .map { |file| Rails.root.join(file.strip).to_s } + .select { |file| File.file?(file) } Gitlab::DowntimeCheck.new.check_and_print(migrations) end diff --git a/lib/tasks/gitlab/dev.rake b/lib/tasks/gitlab/dev.rake new file mode 100644 index 00000000000..7db0779def8 --- /dev/null +++ b/lib/tasks/gitlab/dev.rake @@ -0,0 +1,23 @@ +namespace :gitlab do + namespace :dev do + desc 'Checks if the branch would apply cleanly to EE' + task :ee_compat_check, [:branch] => :environment do |_, args| + opts = + if ENV['CI'] + { branch: ENV['CI_BUILD_REF_NAME'] } + else + unless args[:branch] + puts "Must specify a branch as an argument".color(:red) + exit 1 + end + args + end + + if Gitlab::EeCompatCheck.new(opts || {}).check + exit 0 + else + exit 1 + end + end + end +end diff --git a/lib/tasks/gitlab/generate_docs.rake b/lib/tasks/gitlab/generate_docs.rake deleted file mode 100644 index f6448c38e10..00000000000 --- a/lib/tasks/gitlab/generate_docs.rake +++ /dev/null @@ -1,7 +0,0 @@ -namespace :gitlab do - desc "GitLab | Generate sdocs for project" - task generate_docs: :environment do - system(*%W(bundle exec sdoc -o doc/code app lib)) - end -end - diff --git a/lib/tasks/gitlab/git.rake b/lib/tasks/gitlab/git.rake index f9834a4dae8..cf82134d97e 100644 --- a/lib/tasks/gitlab/git.rake +++ b/lib/tasks/gitlab/git.rake @@ -1,9 +1,8 @@ namespace :gitlab do namespace :git do - desc "GitLab | Git | Repack" task repack: :environment do - failures = perform_git_cmd(%W(git repack -a --quiet), "Repacking repo") + failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} repack -a --quiet), "Repacking repo") if failures.empty? puts "Done".color(:green) else @@ -13,17 +12,17 @@ namespace :gitlab do desc "GitLab | Git | Run garbage collection on all repos" task gc: :environment do - failures = perform_git_cmd(%W(git gc --auto --quiet), "Garbage Collecting") + failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} gc --auto --quiet), "Garbage Collecting") if failures.empty? puts "Done".color(:green) else output_failures(failures) end end - + desc "GitLab | Git | Prune all repos" task prune: :environment do - failures = perform_git_cmd(%W(git prune), "Git Prune") + failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} prune), "Git Prune") if failures.empty? puts "Done".color(:green) else @@ -50,6 +49,5 @@ namespace :gitlab do puts "The following repositories reported errors:".color(:red) failures.each { |f| puts "- #{f}" } end - end end diff --git a/lib/tasks/gitlab/helpers.rake b/lib/tasks/gitlab/helpers.rake new file mode 100644 index 00000000000..dd2d5861481 --- /dev/null +++ b/lib/tasks/gitlab/helpers.rake @@ -0,0 +1,8 @@ +require 'tasks/gitlab/task_helpers' + +# Prevent StateMachine warnings from outputting during a cron task +StateMachines::Machine.ignore_method_conflicts = true if ENV['CRON'] + +namespace :gitlab do + include Gitlab::TaskHelpers +end diff --git a/lib/tasks/gitlab/import.rake b/lib/tasks/gitlab/import.rake index dbdd4e977e8..66e7b7685f7 100644 --- a/lib/tasks/gitlab/import.rake +++ b/lib/tasks/gitlab/import.rake @@ -29,7 +29,7 @@ namespace :gitlab do next end - project = Project.find_with_namespace(path) + project = Project.find_by_full_path(path) if project puts " * #{project.name} (#{repo_path}) exists" @@ -46,7 +46,7 @@ namespace :gitlab do group = Namespace.find_by(path: group_name) # create group namespace unless group - group = Group.new(:name => group_name) + group = Group.new(name: group_name) group.path = group_name group.owner = user if group.save @@ -63,8 +63,7 @@ namespace :gitlab do if project.persisted? puts " * Created #{project.name} (#{repo_path})".color(:green) - project.update_repository_size - project.update_commit_count + ProjectCacheWorker.perform_async(project.id) else puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red) puts " Errors: #{project.errors.messages}".color(:red) diff --git a/lib/tasks/gitlab/import_export.rake b/lib/tasks/gitlab/import_export.rake index c2c6031db67..dd1825c8a9e 100644 --- a/lib/tasks/gitlab/import_export.rake +++ b/lib/tasks/gitlab/import_export.rake @@ -7,7 +7,7 @@ namespace :gitlab do desc "GitLab | Display exported DB structure" task data: :environment do - puts YAML.load_file(Gitlab::ImportExport.config_file)['project_tree'].to_yaml(:SortKeys => true) + puts YAML.load_file(Gitlab::ImportExport.config_file)['project_tree'].to_yaml(SortKeys: true) end end end diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake index dffea8ed155..ae78fe64eb8 100644 --- a/lib/tasks/gitlab/info.rake +++ b/lib/tasks/gitlab/info.rake @@ -2,31 +2,32 @@ namespace :gitlab do namespace :env do desc "GitLab | Show information about GitLab and its environment" task info: :environment do - # check if there is an RVM environment - rvm_version = run_and_match(%W(rvm --version), /[\d\.]+/).try(:to_s) + rvm_version = run_and_match(%w(rvm --version), /[\d\.]+/).try(:to_s) # check Ruby version - ruby_version = run_and_match(%W(ruby --version), /[\d\.p]+/).try(:to_s) + ruby_version = run_and_match(%w(ruby --version), /[\d\.p]+/).try(:to_s) # check Gem version - gem_version = run_command(%W(gem --version)) - # check Bundler version - bunder_version = run_and_match(%W(bundle --version), /[\d\.]+/).try(:to_s) + gem_version = run_command(%w(gem --version)) # check Bundler version - rake_version = run_and_match(%W(rake --version), /[\d\.]+/).try(:to_s) + bunder_version = run_and_match(%w(bundle --version), /[\d\.]+/).try(:to_s) + # check Rake version + rake_version = run_and_match(%w(rake --version), /[\d\.]+/).try(:to_s) + # check redis version + redis_version = run_and_match(%w(redis-cli --version), /redis-cli (\d+\.\d+\.\d+)/).to_a puts "" puts "System information".color(:yellow) puts "System:\t\t#{os_name || "unknown".color(:red)}" - puts "Current User:\t#{run_command(%W(whoami))}" + puts "Current User:\t#{run_command(%w(whoami))}" puts "Using RVM:\t#{rvm_version.present? ? "yes".color(:green) : "no"}" puts "RVM Version:\t#{rvm_version}" if rvm_version.present? puts "Ruby Version:\t#{ruby_version || "unknown".color(:red)}" puts "Gem Version:\t#{gem_version || "unknown".color(:red)}" puts "Bundler Version:#{bunder_version || "unknown".color(:red)}" puts "Rake Version:\t#{rake_version || "unknown".color(:red)}" + puts "Redis Version:\t#{redis_version[1] || "unknown".color(:red)}" puts "Sidekiq Version:#{Sidekiq::VERSION}" - # check database adapter database_adapter = ActiveRecord::Base.connection.adapter_name.downcase @@ -51,8 +52,6 @@ namespace :gitlab do puts "Using Omniauth:\t#{Gitlab.config.omniauth.enabled ? "yes".color(:green) : "no"}" puts "Omniauth Providers: #{omniauth_providers.join(', ')}" if Gitlab.config.omniauth.enabled - - # check Gitolite version gitlab_shell_version_file = "#{Gitlab.config.gitlab_shell.hooks_path}/../VERSION" if File.readable?(gitlab_shell_version_file) @@ -68,7 +67,6 @@ namespace :gitlab do end puts "Hooks:\t\t#{Gitlab.config.gitlab_shell.hooks_path}" puts "Git:\t\t#{Gitlab.config.git.bin_path}" - end end end diff --git a/lib/tasks/gitlab/ldap.rake b/lib/tasks/gitlab/ldap.rake new file mode 100644 index 00000000000..c66a2a263dc --- /dev/null +++ b/lib/tasks/gitlab/ldap.rake @@ -0,0 +1,40 @@ +namespace :gitlab do + namespace :ldap do + desc 'GitLab | LDAP | Rename provider' + task :rename_provider, [:old_provider, :new_provider] => :environment do |_, args| + old_provider = args[:old_provider] || + prompt('What is the old provider? Ex. \'ldapmain\': '.color(:blue)) + new_provider = args[:new_provider] || + prompt('What is the new provider ID? Ex. \'ldapcustom\': '.color(:blue)) + puts '' # Add some separation in the output + + identities = Identity.where(provider: old_provider) + identity_count = identities.count + + if identities.empty? + puts "Found no user identities with '#{old_provider}' provider." + puts 'Please check the provider name and try again.' + exit 1 + end + + plural_id_count = ActionController::Base.helpers.pluralize(identity_count, 'user') + + unless ENV['force'] == 'yes' + puts "#{plural_id_count} with provider '#{old_provider}' will be updated to '#{new_provider}'" + puts 'If the new provider is incorrect, users will be unable to sign in' + ask_to_continue + puts '' + end + + updated_count = identities.update_all(provider: new_provider) + + if updated_count == identity_count + puts 'User identities were successfully updated'.color(:green) + else + plural_updated_count = ActionController::Base.helpers.pluralize(updated_count, 'user') + puts 'Some user identities could not be updated'.color(:red) + puts "Successfully updated #{plural_updated_count} out of #{plural_id_count} total" + end + end + end +end diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index bb7eb852f1b..dd2fda54e62 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -5,44 +5,25 @@ namespace :gitlab do warn_user_is_not_gitlab default_version = Gitlab::Shell.version_required - default_version_tag = 'v' + default_version - args.with_defaults(tag: default_version_tag, repo: "https://gitlab.com/gitlab-org/gitlab-shell.git") + default_version_tag = "v#{default_version}" + args.with_defaults(tag: default_version_tag, repo: 'https://gitlab.com/gitlab-org/gitlab-shell.git') - user = Gitlab.config.gitlab.user - home_dir = Rails.env.test? ? Rails.root.join('tmp/tests') : Gitlab.config.gitlab.user_home gitlab_url = Gitlab.config.gitlab.url # gitlab-shell requires a / at the end of the url gitlab_url += '/' unless gitlab_url.end_with?('/') target_dir = Gitlab.config.gitlab_shell.path - # Clone if needed - if File.directory?(target_dir) - Dir.chdir(target_dir) do - system(*%W(Gitlab.config.git.bin_path} fetch --tags --quiet)) - system(*%W(Gitlab.config.git.bin_path} checkout --quiet #{default_version_tag})) - end - else - system(*%W(#{Gitlab.config.git.bin_path} clone -- #{args.repo} #{target_dir})) - end + checkout_or_clone_tag(tag: default_version_tag, repo: args.repo, target_dir: target_dir) # Make sure we're on the right tag Dir.chdir(target_dir) do - # First try to checkout without fetching - # to avoid stalling tests if the Internet is down. - reseted = reset_to_commit(args) - - unless reseted - system(*%W(#{Gitlab.config.git.bin_path} fetch origin)) - reset_to_commit(args) - end - config = { - user: user, + user: Gitlab.config.gitlab.user, gitlab_url: gitlab_url, - http_settings: {self_signed_cert: false}.stringify_keys, - auth_file: File.join(home_dir, ".ssh", "authorized_keys"), + http_settings: { self_signed_cert: false }.stringify_keys, + auth_file: File.join(user_home, ".ssh", "authorized_keys"), redis: { - bin: %x{which redis-cli}.chomp, + bin: `which redis-cli`.chomp, namespace: "resque:gitlab" }.stringify_keys, log_level: "INFO", @@ -62,23 +43,23 @@ namespace :gitlab do File.open("config.yml", "w+") {|f| f.puts config.to_yaml} # Launch installation process - system(*%W(bin/install) + repository_storage_paths_args) - - # (Re)create hooks - system(*%W(bin/create-hooks) + repository_storage_paths_args) + system(*%w(bin/install) + repository_storage_paths_args) end + # (Re)create hooks + Rake::Task['gitlab:shell:create_hooks'].invoke + # Required for debian packaging with PKGR: Setup .ssh/environment with # the current PATH, so that the correct ruby version gets loaded # Requires to set "PermitUserEnvironment yes" in sshd config (should not # be an issue since it is more than likely that there are no "normal" # user accounts on a gitlab server). The alternative is for the admin to # install a ruby (1.9.3+) in the global path. - File.open(File.join(home_dir, ".ssh", "environment"), "w+") do |f| + File.open(File.join(user_home, ".ssh", "environment"), "w+") do |f| f.puts "PATH=#{ENV['PATH']}" end - Gitlab::Shell.new.generate_and_link_secret_token + Gitlab::Shell.ensure_secret_token! end desc "GitLab | Setup gitlab-shell" @@ -102,6 +83,15 @@ namespace :gitlab do end end end + + desc 'Create or repair repository hooks symlink' + task create_hooks: :environment do + warn_user_is_not_gitlab + + puts 'Creating/Repairing hooks symlinks for all repositories' + system(*%W(#{Gitlab.config.gitlab_shell.path}/bin/create-hooks) + repository_storage_paths_args) + puts 'done'.color(:green) + end end def setup @@ -133,15 +123,4 @@ namespace :gitlab do puts "Quitting...".color(:red) exit 1 end - - def reset_to_commit(args) - tag, status = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} describe -- #{args.tag})) - - unless status.zero? - tag, status = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} describe -- origin/#{args.tag})) - end - - tag = tag.strip - system(*%W(#{Gitlab.config.git.bin_path} reset --hard #{tag})) - end end diff --git a/lib/tasks/gitlab/sidekiq.rake b/lib/tasks/gitlab/sidekiq.rake index 7e2a6668e59..6cbc83b8973 100644 --- a/lib/tasks/gitlab/sidekiq.rake +++ b/lib/tasks/gitlab/sidekiq.rake @@ -1,13 +1,13 @@ namespace :gitlab do namespace :sidekiq do - QUEUE = 'queue:post_receive' + QUEUE = 'queue:post_receive'.freeze desc 'Drop all Sidekiq PostReceive jobs for a given project' - task :drop_post_receive , [:project] => :environment do |t, args| + task :drop_post_receive, [:project] => :environment do |t, args| unless args.project.present? abort "Please specify the project you want to drop PostReceive jobs for:\n rake gitlab:sidekiq:drop_post_receive[group/project]" end - project_path = Project.find_with_namespace(args.project).repository.path_to_repo + project_path = Project.find_by_full_path(args.project).repository.path_to_repo Sidekiq.redis do |redis| unless redis.exists(QUEUE) @@ -21,7 +21,7 @@ namespace :gitlab do # new jobs already. We will repopulate it with the old jobs, skipping the # ones we want to drop. dropped = 0 - while (job = redis.lpop(temp_queue)) do + while (job = redis.lpop(temp_queue)) if repo_path(job) == project_path dropped += 1 else diff --git a/lib/tasks/gitlab/task_helpers.rake b/lib/tasks/gitlab/task_helpers.rake deleted file mode 100644 index 74be413423a..00000000000 --- a/lib/tasks/gitlab/task_helpers.rake +++ /dev/null @@ -1,140 +0,0 @@ -module Gitlab - class TaskAbortedByUserError < StandardError; end -end - -require 'rainbow/ext/string' - -# Prevent StateMachine warnings from outputting during a cron task -StateMachines::Machine.ignore_method_conflicts = true if ENV['CRON'] - -namespace :gitlab do - - # Ask if the user wants to continue - # - # Returns "yes" the user chose to continue - # Raises Gitlab::TaskAbortedByUserError if the user chose *not* to continue - def ask_to_continue - answer = prompt("Do you want to continue (yes/no)? ".color(:blue), %w{yes no}) - raise Gitlab::TaskAbortedByUserError unless answer == "yes" - end - - # Check which OS is running - # - # It will primarily use lsb_relase to determine the OS. - # It has fallbacks to Debian, SuSE, OS X and systems running systemd. - def os_name - os_name = run_command(%W(lsb_release -irs)) - os_name ||= if File.readable?('/etc/system-release') - File.read('/etc/system-release') - end - os_name ||= if File.readable?('/etc/debian_version') - debian_version = File.read('/etc/debian_version') - "Debian #{debian_version}" - end - os_name ||= if File.readable?('/etc/SuSE-release') - File.read('/etc/SuSE-release') - end - os_name ||= if os_x_version = run_command(%W(sw_vers -productVersion)) - "Mac OS X #{os_x_version}" - end - os_name ||= if File.readable?('/etc/os-release') - File.read('/etc/os-release').match(/PRETTY_NAME=\"(.+)\"/)[1] - end - os_name.try(:squish!) - end - - # Prompt the user to input something - # - # message - the message to display before input - # choices - array of strings of acceptable answers or nil for any answer - # - # Returns the user's answer - def prompt(message, choices = nil) - begin - print(message) - answer = STDIN.gets.chomp - end while choices.present? && !choices.include?(answer) - answer - end - - # Runs the given command and matches the output against the given pattern - # - # Returns nil if nothing matched - # Returns the MatchData if the pattern matched - # - # see also #run_command - # see also String#match - def run_and_match(command, regexp) - run_command(command).try(:match, regexp) - end - - # Runs the given command - # - # Returns nil if the command was not found - # Returns the output of the command otherwise - # - # see also #run_and_match - def run_command(command) - output, _ = Gitlab::Popen.popen(command) - output - rescue Errno::ENOENT - '' # if the command does not exist, return an empty string - end - - def uid_for(user_name) - run_command(%W(id -u #{user_name})).chomp.to_i - end - - def gid_for(group_name) - begin - Etc.getgrnam(group_name).gid - rescue ArgumentError # no group - "group #{group_name} doesn't exist" - end - end - - def warn_user_is_not_gitlab - unless @warned_user_not_gitlab - gitlab_user = Gitlab.config.gitlab.user - current_user = run_command(%W(whoami)).chomp - unless current_user == gitlab_user - puts " Warning ".color(:black).background(:yellow) - puts " You are running as user #{current_user.color(:magenta)}, we hope you know what you are doing." - puts " Things may work\/fail for the wrong reasons." - puts " For correct results you should run this as user #{gitlab_user.color(:magenta)}." - puts "" - end - @warned_user_not_gitlab = true - end - end - - # Tries to configure git itself - # - # Returns true if all subcommands were successfull (according to their exit code) - # Returns false if any or all subcommands failed. - def auto_fix_git_config(options) - if !@warned_user_not_gitlab - command_success = options.map do |name, value| - system(*%W(#{Gitlab.config.git.bin_path} config --global #{name} #{value})) - end - - command_success.all? - else - false - end - end - - def all_repos - Gitlab.config.repositories.storages.each do |name, path| - IO.popen(%W(find #{path} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find| - find.each_line do |path| - yield path.chomp - end - end - end - end - - def repository_storage_paths_args - Gitlab.config.repositories.storages.values - end -end diff --git a/lib/tasks/gitlab/task_helpers.rb b/lib/tasks/gitlab/task_helpers.rb new file mode 100644 index 00000000000..2a999ad6959 --- /dev/null +++ b/lib/tasks/gitlab/task_helpers.rb @@ -0,0 +1,187 @@ +require 'rainbow/ext/string' + +module Gitlab + TaskFailedError = Class.new(StandardError) + TaskAbortedByUserError = Class.new(StandardError) + + module TaskHelpers + # Ask if the user wants to continue + # + # Returns "yes" the user chose to continue + # Raises Gitlab::TaskAbortedByUserError if the user chose *not* to continue + def ask_to_continue + answer = prompt("Do you want to continue (yes/no)? ".color(:blue), %w{yes no}) + raise Gitlab::TaskAbortedByUserError unless answer == "yes" + end + + # Check which OS is running + # + # It will primarily use lsb_relase to determine the OS. + # It has fallbacks to Debian, SuSE, OS X and systems running systemd. + def os_name + os_name = run_command(%w(lsb_release -irs)) + os_name ||= + if File.readable?('/etc/system-release') + File.read('/etc/system-release') + elsif File.readable?('/etc/debian_version') + "Debian #{File.read('/etc/debian_version')}" + elsif File.readable?('/etc/SuSE-release') + File.read('/etc/SuSE-release') + elsif os_x_version = run_command(%w(sw_vers -productVersion)) + "Mac OS X #{os_x_version}" + elsif File.readable?('/etc/os-release') + File.read('/etc/os-release').match(/PRETTY_NAME=\"(.+)\"/)[1] + end + + os_name.try(:squish!) + end + + # Prompt the user to input something + # + # message - the message to display before input + # choices - array of strings of acceptable answers or nil for any answer + # + # Returns the user's answer + def prompt(message, choices = nil) + begin + print(message) + answer = STDIN.gets.chomp + end while choices.present? && !choices.include?(answer) + answer + end + + # Runs the given command and matches the output against the given pattern + # + # Returns nil if nothing matched + # Returns the MatchData if the pattern matched + # + # see also #run_command + # see also String#match + def run_and_match(command, regexp) + run_command(command).try(:match, regexp) + end + + # Runs the given command + # + # Returns '' if the command was not found + # Returns the output of the command otherwise + # + # see also #run_and_match + def run_command(command) + output, _ = Gitlab::Popen.popen(command) + output + rescue Errno::ENOENT + '' # if the command does not exist, return an empty string + end + + # Runs the given command and raises a Gitlab::TaskFailedError exception if + # the command does not exit with 0 + # + # Returns the output of the command otherwise + def run_command!(command) + output, status = Gitlab::Popen.popen(command) + + raise Gitlab::TaskFailedError unless status.zero? + + output + end + + def uid_for(user_name) + run_command(%W(id -u #{user_name})).chomp.to_i + end + + def gid_for(group_name) + begin + Etc.getgrnam(group_name).gid + rescue ArgumentError # no group + "group #{group_name} doesn't exist" + end + end + + def warn_user_is_not_gitlab + unless @warned_user_not_gitlab + gitlab_user = Gitlab.config.gitlab.user + current_user = run_command(%w(whoami)).chomp + unless current_user == gitlab_user + puts " Warning ".color(:black).background(:yellow) + puts " You are running as user #{current_user.color(:magenta)}, we hope you know what you are doing." + puts " Things may work\/fail for the wrong reasons." + puts " For correct results you should run this as user #{gitlab_user.color(:magenta)}." + puts "" + end + @warned_user_not_gitlab = true + end + end + + # Tries to configure git itself + # + # Returns true if all subcommands were successfull (according to their exit code) + # Returns false if any or all subcommands failed. + def auto_fix_git_config(options) + if !@warned_user_not_gitlab + command_success = options.map do |name, value| + system(*%W(#{Gitlab.config.git.bin_path} config --global #{name} #{value})) + end + + command_success.all? + else + false + end + end + + def all_repos + Gitlab.config.repositories.storages.each do |name, path| + IO.popen(%W(find #{path} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find| + find.each_line do |path| + yield path.chomp + end + end + end + end + + def repository_storage_paths_args + Gitlab.config.repositories.storages.values + end + + def user_home + Rails.env.test? ? Rails.root.join('tmp/tests') : Gitlab.config.gitlab.user_home + end + + def checkout_or_clone_tag(tag:, repo:, target_dir:) + if Dir.exist?(target_dir) + checkout_tag(tag, target_dir) + else + clone_repo(repo, target_dir) + end + + reset_to_tag(tag, target_dir) + end + + def clone_repo(repo, target_dir) + run_command!(%W[#{Gitlab.config.git.bin_path} clone -- #{repo} #{target_dir}]) + end + + def checkout_tag(tag, target_dir) + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} fetch --tags --quiet]) + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} checkout --quiet #{tag}]) + end + + def reset_to_tag(tag_wanted, target_dir) + tag = + begin + # First try to checkout without fetching + # to avoid stalling tests if the Internet is down. + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} describe -- #{tag_wanted}]) + rescue Gitlab::TaskFailedError + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} fetch origin]) + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} describe -- origin/#{tag_wanted}]) + end + + if tag + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} reset --hard #{tag.strip}]) + else + raise Gitlab::TaskFailedError + end + end + end +end diff --git a/lib/tasks/gitlab/test.rake b/lib/tasks/gitlab/test.rake index 4d4e746503a..523b0fa055b 100644 --- a/lib/tasks/gitlab/test.rake +++ b/lib/tasks/gitlab/test.rake @@ -2,15 +2,15 @@ namespace :gitlab do desc "GitLab | Run all tests" task :test do cmds = [ - %W(rake brakeman), - %W(rake rubocop), - %W(rake spinach), - %W(rake spec), - %W(rake teaspoon) + %w(rake brakeman), + %w(rake rubocop), + %w(rake spinach), + %w(rake spec), + %w(rake karma) ] cmds.each do |cmd| - system({'RAILS_ENV' => 'test', 'force' => 'yes'}, *cmd) or raise("#{cmd} failed!") + system({ 'RAILS_ENV' => 'test', 'force' => 'yes' }, *cmd) || raise("#{cmd} failed!") end end end diff --git a/lib/tasks/gitlab/track_deployment.rake b/lib/tasks/gitlab/track_deployment.rake index 84aa2e8507a..6f101aea303 100644 --- a/lib/tasks/gitlab/track_deployment.rake +++ b/lib/tasks/gitlab/track_deployment.rake @@ -1,8 +1,8 @@ namespace :gitlab do desc 'GitLab | Tracks a deployment in GitLab Performance Monitoring' task track_deployment: :environment do - metric = Gitlab::Metrics::Metric. - new('deployments', version: Gitlab::VERSION) + metric = Gitlab::Metrics::Metric + .new('deployments', version: Gitlab::VERSION) Gitlab::Metrics.submit_metrics([metric.to_hash]) end diff --git a/lib/tasks/gitlab/update_commit_count.rake b/lib/tasks/gitlab/update_commit_count.rake deleted file mode 100644 index 3bd10b0208b..00000000000 --- a/lib/tasks/gitlab/update_commit_count.rake +++ /dev/null @@ -1,20 +0,0 @@ -namespace :gitlab do - desc "GitLab | Update commit count for projects" - task update_commit_count: :environment do - projects = Project.where(commit_count: 0) - puts "#{projects.size} projects need to be updated. This might take a while." - ask_to_continue unless ENV['force'] == 'yes' - - projects.find_each(batch_size: 100) do |project| - print "#{project.name_with_namespace.color(:yellow)} ... " - - unless project.repo_exists? - puts "skipping, because the repo is empty".color(:magenta) - next - end - - project.update_commit_count - puts project.commit_count.to_s.color(:green) - end - end -end diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake index 4f76dad7286..dbdfb335a5c 100644 --- a/lib/tasks/gitlab/update_templates.rake +++ b/lib/tasks/gitlab/update_templates.rake @@ -44,9 +44,9 @@ namespace :gitlab do ), Template.new( "https://gitlab.com/gitlab-org/gitlab-ci-yml.git", - /(\.{1,2}|LICENSE|Pages|\.gitlab-ci.yml)\z/ + /(\.{1,2}|LICENSE|Pages|autodeploy|\.gitlab-ci.yml)\z/ ) - ] + ].freeze def vendor_directory Rails.root.join('vendor') diff --git a/lib/tasks/gitlab/users.rake b/lib/tasks/gitlab/users.rake new file mode 100644 index 00000000000..3a16ace60bd --- /dev/null +++ b/lib/tasks/gitlab/users.rake @@ -0,0 +1,11 @@ +namespace :gitlab do + namespace :users do + desc "GitLab | Clear the authentication token for all users" + task clear_all_authentication_tokens: :environment do |t, args| + # Do small batched updates because these updates will be slow and locking + User.select(:id).find_in_batches(batch_size: 100) do |batch| + User.where(id: batch.map(&:id)).update_all(authentication_token: nil) + end + end + end +end diff --git a/lib/tasks/gitlab/web_hook.rake b/lib/tasks/gitlab/web_hook.rake index 49530e7a372..5a1c8006052 100644 --- a/lib/tasks/gitlab/web_hook.rake +++ b/lib/tasks/gitlab/web_hook.rake @@ -1,7 +1,7 @@ namespace :gitlab do namespace :web_hook do desc "GitLab | Adds a webhook to the projects" - task :add => :environment do + task add: :environment do web_hook_url = ENV['URL'] namespace_path = ENV['NAMESPACE'] @@ -21,7 +21,7 @@ namespace :gitlab do end desc "GitLab | Remove a webhook from the projects" - task :rm => :environment do + task rm: :environment do web_hook_url = ENV['URL'] namespace_path = ENV['NAMESPACE'] @@ -34,7 +34,7 @@ namespace :gitlab do end desc "GitLab | List webhooks" - task :list => :environment do + task list: :environment do namespace_path = ENV['NAMESPACE'] projects = find_projects(namespace_path) diff --git a/lib/tasks/gitlab/workhorse.rake b/lib/tasks/gitlab/workhorse.rake new file mode 100644 index 00000000000..baea94bf8ca --- /dev/null +++ b/lib/tasks/gitlab/workhorse.rake @@ -0,0 +1,23 @@ +namespace :gitlab do + namespace :workhorse do + desc "GitLab | Install or upgrade gitlab-workhorse" + task :install, [:dir] => :environment do |t, args| + warn_user_is_not_gitlab + unless args.dir.present? + abort %(Please specify the directory where you want to install gitlab-workhorse:\n rake "gitlab:workhorse:install[/home/git/gitlab-workhorse]") + end + + tag = "v#{Gitlab::Workhorse.version}" + repo = 'https://gitlab.com/gitlab-org/gitlab-workhorse.git' + + checkout_or_clone_tag(tag: tag, repo: repo, target_dir: args.dir) + + _, status = Gitlab::Popen.popen(%w[which gmake]) + command = status.zero? ? 'gmake' : 'make' + + Dir.chdir(args.dir) do + run_command!([command]) + end + end + end +end diff --git a/lib/tasks/grape.rake b/lib/tasks/grape.rake index 9980e0b7984..ea2698da606 100644 --- a/lib/tasks/grape.rake +++ b/lib/tasks/grape.rake @@ -2,7 +2,11 @@ namespace :grape do desc 'Print compiled grape routes' task routes: :environment do API::API.routes.each do |route| - puts route + puts "#{route.options[:method]} #{route.path} - #{route_description(route.options)}" end end + + def route_description(options) + options[:settings][:description][:description] if options[:settings][:description] + end end diff --git a/lib/tasks/haml-lint.rake b/lib/tasks/haml-lint.rake new file mode 100644 index 00000000000..609dfaa48e3 --- /dev/null +++ b/lib/tasks/haml-lint.rake @@ -0,0 +1,5 @@ +unless Rails.env.production? + require 'haml_lint/rake_task' + + HamlLint::RakeTask.new +end diff --git a/lib/tasks/karma.rake b/lib/tasks/karma.rake new file mode 100644 index 00000000000..40465ea3bf0 --- /dev/null +++ b/lib/tasks/karma.rake @@ -0,0 +1,20 @@ +unless Rails.env.production? + namespace :karma do + desc 'GitLab | Karma | Generate fixtures for JavaScript tests' + RSpec::Core::RakeTask.new(:fixtures) do |t| + ENV['NO_KNAPSACK'] = 'true' + t.pattern = 'spec/javascripts/fixtures/*.rb' + t.rspec_opts = '--format documentation' + end + + desc 'GitLab | Karma | Run JavaScript tests' + task tests: ['yarn:check'] do + sh "yarn run karma" do |ok, res| + abort('rake karma:tests failed') unless ok + end + end + end + + desc 'GitLab | Karma | Shortcut for karma:fixtures and karma:tests' + task karma: ['karma:fixtures', 'karma:tests'] +end diff --git a/lib/tasks/lint.rake b/lib/tasks/lint.rake new file mode 100644 index 00000000000..7b63e93db0e --- /dev/null +++ b/lib/tasks/lint.rake @@ -0,0 +1,8 @@ +unless Rails.env.production? + namespace :lint do + desc "GitLab | lint | Lint JavaScript files using ESLint" + task :javascript do + Rake::Task['eslint'].invoke + end + end +end diff --git a/lib/tasks/migrate/migrate_iids.rake b/lib/tasks/migrate/migrate_iids.rake index 4f2486157b7..fc2cea8c016 100644 --- a/lib/tasks/migrate/migrate_iids.rake +++ b/lib/tasks/migrate/migrate_iids.rake @@ -24,7 +24,7 @@ task migrate_iids: :environment do else print 'F' end - rescue => ex + rescue print 'F' end end diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake index 141a0b74ec0..f5caca3ddbf 100644 --- a/lib/tasks/migrate/setup_postgresql.rake +++ b/lib/tasks/migrate/setup_postgresql.rake @@ -1,8 +1,12 @@ +require Rails.root.join('lib/gitlab/database') +require Rails.root.join('lib/gitlab/database/migration_helpers') require Rails.root.join('db/migrate/20151007120511_namespaces_projects_path_lower_indexes') require Rails.root.join('db/migrate/20151008110232_add_users_lower_username_email_indexes') +require Rails.root.join('db/migrate/20161212142807_add_lower_path_index_to_routes') desc 'GitLab | Sets up PostgreSQL' task setup_postgresql: :environment do NamespacesProjectsPathLowerIndexes.new.up AddUsersLowerUsernameEmailIndexes.new.up + AddLowerPathIndexToRoutes.new.up end diff --git a/lib/tasks/services.rake b/lib/tasks/services.rake index 39541c0b9c6..56b81106c5f 100644 --- a/lib/tasks/services.rake +++ b/lib/tasks/services.rake @@ -76,23 +76,23 @@ namespace :services do end param_hash - end.sort_by { |p| p[:required] ? 0 : 1 } + end + service_hash[:params].sort_by! { |p| p[:required] ? 0 : 1 } - puts "Collected data for: #{service.title}, #{Time.now-service_start}" + puts "Collected data for: #{service.title}, #{Time.now - service_start}" service_hash end doc_start = Time.now doc_path = File.join(Rails.root, 'doc', 'api', 'services.md') - result = ERB.new(services_template, 0 , '>') + result = ERB.new(services_template, 0, '>') .result(OpenStruct.new(services: services).instance_eval { binding }) File.open(doc_path, 'w') do |f| f.write result end - puts "write a new service.md to: #{doc_path.to_s}, #{Time.now-doc_start}" - + puts "write a new service.md to: #{doc_path}, #{Time.now - doc_start}" end end diff --git a/lib/tasks/sidekiq.rake b/lib/tasks/sidekiq.rake index d1f6ed87704..dd9ce86f7ca 100644 --- a/lib/tasks/sidekiq.rake +++ b/lib/tasks/sidekiq.rake @@ -1,21 +1,21 @@ namespace :sidekiq do desc "GitLab | Stop sidekiq" task :stop do - system *%W(bin/background_jobs stop) + system(*%w(bin/background_jobs stop)) end desc "GitLab | Start sidekiq" task :start do - system *%W(bin/background_jobs start) + system(*%w(bin/background_jobs start)) end desc 'GitLab | Restart sidekiq' task :restart do - system *%W(bin/background_jobs restart) + system(*%w(bin/background_jobs restart)) end desc "GitLab | Start sidekiq with launchd on Mac OS X" task :launchd do - system *%W(bin/background_jobs start_no_deamonize) + system(*%w(bin/background_jobs start_no_deamonize)) end end diff --git a/lib/tasks/spec.rake b/lib/tasks/spec.rake index 2cf7a25a0fd..602c60be828 100644 --- a/lib/tasks/spec.rake +++ b/lib/tasks/spec.rake @@ -4,8 +4,8 @@ namespace :spec do desc 'GitLab | Rspec | Run request specs' task :api do cmds = [ - %W(rake gitlab:setup), - %W(rspec spec --tag @api) + %w(rake gitlab:setup), + %w(rspec spec --tag @api) ] run_commands(cmds) end @@ -13,8 +13,8 @@ namespace :spec do desc 'GitLab | Rspec | Run feature specs' task :feature do cmds = [ - %W(rake gitlab:setup), - %W(rspec spec --tag @feature) + %w(rake gitlab:setup), + %w(rspec spec --tag @feature) ] run_commands(cmds) end @@ -22,8 +22,8 @@ namespace :spec do desc 'GitLab | Rspec | Run model specs' task :models do cmds = [ - %W(rake gitlab:setup), - %W(rspec spec --tag @models) + %w(rake gitlab:setup), + %w(rspec spec --tag @models) ] run_commands(cmds) end @@ -31,8 +31,8 @@ namespace :spec do desc 'GitLab | Rspec | Run service specs' task :services do cmds = [ - %W(rake gitlab:setup), - %W(rspec spec --tag @services) + %w(rake gitlab:setup), + %w(rspec spec --tag @services) ] run_commands(cmds) end @@ -40,8 +40,8 @@ namespace :spec do desc 'GitLab | Rspec | Run lib specs' task :lib do cmds = [ - %W(rake gitlab:setup), - %W(rspec spec --tag @lib) + %w(rake gitlab:setup), + %w(rspec spec --tag @lib) ] run_commands(cmds) end @@ -49,8 +49,8 @@ namespace :spec do desc 'GitLab | Rspec | Run other specs' task :other do cmds = [ - %W(rake gitlab:setup), - %W(rspec spec --tag ~@api --tag ~@feature --tag ~@models --tag ~@lib --tag ~@services) + %w(rake gitlab:setup), + %w(rspec spec --tag ~@api --tag ~@feature --tag ~@models --tag ~@lib --tag ~@services) ] run_commands(cmds) end @@ -59,14 +59,14 @@ end desc "GitLab | Run specs" task :spec do cmds = [ - %W(rake gitlab:setup), - %W(rspec spec), + %w(rake gitlab:setup), + %w(rspec spec), ] run_commands(cmds) end def run_commands(cmds) cmds.each do |cmd| - system({'RAILS_ENV' => 'test', 'force' => 'yes'}, *cmd) or raise("#{cmd} failed!") + system({ 'RAILS_ENV' => 'test', 'force' => 'yes' }, *cmd) || raise("#{cmd} failed!") end end diff --git a/lib/tasks/spinach.rake b/lib/tasks/spinach.rake index 8dbfa7751dc..19ff13f06c0 100644 --- a/lib/tasks/spinach.rake +++ b/lib/tasks/spinach.rake @@ -35,7 +35,7 @@ task :spinach do end def run_system_command(cmd) - system({'RAILS_ENV' => 'test', 'force' => 'yes'}, *cmd) + system({ 'RAILS_ENV' => 'test', 'force' => 'yes' }, *cmd) end def run_spinach_command(args) diff --git a/lib/tasks/test.rake b/lib/tasks/test.rake index d3dcbd2c29b..3e01f91d32c 100644 --- a/lib/tasks/test.rake +++ b/lib/tasks/test.rake @@ -7,5 +7,5 @@ end unless Rails.env.production? desc "GitLab | Run all tests on CI with simplecov" - task test_ci: [:rubocop, :brakeman, :teaspoon, :spinach, :spec] + task test_ci: [:rubocop, :brakeman, :karma, :spinach, :spec] end diff --git a/lib/tasks/yarn.rake b/lib/tasks/yarn.rake new file mode 100644 index 00000000000..2ac88a039e7 --- /dev/null +++ b/lib/tasks/yarn.rake @@ -0,0 +1,40 @@ + +namespace :yarn do + desc 'Ensure Yarn is installed' + task :available do + unless system('yarn --version', out: File::NULL) + warn( + 'Error: Yarn executable was not detected in the system.'.color(:red), + 'Download Yarn at https://yarnpkg.com/en/docs/install'.color(:green) + ) + abort + end + end + + desc 'Ensure Node dependencies are installed' + task check: ['yarn:available'] do + unless system('yarn check --ignore-engines', out: File::NULL) + warn( + 'Error: You have unmet dependencies. (`yarn check` command failed)'.color(:red), + 'Run `yarn install` to install missing modules.'.color(:green) + ) + abort + end + end + + desc 'Install Node dependencies with Yarn' + task install: ['yarn:available'] do + unless system('yarn install --pure-lockfile --ignore-engines') + abort 'Error: Unable to install node modules.'.color(:red) + end + end + + desc 'Remove Node dependencies' + task :clobber do + warn 'Purging ./node_modules directory'.color(:red) + FileUtils.rm_rf 'node_modules' + end +end + +desc 'Install Node dependencies with Yarn' +task yarn: ['yarn:install'] |