diff options
Diffstat (limited to 'lib')
218 files changed, 5902 insertions, 2406 deletions
diff --git a/lib/api/access_requests.rb b/lib/api/access_requests.rb index d02b469dac8..87915b19480 100644 --- a/lib/api/access_requests.rb +++ b/lib/api/access_requests.rb @@ -5,32 +5,27 @@ module API helpers ::API::Helpers::MembersHelpers %w[group project].each do |source_type| + params do + requires :id, type: String, desc: "The #{source_type} ID" + end resource source_type.pluralize do - # Get a list of group/project access requests viewable by the authenticated user. - # - # Parameters: - # id (required) - The group/project ID - # - # Example Request: - # GET /groups/:id/access_requests - # GET /projects/:id/access_requests + desc "Gets a list of access requests for a #{source_type}." do + detail 'This feature was introduced in GitLab 8.11.' + success Entities::AccessRequester + end get ":id/access_requests" do source = find_source(source_type, params[:id]) - authorize_admin_source!(source_type, source) - access_requesters = paginate(source.requesters.includes(:user)) + access_requesters = AccessRequestsFinder.new(source).execute!(current_user) + access_requesters = paginate(access_requesters.includes(:user)) - present access_requesters.map(&:user), with: Entities::AccessRequester, access_requesters: access_requesters + present access_requesters.map(&:user), with: Entities::AccessRequester, source: source end - # Request access to the group/project - # - # Parameters: - # id (required) - The group/project ID - # - # Example Request: - # POST /groups/:id/access_requests - # POST /projects/:id/access_requests + desc "Requests access for the authenticated user to a #{source_type}." do + detail 'This feature was introduced in GitLab 8.11.' + success Entities::AccessRequester + end post ":id/access_requests" do source = find_source(source_type, params[:id]) access_requester = source.request_access(current_user) @@ -42,47 +37,34 @@ module API end end - # Approve a group/project access request - # - # Parameters: - # id (required) - The group/project ID - # user_id (required) - The user ID of the access requester - # access_level (optional) - Access level - # - # Example Request: - # PUT /groups/:id/access_requests/:user_id/approve - # PUT /projects/:id/access_requests/:user_id/approve + desc 'Approves an access request for the given user.' do + detail 'This feature was introduced in GitLab 8.11.' + success Entities::Member + end + params do + requires :user_id, type: Integer, desc: 'The user ID of the access requester' + optional :access_level, type: Integer, desc: 'A valid access level (defaults: `30`, developer access level)' + end put ':id/access_requests/:user_id/approve' do - required_attributes! [:user_id] source = find_source(source_type, params[:id]) - authorize_admin_source!(source_type, source) - member = source.requesters.find_by!(user_id: params[:user_id]) - if params[:access_level] - member.update(access_level: params[:access_level]) - end - member.accept_request + member = ::Members::ApproveAccessRequestService.new(source, current_user, declared(params)).execute status :created present member.user, with: Entities::Member, member: member end - # Deny a group/project access request - # - # Parameters: - # id (required) - The group/project ID - # user_id (required) - The user ID of the access requester - # - # Example Request: - # DELETE /groups/:id/access_requests/:user_id - # DELETE /projects/:id/access_requests/:user_id + desc 'Denies an access request for the given user.' do + detail 'This feature was introduced in GitLab 8.11.' + end + params do + requires :user_id, type: Integer, desc: 'The user ID of the access requester' + end delete ":id/access_requests/:user_id" do - required_attributes! [:user_id] source = find_source(source_type, params[:id]) - access_requester = source.requesters.find_by!(user_id: params[:user_id]) - - ::Members::DestroyService.new(access_requester, current_user).execute + ::Members::DestroyService.new(source, current_user, params). + execute(:requesters) end end end diff --git a/lib/api/api.rb b/lib/api/api.rb index d43af3f24e9..67109ceeef9 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -18,31 +18,27 @@ module API end rescue_from :all do |exception| - # lifted from https://github.com/rails/rails/blob/master/actionpack/lib/action_dispatch/middleware/debug_exceptions.rb#L60 - # why is this not wrapped in something reusable? - trace = exception.backtrace - - message = "\n#{exception.class} (#{exception.message}):\n" - message << exception.annoted_source_code.to_s if exception.respond_to?(:annoted_source_code) - message << " " << trace.join("\n ") - - API.logger.add Logger::FATAL, message - rack_response({ 'message' => '500 Internal Server Error' }.to_json, 500) + handle_api_exception(exception) end format :json content_type :txt, "text/plain" # Ensure the namespace is right, otherwise we might load Grape::API::Helpers + helpers ::SentryHelper helpers ::API::Helpers + # Keep in alphabetical order mount ::API::AccessRequests mount ::API::AwardEmoji + mount ::API::Boards mount ::API::Branches + mount ::API::BroadcastMessages mount ::API::Builds - mount ::API::CommitStatuses mount ::API::Commits + mount ::API::CommitStatuses mount ::API::DeployKeys + mount ::API::Deployments mount ::API::Environments mount ::API::Files mount ::API::Groups @@ -50,15 +46,18 @@ module API mount ::API::Issues mount ::API::Keys mount ::API::Labels - mount ::API::LicenseTemplates + mount ::API::Lint mount ::API::Members + mount ::API::MergeRequestDiffs mount ::API::MergeRequests mount ::API::Milestones mount ::API::Namespaces mount ::API::Notes + mount ::API::NotificationSettings + mount ::API::Pipelines mount ::API::ProjectHooks - mount ::API::ProjectSnippets mount ::API::Projects + mount ::API::ProjectSnippets mount ::API::Repositories mount ::API::Runners mount ::API::Services @@ -73,5 +72,10 @@ module API mount ::API::Triggers mount ::API::Users mount ::API::Variables + mount ::API::Version + + route :any, '*path' do + error!('404 Not Found', 404) + end end end diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index 7e67edb203a..8cc7a26f1fa 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -33,46 +33,29 @@ module API # # If the token is revoked, then it raises RevokedError. # - # If the token is not found (nil), then it raises TokenNotFoundError. + # If the token is not found (nil), then it returns nil # # Arguments: # # scopes: (optional) scopes required for this guard. # Defaults to empty array. # - def doorkeeper_guard!(scopes: []) - if (access_token = find_access_token).nil? - raise TokenNotFoundError - - else - case validate_access_token(access_token, scopes) - when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE - raise InsufficientScopeError.new(scopes) - when Oauth2::AccessTokenValidationService::EXPIRED - raise ExpiredError - when Oauth2::AccessTokenValidationService::REVOKED - raise RevokedError - when Oauth2::AccessTokenValidationService::VALID - @current_user = User.find(access_token.resource_owner_id) - end - end - end - def doorkeeper_guard(scopes: []) - if access_token = find_access_token - case validate_access_token(access_token, scopes) - when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE - raise InsufficientScopeError.new(scopes) + access_token = find_access_token + return nil unless access_token + + case validate_access_token(access_token, scopes) + when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE + raise InsufficientScopeError.new(scopes) - when Oauth2::AccessTokenValidationService::EXPIRED - raise ExpiredError + when Oauth2::AccessTokenValidationService::EXPIRED + raise ExpiredError - when Oauth2::AccessTokenValidationService::REVOKED - raise RevokedError + when Oauth2::AccessTokenValidationService::REVOKED + raise RevokedError - when Oauth2::AccessTokenValidationService::VALID - @current_user = User.find(access_token.resource_owner_id) - end + when Oauth2::AccessTokenValidationService::VALID + @current_user = User.find(access_token.resource_owner_id) end end @@ -96,19 +79,6 @@ module API end module ClassMethods - # Installs the doorkeeper guard on the whole Grape API endpoint. - # - # Arguments: - # - # scopes: (optional) scopes required for this guard. - # Defaults to empty array. - # - def guard_all!(scopes: []) - before do - guard! scopes: scopes - end - end - private def install_error_responders(base) diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb index 2efe7e3adf3..e9ccba3b465 100644 --- a/lib/api/award_emoji.rb +++ b/lib/api/award_emoji.rb @@ -1,23 +1,26 @@ module API class AwardEmoji < Grape::API before { authenticate! } - AWARDABLES = [Issue, MergeRequest] + AWARDABLES = %w[issue merge_request snippet] resource :projects do AWARDABLES.each do |awardable_type| - awardable_string = awardable_type.to_s.underscore.pluralize - awardable_id_string = "#{awardable_type.to_s.underscore}_id" + awardable_string = awardable_type.pluralize + awardable_id_string = "#{awardable_type}_id" + + params do + requires :id, type: String, desc: 'The ID of a project' + requires :"#{awardable_id_string}", type: Integer, desc: "The ID of an Issue, Merge Request or Snippet" + end [ ":id/#{awardable_string}/:#{awardable_id_string}/award_emoji", ":id/#{awardable_string}/:#{awardable_id_string}/notes/:note_id/award_emoji" ].each do |endpoint| - # Get a list of project +awardable+ award emoji - # - # Parameters: - # id (required) - The ID of a project - # awardable_id (required) - The ID of an issue or MR - # Example Request: - # GET /projects/:id/issues/:awardable_id/award_emoji + + desc 'Get a list of project +awardable+ award emoji' do + detail 'This feature was introduced in 8.9' + success Entities::AwardEmoji + end get endpoint do if can_read_awardable? awards = paginate(awardable.award_emoji) @@ -27,14 +30,13 @@ module API end end - # Get a specific award emoji - # - # Parameters: - # id (required) - The ID of a project - # awardable_id (required) - The ID of an issue or MR - # award_id (required) - The ID of the award - # Example Request: - # GET /projects/:id/issues/:awardable_id/award_emoji/:award_id + desc 'Get a specific award emoji' do + detail 'This feature was introduced in 8.9' + success Entities::AwardEmoji + end + params do + requires :award_id, type: Integer, desc: 'The ID of the award' + end get "#{endpoint}/:award_id" do if can_read_awardable? present awardable.award_emoji.find(params[:award_id]), with: Entities::AwardEmoji @@ -43,18 +45,15 @@ module API end end - # Award a new Emoji - # - # Parameters: - # id (required) - The ID of a project - # awardable_id (required) - The ID of an issue or mr - # name (required) - The name of a award_emoji (without colons) - # Example Request: - # POST /projects/:id/issues/:awardable_id/award_emoji + desc 'Award a new Emoji' do + detail 'This feature was introduced in 8.9' + success Entities::AwardEmoji + end + params do + requires :name, type: String, desc: 'The name of a award_emoji (without colons)' + end post endpoint do - required_attributes! [:name] - - not_found!('Award Emoji') unless can_read_awardable? + not_found!('Award Emoji') unless can_read_awardable? && can_award_awardable? award = awardable.create_award_emoji(params[:name], current_user) @@ -65,14 +64,13 @@ module API end end - # Delete a +awardables+ award emoji - # - # Parameters: - # id (required) - The ID of a project - # awardable_id (required) - The ID of an issue or MR - # award_emoji_id (required) - The ID of an award emoji - # Example Request: - # DELETE /projects/:id/issues/:issue_id/notes/:note_id/award_emoji/:award_id + desc 'Delete a +awardables+ award emoji' do + detail 'This feature was introduced in 8.9' + success Entities::AwardEmoji + end + params do + requires :award_id, type: Integer, desc: 'The ID of an award emoji' + end delete "#{endpoint}/:award_id" do award = awardable.award_emoji.find(params[:award_id]) @@ -87,27 +85,36 @@ module API helpers do def can_read_awardable? - ability = "read_#{awardable.class.to_s.underscore}".to_sym + can?(current_user, read_ability(awardable), awardable) + end - can?(current_user, ability, awardable) + def can_award_awardable? + awardable.user_can_award?(current_user, params[:name]) end def awardable @awardable ||= begin if params.include?(:note_id) - noteable.notes.find(params[:note_id]) + note_id = params.delete(:note_id) + + awardable.notes.find(note_id) + elsif params.include?(:issue_id) + user_project.issues.find(params[:issue_id]) + elsif params.include?(:merge_request_id) + user_project.merge_requests.find(params[:merge_request_id]) else - noteable + user_project.snippets.find(params[:snippet_id]) end end end - def noteable - if params.include?(:issue_id) - user_project.issues.find(params[:issue_id]) + def read_ability(awardable) + case awardable + when Note + read_ability(awardable.noteable) else - user_project.merge_requests.find(params[:merge_request_id]) + :"read_#{awardable.class.to_s.underscore}" end end end diff --git a/lib/api/boards.rb b/lib/api/boards.rb new file mode 100644 index 00000000000..4ac491edc1b --- /dev/null +++ b/lib/api/boards.rb @@ -0,0 +1,132 @@ +module API + # Boards API + class Boards < Grape::API + before { authenticate! } + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects do + desc 'Get all project boards' do + detail 'This feature was introduced in 8.13' + success Entities::Board + end + get ':id/boards' do + authorize!(:read_board, user_project) + present user_project.boards, with: Entities::Board + end + + params do + requires :board_id, type: Integer, desc: 'The ID of a board' + end + segment ':id/boards/:board_id' do + helpers do + def project_board + board = user_project.boards.first + + if params[:board_id] == board.id + board + else + not_found!('Board') + end + end + + def board_lists + project_board.lists.destroyable + end + end + + desc 'Get the lists of a project board' do + detail 'Does not include `backlog` and `done` lists. This feature was introduced in 8.13' + success Entities::List + end + get '/lists' do + authorize!(:read_board, user_project) + present board_lists, with: Entities::List + end + + desc 'Get a list of a project board' do + detail 'This feature was introduced in 8.13' + success Entities::List + end + params do + requires :list_id, type: Integer, desc: 'The ID of a list' + end + get '/lists/:list_id' do + authorize!(:read_board, user_project) + present board_lists.find(params[:list_id]), with: Entities::List + end + + desc 'Create a new board list' do + detail 'This feature was introduced in 8.13' + success Entities::List + end + params do + requires :label_id, type: Integer, desc: 'The ID of an existing label' + end + post '/lists' do + unless available_labels.exists?(params[:label_id]) + render_api_error!({ error: 'Label not found!' }, 400) + end + + authorize!(:admin_list, user_project) + + service = ::Boards::Lists::CreateService.new(user_project, current_user, + { label_id: params[:label_id] }) + + list = service.execute(project_board) + + if list.valid? + present list, with: Entities::List + else + render_validation_error!(list) + end + end + + desc 'Moves a board list to a new position' do + detail 'This feature was introduced in 8.13' + success Entities::List + end + params do + requires :list_id, type: Integer, desc: 'The ID of a list' + requires :position, type: Integer, desc: 'The position of the list' + end + put '/lists/:list_id' do + list = project_board.lists.movable.find(params[:list_id]) + + authorize!(:admin_list, user_project) + + service = ::Boards::Lists::MoveService.new(user_project, current_user, + { position: params[:position] }) + + if service.execute(list) + present list, with: Entities::List + else + render_api_error!({ error: "List could not be moved!" }, 400) + end + end + + desc 'Delete a board list' do + detail 'This feature was introduced in 8.13' + success Entities::List + end + params do + requires :list_id, type: Integer, desc: 'The ID of a board list' + end + delete "/lists/:list_id" do + authorize!(:admin_list, user_project) + + list = board_lists.find(params[:list_id]) + + service = ::Boards::Lists::DestroyService.new(user_project, current_user) + + if service.execute(list) + present list, with: Entities::List + else + render_api_error!({ error: 'List could not be deleted!' }, 400) + end + end + end + end + end +end diff --git a/lib/api/branches.rb b/lib/api/branches.rb index b615703df93..21a106387f0 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -6,124 +6,100 @@ module API before { authenticate! } before { authorize! :download_code, user_project } + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do - # Get a project repository branches - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # GET /projects/:id/repository/branches + desc 'Get a project repository branches' do + success Entities::RepoBranch + end get ":id/repository/branches" do branches = user_project.repository.branches.sort_by(&:name) present branches, with: Entities::RepoBranch, project: user_project end - # Get a single branch - # - # Parameters: - # id (required) - The ID of a project - # branch (required) - The name of the branch - # Example Request: - # GET /projects/:id/repository/branches/:branch - get ':id/repository/branches/:branch', requirements: { branch: /.+/ } do - @branch = user_project.repository.branches.find { |item| item.name == params[:branch] } - not_found!("Branch") unless @branch + desc 'Get a single branch' do + success Entities::RepoBranch + end + params do + requires :branch, type: String, regexp: /.+/, desc: 'The name of the branch' + end + get ':id/repository/branches/:branch' do + branch = user_project.repository.find_branch(params[:branch]) + not_found!("Branch") unless branch - present @branch, with: Entities::RepoBranch, project: user_project + present branch, with: Entities::RepoBranch, project: user_project end - # Protect a single branch - # # Note: The internal data model moved from `developers_can_{merge,push}` to `allowed_to_{merge,push}` # in `gitlab-org/gitlab-ce!5081`. The API interface has not been changed (to maintain compatibility), # but it works with the changed data model to infer `developers_can_merge` and `developers_can_push`. - # - # Parameters: - # id (required) - The ID of a project - # branch (required) - The name of the branch - # developers_can_push (optional) - Flag if developers can push to that branch - # developers_can_merge (optional) - Flag if developers can merge to that branch - # Example Request: - # PUT /projects/:id/repository/branches/:branch/protect - put ':id/repository/branches/:branch/protect', - requirements: { branch: /.+/ } do + desc 'Protect a single branch' do + success Entities::RepoBranch + end + params do + requires :branch, type: String, regexp: /.+/, desc: 'The name of the branch' + optional :developers_can_push, type: Boolean, desc: 'Flag if developers can push to that branch' + optional :developers_can_merge, type: Boolean, desc: 'Flag if developers can merge to that branch' + end + put ':id/repository/branches/:branch/protect' do authorize_admin_project - @branch = user_project.repository.find_branch(params[:branch]) - not_found!('Branch') unless @branch - protected_branch = user_project.protected_branches.find_by(name: @branch.name) + branch = user_project.repository.find_branch(params[:branch]) + not_found!('Branch') unless branch - developers_can_merge = to_boolean(params[:developers_can_merge]) - developers_can_push = to_boolean(params[:developers_can_push]) + protected_branch = user_project.protected_branches.find_by(name: branch.name) protected_branch_params = { - name: @branch.name + name: branch.name, + developers_can_push: params[:developers_can_push], + developers_can_merge: params[:developers_can_merge] } - # If `developers_can_merge` is switched off, _all_ `DEVELOPER` - # merge_access_levels need to be deleted. - if developers_can_merge == false - protected_branch.merge_access_levels.where(access_level: Gitlab::Access::DEVELOPER).destroy_all - end + service_args = [user_project, current_user, protected_branch_params] - # If `developers_can_push` is switched off, _all_ `DEVELOPER` - # push_access_levels need to be deleted. - if developers_can_push == false - protected_branch.push_access_levels.where(access_level: Gitlab::Access::DEVELOPER).destroy_all - end + protected_branch = if protected_branch + ProtectedBranches::ApiUpdateService.new(*service_args).execute(protected_branch) + else + ProtectedBranches::ApiCreateService.new(*service_args).execute + end - protected_branch_params.merge!( - merge_access_levels_attributes: [{ - access_level: developers_can_merge ? Gitlab::Access::DEVELOPER : Gitlab::Access::MASTER - }], - push_access_levels_attributes: [{ - access_level: developers_can_push ? Gitlab::Access::DEVELOPER : Gitlab::Access::MASTER - }] - ) - - if protected_branch - service = ProtectedBranches::UpdateService.new(user_project, current_user, protected_branch_params) - service.execute(protected_branch) + if protected_branch.valid? + present branch, with: Entities::RepoBranch, project: user_project else - service = ProtectedBranches::CreateService.new(user_project, current_user, protected_branch_params) - service.execute + render_api_error!(protected_branch.errors.full_messages, 422) end - - present @branch, with: Entities::RepoBranch, project: user_project end - # Unprotect a single branch - # - # Parameters: - # id (required) - The ID of a project - # branch (required) - The name of the branch - # Example Request: - # PUT /projects/:id/repository/branches/:branch/unprotect - put ':id/repository/branches/:branch/unprotect', - requirements: { branch: /.+/ } do + desc 'Unprotect a single branch' do + success Entities::RepoBranch + end + params do + requires :branch, type: String, regexp: /.+/, desc: 'The name of the branch' + end + put ':id/repository/branches/:branch/unprotect' do authorize_admin_project - @branch = user_project.repository.find_branch(params[:branch]) - not_found!("Branch") unless @branch - protected_branch = user_project.protected_branches.find_by(name: @branch.name) + branch = user_project.repository.find_branch(params[:branch]) + not_found!("Branch") unless branch + protected_branch = user_project.protected_branches.find_by(name: branch.name) protected_branch.destroy if protected_branch - present @branch, with: Entities::RepoBranch, project: user_project + present branch, with: Entities::RepoBranch, project: user_project end - # Create branch - # - # Parameters: - # id (required) - The ID of a project - # branch_name (required) - The name of the branch - # ref (required) - Create branch from commit sha or existing branch - # Example Request: - # POST /projects/:id/repository/branches + desc 'Create branch' do + success Entities::RepoBranch + end + params do + requires :branch_name, type: String, desc: 'The name of the branch' + requires :ref, type: String, desc: 'Create branch from commit sha or existing branch' + end post ":id/repository/branches" do authorize_push_project result = CreateBranchService.new(user_project, current_user). - execute(params[:branch_name], params[:ref]) + execute(params[:branch_name], params[:ref]) if result[:status] == :success present result[:branch], @@ -134,18 +110,15 @@ module API end end - # Delete branch - # - # Parameters: - # id (required) - The ID of a project - # branch (required) - The name of the branch - # Example Request: - # DELETE /projects/:id/repository/branches/:branch - delete ":id/repository/branches/:branch", - requirements: { branch: /.+/ } do + desc 'Delete a branch' + params do + requires :branch, type: String, regexp: /.+/, desc: 'The name of the branch' + end + delete ":id/repository/branches/:branch" do authorize_push_project + result = DeleteBranchService.new(user_project, current_user). - execute(params[:branch]) + execute(params[:branch]) if result[:status] == :success { diff --git a/lib/api/broadcast_messages.rb b/lib/api/broadcast_messages.rb new file mode 100644 index 00000000000..fb2a4148011 --- /dev/null +++ b/lib/api/broadcast_messages.rb @@ -0,0 +1,99 @@ +module API + class BroadcastMessages < Grape::API + before { authenticate! } + before { authenticated_as_admin! } + + resource :broadcast_messages do + helpers do + def find_message + BroadcastMessage.find(params[:id]) + end + end + + desc 'Get all broadcast messages' do + detail 'This feature was introduced in GitLab 8.12.' + success Entities::BroadcastMessage + end + params do + optional :page, type: Integer, desc: 'Current page number' + optional :per_page, type: Integer, desc: 'Number of messages per page' + end + get do + messages = BroadcastMessage.all + + present paginate(messages), with: Entities::BroadcastMessage + end + + desc 'Create a broadcast message' do + detail 'This feature was introduced in GitLab 8.12.' + success Entities::BroadcastMessage + end + params do + requires :message, type: String, desc: 'Message to display' + optional :starts_at, type: DateTime, desc: 'Starting time', default: -> { Time.zone.now } + optional :ends_at, type: DateTime, desc: 'Ending time', default: -> { 1.hour.from_now } + optional :color, type: String, desc: 'Background color' + optional :font, type: String, desc: 'Foreground color' + end + post do + create_params = declared(params, include_missing: false).to_h + message = BroadcastMessage.create(create_params) + + if message.persisted? + present message, with: Entities::BroadcastMessage + else + render_validation_error!(message) + end + end + + desc 'Get a specific broadcast message' do + detail 'This feature was introduced in GitLab 8.12.' + success Entities::BroadcastMessage + end + params do + requires :id, type: Integer, desc: 'Broadcast message ID' + end + get ':id' do + message = find_message + + present message, with: Entities::BroadcastMessage + end + + desc 'Update a broadcast message' do + detail 'This feature was introduced in GitLab 8.12.' + success Entities::BroadcastMessage + end + params do + requires :id, type: Integer, desc: 'Broadcast message ID' + optional :message, type: String, desc: 'Message to display' + optional :starts_at, type: DateTime, desc: 'Starting time' + optional :ends_at, type: DateTime, desc: 'Ending time' + optional :color, type: String, desc: 'Background color' + optional :font, type: String, desc: 'Foreground color' + end + put ':id' do + message = find_message + update_params = declared(params, include_missing: false).to_h + + if message.update(update_params) + present message, with: Entities::BroadcastMessage + else + render_validation_error!(message) + end + end + + desc 'Delete a broadcast message' do + detail 'This feature was introduced in GitLab 8.12.' + success Entities::BroadcastMessage + end + params do + requires :id, type: Integer, desc: 'Broadcast message ID' + end + delete ':id' do + message = find_message + + present message.destroy, with: Entities::BroadcastMessage + end + end + end +end diff --git a/lib/api/builds.rb b/lib/api/builds.rb index be5a3484ec8..67adca6605f 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -3,15 +3,32 @@ module API class Builds < Grape::API before { authenticate! } + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do - # Get a project builds - # - # Parameters: - # id (required) - The ID of a project - # scope (optional) - The scope of builds to show (one or array of: pending, running, failed, success, canceled; - # if none provided showing all builds) - # Example Request: - # GET /projects/:id/builds + helpers do + params :optional_scope do + optional :scope, types: [String, Array[String]], desc: 'The scope of builds to show', + values: ['pending', 'running', 'failed', 'success', 'canceled'], + coerce_with: ->(scope) { + if scope.is_a?(String) + [scope] + elsif scope.is_a?(Hashie::Mash) + scope.values + else + ['unknown'] + end + } + end + end + + desc 'Get a project builds' do + success Entities::Build + end + params do + use :optional_scope + end get ':id/builds' do builds = user_project.builds.order('id DESC') builds = filter_builds(builds, params[:scope]) @@ -20,15 +37,13 @@ module API user_can_download_artifacts: can?(current_user, :read_build, user_project) end - # Get builds for a specific commit of a project - # - # Parameters: - # id (required) - The ID of a project - # sha (required) - The SHA id of a commit - # scope (optional) - The scope of builds to show (one or array of: pending, running, failed, success, canceled; - # if none provided showing all builds) - # Example Request: - # GET /projects/:id/repository/commits/:sha/builds + desc 'Get builds for a specific commit of a project' do + success Entities::Build + end + params do + requires :sha, type: String, desc: 'The SHA id of a commit' + use :optional_scope + end get ':id/repository/commits/:sha/builds' do authorize_read_builds! @@ -42,13 +57,12 @@ module API user_can_download_artifacts: can?(current_user, :read_build, user_project) end - # Get a specific build of a project - # - # Parameters: - # id (required) - The ID of a project - # build_id (required) - The ID of a build - # Example Request: - # GET /projects/:id/builds/:build_id + desc 'Get a specific build of a project' do + success Entities::Build + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end get ':id/builds/:build_id' do authorize_read_builds! @@ -58,13 +72,12 @@ module API user_can_download_artifacts: can?(current_user, :read_build, user_project) end - # Download the artifacts file from build - # - # Parameters: - # id (required) - The ID of a build - # token (required) - The build authorization token - # Example Request: - # GET /projects/:id/builds/:build_id/artifacts + desc 'Download the artifacts file from build' do + detail 'This feature was introduced in GitLab 8.5' + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end get ':id/builds/:build_id/artifacts' do authorize_read_builds! @@ -73,14 +86,13 @@ module API present_artifacts!(build.artifacts_file) end - # Download the artifacts file from ref_name and job - # - # Parameters: - # id (required) - The ID of a project - # ref_name (required) - The ref from repository - # job (required) - The name for the build - # Example Request: - # GET /projects/:id/builds/artifacts/:ref_name/download?job=name + desc 'Download the artifacts file from build' do + detail 'This feature was introduced in GitLab 8.10' + end + params do + requires :ref_name, type: String, desc: 'The ref from repository' + requires :job, type: String, desc: 'The name for the build' + end get ':id/builds/artifacts/:ref_name/download', requirements: { ref_name: /.+/ } do authorize_read_builds! @@ -91,17 +103,13 @@ module API present_artifacts!(latest_build.artifacts_file) end - # Get a trace of a specific build of a project - # - # Parameters: - # id (required) - The ID of a project - # build_id (required) - The ID of a build - # Example Request: - # GET /projects/:id/build/:build_id/trace - # # TODO: We should use `present_file!` and leave this implementation for backward compatibility (when build trace # is saved in the DB instead of file). But before that, we need to consider how to replace the value of # `runners_token` with some mask (like `xxxxxx`) when sending trace file directly by workhorse. + desc 'Get a trace of a specific build of a project' + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end get ':id/builds/:build_id/trace' do authorize_read_builds! @@ -115,13 +123,12 @@ module API body trace end - # Cancel a specific build of a project - # - # parameters: - # id (required) - the id of a project - # build_id (required) - the id of a build - # example request: - # post /projects/:id/build/:build_id/cancel + desc 'Cancel a specific build of a project' do + success Entities::Build + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end post ':id/builds/:build_id/cancel' do authorize_update_builds! @@ -133,13 +140,12 @@ module API user_can_download_artifacts: can?(current_user, :read_build, user_project) end - # Retry a specific build of a project - # - # parameters: - # id (required) - the id of a project - # build_id (required) - the id of a build - # example request: - # post /projects/:id/build/:build_id/retry + desc 'Retry a specific build of a project' do + success Entities::Build + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end post ':id/builds/:build_id/retry' do authorize_update_builds! @@ -152,13 +158,12 @@ module API user_can_download_artifacts: can?(current_user, :read_build, user_project) end - # Erase build (remove artifacts and build trace) - # - # Parameters: - # id (required) - the id of a project - # build_id (required) - the id of a build - # example Request: - # post /projects/:id/build/:build_id/erase + desc 'Erase build (remove artifacts and build trace)' do + success Entities::Build + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end post ':id/builds/:build_id/erase' do authorize_update_builds! @@ -170,13 +175,12 @@ module API user_can_download_artifacts: can?(current_user, :download_build_artifacts, user_project) end - # Keep the artifacts to prevent them from being deleted - # - # Parameters: - # id (required) - the id of a project - # build_id (required) - The ID of a build - # Example Request: - # POST /projects/:id/builds/:build_id/artifacts/keep + desc 'Keep the artifacts to prevent them from being deleted' do + success Entities::Build + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end post ':id/builds/:build_id/artifacts/keep' do authorize_update_builds! @@ -189,6 +193,27 @@ module API present build, with: Entities::Build, user_can_download_artifacts: can?(current_user, :read_build, user_project) end + + desc 'Trigger a manual build' do + success Entities::Build + detail 'This feature was added in GitLab 8.11' + end + params do + requires :build_id, type: Integer, desc: 'The ID of a Build' + end + post ":id/builds/:build_id/play" do + authorize_read_builds! + + build = get_build!(params[:build_id]) + + bad_request!("Unplayable Build") unless build.playable? + + build.play(current_user) + + status 200 + present build, with: Entities::Build, + user_can_download_artifacts: can?(current_user, :read_build, user_project) + end end helpers do @@ -214,14 +239,6 @@ module API return builds if scope.nil? || scope.empty? available_statuses = ::CommitStatus::AVAILABLE_STATUSES - scope = - if scope.is_a?(String) - [scope] - elsif scope.is_a?(Hashie::Mash) - scope.values - else - ['unknown'] - end unknown = scope - available_statuses render_api_error!('Scope contains invalid value(s)', 400) unless unknown.empty? diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index 4df6ca8333e..f54d4f06627 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -6,17 +6,17 @@ module API resource :projects do before { authenticate! } - # Get a commit's statuses - # - # Parameters: - # id (required) - The ID of a project - # sha (required) - The commit hash - # ref (optional) - The ref - # stage (optional) - The stage - # name (optional) - The name - # all (optional) - Show all statuses, default: false - # Examples: - # GET /projects/:id/repository/commits/:sha/statuses + desc "Get a commit's statuses" do + success Entities::CommitStatus + end + params do + requires :id, type: String, desc: 'The ID of a project' + requires :sha, type: String, desc: 'The commit hash' + optional :ref, type: String, desc: 'The ref' + optional :stage, type: String, desc: 'The stage' + optional :name, type: String, desc: 'The name' + optional :all, type: String, desc: 'Show all statuses, default: false' + end get ':id/repository/commits/:sha/statuses' do authorize!(:read_commit_status, user_project) @@ -31,22 +31,23 @@ module API present paginate(statuses), with: Entities::CommitStatus end - # Post status to commit - # - # Parameters: - # id (required) - The ID of a project - # sha (required) - The commit hash - # ref (optional) - The ref - # state (required) - The state of the status. Can be: pending, running, success, error or failure - # target_url (optional) - The target URL to associate with this status - # description (optional) - A short description of the status - # name or context (optional) - A string label to differentiate this status from the status of other systems. Default: "default" - # Examples: - # POST /projects/:id/statuses/:sha + desc 'Post status to a commit' do + success Entities::CommitStatus + end + params do + requires :id, type: String, desc: 'The ID of a project' + requires :sha, type: String, desc: 'The commit hash' + requires :state, type: String, desc: 'The state of the status', + values: ['pending', 'running', 'success', 'failed', 'canceled'] + optional :ref, type: String, desc: 'The ref' + optional :target_url, type: String, desc: 'The target URL to associate with this status' + optional :description, type: String, desc: 'A short description of the status' + optional :name, type: String, desc: 'A string label to differentiate this status from the status of other systems. Default: "default"' + optional :context, type: String, desc: 'A string label to differentiate this status from the status of other systems. Default: "default"' + end post ':id/statuses/:sha' do authorize! :create_commit_status, user_project - required_attributes! [:state] - attrs = attributes_for_keys [:ref, :target_url, :description, :context, :name] + commit = @project.commit(params[:sha]) not_found! 'Commit' unless commit @@ -58,36 +59,43 @@ module API # the first found branch on that commit ref = params[:ref] - unless ref - branches = @project.repository.branch_names_contains(commit.sha) - not_found! 'References for commit' if branches.none? - ref = branches.first - end + ref ||= @project.repository.branch_names_contains(commit.sha).first + not_found! 'References for commit' unless ref - pipeline = @project.ensure_pipeline(commit.sha, ref, current_user) + name = params[:name] || params[:context] || 'default' - name = params[:name] || params[:context] - status = GenericCommitStatus.running_or_pending.find_by(pipeline: pipeline, name: name, ref: params[:ref]) - status ||= GenericCommitStatus.new(project: @project, pipeline: pipeline, user: current_user) - status.update(attrs) + pipeline = @project.ensure_pipeline(ref, commit.sha, current_user) - case params[:state].to_s - when 'running' - status.run - when 'success' - status.success - when 'failed' - status.drop - when 'canceled' - status.cancel - else - status.status = params[:state].to_s - end + status = GenericCommitStatus.running_or_pending.find_or_initialize_by( + project: @project, + pipeline: pipeline, + user: current_user, + name: name, + ref: ref, + target_url: params[:target_url], + description: params[:description] + ) + + begin + case params[:state].to_s + when 'pending' + status.enqueue! + when 'running' + status.enqueue + status.run! + when 'success' + status.success! + when 'failed' + status.drop! + when 'canceled' + status.cancel! + else + render_api_error!('invalid state', 400) + end - if status.save present status, with: Entities::CommitStatus - else - render_validation_error!(status) + rescue StateMachines::InvalidTransition => e + render_api_error!(e.message, 400) end end end diff --git a/lib/api/commits.rb b/lib/api/commits.rb index b4eaf1813d4..2f2cf769481 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -6,102 +6,149 @@ module API before { authenticate! } before { authorize! :download_code, user_project } + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do - # Get a project repository commits - # - # Parameters: - # id (required) - The ID of a project - # ref_name (optional) - The name of a repository branch or tag, if not given the default branch is used - # since (optional) - Only commits after or in this date will be returned - # until (optional) - Only commits before or in this date will be returned - # Example Request: - # GET /projects/:id/repository/commits + desc 'Get a project repository commits' do + success Entities::RepoCommit + end + params do + optional :ref_name, type: String, desc: 'The name of a repository branch or tag, if not given the default branch is used' + optional :since, type: String, desc: 'Only commits after or in this date will be returned' + optional :until, type: String, desc: 'Only commits before or in this date will be returned' + optional :page, type: Integer, default: 0, desc: 'The page for pagination' + optional :per_page, type: Integer, default: 20, desc: 'The number of results per page' + optional :path, type: String, desc: 'The file path' + end get ":id/repository/commits" do + # TODO remove the next line for 9.0, use DateTime type in the params block datetime_attributes! :since, :until - page = (params[:page] || 0).to_i - per_page = (params[:per_page] || 20).to_i ref = params[:ref_name] || user_project.try(:default_branch) || 'master' - after = params[:since] - before = params[:until] + offset = params[:page] * params[:per_page] + + commits = user_project.repository.commits(ref, + path: params[:path], + limit: params[:per_page], + offset: offset, + after: params[:since], + before: params[:until]) - commits = user_project.repository.commits(ref, limit: per_page, offset: page * per_page, after: after, before: before) present commits, with: Entities::RepoCommit end - # Get a specific commit of a project - # - # Parameters: - # id (required) - The ID of a project - # sha (required) - The commit hash or name of a repository branch or tag - # Example Request: - # GET /projects/:id/repository/commits/:sha + desc 'Commit multiple file changes as one commit' do + success Entities::RepoCommitDetail + detail 'This feature was introduced in GitLab 8.13' + end + params do + requires :id, type: Integer, desc: 'The project ID' + requires :branch_name, type: String, desc: 'The name of branch' + requires :commit_message, type: String, desc: 'Commit message' + requires :actions, type: Array, desc: 'Actions to perform in commit' + optional :author_email, type: String, desc: 'Author email for commit' + optional :author_name, type: String, desc: 'Author name for commit' + end + post ":id/repository/commits" do + authorize! :push_code, user_project + + attrs = declared(params) + attrs[:source_branch] = attrs[:branch_name] + attrs[:target_branch] = attrs[:branch_name] + attrs[:actions].map! do |action| + action[:action] = action[:action].to_sym + action[:file_path].slice!(0) if action[:file_path] && action[:file_path].start_with?('/') + action[:previous_path].slice!(0) if action[:previous_path] && action[:previous_path].start_with?('/') + action + end + + result = ::Files::MultiService.new(user_project, current_user, attrs).execute + + if result[:status] == :success + commit_detail = user_project.repository.commits(result[:result], limit: 1).first + present commit_detail, with: Entities::RepoCommitDetail + else + render_api_error!(result[:message], 400) + end + end + + desc 'Get a specific commit of a project' do + success Entities::RepoCommitDetail + failure [[404, 'Not Found']] + end + params do + requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' + end get ":id/repository/commits/:sha" do - sha = params[:sha] - commit = user_project.commit(sha) + commit = user_project.commit(params[:sha]) + not_found! "Commit" unless commit + present commit, with: Entities::RepoCommitDetail end - # Get the diff for a specific commit of a project - # - # Parameters: - # id (required) - The ID of a project - # sha (required) - The commit or branch name - # Example Request: - # GET /projects/:id/repository/commits/:sha/diff + desc 'Get the diff for a specific commit of a project' do + failure [[404, 'Not Found']] + end + params do + requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' + end get ":id/repository/commits/:sha/diff" do - sha = params[:sha] - commit = user_project.commit(sha) + commit = user_project.commit(params[:sha]) + not_found! "Commit" unless commit + commit.raw_diffs.to_a end - # Get a commit's comments - # - # Parameters: - # id (required) - The ID of a project - # sha (required) - The commit hash - # Examples: - # GET /projects/:id/repository/commits/:sha/comments + desc "Get a commit's comments" do + success Entities::CommitNote + failure [[404, 'Not Found']] + end + params do + requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' + optional :per_page, type: Integer, desc: 'The amount of items per page for paginaion' + optional :page, type: Integer, desc: 'The page number for pagination' + end get ':id/repository/commits/:sha/comments' do - sha = params[:sha] - commit = user_project.commit(sha) + commit = user_project.commit(params[:sha]) + not_found! 'Commit' unless commit notes = Note.where(commit_id: commit.id).order(:created_at) + present paginate(notes), with: Entities::CommitNote end - # Post comment to commit - # - # Parameters: - # id (required) - The ID of a project - # sha (required) - The commit hash - # note (required) - Text of comment - # path (optional) - The file path - # line (optional) - The line number - # line_type (optional) - The type of line (new or old) - # Examples: - # POST /projects/:id/repository/commits/:sha/comments + desc 'Post comment to commit' do + success Entities::CommitNote + end + params do + requires :sha, type: String, regexp: /\A\h{6,40}\z/, desc: "The commit's SHA" + requires :note, type: String, desc: 'The text of the comment' + optional :path, type: String, desc: 'The file path' + given :path do + requires :line, type: Integer, desc: 'The line number' + requires :line_type, type: String, values: ['new', 'old'], default: 'new', desc: 'The type of the line' + end + end post ':id/repository/commits/:sha/comments' do - required_attributes! [:note] - - sha = params[:sha] - commit = user_project.commit(sha) + commit = user_project.commit(params[:sha]) not_found! 'Commit' unless commit + opts = { note: params[:note], noteable_type: 'Commit', commit_id: commit.id } - if params[:path] && params[:line] && params[:line_type] + if params[:path] commit.raw_diffs(all_diffs: true).each do |diff| next unless diff.new_path == params[:path] lines = Gitlab::Diff::Parser.new.parse(diff.diff.each_line) lines.each do |line| - next unless line.new_pos == params[:line].to_i && line.type == params[:line_type] + next unless line.new_pos == params[:line] && line.type == params[:line_type] break opts[:line_code] = Gitlab::Diff::LineCode.generate(diff.new_path, line.new_pos, line.old_pos) end diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb index 825e05fbae3..425df2c176a 100644 --- a/lib/api/deploy_keys.rb +++ b/lib/api/deploy_keys.rb @@ -49,18 +49,23 @@ module API attrs = attributes_for_keys [:title, :key] attrs[:key].strip! if attrs[:key] + # Check for an existing key joined to this project key = user_project.deploy_keys.find_by(key: attrs[:key]) - present key, with: Entities::SSHKey if key + if key + present key, with: Entities::SSHKey + break + end # Check for available deploy keys in other projects key = current_user.accessible_deploy_keys.find_by(key: attrs[:key]) if key user_project.deploy_keys << key present key, with: Entities::SSHKey + break end + # Create a new deploy key key = DeployKey.new attrs - if key.valid? && user_project.deploy_keys << key present key, with: Entities::SSHKey else diff --git a/lib/api/deployments.rb b/lib/api/deployments.rb new file mode 100644 index 00000000000..f782bcaf7e9 --- /dev/null +++ b/lib/api/deployments.rb @@ -0,0 +1,40 @@ +module API + # Deployments RESTfull API endpoints + class Deployments < Grape::API + before { authenticate! } + + params do + requires :id, type: String, desc: 'The project ID' + end + resource :projects do + desc 'Get all deployments of the project' do + detail 'This feature was introduced in GitLab 8.11.' + success Entities::Deployment + end + params do + optional :page, type: Integer, desc: 'Page number of the current request' + optional :per_page, type: Integer, desc: 'Number of items per page' + end + get ':id/deployments' do + authorize! :read_deployment, user_project + + present paginate(user_project.deployments), with: Entities::Deployment + end + + desc 'Gets a specific deployment' do + detail 'This feature was introduced in GitLab 8.11.' + success Entities::Deployment + end + params do + requires :deployment_id, type: Integer, desc: 'The deployment ID' + end + get ':id/deployments/:deployment_id' do + authorize! :read_deployment, user_project + + deployment = user_project.deployments.find(params[:deployment_id]) + + present deployment, with: Entities::Deployment + end + end + end +end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 5df65a2327d..147aaf06b18 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -15,7 +15,7 @@ module API class User < UserBasic expose :created_at expose :is_admin?, as: :is_admin - expose :bio, :location, :skype, :linkedin, :twitter, :website_url + expose :bio, :location, :skype, :linkedin, :twitter, :website_url, :organization end class Identity < Grape::Entity @@ -43,14 +43,13 @@ module API end class Hook < Grape::Entity - expose :id, :url, :created_at + expose :id, :url, :created_at, :push_events, :tag_push_events + expose :enable_ssl_verification end class ProjectHook < Hook - expose :project_id, :push_events - expose :issues_events, :merge_requests_events, :tag_push_events - expose :note_events, :build_events, :pipeline_events - expose :enable_ssl_verification + expose :project_id, :issues_events, :merge_requests_events + expose :note_events, :build_events, :pipeline_events, :wiki_page_events end class BasicProjectDetails < Grape::Entity @@ -76,40 +75,58 @@ module API expose :owner, using: Entities::UserBasic, unless: ->(project, options) { project.group } expose :name, :name_with_namespace expose :path, :path_with_namespace - expose :issues_enabled, :merge_requests_enabled, :wiki_enabled, :builds_enabled, :snippets_enabled, :container_registry_enabled + expose :container_registry_enabled + + # Expose old field names with the new permissions methods to keep API compatible + expose(:issues_enabled) { |project, options| project.feature_available?(:issues, options[:user]) } + expose(:merge_requests_enabled) { |project, options| project.feature_available?(:merge_requests, options[:user]) } + expose(:wiki_enabled) { |project, options| project.feature_available?(:wiki, options[:user]) } + expose(:builds_enabled) { |project, options| project.feature_available?(:builds, options[:user]) } + expose(:snippets_enabled) { |project, options| project.feature_available?(:snippets, options[:user]) } + expose :created_at, :last_activity_at expose :shared_runners_enabled + expose :lfs_enabled?, as: :lfs_enabled expose :creator_id expose :namespace expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda{ |project, options| project.forked? } expose :avatar_url expose :star_count, :forks_count - expose :open_issues_count, if: lambda { |project, options| project.issues_enabled? && project.default_issues_tracker? } + expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:user]) && project.default_issues_tracker? } expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] } expose :public_builds expose :shared_with_groups do |project, options| SharedGroup.represent(project.project_group_links.all, options) end + expose :only_allow_merge_if_build_succeeds + expose :request_access_enabled + expose :only_allow_merge_if_all_discussions_are_resolved end class Member < UserBasic expose :access_level do |user, options| - member = options[:member] || options[:members].find { |m| m.user_id == user.id } + member = options[:member] || options[:source].members.find_by(user_id: user.id) member.access_level end + expose :expires_at do |user, options| + member = options[:member] || options[:source].members.find_by(user_id: user.id) + member.expires_at + end end class AccessRequester < UserBasic expose :requested_at do |user, options| - access_requester = options[:access_requester] || options[:access_requesters].find { |m| m.user_id == user.id } + access_requester = options[:access_requester] || options[:source].requesters.find_by(user_id: user.id) access_requester.requested_at end end class Group < Grape::Entity expose :id, :name, :path, :description, :visibility_level + expose :lfs_enabled?, as: :lfs_enabled expose :avatar_url expose :web_url + expose :request_access_enabled end class GroupDetail < Group @@ -121,7 +138,7 @@ module API expose :name expose :commit do |repo_branch, options| - options[:project].repository.commit(repo_branch.target) + options[:project].repository.commit(repo_branch.dereferenced_target) end expose :protected do |repo_branch, options| @@ -173,6 +190,10 @@ module API # TODO (rspeicher): Deprecated; remove in 9.0 expose(:expires_at) { |snippet| nil } + + expose :web_url do |snippet, options| + Gitlab::UrlBuilder.build(snippet) + end end class ProjectEntity < Grape::Entity @@ -202,6 +223,11 @@ module API expose :user_notes_count expose :upvotes, :downvotes expose :due_date + expose :confidential + + expose :web_url do |issue, options| + Gitlab::UrlBuilder.build(issue) + end end class ExternalIssue < Grape::Entity @@ -219,12 +245,18 @@ module API expose :milestone, using: Entities::Milestone expose :merge_when_build_succeeds expose :merge_status + expose :diff_head_sha, as: :sha + expose :merge_commit_sha expose :subscribed do |merge_request, options| merge_request.subscribed?(options[:current_user]) end expose :user_notes_count expose :should_remove_source_branch?, as: :should_remove_source_branch expose :force_remove_source_branch?, as: :force_remove_source_branch + + expose :web_url do |merge_request, options| + Gitlab::UrlBuilder.build(merge_request) + end end class MergeRequestChanges < MergeRequest @@ -233,6 +265,19 @@ module API end end + class MergeRequestDiff < Grape::Entity + expose :id, :head_commit_sha, :base_commit_sha, :start_commit_sha, + :created_at, :merge_request_id, :state, :real_size + end + + class MergeRequestDiffFull < MergeRequestDiff + expose :commits, using: Entities::RepoCommit + + expose :diffs, using: Entities::RepoDiff do |compare, _| + compare.raw_diffs(all_diffs: true).to_a + end + end + class SSHKey < Grape::Entity expose :id, :title, :key, :created_at, :can_push end @@ -298,7 +343,7 @@ module API end class ProjectGroupLink < Grape::Entity - expose :id, :project_id, :group_id, :group_access + expose :id, :project_id, :group_id, :group_access, :expires_at end class Todo < Grape::Entity @@ -334,7 +379,7 @@ module API expose :access_level expose :notification_level do |member, options| if member.notification_setting - NotificationSetting.levels[member.notification_setting.level] + ::NotificationSetting.levels[member.notification_setting.level] end end end @@ -345,6 +390,21 @@ module API class GroupAccess < MemberAccess end + class NotificationSetting < Grape::Entity + expose :level + expose :events, if: ->(notification_setting, _) { notification_setting.custom? } do + ::NotificationSetting::EMAIL_EVENTS.each do |event| + expose event + end + end + end + + class GlobalNotificationSetting < NotificationSetting + expose :notification_email do |notification_setting, options| + notification_setting.user.notification_email + end + end + class ProjectService < Grape::Entity expose :id, :title, :created_at, :updated_at, :active expose :push_events, :issues_events, :merge_requests_events @@ -372,15 +432,34 @@ module API end end - class Label < Grape::Entity - expose :name, :color, :description + class LabelBasic < Grape::Entity + expose :id, :name, :color, :description + end + + class Label < LabelBasic expose :open_issues_count, :closed_issues_count, :open_merge_requests_count + expose :priority do |label, options| + label.priority(options[:project]) + end expose :subscribed do |label, options| label.subscribed?(options[:current_user]) end end + class List < Grape::Entity + expose :id + expose :label, using: Entities::LabelBasic + expose :position + end + + class Board < Grape::Entity + expose :id + expose :lists, using: Entities::List do |board| + board.lists.destroyable + end + end + class Compare < Grape::Entity expose :commit, using: Entities::RepoCommit do |compare, options| Commit.decorate(compare.commits, nil).last @@ -434,6 +513,9 @@ module API expose :after_sign_out_path expose :container_registry_token_expire_delay expose :repository_storage + expose :repository_storages + expose :koding_enabled + expose :koding_url end class Release < Grape::Entity @@ -445,7 +527,7 @@ module API expose :name, :message expose :commit do |repo_tag, options| - options[:project].repository.commit(repo_tag.target) + options[:project].repository.commit(repo_tag.dereferenced_target) end expose :release, using: Entities::Release do |repo_tag, options| @@ -485,6 +567,10 @@ module API expose :filename, :size end + class PipelineBasic < Grape::Entity + expose :id, :sha, :ref, :status + end + class Build < Grape::Entity expose :id, :status, :stage, :name, :ref, :tag, :coverage expose :created_at, :started_at, :finished_at @@ -492,6 +578,7 @@ module API expose :artifacts_file, using: BuildArtifactFile, if: -> (build, opts) { build.artifacts? } expose :commit, with: RepoCommit expose :runner, with: Runner + expose :pipeline, with: PipelineBasic end class Trigger < Grape::Entity @@ -502,10 +589,29 @@ module API expose :key, :value end - class Environment < Grape::Entity + class Pipeline < PipelineBasic + expose :before_sha, :tag, :yaml_errors + + expose :user, with: Entities::UserBasic + expose :created_at, :updated_at, :started_at, :finished_at, :committed_at + expose :duration + end + + class EnvironmentBasic < Grape::Entity expose :id, :name, :external_url end + class Environment < EnvironmentBasic + expose :project, using: Entities::Project + end + + class Deployment < Grape::Entity + expose :id, :iid, :ref, :sha, :created_at + expose :user, using: Entities::UserBasic + expose :environment, using: Entities::EnvironmentBasic + expose :deployable, using: Entities::Build + end + class RepoLicense < Grape::Entity expose :key, :name, :nickname expose :featured, as: :popular @@ -525,5 +631,10 @@ module API class Template < Grape::Entity expose :name, :content end + + class BroadcastMessage < Grape::Entity + expose :id, :message, :starts_at, :ends_at, :color, :font + expose :active?, as: :active + end end end diff --git a/lib/api/files.rb b/lib/api/files.rb index c1d86f313b0..96510e651a3 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -11,14 +11,16 @@ module API target_branch: attrs[:branch_name], commit_message: attrs[:commit_message], file_content: attrs[:content], - file_content_encoding: attrs[:encoding] + file_content_encoding: attrs[:encoding], + author_email: attrs[:author_email], + author_name: attrs[:author_name] } end def commit_response(attrs) { file_path: attrs[:file_path], - branch_name: attrs[:branch_name], + branch_name: attrs[:branch_name] } end end @@ -96,7 +98,7 @@ module API authorize! :push_code, user_project required_attributes! [:file_path, :branch_name, :content, :commit_message] - attrs = attributes_for_keys [:file_path, :branch_name, :content, :commit_message, :encoding] + attrs = attributes_for_keys [:file_path, :branch_name, :content, :commit_message, :encoding, :author_email, :author_name] result = ::Files::CreateService.new(user_project, current_user, commit_params(attrs)).execute if result[:status] == :success @@ -122,7 +124,7 @@ module API authorize! :push_code, user_project required_attributes! [:file_path, :branch_name, :content, :commit_message] - attrs = attributes_for_keys [:file_path, :branch_name, :content, :commit_message, :encoding] + attrs = attributes_for_keys [:file_path, :branch_name, :content, :commit_message, :encoding, :author_email, :author_name] result = ::Files::UpdateService.new(user_project, current_user, commit_params(attrs)).execute if result[:status] == :success @@ -149,7 +151,7 @@ module API authorize! :push_code, user_project required_attributes! [:file_path, :branch_name, :commit_message] - attrs = attributes_for_keys [:file_path, :branch_name, :commit_message] + attrs = attributes_for_keys [:file_path, :branch_name, :commit_message, :author_email, :author_name] result = ::Files::DeleteService.new(user_project, current_user, commit_params(attrs)).execute if result[:status] == :success diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 9d8b8d737a9..40644fc2adf 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -6,34 +6,52 @@ module API resource :groups do # Get a groups list # + # Parameters: + # skip_groups (optional) - Array of group ids to exclude from list + # all_available (optional, boolean) - Show all group that you have access to # Example Request: # GET /groups get do @groups = if current_user.admin Group.all + elsif params[:all_available] + GroupsFinder.new.execute(current_user) else current_user.groups end @groups = @groups.search(params[:search]) if params[:search].present? + @groups = @groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present? @groups = paginate @groups present @groups, with: Entities::Group end + # Get list of owned groups for authenticated user + # + # Example Request: + # GET /groups/owned + get '/owned' do + @groups = current_user.owned_groups + @groups = paginate @groups + present @groups, with: Entities::Group, user: current_user + end + # Create group. Available only for users who can create groups. # # Parameters: - # name (required) - The name of the group - # path (required) - The path of the group - # description (optional) - The description of the group - # visibility_level (optional) - The visibility level of the group + # name (required) - The name of the group + # path (required) - The path of the group + # description (optional) - The description of the group + # visibility_level (optional) - The visibility level of the group + # lfs_enabled (optional) - Enable/disable LFS for the projects in this group + # request_access_enabled (optional) - Allow users to request member access # Example Request: # POST /groups post do - authorize! :create_group, current_user + authorize! :create_group required_attributes! [:name, :path] - attrs = attributes_for_keys [:name, :path, :description, :visibility_level] + attrs = attributes_for_keys [:name, :path, :description, :visibility_level, :lfs_enabled, :request_access_enabled] @group = Group.new(attrs) if @group.save @@ -47,17 +65,19 @@ module API # Update group. Available only for users who can administrate groups. # # Parameters: - # id (required) - The ID of a group - # path (optional) - The path of the group - # description (optional) - The description of the group - # visibility_level (optional) - The visibility level of the group + # id (required) - The ID of a group + # path (optional) - The path of the group + # description (optional) - The description of the group + # visibility_level (optional) - The visibility level of the group + # lfs_enabled (optional) - Enable/disable LFS for the projects in this group + # request_access_enabled (optional) - Allow users to request member access # Example Request: # PUT /groups/:id put ':id' do group = find_group(params[:id]) authorize! :admin_group, group - attrs = attributes_for_keys [:name, :path, :description, :visibility_level] + attrs = attributes_for_keys [:name, :path, :description, :visibility_level, :lfs_enabled, :request_access_enabled] if ::Groups::UpdateService.new(group, current_user, attrs).execute present group, with: Entities::GroupDetail @@ -97,7 +117,7 @@ module API group = find_group(params[:id]) projects = GroupProjectsFinder.new(group).execute(current_user) projects = paginate projects - present projects, with: Entities::Project + present projects, with: Entities::Project, user: current_user end # Transfer a project to the Group namespace diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index d0469d6602d..3c9d7b1aaef 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -1,24 +1,39 @@ module API module Helpers + include Gitlab::Utils + PRIVATE_TOKEN_HEADER = "HTTP_PRIVATE_TOKEN" PRIVATE_TOKEN_PARAM = :private_token SUDO_HEADER = "HTTP_SUDO" SUDO_PARAM = :sudo - def to_boolean(value) - return true if value =~ /^(true|t|yes|y|1|on)$/i - return false if value =~ /^(false|f|no|n|0|off)$/i + def private_token + params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER] + end + + def warden + env['warden'] + end - nil + # Check the Rails session for valid authentication details + # + # Until CSRF protection is added to the API, disallow this method for + # state-changing endpoints + def find_user_from_warden + warden.try(:authenticate) if %w[GET HEAD].include?(env['REQUEST_METHOD']) end def find_user_by_private_token - token_string = (params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]).to_s - User.find_by_authentication_token(token_string) || User.find_by_personal_access_token(token_string) + token = private_token + return nil unless token.present? + + User.find_by_authentication_token(token) || User.find_by_personal_access_token(token) end def current_user - @current_user ||= (find_user_by_private_token || doorkeeper_guard) + @current_user ||= find_user_by_private_token + @current_user ||= doorkeeper_guard + @current_user ||= find_user_from_warden unless @current_user && Gitlab::UserAccess.new(@current_user).allowed? return nil @@ -51,6 +66,10 @@ module API @project ||= find_project(params[:id]) end + def available_labels + @available_labels ||= LabelsFinder.new(current_user, project_id: user_project.id).execute + end + def find_project(id) project = Project.find_with_namespace(id) || Project.find_by(id: id) @@ -98,7 +117,7 @@ module API end def find_project_label(id) - label = user_project.labels.find_by_id(id) || user_project.labels.find_by_title(id) + label = available_labels.find_by_id(id) || available_labels.find_by_title(id) label || not_found!('Label') end @@ -129,7 +148,7 @@ module API forbidden! unless current_user.is_admin? end - def authorize!(action, subject) + def authorize!(action, subject = nil) forbidden! unless can?(current_user, action, subject) end @@ -148,7 +167,7 @@ module API end def can?(object, action, subject) - abilities.allowed?(object, action, subject) + Ability.allowed?(object, action, subject) end # Checks the occurrences of required attributes, each attribute must be present in the params hash @@ -177,16 +196,11 @@ module API def validate_label_params(params) errors = {} - if params[:labels].present? - params[:labels].split(',').each do |label_name| - label = user_project.labels.create_with( - color: Label::DEFAULT_COLOR).find_or_initialize_by( - title: label_name.strip) + params[:labels].to_s.split(',').each do |label_name| + label = available_labels.find_or_initialize_by(title: label_name.strip) + next if label.valid? - if label.invalid? - errors[label.title] = label.errors - end - end + errors[label.title] = label.errors end errors @@ -269,6 +283,10 @@ module API render_api_error!('304 Not Modified', 304) end + def no_content! + render_api_error!('204 No Content', 204) + end + def render_validation_error!(model) if model.errors.any? render_api_error!(model.errors.messages || '400 Bad Request', 400) @@ -279,6 +297,24 @@ module API error!({ 'message' => message }, status) end + def handle_api_exception(exception) + if sentry_enabled? && report_exception?(exception) + define_params_for_grape_middleware + sentry_context + Raven.capture_exception(exception) + end + + # lifted from https://github.com/rails/rails/blob/master/actionpack/lib/action_dispatch/middleware/debug_exceptions.rb#L60 + trace = exception.backtrace + + message = "\n#{exception.class} (#{exception.message}):\n" + message << exception.annoted_source_code.to_s if exception.respond_to?(:annoted_source_code) + message << " " << trace.join("\n ") + + API.logger.add Logger::FATAL, message + rack_response({ 'message' => '500 Internal Server Error' }.to_json, 500) + end + # Projects helpers def filter_projects(projects) @@ -390,16 +426,8 @@ module API links.join(', ') end - def abilities - @abilities ||= begin - abilities = Six.new - abilities << Ability - abilities - end - end - def secret_token - File.read(Gitlab.config.gitlab_shell.secret_file).chomp + Gitlab::Shell.secret_token end def send_git_blob(repository, blob) @@ -419,5 +447,19 @@ module API Entities::Issue end end + + # The Grape Error Middleware only has access to env but no params. We workaround this by + # defining a method that returns the right value. + def define_params_for_grape_middleware + self.define_singleton_method(:params) { Rack::Request.new(env).params.symbolize_keys } + end + + # We could get a Grape or a standard Ruby exception. We should only report anything that + # is clearly an error. + def report_exception?(exception) + return true unless exception.respond_to?(:status) + + exception.status == 500 + end end end diff --git a/lib/api/internal.rb b/lib/api/internal.rb index d8e9ac406c4..ccf181402f9 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -17,15 +17,20 @@ module API # helpers do + def project_path + @project_path ||= begin + project_path = params[:project].sub(/\.git\z/, '') + Repository.remove_storage_from_path(project_path) + end + end + def wiki? - @wiki ||= params[:project].end_with?('.wiki') && - !Project.find_with_namespace(params[:project]) + @wiki ||= project_path.end_with?('.wiki') && + !Project.find_with_namespace(project_path) end def project @project ||= begin - project_path = params[:project] - # Check for *.wiki repositories. # Strip out the .wiki from the pathname before finding the # project. This applies the correct project permissions to @@ -35,6 +40,14 @@ module API Project.find_with_namespace(project_path) end end + + def ssh_authentication_abilities + [ + :read_project, + :download_code, + :push_code + ] + end end post "/allowed" do @@ -51,9 +64,9 @@ module API access = if wiki? - Gitlab::GitAccessWiki.new(actor, project, protocol) + Gitlab::GitAccessWiki.new(actor, project, protocol, authentication_abilities: ssh_authentication_abilities) else - Gitlab::GitAccess.new(actor, project, protocol) + Gitlab::GitAccess.new(actor, project, protocol, authentication_abilities: ssh_authentication_abilities) end access_status = access.check(params[:action], params[:changes]) @@ -74,6 +87,19 @@ module API response end + post "/lfs_authenticate" do + status 200 + + key = Key.find(params[:key_id]) + token_handler = Gitlab::LfsToken.new(key) + + { + username: token_handler.actor_name, + lfs_token: token_handler.token, + repository_http_path: project.http_url_to_repo + } + end + get "/merge_request_urls" do ::MergeRequests::GetUrlsService.new(project).execute(params[:changes]) end @@ -101,6 +127,35 @@ module API {} end end + + post '/two_factor_recovery_codes' do + status 200 + + key = Key.find_by(id: params[:key_id]) + + unless key + return { 'success' => false, 'message' => 'Could not find the given key' } + end + + if key.is_a?(DeployKey) + return { success: false, message: 'Deploy keys cannot be used to retrieve recovery codes' } + end + + user = key.user + + unless user + return { success: false, message: 'Could not find a user for the given key' } + end + + unless user.two_factor_enabled? + return { success: false, message: 'Two-factor authentication is not enabled for this user' } + end + + codes = user.generate_otp_backup_codes! + user.save! + + { success: true, recovery_codes: codes } + end end end end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 077258faee1..c9689e6f8ef 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -41,7 +41,8 @@ module API issues = current_user.issues.inc_notes_with_associations issues = filter_issues_state(issues, params[:state]) unless params[:state].nil? issues = filter_issues_labels(issues, params[:labels]) unless params[:labels].nil? - issues.reorder(issuable_order_by => issuable_sort) + issues = issues.reorder(issuable_order_by => issuable_sort) + present paginate(issues), with: Entities::Issue, current_user: current_user end end @@ -73,7 +74,11 @@ module API params[:group_id] = group.id params[:milestone_title] = params.delete(:milestone) params[:label_name] = params.delete(:labels) - params[:sort] = "#{params.delete(:order_by)}_#{params.delete(:sort)}" if params[:order_by] && params[:sort] + + if params[:order_by] || params[:sort] + # The Sortable concern takes 'created_desc', not 'created_at_desc' (for example) + params[:sort] = "#{issuable_order_by.sub('_at', '')}_#{issuable_sort}" + end issues = IssuesFinder.new(current_user, params).execute @@ -113,7 +118,8 @@ module API issues = filter_issues_milestone(issues, params[:milestone]) end - issues.reorder(issuable_order_by => issuable_sort) + issues = issues.reorder(issuable_order_by => issuable_sort) + present paginate(issues), with: Entities::Issue, current_user: current_user end @@ -140,12 +146,13 @@ module API # labels (optional) - The labels of an issue # created_at (optional) - Date time string, ISO 8601 formatted # due_date (optional) - Date time string in the format YEAR-MONTH-DAY + # confidential (optional) - Boolean parameter if the issue should be confidential # Example Request: # POST /projects/:id/issues post ':id/issues' do required_attributes! [:title] - keys = [:title, :description, :assignee_id, :milestone_id, :due_date] + keys = [:title, :description, :assignee_id, :milestone_id, :due_date, :confidential] keys << :created_at if current_user.admin? || user_project.owner == current_user attrs = attributes_for_keys(keys) @@ -154,21 +161,19 @@ module API render_api_error!({ labels: errors }, 400) end - project = user_project + attrs[:labels] = params[:labels] if params[:labels] + + # Convert and filter out invalid confidential flags + attrs['confidential'] = to_boolean(attrs['confidential']) + attrs.delete('confidential') if attrs['confidential'].nil? - issue = ::Issues::CreateService.new(project, current_user, attrs.merge(request: request, api: true)).execute + issue = ::Issues::CreateService.new(user_project, current_user, attrs.merge(request: request, api: true)).execute if issue.spam? render_api_error!({ error: 'Spam detected' }, 400) end if issue.valid? - # Find or create labels and attach to issue. Labels are valid because - # we already checked its name, so there can't be an error here - if params[:labels].present? - issue.add_labels_by_names(params[:labels].split(',')) - end - present issue, with: Entities::Issue, current_user: current_user else render_validation_error!(issue) @@ -188,12 +193,13 @@ module API # state_event (optional) - The state event of an issue (close|reopen) # updated_at (optional) - Date time string, ISO 8601 formatted # due_date (optional) - Date time string in the format YEAR-MONTH-DAY + # confidential (optional) - Boolean parameter if the issue should be confidential # Example Request: # PUT /projects/:id/issues/:issue_id put ':id/issues/:issue_id' do issue = user_project.issues.find(params[:issue_id]) authorize! :update_issue, issue - keys = [:title, :description, :assignee_id, :milestone_id, :state_event, :due_date] + keys = [:title, :description, :assignee_id, :milestone_id, :state_event, :due_date, :confidential] keys << :updated_at if current_user.admin? || user_project.owner == current_user attrs = attributes_for_keys(keys) @@ -202,17 +208,15 @@ module API render_api_error!({ labels: errors }, 400) end + attrs[:labels] = params[:labels] if params[:labels] + + # Convert and filter out invalid confidential flags + attrs['confidential'] = to_boolean(attrs['confidential']) + attrs.delete('confidential') if attrs['confidential'].nil? + issue = ::Issues::UpdateService.new(user_project, current_user, attrs).execute(issue) if issue.valid? - # Find or create labels and attach to issue. Labels are valid because - # we already checked its name, so there can't be an error here - if params[:labels] && can?(current_user, :admin_issue, user_project) - issue.remove_labels - # Create and add labels to the new created issue - issue.add_labels_by_names(params[:labels].split(',')) - end - present issue, with: Entities::Issue, current_user: current_user else render_validation_error!(issue) diff --git a/lib/api/keys.rb b/lib/api/keys.rb index 2b723b79504..767f27ef334 100644 --- a/lib/api/keys.rb +++ b/lib/api/keys.rb @@ -4,10 +4,9 @@ module API before { authenticate! } resource :keys do - # Get single ssh key by id. Only available to admin users. - # - # Example Request: - # GET /keys/:id + desc 'Get single ssh key by id. Only available to admin users' do + success Entities::SSHKeyWithUser + end get ":id" do authenticated_as_admin! diff --git a/lib/api/labels.rb b/lib/api/labels.rb index c806829d69e..97218054f37 100644 --- a/lib/api/labels.rb +++ b/lib/api/labels.rb @@ -3,97 +3,97 @@ module API class Labels < Grape::API before { authenticate! } + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do - # Get all labels of the project - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # GET /projects/:id/labels + desc 'Get all labels of the project' do + success Entities::Label + end get ':id/labels' do - present user_project.labels, with: Entities::Label, current_user: current_user + present available_labels, with: Entities::Label, current_user: current_user, project: user_project end - # Creates a new label - # - # Parameters: - # id (required) - The ID of a project - # name (required) - The name of the label to be created - # color (required) - Color of the label given in 6-digit hex - # notation with leading '#' sign (e.g. #FFAABB) - # description (optional) - The description of label to be created - # Example Request: - # POST /projects/:id/labels + desc 'Create a new label' do + success Entities::Label + end + params do + requires :name, type: String, desc: 'The name of the label to be created' + requires :color, type: String, desc: "The color of the label given in 6-digit hex notation with leading '#' sign (e.g. #FFAABB)" + optional :description, type: String, desc: 'The description of label to be created' + optional :priority, type: Integer, desc: 'The priority of the label', allow_blank: true + end post ':id/labels' do authorize! :admin_label, user_project - required_attributes! [:name, :color] - - attrs = attributes_for_keys [:name, :color, :description] - label = user_project.find_label(attrs[:name]) + label = available_labels.find_by(title: params[:name]) conflict!('Label already exists') if label - label = user_project.labels.create(attrs) + priority = params.delete(:priority) + label_params = declared(params, + include_parent_namespaces: false, + include_missing: false).to_h + label = user_project.labels.create(label_params) if label.valid? - present label, with: Entities::Label, current_user: current_user + label.prioritize!(user_project, priority) if priority + present label, with: Entities::Label, current_user: current_user, project: user_project else render_validation_error!(label) end end - # Deletes an existing label - # - # Parameters: - # id (required) - The ID of a project - # name (required) - The name of the label to be deleted - # - # Example Request: - # DELETE /projects/:id/labels + desc 'Delete an existing label' do + success Entities::Label + end + params do + requires :name, type: String, desc: 'The name of the label to be deleted' + end delete ':id/labels' do authorize! :admin_label, user_project - required_attributes! [:name] - label = user_project.find_label(params[:name]) + label = user_project.labels.find_by(title: params[:name]) not_found!('Label') unless label - label.destroy + present label.destroy, with: Entities::Label, current_user: current_user, project: user_project end - # Updates an existing label. At least one optional parameter is required. - # - # Parameters: - # id (required) - The ID of a project - # name (required) - The name of the label to be deleted - # new_name (optional) - The new name of the label - # color (optional) - Color of the label given in 6-digit hex - # notation with leading '#' sign (e.g. #FFAABB) - # description (optional) - The description of label to be created - # Example Request: - # PUT /projects/:id/labels + desc 'Update an existing label. At least one optional parameter is required.' do + success Entities::Label + end + params do + requires :name, type: String, desc: 'The name of the label to be updated' + optional :new_name, type: String, desc: 'The new name of the label' + optional :color, type: String, desc: "The new color of the label given in 6-digit hex notation with leading '#' sign (e.g. #FFAABB)" + optional :description, type: String, desc: 'The new description of label' + optional :priority, type: Integer, desc: 'The priority of the label', allow_blank: true + at_least_one_of :new_name, :color, :description, :priority + end put ':id/labels' do authorize! :admin_label, user_project - required_attributes! [:name] - label = user_project.find_label(params[:name]) + label = user_project.labels.find_by(title: params[:name]) not_found!('Label not found') unless label - attrs = attributes_for_keys [:new_name, :color, :description] - - if attrs.empty? - render_api_error!('Required parameters "new_name" or "color" ' \ - 'missing', - 400) - end - + update_priority = params.key?(:priority) + priority = params.delete(:priority) + label_params = declared(params, + include_parent_namespaces: false, + include_missing: false).to_h # Rename new name to the actual label attribute name - attrs[:name] = attrs.delete(:new_name) if attrs.key?(:new_name) + label_params[:name] = label_params.delete('new_name') if label_params.key?('new_name') - if label.update(attrs) - present label, with: Entities::Label, current_user: current_user - else - render_validation_error!(label) + render_validation_error!(label) unless label.update(label_params) + + if update_priority + if priority.nil? + label.unprioritize!(user_project) + else + label.prioritize!(user_project, priority) + end end + + present label, with: Entities::Label, current_user: current_user, project: user_project end end end diff --git a/lib/api/license_templates.rb b/lib/api/license_templates.rb deleted file mode 100644 index d0552299ed0..00000000000 --- a/lib/api/license_templates.rb +++ /dev/null @@ -1,58 +0,0 @@ -module API - # License Templates API - class LicenseTemplates < Grape::API - PROJECT_TEMPLATE_REGEX = - /[\<\{\[] - (project|description| - one\sline\s.+\swhat\sit\sdoes\.) # matching the start and end is enough here - [\>\}\]]/xi.freeze - YEAR_TEMPLATE_REGEX = /[<{\[](year|yyyy)[>}\]]/i.freeze - FULLNAME_TEMPLATE_REGEX = - /[\<\{\[] - (fullname|name\sof\s(author|copyright\sowner)) - [\>\}\]]/xi.freeze - - # Get the list of the available license templates - # - # Parameters: - # popular - Filter licenses to only the popular ones - # - # Example Request: - # GET /licenses - # GET /licenses?popular=1 - get 'licenses' do - options = { - featured: params[:popular].present? ? true : nil - } - present Licensee::License.all(options), with: Entities::RepoLicense - end - - # Get text for specific license - # - # Parameters: - # key (required) - The key of a license - # project - Copyrighted project name - # fullname - Full name of copyright holder - # - # Example Request: - # GET /licenses/mit - # - get 'licenses/:key', requirements: { key: /[\w\.-]+/ } do - required_attributes! [:key] - - not_found!('License') unless Licensee::License.find(params[:key]) - - # We create a fresh Licensee::License object since we'll modify its - # content in place below. - license = Licensee::License.new(params[:key]) - - license.content.gsub!(YEAR_TEMPLATE_REGEX, Time.now.year.to_s) - license.content.gsub!(PROJECT_TEMPLATE_REGEX, params[:project]) if params[:project].present? - - fullname = params[:fullname].presence || current_user.try(:name) - license.content.gsub!(FULLNAME_TEMPLATE_REGEX, fullname) if fullname - - present license, with: Entities::RepoLicense - end - end -end diff --git a/lib/api/lint.rb b/lib/api/lint.rb new file mode 100644 index 00000000000..ae43a4a3237 --- /dev/null +++ b/lib/api/lint.rb @@ -0,0 +1,21 @@ +module API + class Lint < Grape::API + namespace :ci do + desc 'Validation of .gitlab-ci.yml content' + params do + requires :content, type: String, desc: 'Content of .gitlab-ci.yml' + end + post '/lint' do + error = Ci::GitlabCiYamlProcessor.validation_message(params[:content]) + + status 200 + + if error.blank? + { status: 'valid', errors: [] } + else + { status: 'invalid', errors: [error] } + end + end + end + end +end diff --git a/lib/api/members.rb b/lib/api/members.rb index 2fae83f60b2..b80818f0eb6 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -5,35 +5,32 @@ module API helpers ::API::Helpers::MembersHelpers %w[group project].each do |source_type| + params do + requires :id, type: String, desc: "The #{source_type} ID" + end resource source_type.pluralize do - # Get a list of group/project members viewable by the authenticated user. - # - # Parameters: - # id (required) - The group/project ID - # query - Query string - # - # Example Request: - # GET /groups/:id/members - # GET /projects/:id/members + desc 'Gets a list of group or project members viewable by the authenticated user.' do + success Entities::Member + end + params do + optional :query, type: String, desc: 'A query string to search for members' + end get ":id/members" do source = find_source(source_type, params[:id]) - members = source.members.includes(:user) - members = members.joins(:user).merge(User.search(params[:query])) if params[:query] - members = paginate(members) + users = source.users + users = users.merge(User.search(params[:query])) if params[:query] + users = paginate(users) - present members.map(&:user), with: Entities::Member, members: members + present users, with: Entities::Member, source: source end - # Get a group/project member - # - # Parameters: - # id (required) - The group/project ID - # user_id (required) - The user ID of the member - # - # Example Request: - # GET /groups/:id/members/:user_id - # GET /projects/:id/members/:user_id + desc 'Gets a member of a group or project.' do + success Entities::Member + end + params do + requires :user_id, type: Integer, desc: 'The user ID of the member' + end get ":id/members/:user_id" do source = find_source(source_type, params[:id]) @@ -43,47 +40,34 @@ module API present member.user, with: Entities::Member, member: member end - # Add a new group/project member - # - # Parameters: - # id (required) - The group/project ID - # user_id (required) - The user ID of the new member - # access_level (required) - A valid access level - # - # Example Request: - # POST /groups/:id/members - # POST /projects/:id/members + desc 'Adds a member to a group or project.' do + success Entities::Member + end + params do + requires :user_id, type: Integer, desc: 'The user ID of the new member' + requires :access_level, type: Integer, desc: 'A valid access level (defaults: `30`, developer access level)' + optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY' + end post ":id/members" do source = find_source(source_type, params[:id]) authorize_admin_source!(source_type, source) - required_attributes! [:user_id, :access_level] - - access_requester = source.requesters.find_by(user_id: params[:user_id]) - if access_requester - # We pass current_user = access_requester so that the requester doesn't - # receive a "access denied" email - ::Members::DestroyService.new(access_requester, access_requester.user).execute - end member = source.members.find_by(user_id: params[:user_id]) - # This is to ensure back-compatibility but 409 behavior should be used - # for both project and group members in 9.0! + # We need this explicit check because `source.add_user` doesn't + # currently return the member created so it would return 201 even if + # the member already existed... + # The `source_type == 'group'` check is to ensure back-compatibility + # but 409 behavior should be used for both project and group members in 9.0! conflict!('Member already exists') if source_type == 'group' && member unless member - source.add_user(params[:user_id], params[:access_level], current_user) - member = source.members.find_by(user_id: params[:user_id]) + member = source.add_user(params[:user_id], params[:access_level], current_user: current_user, expires_at: params[:expires_at]) end - if member + if member.persisted? && member.valid? present member.user, with: Entities::Member, member: member else - # Since `source.add_user` doesn't return a member object, we have to - # build a new one and populate its errors in order to render them. - member = source.members.build(attributes_for_keys([:user_id, :access_level])) - member.valid? # populate the errors - # This is to ensure back-compatibility but 400 behavior should be used # for all validation errors in 9.0! render_api_error!('Access level is not known', 422) if member.errors.key?(:access_level) @@ -91,24 +75,22 @@ module API end end - # Update a group/project member - # - # Parameters: - # id (required) - The group/project ID - # user_id (required) - The user ID of the member - # access_level (required) - A valid access level - # - # Example Request: - # PUT /groups/:id/members/:user_id - # PUT /projects/:id/members/:user_id + desc 'Updates a member of a group or project.' do + success Entities::Member + end + params do + requires :user_id, type: Integer, desc: 'The user ID of the new member' + requires :access_level, type: Integer, desc: 'A valid access level' + optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY' + end put ":id/members/:user_id" do source = find_source(source_type, params[:id]) authorize_admin_source!(source_type, source) - required_attributes! [:user_id, :access_level] member = source.members.find_by!(user_id: params[:user_id]) + attrs = attributes_for_keys [:access_level, :expires_at] - if member.update_attributes(access_level: params[:access_level]) + if member.update_attributes(attrs) present member.user, with: Entities::Member, member: member else # This is to ensure back-compatibility but 400 behavior should be used @@ -118,18 +100,12 @@ module API end end - # Remove a group/project member - # - # Parameters: - # id (required) - The group/project ID - # user_id (required) - The user ID of the member - # - # Example Request: - # DELETE /groups/:id/members/:user_id - # DELETE /projects/:id/members/:user_id + desc 'Removes a user from a group or project.' + params do + requires :user_id, type: Integer, desc: 'The user ID of the member' + end delete ":id/members/:user_id" do source = find_source(source_type, params[:id]) - required_attributes! [:user_id] # This is to ensure back-compatibility but find_by! should be used # in that casse in 9.0! @@ -144,7 +120,7 @@ module API if member.nil? { message: "Access revoked", id: params[:user_id].to_i } else - ::Members::DestroyService.new(member, current_user).execute + ::Members::DestroyService.new(source, current_user, declared(params)).execute present member.user, with: Entities::Member, member: member end diff --git a/lib/api/merge_request_diffs.rb b/lib/api/merge_request_diffs.rb new file mode 100644 index 00000000000..07435d78468 --- /dev/null +++ b/lib/api/merge_request_diffs.rb @@ -0,0 +1,45 @@ +module API + # MergeRequestDiff API + class MergeRequestDiffs < Grape::API + before { authenticate! } + + resource :projects do + desc 'Get a list of merge request diff versions' do + detail 'This feature was introduced in GitLab 8.12.' + success Entities::MergeRequestDiff + end + + params do + requires :id, type: String, desc: 'The ID of a project' + requires :merge_request_id, type: Integer, desc: 'The ID of a merge request' + end + + get ":id/merge_requests/:merge_request_id/versions" do + merge_request = user_project.merge_requests. + find(params[:merge_request_id]) + + authorize! :read_merge_request, merge_request + present merge_request.merge_request_diffs, with: Entities::MergeRequestDiff + end + + desc 'Get a single merge request diff version' do + detail 'This feature was introduced in GitLab 8.12.' + success Entities::MergeRequestDiffFull + end + + params do + requires :id, type: String, desc: 'The ID of a project' + requires :merge_request_id, type: Integer, desc: 'The ID of a merge request' + requires :version_id, type: Integer, desc: 'The ID of a merge request diff version' + end + + get ":id/merge_requests/:merge_request_id/versions/:version_id" do + merge_request = user_project.merge_requests. + find(params[:merge_request_id]) + + authorize! :read_merge_request, merge_request + present merge_request.merge_request_diffs.find(params[:version_id]), with: Entities::MergeRequestDiffFull + end + end + end +end diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 2b685621da9..bf8504e1101 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -86,14 +86,11 @@ module API render_api_error!({ labels: errors }, 400) end + attrs[:labels] = params[:labels] if params[:labels] + merge_request = ::MergeRequests::CreateService.new(user_project, current_user, attrs).execute if merge_request.valid? - # Find or create labels and attach to issue - if params[:labels].present? - merge_request.add_labels_by_names(params[:labels].split(",")) - end - present merge_request, with: Entities::MergeRequest, current_user: current_user else handle_merge_request_errors! merge_request.errors @@ -195,15 +192,11 @@ module API render_api_error!({ labels: errors }, 400) end + attrs[:labels] = params[:labels] if params[:labels] + merge_request = ::MergeRequests::UpdateService.new(user_project, current_user, attrs).execute(merge_request) if merge_request.valid? - # Find or create labels and attach to issue - unless params[:labels].nil? - merge_request.remove_labels - merge_request.add_labels_by_names(params[:labels].split(",")) - end - present merge_request, with: Entities::MergeRequest, current_user: current_user else handle_merge_request_errors! merge_request.errors diff --git a/lib/api/milestones.rb b/lib/api/milestones.rb index 7a0cb7c99f3..8984cf8cdcd 100644 --- a/lib/api/milestones.rb +++ b/lib/api/milestones.rb @@ -11,19 +11,25 @@ module API else milestones end end + + params :optional_params do + optional :description, type: String, desc: 'The description of the milestone' + optional :due_date, type: String, desc: 'The due date of the milestone' + end end + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do - # Get a list of project milestones - # - # Parameters: - # id (required) - The ID of a project - # state (optional) - Return "active" or "closed" milestones - # Example Request: - # GET /projects/:id/milestones - # GET /projects/:id/milestones?iid=42 - # GET /projects/:id/milestones?state=active - # GET /projects/:id/milestones?state=closed + desc 'Get a list of project milestones' do + success Entities::Milestone + end + params do + optional :state, type: String, values: %w[active closed all], default: 'all', + desc: 'Return "active", "closed", or "all" milestones' + optional :iid, type: Integer, desc: 'The IID of the milestone' + end get ":id/milestones" do authorize! :read_milestone, user_project @@ -34,34 +40,31 @@ module API present paginate(milestones), with: Entities::Milestone end - # Get a single project milestone - # - # Parameters: - # id (required) - The ID of a project - # milestone_id (required) - The ID of a project milestone - # Example Request: - # GET /projects/:id/milestones/:milestone_id + desc 'Get a single project milestone' do + success Entities::Milestone + end + params do + requires :milestone_id, type: Integer, desc: 'The ID of a project milestone' + end get ":id/milestones/:milestone_id" do authorize! :read_milestone, user_project - @milestone = user_project.milestones.find(params[:milestone_id]) - present @milestone, with: Entities::Milestone + milestone = user_project.milestones.find(params[:milestone_id]) + present milestone, with: Entities::Milestone end - # Create a new project milestone - # - # Parameters: - # id (required) - The ID of the project - # title (required) - The title of the milestone - # description (optional) - The description of the milestone - # due_date (optional) - The due date of the milestone - # Example Request: - # POST /projects/:id/milestones + desc 'Create a new project milestone' do + success Entities::Milestone + end + params do + requires :title, type: String, desc: 'The title of the milestone' + use :optional_params + end post ":id/milestones" do authorize! :admin_milestone, user_project - required_attributes! [:title] - attrs = attributes_for_keys [:title, :description, :due_date] - milestone = ::Milestones::CreateService.new(user_project, current_user, attrs).execute + milestone_params = declared(params, include_parent_namespaces: false) + + milestone = ::Milestones::CreateService.new(user_project, current_user, milestone_params).execute if milestone.valid? present milestone, with: Entities::Milestone @@ -70,22 +73,23 @@ module API end end - # Update an existing project milestone - # - # Parameters: - # id (required) - The ID of a project - # milestone_id (required) - The ID of a project milestone - # title (optional) - The title of a milestone - # description (optional) - The description of a milestone - # due_date (optional) - The due date of a milestone - # state_event (optional) - The state event of the milestone (close|activate) - # Example Request: - # PUT /projects/:id/milestones/:milestone_id + desc 'Update an existing project milestone' do + success Entities::Milestone + end + params do + requires :milestone_id, type: Integer, desc: 'The ID of a project milestone' + optional :title, type: String, desc: 'The title of the milestone' + optional :state_event, type: String, values: %w[close activate], + desc: 'The state event of the milestone ' + use :optional_params + at_least_one_of :title, :description, :due_date, :state_event + end put ":id/milestones/:milestone_id" do authorize! :admin_milestone, user_project - attrs = attributes_for_keys [:title, :description, :due_date, :state_event] - milestone = user_project.milestones.find(params[:milestone_id]) - milestone = ::Milestones::UpdateService.new(user_project, current_user, attrs).execute(milestone) + milestone_params = declared(params, include_parent_namespaces: false, include_missing: false) + + milestone = user_project.milestones.find(milestone_params.delete(:milestone_id)) + milestone = ::Milestones::UpdateService.new(user_project, current_user, milestone_params).execute(milestone) if milestone.valid? present milestone, with: Entities::Milestone @@ -94,22 +98,20 @@ module API end end - # Get all issues for a single project milestone - # - # Parameters: - # id (required) - The ID of a project - # milestone_id (required) - The ID of a project milestone - # Example Request: - # GET /projects/:id/milestones/:milestone_id/issues + desc 'Get all issues for a single project milestone' do + success Entities::Issue + end + params do + requires :milestone_id, type: Integer, desc: 'The ID of a project milestone' + end get ":id/milestones/:milestone_id/issues" do authorize! :read_milestone, user_project - @milestone = user_project.milestones.find(params[:milestone_id]) + milestone = user_project.milestones.find(params[:milestone_id]) finder_params = { project_id: user_project.id, - milestone_title: @milestone.title, - state: 'all' + milestone_title: milestone.title } issues = IssuesFinder.new(current_user, finder_params).execute diff --git a/lib/api/namespaces.rb b/lib/api/namespaces.rb index 50d3729449e..fe981d7b9fa 100644 --- a/lib/api/namespaces.rb +++ b/lib/api/namespaces.rb @@ -4,20 +4,18 @@ module API before { authenticate! } resource :namespaces do - # Get a namespaces list - # - # Example Request: - # GET /namespaces + desc 'Get a namespaces list' do + success Entities::Namespace + end + params do + optional :search, type: String, desc: "Search query for namespaces" + end get do - @namespaces = if current_user.admin - Namespace.all - else - current_user.namespaces - end - @namespaces = @namespaces.search(params[:search]) if params[:search].present? - @namespaces = paginate @namespaces + namespaces = current_user.admin ? Namespace.all : current_user.namespaces + + namespaces = namespaces.search(params[:search]) if params[:search].present? - present @namespaces, with: Entities::Namespace + present paginate(namespaces), with: Entities::Namespace end end end diff --git a/lib/api/notes.rb b/lib/api/notes.rb index 8bfa998dc53..c5c214d4d13 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -83,12 +83,12 @@ module API opts[:created_at] = params[:created_at] end - @note = ::Notes::CreateService.new(user_project, current_user, opts).execute + note = ::Notes::CreateService.new(user_project, current_user, opts).execute - if @note.valid? - present @note, with: Entities::Note + if note.valid? + present note, with: Entities::const_get(note.class.name) else - not_found!("Note #{@note.errors.messages}") + not_found!("Note #{note.errors.messages}") end end diff --git a/lib/api/notification_settings.rb b/lib/api/notification_settings.rb new file mode 100644 index 00000000000..a70a7e71073 --- /dev/null +++ b/lib/api/notification_settings.rb @@ -0,0 +1,97 @@ +module API + # notification_settings API + class NotificationSettings < Grape::API + before { authenticate! } + + helpers ::API::Helpers::MembersHelpers + + resource :notification_settings do + desc 'Get global notification level settings and email, defaults to Participate' do + detail 'This feature was introduced in GitLab 8.12' + success Entities::GlobalNotificationSetting + end + get do + notification_setting = current_user.global_notification_setting + + present notification_setting, with: Entities::GlobalNotificationSetting + end + + desc 'Update global notification level settings and email, defaults to Participate' do + detail 'This feature was introduced in GitLab 8.12' + success Entities::GlobalNotificationSetting + end + params do + optional :level, type: String, desc: 'The global notification level' + optional :notification_email, type: String, desc: 'The email address to send notifications' + NotificationSetting::EMAIL_EVENTS.each do |event| + optional event, type: Boolean, desc: 'Enable/disable this notification' + end + end + put do + notification_setting = current_user.global_notification_setting + + begin + notification_setting.transaction do + new_notification_email = params.delete(:notification_email) + declared_params = declared(params, include_missing: false).to_h + + current_user.update(notification_email: new_notification_email) if new_notification_email + notification_setting.update(declared_params) + end + rescue ArgumentError => e # catch level enum error + render_api_error! e.to_s, 400 + end + + render_validation_error! current_user + render_validation_error! notification_setting + present notification_setting, with: Entities::GlobalNotificationSetting + end + end + + %w[group project].each do |source_type| + resource source_type.pluralize do + desc "Get #{source_type} level notification level settings, defaults to Global" do + detail 'This feature was introduced in GitLab 8.12' + success Entities::NotificationSetting + end + params do + requires :id, type: String, desc: 'The group ID or project ID or project NAMESPACE/PROJECT_NAME' + end + get ":id/notification_settings" do + source = find_source(source_type, params[:id]) + + notification_setting = current_user.notification_settings_for(source) + + present notification_setting, with: Entities::NotificationSetting + end + + desc "Update #{source_type} level notification level settings, defaults to Global" do + detail 'This feature was introduced in GitLab 8.12' + success Entities::NotificationSetting + end + params do + requires :id, type: String, desc: 'The group ID or project ID or project NAMESPACE/PROJECT_NAME' + optional :level, type: String, desc: "The #{source_type} notification level" + NotificationSetting::EMAIL_EVENTS.each do |event| + optional event, type: Boolean, desc: 'Enable/disable this notification' + end + end + put ":id/notification_settings" do + source = find_source(source_type, params.delete(:id)) + notification_setting = current_user.notification_settings_for(source) + + begin + declared_params = declared(params, include_missing: false).to_h + + notification_setting.update(declared_params) + rescue ArgumentError => e # catch level enum error + render_api_error! e.to_s, 400 + end + + render_validation_error! notification_setting + present notification_setting, with: Entities::NotificationSetting + end + end + end + end +end diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb new file mode 100644 index 00000000000..2a0c8e1f2c0 --- /dev/null +++ b/lib/api/pipelines.rb @@ -0,0 +1,77 @@ +module API + class Pipelines < Grape::API + before { authenticate! } + + params do + requires :id, type: String, desc: 'The project ID' + end + resource :projects do + desc 'Get all Pipelines of the project' do + detail 'This feature was introduced in GitLab 8.11.' + success Entities::Pipeline + end + params do + optional :page, type: Integer, desc: 'Page number of the current request' + optional :per_page, type: Integer, desc: 'Number of items per page' + optional :scope, type: String, values: ['running', 'branches', 'tags'], + desc: 'Either running, branches, or tags' + end + get ':id/pipelines' do + authorize! :read_pipeline, user_project + + pipelines = PipelinesFinder.new(user_project).execute(scope: params[:scope]) + present paginate(pipelines), with: Entities::Pipeline + end + + desc 'Gets a specific pipeline for the project' do + detail 'This feature was introduced in GitLab 8.11' + success Entities::Pipeline + end + params do + requires :pipeline_id, type: Integer, desc: 'The pipeline ID' + end + get ':id/pipelines/:pipeline_id' do + authorize! :read_pipeline, user_project + + present pipeline, with: Entities::Pipeline + end + + desc 'Retry failed builds in the pipeline' do + detail 'This feature was introduced in GitLab 8.11.' + success Entities::Pipeline + end + params do + requires :pipeline_id, type: Integer, desc: 'The pipeline ID' + end + post ':id/pipelines/:pipeline_id/retry' do + authorize! :update_pipeline, user_project + + pipeline.retry_failed(current_user) + + present pipeline, with: Entities::Pipeline + end + + desc 'Cancel all builds in the pipeline' do + detail 'This feature was introduced in GitLab 8.11.' + success Entities::Pipeline + end + params do + requires :pipeline_id, type: Integer, desc: 'The pipeline ID' + end + post ':id/pipelines/:pipeline_id/cancel' do + authorize! :update_pipeline, user_project + + pipeline.cancel_running + + status 200 + present pipeline.reload, with: Entities::Pipeline + end + end + + helpers do + def pipeline + @pipeline ||= user_project.pipelines.find(params[:pipeline_id]) + end + end + end +end diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb index 3f63cd678e8..eef343c2ac6 100644 --- a/lib/api/project_hooks.rb +++ b/lib/api/project_hooks.rb @@ -1,110 +1,99 @@ module API # Projects API class ProjectHooks < Grape::API + helpers do + params :project_hook_properties do + requires :url, type: String, desc: "The URL to send the request to" + optional :push_events, type: Boolean, desc: "Trigger hook on push events" + optional :issues_events, type: Boolean, desc: "Trigger hook on issues events" + optional :merge_requests_events, type: Boolean, desc: "Trigger hook on merge request events" + optional :tag_push_events, type: Boolean, desc: "Trigger hook on tag push events" + optional :note_events, type: Boolean, desc: "Trigger hook on note(comment) events" + optional :build_events, type: Boolean, desc: "Trigger hook on build events" + optional :pipeline_events, type: Boolean, desc: "Trigger hook on pipeline events" + optional :wiki_events, type: Boolean, desc: "Trigger hook on wiki events" + optional :enable_ssl_verification, type: Boolean, desc: "Do SSL verification when triggering the hook" + optional :token, type: String, desc: "Secret token to validate received payloads; this will not be returned in the response" + end + end + before { authenticate! } before { authorize_admin_project } + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do - # Get project hooks - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # GET /projects/:id/hooks + desc 'Get project hooks' do + success Entities::ProjectHook + end get ":id/hooks" do - @hooks = paginate user_project.hooks - present @hooks, with: Entities::ProjectHook + hooks = paginate user_project.hooks + + present hooks, with: Entities::ProjectHook end - # Get a project hook - # - # Parameters: - # id (required) - The ID of a project - # hook_id (required) - The ID of a project hook - # Example Request: - # GET /projects/:id/hooks/:hook_id + desc 'Get a project hook' do + success Entities::ProjectHook + end + params do + requires :hook_id, type: Integer, desc: 'The ID of a project hook' + end get ":id/hooks/:hook_id" do - @hook = user_project.hooks.find(params[:hook_id]) - present @hook, with: Entities::ProjectHook + hook = user_project.hooks.find(params[:hook_id]) + present hook, with: Entities::ProjectHook end - # Add hook to project - # - # Parameters: - # id (required) - The ID of a project - # url (required) - The hook URL - # Example Request: - # POST /projects/:id/hooks + desc 'Add hook to project' do + success Entities::ProjectHook + end + params do + use :project_hook_properties + end post ":id/hooks" do - required_attributes! [:url] - attrs = attributes_for_keys [ - :url, - :push_events, - :issues_events, - :merge_requests_events, - :tag_push_events, - :note_events, - :build_events, - :pipeline_events, - :enable_ssl_verification - ] - @hook = user_project.hooks.new(attrs) + new_hook_params = declared(params, include_missing: false, include_parent_namespaces: false).to_h + hook = user_project.hooks.new(new_hook_params) - if @hook.save - present @hook, with: Entities::ProjectHook + if hook.save + present hook, with: Entities::ProjectHook else - if @hook.errors[:url].present? - error!("Invalid url given", 422) - end - not_found!("Project hook #{@hook.errors.messages}") + error!("Invalid url given", 422) if hook.errors[:url].present? + + not_found!("Project hook #{hook.errors.messages}") end end - # Update an existing project hook - # - # Parameters: - # id (required) - The ID of a project - # hook_id (required) - The ID of a project hook - # url (required) - The hook URL - # Example Request: - # PUT /projects/:id/hooks/:hook_id + desc 'Update an existing project hook' do + success Entities::ProjectHook + end + params do + requires :hook_id, type: Integer, desc: "The ID of the hook to update" + use :project_hook_properties + end put ":id/hooks/:hook_id" do - @hook = user_project.hooks.find(params[:hook_id]) - required_attributes! [:url] - attrs = attributes_for_keys [ - :url, - :push_events, - :issues_events, - :merge_requests_events, - :tag_push_events, - :note_events, - :build_events, - :pipeline_events, - :enable_ssl_verification - ] + hook = user_project.hooks.find(params[:hook_id]) + + new_params = declared(params, include_missing: false, include_parent_namespaces: false).to_h + new_params.delete('hook_id') - if @hook.update_attributes attrs - present @hook, with: Entities::ProjectHook + if hook.update_attributes(new_params) + present hook, with: Entities::ProjectHook else - if @hook.errors[:url].present? - error!("Invalid url given", 422) - end - not_found!("Project hook #{@hook.errors.messages}") + error!("Invalid url given", 422) if hook.errors[:url].present? + + not_found!("Project hook #{hook.errors.messages}") end end - # Deletes project hook. This is an idempotent function. - # - # Parameters: - # id (required) - The ID of a project - # hook_id (required) - The ID of hook to delete - # Example Request: - # DELETE /projects/:id/hooks/:hook_id + desc 'Deletes project hook' do + success Entities::ProjectHook + end + params do + requires :hook_id, type: Integer, desc: 'The ID of the hook to delete' + end delete ":id/hooks/:hook_id" do - required_attributes! [:hook_id] - begin - @hook = user_project.hooks.destroy(params[:hook_id]) + present user_project.hooks.destroy(params[:hook_id]), with: Entities::ProjectHook rescue # ProjectHook can raise Error if hook_id not found not_found!("Error deleting hook #{params[:hook_id]}") diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 60cfc103afd..6b856128c2e 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -22,14 +22,25 @@ module API # Example Request: # GET /projects get do - @projects = current_user.authorized_projects - @projects = filter_projects(@projects) - @projects = paginate @projects - if params[:simple] - present @projects, with: Entities::BasicProjectDetails, user: current_user - else - present @projects, with: Entities::ProjectWithAccess, user: current_user - end + projects = current_user.authorized_projects + projects = filter_projects(projects) + projects = paginate projects + entity = params[:simple] ? Entities::BasicProjectDetails : Entities::ProjectWithAccess + + present projects, with: entity, user: current_user + end + + # Get a list of visible projects for authenticated user + # + # Example Request: + # GET /projects/visible + get '/visible' do + projects = ProjectsFinder.new.execute(current_user) + projects = filter_projects(projects) + projects = paginate projects + entity = params[:simple] ? Entities::BasicProjectDetails : Entities::ProjectWithAccess + + present projects, with: entity, user: current_user end # Get an owned projects list for authenticated user @@ -37,10 +48,10 @@ module API # Example Request: # GET /projects/owned get '/owned' do - @projects = current_user.owned_projects - @projects = filter_projects(@projects) - @projects = paginate @projects - present @projects, with: Entities::ProjectWithAccess, user: current_user + projects = current_user.owned_projects + projects = filter_projects(projects) + projects = paginate projects + present projects, with: Entities::ProjectWithAccess, user: current_user end # Gets starred project for the authenticated user @@ -48,10 +59,10 @@ module API # Example Request: # GET /projects/starred get '/starred' do - @projects = current_user.viewable_starred_projects - @projects = filter_projects(@projects) - @projects = paginate @projects - present @projects, with: Entities::Project + projects = current_user.viewable_starred_projects + projects = filter_projects(projects) + projects = paginate projects + present projects, with: Entities::Project, user: current_user end # Get all projects for admin user @@ -60,10 +71,10 @@ module API # GET /projects/all get '/all' do authenticated_as_admin! - @projects = Project.all - @projects = filter_projects(@projects) - @projects = paginate @projects - present @projects, with: Entities::ProjectWithAccess, user: current_user + projects = Project.all + projects = filter_projects(projects) + projects = paginate projects + present projects, with: Entities::ProjectWithAccess, user: current_user end # Get a single project @@ -91,8 +102,8 @@ module API # Create new project # # Parameters: - # name (required) - name for new project - # description (optional) - short project description + # name (required) - name for new project + # description (optional) - short project description # issues_enabled (optional) # merge_requests_enabled (optional) # builds_enabled (optional) @@ -100,30 +111,36 @@ module API # snippets_enabled (optional) # container_registry_enabled (optional) # shared_runners_enabled (optional) - # namespace_id (optional) - defaults to user namespace - # public (optional) - if true same as setting visibility_level = 20 - # visibility_level (optional) - 0 by default + # namespace_id (optional) - defaults to user namespace + # public (optional) - if true same as setting visibility_level = 20 + # visibility_level (optional) - 0 by default # import_url (optional) # public_builds (optional) + # lfs_enabled (optional) + # request_access_enabled (optional) - Allow users to request member access # Example Request # POST /projects post do required_attributes! [:name] - attrs = attributes_for_keys [:name, - :path, + attrs = attributes_for_keys [:builds_enabled, + :container_registry_enabled, :description, + :import_url, :issues_enabled, + :lfs_enabled, :merge_requests_enabled, - :builds_enabled, - :wiki_enabled, - :snippets_enabled, - :container_registry_enabled, - :shared_runners_enabled, + :name, :namespace_id, + :only_allow_merge_if_build_succeeds, + :path, :public, + :public_builds, + :request_access_enabled, + :shared_runners_enabled, + :snippets_enabled, :visibility_level, - :import_url, - :public_builds] + :wiki_enabled, + :only_allow_merge_if_all_discussions_are_resolved] attrs = map_public_to_visibility_level(attrs) @project = ::Projects::CreateService.new(current_user, attrs).execute if @project.saved? @@ -140,10 +157,10 @@ module API # Create new project for a specified user. Only available to admin users. # # Parameters: - # user_id (required) - The ID of a user - # name (required) - name for new project - # description (optional) - short project description - # default_branch (optional) - 'master' by default + # user_id (required) - The ID of a user + # name (required) - name for new project + # description (optional) - short project description + # default_branch (optional) - 'master' by default # issues_enabled (optional) # merge_requests_enabled (optional) # builds_enabled (optional) @@ -151,28 +168,34 @@ module API # snippets_enabled (optional) # container_registry_enabled (optional) # shared_runners_enabled (optional) - # public (optional) - if true same as setting visibility_level = 20 + # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) # import_url (optional) # public_builds (optional) + # lfs_enabled (optional) + # request_access_enabled (optional) - Allow users to request member access # Example Request # POST /projects/user/:user_id post "user/:user_id" do authenticated_as_admin! user = User.find(params[:user_id]) - attrs = attributes_for_keys [:name, - :description, + attrs = attributes_for_keys [:builds_enabled, :default_branch, + :description, + :import_url, :issues_enabled, + :lfs_enabled, :merge_requests_enabled, - :builds_enabled, - :wiki_enabled, - :snippets_enabled, - :shared_runners_enabled, + :name, + :only_allow_merge_if_build_succeeds, :public, + :public_builds, + :request_access_enabled, + :shared_runners_enabled, + :snippets_enabled, :visibility_level, - :import_url, - :public_builds] + :wiki_enabled, + :only_allow_merge_if_all_discussions_are_resolved] attrs = map_public_to_visibility_level(attrs) @project = ::Projects::CreateService.new(user, attrs).execute if @project.saved? @@ -183,16 +206,32 @@ module API end end - # Fork new project for the current user. + # Fork new project for the current user or provided namespace. # # Parameters: # id (required) - The ID of a project + # namespace (optional) - The ID or name of the namespace that the project will be forked into. # Example Request # POST /projects/fork/:id post 'fork/:id' do + attrs = {} + namespace_id = params[:namespace] + + if namespace_id.present? + namespace = Namespace.find_by(id: namespace_id) || Namespace.find_by_path_or_name(namespace_id) + + unless namespace && can?(current_user, :create_projects, namespace) + not_found!('Target Namespace') + end + + attrs[:namespace] = namespace + end + @forked_project = ::Projects::ForkService.new(user_project, - current_user).execute + current_user, + attrs).execute + if @forked_project.errors.any? conflict!(@forked_project.errors.messages) else @@ -218,23 +257,28 @@ module API # public (optional) - if true same as setting visibility_level = 20 # visibility_level (optional) - visibility level of a project # public_builds (optional) + # lfs_enabled (optional) # Example Request # PUT /projects/:id put ':id' do - attrs = attributes_for_keys [:name, - :path, - :description, + attrs = attributes_for_keys [:builds_enabled, + :container_registry_enabled, :default_branch, + :description, :issues_enabled, + :lfs_enabled, :merge_requests_enabled, - :builds_enabled, - :wiki_enabled, - :snippets_enabled, - :container_registry_enabled, - :shared_runners_enabled, + :name, + :only_allow_merge_if_build_succeeds, + :path, :public, + :public_builds, + :request_access_enabled, + :shared_runners_enabled, + :snippets_enabled, :visibility_level, - :public_builds] + :wiki_enabled, + :only_allow_merge_if_all_discussions_are_resolved] attrs = map_public_to_visibility_level(attrs) authorize_admin_project authorize! :rename_project, user_project if attrs[:name].present? @@ -363,23 +407,30 @@ module API # Share project with group # # Parameters: - # id (required) - The ID of a project - # group_id (required) - The ID of a group + # id (required) - The ID of a project + # group_id (required) - The ID of a group # group_access (required) - Level of permissions for sharing + # expires_at (optional) - Share expiration date # # Example Request: # POST /projects/:id/share post ":id/share" do authorize! :admin_project, user_project required_attributes! [:group_id, :group_access] + attrs = attributes_for_keys [:group_id, :group_access, :expires_at] + + group = Group.find_by_id(attrs[:group_id]) + + unless group && can?(current_user, :read_group, group) + not_found!('Group') + end unless user_project.allowed_to_share_with_group? return render_api_error!("The project sharing with group is disabled", 400) end - link = user_project.project_group_links.new - link.group_id = params[:group_id] - link.group_access = params[:group_access] + link = user_project.project_group_links.new(attrs) + if link.save present link, with: Entities::ProjectGroupLink else @@ -405,18 +456,9 @@ module API # Example Request: # GET /projects/search/:query get "/search/:query" do - ids = current_user.authorized_projects.map(&:id) - visibility_levels = [ Gitlab::VisibilityLevel::INTERNAL, Gitlab::VisibilityLevel::PUBLIC ] - projects = Project.where("(id in (?) OR visibility_level in (?)) AND (name LIKE (?))", ids, visibility_levels, "%#{params[:query]}%") - sort = params[:sort] == 'desc' ? 'desc' : 'asc' - - projects = case params["order_by"] - when 'id' then projects.order("id #{sort}") - when 'name' then projects.order("name #{sort}") - when 'created_at' then projects.order("created_at #{sort}") - when 'last_activity_at' then projects.order("last_activity_at #{sort}") - else projects - end + search_service = Search::GlobalService.new(current_user, search: params[:query]).execute + projects = search_service.objects('projects', params[:page]) + projects = projects.reorder(project_order_by => project_sort) present paginate(projects), with: Entities::Project end diff --git a/lib/api/runners.rb b/lib/api/runners.rb index ecc8f2fc5a2..84c19c432b0 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -1,34 +1,39 @@ module API - # Runners API class Runners < Grape::API before { authenticate! } resource :runners do - # Get runners available for user - # - # Example Request: - # GET /runners + desc 'Get runners available for user' do + success Entities::Runner + end + params do + optional :scope, type: String, values: %w[active paused online], + desc: 'The scope of specific runners to show' + end get do runners = filter_runners(current_user.ci_authorized_runners, params[:scope], without: ['specific', 'shared']) present paginate(runners), with: Entities::Runner end - # Get all runners - shared and specific - # - # Example Request: - # GET /runners/all + desc 'Get all runners - shared and specific' do + success Entities::Runner + end + params do + optional :scope, type: String, values: %w[active paused online specific shared], + desc: 'The scope of specific runners to show' + end get 'all' do authenticated_as_admin! runners = filter_runners(Ci::Runner.all, params[:scope]) present paginate(runners), with: Entities::Runner end - # Get runner's details - # - # Parameters: - # id (required) - The ID of ther runner - # Example Request: - # GET /runners/:id + desc "Get runner's details" do + success Entities::RunnerDetails + end + params do + requires :id, type: Integer, desc: 'The ID of the runner' + end get ':id' do runner = get_runner(params[:id]) authenticate_show_runner!(runner) @@ -36,33 +41,37 @@ module API present runner, with: Entities::RunnerDetails, current_user: current_user end - # Update runner's details - # - # Parameters: - # id (required) - The ID of ther runner - # description (optional) - Runner's description - # active (optional) - Runner's status - # tag_list (optional) - Array of tags for runner - # Example Request: - # PUT /runners/:id + desc "Update runner's details" do + success Entities::RunnerDetails + end + params do + requires :id, type: Integer, desc: 'The ID of the runner' + optional :description, type: String, desc: 'The description of the runner' + optional :active, type: Boolean, desc: 'The state of a runner' + optional :tag_list, type: Array[String], desc: 'The list of tags for a runner' + optional :run_untagged, type: Boolean, desc: 'Flag indicating the runner can execute untagged jobs' + optional :locked, type: Boolean, desc: 'Flag indicating the runner is locked' + at_least_one_of :description, :active, :tag_list, :run_untagged, :locked + end put ':id' do - runner = get_runner(params[:id]) + runner = get_runner(params.delete(:id)) authenticate_update_runner!(runner) - attrs = attributes_for_keys [:description, :active, :tag_list, :run_untagged, :locked] - if runner.update(attrs) + runner_params = declared(params, include_missing: false) + + if runner.update(runner_params) present runner, with: Entities::RunnerDetails, current_user: current_user else render_validation_error!(runner) end end - # Remove runner - # - # Parameters: - # id (required) - The ID of ther runner - # Example Request: - # DELETE /runners/:id + desc 'Remove a runner' do + success Entities::Runner + end + params do + requires :id, type: Integer, desc: 'The ID of the runner' + end delete ':id' do runner = get_runner(params[:id]) authenticate_delete_runner!(runner) @@ -72,28 +81,31 @@ module API end end + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do before { authorize_admin_project } - # Get runners available for project - # - # Example Request: - # GET /projects/:id/runners + desc 'Get runners available for project' do + success Entities::Runner + end + params do + optional :scope, type: String, values: %w[active paused online specific shared], + desc: 'The scope of specific runners to show' + end get ':id/runners' do runners = filter_runners(Ci::Runner.owned_or_shared(user_project.id), params[:scope]) present paginate(runners), with: Entities::Runner end - # Enable runner for project - # - # Parameters: - # id (required) - The ID of the project - # runner_id (required) - The ID of the runner - # Example Request: - # POST /projects/:id/runners/:runner_id + desc 'Enable a runner for a project' do + success Entities::Runner + end + params do + requires :runner_id, type: Integer, desc: 'The ID of the runner' + end post ':id/runners' do - required_attributes! [:runner_id] - runner = get_runner(params[:runner_id]) authenticate_enable_runner!(runner) @@ -106,13 +118,12 @@ module API end end - # Disable project's runner - # - # Parameters: - # id (required) - The ID of the project - # runner_id (required) - The ID of the runner - # Example Request: - # DELETE /projects/:id/runners/:runner_id + desc "Disable project's runner" do + success Entities::Runner + end + params do + requires :runner_id, type: Integer, desc: 'The ID of the runner' + end delete ':id/runners/:runner_id' do runner_project = user_project.runner_projects.find_by(runner_id: params[:runner_id]) not_found!('Runner') unless runner_project diff --git a/lib/api/session.rb b/lib/api/session.rb index 56c202f1294..d09400b81f5 100644 --- a/lib/api/session.rb +++ b/lib/api/session.rb @@ -1,19 +1,19 @@ module API - # Users API class Session < Grape::API - # Login to get token - # - # Parameters: - # login (*required) - user login - # email (*required) - user email - # password (required) - user password - # - # Example Request: - # POST /session + desc 'Login to get token' do + success Entities::UserLogin + end + params do + optional :login, type: String, desc: 'The username' + optional :email, type: String, desc: 'The email of the user' + requires :password, type: String, desc: 'The password of the user' + at_least_one_of :login, :email + end post "/session" do user = Gitlab::Auth.find_with_user_password(params[:email] || params[:login], params[:password]) return unauthorized! unless user + return render_api_error!('401 Unauthorized. You have 2FA enabled. Please use a personal access token to access the API', 401) if user.two_factor_enabled? present user, with: Entities::UserLogin end end diff --git a/lib/api/settings.rb b/lib/api/settings.rb index c885fcd7ea3..c4cb1c7924a 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -17,12 +17,12 @@ module API present current_settings, with: Entities::ApplicationSetting end - # Modify applicaiton settings + # Modify application settings # # Example Request: # PUT /application/settings put "application/settings" do - attributes = current_settings.attributes.keys - ["id"] + attributes = ["repository_storage"] + current_settings.attributes.keys - ["id"] attrs = attributes_for_keys(attributes) if current_settings.update_attributes(attrs) diff --git a/lib/api/system_hooks.rb b/lib/api/system_hooks.rb index 22b8f90dc5c..b6bfff9f20f 100644 --- a/lib/api/system_hooks.rb +++ b/lib/api/system_hooks.rb @@ -7,38 +7,41 @@ module API end resource :hooks do - # Get the list of system hooks - # - # Example Request: - # GET /hooks + desc 'Get the list of system hooks' do + success Entities::Hook + end get do - @hooks = SystemHook.all - present @hooks, with: Entities::Hook + hooks = SystemHook.all + + present hooks, with: Entities::Hook end - # Create new system hook - # - # Parameters: - # url (required) - url for system hook - # Example Request - # POST /hooks + desc 'Create a new system hook' do + success Entities::Hook + end + params do + requires :url, type: String, desc: "The URL to send the request to" + optional :token, type: String, desc: 'The token used to validate payloads' + optional :push_events, type: Boolean, desc: "Trigger hook on push events" + optional :tag_push_events, type: Boolean, desc: "Trigger hook on tag push events" + optional :enable_ssl_verification, type: Boolean, desc: "Do SSL verification when triggering the hook" + end post do - attrs = attributes_for_keys [:url] - required_attributes! [:url] - @hook = SystemHook.new attrs - if @hook.save - present @hook, with: Entities::Hook + hook = SystemHook.new declared(params, include_missing: false).to_h + + if hook.save + present hook, with: Entities::Hook else - not_found! + render_validation_error!(hook) end end - # Test a hook - # - # Example Request - # GET /hooks/:id + desc 'Test a hook' + params do + requires :id, type: Integer, desc: 'The ID of the system hook' + end get ":id" do - @hook = SystemHook.find(params[:id]) + hook = SystemHook.find(params[:id]) data = { event_name: "project_create", name: "Ruby", @@ -47,23 +50,21 @@ module API owner_name: "Someone", owner_email: "example@gitlabhq.com" } - @hook.execute(data, 'system_hooks') + hook.execute(data, 'system_hooks') data end - # Delete a hook. This is an idempotent function. - # - # Parameters: - # id (required) - ID of the hook - # Example Request: - # DELETE /hooks/:id + desc 'Delete a hook' do + success Entities::Hook + end + params do + requires :id, type: Integer, desc: 'The ID of the system hook' + end delete ":id" do - begin - @hook = SystemHook.find(params[:id]) - @hook.destroy - rescue - # SystemHook raises an Error if no hook with id found - end + hook = SystemHook.find_by(id: params[:id]) + not_found!('System hook') unless hook + + present hook.destroy, with: Entities::Hook end end end diff --git a/lib/api/tags.rb b/lib/api/tags.rb index 7b675e05fbb..bf2a199ce21 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -4,25 +4,24 @@ module API before { authenticate! } before { authorize! :download_code, user_project } + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do - # Get a project repository tags - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # GET /projects/:id/repository/tags + desc 'Get a project repository tags' do + success Entities::RepoTag + end get ":id/repository/tags" do present user_project.repository.tags.sort_by(&:name).reverse, with: Entities::RepoTag, project: user_project end - # Get a single repository tag - # - # Parameters: - # id (required) - The ID of a project - # tag_name (required) - The name of the tag - # Example Request: - # GET /projects/:id/repository/tags/:tag_name + desc 'Get a single repository tag' do + success Entities::RepoTag + end + params do + requires :tag_name, type: String, desc: 'The name of the tag' + end get ":id/repository/tags/:tag_name", requirements: { tag_name: /.+/ } do tag = user_project.repository.find_tag(params[:tag_name]) not_found!('Tag') unless tag @@ -30,20 +29,21 @@ module API present tag, with: Entities::RepoTag, project: user_project end - # Create tag - # - # Parameters: - # id (required) - The ID of a project - # tag_name (required) - The name of the tag - # ref (required) - Create tag from commit sha or branch - # message (optional) - Specifying a message creates an annotated tag. - # Example Request: - # POST /projects/:id/repository/tags + desc 'Create a new repository tag' do + success Entities::RepoTag + end + params do + requires :tag_name, type: String, desc: 'The name of the tag' + requires :ref, type: String, desc: 'The commit sha or branch name' + optional :message, type: String, desc: 'Specifying a message creates an annotated tag' + optional :release_description, type: String, desc: 'Specifying release notes stored in the GitLab database' + end post ':id/repository/tags' do authorize_push_project - message = params[:message] || nil + create_params = declared(params) + result = CreateTagService.new(user_project, current_user). - execute(params[:tag_name], params[:ref], message, params[:release_description]) + execute(create_params[:tag_name], create_params[:ref], create_params[:message], create_params[:release_description]) if result[:status] == :success present result[:tag], @@ -54,15 +54,13 @@ module API end end - # Delete tag - # - # Parameters: - # id (required) - The ID of a project - # tag_name (required) - The name of the tag - # Example Request: - # DELETE /projects/:id/repository/tags/:tag + desc 'Delete a repository tag' + params do + requires :tag_name, type: String, desc: 'The name of the tag' + end delete ":id/repository/tags/:tag_name", requirements: { tag_name: /.+/ } do authorize_push_project + result = DeleteTagService.new(user_project, current_user). execute(params[:tag_name]) @@ -75,17 +73,16 @@ module API end end - # Add release notes to tag - # - # Parameters: - # id (required) - The ID of a project - # tag_name (required) - The name of the tag - # description (required) - Release notes with markdown support - # Example Request: - # POST /projects/:id/repository/tags/:tag_name/release + desc 'Add a release note to a tag' do + success Entities::Release + end + params do + requires :tag_name, type: String, desc: 'The name of the tag' + requires :description, type: String, desc: 'Release notes with markdown support' + end post ':id/repository/tags/:tag_name/release', requirements: { tag_name: /.+/ } do authorize_push_project - required_attributes! [:description] + result = CreateReleaseService.new(user_project, current_user). execute(params[:tag_name], params[:description]) @@ -96,17 +93,16 @@ module API end end - # Updates a release notes of a tag - # - # Parameters: - # id (required) - The ID of a project - # tag_name (required) - The name of the tag - # description (required) - Release notes with markdown support - # Example Request: - # PUT /projects/:id/repository/tags/:tag_name/release + desc "Update a tag's release note" do + success Entities::Release + end + params do + requires :tag_name, type: String, desc: 'The name of the tag' + requires :description, type: String, desc: 'Release notes with markdown support' + end put ':id/repository/tags/:tag_name/release', requirements: { tag_name: /.+/ } do authorize_push_project - required_attributes! [:description] + result = UpdateReleaseService.new(user_project, current_user). execute(params[:tag_name], params[:description]) diff --git a/lib/api/templates.rb b/lib/api/templates.rb index b9e718147e1..8a53d9c0095 100644 --- a/lib/api/templates.rb +++ b/lib/api/templates.rb @@ -1,39 +1,115 @@ module API class Templates < Grape::API GLOBAL_TEMPLATE_TYPES = { - gitignores: Gitlab::Template::GitignoreTemplate, - gitlab_ci_ymls: Gitlab::Template::GitlabCiYmlTemplate + gitignores: { + klass: Gitlab::Template::GitignoreTemplate, + gitlab_version: 8.8 + }, + gitlab_ci_ymls: { + klass: Gitlab::Template::GitlabCiYmlTemplate, + gitlab_version: 8.9 + } }.freeze + PROJECT_TEMPLATE_REGEX = + /[\<\{\[] + (project|description| + one\sline\s.+\swhat\sit\sdoes\.) # matching the start and end is enough here + [\>\}\]]/xi.freeze + YEAR_TEMPLATE_REGEX = /[<{\[](year|yyyy)[>}\]]/i.freeze + FULLNAME_TEMPLATE_REGEX = + /[\<\{\[] + (fullname|name\sof\s(author|copyright\sowner)) + [\>\}\]]/xi.freeze + DEPRECATION_MESSAGE = ' This endpoint is deprecated and will be removed in GitLab 9.0.'.freeze helpers do + def parsed_license_template + # We create a fresh Licensee::License object since we'll modify its + # content in place below. + template = Licensee::License.new(params[:name]) + + template.content.gsub!(YEAR_TEMPLATE_REGEX, Time.now.year.to_s) + template.content.gsub!(PROJECT_TEMPLATE_REGEX, params[:project]) if params[:project].present? + + fullname = params[:fullname].presence || current_user.try(:name) + template.content.gsub!(FULLNAME_TEMPLATE_REGEX, fullname) if fullname + template + end + def render_response(template_type, template) not_found!(template_type.to_s.singularize) unless template present template, with: Entities::Template end end - GLOBAL_TEMPLATE_TYPES.each do |template_type, klass| - # Get the list of the available template - # - # Example Request: - # GET /gitignores - # GET /gitlab_ci_ymls - get template_type.to_s do - present klass.all, with: Entities::TemplatesList - end - - # Get the text for a specific template present in local filesystem - # - # Parameters: - # name (required) - The name of a template - # - # Example Request: - # GET /gitignores/Elixir - # GET /gitlab_ci_ymls/Ruby - get "#{template_type}/:name" do - required_attributes! [:name] - new_template = klass.find(params[:name]) - render_response(template_type, new_template) + { "licenses" => :deprecated, "templates/licenses" => :ok }.each do |route, status| + desc 'Get the list of the available license template' do + detailed_desc = 'This feature was introduced in GitLab 8.7.' + detailed_desc << DEPRECATION_MESSAGE unless status == :ok + detail detailed_desc + success Entities::RepoLicense + end + params do + optional :popular, type: Boolean, desc: 'If passed, returns only popular licenses' + end + get route do + options = { + featured: declared(params).popular.present? ? true : nil + } + present Licensee::License.all(options), with: Entities::RepoLicense + end + end + + { "licenses/:name" => :deprecated, "templates/licenses/:name" => :ok }.each do |route, status| + desc 'Get the text for a specific license' do + detailed_desc = 'This feature was introduced in GitLab 8.7.' + detailed_desc << DEPRECATION_MESSAGE unless status == :ok + detail detailed_desc + success Entities::RepoLicense + end + params do + requires :name, type: String, desc: 'The name of the template' + end + get route, requirements: { name: /[\w\.-]+/ } do + not_found!('License') unless Licensee::License.find(declared(params).name) + + template = parsed_license_template + + present template, with: Entities::RepoLicense + end + end + + GLOBAL_TEMPLATE_TYPES.each do |template_type, properties| + klass = properties[:klass] + gitlab_version = properties[:gitlab_version] + + { template_type => :deprecated, "templates/#{template_type}" => :ok }.each do |route, status| + desc 'Get the list of the available template' do + detailed_desc = "This feature was introduced in GitLab #{gitlab_version}." + detailed_desc << DEPRECATION_MESSAGE unless status == :ok + detail detailed_desc + success Entities::TemplatesList + end + get route do + present klass.all, with: Entities::TemplatesList + end + end + + { "#{template_type}/:name" => :deprecated, "templates/#{template_type}/:name" => :ok }.each do |route, status| + desc 'Get the text for a specific template present in local filesystem' do + detailed_desc = "This feature was introduced in GitLab #{gitlab_version}." + detailed_desc << DEPRECATION_MESSAGE unless status == :ok + detail detailed_desc + success Entities::Template + end + params do + requires :name, type: String, desc: 'The name of the template' + end + get route do + new_template = klass.find(declared(params).name) + + render_response(template_type, new_template) + end end end end diff --git a/lib/api/todos.rb b/lib/api/todos.rb index 19df13d8aac..832b04a3bb1 100644 --- a/lib/api/todos.rb +++ b/lib/api/todos.rb @@ -8,18 +8,19 @@ module API 'issues' => ->(id) { find_project_issue(id) } } + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do ISSUABLE_TYPES.each do |type, finder| type_id_str = "#{type.singularize}_id".to_sym - # Create a todo on an issuable - # - # Parameters: - # id (required) - The ID of a project - # issuable_id (required) - The ID of an issuable - # Example Request: - # POST /projects/:id/issues/:issuable_id/todo - # POST /projects/:id/merge_requests/:issuable_id/todo + desc 'Create a todo on an issuable' do + success Entities::Todo + end + params do + requires type_id_str, type: Integer, desc: 'The ID of an issuable' + end post ":id/#{type}/:#{type_id_str}/todo" do issuable = instance_exec(params[type_id_str], &finder) todo = TodoService.new.mark_todo(issuable, current_user).first @@ -40,25 +41,21 @@ module API end end - # Get a todo list - # - # Example Request: - # GET /todos - # + desc 'Get a todo list' do + success Entities::Todo + end get do todos = find_todos present paginate(todos), with: Entities::Todo, current_user: current_user end - # Mark a todo as done - # - # Parameters: - # id: (required) - The ID of the todo being marked as done - # - # Example Request: - # DELETE /todos/:id - # + desc 'Mark a todo as done' do + success Entities::Todo + end + params do + requires :id, type: Integer, desc: 'The ID of the todo being marked as done' + end delete ':id' do todo = current_user.todos.find(params[:id]) TodoService.new.mark_todos_as_done([todo], current_user) @@ -66,11 +63,7 @@ module API present todo.reload, with: Entities::Todo, current_user: current_user end - # Mark all todos as done - # - # Example Request: - # DELETE /todos - # + desc 'Mark all todos as done' delete do todos = find_todos TodoService.new.mark_todos_as_done(todos, current_user) diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb index d1d07394e92..9a4f1cd342f 100644 --- a/lib/api/triggers.rb +++ b/lib/api/triggers.rb @@ -1,19 +1,18 @@ module API - # Triggers API class Triggers < Grape::API + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do - # Trigger a GitLab project build - # - # Parameters: - # id (required) - The ID of a CI project - # ref (required) - The name of project's branch or tag - # token (required) - The uniq token of trigger - # variables (optional) - The list of variables to be injected into build - # Example Request: - # POST /projects/:id/trigger/builds + desc 'Trigger a GitLab project build' do + success Entities::TriggerRequest + end + params do + requires :ref, type: String, desc: 'The commit sha or name of a branch or tag' + requires :token, type: String, desc: 'The unique token of trigger' + optional :variables, type: Hash, desc: 'The list of variables to be injected into build' + end post ":id/trigger/builds" do - required_attributes! [:ref, :token] - project = Project.find_with_namespace(params[:id]) || Project.find_by(id: params[:id]) trigger = Ci::Trigger.find_by_token(params[:token].to_s) not_found! unless project && trigger @@ -22,10 +21,6 @@ module API # validate variables variables = params[:variables] if variables - unless variables.is_a?(Hash) - render_api_error!('variables needs to be a hash', 400) - end - unless variables.all? { |key, value| key.is_a?(String) && value.is_a?(String) } render_api_error!('variables needs to be a map of key-valued strings', 400) end @@ -44,31 +39,24 @@ module API end end - # Get triggers list - # - # Parameters: - # id (required) - The ID of a project - # page (optional) - The page number for pagination - # per_page (optional) - The value of items per page to show - # Example Request: - # GET /projects/:id/triggers + desc 'Get triggers list' do + success Entities::Trigger + end get ':id/triggers' do authenticate! authorize! :admin_build, user_project triggers = user_project.triggers.includes(:trigger_requests) - triggers = paginate(triggers) - present triggers, with: Entities::Trigger + present paginate(triggers), with: Entities::Trigger end - # Get specific trigger of a project - # - # Parameters: - # id (required) - The ID of a project - # token (required) - The `token` of a trigger - # Example Request: - # GET /projects/:id/triggers/:token + desc 'Get specific trigger of a project' do + success Entities::Trigger + end + params do + requires :token, type: String, desc: 'The unique token of trigger' + end get ':id/triggers/:token' do authenticate! authorize! :admin_build, user_project @@ -79,12 +67,9 @@ module API present trigger, with: Entities::Trigger end - # Create trigger - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # POST /projects/:id/triggers + desc 'Create a trigger' do + success Entities::Trigger + end post ':id/triggers' do authenticate! authorize! :admin_build, user_project @@ -94,13 +79,12 @@ module API present trigger, with: Entities::Trigger end - # Delete trigger - # - # Parameters: - # id (required) - The ID of a project - # token (required) - The `token` of a trigger - # Example Request: - # DELETE /projects/:id/triggers/:token + desc 'Delete a trigger' do + success Entities::Trigger + end + params do + requires :token, type: String, desc: 'The unique token of trigger' + end delete ':id/triggers/:token' do authenticate! authorize! :admin_build, user_project diff --git a/lib/api/users.rb b/lib/api/users.rb index 8a376d3c2a3..298c401a816 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -10,6 +10,9 @@ module API # GET /users # GET /users?search=Admin # GET /users?username=root + # GET /users?active=true + # GET /users?external=true + # GET /users?blocked=true get do unless can?(current_user, :read_users_list, nil) render_api_error!("Not authorized.", 403) @@ -19,8 +22,10 @@ module API @users = User.where(username: params[:username]) else @users = User.all - @users = @users.active if params[:active].present? + @users = @users.active if to_boolean(params[:active]) @users = @users.search(params[:search]) if params[:search].present? + @users = @users.blocked if to_boolean(params[:blocked]) + @users = @users.external if to_boolean(params[:external]) && current_user.is_admin? @users = paginate @users end @@ -60,6 +65,7 @@ module API # linkedin - Linkedin # twitter - Twitter account # website_url - Website url + # organization - Organization # projects_limit - Number of projects user can create # extern_uid - External authentication provider UID # provider - External provider @@ -74,7 +80,7 @@ module API post do authenticated_as_admin! required_attributes! [:email, :password, :name, :username] - attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :projects_limit, :username, :bio, :location, :can_create_group, :admin, :confirm, :external] + attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :projects_limit, :username, :bio, :location, :can_create_group, :admin, :confirm, :external, :organization] admin = attrs.delete(:admin) confirm = !(attrs.delete(:confirm) =~ /(false|f|no|0)$/i) user = User.build_user(attrs) @@ -111,6 +117,7 @@ module API # linkedin - Linkedin # twitter - Twitter account # website_url - Website url + # organization - Organization # projects_limit - Limit projects each user can create # bio - Bio # location - Location of the user @@ -122,7 +129,7 @@ module API put ":id" do authenticated_as_admin! - attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :website_url, :projects_limit, :username, :bio, :location, :can_create_group, :admin, :external] + attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :website_url, :projects_limit, :username, :bio, :location, :can_create_group, :admin, :external, :organization] user = User.find(params[:id]) not_found!('User') unless user @@ -319,6 +326,26 @@ module API user.activate end end + + desc 'Get contribution events of a specified user' do + detail 'This feature was introduced in GitLab 8.13.' + success Entities::Event + end + params do + requires :id, type: String, desc: 'The user ID' + end + get ':id/events' do + user = User.find_by(id: declared(params).id) + not_found!('User') unless user + + events = user.events. + merge(ProjectsFinder.new.execute(current_user)). + references(:project). + with_associations. + recent + + present paginate(events), with: Entities::Event + end end resource :user do @@ -327,7 +354,7 @@ module API # Example Request: # GET /user get do - present @current_user, with: Entities::UserLogin + present @current_user, with: Entities::UserFull end # Get currently authenticated user's keys diff --git a/lib/api/variables.rb b/lib/api/variables.rb index f6495071a11..b9fb3c21dbb 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -4,27 +4,29 @@ module API before { authenticate! } before { authorize! :admin_build, user_project } + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects do - # Get project variables - # - # Parameters: - # id (required) - The ID of a project - # page (optional) - The page number for pagination - # per_page (optional) - The value of items per page to show - # Example Request: - # GET /projects/:id/variables + desc 'Get project variables' do + success Entities::Variable + end + params do + optional :page, type: Integer, desc: 'The page number for pagination' + optional :per_page, type: Integer, desc: 'The value of items per page to show' + end get ':id/variables' do variables = user_project.variables present paginate(variables), with: Entities::Variable end - # Get specific variable of a project - # - # Parameters: - # id (required) - The ID of a project - # key (required) - The `key` of variable - # Example Request: - # GET /projects/:id/variables/:key + desc 'Get a specific variable from a project' do + success Entities::Variable + end + params do + requires :key, type: String, desc: 'The key of the variable' + end get ':id/variables/:key' do key = params[:key] variable = user_project.variables.find_by(key: key.to_s) @@ -34,18 +36,15 @@ module API present variable, with: Entities::Variable end - # Create a new variable in project - # - # Parameters: - # id (required) - The ID of a project - # key (required) - The key of variable - # value (required) - The value of variable - # Example Request: - # POST /projects/:id/variables + desc 'Create a new variable in a project' do + success Entities::Variable + end + params do + requires :key, type: String, desc: 'The key of the variable' + requires :value, type: String, desc: 'The value of the variable' + end post ':id/variables' do - required_attributes! [:key, :value] - - variable = user_project.variables.create(key: params[:key], value: params[:value]) + variable = user_project.variables.create(declared(params, include_parent_namespaces: false).to_h) if variable.valid? present variable, with: Entities::Variable @@ -54,41 +53,37 @@ module API end end - # Update existing variable of a project - # - # Parameters: - # id (required) - The ID of a project - # key (optional) - The `key` of variable - # value (optional) - New value for `value` field of variable - # Example Request: - # PUT /projects/:id/variables/:key + desc 'Update an existing variable from a project' do + success Entities::Variable + end + params do + optional :key, type: String, desc: 'The key of the variable' + optional :value, type: String, desc: 'The value of the variable' + end put ':id/variables/:key' do - variable = user_project.variables.find_by(key: params[:key].to_s) + variable = user_project.variables.find_by(key: params[:key]) return not_found!('Variable') unless variable - attrs = attributes_for_keys [:value] - if variable.update(attrs) + if variable.update(value: params[:value]) present variable, with: Entities::Variable else render_validation_error!(variable) end end - # Delete existing variable of a project - # - # Parameters: - # id (required) - The ID of a project - # key (required) - The ID of a variable - # Example Request: - # DELETE /projects/:id/variables/:key + desc 'Delete an existing variable from a project' do + success Entities::Variable + end + params do + requires :key, type: String, desc: 'The key of the variable' + end delete ':id/variables/:key' do - variable = user_project.variables.find_by(key: params[:key].to_s) + variable = user_project.variables.find_by(key: params[:key]) return not_found!('Variable') unless variable - variable.destroy - present variable, with: Entities::Variable + present variable.destroy, with: Entities::Variable end end end diff --git a/lib/api/version.rb b/lib/api/version.rb new file mode 100644 index 00000000000..9ba576bd828 --- /dev/null +++ b/lib/api/version.rb @@ -0,0 +1,12 @@ +module API + class Version < Grape::API + before { authenticate! } + + desc 'Get the version information of the GitLab instance.' do + detail 'This feature was introduced in GitLab 8.13.' + end + get '/version' do + { version: Gitlab::VERSION, revision: Gitlab::REVISION } + end + end +end diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index f117fc3d37d..d746070913d 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -2,11 +2,14 @@ require 'yaml' module Backup class Repository + def dump prepare Project.find_each(batch_size: 1000) do |project| $progress.print " * #{project.path_with_namespace} ... " + path_to_project_repo = path_to_repo(project) + path_to_project_bundle = path_to_bundle(project) # Create namespace dir if missing FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.path)) if project.namespace @@ -14,8 +17,22 @@ module Backup if project.empty_repo? $progress.puts "[SKIPPED]".color(:cyan) else - cmd = %W(tar -cf #{path_to_bundle(project)} -C #{path_to_repo(project)} .) + in_path(path_to_project_repo) do |dir| + FileUtils.mkdir_p(path_to_tars(project)) + cmd = %W(tar -cf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir}) + output, status = Gitlab::Popen.popen(cmd) + + unless status.zero? + puts "[FAILED]".color(:red) + puts "failed: #{cmd.join(' ')}" + puts output + abort 'Backup failed' + end + end + + cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_project_repo} bundle create #{path_to_project_bundle} --all) output, status = Gitlab::Popen.popen(cmd) + if status.zero? $progress.puts "[DONE]".color(:green) else @@ -27,19 +44,22 @@ module Backup end wiki = ProjectWiki.new(project) + path_to_wiki_repo = path_to_repo(wiki) + path_to_wiki_bundle = path_to_bundle(wiki) - if File.exist?(path_to_repo(wiki)) + if File.exist?(path_to_wiki_repo) $progress.print " * #{wiki.path_with_namespace} ... " if wiki.repository.empty? $progress.puts " [SKIPPED]".color(:cyan) else - cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_repo(wiki)} bundle create #{path_to_bundle(wiki)} --all) + cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_wiki_repo} bundle create #{path_to_wiki_bundle} --all) output, status = Gitlab::Popen.popen(cmd) if status.zero? $progress.puts " [DONE]".color(:green) else puts " [FAILED]".color(:red) puts "failed: #{cmd.join(' ')}" + puts output abort 'Backup failed' end end @@ -55,45 +75,64 @@ module Backup bk_repos_path = File.join(path, '..', 'repositories.old.' + Time.now.to_i.to_s) FileUtils.mv(path, bk_repos_path) # This is expected from gitlab:check - FileUtils.mkdir_p(path, mode: 2770) + FileUtils.mkdir_p(path, mode: 02770) end Project.find_each(batch_size: 1000) do |project| $progress.print " * #{project.path_with_namespace} ... " + path_to_project_repo = path_to_repo(project) + path_to_project_bundle = path_to_bundle(project) project.ensure_dir_exist - if File.exist?(path_to_bundle(project)) - FileUtils.mkdir_p(path_to_repo(project)) - cmd = %W(tar -xf #{path_to_bundle(project)} -C #{path_to_repo(project)}) + if File.exists?(path_to_project_bundle) + cmd = %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_project_bundle} #{path_to_project_repo}) else - cmd = %W(#{Gitlab.config.git.bin_path} init --bare #{path_to_repo(project)}) + cmd = %W(#{Gitlab.config.git.bin_path} init --bare #{path_to_project_repo}) end - if system(*cmd, silent) + output, status = Gitlab::Popen.popen(cmd) + if status.zero? $progress.puts "[DONE]".color(:green) else puts "[FAILED]".color(:red) puts "failed: #{cmd.join(' ')}" + puts output abort 'Restore failed' end + in_path(path_to_tars(project)) do |dir| + cmd = %W(tar -xf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir}) + + output, status = Gitlab::Popen.popen(cmd) + unless status.zero? + puts "[FAILED]".color(:red) + puts "failed: #{cmd.join(' ')}" + puts output + abort 'Restore failed' + end + end + wiki = ProjectWiki.new(project) + path_to_wiki_repo = path_to_repo(wiki) + path_to_wiki_bundle = path_to_bundle(wiki) - if File.exist?(path_to_bundle(wiki)) + if File.exist?(path_to_wiki_bundle) $progress.print " * #{wiki.path_with_namespace} ... " # If a wiki bundle exists, first remove the empty repo # that was initialized with ProjectWiki.new() and then # try to restore with 'git clone --bare'. - FileUtils.rm_rf(path_to_repo(wiki)) - cmd = %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_bundle(wiki)} #{path_to_repo(wiki)}) + FileUtils.rm_rf(path_to_wiki_repo) + cmd = %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_wiki_bundle} #{path_to_wiki_repo}) - if system(*cmd, silent) + output, status = Gitlab::Popen.popen(cmd) + if status.zero? $progress.puts " [DONE]".color(:green) else puts " [FAILED]".color(:red) puts "failed: #{cmd.join(' ')}" + puts output abort 'Restore failed' end end @@ -101,13 +140,15 @@ module Backup $progress.print 'Put GitLab hooks in repositories dirs'.color(:yellow) cmd = %W(#{Gitlab.config.gitlab_shell.path}/bin/create-hooks) + repository_storage_paths_args - if system(*cmd) + + output, status = Gitlab::Popen.popen(cmd) + if status.zero? $progress.puts " [DONE]".color(:green) else puts " [FAILED]".color(:red) puts "failed: #{cmd}" + puts output end - end protected @@ -117,11 +158,30 @@ module Backup end def path_to_bundle(project) - File.join(backup_repos_path, project.path_with_namespace + ".bundle") + File.join(backup_repos_path, project.path_with_namespace + '.bundle') + end + + def path_to_tars(project, dir = nil) + path = File.join(backup_repos_path, project.path_with_namespace) + + if dir + File.join(path, "#{dir}.tar") + else + path + end end def backup_repos_path - File.join(Gitlab.config.backup.path, "repositories") + File.join(Gitlab.config.backup.path, 'repositories') + end + + def in_path(path) + return unless Dir.exist?(path) + + dir_entries = Dir.entries(path) + %w[annex custom_hooks].each do |entry| + yield(entry) if dir_entries.include?(entry) + end end def prepare diff --git a/lib/banzai.rb b/lib/banzai.rb index 9ebe379f454..35ca234c1ba 100644 --- a/lib/banzai.rb +++ b/lib/banzai.rb @@ -3,6 +3,10 @@ module Banzai Renderer.render(text, context) end + def self.render_field(object, field) + Renderer.render_field(object, field) + end + def self.cache_collection_render(texts_and_contexts) Renderer.cache_collection_render(texts_and_contexts) end diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index d77a5e3ff09..3740d4fb4cd 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -18,10 +18,6 @@ module Banzai @object_sym ||= object_name.to_sym end - def self.object_class_title - @object_title ||= object_class.name.titleize - end - # Public: Find references in text (like `!123` for merge requests) # # AnyReferenceFilter.references_in(text) do |match, id, project_ref, matches| @@ -49,10 +45,6 @@ module Banzai self.class.object_sym end - def object_class_title - self.class.object_class_title - end - def references_in(*args, &block) self.class.references_in(*args, &block) end @@ -72,7 +64,7 @@ module Banzai end end - def project_from_ref_cache(ref) + def project_from_ref_cached(ref) if RequestStore.active? cache = project_refs_cache @@ -110,10 +102,10 @@ module Banzai end elsif element_node?(node) - yield_valid_link(node) do |link, text| + yield_valid_link(node) do |link, inner_html| if ref_pattern && link =~ /\A#{ref_pattern}\z/ replace_link_node_with_href(node, link) do - object_link_filter(link, ref_pattern, link_text: text) + object_link_filter(link, ref_pattern, link_content: inner_html) end next @@ -121,9 +113,9 @@ module Banzai next unless link_pattern - if link == text && text =~ /\A#{link_pattern}/ + if link == inner_html && inner_html =~ /\A#{link_pattern}/ replace_link_node_with_text(node, link) do - object_link_filter(text, link_pattern) + object_link_filter(inner_html, link_pattern) end next @@ -131,7 +123,7 @@ module Banzai if link =~ /\A#{link_pattern}\z/ replace_link_node_with_href(node, link) do - object_link_filter(link, link_pattern, link_text: text) + object_link_filter(link, link_pattern, link_content: inner_html) end next @@ -148,19 +140,19 @@ module Banzai # # text - String text to replace references in. # pattern - Reference pattern to match against. - # link_text - Original content of the link being replaced. + # link_content - Original content of the link being replaced. # # Returns a String with references replaced with links. All links # have `gfm` and `gfm-OBJECT_NAME` class names attached for styling. - def object_link_filter(text, pattern, link_text: nil) + def object_link_filter(text, pattern, link_content: nil) references_in(text, pattern) do |match, id, project_ref, matches| - project = project_from_ref_cache(project_ref) + project = project_from_ref_cached(project_ref) if project && object = find_object_cached(project, id) title = object_link_title(object) klass = reference_class(object_sym) - data = data_attributes_for(link_text || match, project, object) + data = data_attributes_for(link_content || match, project, object) if matches.names.include?("url") && matches[:url] url = matches[:url] @@ -168,11 +160,11 @@ module Banzai url = url_for_object_cached(object, project) end - text = link_text || object_link_text(object, matches) + content = link_content || object_link_text(object, matches) %(<a href="#{url}" #{data} title="#{escape_once(title)}" - class="#{klass}">#{escape_once(text)}</a>) + class="#{klass}">#{content}</a>) else match end @@ -198,7 +190,7 @@ module Banzai end def object_link_title(object) - "#{object_class_title}: #{object.title}" + object.title end def object_link_text(object, matches) @@ -216,8 +208,12 @@ module Banzai @references_per_project ||= begin refs = Hash.new { |hash, key| hash[key] = Set.new } - regex = Regexp.union(object_class.reference_pattern, - object_class.link_reference_pattern) + regex = + if uses_reference_pattern? + Regexp.union(object_class.reference_pattern, object_class.link_reference_pattern) + else + object_class.link_reference_pattern + end nodes.each do |node| node.to_html.scan(regex) do @@ -251,11 +247,27 @@ module Banzai end end - # Returns the projects for the given paths. - def find_projects_for_paths(paths) + def projects_relation_for_paths(paths) Project.where_paths_in(paths).includes(:namespace) end + # Returns projects for the given paths. + def find_projects_for_paths(paths) + if RequestStore.active? + to_query = paths - project_refs_cache.keys + + unless to_query.empty? + projects_relation_for_paths(to_query).each do |project| + get_or_set_cache(project_refs_cache, project.path_with_namespace) { project } + end + end + + project_refs_cache.slice(*paths).values + else + projects_relation_for_paths(paths) + end + end + def current_project_path @current_project_path ||= project.path_with_namespace end @@ -287,6 +299,14 @@ module Banzai value end end + + # There might be special cases like filters + # that should ignore reference pattern + # eg: IssueReferenceFilter when using a external issues tracker + # In those cases this method should be overridden on the filter subclass + def uses_reference_pattern? + true + end end end end diff --git a/lib/banzai/filter/autolink_filter.rb b/lib/banzai/filter/autolink_filter.rb index 799b83b1069..80c844baecd 100644 --- a/lib/banzai/filter/autolink_filter.rb +++ b/lib/banzai/filter/autolink_filter.rb @@ -71,6 +71,14 @@ module Banzai @doc = parse_html(rinku) end + # Return true if any of the UNSAFE_PROTOCOLS strings are included in the URI scheme + def contains_unsafe?(scheme) + return false unless scheme + + scheme = scheme.strip.downcase + Banzai::Filter::SanitizationFilter::UNSAFE_PROTOCOLS.any? { |protocol| scheme.include?(protocol) } + end + # Autolinks any text matching LINK_PATTERN that Rinku didn't already # replace def text_parse @@ -89,17 +97,27 @@ module Banzai doc end - def autolink_filter(text) - text.gsub(LINK_PATTERN) do |match| - # Remove any trailing HTML entities and store them for appending - # outside the link element. The entity must be marked HTML safe in - # order to be output literally rather than escaped. - match.gsub!(/((?:&[\w#]+;)+)\z/, '') - dropped = ($1 || '').html_safe - - options = link_options.merge(href: match) - content_tag(:a, match, options) + dropped + def autolink_match(match) + # start by stripping out dangerous links + begin + uri = Addressable::URI.parse(match) + return match if contains_unsafe?(uri.scheme) + rescue Addressable::URI::InvalidURIError + return match end + + # Remove any trailing HTML entities and store them for appending + # outside the link element. The entity must be marked HTML safe in + # order to be output literally rather than escaped. + match.gsub!(/((?:&[\w#]+;)+)\z/, '') + dropped = ($1 || '').html_safe + + options = link_options.merge(href: match) + content_tag(:a, match, options) + dropped + end + + def autolink_filter(text) + text.gsub(LINK_PATTERN) { |match| autolink_match(match) } end def link_options diff --git a/lib/banzai/filter/commit_range_reference_filter.rb b/lib/banzai/filter/commit_range_reference_filter.rb index bbb88c979cc..4358bf45549 100644 --- a/lib/banzai/filter/commit_range_reference_filter.rb +++ b/lib/banzai/filter/commit_range_reference_filter.rb @@ -35,7 +35,7 @@ module Banzai end def object_link_title(range) - range.reference_title + nil end end end diff --git a/lib/banzai/filter/commit_reference_filter.rb b/lib/banzai/filter/commit_reference_filter.rb index 2ce1816672b..a26dd09c25a 100644 --- a/lib/banzai/filter/commit_reference_filter.rb +++ b/lib/banzai/filter/commit_reference_filter.rb @@ -28,10 +28,6 @@ module Banzai only_path: context[:only_path]) end - def object_link_title(commit) - commit.link_title - end - def object_link_text_extras(object, matches) extras = super diff --git a/lib/banzai/filter/emoji_filter.rb b/lib/banzai/filter/emoji_filter.rb index 2492b5213ac..a8c1ca0c60a 100644 --- a/lib/banzai/filter/emoji_filter.rb +++ b/lib/banzai/filter/emoji_filter.rb @@ -1,6 +1,6 @@ module Banzai module Filter - # HTML filter that replaces :emoji: with images. + # HTML filter that replaces :emoji: and unicode with images. # # Based on HTML::Pipeline::EmojiFilter # @@ -13,16 +13,17 @@ module Banzai def call search_text_nodes(doc).each do |node| content = node.to_html - next unless content.include?(':') next if has_ancestor?(node, IGNORED_ANCESTOR_TAGS) - html = emoji_image_filter(content) + next unless content.include?(':') || node.text.match(emoji_unicode_pattern) + + html = emoji_name_image_filter(content) + html = emoji_unicode_image_filter(html) next if html == content node.replace(html) end - doc end @@ -31,18 +32,38 @@ module Banzai # text - String text to replace :emoji: in. # # Returns a String with :emoji: replaced with images. - def emoji_image_filter(text) + def emoji_name_image_filter(text) text.gsub(emoji_pattern) do |match| name = $1 - "<img class='emoji' title=':#{name}:' alt=':#{name}:' src='#{emoji_url(name)}' height='20' width='20' align='absmiddle' />" + emoji_image_tag(name, emoji_url(name)) end end + # Replace unicode emoji with corresponding images if they exist. + # + # text - String text to replace unicode emoji in. + # + # Returns a String with unicode emoji replaced with images. + def emoji_unicode_image_filter(text) + text.gsub(emoji_unicode_pattern) do |moji| + emoji_image_tag(Gitlab::Emoji.emojis_by_moji[moji]['name'], emoji_unicode_url(moji)) + end + end + + def emoji_image_tag(emoji_name, emoji_url) + "<img class='emoji' title=':#{emoji_name}:' alt=':#{emoji_name}:' src='#{emoji_url}' height='20' width='20' align='absmiddle' />" + end + # Build a regexp that matches all valid :emoji: names. def self.emoji_pattern @emoji_pattern ||= /:(#{Gitlab::Emoji.emojis_names.map { |name| Regexp.escape(name) }.join('|')}):/ end + # Build a regexp that matches all valid unicode emojis names. + def self.emoji_unicode_pattern + @emoji_unicode_pattern ||= /(#{Gitlab::Emoji.emojis_unicodes.map { |moji| Regexp.escape(moji) }.join('|')})/ + end + private def emoji_url(name) @@ -60,6 +81,18 @@ module Banzai end end + def emoji_unicode_url(moji) + emoji_unicode_path = emoji_unicode_filename(moji) + + if context[:asset_host] + url_to_image(emoji_unicode_path) + elsif context[:asset_root] + File.join(context[:asset_root], url_to_image(emoji_unicode_path)) + else + url_to_image(emoji_unicode_path) + end + end + def url_to_image(image) ActionController::Base.helpers.url_to_image(image) end @@ -71,6 +104,14 @@ module Banzai def emoji_filename(name) "#{Gitlab::Emoji.emoji_filename(name)}.png" end + + def emoji_unicode_pattern + self.class.emoji_unicode_pattern + end + + def emoji_unicode_filename(name) + "#{Gitlab::Emoji.emoji_unicode_filename(name)}.png" + end end end end diff --git a/lib/banzai/filter/external_issue_reference_filter.rb b/lib/banzai/filter/external_issue_reference_filter.rb index eaa702952cc..dce4de3ceaf 100644 --- a/lib/banzai/filter/external_issue_reference_filter.rb +++ b/lib/banzai/filter/external_issue_reference_filter.rb @@ -8,7 +8,7 @@ module Banzai # Public: Find `JIRA-123` issue references in text # - # ExternalIssueReferenceFilter.references_in(text) do |match, issue| + # ExternalIssueReferenceFilter.references_in(text, pattern) do |match, issue| # "<a href=...>##{issue}</a>" # end # @@ -17,8 +17,8 @@ module Banzai # Yields the String match and the String issue reference. # # Returns a String replaced with the return of the block. - def self.references_in(text) - text.gsub(ExternalIssue.reference_pattern) do |match| + def self.references_in(text, pattern) + text.gsub(pattern) do |match| yield match, $~[:issue] end end @@ -27,7 +27,7 @@ module Banzai # Early return if the project isn't using an external tracker return doc if project.nil? || default_issues_tracker? - ref_pattern = ExternalIssue.reference_pattern + ref_pattern = issue_reference_pattern ref_start_pattern = /\A#{ref_pattern}\z/ each_node do |node| @@ -37,10 +37,10 @@ module Banzai end elsif element_node?(node) - yield_valid_link(node) do |link, text| + yield_valid_link(node) do |link, inner_html| if link =~ ref_start_pattern replace_link_node_with_href(node, link) do - issue_link_filter(link, link_text: text) + issue_link_filter(link, link_content: inner_html) end end end @@ -54,13 +54,14 @@ module Banzai # issue's details page. # # text - String text to replace references in. + # link_content - Original content of the link being replaced. # # Returns a String with `JIRA-123` references replaced with links. All # links have `gfm` and `gfm-issue` class names attached for styling. - def issue_link_filter(text, link_text: nil) + def issue_link_filter(text, link_content: nil) project = context[:project] - self.class.references_in(text) do |match, id| + self.class.references_in(text, issue_reference_pattern) do |match, id| ExternalIssue.new(id, project) url = url_for_issue(id, project, only_path: context[:only_path]) @@ -69,11 +70,11 @@ module Banzai klass = reference_class(:issue) data = data_attribute(project: project.id, external_issue: id) - text = link_text || match + content = link_content || match %(<a href="#{url}" #{data} title="#{escape_once(title)}" - class="#{klass}">#{escape_once(text)}</a>) + class="#{klass}">#{content}</a>) end end @@ -82,18 +83,21 @@ module Banzai end def default_issues_tracker? - if RequestStore.active? - default_issues_tracker_cache[project.id] ||= - project.default_issues_tracker? - else - project.default_issues_tracker? - end + external_issues_cached(:default_issues_tracker?) + end + + def issue_reference_pattern + external_issues_cached(:issue_reference_pattern) end private - def default_issues_tracker_cache - RequestStore[:banzai_default_issues_tracker_cache] ||= {} + def external_issues_cached(attribute) + return project.public_send(attribute) unless RequestStore.active? + + cached_attributes = RequestStore[:banzai_external_issues_tracker_attributes] ||= Hash.new { |h, k| h[k] = {} } + cached_attributes[project.id][attribute] = project.public_send(attribute) if cached_attributes[project.id][attribute].nil? + cached_attributes[project.id][attribute] end end end diff --git a/lib/banzai/filter/external_link_filter.rb b/lib/banzai/filter/external_link_filter.rb index 0a29c547a4d..2f19b59e725 100644 --- a/lib/banzai/filter/external_link_filter.rb +++ b/lib/banzai/filter/external_link_filter.rb @@ -3,10 +3,17 @@ module Banzai # HTML Filter to modify the attributes of external links class ExternalLinkFilter < HTML::Pipeline::Filter def call - # Skip non-HTTP(S) links and internal links - doc.xpath("descendant-or-self::a[starts-with(@href, 'http') and not(starts-with(@href, '#{internal_url}'))]").each do |node| - node.set_attribute('rel', 'nofollow noreferrer') - node.set_attribute('target', '_blank') + links.each do |node| + href = href_to_lowercase_scheme(node["href"].to_s) + + unless node["href"].to_s == href + node.set_attribute('href', href) + end + + if href =~ /\Ahttp(s)?:\/\// && external_url?(href) + node.set_attribute('rel', 'nofollow noreferrer') + node.set_attribute('target', '_blank') + end end doc @@ -14,6 +21,25 @@ module Banzai private + def links + query = 'descendant-or-self::a[@href and not(@href = "")]' + doc.xpath(query) + end + + def href_to_lowercase_scheme(href) + scheme_match = href.match(/\A(\w+):\/\//) + + if scheme_match + scheme_match.to_s.downcase + scheme_match.post_match + else + href + end + end + + def external_url?(url) + !url.start_with?(internal_url) + end + def internal_url @internal_url ||= Gitlab.config.gitlab.url end diff --git a/lib/banzai/filter/html_entity_filter.rb b/lib/banzai/filter/html_entity_filter.rb new file mode 100644 index 00000000000..f3bd587c28b --- /dev/null +++ b/lib/banzai/filter/html_entity_filter.rb @@ -0,0 +1,12 @@ +require 'erb' + +module Banzai + module Filter + # Text filter that escapes these HTML entities: & " < > + class HtmlEntityFilter < HTML::Pipeline::TextFilter + def call + ERB::Util.html_escape_once(text) + end + end + end +end diff --git a/lib/banzai/filter/issue_reference_filter.rb b/lib/banzai/filter/issue_reference_filter.rb index 4042e9a4c25..4d1bc687696 100644 --- a/lib/banzai/filter/issue_reference_filter.rb +++ b/lib/banzai/filter/issue_reference_filter.rb @@ -4,6 +4,10 @@ module Banzai # issues that do not exist are ignored. # # This filter supports cross-project references. + # + # When external issues tracker like Jira is activated we should not + # use issue reference pattern, but we should still be able + # to reference issues from other GitLab projects. class IssueReferenceFilter < AbstractReferenceFilter self.reference_type = :issue @@ -11,6 +15,10 @@ module Banzai Issue end + def uses_reference_pattern? + context[:project].default_issues_tracker? + end + def find_object(project, iid) issues_per_project[project][iid] end @@ -66,7 +74,7 @@ module Banzai end end - def find_projects_for_paths(paths) + def projects_relation_for_paths(paths) super(paths).includes(:gitlab_issue_tracker_service) end end diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb index e258dc8e2bf..9f9a96cdc65 100644 --- a/lib/banzai/filter/label_reference_filter.rb +++ b/lib/banzai/filter/label_reference_filter.rb @@ -9,7 +9,7 @@ module Banzai end def find_object(project, id) - project.labels.find(id) + find_labels(project).find(id) end def self.references_in(text, pattern = Label.reference_pattern) @@ -35,7 +35,11 @@ module Banzai return unless project label_params = label_params(label_id, label_name) - project.labels.find_by(label_params) + find_labels(project).find_by(label_params) + end + + def find_labels(project) + LabelsFinder.new(nil, project_id: project.id).execute(skip_authorization: true) end # Parameters to pass to `Label.find_by` based on the given arguments @@ -60,16 +64,58 @@ module Banzai end def object_link_text(object, matches) - if context[:project] == object.project - LabelsHelper.render_colored_label(object) + if same_group?(object) && namespace_match?(matches) + render_same_project_label(object) + elsif same_project?(object) + render_same_project_label(object) else - LabelsHelper.render_colored_cross_project_label(object) + render_cross_project_label(object, matches) end end + def same_group?(object) + object.is_a?(GroupLabel) && object.group == project.group + end + + def namespace_match?(matches) + matches[:project].blank? || matches[:project] == project.path_with_namespace + end + + def same_project?(object) + object.is_a?(ProjectLabel) && object.project == project + end + + def user + context[:current_user] || context[:author] + end + + def project + context[:project] + end + + def render_same_project_label(object) + LabelsHelper.render_colored_label(object) + end + + def render_cross_project_label(object, matches) + source_project = + if matches[:project] + Project.find_with_namespace(matches[:project]) + else + object.project + end + + LabelsHelper.render_colored_cross_project_label(object, source_project) + end + def unescape_html_entities(text) CGI.unescapeHTML(text.to_s) end + + def object_link_title(object) + # use title of wrapped element instead + nil + end end end end diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb index ca686c87d97..58fff496d00 100644 --- a/lib/banzai/filter/milestone_reference_filter.rb +++ b/lib/banzai/filter/milestone_reference_filter.rb @@ -59,6 +59,10 @@ module Banzai html_safe end end + + def object_link_title(object) + nil + end end end end diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index bf058241cda..84bfeac8041 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -52,7 +52,7 @@ module Banzai end def reference_class(type) - "gfm gfm-#{type}" + "gfm gfm-#{type} has-tooltip" end # Ensure that a :project key exists in context @@ -85,14 +85,14 @@ module Banzai @nodes ||= each_node.to_a end - # Yields the link's URL and text whenever the node is a valid <a> tag. + # Yields the link's URL and inner HTML whenever the node is a valid <a> tag. def yield_valid_link(node) link = CGI.unescape(node.attr('href').to_s) - text = node.text + inner_html = node.inner_html return unless link.force_encoding('UTF-8').valid_encoding? - yield link, text + yield link, inner_html end def replace_text_when_pattern_matches(node, pattern) diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb index 4fa8d05481f..f09d78be0ce 100644 --- a/lib/banzai/filter/relative_link_filter.rb +++ b/lib/banzai/filter/relative_link_filter.rb @@ -52,8 +52,8 @@ module Banzai relative_url_root, context[:project].path_with_namespace, uri_type(file_path), - ref, - file_path + Addressable::URI.escape(ref), + Addressable::URI.escape(file_path) ].compact.join('/').squeeze('/').chomp('/') uri diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index 6e13282d5f4..af1e575fc89 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -7,7 +7,7 @@ module Banzai UNSAFE_PROTOCOLS = %w(data javascript vbscript).freeze def whitelist - whitelist = super.dup + whitelist = super customize_whitelist(whitelist) @@ -25,7 +25,7 @@ module Banzai return if customized?(whitelist[:transformers]) # Allow code highlighting - whitelist[:attributes]['pre'] = %w(class) + whitelist[:attributes]['pre'] = %w(class v-pre) whitelist[:attributes]['span'] = %w(class) # Allow table alignment @@ -42,58 +42,58 @@ module Banzai # Allow any protocol in `a` elements... whitelist[:protocols].delete('a') - whitelist[:transformers] = whitelist[:transformers].dup - # ...but then remove links with unsafe protocols - whitelist[:transformers].push(remove_unsafe_links) + whitelist[:transformers].push(self.class.remove_unsafe_links) # Remove `rel` attribute from `a` elements - whitelist[:transformers].push(remove_rel) + whitelist[:transformers].push(self.class.remove_rel) # Remove `class` attribute from non-highlight spans - whitelist[:transformers].push(clean_spans) + whitelist[:transformers].push(self.class.clean_spans) whitelist end - def remove_unsafe_links - lambda do |env| - node = env[:node] + class << self + def remove_unsafe_links + lambda do |env| + node = env[:node] - return unless node.name == 'a' - return unless node.has_attribute?('href') + return unless node.name == 'a' + return unless node.has_attribute?('href') - begin - uri = Addressable::URI.parse(node['href']) - uri.scheme = uri.scheme.strip.downcase if uri.scheme + begin + uri = Addressable::URI.parse(node['href']) + uri.scheme = uri.scheme.strip.downcase if uri.scheme - node.remove_attribute('href') if UNSAFE_PROTOCOLS.include?(uri.scheme) - rescue Addressable::URI::InvalidURIError - node.remove_attribute('href') + node.remove_attribute('href') if UNSAFE_PROTOCOLS.include?(uri.scheme) + rescue Addressable::URI::InvalidURIError + node.remove_attribute('href') + end end end - end - def remove_rel - lambda do |env| - if env[:node_name] == 'a' - env[:node].remove_attribute('rel') + def remove_rel + lambda do |env| + if env[:node_name] == 'a' + env[:node].remove_attribute('rel') + end end end - end - def clean_spans - lambda do |env| - node = env[:node] + def clean_spans + lambda do |env| + node = env[:node] - return unless node.name == 'span' - return unless node.has_attribute?('class') + return unless node.name == 'span' + return unless node.has_attribute?('class') - unless has_ancestor?(node, 'pre') - node.remove_attribute('class') - end + unless node.ancestors.any? { |n| n.name.casecmp('pre').zero? } + node.remove_attribute('class') + end - { node_whitelist: [node] } + { node_whitelist: [node] } + end end end end diff --git a/lib/banzai/filter/set_direction_filter.rb b/lib/banzai/filter/set_direction_filter.rb new file mode 100644 index 00000000000..c2976aeb7c6 --- /dev/null +++ b/lib/banzai/filter/set_direction_filter.rb @@ -0,0 +1,15 @@ +module Banzai + module Filter + # HTML filter that sets dir="auto" for RTL languages support + class SetDirectionFilter < HTML::Pipeline::Filter + def call + # select these elements just on top level of the document + doc.xpath('p|h1|h2|h3|h4|h5|h6|ol|ul[not(@class="section-nav")]|blockquote|table').each do |el| + el['dir'] = 'auto' + end + + doc + end + end + end +end diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index fcdb496aed2..026b81ac175 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -30,7 +30,7 @@ module Banzai # users can still access an issue/comment/etc. end - highlighted = %(<pre class="#{css_classes}"><code>#{code}</code></pre>) + highlighted = %(<pre class="#{css_classes}" v-pre="true"><code>#{code}</code></pre>) # Extracted to a method to measure it replace_parent_pre_element(node, highlighted) diff --git a/lib/banzai/filter/task_list_filter.rb b/lib/banzai/filter/task_list_filter.rb index 66608c9859c..9fa5f589f3e 100644 --- a/lib/banzai/filter/task_list_filter.rb +++ b/lib/banzai/filter/task_list_filter.rb @@ -2,27 +2,7 @@ require 'task_list/filter' module Banzai module Filter - # Work around a bug in the default TaskList::Filter that adds a `task-list` - # class to every list element, regardless of whether or not it contains a - # task list. - # - # This is a (hopefully) temporary fix, pending a new release of the - # task_list gem. - # - # See https://github.com/github/task_list/pull/60 class TaskListFilter < TaskList::Filter - def add_css_class_with_fix(node, *new_class_names) - if new_class_names.include?('task-list') - # Don't add class to all lists - return - elsif new_class_names.include?('task-list-item') - add_css_class_without_fix(node.parent, 'task-list') - end - - add_css_class_without_fix(node, *new_class_names) - end - - alias_method_chain :add_css_class, :fix end end end diff --git a/lib/banzai/filter/user_reference_filter.rb b/lib/banzai/filter/user_reference_filter.rb index e1ca7f4d24b..f842b1fb779 100644 --- a/lib/banzai/filter/user_reference_filter.rb +++ b/lib/banzai/filter/user_reference_filter.rb @@ -35,10 +35,10 @@ module Banzai user_link_filter(content) end elsif element_node?(node) - yield_valid_link(node) do |link, text| + yield_valid_link(node) do |link, inner_html| if link =~ ref_pattern_start replace_link_node_with_href(node, link) do - user_link_filter(link, link_text: text) + user_link_filter(link, link_content: inner_html) end end end @@ -52,15 +52,16 @@ module Banzai # user's profile page. # # text - String text to replace references in. + # link_content - Original content of the link being replaced. # # Returns a String with `@user` references replaced with links. All links # have `gfm` and `gfm-project_member` class names attached for styling. - def user_link_filter(text, link_text: nil) + def user_link_filter(text, link_content: nil) self.class.references_in(text) do |match, username| if username == 'all' - link_to_all(link_text: link_text) + link_to_all(link_content: link_content) elsif namespace = namespaces[username] - link_to_namespace(namespace, link_text: link_text) || match + link_to_namespace(namespace, link_content: link_content) || match else match end @@ -102,45 +103,49 @@ module Banzai reference_class(:project_member) end - def link_to_all(link_text: nil) + def link_to_all(link_content: nil) project = context[:project] author = context[:author] - url = urls.namespace_project_url(project.namespace, project, - only_path: context[:only_path]) + if author && !project.team.member?(author) + link_content + else + url = urls.namespace_project_url(project.namespace, project, + only_path: context[:only_path]) - data = data_attribute(project: project.id, author: author.try(:id)) - text = link_text || User.reference_prefix + 'all' + data = data_attribute(project: project.id, author: author.try(:id)) + content = link_content || User.reference_prefix + 'all' - link_tag(url, data, text, 'All Project and Group Members') + link_tag(url, data, content, 'All Project and Group Members') + end end - def link_to_namespace(namespace, link_text: nil) + def link_to_namespace(namespace, link_content: nil) if namespace.is_a?(Group) - link_to_group(namespace.path, namespace, link_text: link_text) + link_to_group(namespace.path, namespace, link_content: link_content) else - link_to_user(namespace.path, namespace, link_text: link_text) + link_to_user(namespace.path, namespace, link_content: link_content) end end - def link_to_group(group, namespace, link_text: nil) + def link_to_group(group, namespace, link_content: nil) url = urls.group_url(group, only_path: context[:only_path]) data = data_attribute(group: namespace.id) - text = link_text || Group.reference_prefix + group + content = link_content || Group.reference_prefix + group - link_tag(url, data, text, namespace.name) + link_tag(url, data, content, namespace.name) end - def link_to_user(user, namespace, link_text: nil) + def link_to_user(user, namespace, link_content: nil) url = urls.user_url(user, only_path: context[:only_path]) data = data_attribute(user: namespace.owner_id) - text = link_text || User.reference_prefix + user + content = link_content || User.reference_prefix + user - link_tag(url, data, text, namespace.owner_name) + link_tag(url, data, content, namespace.owner_name) end - def link_tag(url, data, text, title) - %(<a href="#{url}" #{data} class="#{link_class}" title="#{escape_once(title)}">#{escape_once(text)}</a>) + def link_tag(url, data, link_content, title) + %(<a href="#{url}" #{data} class="#{link_class}" title="#{escape_once(title)}">#{link_content}</a>) end end end diff --git a/lib/banzai/filter/wiki_link_filter/rewriter.rb b/lib/banzai/filter/wiki_link_filter/rewriter.rb index 2e2c8da311e..e7a1ec8457d 100644 --- a/lib/banzai/filter/wiki_link_filter/rewriter.rb +++ b/lib/banzai/filter/wiki_link_filter/rewriter.rb @@ -31,6 +31,7 @@ module Banzai def apply_relative_link_rules! if @uri.relative? && @uri.path.present? link = ::File.join(@wiki_base_path, @uri.path) + link = "#{link}##{@uri.fragment}" if @uri.fragment @uri = Addressable::URI.parse(link) end end diff --git a/lib/banzai/note_renderer.rb b/lib/banzai/note_renderer.rb index bab6a9934d1..2b7c10f1a0e 100644 --- a/lib/banzai/note_renderer.rb +++ b/lib/banzai/note_renderer.rb @@ -3,7 +3,7 @@ module Banzai # Renders a collection of Note instances. # # notes - The notes to render. - # project - The project to use for rendering/redacting. + # project - The project to use for redacting. # user - The user viewing the notes. # path - The request path. # wiki - The project's wiki. @@ -13,8 +13,7 @@ module Banzai user, requested_path: path, project_wiki: wiki, - ref: git_ref, - pipeline: :note) + ref: git_ref) renderer.render(notes, :note) end diff --git a/lib/banzai/object_renderer.rb b/lib/banzai/object_renderer.rb index 9aef807c152..9f8eb0931b8 100644 --- a/lib/banzai/object_renderer.rb +++ b/lib/banzai/object_renderer.rb @@ -1,28 +1,32 @@ module Banzai - # Class for rendering multiple objects (e.g. Note instances) in a single pass. + # Class for rendering multiple objects (e.g. Note instances) in a single pass, + # using +render_field+ to benefit from caching in the database. Rendering and + # redaction are both performed. # - # Rendered Markdown is stored in an attribute in every object based on the - # name of the attribute containing the Markdown. For example, when the - # attribute `note` is rendered the HTML is stored in `note_html`. + # The unredacted HTML is generated according to the usual +render_field+ + # policy, so specify the pipeline and any other context options on the model. + # + # The *redacted* (i.e., suitable for use) HTML is placed in an attribute + # named "redacted_<foo>", where <foo> is the name of the cache field for the + # chosen attribute. + # + # As an example, rendering the attribute `note` would place the unredacted + # HTML into `note_html` and the redacted HTML into `redacted_note_html`. class ObjectRenderer attr_reader :project, :user - # Make sure to set the appropriate pipeline in the `raw_context` attribute - # (e.g. `:note` for Note instances). - # - # project - A Project to use for rendering and redacting Markdown. + # project - A Project to use for redacting Markdown. # user - The user viewing the Markdown/HTML documents, if any. - # context - A Hash containing extra attributes to use in the rendering - # pipeline. - def initialize(project, user = nil, raw_context = {}) + # context - A Hash containing extra attributes to use during redaction + def initialize(project, user = nil, redaction_context = {}) @project = project @user = user - @raw_context = raw_context + @redaction_context = redaction_context end # Renders and redacts an Array of objects. # - # objects - The objects to render + # objects - The objects to render. # attribute - The attribute containing the raw Markdown to render. # # Returns the same input objects. @@ -32,7 +36,7 @@ module Banzai objects.each_with_index do |object, index| redacted_data = redacted[index] - object.__send__("#{attribute}_html=", redacted_data[:document].to_html.html_safe) + object.__send__("redacted_#{attribute}_html=", redacted_data[:document].to_html.html_safe) object.user_visible_reference_count = redacted_data[:visible_reference_count] end end @@ -53,12 +57,8 @@ module Banzai # Returns a Banzai context for the given object and attribute. def context_for(object, attribute) - context = base_context.merge(cache_key: [object, attribute]) - - if object.respond_to?(:author) - context[:author] = object.author - end - + context = base_context.dup + context = context.merge(object.banzai_render_context(attribute)) context end @@ -66,21 +66,16 @@ module Banzai # # Returns an Array of `Nokogiri::HTML::Document`. def render_attributes(objects, attribute) - strings_and_contexts = objects.map do |object| + objects.map do |object| + string = Banzai.render_field(object, attribute) context = context_for(object, attribute) - string = object.__send__(attribute) - - { text: string, context: context } - end - - Banzai.cache_collection_render(strings_and_contexts).each_with_index.map do |html, index| - Banzai::Pipeline[:relative_link].to_document(html, strings_and_contexts[index][:context]) + Banzai::Pipeline[:relative_link].to_document(string, context) end end def base_context - @base_context ||= @raw_context.merge(current_user: user, project: project) + @base_context ||= @redaction_context.merge(current_user: user, project: project) end end end diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index 8d94b199c66..5da2d0b008c 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -25,7 +25,9 @@ module Banzai Filter::MilestoneReferenceFilter, Filter::TaskListFilter, - Filter::InlineDiffFilter + Filter::InlineDiffFilter, + + Filter::SetDirectionFilter ] end diff --git a/lib/banzai/pipeline/single_line_pipeline.rb b/lib/banzai/pipeline/single_line_pipeline.rb index ba2555df98d..1929099931b 100644 --- a/lib/banzai/pipeline/single_line_pipeline.rb +++ b/lib/banzai/pipeline/single_line_pipeline.rb @@ -3,6 +3,7 @@ module Banzai class SingleLinePipeline < GfmPipeline def self.filters @filters ||= FilterArray[ + Filter::HtmlEntityFilter, Filter::SanitizationFilter, Filter::EmojiFilter, diff --git a/lib/banzai/redactor.rb b/lib/banzai/redactor.rb index 0df3a72d1c4..de3ebe72720 100644 --- a/lib/banzai/redactor.rb +++ b/lib/banzai/redactor.rb @@ -41,10 +41,10 @@ module Banzai next if visible.include?(node) doc_data[:visible_reference_count] -= 1 - # The reference should be replaced by the original text, - # which is not always the same as the rendered text. - text = node.attr('data-original') || node.text - node.replace(text) + # The reference should be replaced by the original link's content, + # which is not always the same as the rendered one. + content = node.attr('data-original') || node.inner_html + node.replace(content) end end diff --git a/lib/banzai/reference_parser/base_parser.rb b/lib/banzai/reference_parser/base_parser.rb index 6cf218aaa0d..d8a855ec1fe 100644 --- a/lib/banzai/reference_parser/base_parser.rb +++ b/lib/banzai/reference_parser/base_parser.rb @@ -63,12 +63,7 @@ module Banzai nodes.select do |node| if node.has_attribute?(project_attr) node_id = node.attr(project_attr).to_i - - if project && project.id == node_id - true - else - can?(user, :read_project, projects[node_id]) - end + can_read_reference?(user, projects[node_id]) else true end @@ -79,7 +74,11 @@ module Banzai def referenced_by(nodes) ids = unique_attribute_values(nodes, self.class.data_attribute) - references_relation.where(id: ids) + if ids.empty? + references_relation.none + else + references_relation.where(id: ids) + end end # Returns the ActiveRecord::Relation to use for querying references in the @@ -211,7 +210,7 @@ module Banzai end def can?(user, permission, subject) - Ability.abilities.allowed?(user, permission, subject) + Ability.allowed?(user, permission, subject) end def find_projects_for_hash_keys(hash) @@ -222,6 +221,15 @@ module Banzai attr_reader :current_user, :project + # When a feature is disabled or visible only for + # team members we should not allow team members + # see reference comments. + # Override this method on subclasses + # to check if user can read resource + def can_read_reference?(user, ref_project) + raise NotImplementedError + end + def lazy(&block) Gitlab::Lazy.new(&block) end diff --git a/lib/banzai/reference_parser/commit_parser.rb b/lib/banzai/reference_parser/commit_parser.rb index 0fee9d267de..8c54a041cb8 100644 --- a/lib/banzai/reference_parser/commit_parser.rb +++ b/lib/banzai/reference_parser/commit_parser.rb @@ -29,6 +29,12 @@ module Banzai commits end + + private + + def can_read_reference?(user, ref_project) + can?(user, :download_code, ref_project) + end end end end diff --git a/lib/banzai/reference_parser/commit_range_parser.rb b/lib/banzai/reference_parser/commit_range_parser.rb index 69d01f8db15..0878b6afba3 100644 --- a/lib/banzai/reference_parser/commit_range_parser.rb +++ b/lib/banzai/reference_parser/commit_range_parser.rb @@ -33,6 +33,12 @@ module Banzai range.valid_commits? ? range : nil end + + private + + def can_read_reference?(user, ref_project) + can?(user, :download_code, ref_project) + end end end end diff --git a/lib/banzai/reference_parser/external_issue_parser.rb b/lib/banzai/reference_parser/external_issue_parser.rb index a1264db2111..6e7b7669578 100644 --- a/lib/banzai/reference_parser/external_issue_parser.rb +++ b/lib/banzai/reference_parser/external_issue_parser.rb @@ -20,6 +20,12 @@ module Banzai def issue_ids_per_project(nodes) gather_attributes_per_project(nodes, self.class.data_attribute) end + + private + + def can_read_reference?(user, ref_project) + can?(user, :read_issue, ref_project) + end end end end diff --git a/lib/banzai/reference_parser/label_parser.rb b/lib/banzai/reference_parser/label_parser.rb index e5d1eb11d7f..aa76c64ac5f 100644 --- a/lib/banzai/reference_parser/label_parser.rb +++ b/lib/banzai/reference_parser/label_parser.rb @@ -6,6 +6,12 @@ module Banzai def references_relation Label end + + private + + def can_read_reference?(user, ref_project) + can?(user, :read_label, ref_project) + end end end end diff --git a/lib/banzai/reference_parser/merge_request_parser.rb b/lib/banzai/reference_parser/merge_request_parser.rb index c9a9ca79c09..40451947e6c 100644 --- a/lib/banzai/reference_parser/merge_request_parser.rb +++ b/lib/banzai/reference_parser/merge_request_parser.rb @@ -6,6 +6,12 @@ module Banzai def references_relation MergeRequest.includes(:author, :assignee, :target_project) end + + private + + def can_read_reference?(user, ref_project) + can?(user, :read_merge_request, ref_project) + end end end end diff --git a/lib/banzai/reference_parser/milestone_parser.rb b/lib/banzai/reference_parser/milestone_parser.rb index a000ac61e5c..d3968d6b229 100644 --- a/lib/banzai/reference_parser/milestone_parser.rb +++ b/lib/banzai/reference_parser/milestone_parser.rb @@ -6,6 +6,12 @@ module Banzai def references_relation Milestone end + + private + + def can_read_reference?(user, ref_project) + can?(user, :read_milestone, ref_project) + end end end end diff --git a/lib/banzai/reference_parser/snippet_parser.rb b/lib/banzai/reference_parser/snippet_parser.rb index fa71b3c952a..63b592137bb 100644 --- a/lib/banzai/reference_parser/snippet_parser.rb +++ b/lib/banzai/reference_parser/snippet_parser.rb @@ -6,6 +6,12 @@ module Banzai def references_relation Snippet end + + private + + def can_read_reference?(user, ref_project) + can?(user, :read_project_snippet, ref_project) + end end end end diff --git a/lib/banzai/reference_parser/user_parser.rb b/lib/banzai/reference_parser/user_parser.rb index 863f5725d3b..7adaffa19c1 100644 --- a/lib/banzai/reference_parser/user_parser.rb +++ b/lib/banzai/reference_parser/user_parser.rb @@ -30,22 +30,36 @@ module Banzai nodes.each do |node| if node.has_attribute?(group_attr) - node_group = groups[node.attr(group_attr).to_i] - - if node_group && - can?(user, :read_group, node_group) - visible << node - end - # Remaining nodes will be processed by the parent class' - # implementation of this method. + next unless can_read_group_reference?(node, user, groups) + visible << node + elsif can_read_project_reference?(node) + visible << node else remaining << node end end + # If project does not belong to a group + # and does not have the same project id as the current project + # base class will check if user can read the project that contains + # the user reference. visible + super(current_user, remaining) end + # Check if project belongs to a group which + # user can read. + def can_read_group_reference?(node, user, groups) + node_group = groups[node.attr('data-group').to_i] + + node_group && can?(user, :read_group, node_group) + end + + def can_read_project_reference?(node) + node_id = node.attr('data-project').to_i + + project && project.id == node_id + end + def nodes_user_can_reference(current_user, nodes) project_attr = 'data-project' author_attr = 'data-author' @@ -88,6 +102,10 @@ module Banzai collection_objects_for_ids(Project, ids). flat_map { |p| p.team.members.to_a } end + + def can_read_reference?(user, ref_project) + can?(user, :read_project, ref_project) + end end end end diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index a4ae27eefd8..f31fb6c3f71 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -1,6 +1,6 @@ module Banzai module Renderer - extend self + module_function # Convert a Markdown String into an HTML-safe String of HTML # @@ -31,6 +31,34 @@ module Banzai end end + # Convert a Markdown-containing field on an object into an HTML-safe String + # of HTML. This method is analogous to calling render(object.field), but it + # can cache the rendered HTML in the object, rather than Redis. + # + # The context to use is learned from the passed-in object by calling + # #banzai_render_context(field), and cannot be changed. Use #render, passing + # it the field text, if a custom rendering is needed. The generated context + # is returned along with the HTML. + def render_field(object, field) + html_field = object.markdown_cache_field_for(field) + + html = object.__send__(html_field) + return html if html.present? + + html = cacheless_render_field(object, field) + update_object(object, html_field, html) unless object.new_record? || object.destroyed? + + html + end + + # Same as +render_field+, but without consulting or updating the cache field + def cacheless_render_field(object, field) + text = object.__send__(field) + context = object.banzai_render_context(field) + + cacheless_render(text, context) + end + # Perform multiple render from an Array of Markdown String into an # Array of HTML-safe String of HTML. # @@ -113,8 +141,6 @@ module Banzai end.html_safe end - private - def cacheless_render(text, context = {}) Gitlab::Metrics.measure(:banzai_cacheless_render) do result = render_result(text, context) @@ -140,5 +166,9 @@ module Banzai return unless cache_key Rails.cache.send(:expanded_key, full_cache_key(cache_key, pipeline_name)) end + + def update_object(object, html_field, html) + object.update_column(html_field, html) + end end end diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb index 17bb99a2ae5..a6b9beecded 100644 --- a/lib/ci/api/api.rb +++ b/lib/ci/api/api.rb @@ -9,22 +9,14 @@ module Ci end rescue_from :all do |exception| - # lifted from https://github.com/rails/rails/blob/master/actionpack/lib/action_dispatch/middleware/debug_exceptions.rb#L60 - # why is this not wrapped in something reusable? - trace = exception.backtrace - - message = "\n#{exception.class} (#{exception.message}):\n" - message << exception.annoted_source_code.to_s if exception.respond_to?(:annoted_source_code) - message << " " << trace.join("\n ") - - API.logger.add Logger::FATAL, message - rack_response({ 'message' => '500 Internal Server Error' }, 500) + handle_api_exception(exception) end content_type :txt, 'text/plain' content_type :json, 'application/json' format :json + helpers ::SentryHelper helpers ::Ci::API::Helpers helpers ::API::Helpers helpers Gitlab::CurrentSettings diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 260ac81f5fa..ed87a2603e8 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -12,17 +12,21 @@ module Ci # POST /builds/register post "register" do authenticate_runner! - update_runner_last_contact - update_runner_info required_attributes! [:token] not_found! unless current_runner.active? + update_runner_info build = Ci::RegisterBuildService.new.execute(current_runner) if build + Gitlab::Metrics.add_event(:build_found, + project: build.project.path_with_namespace) + present build, with: Entities::BuildDetails else - not_found! + Gitlab::Metrics.add_event(:build_not_found) + + build_not_found! end end @@ -36,12 +40,16 @@ module Ci # PUT /builds/:id put ":id" do authenticate_runner! - update_runner_last_contact build = Ci::Build.where(runner_id: current_runner.id).running.find(params[:id]) forbidden!('Build has been erased!') if build.erased? + update_runner_info + build.update_attributes(trace: params[:trace]) if params[:trace] + Gitlab::Metrics.add_event(:update_build, + project: build.project.path_with_namespace) + case params[:state].to_s when 'success' build.success @@ -93,6 +101,7 @@ module Ci # POST /builds/:id/artifacts/authorize post ":id/artifacts/authorize" do require_gitlab_workhorse! + Gitlab::Workhorse.verify_api_request!(headers) not_allowed! unless Gitlab.config.artifacts.enabled build = Ci::Build.find_by_id(params[:id]) not_found! unless build @@ -105,7 +114,8 @@ module Ci end status 200 - { TempPath: ArtifactUploader.artifacts_upload_path } + content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE + Gitlab::Workhorse.artifact_upload_ok end # Upload artifacts to build - Runners only diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb index 3f5bdaba3f5..66c05773b68 100644 --- a/lib/ci/api/entities.rb +++ b/lib/ci/api/entities.rb @@ -15,6 +15,15 @@ module Ci expose :filename, :size end + class BuildOptions < Grape::Entity + expose :image + expose :services + expose :artifacts + expose :cache + expose :dependencies + expose :after_script + end + class Build < Grape::Entity expose :id, :ref, :tag, :sha, :status expose :name, :token, :stage diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb index 199d62d9b8a..e608f5f6cad 100644 --- a/lib/ci/api/helpers.rb +++ b/lib/ci/api/helpers.rb @@ -3,7 +3,7 @@ module Ci module Helpers BUILD_TOKEN_HEADER = "HTTP_BUILD_TOKEN" BUILD_TOKEN_PARAM = :token - UPDATE_RUNNER_EVERY = 60 + UPDATE_RUNNER_EVERY = 10 * 60 def authenticate_runners! forbidden! unless runner_registration_token_valid? @@ -14,19 +14,45 @@ module Ci end def authenticate_build_token!(build) - token = (params[BUILD_TOKEN_PARAM] || env[BUILD_TOKEN_HEADER]).to_s - forbidden! unless token && build.valid_token?(token) + forbidden! unless build_token_valid?(build) end def runner_registration_token_valid? - params[:token] == current_application_settings.runners_registration_token + ActiveSupport::SecurityUtils.variable_size_secure_compare( + params[:token], + current_application_settings.runners_registration_token) + end + + def build_token_valid?(build) + token = (params[BUILD_TOKEN_PARAM] || env[BUILD_TOKEN_HEADER]).to_s + + # We require to also check `runners_token` to maintain compatibility with old version of runners + token && (build.valid_token?(token) || build.project.valid_runners_token?(token)) end - def update_runner_last_contact - # Use a random threshold to prevent beating DB updates + def update_runner_info + return unless update_runner? + + current_runner.contacted_at = Time.now + current_runner.assign_attributes(get_runner_version_from_params) + current_runner.save if current_runner.changed? + end + + def update_runner? + # Use a random threshold to prevent beating DB updates. + # It generates a distribution between [40m, 80m]. + # contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY) - if current_runner.contacted_at.nil? || Time.now - current_runner.contacted_at >= contacted_at_max_age - current_runner.update_attributes(contacted_at: Time.now) + + current_runner.contacted_at.nil? || + (Time.now - current_runner.contacted_at) >= contacted_at_max_age + end + + def build_not_found! + if headers['User-Agent'].match(/gitlab-ci-multi-runner \d+\.\d+\.\d+(~beta\.\d+\.g[0-9a-f]+)? /) + no_content! + else + not_found! end end @@ -39,11 +65,6 @@ module Ci attributes_for_keys(["name", "version", "revision", "platform", "architecture"], params["info"]) end - def update_runner_info - current_runner.assign_attributes(get_runner_version_from_params) - current_runner.save if current_runner.changed? - end - def max_artifacts_size current_application_settings.max_artifacts_size.megabytes.to_i end diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 47efd5bd9f2..3e33c9399e2 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -4,7 +4,7 @@ module Ci include Gitlab::Ci::Config::Node::LegacyValidationHelpers - attr_reader :path, :cache, :stages + attr_reader :path, :cache, :stages, :jobs def initialize(config, path = nil) @ci_config = Gitlab::Ci::Config.new(config) @@ -55,29 +55,36 @@ module Ci { stage_idx: @stages.index(job[:stage]), stage: job[:stage], - ## - # Refactoring note: - # - before script behaves differently than after script - # - after script returns an array of commands - # - before script should be a concatenated command - commands: [job[:before_script] || @before_script, job[:script]].flatten.compact.join("\n"), + commands: job[:commands], tag_list: job[:tags] || [], name: job[:name].to_s, allow_failure: job[:allow_failure] || false, when: job[:when] || 'on_success', - environment: job[:environment], + environment: job[:environment_name], yaml_variables: yaml_variables(name), options: { - image: job[:image] || @image, - services: job[:services] || @services, + image: job[:image], + services: job[:services], artifacts: job[:artifacts], - cache: job[:cache] || @cache, + cache: job[:cache], dependencies: job[:dependencies], - after_script: job[:after_script] || @after_script, + after_script: job[:after_script], + environment: job[:environment], }.compact } end + def self.validation_message(content) + return 'Please provide content of .gitlab-ci.yml' if content.blank? + + begin + Ci::GitlabCiYamlProcessor.new(content) + nil + rescue ValidationError, Psych::SyntaxError => e + e.message + end + end + private def initial_parsing @@ -102,6 +109,7 @@ module Ci validate_job_stage!(name, job) validate_job_dependencies!(name, job) + validate_job_environment!(name, job) end end @@ -143,6 +151,35 @@ module Ci end end + def validate_job_environment!(name, job) + return unless job[:environment] + return unless job[:environment].is_a?(Hash) + + environment = job[:environment] + validate_on_stop_job!(name, environment, environment[:on_stop]) + end + + def validate_on_stop_job!(name, environment, on_stop) + return unless on_stop + + on_stop_job = @jobs[on_stop.to_sym] + unless on_stop_job + raise ValidationError, "#{name} job: on_stop job #{on_stop} is not defined" + end + + unless on_stop_job[:environment] + raise ValidationError, "#{name} job: on_stop job #{on_stop} does not have environment defined" + end + + unless on_stop_job[:environment][:name] == environment[:name] + raise ValidationError, "#{name} job: on_stop job #{on_stop} have different environment name" + end + + unless on_stop_job[:environment][:action] == 'stop' + raise ValidationError, "#{name} job: on_stop job #{on_stop} needs to have action stop defined" + end + end + def process?(only_params, except_params, ref, tag, trigger_request) if only_params.present? return false unless matching?(only_params, ref, tag, trigger_request) diff --git a/lib/ci/mask_secret.rb b/lib/ci/mask_secret.rb new file mode 100644 index 00000000000..997377abc55 --- /dev/null +++ b/lib/ci/mask_secret.rb @@ -0,0 +1,10 @@ +module Ci::MaskSecret + class << self + def mask!(value, token) + return value unless value.present? && token.present? + + value.gsub!(token, 'x' * token.length) + value + end + end +end diff --git a/lib/ci/version_info.rb b/lib/ci/version_info.rb deleted file mode 100644 index 2a87c91db5e..00000000000 --- a/lib/ci/version_info.rb +++ /dev/null @@ -1,52 +0,0 @@ -class VersionInfo - include Comparable - - attr_reader :major, :minor, :patch - - def self.parse(str) - if str && m = str.match(/(\d+)\.(\d+)\.(\d+)/) - VersionInfo.new(m[1].to_i, m[2].to_i, m[3].to_i) - else - VersionInfo.new - end - end - - def initialize(major = 0, minor = 0, patch = 0) - @major = major - @minor = minor - @patch = patch - end - - def <=>(other) - return unless other.is_a? VersionInfo - return unless valid? && other.valid? - - if other.major < @major - 1 - elsif @major < other.major - -1 - elsif other.minor < @minor - 1 - elsif @minor < other.minor - -1 - elsif other.patch < @patch - 1 - elsif @patch < other.patch - -1 - else - 0 - end - end - - def to_s - if valid? - "%d.%d.%d" % [@major, @minor, @patch] - else - "Unknown" - end - end - - def valid? - @major >= 0 && @minor >= 0 && @patch >= 0 && @major + @minor + @patch > 0 - end -end diff --git a/lib/constraints/constrainer_helper.rb b/lib/constraints/constrainer_helper.rb new file mode 100644 index 00000000000..ab07a6793d9 --- /dev/null +++ b/lib/constraints/constrainer_helper.rb @@ -0,0 +1,15 @@ +module ConstrainerHelper + def extract_resource_path(path) + id = path.dup + id.sub!(/\A#{relative_url_root}/, '') if relative_url_root + id.sub(/\A\/+/, '').sub(/\/+\z/, '').sub(/.atom\z/, '') + end + + private + + def relative_url_root + if defined?(Gitlab::Application.config.relative_url_root) + Gitlab::Application.config.relative_url_root + end + end +end diff --git a/lib/constraints/group_url_constrainer.rb b/lib/constraints/group_url_constrainer.rb new file mode 100644 index 00000000000..2af6e1a11c8 --- /dev/null +++ b/lib/constraints/group_url_constrainer.rb @@ -0,0 +1,15 @@ +require_relative 'constrainer_helper' + +class GroupUrlConstrainer + include ConstrainerHelper + + def matches?(request) + id = extract_resource_path(request.path) + + if id =~ Gitlab::Regex.namespace_regex + Group.find_by(path: id).present? + else + false + end + end +end diff --git a/lib/constraints/user_url_constrainer.rb b/lib/constraints/user_url_constrainer.rb new file mode 100644 index 00000000000..4d722ad5af2 --- /dev/null +++ b/lib/constraints/user_url_constrainer.rb @@ -0,0 +1,15 @@ +require_relative 'constrainer_helper' + +class UserUrlConstrainer + include ConstrainerHelper + + def matches?(request) + id = extract_resource_path(request.path) + + if id =~ Gitlab::Regex.namespace_regex + User.find_by('lower(username) = ?', id.downcase).present? + else + false + end + end +end diff --git a/lib/event_filter.rb b/lib/event_filter.rb index 668d2fa41b3..21f6a9a762b 100644 --- a/lib/event_filter.rb +++ b/lib/event_filter.rb @@ -2,8 +2,8 @@ class EventFilter attr_accessor :params class << self - def default_filter - %w{ push issues merge_requests team} + def all + 'all' end def push @@ -35,18 +35,28 @@ class EventFilter return events unless params.present? filter = params.dup - actions = [] - actions << Event::PUSHED if filter.include? 'push' - actions << Event::MERGED if filter.include? 'merged' - if filter.include? 'team' - actions << Event::JOINED - actions << Event::LEFT + case filter + when EventFilter.push + actions = [Event::PUSHED] + when EventFilter.merged + actions = [Event::MERGED] + when EventFilter.comments + actions = [Event::COMMENTED] + when EventFilter.team + actions = [Event::JOINED, Event::LEFT, Event::EXPIRED] + when EventFilter.all + actions = [ + Event::PUSHED, + Event::MERGED, + Event::COMMENTED, + Event::JOINED, + Event::LEFT, + Event::EXPIRED + ] end - actions << Event::COMMENTED if filter.include? 'comments' - events.where(action: actions) end diff --git a/lib/expand_variables.rb b/lib/expand_variables.rb new file mode 100644 index 00000000000..7b1533d0d32 --- /dev/null +++ b/lib/expand_variables.rb @@ -0,0 +1,17 @@ +module ExpandVariables + class << self + def expand(value, variables) + # Convert hash array to variables + if variables.is_a?(Array) + variables = variables.reduce({}) do |hash, variable| + hash[variable[:key]] = variable[:value] + hash + end + end + + value.gsub(/\$([a-zA-Z_][a-zA-Z0-9_]*)|\${\g<1>}|%\g<1>%/) do + variables[$1 || $2] + end + end + end +end diff --git a/lib/extracts_path.rb b/lib/extracts_path.rb index 84688f6646e..82551f1f222 100644 --- a/lib/extracts_path.rb +++ b/lib/extracts_path.rb @@ -52,8 +52,7 @@ module ExtractsPath # Append a trailing slash if we only get a ref and no file path id += '/' unless id.ends_with?('/') - valid_refs = @project.repository.ref_names - valid_refs.select! { |v| id.start_with?("#{v}/") } + valid_refs = ref_names.select { |v| id.start_with?("#{v}/") } if valid_refs.length == 0 # No exact ref match, so just try our best @@ -74,6 +73,19 @@ module ExtractsPath pair end + # If we have an ID of 'foo.atom', and the controller provides Atom and HTML + # formats, then we have to check if the request was for the Atom version of + # the ID without the '.atom' suffix, or the HTML version of the ID including + # the suffix. We only check this if the version including the suffix doesn't + # match, so it is possible to create a branch which has an unroutable Atom + # feed. + def extract_ref_without_atom(id) + id_without_atom = id.sub(/\.atom$/, '') + valid_refs = ref_names.select { |v| "#{id_without_atom}/".start_with?("#{v}/") } + + valid_refs.max_by(&:length) + end + # Assigns common instance variables for views working with Git tree-ish objects # # Assignments are: @@ -86,21 +98,29 @@ module ExtractsPath # If the :id parameter appears to be requesting a specific response format, # that will be handled as well. # + # If there is no path and the ref doesn't exist in the repo, try to resolve + # the ref without an '.atom' suffix. If _that_ ref is found, set the request's + # format to Atom manually. + # # Automatically renders `not_found!` if a valid tree path could not be # resolved (e.g., when a user inserts an invalid path or ref). def assign_ref_vars # assign allowed options - allowed_options = ["filter_ref", "extended_sha1"] + allowed_options = ["filter_ref"] @options = params.select {|key, value| allowed_options.include?(key) && !value.blank? } @options = HashWithIndifferentAccess.new(@options) - @id = Addressable::URI.normalize_component(get_id) + @id = get_id @ref, @path = extract_ref(@id) @repo = @project.repository - if @options[:extended_sha1].blank? + + @commit = @repo.commit(@ref) + + if @path.empty? && !@commit && @id.ends_with?('.atom') + @id = @ref = extract_ref_without_atom(@id) @commit = @repo.commit(@ref) - else - @commit = @repo.commit(@options[:extended_sha1]) + + request.format = :atom if @commit end raise InvalidPathError unless @commit @@ -119,9 +139,16 @@ module ExtractsPath private + # overriden in subclasses, do not remove def get_id id = params[:id] || params[:ref] id += "/" + params[:path] unless params[:path].blank? id end + + def ref_names + return [] unless @project + + @ref_names ||= @project.repository.ref_names + end end diff --git a/lib/generators/rails/post_deployment_migration/post_deployment_migration_generator.rb b/lib/generators/rails/post_deployment_migration/post_deployment_migration_generator.rb new file mode 100644 index 00000000000..7cb4bccb23c --- /dev/null +++ b/lib/generators/rails/post_deployment_migration/post_deployment_migration_generator.rb @@ -0,0 +1,15 @@ +require 'rails/generators' + +module Rails + class PostDeploymentMigrationGenerator < Rails::Generators::NamedBase + def create_migration_file + timestamp = Time.now.strftime('%Y%m%d%H%I%S') + + template "migration.rb", "db/post_migrate/#{timestamp}_#{file_name}.rb" + end + + def migration_class_name + file_name.camelize + end + end +end diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb index a533bac2692..9b484a2ecfd 100644 --- a/lib/gitlab/access.rb +++ b/lib/gitlab/access.rb @@ -53,6 +53,10 @@ module Gitlab } end + def sym_options_with_owner + sym_options.merge(owner: OWNER) + end + def protection_options { "Not protected: Both developers and masters can push new commits, force push, or delete the branch." => PROTECTION_NONE, diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index db1704af75e..aca5d0020cf 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -1,22 +1,22 @@ module Gitlab module Auth - Result = Struct.new(:user, :type) + class MissingPersonalTokenError < StandardError; end class << self def find_for_git_client(login, password, project:, ip:) raise "Must provide an IP for rate limiting" if ip.nil? - result = Result.new + result = + service_request_check(login, password, project) || + build_access_token_check(login, password) || + user_with_password_for_git(login, password) || + oauth_access_token_check(login, password) || + lfs_token_check(login, password) || + personal_access_token_check(login, password) || + Gitlab::Auth::Result.new - if valid_ci_request?(login, password, project) - result.type = :ci - elsif result.user = find_with_user_password(login, password) - result.type = :gitlab_or_ldap - elsif result.user = oauth_access_token_check(login, password) - result.type = :oauth - end + rate_limit!(ip, success: result.success?, login: login) - rate_limit!(ip, success: !!result.user || (result.type == :ci), login: login) result end @@ -58,30 +58,117 @@ module Gitlab private - def valid_ci_request?(login, password, project) + def service_request_check(login, password, project) matched_login = /(?<service>^[a-zA-Z]*-ci)-token$/.match(login) - return false unless project && matched_login.present? + return unless project && matched_login.present? underscored_service = matched_login['service'].underscore - if underscored_service == 'gitlab_ci' - project && project.valid_build_token?(password) - elsif Service.available_services_names.include?(underscored_service) + if Service.available_services_names.include?(underscored_service) # We treat underscored_service as a trusted input because it is included # in the Service.available_services_names whitelist. service = project.public_send("#{underscored_service}_service") - service && service.activated? && service.valid_token?(password) + if service && service.activated? && service.valid_token?(password) + Gitlab::Auth::Result.new(nil, project, :ci, build_authentication_abilities) + end end end + def user_with_password_for_git(login, password) + user = find_with_user_password(login, password) + return unless user + + raise Gitlab::Auth::MissingPersonalTokenError if user.two_factor_enabled? + + Gitlab::Auth::Result.new(user, nil, :gitlab_or_ldap, full_authentication_abilities) + end + def oauth_access_token_check(login, password) if login == "oauth2" && password.present? token = Doorkeeper::AccessToken.by_token(password) - token && token.accessible? && User.find_by(id: token.resource_owner_id) + if token && token.accessible? + user = User.find_by(id: token.resource_owner_id) + Gitlab::Auth::Result.new(user, nil, :oauth, read_authentication_abilities) + end + end + end + + def personal_access_token_check(login, password) + if login && password + user = User.find_by_personal_access_token(password) + validation = User.by_login(login) + Gitlab::Auth::Result.new(user, nil, :personal_token, full_authentication_abilities) if user.present? && user == validation + end + end + + def lfs_token_check(login, password) + deploy_key_matches = login.match(/\Alfs\+deploy-key-(\d+)\z/) + + actor = + if deploy_key_matches + DeployKey.find(deploy_key_matches[1]) + else + User.by_login(login) + end + + return unless actor + + token_handler = Gitlab::LfsToken.new(actor) + + authentication_abilities = + if token_handler.user? + full_authentication_abilities + else + read_authentication_abilities + end + + Result.new(actor, nil, token_handler.type, authentication_abilities) if Devise.secure_compare(token_handler.token, password) + end + + def build_access_token_check(login, password) + return unless login == 'gitlab-ci-token' + return unless password + + build = ::Ci::Build.running.find_by_token(password) + return unless build + return unless build.project.builds_enabled? + + if build.user + # If user is assigned to build, use restricted credentials of user + Gitlab::Auth::Result.new(build.user, build.project, :build, build_authentication_abilities) + else + # Otherwise use generic CI credentials (backward compatibility) + Gitlab::Auth::Result.new(nil, build.project, :ci, build_authentication_abilities) end end + + public + + def build_authentication_abilities + [ + :read_project, + :build_download_code, + :build_read_container_image, + :build_create_container_image + ] + end + + def read_authentication_abilities + [ + :read_project, + :download_code, + :read_container_image + ] + end + + def full_authentication_abilities + read_authentication_abilities + [ + :push_code, + :create_container_image + ] + end end end end diff --git a/lib/gitlab/auth/result.rb b/lib/gitlab/auth/result.rb new file mode 100644 index 00000000000..6be7f690676 --- /dev/null +++ b/lib/gitlab/auth/result.rb @@ -0,0 +1,21 @@ +module Gitlab + module Auth + Result = Struct.new(:actor, :project, :type, :authentication_abilities) do + def ci?(for_project) + type == :ci && + project && + project == for_project + end + + def lfs_deploy_token?(for_project) + type == :lfs_deploy_token && + actor && + actor.projects.include?(for_project) + end + + def success? + actor.present? || type == :ci + end + end + end +end diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb index 839a4fa30d5..82e194c1af1 100644 --- a/lib/gitlab/backend/shell.rb +++ b/lib/gitlab/backend/shell.rb @@ -6,16 +6,56 @@ module Gitlab KeyAdder = Struct.new(:io) do def add_key(id, key) - key.gsub!(/[[:space:]]+/, ' ').strip! + key = Gitlab::Shell.strip_key(key) + # Newline and tab are part of the 'protocol' used to transmit id+key to the other end + if key.include?("\t") || key.include?("\n") + raise Error.new("Invalid key: #{key.inspect}") + end + io.puts("#{id}\t#{key}") end end class << self + def secret_token + @secret_token ||= begin + File.read(Gitlab.config.gitlab_shell.secret_file).chomp + end + end + + def ensure_secret_token! + return if File.exist?(File.join(Gitlab.config.gitlab_shell.path, '.gitlab_shell_secret')) + + generate_and_link_secret_token + end + def version_required @version_required ||= File.read(Rails.root. join('GITLAB_SHELL_VERSION')).strip end + + def strip_key(key) + key.split(/ /)[0, 2].join(' ') + end + + private + + # Create (if necessary) and link the secret token file + def generate_and_link_secret_token + secret_file = Gitlab.config.gitlab_shell.secret_file + shell_path = Gitlab.config.gitlab_shell.path + + unless File.size?(secret_file) + # Generate a new token of 16 random hexadecimal characters and store it in secret_file. + @secret_token = SecureRandom.hex(16) + File.write(secret_file, @secret_token) + end + + link_path = File.join(shell_path, '.gitlab_shell_secret') + if File.exist?(shell_path) && !File.exist?(link_path) + FileUtils.symlink(secret_file, link_path) + end + end end # Init new repository @@ -87,19 +127,6 @@ module Gitlab 'rm-project', storage, "#{name}.git"]) end - # Gc repository - # - # storage - project storage path - # path - project path with namespace - # - # Ex. - # gc("/path/to/storage", "gitlab/gitlab-ci") - # - def gc(storage, path) - Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'gc', - storage, "#{path}.git"]) - end - # Add new key to gitlab-shell # # Ex. @@ -107,7 +134,7 @@ module Gitlab # def add_key(key_id, key_content) Gitlab::Utils.system_silent([gitlab_shell_keys_path, - 'add-key', key_id, key_content]) + 'add-key', key_id, self.class.strip_key(key_content)]) end # Batch-add keys to authorized_keys @@ -192,21 +219,6 @@ module Gitlab File.exist?(full_path(storage, dir_name)) end - # Create (if necessary) and link the secret token file - def generate_and_link_secret_token - secret_file = Gitlab.config.gitlab_shell.secret_file - unless File.exist? secret_file - # Generate a new token of 16 random hexadecimal characters and store it in secret_file. - token = SecureRandom.hex(16) - File.write(secret_file, token) - end - - link_path = File.join(gitlab_shell_path, '.gitlab_shell_secret') - if File.exist?(gitlab_shell_path) && !File.exist?(link_path) - FileUtils.symlink(secret_file, link_path) - end - end - protected def gitlab_shell_path diff --git a/lib/gitlab/badge/coverage/report.rb b/lib/gitlab/badge/coverage/report.rb index 3d56ea3e47a..9a0482306b7 100644 --- a/lib/gitlab/badge/coverage/report.rb +++ b/lib/gitlab/badge/coverage/report.rb @@ -12,10 +12,7 @@ module Gitlab @ref = ref @job = job - @pipeline = @project.pipelines - .where(ref: @ref) - .where(sha: @project.commit(@ref).try(:sha)) - .first + @pipeline = @project.pipelines.latest_successful_for(@ref) end def entity diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 7beaecd1cf0..f4b5097adb1 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -21,7 +21,7 @@ module Gitlab private - def gl_user_id(project, bitbucket_id) + def gitlab_user_id(project, bitbucket_id) if bitbucket_id user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", bitbucket_id.to_s) (user && user.id) || project.creator_id @@ -74,7 +74,7 @@ module Gitlab description: body, title: issue["title"], state: %w(resolved invalid duplicate wontfix closed).include?(issue["status"]) ? 'closed' : 'opened', - author_id: gl_user_id(project, reporter) + author_id: gitlab_user_id(project, reporter) ) end rescue ActiveRecord::RecordInvalid => e diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb index 4b32eb966aa..cb1065223d4 100644 --- a/lib/gitlab/checks/change_access.rb +++ b/lib/gitlab/checks/change_access.rb @@ -23,6 +23,7 @@ module Gitlab protected def protected_branch_checks + return unless @branch_name return unless project.protected_branch?(@branch_name) if forced_push? && user_access.cannot_do_action?(:force_push_code_to_protected_branches) diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index ae82c0db3f1..bbfa6cf7d05 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -14,7 +14,7 @@ module Gitlab @config = Loader.new(config).load! @global = Node::Global.new(@config) - @global.process! + @global.compose! end def valid? diff --git a/lib/gitlab/ci/config/node/configurable.rb b/lib/gitlab/ci/config/node/configurable.rb index 2de82d40c9d..6b7ab2fdaf2 100644 --- a/lib/gitlab/ci/config/node/configurable.rb +++ b/lib/gitlab/ci/config/node/configurable.rb @@ -23,9 +23,9 @@ module Gitlab end end - private + def compose!(deps = nil) + return unless valid? - def compose! self.class.nodes.each do |key, factory| factory .value(@config[key]) @@ -33,6 +33,12 @@ module Gitlab @entries[key] = factory.create! end + + yield if block_given? + + @entries.each_value do |entry| + entry.compose!(deps) + end end class_methods do diff --git a/lib/gitlab/ci/config/node/entry.rb b/lib/gitlab/ci/config/node/entry.rb index 0c782c422b5..8717eabf81e 100644 --- a/lib/gitlab/ci/config/node/entry.rb +++ b/lib/gitlab/ci/config/node/entry.rb @@ -20,11 +20,14 @@ module Gitlab @validator.validate(:new) end - def process! + def [](key) + @entries[key] || Node::Undefined.new + end + + def compose!(deps = nil) return unless valid? - compose! - descendants.each(&:process!) + yield if block_given? end def leaf? @@ -73,11 +76,6 @@ module Gitlab def self.validator Validator end - - private - - def compose! - end end end end diff --git a/lib/gitlab/ci/config/node/environment.rb b/lib/gitlab/ci/config/node/environment.rb new file mode 100644 index 00000000000..9a95ef43628 --- /dev/null +++ b/lib/gitlab/ci/config/node/environment.rb @@ -0,0 +1,82 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Entry that represents an environment. + # + class Environment < Entry + include Validatable + + ALLOWED_KEYS = %i[name url action on_stop] + + validations do + validate do + unless hash? || string? + errors.add(:config, 'should be a hash or a string') + end + end + + validates :name, presence: true + validates :name, + type: { + with: String, + message: Gitlab::Regex.environment_name_regex_message } + + validates :name, + format: { + with: Gitlab::Regex.environment_name_regex, + message: Gitlab::Regex.environment_name_regex_message } + + with_options if: :hash? do + validates :config, allowed_keys: ALLOWED_KEYS + + validates :url, + length: { maximum: 255 }, + addressable_url: true, + allow_nil: true + + validates :action, + inclusion: { in: %w[start stop], message: 'should be start or stop' }, + allow_nil: true + + validates :on_stop, type: String, allow_nil: true + end + end + + def hash? + @config.is_a?(Hash) + end + + def string? + @config.is_a?(String) + end + + def name + value[:name] + end + + def url + value[:url] + end + + def action + value[:action] || 'start' + end + + def on_stop + value[:on_stop] + end + + def value + case @config + when String then { name: @config, action: 'start' } + when Hash then @config + else {} + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/factory.rb b/lib/gitlab/ci/config/node/factory.rb index 707b052e6a8..5387f29ad59 100644 --- a/lib/gitlab/ci/config/node/factory.rb +++ b/lib/gitlab/ci/config/node/factory.rb @@ -37,8 +37,8 @@ module Gitlab # See issue #18775. # if @value.nil? - Node::Undefined.new( - fabricate_undefined + Node::Unspecified.new( + fabricate_unspecified ) else fabricate(@node, @value) @@ -47,13 +47,13 @@ module Gitlab private - def fabricate_undefined + def fabricate_unspecified ## # If node has a default value we fabricate concrete node # with default value. # if @node.default.nil? - fabricate(Node::Null) + fabricate(Node::Undefined) else fabricate(@node, @node.default) end diff --git a/lib/gitlab/ci/config/node/global.rb b/lib/gitlab/ci/config/node/global.rb index ccd539fb003..2a2943c9288 100644 --- a/lib/gitlab/ci/config/node/global.rb +++ b/lib/gitlab/ci/config/node/global.rb @@ -36,15 +36,15 @@ module Gitlab helpers :before_script, :image, :services, :after_script, :variables, :stages, :types, :cache, :jobs - private - - def compose! - super - - compose_jobs! - compose_deprecated_entries! + def compose!(_deps = nil) + super(self) do + compose_jobs! + compose_deprecated_entries! + end end + private + def compose_jobs! factory = Node::Factory.new(Node::Jobs) .value(@config.except(*self.class.nodes.keys)) diff --git a/lib/gitlab/ci/config/node/hidden_job.rb b/lib/gitlab/ci/config/node/hidden.rb index 073044b66f8..fe4ee8a7fc6 100644 --- a/lib/gitlab/ci/config/node/hidden_job.rb +++ b/lib/gitlab/ci/config/node/hidden.rb @@ -5,11 +5,10 @@ module Gitlab ## # Entry that represents a hidden CI/CD job. # - class HiddenJob < Entry + class Hidden < Entry include Validatable validations do - validates :config, type: Hash validates :config, presence: true end diff --git a/lib/gitlab/ci/config/node/job.rb b/lib/gitlab/ci/config/node/job.rb index e84737acbb9..603334d6793 100644 --- a/lib/gitlab/ci/config/node/job.rb +++ b/lib/gitlab/ci/config/node/job.rb @@ -13,7 +13,7 @@ module Gitlab type stage when artifacts cache dependencies before_script after_script variables environment] - attributes :tags, :allow_failure, :when, :environment, :dependencies + attributes :tags, :allow_failure, :when, :dependencies validations do validates :config, allowed_keys: ALLOWED_KEYS @@ -29,58 +29,65 @@ module Gitlab inclusion: { in: %w[on_success on_failure always manual], message: 'should be on_success, on_failure, ' \ 'always or manual' } - validates :environment, - type: { - with: String, - message: Gitlab::Regex.environment_name_regex_message } - validates :environment, - format: { - with: Gitlab::Regex.environment_name_regex, - message: Gitlab::Regex.environment_name_regex_message } validates :dependencies, array_of_strings: true end end - node :before_script, Script, + node :before_script, Node::Script, description: 'Global before script overridden in this job.' - node :script, Commands, + node :script, Node::Commands, description: 'Commands that will be executed in this job.' - node :stage, Stage, + node :stage, Node::Stage, description: 'Pipeline stage this job will be executed into.' - node :type, Stage, + node :type, Node::Stage, description: 'Deprecated: stage this job will be executed into.' - node :after_script, Script, + node :after_script, Node::Script, description: 'Commands that will be executed when finishing job.' - node :cache, Cache, + node :cache, Node::Cache, description: 'Cache definition for this job.' - node :image, Image, + node :image, Node::Image, description: 'Image that will be used to execute this job.' - node :services, Services, + node :services, Node::Services, description: 'Services that will be used to execute this job.' - node :only, Trigger, + node :only, Node::Trigger, description: 'Refs policy this job will be executed for.' - node :except, Trigger, + node :except, Node::Trigger, description: 'Refs policy this job will be executed for.' - node :variables, Variables, + node :variables, Node::Variables, description: 'Environment variables available for this job.' - node :artifacts, Artifacts, + node :artifacts, Node::Artifacts, description: 'Artifacts configuration for this job.' + node :environment, Node::Environment, + description: 'Environment configuration for this job.' + helpers :before_script, :script, :stage, :type, :after_script, :cache, :image, :services, :only, :except, :variables, - :artifacts + :artifacts, :commands, :environment + + def compose!(deps = nil) + super do + if type_defined? && !stage_defined? + @entries[:stage] = @entries[:type] + end + + @entries.delete(:type) + end + + inherit!(deps) + end def name @metadata[:name] @@ -90,12 +97,30 @@ module Gitlab @config.merge(to_hash.compact) end + def commands + (before_script_value.to_a + script_value.to_a).join("\n") + end + private + def inherit!(deps) + return unless deps + + self.class.nodes.each_key do |key| + global_entry = deps[key] + job_entry = @entries[key] + + if global_entry.specified? && !job_entry.specified? + @entries[key] = global_entry + end + end + end + def to_hash { name: name, before_script: before_script, script: script, + commands: commands, image: image, services: services, stage: stage, @@ -103,19 +128,11 @@ module Gitlab only: only, except: except, variables: variables_defined? ? variables : nil, + environment: environment_defined? ? environment : nil, + environment_name: environment_defined? ? environment[:name] : nil, artifacts: artifacts, after_script: after_script } end - - def compose! - super - - if type_defined? && !stage_defined? - @entries[:stage] = @entries[:type] - end - - @entries.delete(:type) - end end end end diff --git a/lib/gitlab/ci/config/node/jobs.rb b/lib/gitlab/ci/config/node/jobs.rb index 51683c82ceb..d10e80d1a7d 100644 --- a/lib/gitlab/ci/config/node/jobs.rb +++ b/lib/gitlab/ci/config/node/jobs.rb @@ -26,19 +26,23 @@ module Gitlab name.to_s.start_with?('.') end - private - - def compose! - @config.each do |name, config| - node = hidden?(name) ? Node::HiddenJob : Node::Job - - factory = Node::Factory.new(node) - .value(config || {}) - .metadata(name: name) - .with(key: name, parent: self, - description: "#{name} job definition.") + def compose!(deps = nil) + super do + @config.each do |name, config| + node = hidden?(name) ? Node::Hidden : Node::Job + + factory = Node::Factory.new(node) + .value(config || {}) + .metadata(name: name) + .with(key: name, parent: self, + description: "#{name} job definition.") + + @entries[name] = factory.create! + end - @entries[name] = factory.create! + @entries.each_value do |entry| + entry.compose!(deps) + end end end end diff --git a/lib/gitlab/ci/config/node/null.rb b/lib/gitlab/ci/config/node/null.rb deleted file mode 100644 index 88a5f53f13c..00000000000 --- a/lib/gitlab/ci/config/node/null.rb +++ /dev/null @@ -1,34 +0,0 @@ -module Gitlab - module Ci - class Config - module Node - ## - # This class represents an undefined node. - # - # Implements the Null Object pattern. - # - class Null < Entry - def value - nil - end - - def valid? - true - end - - def errors - [] - end - - def specified? - false - end - - def relevant? - false - end - end - end - end - end -end diff --git a/lib/gitlab/ci/config/node/undefined.rb b/lib/gitlab/ci/config/node/undefined.rb index 45fef8c3ae5..33e78023539 100644 --- a/lib/gitlab/ci/config/node/undefined.rb +++ b/lib/gitlab/ci/config/node/undefined.rb @@ -3,15 +3,34 @@ module Gitlab class Config module Node ## - # This class represents an unspecified entry node. + # This class represents an undefined node. # - # It decorates original entry adding method that indicates it is - # unspecified. + # Implements the Null Object pattern. # - class Undefined < SimpleDelegator + class Undefined < Entry + def initialize(*) + super(nil) + end + + def value + nil + end + + def valid? + true + end + + def errors + [] + end + def specified? false end + + def relevant? + false + end end end end diff --git a/lib/gitlab/ci/config/node/unspecified.rb b/lib/gitlab/ci/config/node/unspecified.rb new file mode 100644 index 00000000000..a7d1f6131b8 --- /dev/null +++ b/lib/gitlab/ci/config/node/unspecified.rb @@ -0,0 +1,19 @@ +module Gitlab + module Ci + class Config + module Node + ## + # This class represents an unspecified entry node. + # + # It decorates original entry adding method that indicates it is + # unspecified. + # + class Unspecified < SimpleDelegator + def specified? + false + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline_duration.rb b/lib/gitlab/ci/pipeline_duration.rb new file mode 100644 index 00000000000..a210e76acaa --- /dev/null +++ b/lib/gitlab/ci/pipeline_duration.rb @@ -0,0 +1,141 @@ +module Gitlab + module Ci + # # Introduction - total running time + # + # The problem this module is trying to solve is finding the total running + # time amongst all the jobs, excluding retries and pending (queue) time. + # We could reduce this problem down to finding the union of periods. + # + # So each job would be represented as a `Period`, which consists of + # `Period#first` as when the job started and `Period#last` as when the + # job was finished. A simple example here would be: + # + # * A (1, 3) + # * B (2, 4) + # * C (6, 7) + # + # Here A begins from 1, and ends to 3. B begins from 2, and ends to 4. + # C begins from 6, and ends to 7. Visually it could be viewed as: + # + # 0 1 2 3 4 5 6 7 + # AAAAAAA + # BBBBBBB + # CCCC + # + # The union of A, B, and C would be (1, 4) and (6, 7), therefore the + # total running time should be: + # + # (4 - 1) + (7 - 6) => 4 + # + # # The Algorithm + # + # The algorithm used here for union would be described as follow. + # First we make sure that all periods are sorted by `Period#first`. + # Then we try to merge periods by iterating through the first period + # to the last period. The goal would be merging all overlapped periods + # so that in the end all the periods are discrete. When all periods + # are discrete, we're free to just sum all the periods to get real + # running time. + # + # Here we begin from A, and compare it to B. We could find that + # before A ends, B already started. That is `B.first <= A.last` + # that is `2 <= 3` which means A and B are overlapping! + # + # When we found that two periods are overlapping, we would need to merge + # them into a new period and disregard the old periods. To make a new + # period, we take `A.first` as the new first because remember? we sorted + # them, so `A.first` must be smaller or equal to `B.first`. And we take + # `[A.last, B.last].max` as the new last because we want whoever ended + # later. This could be broken into two cases: + # + # 0 1 2 3 4 + # AAAAAAA + # BBBBBBB + # + # Or: + # + # 0 1 2 3 4 + # AAAAAAAAAA + # BBBB + # + # So that we need to take whoever ends later. Back to our example, + # after merging and discard A and B it could be visually viewed as: + # + # 0 1 2 3 4 5 6 7 + # DDDDDDDDDD + # CCCC + # + # Now we could go on and compare the newly created D and the old C. + # We could figure out that D and C are not overlapping by checking + # `C.first <= D.last` is `false`. Therefore we need to keep both C + # and D. The example would end here because there are no more jobs. + # + # After having the union of all periods, we just need to sum the length + # of all periods to get total time. + # + # (4 - 1) + (7 - 6) => 4 + # + # That is 4 is the answer in the example. + module PipelineDuration + extend self + + Period = Struct.new(:first, :last) do + def duration + last - first + end + end + + def from_pipeline(pipeline) + status = %w[success failed running canceled] + builds = pipeline.builds.latest. + where(status: status).where.not(started_at: nil).order(:started_at) + + from_builds(builds) + end + + def from_builds(builds) + now = Time.now + + periods = builds.map do |b| + Period.new(b.started_at, b.finished_at || now) + end + + from_periods(periods) + end + + # periods should be sorted by `first` + def from_periods(periods) + process_duration(process_periods(periods)) + end + + private + + def process_periods(periods) + return periods if periods.empty? + + periods.drop(1).inject([periods.first]) do |result, current| + previous = result.last + + if overlap?(previous, current) + result[-1] = merge(previous, current) + result + else + result << current + end + end + end + + def overlap?(previous, current) + current.first <= previous.last + end + + def merge(previous, current) + Period.new(previous.first, [previous.last, current.last].max) + end + + def process_duration(periods) + periods.sum(&:duration) + end + end + end +end diff --git a/lib/gitlab/ci/trace_reader.rb b/lib/gitlab/ci/trace_reader.rb new file mode 100644 index 00000000000..37e51536e8f --- /dev/null +++ b/lib/gitlab/ci/trace_reader.rb @@ -0,0 +1,49 @@ +module Gitlab + module Ci + # This was inspired from: http://stackoverflow.com/a/10219411/1520132 + class TraceReader + BUFFER_SIZE = 4096 + + attr_accessor :path, :buffer_size + + def initialize(new_path, buffer_size: BUFFER_SIZE) + self.path = new_path + self.buffer_size = Integer(buffer_size) + end + + def read(last_lines: nil) + if last_lines + read_last_lines(last_lines) + else + File.read(path) + end + end + + def read_last_lines(max_lines) + File.open(path) do |file| + chunks = [] + pos = lines = 0 + max = file.size + + # We want an extra line to make sure fist line has full contents + while lines <= max_lines && pos < max + pos += buffer_size + + buf = if pos <= max + file.seek(-pos, IO::SEEK_END) + file.read(buffer_size) + else # Reached the head, read only left + file.seek(0) + file.read(buffer_size - (pos - max)) + end + + lines += buf.count("\n") + chunks.unshift(buf) + end + + chunks.join.lines.last(max_lines).join + end + end + end + end +end diff --git a/lib/gitlab/conflict/file.rb b/lib/gitlab/conflict/file.rb new file mode 100644 index 00000000000..c843315782d --- /dev/null +++ b/lib/gitlab/conflict/file.rb @@ -0,0 +1,245 @@ +module Gitlab + module Conflict + class File + include Gitlab::Routing.url_helpers + include IconsHelper + + class MissingResolution < ResolutionError + end + + CONTEXT_LINES = 3 + + attr_reader :merge_file_result, :their_path, :our_path, :our_mode, :merge_request, :repository + + def initialize(merge_file_result, conflict, merge_request:) + @merge_file_result = merge_file_result + @their_path = conflict[:theirs][:path] + @our_path = conflict[:ours][:path] + @our_mode = conflict[:ours][:mode] + @merge_request = merge_request + @repository = merge_request.project.repository + @match_line_headers = {} + end + + def content + merge_file_result[:data] + end + + def our_blob + @our_blob ||= repository.blob_at(merge_request.diff_refs.head_sha, our_path) + end + + def type + lines unless @type + + @type.inquiry + end + + # Array of Gitlab::Diff::Line objects + def lines + return @lines if defined?(@lines) + + begin + @type = 'text' + @lines = Gitlab::Conflict::Parser.new.parse(content, + our_path: our_path, + their_path: their_path, + parent_file: self) + rescue Gitlab::Conflict::Parser::ParserError + @type = 'text-editor' + @lines = nil + end + end + + def resolve_lines(resolution) + section_id = nil + + lines.map do |line| + unless line.type + section_id = nil + next line + end + + section_id ||= line_code(line) + + case resolution[section_id] + when 'head' + next unless line.type == 'new' + when 'origin' + next unless line.type == 'old' + else + raise MissingResolution, "Missing resolution for section ID: #{section_id}" + end + + line + end.compact + end + + def resolve_content(resolution) + if resolution == content + raise MissingResolution, "Resolved content has no changes for file #{our_path}" + end + + resolution + end + + def highlight_lines! + their_file = lines.reject { |line| line.type == 'new' }.map(&:text).join("\n") + our_file = lines.reject { |line| line.type == 'old' }.map(&:text).join("\n") + + their_highlight = Gitlab::Highlight.highlight(their_path, their_file, repository: repository).lines + our_highlight = Gitlab::Highlight.highlight(our_path, our_file, repository: repository).lines + + lines.each do |line| + if line.type == 'old' + line.rich_text = their_highlight[line.old_line - 1].try(:html_safe) + else + line.rich_text = our_highlight[line.new_line - 1].try(:html_safe) + end + end + end + + def sections + return @sections if @sections + + chunked_lines = lines.chunk { |line| line.type.nil? }.to_a + match_line = nil + + sections_count = chunked_lines.size + + @sections = chunked_lines.flat_map.with_index do |(no_conflict, lines), i| + section = nil + + # We need to reduce context sections to CONTEXT_LINES. Conflict sections are + # always shown in full. + if no_conflict + conflict_before = i > 0 + conflict_after = (sections_count - i) > 1 + + if conflict_before && conflict_after + # Create a gap in a long context section. + if lines.length > CONTEXT_LINES * 2 + head_lines = lines.first(CONTEXT_LINES) + tail_lines = lines.last(CONTEXT_LINES) + + # Ensure any existing match line has text for all lines up to the last + # line of its context. + update_match_line_text(match_line, head_lines.last) + + # Insert a new match line after the created gap. + match_line = create_match_line(tail_lines.first) + + section = [ + { conflict: false, lines: head_lines }, + { conflict: false, lines: tail_lines.unshift(match_line) } + ] + end + elsif conflict_after + tail_lines = lines.last(CONTEXT_LINES) + + # Create a gap and insert a match line at the start. + if lines.length > tail_lines.length + match_line = create_match_line(tail_lines.first) + + tail_lines.unshift(match_line) + end + + lines = tail_lines + elsif conflict_before + # We're at the end of the file (no conflicts after), so just remove extra + # trailing lines. + lines = lines.first(CONTEXT_LINES) + end + end + + # We want to update the match line's text every time unless we've already + # created a gap and its corresponding match line. + update_match_line_text(match_line, lines.last) unless section + + section ||= { conflict: !no_conflict, lines: lines } + section[:id] = line_code(lines.first) unless no_conflict + section + end + end + + def line_code(line) + Gitlab::Diff::LineCode.generate(our_path, line.new_pos, line.old_pos) + end + + def create_match_line(line) + Gitlab::Diff::Line.new('', 'match', line.index, line.old_pos, line.new_pos) + end + + # Any line beginning with a letter, an underscore, or a dollar can be used in a + # match line header. Only context sections can contain match lines, as match lines + # have to exist in both versions of the file. + def find_match_line_header(index) + return @match_line_headers[index] if @match_line_headers.key?(index) + + @match_line_headers[index] = begin + if index >= 0 + line = lines[index] + + if line.type.nil? && line.text.match(/\A[A-Za-z$_]/) + " #{line.text}" + else + find_match_line_header(index - 1) + end + end + end + end + + # Set the match line's text for the current line. A match line takes its start + # position and context header (where present) from itself, and its end position from + # the line passed in. + def update_match_line_text(match_line, line) + return unless match_line + + header = find_match_line_header(match_line.index - 1) + + match_line.text = "@@ -#{match_line.old_pos},#{line.old_pos} +#{match_line.new_pos},#{line.new_pos} @@#{header}" + end + + def as_json(opts = {}) + json_hash = { + old_path: their_path, + new_path: our_path, + blob_icon: file_type_icon_class('file', our_mode, our_path), + blob_path: namespace_project_blob_path(merge_request.project.namespace, + merge_request.project, + ::File.join(merge_request.diff_refs.head_sha, our_path)) + } + + json_hash.tap do |json_hash| + if opts[:full_content] + json_hash[:content] = content + json_hash[:blob_ace_mode] = our_blob && our_blob.language.try(:ace_mode) + else + json_hash[:sections] = sections if type.text? + json_hash[:type] = type + json_hash[:content_path] = content_path + end + end + end + + def content_path + conflict_for_path_namespace_project_merge_request_path(merge_request.project.namespace, + merge_request.project, + merge_request, + old_path: their_path, + new_path: our_path) + end + + # Don't try to print merge_request or repository. + def inspect + instance_variables = [:merge_file_result, :their_path, :our_path, :our_mode, :type].map do |instance_variable| + value = instance_variable_get("@#{instance_variable}") + + "#{instance_variable}=\"#{value}\"" + end + + "#<#{self.class} #{instance_variables.join(' ')}>" + end + end + end +end diff --git a/lib/gitlab/conflict/file_collection.rb b/lib/gitlab/conflict/file_collection.rb new file mode 100644 index 00000000000..fa5bd4649d4 --- /dev/null +++ b/lib/gitlab/conflict/file_collection.rb @@ -0,0 +1,61 @@ +module Gitlab + module Conflict + class FileCollection + class ConflictSideMissing < StandardError + end + + attr_reader :merge_request, :our_commit, :their_commit + + def initialize(merge_request) + @merge_request = merge_request + @our_commit = merge_request.source_branch_head.raw.raw_commit + @their_commit = merge_request.target_branch_head.raw.raw_commit + end + + def repository + merge_request.project.repository + end + + def merge_index + @merge_index ||= repository.rugged.merge_commits(our_commit, their_commit) + end + + def files + @files ||= merge_index.conflicts.map do |conflict| + raise ConflictSideMissing unless conflict[:theirs] && conflict[:ours] + + Gitlab::Conflict::File.new(merge_index.merge_file(conflict[:ours][:path]), + conflict, + merge_request: merge_request) + end + end + + def file_for_path(old_path, new_path) + files.find { |file| file.their_path == old_path && file.our_path == new_path } + end + + def as_json(opts = nil) + { + target_branch: merge_request.target_branch, + source_branch: merge_request.source_branch, + commit_sha: merge_request.diff_head_sha, + commit_message: default_commit_message, + files: files + } + end + + def default_commit_message + conflict_filenames = merge_index.conflicts.map do |conflict| + "# #{conflict[:ours][:path]}" + end + + <<EOM.chomp +Merge branch '#{merge_request.target_branch}' into '#{merge_request.source_branch}' + +# Conflicts: +#{conflict_filenames.join("\n")} +EOM + end + end + end +end diff --git a/lib/gitlab/conflict/parser.rb b/lib/gitlab/conflict/parser.rb new file mode 100644 index 00000000000..ddd657903fb --- /dev/null +++ b/lib/gitlab/conflict/parser.rb @@ -0,0 +1,76 @@ +module Gitlab + module Conflict + class Parser + class UnresolvableError < StandardError + end + + class UnmergeableFile < UnresolvableError + end + + class UnsupportedEncoding < UnresolvableError + end + + # Recoverable errors - the conflict can be resolved in an editor, but not with + # sections. + class ParserError < StandardError + end + + class UnexpectedDelimiter < ParserError + end + + class MissingEndDelimiter < ParserError + end + + def parse(text, our_path:, their_path:, parent_file: nil) + raise UnmergeableFile if text.blank? # Typically a binary file + raise UnmergeableFile if text.length > 200.kilobytes + + begin + text.to_json + rescue Encoding::UndefinedConversionError + raise UnsupportedEncoding + end + + line_obj_index = 0 + line_old = 1 + line_new = 1 + type = nil + lines = [] + conflict_start = "<<<<<<< #{our_path}" + conflict_middle = '=======' + conflict_end = ">>>>>>> #{their_path}" + + text.each_line.map do |line| + full_line = line.delete("\n") + + if full_line == conflict_start + raise UnexpectedDelimiter unless type.nil? + + type = 'new' + elsif full_line == conflict_middle + raise UnexpectedDelimiter unless type == 'new' + + type = 'old' + elsif full_line == conflict_end + raise UnexpectedDelimiter unless type == 'old' + + type = nil + elsif line[0] == '\\' + type = 'nonewline' + lines << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new, parent_file: parent_file) + else + lines << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new, parent_file: parent_file) + line_old += 1 if type != 'new' + line_new += 1 if type != 'old' + + line_obj_index += 1 + end + end + + raise MissingEndDelimiter unless type.nil? + + lines + end + end + end +end diff --git a/lib/gitlab/conflict/resolution_error.rb b/lib/gitlab/conflict/resolution_error.rb new file mode 100644 index 00000000000..a0f2006bc24 --- /dev/null +++ b/lib/gitlab/conflict/resolution_error.rb @@ -0,0 +1,6 @@ +module Gitlab + module Conflict + class ResolutionError < StandardError + end + end +end diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index 9dc2602867e..7e3d5647b39 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -1,47 +1,44 @@ module Gitlab class ContributionsCalendar - attr_reader :timestamps, :projects, :user + attr_reader :contributor + attr_reader :current_user + attr_reader :projects - def initialize(projects, user) - @projects = projects - @user = user + def initialize(contributor, current_user = nil) + @contributor = contributor + @current_user = current_user + @projects = ContributedProjectsFinder.new(contributor).execute(current_user) end - def timestamps - return @timestamps if @timestamps.present? + def activity_dates + return @activity_dates if @activity_dates.present? - @timestamps = {} + # Can't use Event.contributions here because we need to check 3 different + # project_features for the (currently) 3 different contribution types date_from = 1.year.ago + repo_events = event_counts(date_from, :repository). + having(action: Event::PUSHED) + issue_events = event_counts(date_from, :issues). + having(action: [Event::CREATED, Event::CLOSED], target_type: "Issue") + mr_events = event_counts(date_from, :merge_requests). + having(action: [Event::MERGED, Event::CREATED, Event::CLOSED], target_type: "MergeRequest") - events = Event.reorder(nil).contributions.where(author_id: user.id). - where("created_at > ?", date_from).where(project_id: projects). - group('date(created_at)'). - select('date(created_at) as date, count(id) as total_amount'). - map(&:attributes) + union = Gitlab::SQL::Union.new([repo_events, issue_events, mr_events]) + events = Event.find_by_sql(union.to_sql).map(&:attributes) - dates = (1.year.ago.to_date..Date.today).to_a - - dates.each do |date| - date_id = date.to_time.to_i.to_s - @timestamps[date_id] = 0 - day_events = events.find { |day_events| day_events["date"] == date } - - if day_events - @timestamps[date_id] = day_events["total_amount"] - end + @activity_events = events.each_with_object(Hash.new {|h, k| h[k] = 0 }) do |event, activities| + activities[event["date"]] += event["total_amount"] end - - @timestamps end def events_by_date(date) - events = Event.contributions.where(author_id: user.id). - where("created_at > ? AND created_at < ?", date.beginning_of_day, date.end_of_day). + events = Event.contributions.where(author_id: contributor.id). + where(created_at: date.beginning_of_day..date.end_of_day). where(project_id: projects) - events.select do |event| - event.push? || event.issue? || event.merge_request? - end + # Use visible_to_user? instead of the complicated logic in activity_dates + # because we're only viewing the events for a single day. + events.select {|event| event.visible_to_user?(current_user) } end def starting_year @@ -51,5 +48,30 @@ module Gitlab def starting_month Date.today.month end + + private + + def event_counts(date_from, feature) + t = Event.arel_table + + # re-running the contributed projects query in each union is expensive, so + # use IN(project_ids...) instead. It's the intersection of two users so + # the list will be (relatively) short + @contributed_project_ids ||= projects.uniq.pluck(:id) + authed_projects = Project.where(id: @contributed_project_ids). + with_feature_available_for_user(feature, current_user). + reorder(nil). + select(:id) + + conditions = t[:created_at].gteq(date_from.beginning_of_day). + and(t[:created_at].lteq(Date.today.end_of_day)). + and(t[:author_id].eq(contributor.id)) + + Event.reorder(nil). + select(t[:project_id], t[:target_type], t[:action], 'date(created_at) AS date', 'count(id) as total_amount'). + group(t[:project_id], t[:target_type], t[:action], 'date(created_at)'). + where(conditions). + having(t[:project_id].in(Arel::Nodes::SqlLiteral.new(authed_projects.to_sql))) + end end end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 735331df66c..ef9160d6437 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -30,6 +30,7 @@ module Gitlab signup_enabled: Settings.gitlab['signup_enabled'], signin_enabled: Settings.gitlab['signin_enabled'], gravatar_enabled: Settings.gravatar['enabled'], + koding_enabled: false, sign_in_text: nil, after_sign_up_text: nil, help_page_text: nil, @@ -40,7 +41,7 @@ module Gitlab default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'], default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'], domain_whitelist: Settings.gitlab['domain_whitelist'], - import_sources: %w[github bitbucket gitlab gitorious google_code fogbugz git gitlab_project], + import_sources: %w[github bitbucket gitlab google_code fogbugz git gitlab_project], shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], max_artifacts_size: Settings.artifacts['max_size'], require_two_factor_authentication: false, @@ -58,10 +59,8 @@ module Gitlab # When the DBMS is not available, an exception (e.g. PG::ConnectionBad) is raised active_db_connection = ActiveRecord::Base.connection.active? rescue false - ENV['USE_DB'] != 'false' && active_db_connection && - ActiveRecord::Base.connection.table_exists?('application_settings') - + ActiveRecord::Base.connection.table_exists?('application_settings') rescue ActiveRecord::NoDatabaseError false end diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb index 4f81863da35..d76aa38f741 100644 --- a/lib/gitlab/data_builder/push.rb +++ b/lib/gitlab/data_builder/push.rb @@ -83,7 +83,7 @@ module Gitlab tag = repository.find_tag(tag_name) if tag - commit = repository.commit(tag.target) + commit = repository.commit(tag.dereferenced_target) commit.try(:sha) end else diff --git a/lib/gitlab/database/date_time.rb b/lib/gitlab/database/date_time.rb new file mode 100644 index 00000000000..b6a89f715fd --- /dev/null +++ b/lib/gitlab/database/date_time.rb @@ -0,0 +1,27 @@ +module Gitlab + module Database + module DateTime + # Find the first of the `end_time_attrs` that isn't `NULL`. Subtract from it + # the first of the `start_time_attrs` that isn't NULL. `SELECT` the resulting interval + # along with an alias specified by the `as` parameter. + # + # Note: For MySQL, the interval is returned in seconds. + # For PostgreSQL, the interval is returned as an INTERVAL type. + def subtract_datetimes(query_so_far, end_time_attrs, start_time_attrs, as) + diff_fn = if Gitlab::Database.postgresql? + Arel::Nodes::Subtraction.new( + Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(end_time_attrs)), + Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(start_time_attrs))) + elsif Gitlab::Database.mysql? + Arel::Nodes::NamedFunction.new( + "TIMESTAMPDIFF", + [Arel.sql('second'), + Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(start_time_attrs)), + Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(end_time_attrs))]) + end + + query_so_far.project(diff_fn.as(as)) + end + end + end +end diff --git a/lib/gitlab/database/median.rb b/lib/gitlab/database/median.rb new file mode 100644 index 00000000000..1444d25ebc7 --- /dev/null +++ b/lib/gitlab/database/median.rb @@ -0,0 +1,112 @@ +# https://www.periscopedata.com/blog/medians-in-sql.html +module Gitlab + module Database + module Median + def median_datetime(arel_table, query_so_far, column_sym) + median_queries = + if Gitlab::Database.postgresql? + pg_median_datetime_sql(arel_table, query_so_far, column_sym) + elsif Gitlab::Database.mysql? + mysql_median_datetime_sql(arel_table, query_so_far, column_sym) + end + + results = Array.wrap(median_queries).map do |query| + ActiveRecord::Base.connection.execute(query) + end + extract_median(results).presence + end + + def extract_median(results) + result = results.compact.first + + if Gitlab::Database.postgresql? + result = result.first.presence + median = result['median'] if result + median.to_f if median + elsif Gitlab::Database.mysql? + result.to_a.flatten.first + end + end + + def mysql_median_datetime_sql(arel_table, query_so_far, column_sym) + query = arel_table. + from(arel_table.project(Arel.sql('*')).order(arel_table[column_sym]).as(arel_table.table_name)). + project(average([arel_table[column_sym]], 'median')). + where( + Arel::Nodes::Between.new( + Arel.sql("(select @row_id := @row_id + 1)"), + Arel::Nodes::And.new( + [Arel.sql('@ct/2.0'), + Arel.sql('@ct/2.0 + 1')] + ) + ) + ). + # Disallow negative values + where(arel_table[column_sym].gteq(0)) + + [ + Arel.sql("CREATE TEMPORARY TABLE IF NOT EXISTS #{query_so_far.to_sql}"), + Arel.sql("set @ct := (select count(1) from #{arel_table.table_name});"), + Arel.sql("set @row_id := 0;"), + query.to_sql, + Arel.sql("DROP TEMPORARY TABLE IF EXISTS #{arel_table.table_name};") + ] + end + + def pg_median_datetime_sql(arel_table, query_so_far, column_sym) + # Create a CTE with the column we're operating on, row number (after sorting by the column + # we're operating on), and count of the table we're operating on (duplicated across) all rows + # of the CTE. For example, if we're looking to find the median of the `projects.star_count` + # column, the CTE might look like this: + # + # star_count | row_id | ct + # ------------+--------+---- + # 5 | 1 | 3 + # 9 | 2 | 3 + # 15 | 3 | 3 + cte_table = Arel::Table.new("ordered_records") + cte = Arel::Nodes::As.new( + cte_table, + arel_table. + project( + arel_table[column_sym].as(column_sym.to_s), + Arel::Nodes::Over.new(Arel::Nodes::NamedFunction.new("row_number", []), + Arel::Nodes::Window.new.order(arel_table[column_sym])).as('row_id'), + arel_table.project("COUNT(1)").as('ct')). + # Disallow negative values + where(arel_table[column_sym].gteq(zero_interval))) + + # From the CTE, select either the middle row or the middle two rows (this is accomplished + # by 'where cte.row_id between cte.ct / 2.0 AND cte.ct / 2.0 + 1'). Find the average of the + # selected rows, and this is the median value. + cte_table.project(average([extract_epoch(cte_table[column_sym])], "median")). + where( + Arel::Nodes::Between.new( + cte_table[:row_id], + Arel::Nodes::And.new( + [(cte_table[:ct] / Arel.sql('2.0')), + (cte_table[:ct] / Arel.sql('2.0') + 1)] + ) + ) + ). + with(query_so_far, cte). + to_sql + end + + private + + def average(args, as) + Arel::Nodes::NamedFunction.new("AVG", args, as) + end + + def extract_epoch(arel_attribute) + Arel.sql(%Q{EXTRACT(EPOCH FROM "#{arel_attribute.relation.name}"."#{arel_attribute.name}")}) + end + + # Need to cast '0' to an INTERVAL before we can check if the interval is positive + def zero_interval + Arel::Nodes::NamedFunction.new("CAST", [Arel.sql("'0' AS INTERVAL")]) + end + end + end +end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 927f9dad20b..0bd6e148ba8 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -129,12 +129,14 @@ module Gitlab # column - The name of the column to add. # type - The column type (e.g. `:integer`). # default - The default value for the column. + # limit - Sets a column limit. For example, for :integer, the default is + # 4-bytes. Set `limit: 8` to allow 8-byte integers. # allow_null - When set to `true` the column will allow NULL values, the # default is to not allow NULL values. # # This method can also take a block which is passed directly to the # `update_column_in_batches` method. - def add_column_with_default(table, column, type, default:, allow_null: false, &block) + def add_column_with_default(table, column, type, default:, limit: nil, allow_null: false, &block) if transaction_open? raise 'add_column_with_default can not be run inside a transaction, ' \ 'you can disable transactions by calling disable_ddl_transaction! ' \ @@ -144,7 +146,11 @@ module Gitlab disable_statement_timeout transaction do - add_column(table, column, type, default: nil) + if limit + add_column(table, column, type, default: nil, limit: limit) + else + add_column(table, column, type, default: nil) + end # Changing the default before the update ensures any newly inserted # rows already use the proper default value. diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index e47df508ca2..ce85e5e0123 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -125,6 +125,10 @@ module Gitlab repository.blob_at(commit.id, file_path) end + + def cache_key + "#{file_path}-#{new_file}-#{deleted_file}-#{renamed_file}" + end end end end diff --git a/lib/gitlab/diff/file_collection/merge_request.rb b/lib/gitlab/diff/file_collection/merge_request_diff.rb index 4f946908e2f..dc4d47c878b 100644 --- a/lib/gitlab/diff/file_collection/merge_request.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff.rb @@ -1,14 +1,14 @@ module Gitlab module Diff module FileCollection - class MergeRequest < Base - def initialize(merge_request, diff_options:) - @merge_request = merge_request + class MergeRequestDiff < Base + def initialize(merge_request_diff, diff_options:) + @merge_request_diff = merge_request_diff - super(merge_request, - project: merge_request.project, + super(merge_request_diff, + project: merge_request_diff.project, diff_options: diff_options, - diff_refs: merge_request.diff_refs) + diff_refs: merge_request_diff.diff_refs) end def diff_files @@ -35,16 +35,16 @@ module Gitlab # for the highlighted ones, so we just skip their execution. # If the highlighted diff files lines are not cached we calculate and cache them. # - # The content of the cache is a Hash where the key correspond to the file_path and the values are Arrays of + # The content of the cache is a Hash where the key identifies the file and the values are Arrays of # hashes that represent serialized diff lines. # def cache_highlight!(diff_file) - file_path = diff_file.file_path + item_key = diff_file.cache_key - if highlight_cache[file_path] - highlight_diff_file_from_cache!(diff_file, highlight_cache[file_path]) + if highlight_cache[item_key] + highlight_diff_file_from_cache!(diff_file, highlight_cache[item_key]) else - highlight_cache[file_path] = diff_file.highlighted_diff_lines.map(&:to_hash) + highlight_cache[item_key] = diff_file.highlighted_diff_lines.map(&:to_hash) end end @@ -61,11 +61,11 @@ module Gitlab end def cacheable? - @merge_request.merge_request_diff.present? + @merge_request_diff.present? end def cache_key - [@merge_request.merge_request_diff, 'highlighted-diff-files', diff_options] + [@merge_request_diff, 'highlighted-diff-files', diff_options] end end end diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index cf097e0d0de..80a146b4a5a 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -2,11 +2,13 @@ module Gitlab module Diff class Line attr_reader :type, :index, :old_pos, :new_pos + attr_writer :rich_text attr_accessor :text - def initialize(text, type, index, old_pos, new_pos) + def initialize(text, type, index, old_pos, new_pos, parent_file: nil) @text, @type, @index = text, type, index @old_pos, @new_pos = old_pos, new_pos + @parent_file = parent_file end def self.init_from_hash(hash) @@ -43,9 +45,25 @@ module Gitlab type == 'old' end + def rich_text + @parent_file.highlight_lines! if @parent_file && !@rich_text + + @rich_text + end + def meta? type == 'match' || type == 'nonewline' end + + def as_json(opts = nil) + { + type: type, + old_line: old_line, + new_line: new_line, + text: text, + rich_text: rich_text || text + } + end end end end diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb index 2fdcf8d7838..ecf62dead35 100644 --- a/lib/gitlab/diff/position.rb +++ b/lib/gitlab/diff/position.rb @@ -139,13 +139,19 @@ module Gitlab private def find_diff_file(repository) - diffs = Gitlab::Git::Compare.new( - repository.raw_repository, - start_sha, - head_sha - ).diffs(paths: paths) + # We're at the initial commit, so just get that as we can't compare to anything. + if Gitlab::Git.blank_ref?(start_sha) + compare = Gitlab::Git::Commit.find(repository.raw_repository, head_sha) + else + compare = Gitlab::Git::Compare.new( + repository.raw_repository, + start_sha, + head_sha + ) + end + + diff = compare.diffs(paths: paths).first - diff = diffs.first return unless diff Gitlab::Diff::File.new(diff, repository: repository, diff_refs: diff_refs) diff --git a/lib/gitlab/downtime_check/message.rb b/lib/gitlab/downtime_check/message.rb index 4446e921e0d..40a4815a9a0 100644 --- a/lib/gitlab/downtime_check/message.rb +++ b/lib/gitlab/downtime_check/message.rb @@ -1,10 +1,10 @@ module Gitlab class DowntimeCheck class Message - attr_reader :path, :offline, :reason + attr_reader :path, :offline - OFFLINE = "\e[32moffline\e[0m" - ONLINE = "\e[31monline\e[0m" + OFFLINE = "\e[31moffline\e[0m" + ONLINE = "\e[32monline\e[0m" # path - The file path of the migration. # offline - When set to `true` the migration will require downtime. @@ -19,10 +19,21 @@ module Gitlab label = offline ? OFFLINE : ONLINE message = "[#{label}]: #{path}" - message += ": #{reason}" if reason + + if reason? + message += ":\n\n#{reason}\n\n" + end message end + + def reason? + @reason.present? + end + + def reason + @reason.strip.lines.map(&:strip).join("\n") + end end end end diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb new file mode 100644 index 00000000000..f4d1505ea91 --- /dev/null +++ b/lib/gitlab/ee_compat_check.rb @@ -0,0 +1,271 @@ +# rubocop: disable Rails/Output +module Gitlab + # Checks if a set of migrations requires downtime or not. + class EeCompatCheck + CE_REPO = 'https://gitlab.com/gitlab-org/gitlab-ce.git'.freeze + EE_REPO = 'https://gitlab.com/gitlab-org/gitlab-ee.git'.freeze + CHECK_DIR = Rails.root.join('ee_compat_check') + MAX_FETCH_DEPTH = 500 + IGNORED_FILES_REGEX = /(VERSION|CHANGELOG\.md:\d+)/.freeze + + attr_reader :repo_dir, :patches_dir, :ce_repo, :ce_branch + + def initialize(branch:, ce_repo: CE_REPO) + @repo_dir = CHECK_DIR.join('repo') + @patches_dir = CHECK_DIR.join('patches') + @ce_branch = branch + @ce_repo = ce_repo + end + + def check + ensure_ee_repo + ensure_patches_dir + + generate_patch(ce_branch, ce_patch_full_path) + + Dir.chdir(repo_dir) do + step("In the #{repo_dir} directory") + + status = catch(:halt_check) do + ce_branch_compat_check! + delete_ee_branch_locally! + ee_branch_presence_check! + ee_branch_compat_check! + end + + delete_ee_branch_locally! + + if status.nil? + true + else + false + end + end + end + + private + + def ensure_ee_repo + if Dir.exist?(repo_dir) + step("#{repo_dir} already exists") + else + cmd = %W[git clone --branch master --single-branch --depth 200 #{EE_REPO} #{repo_dir}] + step("Cloning #{EE_REPO} into #{repo_dir}", cmd) + end + end + + def ensure_patches_dir + FileUtils.mkdir_p(patches_dir) + end + + def generate_patch(branch, patch_path) + FileUtils.rm(patch_path, force: true) + + depth = 0 + loop do + depth += 50 + cmd = %W[git fetch --depth #{depth} origin --prune +refs/heads/master:refs/remotes/origin/master] + Gitlab::Popen.popen(cmd) + _, status = Gitlab::Popen.popen(%w[git merge-base FETCH_HEAD HEAD]) + + raise "#{branch} is too far behind master, please rebase it!" if depth >= MAX_FETCH_DEPTH + break if status.zero? + end + + step("Generating the patch against master in #{patch_path}") + output, status = Gitlab::Popen.popen(%w[git format-patch FETCH_HEAD --stdout]) + throw(:halt_check, :ko) unless status.zero? + + File.write(patch_path, output) + throw(:halt_check, :ko) unless File.exist?(patch_path) + end + + def ce_branch_compat_check! + if check_patch(ce_patch_full_path).zero? + puts applies_cleanly_msg(ce_branch) + throw(:halt_check) + end + end + + def ee_branch_presence_check! + status = step("Fetching origin/#{ee_branch}", %W[git fetch origin #{ee_branch}]) + + unless status.zero? + puts + puts ce_branch_doesnt_apply_cleanly_and_no_ee_branch_msg + + throw(:halt_check, :ko) + end + end + + def ee_branch_compat_check! + step("Checking out origin/#{ee_branch}", %W[git checkout -b #{ee_branch} FETCH_HEAD]) + + generate_patch(ee_branch, ee_patch_full_path) + + unless check_patch(ee_patch_full_path).zero? + puts + puts ee_branch_doesnt_apply_cleanly_msg + + throw(:halt_check, :ko) + end + + puts + puts applies_cleanly_msg(ee_branch) + end + + def check_patch(patch_path) + step("Checking out master", %w[git checkout master]) + step("Reseting to latest master", %w[git reset --hard origin/master]) + + step("Checking if #{patch_path} applies cleanly to EE/master") + output, status = Gitlab::Popen.popen(%W[git apply --check #{patch_path}]) + + unless status.zero? + failed_files = output.lines.reduce([]) do |memo, line| + if line.start_with?('error: patch failed:') + file = line.sub(/\Aerror: patch failed: /, '') + memo << file unless file =~ IGNORED_FILES_REGEX + end + memo + end + + if failed_files.empty? + status = 0 + else + puts "\nConflicting files:" + failed_files.each do |file| + puts " - #{file}" + end + end + end + + status + end + + def delete_ee_branch_locally! + command(%w[git checkout master]) + step("Deleting the local #{ee_branch} branch", %W[git branch -D #{ee_branch}]) + end + + def ce_patch_name + @ce_patch_name ||= "#{ce_branch}.patch" + end + + def ce_patch_full_path + @ce_patch_full_path ||= patches_dir.join(ce_patch_name) + end + + def ee_branch + @ee_branch ||= "#{ce_branch}-ee" + end + + def ee_patch_name + @ee_patch_name ||= "#{ee_branch}.patch" + end + + def ee_patch_full_path + @ee_patch_full_path ||= patches_dir.join(ee_patch_name) + end + + def step(desc, cmd = nil) + puts "\n=> #{desc}\n" + + if cmd + start = Time.now + puts "\n$ #{cmd.join(' ')}" + status = command(cmd) + puts "\nFinished in #{Time.now - start} seconds" + status + end + end + + def command(cmd) + output, status = Gitlab::Popen.popen(cmd) + puts output + + status + end + + def applies_cleanly_msg(branch) + <<-MSG.strip_heredoc + ================================================================= + 🎉 Congratulations!! 🎉 + + The #{branch} branch applies cleanly to EE/master! + + Much ❤️!! + =================================================================\n + MSG + end + + def ce_branch_doesnt_apply_cleanly_and_no_ee_branch_msg + <<-MSG.strip_heredoc + ================================================================= + 💥 Oh no! 💥 + + The #{ce_branch} branch does not apply cleanly to the current + EE/master, and no #{ee_branch} branch was found in the EE repository. + + Please create a #{ee_branch} branch that includes changes from + #{ce_branch} but also specific changes than can be applied cleanly + to EE/master. + + There are different ways to create such branch: + + 1. Create a new branch based on the CE branch and rebase it on top of EE/master + + # In the EE repo + $ git fetch #{ce_repo} #{ce_branch} + $ git checkout -b #{ee_branch} FETCH_HEAD + + # You can squash the #{ce_branch} commits into a single "Port of #{ce_branch} to EE" commit + # before rebasing to limit the conflicts-resolving steps during the rebase + $ git fetch origin + $ git rebase origin/master + + At this point you will likely have conflicts. + Solve them, and continue/finish the rebase. + + You can squash the #{ce_branch} commits into a single "Port of #{ce_branch} to EE". + + 2. Create a new branch from master and cherry-pick your CE commits + + # In the EE repo + $ git fetch origin + $ git checkout -b #{ee_branch} origin/master + $ git fetch #{ce_repo} #{ce_branch} + $ git cherry-pick SHA # Repeat for all the commits you want to pick + + You can squash the #{ce_branch} commits into a single "Port of #{ce_branch} to EE" commit. + + Don't forget to push your branch to #{EE_REPO}: + + # In the EE repo + $ git push origin #{ee_branch} + + You can then retry this failed build, and hopefully it should pass. + + Stay 💪 ! + =================================================================\n + MSG + end + + def ee_branch_doesnt_apply_cleanly_msg + <<-MSG.strip_heredoc + ================================================================= + 💥 Oh no! 💥 + + The #{ce_branch} does not apply cleanly to the current + EE/master, and even though a #{ee_branch} branch exists in the EE + repository, it does not apply cleanly either to EE/master! + + Please update the #{ee_branch}, push it again to #{EE_REPO}, and + retry this build. + + Stay 💪 ! + =================================================================\n + MSG + end + end +end diff --git a/lib/gitlab/email/handler/base_handler.rb b/lib/gitlab/email/handler/base_handler.rb index b7ed11cb638..7cccf465334 100644 --- a/lib/gitlab/email/handler/base_handler.rb +++ b/lib/gitlab/email/handler/base_handler.rb @@ -45,6 +45,7 @@ module Gitlab def verify_record!(record:, invalid_exception:, record_name:) return if record.persisted? + return if record.errors.key?(:commands_only) error_title = "The #{record_name} could not be created for the following reasons:" diff --git a/lib/gitlab/email/handler/create_issue_handler.rb b/lib/gitlab/email/handler/create_issue_handler.rb index 4e6566af8ab..9f90a3ec2b2 100644 --- a/lib/gitlab/email/handler/create_issue_handler.rb +++ b/lib/gitlab/email/handler/create_issue_handler.rb @@ -5,16 +5,16 @@ module Gitlab module Email module Handler class CreateIssueHandler < BaseHandler - attr_reader :project_path, :authentication_token + attr_reader :project_path, :incoming_email_token def initialize(mail, mail_key) super(mail, mail_key) - @project_path, @authentication_token = + @project_path, @incoming_email_token = mail_key && mail_key.split('+', 2) end def can_handle? - !authentication_token.nil? + !incoming_email_token.nil? end def execute @@ -29,7 +29,7 @@ module Gitlab end def author - @author ||= User.find_by(authentication_token: authentication_token) + @author ||= User.find_by(incoming_email_token: incoming_email_token) end def project diff --git a/lib/gitlab/email/handler/create_note_handler.rb b/lib/gitlab/email/handler/create_note_handler.rb index 06dae31cc27..447c7a6a6b9 100644 --- a/lib/gitlab/email/handler/create_note_handler.rb +++ b/lib/gitlab/email/handler/create_note_handler.rb @@ -46,7 +46,9 @@ module Gitlab noteable_type: sent_notification.noteable_type, noteable_id: sent_notification.noteable_id, commit_id: sent_notification.commit_id, - line_code: sent_notification.line_code + line_code: sent_notification.line_code, + position: sent_notification.position, + type: sent_notification.note_type ).execute end end diff --git a/lib/gitlab/emoji.rb b/lib/gitlab/emoji.rb index b63213ae208..bbbca8acc40 100644 --- a/lib/gitlab/emoji.rb +++ b/lib/gitlab/emoji.rb @@ -10,12 +10,20 @@ module Gitlab Gemojione.index.instance_variable_get(:@emoji_by_moji) end + def emojis_unicodes + emojis_by_moji.keys + end + def emojis_names - emojis.keys.sort + emojis.keys end def emoji_filename(name) emojis[name]["unicode"] end + + def emoji_unicode_filename(moji) + emojis_by_moji[moji]["unicode"] + end end end diff --git a/lib/gitlab/production_logger.rb b/lib/gitlab/environment_logger.rb index 89ce7144b1b..407cc572656 100644 --- a/lib/gitlab/production_logger.rb +++ b/lib/gitlab/environment_logger.rb @@ -1,7 +1,7 @@ module Gitlab - class ProductionLogger < Gitlab::Logger + class EnvironmentLogger < Gitlab::Logger def self.file_name_noext - 'production' + Rails.env end end end diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index ffe49364379..2dd42704396 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -1,59 +1,52 @@ +require 'securerandom' + module Gitlab # This class implements an 'exclusive lease'. We call it a 'lease' # because it has a set expiry time. We call it 'exclusive' because only # one caller may obtain a lease for a given key at a time. The # implementation is intended to work across GitLab processes and across - # servers. It is a 'cheap' alternative to using SQL queries and updates: + # servers. It is a cheap alternative to using SQL queries and updates: # you do not need to change the SQL schema to start using # ExclusiveLease. # - # It is important to choose the timeout wisely. If the timeout is very - # high (1 hour) then the throughput of your operation gets very low (at - # most once an hour). If the timeout is lower than how long your - # operation may take then you cannot count on exclusivity. For example, - # if the timeout is 10 seconds and you do an operation which may take 20 - # seconds then two overlapping operations may hold a lease for the same - # key at the same time. - # - # This class has no 'cancel' method. I originally decided against adding - # it because it would add complexity and a false sense of security. The - # complexity: instead of setting '1' we would have to set a UUID, and to - # delete it we would have to execute Lua on the Redis server to only - # delete the key if the value was our own UUID. Otherwise there is a - # chance that when you intend to cancel your lease you actually delete - # someone else's. The false sense of security: you cannot design your - # system to rely too much on the lease being cancelled after use because - # the calling (Ruby) process may crash or be killed. You _cannot_ count - # on begin/ensure blocks to cancel a lease, because the 'ensure' does - # not always run. Think of 'kill -9' from the Unicorn master for - # instance. - # - # If you find that leases are getting in your way, ask yourself: would - # it be enough to lower the lease timeout? Another thing that might be - # appropriate is to only use a lease for bulk/automated operations, and - # to ignore the lease when you get a single 'manual' user request (a - # button click). - # class ExclusiveLease + LUA_CANCEL_SCRIPT = <<-EOS + local key, uuid = KEYS[1], ARGV[1] + if redis.call("get", key) == uuid then + redis.call("del", key) + end + EOS + + def self.cancel(key, uuid) + Gitlab::Redis.with do |redis| + redis.eval(LUA_CANCEL_SCRIPT, keys: [redis_key(key)], argv: [uuid]) + end + end + + def self.redis_key(key) + "gitlab:exclusive_lease:#{key}" + end + def initialize(key, timeout:) - @key, @timeout = key, timeout + @redis_key = self.class.redis_key(key) + @timeout = timeout + @uuid = SecureRandom.uuid end - # Try to obtain the lease. Return true on success, + # Try to obtain the lease. Return lease UUID on success, # false if the lease is already taken. def try_obtain # Performing a single SET is atomic Gitlab::Redis.with do |redis| - !!redis.set(redis_key, '1', nx: true, ex: @timeout) + redis.set(@redis_key, @uuid, nx: true, ex: @timeout) && @uuid end end - # No #cancel method. See comments above! - - private - - def redis_key - "gitlab:exclusive_lease:#{@key}" + # Returns true if the key for this lease is set. + def exists? + Gitlab::Redis.with do |redis| + redis.exists(@redis_key) + end end end end diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb index 501d5a95547..222bcdcbf9c 100644 --- a/lib/gitlab/fogbugz_import/importer.rb +++ b/lib/gitlab/fogbugz_import/importer.rb @@ -74,8 +74,8 @@ module Gitlab end def create_label(name) - color = nice_label_color(name) - Label.create!(project_id: project.id, title: name, color: color) + params = { title: name, color: nice_label_color(name) } + ::Labels::FindOrCreateService.new(nil, project, params).execute(skip_authorization: true) end def user_info(person_id) @@ -122,25 +122,21 @@ module Gitlab author_id = user_info(bug['ixPersonOpenedBy'])[:gitlab_id] || project.creator_id issue = Issue.create!( - project_id: project.id, - title: bug['sTitle'], - description: body, - author_id: author_id, - assignee_id: assignee_id, - state: bug['fOpen'] == 'true' ? 'opened' : 'closed' + iid: bug['ixBug'], + project_id: project.id, + title: bug['sTitle'], + description: body, + author_id: author_id, + assignee_id: assignee_id, + state: bug['fOpen'] == 'true' ? 'opened' : 'closed', + created_at: date, + updated_at: DateTime.parse(bug['dtLastUpdated']) ) - issue.add_labels_by_names(labels) - if issue.iid != bug['ixBug'] - issue.update_attribute(:iid, bug['ixBug']) - end + issue_labels = ::LabelsFinder.new(nil, project_id: project.id, title: labels).execute(skip_authorization: true) + issue.update_attribute(:label_ids, issue_labels.pluck(:id)) import_issue_comments(issue, comments) - - issue.update_attribute(:created_at, date) - - last_update = DateTime.parse(bug['dtLastUpdated']) - issue.update_attribute(:updated_at, last_update) end end diff --git a/lib/gitlab/gfm/reference_rewriter.rb b/lib/gitlab/gfm/reference_rewriter.rb index 78d7a4f27cf..a7c596dced0 100644 --- a/lib/gitlab/gfm/reference_rewriter.rb +++ b/lib/gitlab/gfm/reference_rewriter.rb @@ -58,7 +58,7 @@ module Gitlab referable = find_referable(reference) return reference unless referable - cross_reference = referable.to_reference(target_project) + cross_reference = build_cross_reference(referable, target_project) return reference if reference == cross_reference new_text = before + cross_reference + after @@ -72,6 +72,14 @@ module Gitlab extractor.all.first end + def build_cross_reference(referable, target_project) + if referable.respond_to?(:project) + referable.to_reference(target_project) + else + referable.to_reference(@source_project, target_project) + end + end + def substitution_valid?(substituted) @original_html == markdown(substituted) end diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 7584efe4fa8..3cd515e4a3a 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -18,6 +18,16 @@ module Gitlab end end + def committer_hash(email:, name:) + return if email.nil? || name.nil? + + { + email: email, + name: name, + time: Time.now + } + end + def tag_name(ref) ref = ref.to_s if self.tag_ref?(ref) diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb index 9b681e636c7..bd90d24a2ec 100644 --- a/lib/gitlab/git/hook.rb +++ b/lib/gitlab/git/hook.rb @@ -17,11 +17,13 @@ module Gitlab def trigger(gl_id, oldrev, newrev, ref) return [true, nil] unless exists? - case name - when "pre-receive", "post-receive" - call_receive_hook(gl_id, oldrev, newrev, ref) - when "update" - call_update_hook(gl_id, oldrev, newrev, ref) + Bundler.with_clean_env do + case name + when "pre-receive", "post-receive" + call_receive_hook(gl_id, oldrev, newrev, ref) + when "update" + call_update_hook(gl_id, oldrev, newrev, ref) + end end end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index f7432df5557..64b5c4b98dc 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -2,50 +2,52 @@ # class return an instance of `GitlabAccessStatus` module Gitlab class GitAccess + UnauthorizedError = Class.new(StandardError) + + ERROR_MESSAGES = { + upload: 'You are not allowed to upload code for this project.', + download: 'You are not allowed to download code from this project.', + deploy_key: 'Deploy keys are not allowed to push code.', + no_repo: 'A repository for this project does not exist yet.' + } + DOWNLOAD_COMMANDS = %w{ git-upload-pack git-upload-archive } PUSH_COMMANDS = %w{ git-receive-pack } + ALL_COMMANDS = DOWNLOAD_COMMANDS + PUSH_COMMANDS - attr_reader :actor, :project, :protocol, :user_access + attr_reader :actor, :project, :protocol, :user_access, :authentication_abilities - def initialize(actor, project, protocol) + def initialize(actor, project, protocol, authentication_abilities:) @actor = actor @project = project @protocol = protocol + @authentication_abilities = authentication_abilities @user_access = UserAccess.new(user, project: project) end def check(cmd, changes) - return build_status_object(false, "Git access over #{protocol.upcase} is not allowed") unless protocol_allowed? - - unless actor - return build_status_object(false, "No user or key was provided.") - end - - if user && !user_access.allowed? - return build_status_object(false, "Your account has been blocked.") - end - - unless project && (user_access.can_read_project? || deploy_key_can_read_project?) - return build_status_object(false, 'The project you were looking for could not be found.') - end + check_protocol! + check_active_user! + check_project_accessibility! + check_command_existence!(cmd) case cmd when *DOWNLOAD_COMMANDS download_access_check when *PUSH_COMMANDS push_access_check(changes) - else - build_status_object(false, "The command you're trying to execute is not allowed.") end + + build_status_object(true) + rescue UnauthorizedError => ex + build_status_object(false, ex.message) end def download_access_check if user user_download_access_check - elsif deploy_key - build_status_object(true) - else - raise 'Wrong actor' + elsif deploy_key.nil? && !Guest.can?(:download_code, project) + raise UnauthorizedError, ERROR_MESSAGES[:download] end end @@ -59,21 +61,35 @@ module Gitlab elsif deploy_key deploy_key_push_access_check(changes) else - raise 'Wrong actor' + raise UnauthorizedError, ERROR_MESSAGES[deploy_key ? :deploy_key : :upload] end end def user_download_access_check - unless user_access.can_do_action?(:download_code) - return build_status_object(false, "You are not allowed to download code from this project.") + unless user_can_download_code? || build_can_download_code? + raise UnauthorizedError, ERROR_MESSAGES[:download] end + end - build_status_object(true) + def user_can_download_code? + authentication_abilities.include?(:download_code) && user_access.can_do_action?(:download_code) + end + + def build_can_download_code? + authentication_abilities.include?(:build_download_code) && user_access.can_do_action?(:build_download_code) end def user_push_access_check(changes) + unless authentication_abilities.include?(:push_code) + raise UnauthorizedError, ERROR_MESSAGES[:upload] + end + if changes.blank? - return build_status_object(true) + return # Allow access. + end + + unless project.repository.exists? + raise UnauthorizedError, ERROR_MESSAGES[:no_repo] end changes_list = Gitlab::ChangesList.new(changes) @@ -83,11 +99,9 @@ module Gitlab status = change_access_check(change) unless status.allowed? # If user does not have access to make at least one change - cancel all push - return status + raise UnauthorizedError, status.message end end - - build_status_object(true) end def deploy_key_push_access_check(changes) @@ -108,6 +122,30 @@ module Gitlab private + def check_protocol! + unless protocol_allowed? + raise UnauthorizedError, "Git access over #{protocol.upcase} is not allowed" + end + end + + def check_active_user! + if user && !user_access.allowed? + raise UnauthorizedError, "Your account has been blocked." + end + end + + def check_project_accessibility! + if project.blank? || !can_read_project? + raise UnauthorizedError, 'The project you were looking for could not be found.' + end + end + + def check_command_existence!(cmd) + unless ALL_COMMANDS.include?(cmd) + raise UnauthorizedError, "The command you're trying to execute is not allowed." + end + end + def matching_merge_request?(newrev, branch_name) Checks::MatchingMergeRequest.new(newrev, branch_name, project).match? end @@ -125,6 +163,16 @@ module Gitlab end end + def can_read_project? + if user + user_access.can_read_project? + elsif deploy_key + deploy_key_can_read_project? + else + Guest.can?(:read_project, project) + end + end + protected def user diff --git a/lib/gitlab/github_import/base_formatter.rb b/lib/gitlab/github_import/base_formatter.rb index 72992baffd4..6dbae64a9fe 100644 --- a/lib/gitlab/github_import/base_formatter.rb +++ b/lib/gitlab/github_import/base_formatter.rb @@ -10,16 +10,23 @@ module Gitlab end def create! - self.klass.create!(self.attributes) + project.public_send(project_association).find_or_create_by!(find_condition) do |record| + record.attributes = attributes + end end private - def gl_user_id(github_id) + def gitlab_user_id(github_id) User.joins(:identities). find_by("identities.extern_uid = ? AND identities.provider = 'github'", github_id.to_s). try(:id) end + + def gitlab_author_id + return @gitlab_author_id if defined?(@gitlab_author_id) + @gitlab_author_id = gitlab_user_id(raw_data.user.id) + end end end end diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 084e514492c..85df6547a67 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -52,7 +52,7 @@ module Gitlab def method_missing(method, *args, &block) if api.respond_to?(method) - request { api.send(method, *args, &block) } + request(method, *args, &block) else super(method, *args, &block) end @@ -99,20 +99,31 @@ module Gitlab rate_limit.resets_in + GITHUB_SAFE_SLEEP_TIME end - def request + def request(method, *args, &block) sleep rate_limit_sleep_time if rate_limit_exceed? - data = yield + data = api.send(method, *args) + return data unless data.is_a?(Array) last_response = api.last_response + if block_given? + yield data + # api.last_response could change while we're yielding (e.g. fetching labels for each PR) + # so we cache our own last response + each_response_page(last_response, &block) + else + each_response_page(last_response) { |page| data.concat(page) } + data + end + end + + def each_response_page(last_response) while last_response.rels[:next] sleep rate_limit_sleep_time if rate_limit_exceed? last_response = last_response.rels[:next].get - data.concat(last_response.data) if last_response.data.is_a?(Array) + yield last_response.data if last_response.data.is_a?(Array) end - - data end end end diff --git a/lib/gitlab/github_import/comment_formatter.rb b/lib/gitlab/github_import/comment_formatter.rb index 2c1b94ef2cd..2bddcde2b7c 100644 --- a/lib/gitlab/github_import/comment_formatter.rb +++ b/lib/gitlab/github_import/comment_formatter.rb @@ -21,7 +21,7 @@ module Gitlab end def author_id - gl_user_id(raw_data.user.id) || project.creator_id + gitlab_author_id || project.creator_id end def body @@ -52,7 +52,11 @@ module Gitlab end def note - formatter.author_line(author) + body + if gitlab_author_id + body + else + formatter.author_line(author) + body + end end def type diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 9ddc8905bd6..90cf38a8513 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -3,12 +3,14 @@ module Gitlab class Importer include Gitlab::ShellAdapter - attr_reader :client, :project, :repo, :repo_url + attr_reader :client, :errors, :project, :repo, :repo_url def initialize(project) @project = project @repo = project.import_source @repo_url = project.import_url + @errors = [] + @labels = {} if credentials @client = Client.new(credentials[:user]) @@ -18,8 +20,17 @@ module Gitlab end def execute - import_labels && import_milestones && import_issues && - import_pull_requests && import_wiki + import_labels + import_milestones + import_issues + import_pull_requests + import_comments(:issues) + import_comments(:pull_requests) + import_wiki + import_releases + handle_errors + + true end private @@ -28,63 +39,79 @@ module Gitlab @credentials ||= project.import_data.credentials if project.import_data end + def handle_errors + return unless errors.any? + + project.update_column(:import_error, { + message: 'The remote data could not be fully imported.', + errors: errors + }.to_json) + end + def import_labels - labels = client.labels(repo, per_page: 100) - labels.each { |raw| LabelFormatter.new(project, raw).create! } + fetch_resources(:labels, repo, per_page: 100) do |labels| + labels.each do |raw| + begin + LabelFormatter.new(project, raw).create! + rescue => e + errors << { type: :label, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + end + end + end - true - rescue ActiveRecord::RecordInvalid => e - raise Projects::ImportService::Error, e.message + cache_labels! end def import_milestones - milestones = client.milestones(repo, state: :all, per_page: 100) - milestones.each { |raw| MilestoneFormatter.new(project, raw).create! } - - true - rescue ActiveRecord::RecordInvalid => e - raise Projects::ImportService::Error, e.message + fetch_resources(:milestones, repo, state: :all, per_page: 100) do |milestones| + milestones.each do |raw| + begin + MilestoneFormatter.new(project, raw).create! + rescue => e + errors << { type: :milestone, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + end + end + end end def import_issues - issues = client.issues(repo, state: :all, sort: :created, direction: :asc, per_page: 100) + fetch_resources(:issues, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |issues| + issues.each do |raw| + gh_issue = IssueFormatter.new(project, raw) - issues.each do |raw| - gh_issue = IssueFormatter.new(project, raw) - - if gh_issue.valid? - issue = gh_issue.create! - apply_labels(issue) - import_comments(issue) if gh_issue.has_comments? + if gh_issue.valid? + begin + issue = gh_issue.create! + apply_labels(issue, raw) + rescue => e + errors << { type: :issue, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + end + end end end - - true - rescue ActiveRecord::RecordInvalid => e - raise Projects::ImportService::Error, e.message end def import_pull_requests - pull_requests = client.pull_requests(repo, state: :all, sort: :created, direction: :asc, per_page: 100) - pull_requests = pull_requests.map { |raw| PullRequestFormatter.new(project, raw) }.select(&:valid?) - - pull_requests.each do |pull_request| - begin - restore_source_branch(pull_request) unless pull_request.source_branch_exists? - restore_target_branch(pull_request) unless pull_request.target_branch_exists? - - merge_request = pull_request.create! - apply_labels(merge_request) - import_comments(merge_request) - import_comments_on_diff(merge_request) - rescue ActiveRecord::RecordInvalid => e - raise Projects::ImportService::Error, e.message - ensure - clean_up_restored_branches(pull_request) + fetch_resources(:pull_requests, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |pull_requests| + pull_requests.each do |raw| + pull_request = PullRequestFormatter.new(project, raw) + next unless pull_request.valid? + + begin + restore_source_branch(pull_request) unless pull_request.source_branch_exists? + restore_target_branch(pull_request) unless pull_request.target_branch_exists? + + merge_request = pull_request.create! + apply_labels(merge_request, raw) + rescue => e + errors << { type: :pull_request, url: Gitlab::UrlSanitizer.sanitize(pull_request.url), errors: e.message } + ensure + clean_up_restored_branches(pull_request) + end end end - true + project.repository.after_remove_branch end def restore_source_branch(pull_request) @@ -98,63 +125,162 @@ module Gitlab def remove_branch(name) project.repository.delete_branch(name) rescue Rugged::ReferenceError - nil + errors << { type: :remove_branch, name: name } end def clean_up_restored_branches(pull_request) remove_branch(pull_request.source_branch_name) unless pull_request.source_branch_exists? remove_branch(pull_request.target_branch_name) unless pull_request.target_branch_exists? - - project.repository.after_remove_branch end - def apply_labels(issuable) - issue = client.issue(repo, issuable.iid) + def apply_labels(issuable, raw_issuable) + # GH returns labels for issues but not for pull requests! + labels = if issuable.is_a?(MergeRequest) + client.labels_for_issue(repo, raw_issuable.number) + else + raw_issuable.labels + end - if issue.labels.count > 0 - label_ids = issue.labels.map do |raw| - Label.find_by(LabelFormatter.new(project, raw).attributes).try(:id) - end + if labels.count > 0 + label_ids = labels + .map { |attrs| @labels[attrs.name] } + .compact issuable.update_attribute(:label_ids, label_ids) end end - def import_comments(issuable) - comments = client.issue_comments(repo, issuable.iid, per_page: 100) - create_comments(issuable, comments) + def import_comments(issuable_type) + resource_type = "#{issuable_type}_comments".to_sym + + # Two notes here: + # 1. We don't have a distinctive attribute for comments (unlike issues iid), so we fetch the last inserted note, + # compare it against every comment in the current imported page until we find match, and that's where start importing + # 2. GH returns comments for _both_ issues and PRs through issues_comments API, while pull_requests_comments returns + # only comments on diffs, so select last note not based on noteable_type but on line_code + line_code_is = issuable_type == :pull_requests ? 'NOT NULL' : 'NULL' + last_note = project.notes.where("line_code IS #{line_code_is}").last + + fetch_resources(resource_type, repo, per_page: 100) do |comments| + if last_note + discard_inserted_comments(comments, last_note) + last_note = nil + end + + create_comments(comments) + end end - def import_comments_on_diff(merge_request) - comments = client.pull_request_comments(repo, merge_request.iid, per_page: 100) - create_comments(merge_request, comments) + def create_comments(comments) + ActiveRecord::Base.no_touching do + comments.each do |raw| + begin + comment = CommentFormatter.new(project, raw) + # GH does not return info about comment's parent, so we guess it by checking its URL! + *_, parent, iid = URI(raw.html_url).path.split('/') + issuable_class = parent == 'issues' ? Issue : MergeRequest + issuable = issuable_class.find_by_iid(iid) + next unless issuable + + issuable.notes.create!(comment.attributes) + rescue => e + errors << { type: :comment, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + end + end + end end - def create_comments(issuable, comments) - comments.each do |raw| - comment = CommentFormatter.new(project, raw) - issuable.notes.create!(comment.attributes) + def discard_inserted_comments(comments, last_note) + last_note_attrs = nil + + cut_off_index = comments.find_index do |raw| + comment = CommentFormatter.new(project, raw) + comment_attrs = comment.attributes + last_note_attrs ||= last_note.slice(*comment_attrs.keys) + + comment_attrs.with_indifferent_access == last_note_attrs end + + # No matching resource in the collection, which means we got halted right on the end of the last page, so all good + return unless cut_off_index + + # Otherwise, remove the resources we've already inserted + comments.shift(cut_off_index + 1) end def import_wiki - unless project.wiki_enabled? + unless project.wiki.repository_exists? wiki = WikiFormatter.new(project) gitlab_shell.import_repository(project.repository_storage_path, wiki.path_with_namespace, wiki.import_url) - project.update_attribute(:wiki_enabled, true) end - - true rescue Gitlab::Shell::Error => e # GitHub error message when the wiki repo has not been created, # this means that repo has wiki enabled, but have no pages. So, # we can skip the import. if e.message !~ /repository not exported/ - raise Projects::ImportService::Error, e.message - else - true + errors << { type: :wiki, errors: e.message } + end + end + + def import_releases + fetch_resources(:releases, repo, per_page: 100) do |releases| + releases.each do |raw| + begin + gh_release = ReleaseFormatter.new(project, raw) + gh_release.create! if gh_release.valid? + rescue => e + errors << { type: :release, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + end + end + end + end + + def cache_labels! + project.labels.select(:id, :title).find_each do |label| + @labels[label.title] = label.id end end + + def fetch_resources(resource_type, *opts) + return if imported?(resource_type) + + opts.last.merge!(page: current_page(resource_type)) + + client.public_send(resource_type, *opts) do |resources| + yield resources + increment_page(resource_type) + end + + imported!(resource_type) + end + + def imported?(resource_type) + Rails.cache.read("#{cache_key_prefix}:#{resource_type}:imported") + end + + def imported!(resource_type) + Rails.cache.write("#{cache_key_prefix}:#{resource_type}:imported", true, ex: 1.day) + end + + def increment_page(resource_type) + key = "#{cache_key_prefix}:#{resource_type}:current-page" + + # Rails.cache.increment calls INCRBY directly on the value stored under the key, which is + # a serialized ActiveSupport::Cache::Entry, so it will return an error by Redis, hence this ugly work-around + page = Rails.cache.read(key) + page += 1 + Rails.cache.write(key, page) + + page + end + + def current_page(resource_type) + Rails.cache.fetch("#{cache_key_prefix}:#{resource_type}:current-page", ex: 1.day) { 1 } + end + + def cache_key_prefix + @cache_key_prefix ||= "github-import:#{project.id}" + end end end end diff --git a/lib/gitlab/github_import/issue_formatter.rb b/lib/gitlab/github_import/issue_formatter.rb index 835ec858b35..8c32ac59fc5 100644 --- a/lib/gitlab/github_import/issue_formatter.rb +++ b/lib/gitlab/github_import/issue_formatter.rb @@ -12,7 +12,7 @@ module Gitlab author_id: author_id, assignee_id: assignee_id, created_at: raw_data.created_at, - updated_at: updated_at + updated_at: raw_data.updated_at } end @@ -20,8 +20,12 @@ module Gitlab raw_data.comments > 0 end - def klass - Issue + def project_association + :issues + end + + def find_condition + { iid: number } end def number @@ -40,7 +44,7 @@ module Gitlab def assignee_id if assigned? - gl_user_id(raw_data.assignee.id) + gitlab_user_id(raw_data.assignee.id) end end @@ -49,7 +53,7 @@ module Gitlab end def author_id - gl_user_id(raw_data.user.id) || project.creator_id + gitlab_author_id || project.creator_id end def body @@ -57,7 +61,11 @@ module Gitlab end def description - @formatter.author_line(author) + body + if gitlab_author_id + body + else + formatter.author_line(author) + body + end end def milestone @@ -69,10 +77,6 @@ module Gitlab def state raw_data.state == 'closed' ? 'closed' : 'opened' end - - def updated_at - state == 'closed' ? raw_data.closed_at : raw_data.updated_at - end end end end diff --git a/lib/gitlab/github_import/label_formatter.rb b/lib/gitlab/github_import/label_formatter.rb index 9f18244e7d7..211ccdc51bb 100644 --- a/lib/gitlab/github_import/label_formatter.rb +++ b/lib/gitlab/github_import/label_formatter.rb @@ -9,8 +9,18 @@ module Gitlab } end - def klass - Label + def project_association + :labels + end + + def create! + params = attributes.except(:project) + service = ::Labels::FindOrCreateService.new(nil, project, params) + label = service.execute(skip_authorization: true) + + raise ActiveRecord::RecordInvalid.new(label) unless label.persisted? + + label end private diff --git a/lib/gitlab/github_import/milestone_formatter.rb b/lib/gitlab/github_import/milestone_formatter.rb index 53d4b3102d1..401dd962521 100644 --- a/lib/gitlab/github_import/milestone_formatter.rb +++ b/lib/gitlab/github_import/milestone_formatter.rb @@ -3,50 +3,30 @@ module Gitlab class MilestoneFormatter < BaseFormatter def attributes { - iid: number, + iid: raw_data.number, project: project, - title: title, - description: description, - due_date: due_date, + title: raw_data.title, + description: raw_data.description, + due_date: raw_data.due_on, state: state, - created_at: created_at, - updated_at: updated_at + created_at: raw_data.created_at, + updated_at: raw_data.updated_at } end - def klass - Milestone + def project_association + :milestones end - private - - def number - raw_data.number - end - - def title - raw_data.title - end - - def description - raw_data.description + def find_condition + { iid: raw_data.number } end - def due_date - raw_data.due_on - end + private def state raw_data.state == 'closed' ? 'closed' : 'active' end - - def created_at - raw_data.created_at - end - - def updated_at - state == 'closed' ? raw_data.closed_at : raw_data.updated_at - end end end end diff --git a/lib/gitlab/github_import/project_creator.rb b/lib/gitlab/github_import/project_creator.rb index f4221003db5..a2410068845 100644 --- a/lib/gitlab/github_import/project_creator.rb +++ b/lib/gitlab/github_import/project_creator.rb @@ -1,10 +1,11 @@ module Gitlab module GithubImport class ProjectCreator - attr_reader :repo, :namespace, :current_user, :session_data + attr_reader :repo, :name, :namespace, :current_user, :session_data - def initialize(repo, namespace, current_user, session_data) + def initialize(repo, name, namespace, current_user, session_data) @repo = repo + @name = name @namespace = namespace @current_user = current_user @session_data = session_data @@ -13,17 +14,36 @@ module Gitlab def execute ::Projects::CreateService.new( current_user, - name: repo.name, - path: repo.name, + name: name, + path: name, description: repo.description, namespace_id: namespace.id, - visibility_level: repo.private ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::VisibilityLevel::PUBLIC, + visibility_level: visibility_level, import_type: "github", import_source: repo.full_name, - import_url: repo.clone_url.sub("https://", "https://#{@session_data[:github_access_token]}@"), - wiki_enabled: !repo.has_wiki? # If repo has wiki we'll import it later + import_url: import_url, + skip_wiki: skip_wiki ).execute end + + private + + def import_url + repo.clone_url.sub('https://', "https://#{session_data[:github_access_token]}@") + end + + def visibility_level + repo.private ? Gitlab::VisibilityLevel::PRIVATE : ApplicationSetting.current.default_project_visibility + end + + # + # If the GitHub project repository has wiki, we should not create the + # default wiki. Otherwise the GitHub importer will fail because the wiki + # repository already exist. + # + def skip_wiki + repo.has_wiki? + end end end end diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb index b84538a090a..b9a227fb11a 100644 --- a/lib/gitlab/github_import/pull_request_formatter.rb +++ b/lib/gitlab/github_import/pull_request_formatter.rb @@ -20,12 +20,16 @@ module Gitlab author_id: author_id, assignee_id: assignee_id, created_at: raw_data.created_at, - updated_at: updated_at + updated_at: raw_data.updated_at } end - def klass - MergeRequest + def project_association + :merge_requests + end + + def find_condition + { iid: number } end def number @@ -56,6 +60,10 @@ module Gitlab end end + def url + raw_data.url + end + private def assigned? @@ -64,7 +72,7 @@ module Gitlab def assignee_id if assigned? - gl_user_id(raw_data.assignee.id) + gitlab_user_id(raw_data.assignee.id) end end @@ -73,7 +81,7 @@ module Gitlab end def author_id - gl_user_id(raw_data.user.id) || project.creator_id + gitlab_author_id || project.creator_id end def body @@ -81,7 +89,11 @@ module Gitlab end def description - formatter.author_line(author) + body + if gitlab_author_id + body + else + formatter.author_line(author) + body + end end def milestone @@ -99,15 +111,6 @@ module Gitlab 'opened' end end - - def updated_at - case state - when 'merged' then raw_data.merged_at - when 'closed' then raw_data.closed_at - else - raw_data.updated_at - end - end end end end diff --git a/lib/gitlab/github_import/release_formatter.rb b/lib/gitlab/github_import/release_formatter.rb new file mode 100644 index 00000000000..1ad702a6058 --- /dev/null +++ b/lib/gitlab/github_import/release_formatter.rb @@ -0,0 +1,27 @@ +module Gitlab + module GithubImport + class ReleaseFormatter < BaseFormatter + def attributes + { + project: project, + tag: raw_data.tag_name, + description: raw_data.body, + created_at: raw_data.created_at, + updated_at: raw_data.created_at + } + end + + def project_association + :releases + end + + def find_condition + { tag: raw_data.tag_name } + end + + def valid? + !raw_data.draft + end + end + end +end diff --git a/lib/gitlab/gitlab_import/importer.rb b/lib/gitlab/gitlab_import/importer.rb index 46d40f75be6..e44d7934fda 100644 --- a/lib/gitlab/gitlab_import/importer.rb +++ b/lib/gitlab/gitlab_import/importer.rb @@ -41,7 +41,8 @@ module Gitlab title: issue["title"], state: issue["state"], updated_at: issue["updated_at"], - author_id: gl_user_id(project, issue["author"]["id"]) + author_id: gitlab_user_id(project, issue["author"]["id"]), + confidential: issue["confidential"] ) end end @@ -51,7 +52,7 @@ module Gitlab private - def gl_user_id(project, gitlab_id) + def gitlab_user_id(project, gitlab_id) user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'gitlab'", gitlab_id.to_s) (user && user.id) || project.creator_id end diff --git a/lib/gitlab/gitorious_import.rb b/lib/gitlab/gitorious_import.rb deleted file mode 100644 index 8d0132a744c..00000000000 --- a/lib/gitlab/gitorious_import.rb +++ /dev/null @@ -1,5 +0,0 @@ -module Gitlab - module GitoriousImport - GITORIOUS_HOST = "https://gitorious.org" - end -end diff --git a/lib/gitlab/gitorious_import/client.rb b/lib/gitlab/gitorious_import/client.rb deleted file mode 100644 index 99fe5bdebfc..00000000000 --- a/lib/gitlab/gitorious_import/client.rb +++ /dev/null @@ -1,29 +0,0 @@ -module Gitlab - module GitoriousImport - class Client - attr_reader :repo_list - - def initialize(repo_list) - @repo_list = repo_list - end - - def authorize_url(redirect_uri) - "#{GITORIOUS_HOST}/gitlab-import?callback_url=#{redirect_uri}" - end - - def repos - @repos ||= repo_names.map { |full_name| GitoriousImport::Repository.new(full_name) } - end - - def repo(id) - repos.find { |repo| repo.id == id } - end - - private - - def repo_names - repo_list.to_s.split(',').map(&:strip).reject(&:blank?) - end - end - end -end diff --git a/lib/gitlab/gitorious_import/project_creator.rb b/lib/gitlab/gitorious_import/project_creator.rb deleted file mode 100644 index 8e22aa9286d..00000000000 --- a/lib/gitlab/gitorious_import/project_creator.rb +++ /dev/null @@ -1,27 +0,0 @@ -module Gitlab - module GitoriousImport - class ProjectCreator - attr_reader :repo, :namespace, :current_user - - def initialize(repo, namespace, current_user) - @repo = repo - @namespace = namespace - @current_user = current_user - end - - def execute - ::Projects::CreateService.new( - current_user, - name: repo.name, - path: repo.path, - description: repo.description, - namespace_id: namespace.id, - visibility_level: Gitlab::VisibilityLevel::PUBLIC, - import_type: "gitorious", - import_source: repo.full_name, - import_url: repo.import_url - ).execute - end - end - end -end diff --git a/lib/gitlab/gitorious_import/repository.rb b/lib/gitlab/gitorious_import/repository.rb deleted file mode 100644 index c88f1ae358d..00000000000 --- a/lib/gitlab/gitorious_import/repository.rb +++ /dev/null @@ -1,35 +0,0 @@ -module Gitlab - module GitoriousImport - Repository = Struct.new(:full_name) do - def id - Digest::SHA1.hexdigest(full_name) - end - - def namespace - segments.first - end - - def path - segments.last - end - - def name - path.titleize - end - - def description - "" - end - - def import_url - "#{GITORIOUS_HOST}/#{full_name}.git" - end - - private - - def segments - full_name.split('/') - end - end - end -end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index c5a11148d33..2c21804fe7a 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -11,7 +11,6 @@ module Gitlab if current_user gon.current_user_id = current_user.id - gon.api_token = current_user.private_token end end end diff --git a/lib/gitlab/google_code_import/importer.rb b/lib/gitlab/google_code_import/importer.rb index 62da327931f..1f4edc36928 100644 --- a/lib/gitlab/google_code_import/importer.rb +++ b/lib/gitlab/google_code_import/importer.rb @@ -92,19 +92,17 @@ module Gitlab end issue = Issue.create!( - project_id: project.id, - title: raw_issue["title"], - description: body, - author_id: project.creator_id, - assignee_id: assignee_id, - state: raw_issue["state"] == "closed" ? "closed" : "opened" + iid: raw_issue['id'], + project_id: project.id, + title: raw_issue['title'], + description: body, + author_id: project.creator_id, + assignee_id: assignee_id, + state: raw_issue['state'] == 'closed' ? 'closed' : 'opened' ) - issue.add_labels_by_names(labels) - - if issue.iid != raw_issue["id"] - issue.update_attribute(:iid, raw_issue["id"]) - end + issue_labels = ::LabelsFinder.new(nil, project_id: project.id, title: labels).execute(skip_authorization: true) + issue.update_attribute(:label_ids, issue_labels.pluck(:id)) import_issue_comments(issue, comments) end @@ -236,8 +234,8 @@ module Gitlab end def create_label(name) - color = nice_label_color(name) - Label.create!(project_id: project.id, name: name, color: color) + params = { name: name, color: nice_label_color(name) } + ::Labels::FindOrCreateService.new(nil, project, params).execute(skip_authorization: true) end def format_content(raw_content) diff --git a/lib/gitlab/identifier.rb b/lib/gitlab/identifier.rb index 3e5d728f3bc..f8809db21aa 100644 --- a/lib/gitlab/identifier.rb +++ b/lib/gitlab/identifier.rb @@ -5,19 +5,61 @@ module Gitlab def identify(identifier, project, newrev) if identifier.blank? # Local push from gitlab - email = project.commit(newrev).author_email rescue nil - User.find_by(email: email) if email - + identify_using_commit(project, newrev) elsif identifier =~ /\Auser-\d+\Z/ # git push over http - user_id = identifier.gsub("user-", "") - User.find_by(id: user_id) - + identify_using_user(identifier) elsif identifier =~ /\Akey-\d+\Z/ # git push over ssh - key_id = identifier.gsub("key-", "") - Key.find_by(id: key_id).try(:user) + identify_using_ssh_key(identifier) + end + end + + # Tries to identify a user based on a commit SHA. + def identify_using_commit(project, ref) + commit = project.commit(ref) + + return if !commit || !commit.author_email + + email = commit.author_email + + identify_with_cache(:email, email) do + User.find_by(email: email) end end + + # Tries to identify a user based on a user identifier (e.g. "user-123"). + def identify_using_user(identifier) + user_id = identifier.gsub("user-", "") + + identify_with_cache(:user, user_id) do + User.find_by(id: user_id) + end + end + + # Tries to identify a user based on an SSH key identifier (e.g. "key-123"). + def identify_using_ssh_key(identifier) + key_id = identifier.gsub("key-", "") + + identify_with_cache(:ssh_key, key_id) do + User.find_by_ssh_key_id(key_id) + end + end + + def identify_with_cache(category, key) + if identification_cache[category].key?(key) + identification_cache[category][key] + else + identification_cache[category][key] = yield + end + end + + def identification_cache + @identification_cache ||= { + email: {}, + user: {}, + ssh_key: {} + } + end end end diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index bb562bdcd2c..eb667a85b78 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -2,7 +2,8 @@ module Gitlab module ImportExport extend self - VERSION = '0.1.3' + # For every version update, the version history in import_export.md has to be kept up to date. + VERSION = '0.1.5' FILENAME_LIMIT = 50 def export_path(relative_path:) diff --git a/lib/gitlab/import_export/attribute_cleaner.rb b/lib/gitlab/import_export/attribute_cleaner.rb new file mode 100644 index 00000000000..34169319b26 --- /dev/null +++ b/lib/gitlab/import_export/attribute_cleaner.rb @@ -0,0 +1,28 @@ +module Gitlab + module ImportExport + class AttributeCleaner + ALLOWED_REFERENCES = RelationFactory::PROJECT_REFERENCES + RelationFactory::USER_REFERENCES + ['group_id'] + + def self.clean(*args) + new(*args).clean + end + + def initialize(relation_hash:, relation_class:) + @relation_hash = relation_hash + @relation_class = relation_class + end + + def clean + @relation_hash.reject do |key, _value| + prohibited_key?(key) || !@relation_class.attribute_method?(key) + end.except('id') + end + + private + + def prohibited_key?(key) + key.end_with?('_id') && !ALLOWED_REFERENCES.include?(key) + end + end + end +end diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index e522a0fc8f6..f00c7460e82 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -1,6 +1,8 @@ module Gitlab module ImportExport module CommandLineUtil + DEFAULT_MODE = 0700 + def tar_czf(archive:, dir:) tar_with_options(archive: archive, dir: dir, options: 'czf') end @@ -21,6 +23,11 @@ module Gitlab execute(%W(#{Gitlab.config.gitlab_shell.path}/bin/create-hooks) + repository_storage_paths_args) end + def mkdir_p(path) + FileUtils.mkdir_p(path, mode: DEFAULT_MODE) + FileUtils.chmod(DEFAULT_MODE, path) + end + private def tar_with_options(archive:, dir:, options:) @@ -45,7 +52,7 @@ module Gitlab # if we are copying files, create the destination folder destination_folder = File.file?(source) ? File.dirname(destination) : destination - FileUtils.mkdir_p(destination_folder) + mkdir_p(destination_folder) FileUtils.copy_entry(source, destination) true end diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index eca6e5b6d51..ffd17118c91 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -15,7 +15,7 @@ module Gitlab end def import - FileUtils.mkdir_p(@shared.export_path) + mkdir_p(@shared.export_path) wait_for_archived_file do decompress_archive @@ -43,6 +43,14 @@ module Gitlab raise Projects::ImportService::Error.new("Unable to decompress #{@archive_file} into #{@shared.export_path}") unless result + remove_symlinks! + end + + def remove_symlinks! + Dir["#{@shared.export_path}/**/*"].each do |path| + FileUtils.rm(path) if File.lstat(path).symlink? + end + true end end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 1da51043611..e6ecd118609 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -1,15 +1,21 @@ # Model relationships to be included in the project import/export project_tree: + - labels: + :priorities + - milestones: + - :events - issues: - :events - notes: - :author - :events - label_links: - - :label + - label: + :priorities - milestone: - :events - snippets: + - :award_emoji - notes: :author - :releases @@ -22,7 +28,8 @@ project_tree: - :merge_request_diff - :events - label_links: - - :label + - label: + :priorities - milestone: - :events - pipelines: @@ -35,19 +42,15 @@ project_tree: - :deploy_keys - :services - :hooks - - :protected_branches - - :labels - - milestones: - - :events + - protected_branches: + - :merge_access_levels + - :push_access_levels + - :project_feature # Only include the following attributes for the models specified. included_attributes: project: - :description - - :issues_enabled - - :merge_requests_enabled - - :wiki_enabled - - :snippets_enabled - :visibility_level - :archived user: @@ -67,9 +70,17 @@ excluded_attributes: - :milestone_id merge_requests: - :milestone_id + award_emoji: + - :awardable_id methods: + labels: + - :type + label: + - :type statuses: - :type + services: + - :type merge_request_diff: - - :utf8_st_diffs
\ No newline at end of file + - :utf8_st_diffs diff --git a/lib/gitlab/import_export/json_hash_builder.rb b/lib/gitlab/import_export/json_hash_builder.rb index 0cc10f40087..48c09dafcb6 100644 --- a/lib/gitlab/import_export/json_hash_builder.rb +++ b/lib/gitlab/import_export/json_hash_builder.rb @@ -65,11 +65,17 @@ module Gitlab # +value+ existing model to be included in the hash # +parsed_hash+ the original hash def parse_hash(value) + return nil if already_contains_methods?(value) + @attributes_finder.parse(value) do |hash| { include: hash_or_merge(value, hash) } end end + def already_contains_methods?(value) + value.is_a?(Hash) && value.values.detect { |val| val[:methods]} + end + # Adds new model configuration to an existing hash with key +current_key+ # It may include exceptions or other attribute detail configuration, parsed by +@attributes_finder+ # diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index 36c4cf6efa0..b790733f4a7 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -55,7 +55,12 @@ module Gitlab end def member_hash(member) - member.except('id').merge(source_id: @project.id, importing: true) + parsed_hash(member).merge('source_id' => @project.id, 'importing' => true) + end + + def parsed_hash(member) + Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: member.deep_stringify_keys, + relation_class: ProjectMember) end def find_project_user_query(member) diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index c7b3551b84c..c551321c18d 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -9,8 +9,14 @@ module Gitlab end def restore - json = IO.read(@path) - @tree_hash = ActiveSupport::JSON.decode(json) + begin + json = IO.read(@path) + @tree_hash = ActiveSupport::JSON.decode(json) + rescue => e + Rails.logger.error("Import/Export error: #{e.message}") + raise Gitlab::ImportExport::Error.new('Incorrect JSON format') + end + @project_members = @tree_hash.delete('project_members') ActiveRecord::Base.no_touching do @@ -61,11 +67,17 @@ module Gitlab def restore_project return @project unless @tree_hash - project_params = @tree_hash.reject { |_key, value| value.is_a?(Array) } @project.update(project_params) @project end + def project_params + @tree_hash.reject do |key, value| + # return params that are not 1 to many or 1 to 1 relations + value.is_a?(Array) || key == key.singularize + end + end + # Given a relation hash containing one or more models and its relationships, # loops through each model and each object from a model type and # and assigns its correspondent attributes hash from +tree_hash+ @@ -104,13 +116,18 @@ module Gitlab def create_relation(relation, relation_hash_list) relation_array = [relation_hash_list].flatten.map do |relation_hash| Gitlab::ImportExport::RelationFactory.create(relation_sym: relation.to_sym, - relation_hash: relation_hash.merge('project_id' => restored_project.id), + relation_hash: parsed_relation_hash(relation_hash), members_mapper: members_mapper, - user: @user) + user: @user, + project_id: restored_project.id) end relation_hash_list.is_a?(Array) ? relation_array : relation_array.first end + + def parsed_relation_hash(relation_hash) + relation_hash.merge!('group_id' => restored_project.group.try(:id), 'project_id' => restored_project.id) + end end end end diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb index 9153088e966..2fbf437ec26 100644 --- a/lib/gitlab/import_export/project_tree_saver.rb +++ b/lib/gitlab/import_export/project_tree_saver.rb @@ -1,6 +1,8 @@ module Gitlab module ImportExport class ProjectTreeSaver + include Gitlab::ImportExport::CommandLineUtil + attr_reader :full_path def initialize(project:, shared:) @@ -10,7 +12,7 @@ module Gitlab end def save - FileUtils.mkdir_p(@shared.export_path) + mkdir_p(@shared.export_path) File.write(full_path, project_json_tree) true diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index b0726268ca6..a0e80fccad9 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -7,23 +7,30 @@ module Gitlab variables: 'Ci::Variable', triggers: 'Ci::Trigger', builds: 'Ci::Build', - hooks: 'ProjectHook' }.freeze + hooks: 'ProjectHook', + merge_access_levels: 'ProtectedBranch::MergeAccessLevel', + push_access_levels: 'ProtectedBranch::PushAccessLevel', + labels: :project_labels, + priorities: :label_priorities, + label: :project_label }.freeze - USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id].freeze + USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id created_by_id].freeze + + PROJECT_REFERENCES = %w[project_id source_project_id gl_project_id target_project_id].freeze BUILD_MODELS = %w[Ci::Build commit_status].freeze IMPORTED_OBJECT_MAX_RETRIES = 5.freeze - EXISTING_OBJECT_CHECK = %i[milestone milestones label labels].freeze + EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels project_label group_label].freeze def self.create(*args) new(*args).create end - def initialize(relation_sym:, relation_hash:, members_mapper:, user:) + def initialize(relation_sym:, relation_hash:, members_mapper:, user:, project_id:) @relation_name = OVERRIDES[relation_sym] || relation_sym - @relation_hash = relation_hash.except('id', 'noteable_id') + @relation_hash = relation_hash.except('noteable_id').merge('project_id' => project_id) @members_mapper = members_mapper @user = user @imported_object_retries = 0 @@ -50,6 +57,8 @@ module Gitlab update_user_references update_project_references + + handle_group_label if group_label? reset_ci_tokens if @relation_name == 'Ci::Trigger' @relation_hash['data'].deep_symbolize_keys! if @relation_name == :events && @relation_hash['data'] set_st_diffs if @relation_name == :merge_request_diff @@ -117,6 +126,20 @@ module Gitlab @relation_hash['target_project_id'] && @relation_hash['target_project_id'] == @relation_hash['source_project_id'] end + def group_label? + @relation_hash['type'] == 'GroupLabel' + end + + def handle_group_label + # If there's no group, move the label to a project label + if @relation_hash['group_id'] + @relation_hash['project_id'] = nil + @relation_name = :group_label + else + @relation_hash['type'] = 'ProjectLabel' + end + end + def reset_ci_tokens return unless Gitlab::ImportExport.reset_tokens? @@ -149,7 +172,8 @@ module Gitlab end def parsed_relation_hash - @relation_hash.reject { |k, _v| !relation_class.attribute_method?(k) } + @parsed_relation_hash ||= Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: @relation_hash, + relation_class: relation_class) end def set_st_diffs @@ -161,14 +185,36 @@ module Gitlab # Otherwise always create the record, skipping the extra SELECT clause. @existing_or_new_object ||= begin if EXISTING_OBJECT_CHECK.include?(@relation_name) - existing_object = relation_class.find_or_initialize_by(parsed_relation_hash.slice('title', 'project_id')) - existing_object.assign_attributes(parsed_relation_hash) + attribute_hash = attribute_hash_for(['events', 'priorities']) + + existing_object.assign_attributes(attribute_hash) if attribute_hash.any? + existing_object else relation_class.new(parsed_relation_hash) end end end + + def attribute_hash_for(attributes) + attributes.inject({}) do |hash, value| + hash[value] = parsed_relation_hash.delete(value) if parsed_relation_hash[value] + hash + end + end + + def existing_object + @existing_object ||= + begin + finder_attributes = @relation_name == :group_label ? %w[title group_id] : %w[title project_id] + finder_hash = parsed_relation_hash.slice(*finder_attributes) + existing_object = relation_class.find_or_create_by(finder_hash) + # Done in two steps, as MySQL behaves differently than PostgreSQL using + # the +find_or_create_by+ method and does not return the ID the second time. + existing_object.update!(parsed_relation_hash) + existing_object + end + end end end end diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index 6d9379acf25..48a9a6fa5e2 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -12,7 +12,7 @@ module Gitlab def restore return true unless File.exist?(@path_to_bundle) - FileUtils.mkdir_p(path_to_repo) + mkdir_p(path_to_repo) git_unbundle(repo_path: path_to_repo, bundle_path: @path_to_bundle) && repo_restore_hooks rescue => e @@ -22,10 +22,6 @@ module Gitlab private - def repos_path - Gitlab.config.gitlab_shell.repos_path - end - def path_to_repo @project.repository.path_to_repo end diff --git a/lib/gitlab/import_export/repo_saver.rb b/lib/gitlab/import_export/repo_saver.rb index 331e14021e6..a7028a32570 100644 --- a/lib/gitlab/import_export/repo_saver.rb +++ b/lib/gitlab/import_export/repo_saver.rb @@ -20,7 +20,7 @@ module Gitlab private def bundle_to_disk - FileUtils.mkdir_p(@shared.export_path) + mkdir_p(@shared.export_path) git_bundle(repo_path: path_to_repo, bundle_path: @full_path) rescue => e @shared.error(e) diff --git a/lib/gitlab/import_export/version_checker.rb b/lib/gitlab/import_export/version_checker.rb index de3fe6d822e..bd3c3ee3b2f 100644 --- a/lib/gitlab/import_export/version_checker.rb +++ b/lib/gitlab/import_export/version_checker.rb @@ -24,12 +24,19 @@ module Gitlab end def verify_version!(version) - if Gem::Version.new(version) > Gem::Version.new(Gitlab::ImportExport.version) - raise Gitlab::ImportExport::Error.new("Import version mismatch: Required <= #{Gitlab::ImportExport.version} but was #{version}") + if different_version?(version) + raise Gitlab::ImportExport::Error.new("Import version mismatch: Required #{Gitlab::ImportExport.version} but was #{version}") else true end end + + def different_version?(version) + Gem::Version.new(version) != Gem::Version.new(Gitlab::ImportExport.version) + rescue => e + Rails.logger.error("Import/Export error: #{e.message}") + raise Gitlab::ImportExport::Error.new('Incorrect VERSION format') + end end end end diff --git a/lib/gitlab/import_export/version_saver.rb b/lib/gitlab/import_export/version_saver.rb index 9b642d740b7..7cf88298642 100644 --- a/lib/gitlab/import_export/version_saver.rb +++ b/lib/gitlab/import_export/version_saver.rb @@ -1,12 +1,14 @@ module Gitlab module ImportExport class VersionSaver + include Gitlab::ImportExport::CommandLineUtil + def initialize(shared:) @shared = shared end def save - FileUtils.mkdir_p(@shared.export_path) + mkdir_p(@shared.export_path) File.write(version_file, Gitlab::ImportExport.version, mode: 'w') rescue => e diff --git a/lib/gitlab/import_export/wiki_repo_saver.rb b/lib/gitlab/import_export/wiki_repo_saver.rb index 6107420e4dd..1e6722a7bba 100644 --- a/lib/gitlab/import_export/wiki_repo_saver.rb +++ b/lib/gitlab/import_export/wiki_repo_saver.rb @@ -9,7 +9,7 @@ module Gitlab end def bundle_to_disk(full_path) - FileUtils.mkdir_p(@shared.export_path) + mkdir_p(@shared.export_path) git_bundle(repo_path: path_to_repo, bundle_path: full_path) rescue => e @shared.error(e) diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index 59a05411fe9..94261b7eeed 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -14,13 +14,12 @@ module Gitlab def options { - 'GitHub' => 'github', - 'Bitbucket' => 'bitbucket', - 'GitLab.com' => 'gitlab', - 'Gitorious.org' => 'gitorious', - 'Google Code' => 'google_code', - 'FogBugz' => 'fogbugz', - 'Repo by URL' => 'git', + 'GitHub' => 'github', + 'Bitbucket' => 'bitbucket', + 'GitLab.com' => 'gitlab', + 'Google Code' => 'google_code', + 'FogBugz' => 'fogbugz', + 'Repo by URL' => 'git', 'GitLab export' => 'gitlab_project' } end diff --git a/lib/gitlab/incoming_email.rb b/lib/gitlab/incoming_email.rb index d7be50bd437..801dfde9a36 100644 --- a/lib/gitlab/incoming_email.rb +++ b/lib/gitlab/incoming_email.rb @@ -1,5 +1,7 @@ module Gitlab module IncomingEmail + WILDCARD_PLACEHOLDER = '%{key}'.freeze + class << self FALLBACK_MESSAGE_ID_REGEX = /\Areply\-(.+)@#{Gitlab.config.gitlab.host}\Z/.freeze @@ -7,8 +9,16 @@ module Gitlab config.enabled && config.address end + def supports_wildcard? + config.address && config.address.include?(WILDCARD_PLACEHOLDER) + end + + def supports_issue_creation? + enabled? && supports_wildcard? + end + def reply_address(key) - config.address.gsub('%{key}', key) + config.address.gsub(WILDCARD_PLACEHOLDER, key) end def key_from_address(address) diff --git a/lib/gitlab/issues_labels.rb b/lib/gitlab/issues_labels.rb index 1bec6088292..b8ca7f2f55f 100644 --- a/lib/gitlab/issues_labels.rb +++ b/lib/gitlab/issues_labels.rb @@ -18,8 +18,8 @@ module Gitlab { title: "enhancement", color: green } ] - labels.each do |label| - project.labels.create(label) + labels.each do |params| + ::Labels::FindOrCreateService.new(nil, project, params).execute(skip_authorization: true) end end end diff --git a/lib/gitlab/ldap/access.rb b/lib/gitlab/ldap/access.rb index 2f326d00a2f..7e06bd2b0fb 100644 --- a/lib/gitlab/ldap/access.rb +++ b/lib/gitlab/ldap/access.rb @@ -51,8 +51,6 @@ module Gitlab user.ldap_block false end - rescue - false end def adapter diff --git a/lib/gitlab/ldap/adapter.rb b/lib/gitlab/ldap/adapter.rb index 9a5bcfb5c9b..8b38cfaefb6 100644 --- a/lib/gitlab/ldap/adapter.rb +++ b/lib/gitlab/ldap/adapter.rb @@ -23,31 +23,7 @@ module Gitlab end def users(field, value, limit = nil) - if field.to_sym == :dn - options = { - base: value, - scope: Net::LDAP::SearchScope_BaseObject - } - else - options = { - base: config.base, - filter: Net::LDAP::Filter.eq(field, value) - } - end - - if config.user_filter.present? - user_filter = Net::LDAP::Filter.construct(config.user_filter) - - options[:filter] = if options[:filter] - Net::LDAP::Filter.join(options[:filter], user_filter) - else - user_filter - end - end - - if limit.present? - options.merge!(size: limit) - end + options = user_options(field, value, limit) entries = ldap_search(options).select do |entry| entry.respond_to? config.uid @@ -86,10 +62,49 @@ module Gitlab results end end + rescue Net::LDAP::Error => error + Rails.logger.warn("LDAP search raised exception #{error.class}: #{error.message}") + [] rescue Timeout::Error Rails.logger.warn("LDAP search timed out after #{config.timeout} seconds") [] end + + private + + def user_options(field, value, limit) + options = { attributes: user_attributes } + options[:size] = limit if limit + + if field.to_sym == :dn + options[:base] = value + options[:scope] = Net::LDAP::SearchScope_BaseObject + options[:filter] = user_filter + else + options[:base] = config.base + options[:filter] = user_filter(Net::LDAP::Filter.eq(field, value)) + end + + options + end + + def user_filter(filter = nil) + if config.user_filter.present? + user_filter = Net::LDAP::Filter.construct(config.user_filter) + end + + if user_filter && filter + Net::LDAP::Filter.join(filter, user_filter) + elsif user_filter + user_filter + else + filter + end + end + + def user_attributes + %W(#{config.uid} cn mail dn) + end end end end diff --git a/lib/gitlab/ldap/config.rb b/lib/gitlab/ldap/config.rb index f9bb5775323..6ea069d26df 100644 --- a/lib/gitlab/ldap/config.rb +++ b/lib/gitlab/ldap/config.rb @@ -92,6 +92,10 @@ module Gitlab options['timeout'].to_i end + def has_auth? + options['password'] || options['bind_dn'] + end + protected def base_config @@ -122,10 +126,6 @@ module Gitlab } } end - - def has_auth? - options['password'] || options['bind_dn'] - end end end end diff --git a/lib/gitlab/lfs_token.rb b/lib/gitlab/lfs_token.rb new file mode 100644 index 00000000000..5f67e97fa2a --- /dev/null +++ b/lib/gitlab/lfs_token.rb @@ -0,0 +1,48 @@ +module Gitlab + class LfsToken + attr_accessor :actor + + TOKEN_LENGTH = 50 + EXPIRY_TIME = 1800 + + def initialize(actor) + @actor = + case actor + when DeployKey, User + actor + when Key + actor.user + else + raise 'Bad Actor' + end + end + + def token + Gitlab::Redis.with do |redis| + token = redis.get(redis_key) + token ||= Devise.friendly_token(TOKEN_LENGTH) + redis.set(redis_key, token, ex: EXPIRY_TIME) + + token + end + end + + def user? + actor.is_a?(User) + end + + def type + actor.is_a?(User) ? :lfs_token : :lfs_deploy_token + end + + def actor_name + actor.is_a?(User) ? actor.username : "lfs+deploy-key-#{actor.id}" + end + + private + + def redis_key + "gitlab:lfs_token:#{actor.class.name.underscore}_#{actor.id}" if actor + end + end +end diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb index 12999a90a29..a5220d92312 100644 --- a/lib/gitlab/mail_room.rb +++ b/lib/gitlab/mail_room.rb @@ -33,7 +33,12 @@ module Gitlab config[:mailbox] = 'inbox' if config[:mailbox].nil? if config[:enabled] && config[:address] - config[:redis_url] = Gitlab::Redis.new(rails_env).url + gitlab_redis = Gitlab::Redis.new(rails_env) + config[:redis_url] = gitlab_redis.url + + if gitlab_redis.sentinels? + config[:sentinels] = gitlab_redis.sentinels + end end config diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index 41fcd971c22..3d1ba33ec68 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -124,6 +124,15 @@ module Gitlab trans.action = action if trans end + # Tracks an event. + # + # See `Gitlab::Metrics::Transaction#add_event` for more details. + def self.add_event(*args) + trans = current_transaction + + trans.add_event(*args) if trans + end + # Returns the prefix to use for the name of a series. def self.series_prefix @series_prefix ||= Sidekiq.server? ? 'sidekiq_' : 'rails_' diff --git a/lib/gitlab/metrics/metric.rb b/lib/gitlab/metrics/metric.rb index f23d67e1e38..bd0afe53c51 100644 --- a/lib/gitlab/metrics/metric.rb +++ b/lib/gitlab/metrics/metric.rb @@ -4,15 +4,20 @@ module Gitlab class Metric JITTER_RANGE = 0.000001..0.001 - attr_reader :series, :values, :tags + attr_reader :series, :values, :tags, :type # series - The name of the series (as a String) to store the metric in. # values - A Hash containing the values to store. # tags - A Hash containing extra tags to add to the metrics. - def initialize(series, values, tags = {}) + def initialize(series, values, tags = {}, type = :metric) @values = values @series = series @tags = tags + @type = type + end + + def event? + type == :event end # Returns a Hash in a format that can be directly written to InfluxDB. diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb index e61670f491c..01c96a6fe96 100644 --- a/lib/gitlab/metrics/rack_middleware.rb +++ b/lib/gitlab/metrics/rack_middleware.rb @@ -4,6 +4,17 @@ module Gitlab class RackMiddleware CONTROLLER_KEY = 'action_controller.instance' ENDPOINT_KEY = 'api.endpoint' + CONTENT_TYPES = { + 'text/html' => :html, + 'text/plain' => :txt, + 'application/json' => :json, + 'text/js' => :js, + 'application/atom+xml' => :atom, + 'image/png' => :png, + 'image/jpeg' => :jpeg, + 'image/gif' => :gif, + 'image/svg+xml' => :svg + } def initialize(app) @app = app @@ -17,6 +28,10 @@ module Gitlab begin retval = trans.run { @app.call(env) } + rescue Exception => error # rubocop: disable Lint/RescueException + trans.add_event(:rails_exception) + + raise error # Even in the event of an error we want to submit any metrics we # might've gathered up to this point. ensure @@ -42,8 +57,15 @@ module Gitlab end def tag_controller(trans, env) - controller = env[CONTROLLER_KEY] - trans.action = "#{controller.class.name}##{controller.action_name}" + controller = env[CONTROLLER_KEY] + action = "#{controller.class.name}##{controller.action_name}" + suffix = CONTENT_TYPES[controller.content_type] + + if suffix && suffix != :html + action += ".#{suffix}" + end + + trans.action = action end def tag_endpoint(trans, env) diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb index a1240fd33ee..f9dd8e41912 100644 --- a/lib/gitlab/metrics/sidekiq_middleware.rb +++ b/lib/gitlab/metrics/sidekiq_middleware.rb @@ -11,6 +11,10 @@ module Gitlab # Old gitlad-shell messages don't provide enqueued_at/created_at attributes trans.set(:sidekiq_queue_duration, Time.now.to_f - (message['enqueued_at'] || message['created_at'] || 0)) trans.run { yield } + rescue Exception => error # rubocop: disable Lint/RescueException + trans.add_event(:sidekiq_exception) + + raise error ensure trans.finish end diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index 968f3218950..7bc16181be6 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -4,7 +4,10 @@ module Gitlab class Transaction THREAD_KEY = :_gitlab_metrics_transaction - attr_reader :tags, :values, :methods + # The series to store events (e.g. Git pushes) in. + EVENT_SERIES = 'events' + + attr_reader :tags, :values, :method, :metrics attr_accessor :action @@ -55,6 +58,20 @@ module Gitlab @metrics << Metric.new("#{Metrics.series_prefix}#{series}", values, tags) end + # Tracks a business level event + # + # Business level events including events such as Git pushes, Emails being + # sent, etc. + # + # event_name - The name of the event (e.g. "git_push"). + # tags - A set of tags to attach to the event. + def add_event(event_name, tags = {}) + @metrics << Metric.new(EVENT_SERIES, + { count: 1 }, + { event: event_name }.merge(tags), + :event) + end + # Returns a MethodCall object for the given name. def method_call_for(name) unless method = @methods[name] @@ -101,7 +118,7 @@ module Gitlab submit_hashes = submit.map do |metric| hash = metric.to_hash - hash[:tags][:action] ||= @action if @action + hash[:tags][:action] ||= @action if @action && !metric.event? hash end diff --git a/lib/gitlab/middleware/rails_queue_duration.rb b/lib/gitlab/middleware/rails_queue_duration.rb index 56608b1b276..5d2d7d0026c 100644 --- a/lib/gitlab/middleware/rails_queue_duration.rb +++ b/lib/gitlab/middleware/rails_queue_duration.rb @@ -11,7 +11,7 @@ module Gitlab def call(env) trans = Gitlab::Metrics.current_transaction - proxy_start = env['HTTP_GITLAB_WORHORSE_PROXY_START'].presence + proxy_start = env['HTTP_GITLAB_WORKHORSE_PROXY_START'].presence if trans && proxy_start # Time in milliseconds since gitlab-workhorse started the request trans.set(:rails_queue_duration, Time.now.to_f * 1_000 - proxy_start.to_f / 1_000_000) diff --git a/lib/gitlab/optimistic_locking.rb b/lib/gitlab/optimistic_locking.rb new file mode 100644 index 00000000000..879d46446b3 --- /dev/null +++ b/lib/gitlab/optimistic_locking.rb @@ -0,0 +1,19 @@ +module Gitlab + module OptimisticLocking + extend self + + def retry_lock(subject, retries = 100, &block) + loop do + begin + ActiveRecord::Base.transaction do + return block.call(subject) + end + rescue ActiveRecord::StaleObjectError + retries -= 1 + raise unless retries >= 0 + subject.reload + end + end + end + end +end diff --git a/lib/gitlab/popen.rb b/lib/gitlab/popen.rb index ca23ccef25b..cc74bb29087 100644 --- a/lib/gitlab/popen.rb +++ b/lib/gitlab/popen.rb @@ -18,18 +18,18 @@ module Gitlab FileUtils.mkdir_p(path) end - @cmd_output = "" - @cmd_status = 0 + cmd_output = "" + cmd_status = 0 Open3.popen3(vars, *cmd, options) do |stdin, stdout, stderr, wait_thr| - # We are not using stdin so we should close it, in case the command we - # are running waits for input. + yield(stdin) if block_given? stdin.close - @cmd_output << stdout.read - @cmd_output << stderr.read - @cmd_status = wait_thr.value.exitstatus + + cmd_output << stdout.read + cmd_output << stderr.read + cmd_status = wait_thr.value.exitstatus end - [@cmd_output, @cmd_status] + [cmd_output, cmd_status] end end end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 183bd10d6a3..b8326a64b22 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -5,11 +5,7 @@ module Gitlab def initialize(current_user, project, query, repository_ref = nil) @current_user = current_user @project = project - @repository_ref = if repository_ref.present? - repository_ref - else - nil - end + @repository_ref = repository_ref.presence @query = query end @@ -28,11 +24,6 @@ module Gitlab end end - def total_count - @total_count ||= issues_count + merge_requests_count + blobs_count + - notes_count + wiki_blobs_count + commits_count - end - def blobs_count @blobs_count ||= blobs.count end @@ -52,37 +43,31 @@ module Gitlab private def blobs - if project.empty_repo? || query.blank? - [] - else - project.repository.search_files(query, repository_ref) - end + @blobs ||= project.repository.search_files(query, repository_ref) end def wiki_blobs - if project.wiki_enabled? && query.present? - project_wiki = ProjectWiki.new(project) + @wiki_blobs ||= begin + if project.wiki_enabled? && query.present? + project_wiki = ProjectWiki.new(project) - unless project_wiki.empty? - project_wiki.search_files(query) + unless project_wiki.empty? + project_wiki.search_files(query) + else + [] + end else [] end - else - [] end end def notes - project.notes.user.search(query, as_user: @current_user).order('updated_at DESC') + @notes ||= project.notes.user.search(query, as_user: @current_user).order('updated_at DESC') end def commits - if project.empty_repo? || query.blank? - [] - else - project.repository.find_commits_by_message(query).compact - end + @commits ||= project.repository.find_commits_by_message(query) end def project_ids_relation diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb index 9376b54f43b..9226da2d6b1 100644 --- a/lib/gitlab/redis.rb +++ b/lib/gitlab/redis.rb @@ -9,35 +9,45 @@ module Gitlab SIDEKIQ_NAMESPACE = 'resque:gitlab' MAILROOM_NAMESPACE = 'mail_room:gitlab' DEFAULT_REDIS_URL = 'redis://localhost:6379' - - # To be thread-safe we must be careful when writing the class instance - # variables @url and @pool. Because @pool depends on @url we need two - # mutexes to prevent deadlock. - PARAMS_MUTEX = Mutex.new - POOL_MUTEX = Mutex.new - private_constant :PARAMS_MUTEX, :POOL_MUTEX + CONFIG_FILE = File.expand_path('../../config/resque.yml', __dir__) class << self + # Do NOT cache in an instance variable. Result may be mutated by caller. def params - @params || PARAMS_MUTEX.synchronize { @params = new.params } + new.params end + # Do NOT cache in an instance variable. Result may be mutated by caller. # @deprecated Use .params instead to get sentinel support def url new.url end def with - if @pool.nil? - POOL_MUTEX.synchronize do - @pool = ConnectionPool.new { ::Redis.new(params) } - end - end + @pool ||= ConnectionPool.new(size: pool_size) { ::Redis.new(params) } @pool.with { |redis| yield redis } end - def reset_params! - @params = nil + def pool_size + if Sidekiq.server? + # the pool will be used in a multi-threaded context + Sidekiq.options[:concurrency] + 5 + else + # probably this is a Unicorn process, so single threaded + 5 + end + end + + def _raw_config + return @_raw_config if defined?(@_raw_config) + + begin + @_raw_config = File.read(CONFIG_FILE).freeze + rescue Errno::ENOENT + @_raw_config = false + end + + @_raw_config end end @@ -53,6 +63,14 @@ module Gitlab raw_config_hash[:url] end + def sentinels + raw_config_hash[:sentinels] + end + + def sentinels? + sentinels && !sentinels.empty? + end + private def redis_store_options @@ -83,12 +101,7 @@ module Gitlab end def fetch_config - file = config_file - File.exist?(file) ? YAML.load_file(file)[@rails_env] : false - end - - def config_file - File.expand_path('../../../config/resque.yml', __FILE__) + self.class._raw_config ? YAML.load(self.class._raw_config)[@rails_env] : false end end end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index ffad5e17c78..cb1659f9cee 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -2,15 +2,19 @@ module Gitlab module Regex extend self - NAMESPACE_REGEX_STR = '(?:[a-zA-Z0-9_\.][a-zA-Z0-9_\-\.]*[a-zA-Z0-9_\-]|[a-zA-Z0-9_])'.freeze + NAMESPACE_REGEX_STR = '(?:[a-zA-Z0-9_\.][a-zA-Z0-9_\-\.]*[a-zA-Z0-9_\-]|[a-zA-Z0-9_])(?<!\.git|\.atom)'.freeze def namespace_regex @namespace_regex ||= /\A#{NAMESPACE_REGEX_STR}\z/.freeze end + def namespace_route_regex + @namespace_route_regex ||= /#{NAMESPACE_REGEX_STR}/.freeze + end + def namespace_regex_message "can contain only letters, digits, '_', '-' and '.'. " \ - "Cannot start with '-' or end in '.'." \ + "Cannot start with '-' or end in '.', '.git' or '.atom'." \ end def namespace_name_regex @@ -44,7 +48,7 @@ module Gitlab end def file_name_regex_message - "can contain only letters, digits, '_', '-', '@' and '.'. " + "can contain only letters, digits, '_', '-', '@' and '.'." end def file_path_regex @@ -52,7 +56,7 @@ module Gitlab end def file_path_regex_message - "can contain only letters, digits, '_', '-', '@' and '.'. Separate directories with a '/'. " + "can contain only letters, digits, '_', '-', '@' and '.'. Separate directories with a '/'." end def directory_traversal_regex @@ -60,7 +64,7 @@ module Gitlab end def directory_traversal_regex_message - "cannot include directory traversal. " + "cannot include directory traversal." end def archive_formats_regex @@ -96,11 +100,11 @@ module Gitlab end def environment_name_regex - @environment_name_regex ||= /\A[a-zA-Z0-9_-]+\z/.freeze + @environment_name_regex ||= /\A[a-zA-Z0-9_\\\/\${}. -]+\z/.freeze end def environment_name_regex_message - "can contain only letters, digits, '-' and '_'." + "can contain only letters, digits, '-', '_', '/', '$', '{', '}', '.' and spaces" end end end diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index f8ab2b1f09e..2690938fe82 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -27,11 +27,6 @@ module Gitlab end end - def total_count - @total_count ||= projects_count + issues_count + merge_requests_count + - milestones_count - end - def projects_count @projects_count ||= projects.count end @@ -48,10 +43,6 @@ module Gitlab @milestones_count ||= milestones.count end - def empty? - total_count.zero? - end - private def projects diff --git a/lib/gitlab/sentry.rb b/lib/gitlab/sentry.rb new file mode 100644 index 00000000000..117fc508135 --- /dev/null +++ b/lib/gitlab/sentry.rb @@ -0,0 +1,27 @@ +module Gitlab + module Sentry + def self.enabled? + Rails.env.production? && current_application_settings.sentry_enabled? + end + + def self.context(current_user = nil) + return unless self.enabled? + + if current_user + Raven.user_context( + id: current_user.id, + email: current_user.email, + username: current_user.username, + ) + end + end + + def self.program_context + if Sidekiq.server? + 'sidekiq' + else + 'rails' + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/arguments_logger.rb b/lib/gitlab/sidekiq_middleware/arguments_logger.rb index 7813091ec7b..82a59a7a87e 100644 --- a/lib/gitlab/sidekiq_middleware/arguments_logger.rb +++ b/lib/gitlab/sidekiq_middleware/arguments_logger.rb @@ -2,7 +2,7 @@ module Gitlab module SidekiqMiddleware class ArgumentsLogger def call(worker, job, queue) - Sidekiq.logger.info "arguments: #{job['args']}" + Sidekiq.logger.info "arguments: #{JSON.dump(job['args'])}" yield end end diff --git a/lib/gitlab/slash_commands/command_definition.rb b/lib/gitlab/slash_commands/command_definition.rb new file mode 100644 index 00000000000..60d35be2599 --- /dev/null +++ b/lib/gitlab/slash_commands/command_definition.rb @@ -0,0 +1,57 @@ +module Gitlab + module SlashCommands + class CommandDefinition + attr_accessor :name, :aliases, :description, :params, :condition_block, :action_block + + def initialize(name, attributes = {}) + @name = name + + @aliases = attributes[:aliases] || [] + @description = attributes[:description] || '' + @params = attributes[:params] || [] + @condition_block = attributes[:condition_block] + @action_block = attributes[:action_block] + end + + def all_names + [name, *aliases] + end + + def noop? + action_block.nil? + end + + def available?(opts) + return true unless condition_block + + context = OpenStruct.new(opts) + context.instance_exec(&condition_block) + end + + def execute(context, opts, arg) + return if noop? || !available?(opts) + + if arg.present? + context.instance_exec(arg, &action_block) + elsif action_block.arity == 0 + context.instance_exec(&action_block) + end + end + + def to_h(opts) + desc = description + if desc.respond_to?(:call) + context = OpenStruct.new(opts) + desc = context.instance_exec(&desc) rescue '' + end + + { + name: name, + aliases: aliases, + description: desc, + params: params + } + end + end + end +end diff --git a/lib/gitlab/slash_commands/dsl.rb b/lib/gitlab/slash_commands/dsl.rb new file mode 100644 index 00000000000..50b0937d267 --- /dev/null +++ b/lib/gitlab/slash_commands/dsl.rb @@ -0,0 +1,98 @@ +module Gitlab + module SlashCommands + module Dsl + extend ActiveSupport::Concern + + included do + cattr_accessor :command_definitions, instance_accessor: false do + [] + end + + cattr_accessor :command_definitions_by_name, instance_accessor: false do + {} + end + end + + class_methods do + # Allows to give a description to the next slash command. + # This description is shown in the autocomplete menu. + # It accepts a block that will be evaluated with the context given to + # `CommandDefintion#to_h`. + # + # Example: + # + # desc do + # "This is a dynamic description for #{noteable.to_ability_name}" + # end + # command :command_key do |arguments| + # # Awesome code block + # end + def desc(text = '', &block) + @description = block_given? ? block : text + end + + # Allows to define params for the next slash command. + # These params are shown in the autocomplete menu. + # + # Example: + # + # params "~label ~label2" + # command :command_key do |arguments| + # # Awesome code block + # end + def params(*params) + @params = params + end + + # Allows to define conditions that must be met in order for the command + # to be returned by `.command_names` & `.command_definitions`. + # It accepts a block that will be evaluated with the context given to + # `CommandDefintion#to_h`. + # + # Example: + # + # condition do + # project.public? + # end + # command :command_key do |arguments| + # # Awesome code block + # end + def condition(&block) + @condition_block = block + end + + # Registers a new command which is recognizeable from body of email or + # comment. + # It accepts aliases and takes a block. + # + # Example: + # + # command :my_command, :alias_for_my_command do |arguments| + # # Awesome code block + # end + def command(*command_names, &block) + name, *aliases = command_names + + definition = CommandDefinition.new( + name, + aliases: aliases, + description: @description, + params: @params, + condition_block: @condition_block, + action_block: block + ) + + self.command_definitions << definition + + definition.all_names.each do |name| + self.command_definitions_by_name[name] = definition + end + + @description = nil + @params = nil + @condition_block = nil + end + end + end + end +end diff --git a/lib/gitlab/slash_commands/extractor.rb b/lib/gitlab/slash_commands/extractor.rb new file mode 100644 index 00000000000..a672e5e4855 --- /dev/null +++ b/lib/gitlab/slash_commands/extractor.rb @@ -0,0 +1,122 @@ +module Gitlab + module SlashCommands + # This class takes an array of commands that should be extracted from a + # given text. + # + # ``` + # extractor = Gitlab::SlashCommands::Extractor.new([:open, :assign, :labels]) + # ``` + class Extractor + attr_reader :command_definitions + + def initialize(command_definitions) + @command_definitions = command_definitions + end + + # Extracts commands from content and return an array of commands. + # The array looks like the following: + # [ + # ['command1'], + # ['command3', 'arg1 arg2'], + # ] + # The command and the arguments are stripped. + # The original command text is removed from the given `content`. + # + # Usage: + # ``` + # extractor = Gitlab::SlashCommands::Extractor.new([:open, :assign, :labels]) + # msg = %(hello\n/labels ~foo ~"bar baz"\nworld) + # commands = extractor.extract_commands(msg) #=> [['labels', '~foo ~"bar baz"']] + # msg #=> "hello\nworld" + # ``` + def extract_commands(content, opts = {}) + return [content, []] unless content + + content = content.dup + + commands = [] + + content.delete!("\r") + content.gsub!(commands_regex(opts)) do + if $~[:cmd] + commands << [$~[:cmd], $~[:arg]].reject(&:blank?) + '' + else + $~[0] + end + end + + [content.strip, commands] + end + + private + + # Builds a regular expression to match known commands. + # First match group captures the command name and + # second match group captures its arguments. + # + # It looks something like: + # + # /^\/(?<cmd>close|reopen|...)(?:( |$))(?<arg>[^\/\n]*)(?:\n|$)/ + def commands_regex(opts) + names = command_names(opts).map(&:to_s) + + @commands_regex ||= %r{ + (?<code> + # Code blocks: + # ``` + # Anything, including `/cmd arg` which are ignored by this filter + # ``` + + ^``` + .+? + \n```$ + ) + | + (?<html> + # HTML block: + # <tag> + # Anything, including `/cmd arg` which are ignored by this filter + # </tag> + + ^<[^>]+?>\n + .+? + \n<\/[^>]+?>$ + ) + | + (?<html> + # Quote block: + # >>> + # Anything, including `/cmd arg` which are ignored by this filter + # >>> + + ^>>> + .+? + \n>>>$ + ) + | + (?: + # Command not in a blockquote, blockcode, or HTML tag: + # /close + + ^\/ + (?<cmd>#{Regexp.union(names)}) + (?: + [ ] + (?<arg>[^\/\n]*) + )? + (?:\n|$) + ) + }mx + end + + def command_names(opts) + command_definitions.flat_map do |command| + next if command.noop? + + command.all_names + end.compact + end + end + end +end diff --git a/lib/gitlab/snippet_search_results.rb b/lib/gitlab/snippet_search_results.rb index e0e74ff8359..9e01f02029c 100644 --- a/lib/gitlab/snippet_search_results.rb +++ b/lib/gitlab/snippet_search_results.rb @@ -20,10 +20,6 @@ module Gitlab end end - def total_count - @total_count ||= snippet_titles_count + snippet_blobs_count - end - def snippet_titles_count @snippet_titles_count ||= snippet_titles.count end diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index fe65c246101..99d0c28e749 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -22,6 +22,8 @@ module Gitlab note_url when WikiPage wiki_page_url + when ProjectSnippet + project_snippet_url(object) else raise NotImplementedError.new("No URL builder defined for #{object.class}") end diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index d13fe0ef8a9..4c395b4266e 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -7,11 +7,19 @@ module Gitlab # @param cmd [Array<String>] # @return [Boolean] def system_silent(cmd) - Popen::popen(cmd).last.zero? + Popen.popen(cmd).last.zero? end def force_utf8(str) str.force_encoding(Encoding::UTF_8) end + + def to_boolean(value) + return value if [true, false].include?(value) + return true if value =~ /^(true|t|yes|y|1|on)$/i + return false if value =~ /^(false|f|no|n|0|off)$/i + + nil + end end end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index c6826a09bd2..594439a5d4b 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -1,19 +1,38 @@ require 'base64' require 'json' +require 'securerandom' module Gitlab class Workhorse SEND_DATA_HEADER = 'Gitlab-Workhorse-Send-Data' VERSION_FILE = 'GITLAB_WORKHORSE_VERSION' + INTERNAL_API_CONTENT_TYPE = 'application/vnd.gitlab-workhorse+json' + INTERNAL_API_REQUEST_HEADER = 'Gitlab-Workhorse-Api-Request' + + # Supposedly the effective key size for HMAC-SHA256 is 256 bits, i.e. 32 + # bytes https://tools.ietf.org/html/rfc4868#section-2.6 + SECRET_LENGTH = 32 class << self def git_http_ok(repository, user) { - 'GL_ID' => Gitlab::GlId.gl_id(user), - 'RepoPath' => repository.path_to_repo, + GL_ID: Gitlab::GlId.gl_id(user), + RepoPath: repository.path_to_repo, + } + end + + def lfs_upload_ok(oid, size) + { + StoreLFSPath: "#{Gitlab.config.lfs.storage_path}/tmp/upload", + LfsOid: oid, + LfsSize: size, } end + def artifact_upload_ok + { TempPath: ArtifactUploader.artifacts_upload_path } + end + def send_git_blob(repository, blob) params = { 'RepoPath' => repository.path_to_repo, @@ -41,7 +60,7 @@ module Gitlab def send_git_diff(repository, diff_refs) params = { 'RepoPath' => repository.path_to_repo, - 'ShaFrom' => diff_refs.start_sha, + 'ShaFrom' => diff_refs.base_sha, 'ShaTo' => diff_refs.head_sha } @@ -54,7 +73,7 @@ module Gitlab def send_git_patch(repository, diff_refs) params = { 'RepoPath' => repository.path_to_repo, - 'ShaFrom' => diff_refs.start_sha, + 'ShaFrom' => diff_refs.base_sha, 'ShaTo' => diff_refs.head_sha } @@ -81,6 +100,35 @@ module Gitlab path.readable? ? path.read.chomp : 'unknown' end + def secret + @secret ||= begin + bytes = Base64.strict_decode64(File.read(secret_path).chomp) + raise "#{secret_path} does not contain #{SECRET_LENGTH} bytes" if bytes.length != SECRET_LENGTH + bytes + end + end + + def write_secret + bytes = SecureRandom.random_bytes(SECRET_LENGTH) + File.open(secret_path, 'w:BINARY', 0600) do |f| + f.chmod(0600) # If the file already existed, the '0600' passed to 'open' above was a no-op. + f.write(Base64.strict_encode64(bytes)) + end + end + + def verify_api_request!(request_headers) + JWT.decode( + request_headers[INTERNAL_API_REQUEST_HEADER], + secret, + true, + { iss: 'gitlab-workhorse', verify_iss: true, algorithm: 'HS256' }, + ) + end + + def secret_path + Rails.root.join('.gitlab_workhorse_secret') + end + protected def encode(hash) diff --git a/lib/tasks/.gitkeep b/lib/tasks/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 --- a/lib/tasks/.gitkeep +++ /dev/null diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake index 2214f855200..78ae187817a 100644 --- a/lib/tasks/cache.rake +++ b/lib/tasks/cache.rake @@ -1,22 +1,33 @@ namespace :cache do - CLEAR_BATCH_SIZE = 1000 # There seems to be no speedup when pushing beyond 1,000 - REDIS_SCAN_START_STOP = '0' # Magic value, see http://redis.io/commands/scan + namespace :clear do + REDIS_CLEAR_BATCH_SIZE = 1000 # There seems to be no speedup when pushing beyond 1,000 + REDIS_SCAN_START_STOP = '0' # Magic value, see http://redis.io/commands/scan - desc "GitLab | Clear redis cache" - task :clear => :environment do - Gitlab::Redis.with do |redis| - cursor = REDIS_SCAN_START_STOP - loop do - cursor, keys = redis.scan( - cursor, - match: "#{Gitlab::Redis::CACHE_NAMESPACE}*", - count: CLEAR_BATCH_SIZE - ) - - redis.del(*keys) if keys.any? - - break if cursor == REDIS_SCAN_START_STOP + desc "GitLab | Clear redis cache" + task redis: :environment do + Gitlab::Redis.with do |redis| + cursor = REDIS_SCAN_START_STOP + loop do + cursor, keys = redis.scan( + cursor, + match: "#{Gitlab::Redis::CACHE_NAMESPACE}*", + count: REDIS_CLEAR_BATCH_SIZE + ) + + redis.del(*keys) if keys.any? + + break if cursor == REDIS_SCAN_START_STOP + end end end + + desc "GitLab | Clear database cache (in the background)" + task db: :environment do + ClearDatabaseCacheWorker.perform_async + end + + task all: [:db, :redis] end + + task clear: 'cache:clear:redis' end diff --git a/lib/tasks/ci/.gitkeep b/lib/tasks/ci/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 --- a/lib/tasks/ci/.gitkeep +++ /dev/null diff --git a/lib/tasks/ee_compat_check.rake b/lib/tasks/ee_compat_check.rake new file mode 100644 index 00000000000..f494fa5c5c2 --- /dev/null +++ b/lib/tasks/ee_compat_check.rake @@ -0,0 +1,4 @@ +desc 'Checks if the branch would apply cleanly to EE' +task ee_compat_check: :environment do + Rake::Task['gitlab:dev:ee_compat_check'].invoke +end diff --git a/lib/tasks/eslint.rake b/lib/tasks/eslint.rake new file mode 100644 index 00000000000..d43cbad1909 --- /dev/null +++ b/lib/tasks/eslint.rake @@ -0,0 +1,7 @@ +unless Rails.env.production? + desc "GitLab | Run ESLint" + task :eslint do + system("npm", "run", "eslint") + end +end + diff --git a/lib/tasks/flog.rake b/lib/tasks/flog.rake deleted file mode 100644 index 3bfe999ae74..00000000000 --- a/lib/tasks/flog.rake +++ /dev/null @@ -1,25 +0,0 @@ -desc 'Code complexity analyze via flog' -task :flog do - output = %x(bundle exec flog -m app/ lib/gitlab) - exit_code = 0 - minimum_score = 70 - output = output.lines - - # Skip total complexity score - output.shift - - # Skip some trash info - output.shift - - output.each do |line| - score, method = line.split(" ") - score = score.to_i - - if score > minimum_score - exit_code = 1 - puts "High complexity in #{method}. Score: #{score}" - end - end - - exit exit_code -end diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index b43ee5b3383..a9f1255e8cf 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -51,6 +51,7 @@ namespace :gitlab do $progress.puts 'done'.color(:green) Rake::Task['gitlab:backup:db:restore'].invoke end + Rake::Task['gitlab:backup:repo:restore'].invoke unless backup.skipped?('repositories') Rake::Task['gitlab:backup:uploads:restore'].invoke unless backup.skipped?('uploads') Rake::Task['gitlab:backup:builds:restore'].invoke unless backup.skipped?('builds') @@ -58,6 +59,7 @@ namespace :gitlab do Rake::Task['gitlab:backup:lfs:restore'].invoke unless backup.skipped?('lfs') Rake::Task['gitlab:backup:registry:restore'].invoke unless backup.skipped?('registry') Rake::Task['gitlab:shell:setup'].invoke + Rake::Task['cache:clear'].invoke backup.cleanup end diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 5f4a6bbfa35..35c4194e87c 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -671,7 +671,7 @@ namespace :gitlab do "Enable mail_room in the init.d configuration." ) for_more_information( - "doc/incoming_email/README.md" + "doc/administration/reply_by_email.md" ) fix_and_rerun end @@ -690,7 +690,7 @@ namespace :gitlab do "Enable mail_room in your Procfile." ) for_more_information( - "doc/incoming_email/README.md" + "doc/administration/reply_by_email.md" ) fix_and_rerun end @@ -747,7 +747,7 @@ namespace :gitlab do "Check that the information in config/gitlab.yml is correct" ) for_more_information( - "doc/incoming_email/README.md" + "doc/administration/reply_by_email.md" ) fix_and_rerun end @@ -760,7 +760,7 @@ namespace :gitlab do end namespace :ldap do - task :check, [:limit] => :environment do |t, args| + task :check, [:limit] => :environment do |_, args| # Only show up to 100 results because LDAP directories can be very big. # This setting only affects the `rake gitlab:check` script. args.with_defaults(limit: 100) @@ -768,7 +768,7 @@ namespace :gitlab do start_checking "LDAP" if Gitlab::LDAP::Config.enabled? - print_users(args.limit) + check_ldap(args.limit) else puts 'LDAP is disabled in config/gitlab.yml' end @@ -776,21 +776,42 @@ namespace :gitlab do finished_checking "LDAP" end - def print_users(limit) - puts "LDAP users with access to your GitLab server (only showing the first #{limit} results)" - + def check_ldap(limit) servers = Gitlab::LDAP::Config.providers servers.each do |server| puts "Server: #{server}" - Gitlab::LDAP::Adapter.open(server) do |adapter| - users = adapter.users(adapter.config.uid, '*', limit) - users.each do |user| - puts "\tDN: #{user.dn}\t #{adapter.config.uid}: #{user.uid}" + + begin + Gitlab::LDAP::Adapter.open(server) do |adapter| + check_ldap_auth(adapter) + + puts "LDAP users with access to your GitLab server (only showing the first #{limit} results)" + + users = adapter.users(adapter.config.uid, '*', limit) + users.each do |user| + puts "\tDN: #{user.dn}\t #{adapter.config.uid}: #{user.uid}" + end end + rescue Net::LDAP::ConnectionRefusedError, Errno::ECONNREFUSED => e + puts "Could not connect to the LDAP server: #{e.message}".color(:red) end end end + + def check_ldap_auth(adapter) + auth = adapter.config.has_auth? + + if auth && adapter.ldap.bind + message = 'Success'.color(:green) + elsif auth + message = 'Failed. Check `bind_dn` and `password` configuration values'.color(:red) + else + message = 'Anonymous. No `bind_dn` or `password` configured'.color(:yellow) + end + + puts "LDAP authentication... #{message}" + end end namespace :repo do diff --git a/lib/tasks/gitlab/dev.rake b/lib/tasks/gitlab/dev.rake new file mode 100644 index 00000000000..3117075b08b --- /dev/null +++ b/lib/tasks/gitlab/dev.rake @@ -0,0 +1,26 @@ +namespace :gitlab do + namespace :dev do + desc 'Checks if the branch would apply cleanly to EE' + task :ee_compat_check, [:branch] => :environment do |_, args| + opts = + if ENV['CI'] + { + branch: ENV['CI_BUILD_REF_NAME'], + ce_repo: ENV['CI_BUILD_REPO'] + } + else + unless args[:branch] + puts "Must specify a branch as an argument".color(:red) + exit 1 + end + args + end + + if Gitlab::EeCompatCheck.new(opts || {}).check + exit 0 + else + exit 1 + end + end + end +end diff --git a/lib/tasks/gitlab/generate_docs.rake b/lib/tasks/gitlab/generate_docs.rake deleted file mode 100644 index f6448c38e10..00000000000 --- a/lib/tasks/gitlab/generate_docs.rake +++ /dev/null @@ -1,7 +0,0 @@ -namespace :gitlab do - desc "GitLab | Generate sdocs for project" - task generate_docs: :environment do - system(*%W(bundle exec sdoc -o doc/code app lib)) - end -end - diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index bb7eb852f1b..58761a129d4 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -63,11 +63,11 @@ namespace :gitlab do # Launch installation process system(*%W(bin/install) + repository_storage_paths_args) - - # (Re)create hooks - system(*%W(bin/create-hooks) + repository_storage_paths_args) end + # (Re)create hooks + Rake::Task['gitlab:shell:create_hooks'].invoke + # Required for debian packaging with PKGR: Setup .ssh/environment with # the current PATH, so that the correct ruby version gets loaded # Requires to set "PermitUserEnvironment yes" in sshd config (should not @@ -78,7 +78,7 @@ namespace :gitlab do f.puts "PATH=#{ENV['PATH']}" end - Gitlab::Shell.new.generate_and_link_secret_token + Gitlab::Shell.ensure_secret_token! end desc "GitLab | Setup gitlab-shell" @@ -102,6 +102,15 @@ namespace :gitlab do end end end + + desc 'Create or repair repository hooks symlink' + task create_hooks: :environment do + warn_user_is_not_gitlab + + puts 'Creating/Repairing hooks symlinks for all repositories' + system(*%W(#{Gitlab.config.gitlab_shell.path}/bin/create-hooks) + repository_storage_paths_args) + puts 'done'.color(:green) + end end def setup diff --git a/lib/tasks/gitlab/users.rake b/lib/tasks/gitlab/users.rake new file mode 100644 index 00000000000..3a16ace60bd --- /dev/null +++ b/lib/tasks/gitlab/users.rake @@ -0,0 +1,11 @@ +namespace :gitlab do + namespace :users do + desc "GitLab | Clear the authentication token for all users" + task clear_all_authentication_tokens: :environment do |t, args| + # Do small batched updates because these updates will be slow and locking + User.select(:id).find_in_batches(batch_size: 100) do |batch| + User.where(id: batch.map(&:id)).update_all(authentication_token: nil) + end + end + end +end diff --git a/lib/tasks/haml-lint.rake b/lib/tasks/haml-lint.rake new file mode 100644 index 00000000000..609dfaa48e3 --- /dev/null +++ b/lib/tasks/haml-lint.rake @@ -0,0 +1,5 @@ +unless Rails.env.production? + require 'haml_lint/rake_task' + + HamlLint::RakeTask.new +end diff --git a/lib/tasks/lint.rake b/lib/tasks/lint.rake new file mode 100644 index 00000000000..32b668df3bf --- /dev/null +++ b/lib/tasks/lint.rake @@ -0,0 +1,9 @@ +unless Rails.env.production? + namespace :lint do + desc "GitLab | lint | Lint JavaScript files using ESLint" + task :javascript do + Rake::Task['eslint'].invoke + end + end +end + diff --git a/lib/tasks/teaspoon.rake b/lib/tasks/teaspoon.rake new file mode 100644 index 00000000000..08caedd7ff3 --- /dev/null +++ b/lib/tasks/teaspoon.rake @@ -0,0 +1,25 @@ +unless Rails.env.production? + Rake::Task['teaspoon'].clear if Rake::Task.task_defined?('teaspoon') + + namespace :teaspoon do + desc 'GitLab | Teaspoon | Generate fixtures for JavaScript tests' + RSpec::Core::RakeTask.new(:fixtures) do |t| + ENV['NO_KNAPSACK'] = 'true' + t.pattern = 'spec/javascripts/fixtures/*.rb' + t.rspec_opts = '--format documentation' + end + + desc 'GitLab | Teaspoon | Run JavaScript tests' + task :tests do + require "teaspoon/console" + options = {} + abort('rake teaspoon:tests failed') if Teaspoon::Console.new(options).failures? + end + end + + desc 'GitLab | Teaspoon | Shortcut for teaspoon:fixtures and teaspoon:tests' + task :teaspoon do + Rake::Task['teaspoon:fixtures'].invoke + Rake::Task['teaspoon:tests'].invoke + end +end |