diff options
Diffstat (limited to 'lib')
329 files changed, 11615 insertions, 3509 deletions
diff --git a/lib/api/access_requests.rb b/lib/api/access_requests.rb index 87915b19480..789f45489eb 100644 --- a/lib/api/access_requests.rb +++ b/lib/api/access_requests.rb @@ -1,5 +1,7 @@ module API class AccessRequests < Grape::API + include PaginationParams + before { authenticate! } helpers ::API::Helpers::MembersHelpers @@ -13,6 +15,9 @@ module API detail 'This feature was introduced in GitLab 8.11.' success Entities::AccessRequester end + params do + use :pagination + end get ":id/access_requests" do source = find_source(source_type, params[:id]) @@ -48,7 +53,7 @@ module API put ':id/access_requests/:user_id/approve' do source = find_source(source_type, params[:id]) - member = ::Members::ApproveAccessRequestService.new(source, current_user, declared(params)).execute + member = ::Members::ApproveAccessRequestService.new(source, current_user, declared_params).execute status :created present member.user, with: Entities::Member, member: member diff --git a/lib/api/api.rb b/lib/api/api.rb index 67109ceeef9..6cf6b501021 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -3,6 +3,8 @@ module API include APIGuard version 'v3', using: :path + before { allow_access_with_scope :api } + rescue_from Gitlab::Access::AccessDeniedError do rack_response({ 'message' => '403 Forbidden' }.to_json, 403) end @@ -12,7 +14,11 @@ module API end # Retain 405 error rather than a 500 error for Grape 0.15.0+. - # See: https://github.com/ruby-grape/grape/commit/252bfd27c320466ec3c0751812cf44245e97e5de + # https://github.com/ruby-grape/grape/blob/a3a28f5b5dfbb2797442e006dbffd750b27f2a76/UPGRADING.md#changes-to-method-not-allowed-routes + rescue_from Grape::Exceptions::MethodNotAllowed do |e| + error! e.message, e.status, e.headers + end + rescue_from Grape::Exceptions::Base do |e| error! e.message, e.status, e.headers end @@ -64,6 +70,7 @@ module API mount ::API::Session mount ::API::Settings mount ::API::SidekiqMetrics + mount ::API::Snippets mount ::API::Subscriptions mount ::API::SystemHooks mount ::API::Tags diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index 8cc7a26f1fa..df6db140d0e 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -6,6 +6,9 @@ module API module APIGuard extend ActiveSupport::Concern + PRIVATE_TOKEN_HEADER = "HTTP_PRIVATE_TOKEN" + PRIVATE_TOKEN_PARAM = :private_token + included do |base| # OAuth2 Resource Server Authentication use Rack::OAuth2::Server::Resource::Bearer, 'The API' do |request| @@ -44,27 +47,60 @@ module API access_token = find_access_token return nil unless access_token - case validate_access_token(access_token, scopes) - when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE + case AccessTokenValidationService.new(access_token).validate(scopes: scopes) + when AccessTokenValidationService::INSUFFICIENT_SCOPE raise InsufficientScopeError.new(scopes) - when Oauth2::AccessTokenValidationService::EXPIRED + when AccessTokenValidationService::EXPIRED raise ExpiredError - when Oauth2::AccessTokenValidationService::REVOKED + when AccessTokenValidationService::REVOKED raise RevokedError - when Oauth2::AccessTokenValidationService::VALID + when AccessTokenValidationService::VALID @current_user = User.find(access_token.resource_owner_id) end end + def find_user_by_private_token(scopes: []) + token_string = (params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER]).to_s + + return nil unless token_string.present? + + find_user_by_authentication_token(token_string) || find_user_by_personal_access_token(token_string, scopes) + end + def current_user @current_user end + # Set the authorization scope(s) allowed for the current request. + # + # Note: A call to this method adds to any previous scopes in place. This is done because + # `Grape` callbacks run from the outside-in: the top-level callback (API::API) runs first, then + # the next-level callback (API::API::Users, for example) runs. All these scopes are valid for the + # given endpoint (GET `/api/users` is accessible by the `api` and `read_user` scopes), and so they + # need to be stored. + def allow_access_with_scope(*scopes) + @scopes ||= [] + @scopes.concat(scopes.map(&:to_s)) + end + private + def find_user_by_authentication_token(token_string) + User.find_by_authentication_token(token_string) + end + + def find_user_by_personal_access_token(token_string, scopes) + access_token = PersonalAccessToken.active.find_by_token(token_string) + return unless access_token + + if AccessTokenValidationService.new(access_token).include_any_scope?(scopes) + User.find(access_token.user_id) + end + end + def find_access_token @access_token ||= Doorkeeper.authenticate(doorkeeper_request, Doorkeeper.configuration.access_token_methods) end @@ -72,10 +108,6 @@ module API def doorkeeper_request @doorkeeper_request ||= ActionDispatch::Request.new(env) end - - def validate_access_token(access_token, scopes) - Oauth2::AccessTokenValidationService.validate(access_token, scopes: scopes) - end end module ClassMethods diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb index e9ccba3b465..58a4df54bea 100644 --- a/lib/api/award_emoji.rb +++ b/lib/api/award_emoji.rb @@ -1,5 +1,7 @@ module API class AwardEmoji < Grape::API + include PaginationParams + before { authenticate! } AWARDABLES = %w[issue merge_request snippet] @@ -21,6 +23,9 @@ module API detail 'This feature was introduced in 8.9' success Entities::AwardEmoji end + params do + use :pagination + end get endpoint do if can_read_awardable? awards = paginate(awardable.award_emoji) diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 21a106387f0..0950c3d2e88 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -23,9 +23,9 @@ module API success Entities::RepoBranch end params do - requires :branch, type: String, regexp: /.+/, desc: 'The name of the branch' + requires :branch, type: String, desc: 'The name of the branch' end - get ':id/repository/branches/:branch' do + get ':id/repository/branches/:branch', requirements: { branch: /.+/ } do branch = user_project.repository.find_branch(params[:branch]) not_found!("Branch") unless branch @@ -39,11 +39,11 @@ module API success Entities::RepoBranch end params do - requires :branch, type: String, regexp: /.+/, desc: 'The name of the branch' + requires :branch, type: String, desc: 'The name of the branch' optional :developers_can_push, type: Boolean, desc: 'Flag if developers can push to that branch' optional :developers_can_merge, type: Boolean, desc: 'Flag if developers can merge to that branch' end - put ':id/repository/branches/:branch/protect' do + put ':id/repository/branches/:branch/protect', requirements: { branch: /.+/ } do authorize_admin_project branch = user_project.repository.find_branch(params[:branch]) @@ -76,9 +76,9 @@ module API success Entities::RepoBranch end params do - requires :branch, type: String, regexp: /.+/, desc: 'The name of the branch' + requires :branch, type: String, desc: 'The name of the branch' end - put ':id/repository/branches/:branch/unprotect' do + put ':id/repository/branches/:branch/unprotect', requirements: { branch: /.+/ } do authorize_admin_project branch = user_project.repository.find_branch(params[:branch]) @@ -112,9 +112,9 @@ module API desc 'Delete a branch' params do - requires :branch, type: String, regexp: /.+/, desc: 'The name of the branch' + requires :branch, type: String, desc: 'The name of the branch' end - delete ":id/repository/branches/:branch" do + delete ":id/repository/branches/:branch", requirements: { branch: /.+/ } do authorize_push_project result = DeleteBranchService.new(user_project, current_user). @@ -128,6 +128,18 @@ module API render_api_error!(result[:message], result[:return_code]) end end + + # Delete all merged branches + # + # Parameters: + # id (required) - The ID of a project + # Example Request: + # DELETE /projects/:id/repository/branches/delete_merged + delete ":id/repository/merged_branches" do + DeleteMergedBranchesService.new(user_project, current_user).async_execute + + status(200) + end end end end diff --git a/lib/api/broadcast_messages.rb b/lib/api/broadcast_messages.rb index fb2a4148011..1217002bf8e 100644 --- a/lib/api/broadcast_messages.rb +++ b/lib/api/broadcast_messages.rb @@ -1,5 +1,7 @@ module API class BroadcastMessages < Grape::API + include PaginationParams + before { authenticate! } before { authenticated_as_admin! } @@ -15,8 +17,7 @@ module API success Entities::BroadcastMessage end params do - optional :page, type: Integer, desc: 'Current page number' - optional :per_page, type: Integer, desc: 'Number of messages per page' + use :pagination end get do messages = BroadcastMessage.all @@ -36,8 +37,7 @@ module API optional :font, type: String, desc: 'Foreground color' end post do - create_params = declared(params, include_missing: false).to_h - message = BroadcastMessage.create(create_params) + message = BroadcastMessage.create(declared_params(include_missing: false)) if message.persisted? present message, with: Entities::BroadcastMessage @@ -73,9 +73,8 @@ module API end put ':id' do message = find_message - update_params = declared(params, include_missing: false).to_h - if message.update(update_params) + if message.update(declared_params(include_missing: false)) present message, with: Entities::BroadcastMessage else render_validation_error!(message) diff --git a/lib/api/builds.rb b/lib/api/builds.rb index 67adca6605f..af61be343be 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -1,6 +1,7 @@ module API - # Projects builds API class Builds < Grape::API + include PaginationParams + before { authenticate! } params do @@ -28,6 +29,7 @@ module API end params do use :optional_scope + use :pagination end get ':id/builds' do builds = user_project.builds.order('id DESC') @@ -41,8 +43,9 @@ module API success Entities::Build end params do - requires :sha, type: String, desc: 'The SHA id of a commit' + requires :sha, type: String, desc: 'The SHA id of a commit' use :optional_scope + use :pagination end get ':id/repository/commits/:sha/builds' do authorize_read_builds! diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index f54d4f06627..b6e6820c3f4 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -1,9 +1,10 @@ require 'mime/types' module API - # Project commit statuses API class CommitStatuses < Grape::API resource :projects do + include PaginationParams + before { authenticate! } desc "Get a commit's statuses" do @@ -16,6 +17,7 @@ module API optional :stage, type: String, desc: 'The stage' optional :name, type: String, desc: 'The name' optional :all, type: String, desc: 'Show all statuses, default: false' + use :pagination end get ':id/repository/commits/:sha/statuses' do authorize!(:read_commit_status, user_project) @@ -76,8 +78,10 @@ module API description: params[:description] ) + render_validation_error!(status) if status.invalid? + begin - case params[:state].to_s + case params[:state] when 'pending' status.enqueue! when 'running' diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 2f2cf769481..e6d707f3c3d 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -1,8 +1,9 @@ require 'mime/types' module API - # Projects commits API class Commits < Grape::API + include PaginationParams + before { authenticate! } before { authorize! :download_code, user_project } @@ -43,17 +44,16 @@ module API detail 'This feature was introduced in GitLab 8.13' end params do - requires :id, type: Integer, desc: 'The project ID' requires :branch_name, type: String, desc: 'The name of branch' requires :commit_message, type: String, desc: 'Commit message' - requires :actions, type: Array, desc: 'Actions to perform in commit' + requires :actions, type: Array[Hash], desc: 'Actions to perform in commit' optional :author_email, type: String, desc: 'Author email for commit' optional :author_name, type: String, desc: 'Author name for commit' end post ":id/repository/commits" do authorize! :push_code, user_project - attrs = declared(params) + attrs = declared_params attrs[:source_branch] = attrs[:branch_name] attrs[:target_branch] = attrs[:branch_name] attrs[:actions].map! do |action| @@ -107,9 +107,8 @@ module API failure [[404, 'Not Found']] end params do + use :pagination requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' - optional :per_page, type: Integer, desc: 'The amount of items per page for paginaion' - optional :page, type: Integer, desc: 'The page number for pagination' end get ':id/repository/commits/:sha/comments' do commit = user_project.commit(params[:sha]) @@ -120,6 +119,41 @@ module API present paginate(notes), with: Entities::CommitNote end + desc 'Cherry pick commit into a branch' do + detail 'This feature was introduced in GitLab 8.15' + success Entities::RepoCommit + end + params do + requires :sha, type: String, desc: 'A commit sha to be cherry picked' + requires :branch, type: String, desc: 'The name of the branch' + end + post ':id/repository/commits/:sha/cherry_pick' do + authorize! :push_code, user_project + + commit = user_project.commit(params[:sha]) + not_found!('Commit') unless commit + + branch = user_project.repository.find_branch(params[:branch]) + not_found!('Branch') unless branch + + commit_params = { + commit: commit, + create_merge_request: false, + source_project: user_project, + source_branch: commit.cherry_pick_branch_name, + target_branch: params[:branch] + } + + result = ::Commits::CherryPickService.new(user_project, current_user, commit_params).execute + + if result[:status] == :success + branch = user_project.repository.find_branch(params[:branch]) + present user_project.repository.commit(branch.dereferenced_target), with: Entities::RepoCommit + else + render_api_error!(result[:message], 400) + end + end + desc 'Post comment to commit' do success Entities::CommitNote end diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb index 425df2c176a..f6cb17bafd8 100644 --- a/lib/api/deploy_keys.rb +++ b/lib/api/deploy_keys.rb @@ -82,7 +82,7 @@ module API end post ":id/#{path}/:key_id/enable" do key = ::Projects::EnableDeployKeyService.new(user_project, - current_user, declared(params)).execute + current_user, declared_params).execute if key present key, with: Entities::SSHKey @@ -105,15 +105,19 @@ module API present key.deploy_key, with: Entities::SSHKey end - desc 'Delete existing deploy key of currently authenticated user' do + desc 'Delete deploy key for a project' do success Key end params do requires :key_id, type: Integer, desc: 'The ID of the deploy key' end delete ":id/#{path}/:key_id" do - key = user_project.deploy_keys.find(params[:key_id]) - key.destroy + key = user_project.deploy_keys_projects.find_by(deploy_key_id: params[:key_id]) + if key + key.destroy + else + not_found!('Deploy Key') + end end end end diff --git a/lib/api/deployments.rb b/lib/api/deployments.rb index f782bcaf7e9..c5feb49b22f 100644 --- a/lib/api/deployments.rb +++ b/lib/api/deployments.rb @@ -1,6 +1,8 @@ module API # Deployments RESTfull API endpoints class Deployments < Grape::API + include PaginationParams + before { authenticate! } params do @@ -12,8 +14,7 @@ module API success Entities::Deployment end params do - optional :page, type: Integer, desc: 'Page number of the current request' - optional :per_page, type: Integer, desc: 'Number of items per page' + use :pagination end get ':id/deployments' do authorize! :read_deployment, user_project diff --git a/lib/api/entities.rb b/lib/api/entities.rb index ab9d2d54f4b..9f59939e9ae 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -22,7 +22,7 @@ module API expose :provider, :extern_uid end - class UserFull < User + class UserPublic < User expose :last_sign_in_at expose :confirmed_at expose :email @@ -34,7 +34,7 @@ module API expose :external end - class UserLogin < UserFull + class UserWithPrivateToken < UserPublic expose :private_token end @@ -43,14 +43,13 @@ module API end class Hook < Grape::Entity - expose :id, :url, :created_at + expose :id, :url, :created_at, :push_events, :tag_push_events + expose :enable_ssl_verification end class ProjectHook < Hook - expose :project_id, :push_events - expose :issues_events, :merge_requests_events, :tag_push_events + expose :project_id, :issues_events, :merge_requests_events expose :note_events, :build_events, :pipeline_events, :wiki_page_events - expose :enable_ssl_verification end class BasicProjectDetails < Grape::Entity @@ -79,21 +78,21 @@ module API expose :container_registry_enabled # Expose old field names with the new permissions methods to keep API compatible - expose(:issues_enabled) { |project, options| project.feature_available?(:issues, options[:user]) } - expose(:merge_requests_enabled) { |project, options| project.feature_available?(:merge_requests, options[:user]) } - expose(:wiki_enabled) { |project, options| project.feature_available?(:wiki, options[:user]) } - expose(:builds_enabled) { |project, options| project.feature_available?(:builds, options[:user]) } - expose(:snippets_enabled) { |project, options| project.feature_available?(:snippets, options[:user]) } + expose(:issues_enabled) { |project, options| project.feature_available?(:issues, options[:current_user]) } + expose(:merge_requests_enabled) { |project, options| project.feature_available?(:merge_requests, options[:current_user]) } + expose(:wiki_enabled) { |project, options| project.feature_available?(:wiki, options[:current_user]) } + expose(:builds_enabled) { |project, options| project.feature_available?(:builds, options[:current_user]) } + expose(:snippets_enabled) { |project, options| project.feature_available?(:snippets, options[:current_user]) } expose :created_at, :last_activity_at expose :shared_runners_enabled expose :lfs_enabled?, as: :lfs_enabled expose :creator_id - expose :namespace + expose :namespace, using: 'API::Entities::Namespace' expose :forked_from_project, using: Entities::BasicProjectDetails, if: lambda{ |project, options| project.forked? } expose :avatar_url expose :star_count, :forks_count - expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:user]) && project.default_issues_tracker? } + expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:current_user]) && project.default_issues_tracker? } expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] } expose :public_builds expose :shared_with_groups do |project, options| @@ -101,6 +100,17 @@ module API end expose :only_allow_merge_if_build_succeeds expose :request_access_enabled + expose :only_allow_merge_if_all_discussions_are_resolved + + expose :statistics, using: 'API::Entities::ProjectStatistics', if: :statistics + end + + class ProjectStatistics < Grape::Entity + expose :commit_count + expose :storage_size + expose :repository_size + expose :lfs_objects_size + expose :build_artifacts_size end class Member < UserBasic @@ -127,6 +137,15 @@ module API expose :avatar_url expose :web_url expose :request_access_enabled + + expose :statistics, if: :statistics do + with_options format_with: -> (value) { value.to_i } do + expose :storage_size + expose :repository_size + expose :lfs_objects_size + expose :build_artifacts_size + end + end end class GroupDetail < Group @@ -141,8 +160,12 @@ module API options[:project].repository.commit(repo_branch.dereferenced_target) end + expose :merged do |repo_branch, options| + options[:project].repository.merged_to_root_ref?(repo_branch.name) + end + expose :protected do |repo_branch, options| - options[:project].protected_branch? repo_branch.name + options[:project].protected_branch?(repo_branch.name) end expose :developers_can_push do |repo_branch, options| @@ -159,7 +182,7 @@ module API end class RepoTreeObject < Grape::Entity - expose :id, :name, :type + expose :id, :name, :type, :path expose :mode do |obj, options| filemode = obj.mode.to_s(8) @@ -170,6 +193,7 @@ module API class RepoCommit < Grape::Entity expose :id, :short_id, :title, :author_name, :author_email, :created_at + expose :committer_name, :committer_email expose :safe_message, as: :message end @@ -196,6 +220,19 @@ module API end end + class PersonalSnippet < Grape::Entity + expose :id, :title, :file_name + expose :author, using: Entities::UserBasic + expose :updated_at, :created_at + + expose :web_url do |snippet| + Gitlab::UrlBuilder.build(snippet) + end + expose :raw_url do |snippet| + Gitlab::UrlBuilder.build(snippet) + "/raw" + end + end + class ProjectEntity < Grape::Entity expose :id, :iid expose(:project_id) { |entity| entity.project.id } @@ -210,6 +247,7 @@ module API class Milestone < ProjectEntity expose :due_date + expose :start_date end class Issue < ProjectEntity @@ -218,7 +256,7 @@ module API expose :assignee, :author, using: Entities::UserBasic expose :subscribed do |issue, options| - issue.subscribed?(options[:current_user]) + issue.subscribed?(options[:current_user], options[:project] || issue.project) end expose :user_notes_count expose :upvotes, :downvotes @@ -230,6 +268,13 @@ module API end end + class IssuableTimeStats < Grape::Entity + expose :time_estimate + expose :total_time_spent + expose :human_time_estimate + expose :human_total_time_spent + end + class ExternalIssue < Grape::Entity expose :title expose :id @@ -248,7 +293,7 @@ module API expose :diff_head_sha, as: :sha expose :merge_commit_sha expose :subscribed do |merge_request, options| - merge_request.subscribed?(options[:current_user]) + merge_request.subscribed?(options[:current_user], options[:project]) end expose :user_notes_count expose :should_remove_source_branch?, as: :should_remove_source_branch @@ -279,11 +324,11 @@ module API end class SSHKey < Grape::Entity - expose :id, :title, :key, :created_at + expose :id, :title, :key, :created_at, :can_push end class SSHKeyWithUser < SSHKey - expose :user, using: Entities::UserFull + expose :user, using: Entities::UserPublic end class Note < Grape::Entity @@ -372,7 +417,7 @@ module API end class Namespace < Grape::Entity - expose :id, :path, :kind + expose :id, :name, :path, :kind end class MemberAccess < Grape::Entity @@ -421,26 +466,40 @@ module API class ProjectWithAccess < Project expose :permissions do expose :project_access, using: Entities::ProjectAccess do |project, options| - project.project_members.find_by(user_id: options[:user].id) + project.project_members.find_by(user_id: options[:current_user].id) end expose :group_access, using: Entities::GroupAccess do |project, options| if project.group - project.group.group_members.find_by(user_id: options[:user].id) + project.group.group_members.find_by(user_id: options[:current_user].id) end end end end class LabelBasic < Grape::Entity - expose :name, :color, :description + expose :id, :name, :color, :description end class Label < LabelBasic - expose :open_issues_count, :closed_issues_count, :open_merge_requests_count + expose :open_issues_count do |label, options| + label.open_issues_count(options[:current_user]) + end + + expose :closed_issues_count do |label, options| + label.closed_issues_count(options[:current_user]) + end + + expose :open_merge_requests_count do |label, options| + label.open_merge_requests_count(options[:current_user]) + end + + expose :priority do |label, options| + label.priority(options[:project]) + end expose :subscribed do |label, options| - label.subscribed?(options[:current_user]) + label.subscribed?(options[:current_user], options[:project]) end end @@ -510,8 +569,11 @@ module API expose :after_sign_out_path expose :container_registry_token_expire_delay expose :repository_storage + expose :repository_storages expose :koding_enabled expose :koding_url + expose :plantuml_enabled + expose :plantuml_url end class Release < Grape::Entity @@ -591,10 +653,11 @@ module API expose :user, with: Entities::UserBasic expose :created_at, :updated_at, :started_at, :finished_at, :committed_at expose :duration + expose :coverage end class EnvironmentBasic < Grape::Entity - expose :id, :name, :external_url + expose :id, :name, :slug, :external_url end class Environment < EnvironmentBasic diff --git a/lib/api/environments.rb b/lib/api/environments.rb index 819f80d8365..1a7e68f0528 100644 --- a/lib/api/environments.rb +++ b/lib/api/environments.rb @@ -1,6 +1,9 @@ module API # Environments RESTfull API endpoints class Environments < Grape::API + include ::API::Helpers::CustomValidators + include PaginationParams + before { authenticate! } params do @@ -12,8 +15,7 @@ module API success Entities::Environment end params do - optional :page, type: Integer, desc: 'Page number of the current request' - optional :per_page, type: Integer, desc: 'Number of items per page' + use :pagination end get ':id/environments' do authorize! :read_environment, user_project @@ -28,12 +30,12 @@ module API params do requires :name, type: String, desc: 'The name of the environment to be created' optional :external_url, type: String, desc: 'URL on which this deployment is viewable' + optional :slug, absence: { message: "is automatically generated and cannot be changed" } end post ':id/environments' do authorize! :create_environment, user_project - create_params = declared(params, include_parent_namespaces: false).to_h - environment = user_project.environments.create(create_params) + environment = user_project.environments.create(declared_params) if environment.persisted? present environment, with: Entities::Environment @@ -50,13 +52,14 @@ module API requires :environment_id, type: Integer, desc: 'The environment ID' optional :name, type: String, desc: 'The new environment name' optional :external_url, type: String, desc: 'The new URL on which this deployment is viewable' + optional :slug, absence: { message: "is automatically generated and cannot be changed" } end put ':id/environments/:environment_id' do authorize! :update_environment, user_project environment = user_project.environments.find(params[:environment_id]) - - update_params = declared(params, include_missing: false).extract!(:name, :external_url).to_h + + update_params = declared_params(include_missing: false).extract!(:name, :external_url) if environment.update(update_params) present environment, with: Entities::Environment else diff --git a/lib/api/files.rb b/lib/api/files.rb index 96510e651a3..2e79e22e649 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -1,8 +1,6 @@ module API # Projects API class Files < Grape::API - before { authenticate! } - helpers do def commit_params(attrs) { @@ -23,140 +21,107 @@ module API branch_name: attrs[:branch_name] } end + + params :simple_file_params do + requires :file_path, type: String, desc: 'The path to new file. Ex. lib/class.rb' + requires :branch_name, type: String, desc: 'The name of branch' + requires :commit_message, type: String, desc: 'Commit Message' + optional :author_email, type: String, desc: 'The email of the author' + optional :author_name, type: String, desc: 'The name of the author' + end + + params :extended_file_params do + use :simple_file_params + requires :content, type: String, desc: 'File content' + optional :encoding, type: String, values: %w[base64], desc: 'File encoding' + end end + params do + requires :id, type: String, desc: 'The project ID' + end resource :projects do - # Get file from repository - # File content is Base64 encoded - # - # Parameters: - # file_path (required) - The path to the file. Ex. lib/class.rb - # ref (required) - The name of branch, tag or commit - # - # Example Request: - # GET /projects/:id/repository/files - # - # Example response: - # { - # "file_name": "key.rb", - # "file_path": "app/models/key.rb", - # "size": 1476, - # "encoding": "base64", - # "content": "IyA9PSBTY2hlbWEgSW5mb3...", - # "ref": "master", - # "blob_id": "79f7bbd25901e8334750839545a9bd021f0e4c83", - # "commit_id": "d5a3ff139356ce33e37e73add446f16869741b50", - # "last_commit_id": "570e7b2abdd848b95f2f578043fc23bd6f6fd24d", - # } - # + desc 'Get a file from repository' + params do + requires :file_path, type: String, desc: 'The path to the file. Ex. lib/class.rb' + requires :ref, type: String, desc: 'The name of branch, tag, or commit' + end get ":id/repository/files" do authorize! :download_code, user_project - required_attributes! [:file_path, :ref] - attrs = attributes_for_keys [:file_path, :ref] - ref = attrs.delete(:ref) - file_path = attrs.delete(:file_path) - - commit = user_project.commit(ref) - not_found! 'Commit' unless commit + commit = user_project.commit(params[:ref]) + not_found!('Commit') unless commit repo = user_project.repository - blob = repo.blob_at(commit.sha, file_path) + blob = repo.blob_at(commit.sha, params[:file_path]) + not_found!('File') unless blob - if blob - blob.load_all_data!(repo) - status(200) + blob.load_all_data!(repo) + status(200) - { - file_name: blob.name, - file_path: blob.path, - size: blob.size, - encoding: "base64", - content: Base64.strict_encode64(blob.data), - ref: ref, - blob_id: blob.id, - commit_id: commit.id, - last_commit_id: repo.last_commit_for_path(commit.sha, file_path).id - } - else - not_found! 'File' - end + { + file_name: blob.name, + file_path: blob.path, + size: blob.size, + encoding: "base64", + content: Base64.strict_encode64(blob.data), + ref: params[:ref], + blob_id: blob.id, + commit_id: commit.id, + last_commit_id: repo.last_commit_id_for_path(commit.sha, params[:file_path]) + } end - # Create new file in repository - # - # Parameters: - # file_path (required) - The path to new file. Ex. lib/class.rb - # branch_name (required) - The name of branch - # content (required) - File content - # commit_message (required) - Commit message - # - # Example Request: - # POST /projects/:id/repository/files - # + desc 'Create new file in repository' + params do + use :extended_file_params + end post ":id/repository/files" do authorize! :push_code, user_project - required_attributes! [:file_path, :branch_name, :content, :commit_message] - attrs = attributes_for_keys [:file_path, :branch_name, :content, :commit_message, :encoding, :author_email, :author_name] - result = ::Files::CreateService.new(user_project, current_user, commit_params(attrs)).execute + file_params = declared_params(include_missing: false) + result = ::Files::CreateService.new(user_project, current_user, commit_params(file_params)).execute if result[:status] == :success status(201) - commit_response(attrs) + commit_response(file_params) else render_api_error!(result[:message], 400) end end - # Update existing file in repository - # - # Parameters: - # file_path (optional) - The path to file. Ex. lib/class.rb - # branch_name (required) - The name of branch - # content (required) - File content - # commit_message (required) - Commit message - # - # Example Request: - # PUT /projects/:id/repository/files - # + desc 'Update existing file in repository' + params do + use :extended_file_params + end put ":id/repository/files" do authorize! :push_code, user_project - required_attributes! [:file_path, :branch_name, :content, :commit_message] - attrs = attributes_for_keys [:file_path, :branch_name, :content, :commit_message, :encoding, :author_email, :author_name] - result = ::Files::UpdateService.new(user_project, current_user, commit_params(attrs)).execute + file_params = declared_params(include_missing: false) + result = ::Files::UpdateService.new(user_project, current_user, commit_params(file_params)).execute if result[:status] == :success status(200) - commit_response(attrs) + commit_response(file_params) else http_status = result[:http_status] || 400 render_api_error!(result[:message], http_status) end end - # Delete existing file in repository - # - # Parameters: - # file_path (optional) - The path to file. Ex. lib/class.rb - # branch_name (required) - The name of branch - # content (required) - File content - # commit_message (required) - Commit message - # - # Example Request: - # DELETE /projects/:id/repository/files - # + desc 'Delete an existing file in repository' + params do + use :simple_file_params + end delete ":id/repository/files" do authorize! :push_code, user_project - required_attributes! [:file_path, :branch_name, :commit_message] - attrs = attributes_for_keys [:file_path, :branch_name, :commit_message, :author_email, :author_name] - result = ::Files::DeleteService.new(user_project, current_user, commit_params(attrs)).execute + file_params = declared_params(include_missing: false) + result = ::Files::DeleteService.new(user_project, current_user, commit_params(file_params)).execute if result[:status] == :success status(200) - commit_response(attrs) + commit_response(file_params) else render_api_error!(result[:message], 400) end diff --git a/lib/api/groups.rb b/lib/api/groups.rb index bfb89475025..7682d286866 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -1,127 +1,171 @@ module API - # groups API class Groups < Grape::API + include PaginationParams + before { authenticate! } + helpers do + params :optional_params do + optional :description, type: String, desc: 'The description of the group' + optional :visibility_level, type: Integer, desc: 'The visibility level of the group' + optional :lfs_enabled, type: Boolean, desc: 'Enable/disable LFS for the projects in this group' + optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access' + end + + params :statistics_params do + optional :statistics, type: Boolean, default: false, desc: 'Include project statistics' + end + + def present_groups(groups, options = {}) + options = options.reverse_merge( + with: Entities::Group, + current_user: current_user, + ) + + groups = groups.with_statistics if options[:statistics] + present paginate(groups), options + end + end + resource :groups do - # Get a groups list - # - # Parameters: - # skip_groups (optional) - Array of group ids to exclude from list - # Example Request: - # GET /groups + desc 'Get a groups list' do + success Entities::Group + end + params do + use :statistics_params + optional :skip_groups, type: Array[Integer], desc: 'Array of group ids to exclude from list' + optional :all_available, type: Boolean, desc: 'Show all group that you have access to' + optional :search, type: String, desc: 'Search for a specific group' + optional :order_by, type: String, values: %w[name path], default: 'name', desc: 'Order by name or path' + optional :sort, type: String, values: %w[asc desc], default: 'asc', desc: 'Sort by asc (ascending) or desc (descending)' + use :pagination + end get do - @groups = if current_user.admin - Group.all - else - current_user.groups - end - - @groups = @groups.search(params[:search]) if params[:search].present? - @groups = @groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present? - @groups = paginate @groups - present @groups, with: Entities::Group - end - - # Create group. Available only for users who can create groups. - # - # Parameters: - # name (required) - The name of the group - # path (required) - The path of the group - # description (optional) - The description of the group - # visibility_level (optional) - The visibility level of the group - # lfs_enabled (optional) - Enable/disable LFS for the projects in this group - # request_access_enabled (optional) - Allow users to request member access - # Example Request: - # POST /groups + groups = if current_user.admin + Group.all + elsif params[:all_available] + GroupsFinder.new.execute(current_user) + else + current_user.groups + end + + groups = groups.search(params[:search]) if params[:search].present? + groups = groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present? + groups = groups.reorder(params[:order_by] => params[:sort]) + + present_groups groups, statistics: params[:statistics] && current_user.is_admin? + end + + desc 'Get list of owned groups for authenticated user' do + success Entities::Group + end + params do + use :pagination + use :statistics_params + end + get '/owned' do + present_groups current_user.owned_groups, statistics: params[:statistics] + end + + desc 'Create a group. Available only for users who can create groups.' do + success Entities::Group + end + params do + requires :name, type: String, desc: 'The name of the group' + requires :path, type: String, desc: 'The path of the group' + use :optional_params + end post do authorize! :create_group - required_attributes! [:name, :path] - attrs = attributes_for_keys [:name, :path, :description, :visibility_level, :lfs_enabled, :request_access_enabled] - @group = Group.new(attrs) + group = ::Groups::CreateService.new(current_user, declared_params(include_missing: false)).execute - if @group.save - @group.add_owner(current_user) - present @group, with: Entities::Group + if group.persisted? + present group, with: Entities::Group, current_user: current_user else - render_api_error!("Failed to save group #{@group.errors.messages}", 400) + render_api_error!("Failed to save group #{group.errors.messages}", 400) end end + end - # Update group. Available only for users who can administrate groups. - # - # Parameters: - # id (required) - The ID of a group - # path (optional) - The path of the group - # description (optional) - The description of the group - # visibility_level (optional) - The visibility level of the group - # lfs_enabled (optional) - Enable/disable LFS for the projects in this group - # request_access_enabled (optional) - Allow users to request member access - # Example Request: - # PUT /groups/:id + params do + requires :id, type: String, desc: 'The ID of a group' + end + resource :groups do + desc 'Update a group. Available only for users who can administrate groups.' do + success Entities::Group + end + params do + optional :name, type: String, desc: 'The name of the group' + optional :path, type: String, desc: 'The path of the group' + use :optional_params + at_least_one_of :name, :path, :description, :visibility_level, + :lfs_enabled, :request_access_enabled + end put ':id' do - group = find_group(params[:id]) + group = find_group!(params[:id]) authorize! :admin_group, group - attrs = attributes_for_keys [:name, :path, :description, :visibility_level, :lfs_enabled, :request_access_enabled] - - if ::Groups::UpdateService.new(group, current_user, attrs).execute - present group, with: Entities::GroupDetail + if ::Groups::UpdateService.new(group, current_user, declared_params(include_missing: false)).execute + present group, with: Entities::GroupDetail, current_user: current_user else render_validation_error!(group) end end - # Get a single group, with containing projects - # - # Parameters: - # id (required) - The ID of a group - # Example Request: - # GET /groups/:id + desc 'Get a single group, with containing projects.' do + success Entities::GroupDetail + end get ":id" do - group = find_group(params[:id]) - present group, with: Entities::GroupDetail + group = find_group!(params[:id]) + present group, with: Entities::GroupDetail, current_user: current_user end - # Remove group - # - # Parameters: - # id (required) - The ID of a group - # Example Request: - # DELETE /groups/:id + desc 'Remove a group.' delete ":id" do - group = find_group(params[:id]) + group = find_group!(params[:id]) authorize! :admin_group, group DestroyGroupService.new(group, current_user).execute end - # Get a list of projects in this group - # - # Example Request: - # GET /groups/:id/projects + desc 'Get a list of projects in this group.' do + success Entities::Project + end + params do + optional :archived, type: Boolean, default: false, desc: 'Limit by archived status' + optional :visibility, type: String, values: %w[public internal private], + desc: 'Limit by visibility' + optional :search, type: String, desc: 'Return list of authorized projects matching the search criteria' + optional :order_by, type: String, values: %w[id name path created_at updated_at last_activity_at], + default: 'created_at', desc: 'Return projects ordered by field' + optional :sort, type: String, values: %w[asc desc], default: 'desc', + desc: 'Return projects sorted in ascending and descending order' + optional :simple, type: Boolean, default: false, + desc: 'Return only the ID, URL, name, and path of each project' + use :pagination + end get ":id/projects" do - group = find_group(params[:id]) + group = find_group!(params[:id]) projects = GroupProjectsFinder.new(group).execute(current_user) - projects = paginate projects - present projects, with: Entities::Project, user: current_user + projects = filter_projects(projects) + entity = params[:simple] ? Entities::BasicProjectDetails : Entities::Project + present paginate(projects), with: entity, current_user: current_user end - # Transfer a project to the Group namespace - # - # Parameters: - # id - group id - # project_id - project id - # Example Request: - # POST /groups/:id/projects/:project_id + desc 'Transfer a project to the group namespace. Available only for admin.' do + success Entities::GroupDetail + end + params do + requires :project_id, type: String, desc: 'The ID or path of the project' + end post ":id/projects/:project_id" do authenticated_as_admin! - group = Group.find_by(id: params[:id]) - project = Project.find(params[:project_id]) + group = find_group!(params[:id]) + project = find_project!(params[:project_id]) result = ::Projects::TransferService.new(project, current_user).execute(group) if result - present group + present group, with: Entities::GroupDetail, current_user: current_user else render_api_error!("Failed to transfer project #{project.errors.messages}", 400) end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 3c9d7b1aaef..a1d7b323f4f 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -1,113 +1,74 @@ module API module Helpers include Gitlab::Utils + include Helpers::Pagination - PRIVATE_TOKEN_HEADER = "HTTP_PRIVATE_TOKEN" - PRIVATE_TOKEN_PARAM = :private_token SUDO_HEADER = "HTTP_SUDO" SUDO_PARAM = :sudo - def private_token - params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER] - end - - def warden - env['warden'] - end - - # Check the Rails session for valid authentication details - # - # Until CSRF protection is added to the API, disallow this method for - # state-changing endpoints - def find_user_from_warden - warden.try(:authenticate) if %w[GET HEAD].include?(env['REQUEST_METHOD']) - end - - def find_user_by_private_token - token = private_token - return nil unless token.present? - - User.find_by_authentication_token(token) || User.find_by_personal_access_token(token) + def declared_params(options = {}) + options = { include_parent_namespaces: false }.merge(options) + declared(params, options).to_h.symbolize_keys end def current_user - @current_user ||= find_user_by_private_token - @current_user ||= doorkeeper_guard - @current_user ||= find_user_from_warden + return @current_user if defined?(@current_user) - unless @current_user && Gitlab::UserAccess.new(@current_user).allowed? - return nil - end - - identifier = sudo_identifier() + @current_user = initial_current_user - # If the sudo is the current user do nothing - if identifier && !(@current_user.id == identifier || @current_user.username == identifier) - forbidden!('Must be admin to use sudo') unless @current_user.is_admin? - @current_user = User.by_username_or_id(identifier) - not_found!("No user id or username for: #{identifier}") if @current_user.nil? - end + sudo! @current_user end - def sudo_identifier - identifier ||= params[SUDO_PARAM] || env[SUDO_HEADER] - - # Regex for integers - if !!(identifier =~ /\A[0-9]+\z/) - identifier.to_i - else - identifier - end + def sudo? + initial_current_user != current_user end def user_project - @project ||= find_project(params[:id]) + @project ||= find_project!(params[:id]) end def available_labels @available_labels ||= LabelsFinder.new(current_user, project_id: user_project.id).execute end - def find_project(id) - project = Project.find_with_namespace(id) || Project.find_by(id: id) - - if can?(current_user, :read_project, project) - project + def find_user(id) + if id =~ /^\d+$/ + User.find_by(id: id) else - not_found!('Project') + User.find_by(username: id) end end - def project_service - @project_service ||= begin - underscored_service = params[:service_slug].underscore - - if Service.available_services_names.include?(underscored_service) - user_project.build_missing_services - - service_method = "#{underscored_service}_service" - - send_service(service_method) - end + def find_project(id) + if id =~ /^\d+$/ + Project.find_by(id: id) + else + Project.find_with_namespace(id) end - - @project_service || not_found!("Service") end - def send_service(service_method) - user_project.send(service_method) - end + def find_project!(id) + project = find_project(id) - def service_attributes - @service_attributes ||= project_service.fields.inject([]) do |arr, hash| - arr << hash[:name].to_sym + if can?(current_user, :read_project, project) + project + else + not_found!('Project') end end def find_group(id) - group = Group.find_by(path: id) || Group.find_by(id: id) + if id =~ /^\d+$/ + Group.find_by(id: id) + else + Group.find_by_full_path(id) + end + end + + def find_group!(id) + group = find_group(id) if can?(current_user, :read_group, group) group @@ -122,21 +83,27 @@ module API end def find_project_issue(id) - issue = user_project.issues.find(id) - not_found! unless can?(current_user, :read_issue, issue) - issue + IssuesFinder.new(current_user, project_id: user_project.id).find(id) end - def paginate(relation) - relation.page(params[:page]).per(params[:per_page].to_i).tap do |data| - add_pagination_headers(data) - end + def find_project_merge_request(id) + MergeRequestsFinder.new(current_user, project_id: user_project.id).find(id) + end + + def find_merge_request_with_access(id, access_level = :read_merge_request) + merge_request = user_project.merge_requests.find(id) + authorize! access_level, merge_request + merge_request end def authenticate! unauthorized! unless current_user end + def authenticate_non_get! + authenticate! unless %w[GET HEAD].include?(route.request_method) + end + def authenticate_by_gitlab_shell_token! input = params['secret_token'].try(:chomp) unless Devise.secure_compare(secret_token, input) @@ -145,6 +112,7 @@ module API end def authenticated_as_admin! + authenticate! forbidden! unless current_user.is_admin? end @@ -192,20 +160,6 @@ module API ActionController::Parameters.new(attrs).permit! end - # Helper method for validating all labels against its names - def validate_label_params(params) - errors = {} - - params[:labels].to_s.split(',').each do |label_name| - label = available_labels.find_or_initialize_by(title: label_name.strip) - next if label.valid? - - errors[label.title] = label.errors - end - - errors - end - # Checks the occurrences of datetime attributes, each attribute if present in the params hash must be in ISO 8601 # format (YYYY-MM-DDTHH:MM:SSZ) or a Bad Request error is invoked. # @@ -222,22 +176,6 @@ module API end end - def issuable_order_by - if params["order_by"] == 'updated_at' - 'updated_at' - else - 'created_at' - end - end - - def issuable_sort - if params["sort"] == 'asc' - :asc - else - :desc - end - end - def filter_by_iid(items, iid) items.where(iid: iid) end @@ -294,7 +232,7 @@ module API end def render_api_error!(message, status) - error!({ 'message' => message }, status) + error!({ 'message' => message }, status, header) end def handle_api_exception(exception) @@ -315,14 +253,9 @@ module API rack_response({ 'message' => '500 Internal Server Error' }.to_json, 500) end - # Projects helpers + # project helpers def filter_projects(projects) - # If the archived parameter is passed, limit results accordingly - if params[:archived].present? - projects = projects.where(archived: to_boolean(params[:archived])) - end - if params[:search].present? projects = projects.search(params[:search]) end @@ -331,25 +264,8 @@ module API projects = projects.search_by_visibility(params[:visibility]) end - projects.reorder(project_order_by => project_sort) - end - - def project_order_by - order_fields = %w(id name path created_at updated_at last_activity_at) - - if order_fields.include?(params['order_by']) - params['order_by'] - else - 'created_at' - end - end - - def project_sort - if params["sort"] == 'asc' - :asc - else - :desc - end + projects = projects.where(archived: params[:archived]) + projects.reorder(params[:order_by] => params[:sort]) end # file helpers @@ -388,42 +304,66 @@ module API header['X-Sendfile'] = path body else - file FileStreamer.new(path) + path end end private - def add_pagination_headers(paginated_data) - header 'X-Total', paginated_data.total_count.to_s - header 'X-Total-Pages', paginated_data.total_pages.to_s - header 'X-Per-Page', paginated_data.limit_value.to_s - header 'X-Page', paginated_data.current_page.to_s - header 'X-Next-Page', paginated_data.next_page.to_s - header 'X-Prev-Page', paginated_data.prev_page.to_s - header 'Link', pagination_links(paginated_data) + def private_token + params[APIGuard::PRIVATE_TOKEN_PARAM] || env[APIGuard::PRIVATE_TOKEN_HEADER] + end + + def warden + env['warden'] + end + + # Check the Rails session for valid authentication details + # + # Until CSRF protection is added to the API, disallow this method for + # state-changing endpoints + def find_user_from_warden + warden.try(:authenticate) if %w[GET HEAD].include?(env['REQUEST_METHOD']) end - def pagination_links(paginated_data) - request_url = request.url.split('?').first - request_params = params.clone - request_params[:per_page] = paginated_data.limit_value + def initial_current_user + return @initial_current_user if defined?(@initial_current_user) + + @initial_current_user ||= find_user_by_private_token(scopes: @scopes) + @initial_current_user ||= doorkeeper_guard(scopes: @scopes) + @initial_current_user ||= find_user_from_warden + + unless @initial_current_user && Gitlab::UserAccess.new(@initial_current_user).allowed? + @initial_current_user = nil + end + + @initial_current_user + end - links = [] + def sudo! + return unless sudo_identifier + return unless initial_current_user - request_params[:page] = paginated_data.current_page - 1 - links << %(<#{request_url}?#{request_params.to_query}>; rel="prev") unless paginated_data.first_page? + unless initial_current_user.is_admin? + forbidden!('Must be admin to use sudo') + end - request_params[:page] = paginated_data.current_page + 1 - links << %(<#{request_url}?#{request_params.to_query}>; rel="next") unless paginated_data.last_page? + # Only private tokens should be used for the SUDO feature + unless private_token == initial_current_user.private_token + forbidden!('Private token must be specified in order to use sudo') + end - request_params[:page] = 1 - links << %(<#{request_url}?#{request_params.to_query}>; rel="first") + sudoed_user = find_user(sudo_identifier) - request_params[:page] = paginated_data.total_pages - links << %(<#{request_url}?#{request_params.to_query}>; rel="last") + if sudoed_user + @current_user = sudoed_user + else + not_found!("No user id or username for: #{sudo_identifier}") + end + end - links.join(', ') + def sudo_identifier + @sudo_identifier ||= params[SUDO_PARAM] || env[SUDO_HEADER] end def secret_token diff --git a/lib/api/helpers/custom_validators.rb b/lib/api/helpers/custom_validators.rb new file mode 100644 index 00000000000..0a8f3073a50 --- /dev/null +++ b/lib/api/helpers/custom_validators.rb @@ -0,0 +1,14 @@ +module API + module Helpers + module CustomValidators + class Absence < Grape::Validations::Base + def validate_param!(attr_name, params) + return if params.respond_to?(:key?) && !params.key?(attr_name) + raise Grape::Exceptions::Validation, params: [@scope.full_name(attr_name)], message: message(:absence) + end + end + end + end +end + +Grape::Validations.register_validator(:absence, ::API::Helpers::CustomValidators::Absence) diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb new file mode 100644 index 00000000000..e8975eb57e0 --- /dev/null +++ b/lib/api/helpers/internal_helpers.rb @@ -0,0 +1,65 @@ +module API + module Helpers + module InternalHelpers + # Project paths may be any of the following: + # * /repository/storage/path/namespace/project + # * /namespace/project + # * namespace/project + # + # In addition, they may have a '.git' extension and multiple namespaces + # + # Transform all these cases to 'namespace/project' + def clean_project_path(project_path, storage_paths = Repository.storages.values) + project_path = project_path.sub(/\.git\z/, '') + + storage_paths.each do |storage_path| + storage_path = File.expand_path(storage_path) + + if project_path.start_with?(storage_path) + project_path = project_path.sub(storage_path, '') + break + end + end + + project_path.sub(/\A\//, '') + end + + def project_path + @project_path ||= clean_project_path(params[:project]) + end + + def wiki? + @wiki ||= project_path.end_with?('.wiki') && + !Project.find_with_namespace(project_path) + end + + def project + @project ||= begin + # Check for *.wiki repositories. + # Strip out the .wiki from the pathname before finding the + # project. This applies the correct project permissions to + # the wiki repository as well. + project_path.chomp!('.wiki') if wiki? + + Project.find_with_namespace(project_path) + end + end + + def ssh_authentication_abilities + [ + :read_project, + :download_code, + :push_code + ] + end + + def parse_allowed_environment_variables + return if params[:env].blank? + + JSON.parse(params[:env]) + + rescue JSON::ParserError + end + end + end +end diff --git a/lib/api/helpers/members_helpers.rb b/lib/api/helpers/members_helpers.rb index 90114f6f667..d9cae1501f8 100644 --- a/lib/api/helpers/members_helpers.rb +++ b/lib/api/helpers/members_helpers.rb @@ -2,7 +2,7 @@ module API module Helpers module MembersHelpers def find_source(source_type, id) - public_send("find_#{source_type}", id) + public_send("find_#{source_type}!", id) end def authorize_admin_source!(source_type, source) diff --git a/lib/api/helpers/pagination.rb b/lib/api/helpers/pagination.rb new file mode 100644 index 00000000000..2199eea7e5f --- /dev/null +++ b/lib/api/helpers/pagination.rb @@ -0,0 +1,45 @@ +module API + module Helpers + module Pagination + def paginate(relation) + relation.page(params[:page]).per(params[:per_page].to_i).tap do |data| + add_pagination_headers(data) + end + end + + private + + def add_pagination_headers(paginated_data) + header 'X-Total', paginated_data.total_count.to_s + header 'X-Total-Pages', paginated_data.total_pages.to_s + header 'X-Per-Page', paginated_data.limit_value.to_s + header 'X-Page', paginated_data.current_page.to_s + header 'X-Next-Page', paginated_data.next_page.to_s + header 'X-Prev-Page', paginated_data.prev_page.to_s + header 'Link', pagination_links(paginated_data) + end + + def pagination_links(paginated_data) + request_url = request.url.split('?').first + request_params = params.clone + request_params[:per_page] = paginated_data.limit_value + + links = [] + + request_params[:page] = paginated_data.current_page - 1 + links << %(<#{request_url}?#{request_params.to_query}>; rel="prev") unless paginated_data.first_page? + + request_params[:page] = paginated_data.current_page + 1 + links << %(<#{request_url}?#{request_params.to_query}>; rel="next") unless paginated_data.last_page? + + request_params[:page] = 1 + links << %(<#{request_url}?#{request_params.to_query}>; rel="first") + + request_params[:page] = paginated_data.total_pages + links << %(<#{request_url}?#{request_params.to_query}>; rel="last") + + links.join(', ') + end + end + end +end diff --git a/lib/api/internal.rb b/lib/api/internal.rb index ccf181402f9..d235977fbd8 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -3,6 +3,8 @@ module API class Internal < Grape::API before { authenticate_by_gitlab_shell_token! } + helpers ::API::Helpers::InternalHelpers + namespace 'internal' do # Check if git command is allowed to project # @@ -14,42 +16,6 @@ module API # ref - branch name # forced_push - forced_push # protocol - Git access protocol being used, e.g. HTTP or SSH - # - - helpers do - def project_path - @project_path ||= begin - project_path = params[:project].sub(/\.git\z/, '') - Repository.remove_storage_from_path(project_path) - end - end - - def wiki? - @wiki ||= project_path.end_with?('.wiki') && - !Project.find_with_namespace(project_path) - end - - def project - @project ||= begin - # Check for *.wiki repositories. - # Strip out the .wiki from the pathname before finding the - # project. This applies the correct project permissions to - # the wiki repository as well. - project_path.chomp!('.wiki') if wiki? - - Project.find_with_namespace(project_path) - end - end - - def ssh_authentication_abilities - [ - :read_project, - :download_code, - :push_code - ] - end - end - post "/allowed" do status 200 @@ -62,11 +28,17 @@ module API protocol = params[:protocol] + actor.update_last_used_at if actor.is_a?(Key) + access = if wiki? Gitlab::GitAccessWiki.new(actor, project, protocol, authentication_abilities: ssh_authentication_abilities) else - Gitlab::GitAccess.new(actor, project, protocol, authentication_abilities: ssh_authentication_abilities) + Gitlab::GitAccess.new(actor, + project, + protocol, + authentication_abilities: ssh_authentication_abilities, + env: parse_allowed_environment_variables) end access_status = access.check(params[:action], params[:changes]) @@ -91,6 +63,8 @@ module API status 200 key = Key.find(params[:key_id]) + key.update_last_used_at + token_handler = Gitlab::LfsToken.new(key) { @@ -133,7 +107,9 @@ module API key = Key.find_by(id: params[:key_id]) - unless key + if key + key.update_last_used_at + else return { 'success' => false, 'message' => 'Could not find the given key' } end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index c9689e6f8ef..fe016c1ec0a 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -1,260 +1,225 @@ module API - # Issues API class Issues < Grape::API + include PaginationParams + before { authenticate! } helpers do - def filter_issues_state(issues, state) - case state - when 'opened' then issues.opened - when 'closed' then issues.closed - else issues + def find_issues(args = {}) + args = params.merge(args) + + args.delete(:id) + args[:milestone_title] = args.delete(:milestone) + + match_all_labels = args.delete(:match_all_labels) + labels = args.delete(:labels) + args[:label_name] = labels if match_all_labels + + args[:search] = "#{Issue.reference_prefix}#{args.delete(:iid)}" if args.key?(:iid) + + issues = IssuesFinder.new(current_user, args).execute.inc_notes_with_associations + + # TODO: Remove in 9.0 pass `label_name: args.delete(:labels)` to IssuesFinder + if !match_all_labels && labels.present? + issues = issues.includes(:labels).where('labels.title' => labels.split(',')) end + + issues.reorder(args[:order_by] => args[:sort]) end - def filter_issues_labels(issues, labels) - issues.includes(:labels).where('labels.title' => labels.split(',')) + params :issues_params do + optional :labels, type: String, desc: 'Comma-separated list of label names' + optional :milestone, type: String, desc: 'Milestone title' + optional :order_by, type: String, values: %w[created_at updated_at], default: 'created_at', + desc: 'Return issues ordered by `created_at` or `updated_at` fields.' + optional :sort, type: String, values: %w[asc desc], default: 'desc', + desc: 'Return issues sorted in `asc` or `desc` order.' + optional :milestone, type: String, desc: 'Return issues for a specific milestone' + use :pagination end - def filter_issues_milestone(issues, milestone) - issues.includes(:milestone).where('milestones.title' => milestone) + params :issue_params do + optional :description, type: String, desc: 'The description of an issue' + optional :assignee_id, type: Integer, desc: 'The ID of a user to assign issue' + optional :milestone_id, type: Integer, desc: 'The ID of a milestone to assign issue' + optional :labels, type: String, desc: 'Comma-separated list of label names' + optional :due_date, type: String, desc: 'Date time string in the format YEAR-MONTH-DAY' + optional :confidential, type: Boolean, desc: 'Boolean parameter if the issue should be confidential' end end resource :issues do - # Get currently authenticated user's issues - # - # Parameters: - # state (optional) - Return "opened" or "closed" issues - # labels (optional) - Comma-separated list of label names - # order_by (optional) - Return requests ordered by `created_at` or `updated_at` fields. Default is `created_at` - # sort (optional) - Return requests sorted in `asc` or `desc` order. Default is `desc` - # - # Example Requests: - # GET /issues - # GET /issues?state=opened - # GET /issues?state=closed - # GET /issues?labels=foo - # GET /issues?labels=foo,bar - # GET /issues?labels=foo,bar&state=opened + desc "Get currently authenticated user's issues" do + success Entities::Issue + end + params do + optional :state, type: String, values: %w[opened closed all], default: 'all', + desc: 'Return opened, closed, or all issues' + use :issues_params + end get do - issues = current_user.issues.inc_notes_with_associations - issues = filter_issues_state(issues, params[:state]) unless params[:state].nil? - issues = filter_issues_labels(issues, params[:labels]) unless params[:labels].nil? - issues = issues.reorder(issuable_order_by => issuable_sort) + issues = find_issues(scope: 'authored') present paginate(issues), with: Entities::Issue, current_user: current_user end end + params do + requires :id, type: String, desc: 'The ID of a group' + end resource :groups do - # Get a list of group issues - # - # Parameters: - # id (required) - The ID of a group - # state (optional) - Return "opened" or "closed" issues - # labels (optional) - Comma-separated list of label names - # milestone (optional) - Milestone title - # order_by (optional) - Return requests ordered by `created_at` or `updated_at` fields. Default is `created_at` - # sort (optional) - Return requests sorted in `asc` or `desc` order. Default is `desc` - # - # Example Requests: - # GET /groups/:id/issues - # GET /groups/:id/issues?state=opened - # GET /groups/:id/issues?state=closed - # GET /groups/:id/issues?labels=foo - # GET /groups/:id/issues?labels=foo,bar - # GET /groups/:id/issues?labels=foo,bar&state=opened - # GET /groups/:id/issues?milestone=1.0.0 - # GET /groups/:id/issues?milestone=1.0.0&state=closed + desc 'Get a list of group issues' do + success Entities::Issue + end + params do + optional :state, type: String, values: %w[opened closed all], default: 'opened', + desc: 'Return opened, closed, or all issues' + use :issues_params + end get ":id/issues" do - group = find_group(params[:id]) - - params[:state] ||= 'opened' - params[:group_id] = group.id - params[:milestone_title] = params.delete(:milestone) - params[:label_name] = params.delete(:labels) - - if params[:order_by] || params[:sort] - # The Sortable concern takes 'created_desc', not 'created_at_desc' (for example) - params[:sort] = "#{issuable_order_by.sub('_at', '')}_#{issuable_sort}" - end + group = find_group!(params[:id]) - issues = IssuesFinder.new(current_user, params).execute + issues = find_issues(group_id: group.id, state: params[:state] || 'opened', match_all_labels: true) present paginate(issues), with: Entities::Issue, current_user: current_user end end + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do - # Get a list of project issues - # - # Parameters: - # id (required) - The ID of a project - # iid (optional) - Return the project issue having the given `iid` - # state (optional) - Return "opened" or "closed" issues - # labels (optional) - Comma-separated list of label names - # milestone (optional) - Milestone title - # order_by (optional) - Return requests ordered by `created_at` or `updated_at` fields. Default is `created_at` - # sort (optional) - Return requests sorted in `asc` or `desc` order. Default is `desc` - # - # Example Requests: - # GET /projects/:id/issues - # GET /projects/:id/issues?state=opened - # GET /projects/:id/issues?state=closed - # GET /projects/:id/issues?labels=foo - # GET /projects/:id/issues?labels=foo,bar - # GET /projects/:id/issues?labels=foo,bar&state=opened - # GET /projects/:id/issues?milestone=1.0.0 - # GET /projects/:id/issues?milestone=1.0.0&state=closed - # GET /issues?iid=42 - get ":id/issues" do - issues = user_project.issues.inc_notes_with_associations.visible_to_user(current_user) - issues = filter_issues_state(issues, params[:state]) unless params[:state].nil? - issues = filter_issues_labels(issues, params[:labels]) unless params[:labels].nil? - issues = filter_by_iid(issues, params[:iid]) unless params[:iid].nil? + include TimeTrackingEndpoints - unless params[:milestone].nil? - issues = filter_issues_milestone(issues, params[:milestone]) - end + desc 'Get a list of project issues' do + success Entities::Issue + end + params do + optional :state, type: String, values: %w[opened closed all], default: 'all', + desc: 'Return opened, closed, or all issues' + optional :iid, type: Integer, desc: 'Return the issue having the given `iid`' + use :issues_params + end + get ":id/issues" do + project = find_project(params[:id]) - issues = issues.reorder(issuable_order_by => issuable_sort) + issues = find_issues(project_id: project.id) - present paginate(issues), with: Entities::Issue, current_user: current_user + present paginate(issues), with: Entities::Issue, current_user: current_user, project: user_project end - # Get a single project issue - # - # Parameters: - # id (required) - The ID of a project - # issue_id (required) - The ID of a project issue - # Example Request: - # GET /projects/:id/issues/:issue_id + desc 'Get a single project issue' do + success Entities::Issue + end + params do + requires :issue_id, type: Integer, desc: 'The ID of a project issue' + end get ":id/issues/:issue_id" do - @issue = find_project_issue(params[:issue_id]) - present @issue, with: Entities::Issue, current_user: current_user + issue = find_project_issue(params[:issue_id]) + present issue, with: Entities::Issue, current_user: current_user, project: user_project end - # Create a new project issue - # - # Parameters: - # id (required) - The ID of a project - # title (required) - The title of an issue - # description (optional) - The description of an issue - # assignee_id (optional) - The ID of a user to assign issue - # milestone_id (optional) - The ID of a milestone to assign issue - # labels (optional) - The labels of an issue - # created_at (optional) - Date time string, ISO 8601 formatted - # due_date (optional) - Date time string in the format YEAR-MONTH-DAY - # confidential (optional) - Boolean parameter if the issue should be confidential - # Example Request: - # POST /projects/:id/issues + desc 'Create a new project issue' do + success Entities::Issue + end + params do + requires :title, type: String, desc: 'The title of an issue' + optional :created_at, type: DateTime, + desc: 'Date time when the issue was created. Available only for admins and project owners.' + optional :merge_request_for_resolving_discussions, type: Integer, + desc: 'The IID of a merge request for which to resolve discussions' + use :issue_params + end post ':id/issues' do - required_attributes! [:title] - - keys = [:title, :description, :assignee_id, :milestone_id, :due_date, :confidential] - keys << :created_at if current_user.admin? || user_project.owner == current_user - attrs = attributes_for_keys(keys) - - # Validate label names in advance - if (errors = validate_label_params(params)).any? - render_api_error!({ labels: errors }, 400) + # Setting created_at time only allowed for admins and project owners + unless current_user.admin? || user_project.owner == current_user + params.delete(:created_at) end - attrs[:labels] = params[:labels] if params[:labels] - - # Convert and filter out invalid confidential flags - attrs['confidential'] = to_boolean(attrs['confidential']) - attrs.delete('confidential') if attrs['confidential'].nil? + issue_params = declared_params(include_missing: false) - issue = ::Issues::CreateService.new(user_project, current_user, attrs.merge(request: request, api: true)).execute + if merge_request_iid = params[:merge_request_for_resolving_discussions] + issue_params[:merge_request_for_resolving_discussions] = MergeRequestsFinder.new(current_user, project_id: user_project.id). + execute. + find_by(iid: merge_request_iid) + end + issue = ::Issues::CreateService.new(user_project, + current_user, + issue_params.merge(request: request, api: true)).execute if issue.spam? render_api_error!({ error: 'Spam detected' }, 400) end if issue.valid? - present issue, with: Entities::Issue, current_user: current_user + present issue, with: Entities::Issue, current_user: current_user, project: user_project else render_validation_error!(issue) end end - # Update an existing issue - # - # Parameters: - # id (required) - The ID of a project - # issue_id (required) - The ID of a project issue - # title (optional) - The title of an issue - # description (optional) - The description of an issue - # assignee_id (optional) - The ID of a user to assign issue - # milestone_id (optional) - The ID of a milestone to assign issue - # labels (optional) - The labels of an issue - # state_event (optional) - The state event of an issue (close|reopen) - # updated_at (optional) - Date time string, ISO 8601 formatted - # due_date (optional) - Date time string in the format YEAR-MONTH-DAY - # confidential (optional) - Boolean parameter if the issue should be confidential - # Example Request: - # PUT /projects/:id/issues/:issue_id + desc 'Update an existing issue' do + success Entities::Issue + end + params do + requires :issue_id, type: Integer, desc: 'The ID of a project issue' + optional :title, type: String, desc: 'The title of an issue' + optional :updated_at, type: DateTime, + desc: 'Date time when the issue was updated. Available only for admins and project owners.' + optional :state_event, type: String, values: %w[reopen close], desc: 'State of the issue' + use :issue_params + at_least_one_of :title, :description, :assignee_id, :milestone_id, + :labels, :created_at, :due_date, :confidential, :state_event + end put ':id/issues/:issue_id' do - issue = user_project.issues.find(params[:issue_id]) + issue = user_project.issues.find(params.delete(:issue_id)) authorize! :update_issue, issue - keys = [:title, :description, :assignee_id, :milestone_id, :state_event, :due_date, :confidential] - keys << :updated_at if current_user.admin? || user_project.owner == current_user - attrs = attributes_for_keys(keys) - # Validate label names in advance - if (errors = validate_label_params(params)).any? - render_api_error!({ labels: errors }, 400) + # Setting created_at time only allowed for admins and project owners + unless current_user.admin? || user_project.owner == current_user + params.delete(:updated_at) end - attrs[:labels] = params[:labels] if params[:labels] - - # Convert and filter out invalid confidential flags - attrs['confidential'] = to_boolean(attrs['confidential']) - attrs.delete('confidential') if attrs['confidential'].nil? - - issue = ::Issues::UpdateService.new(user_project, current_user, attrs).execute(issue) + issue = ::Issues::UpdateService.new(user_project, + current_user, + declared_params(include_missing: false)).execute(issue) if issue.valid? - present issue, with: Entities::Issue, current_user: current_user + present issue, with: Entities::Issue, current_user: current_user, project: user_project else render_validation_error!(issue) end end - # Move an existing issue - # - # Parameters: - # id (required) - The ID of a project - # issue_id (required) - The ID of a project issue - # to_project_id (required) - The ID of the new project - # Example Request: - # POST /projects/:id/issues/:issue_id/move + desc 'Move an existing issue' do + success Entities::Issue + end + params do + requires :issue_id, type: Integer, desc: 'The ID of a project issue' + requires :to_project_id, type: Integer, desc: 'The ID of the new project' + end post ':id/issues/:issue_id/move' do - required_attributes! [:to_project_id] + issue = user_project.issues.find_by(id: params[:issue_id]) + not_found!('Issue') unless issue - issue = user_project.issues.find(params[:issue_id]) - new_project = Project.find(params[:to_project_id]) + new_project = Project.find_by(id: params[:to_project_id]) + not_found!('Project') unless new_project begin issue = ::Issues::MoveService.new(user_project, current_user).execute(issue, new_project) - present issue, with: Entities::Issue, current_user: current_user + present issue, with: Entities::Issue, current_user: current_user, project: user_project rescue ::Issues::MoveService::MoveError => error render_api_error!(error.message, 400) end end - # - # Delete a project issue - # - # Parameters: - # id (required) - The ID of a project - # issue_id (required) - The ID of a project issue - # Example Request: - # DELETE /projects/:id/issues/:issue_id + desc 'Delete a project issue' + params do + requires :issue_id, type: Integer, desc: 'The ID of a project issue' + end delete ":id/issues/:issue_id" do issue = user_project.issues.find_by(id: params[:issue_id]) + not_found!('Issue') unless issue authorize!(:destroy_issue, issue) issue.destroy diff --git a/lib/api/labels.rb b/lib/api/labels.rb index 326e1e7ae00..652786d4e3e 100644 --- a/lib/api/labels.rb +++ b/lib/api/labels.rb @@ -11,7 +11,7 @@ module API success Entities::Label end get ':id/labels' do - present available_labels, with: Entities::Label, current_user: current_user + present available_labels, with: Entities::Label, current_user: current_user, project: user_project end desc 'Create a new label' do @@ -21,17 +21,20 @@ module API requires :name, type: String, desc: 'The name of the label to be created' requires :color, type: String, desc: "The color of the label given in 6-digit hex notation with leading '#' sign (e.g. #FFAABB)" optional :description, type: String, desc: 'The description of label to be created' + optional :priority, type: Integer, desc: 'The priority of the label', allow_blank: true end post ':id/labels' do authorize! :admin_label, user_project - label = user_project.find_label(params[:name]) + label = available_labels.find_by(title: params[:name]) conflict!('Label already exists') if label - label = user_project.labels.create(declared(params, include_parent_namespaces: false).to_h) + priority = params.delete(:priority) + label = user_project.labels.create(declared_params(include_missing: false)) if label.valid? - present label, with: Entities::Label, current_user: current_user + label.prioritize!(user_project, priority) if priority + present label, with: Entities::Label, current_user: current_user, project: user_project else render_validation_error!(label) end @@ -46,10 +49,10 @@ module API delete ':id/labels' do authorize! :admin_label, user_project - label = user_project.find_label(params[:name]) + label = user_project.labels.find_by(title: params[:name]) not_found!('Label') unless label - present label.destroy, with: Entities::Label, current_user: current_user + present label.destroy, with: Entities::Label, current_user: current_user, project: user_project end desc 'Update an existing label. At least one optional parameter is required.' do @@ -60,25 +63,32 @@ module API optional :new_name, type: String, desc: 'The new name of the label' optional :color, type: String, desc: "The new color of the label given in 6-digit hex notation with leading '#' sign (e.g. #FFAABB)" optional :description, type: String, desc: 'The new description of label' - at_least_one_of :new_name, :color, :description + optional :priority, type: Integer, desc: 'The priority of the label', allow_blank: true + at_least_one_of :new_name, :color, :description, :priority end put ':id/labels' do authorize! :admin_label, user_project - label = user_project.find_label(params[:name]) + label = user_project.labels.find_by(title: params[:name]) not_found!('Label not found') unless label - update_params = declared(params, - include_parent_namespaces: false, - include_missing: false).to_h + update_priority = params.key?(:priority) + priority = params.delete(:priority) + label_params = declared_params(include_missing: false) # Rename new name to the actual label attribute name - update_params['name'] = update_params.delete('new_name') if update_params.key?('new_name') + label_params[:name] = label_params.delete(:new_name) if label_params.key?(:new_name) - if label.update(update_params) - present label, with: Entities::Label, current_user: current_user - else - render_validation_error!(label) + render_validation_error!(label) unless label.update(label_params) + + if update_priority + if priority.nil? + label.unprioritize!(user_project) + else + label.prioritize!(user_project, priority) + end end + + present label, with: Entities::Label, current_user: current_user, project: user_project end end end diff --git a/lib/api/members.rb b/lib/api/members.rb index b80818f0eb6..d85f1f78cd6 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -1,5 +1,7 @@ module API class Members < Grape::API + include PaginationParams + before { authenticate! } helpers ::API::Helpers::MembersHelpers @@ -14,15 +16,15 @@ module API end params do optional :query, type: String, desc: 'A query string to search for members' + use :pagination end get ":id/members" do source = find_source(source_type, params[:id]) users = source.users users = users.merge(User.search(params[:query])) if params[:query] - users = paginate(users) - present users, with: Entities::Member, source: source + present paginate(users), with: Entities::Member, source: source end desc 'Gets a member of a group or project.' do @@ -120,7 +122,7 @@ module API if member.nil? { message: "Access revoked", id: params[:user_id].to_i } else - ::Members::DestroyService.new(source, current_user, declared(params)).execute + ::Members::DestroyService.new(source, current_user, declared_params).execute present member.user, with: Entities::Member, member: member end diff --git a/lib/api/merge_request_diffs.rb b/lib/api/merge_request_diffs.rb index 07435d78468..bc3d69f6904 100644 --- a/lib/api/merge_request_diffs.rb +++ b/lib/api/merge_request_diffs.rb @@ -15,10 +15,8 @@ module API end get ":id/merge_requests/:merge_request_id/versions" do - merge_request = user_project.merge_requests. - find(params[:merge_request_id]) + merge_request = find_merge_request_with_access(params[:merge_request_id]) - authorize! :read_merge_request, merge_request present merge_request.merge_request_diffs, with: Entities::MergeRequestDiff end @@ -34,10 +32,8 @@ module API end get ":id/merge_requests/:merge_request_id/versions/:version_id" do - merge_request = user_project.merge_requests. - find(params[:merge_request_id]) + merge_request = find_merge_request_with_access(params[:merge_request_id]) - authorize! :read_merge_request, merge_request present merge_request.merge_request_diffs.find(params[:version_id]), with: Entities::MergeRequestDiffFull end end diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index bf8504e1101..7ffb38e62da 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -1,9 +1,17 @@ module API - # MergeRequest API class MergeRequests < Grape::API + include PaginationParams + + DEPRECATION_MESSAGE = 'This endpoint is deprecated and will be removed in GitLab 9.0.'.freeze + before { authenticate! } + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do + include TimeTrackingEndpoints + helpers do def handle_merge_request_errors!(errors) if errors[:project_access].any? @@ -18,92 +26,79 @@ module API render_api_error!(errors, 400) end + + params :optional_params do + optional :description, type: String, desc: 'The description of the merge request' + optional :assignee_id, type: Integer, desc: 'The ID of a user to assign the merge request' + optional :milestone_id, type: Integer, desc: 'The ID of a milestone to assign the merge request' + optional :labels, type: String, desc: 'Comma-separated list of label names' + optional :remove_source_branch, type: Boolean, desc: 'Remove source branch when merging' + end end - # List merge requests - # - # Parameters: - # id (required) - The ID of a project - # iid (optional) - Return the project MR having the given `iid` - # state (optional) - Return requests "merged", "opened" or "closed" - # order_by (optional) - Return requests ordered by `created_at` or `updated_at` fields. Default is `created_at` - # sort (optional) - Return requests sorted in `asc` or `desc` order. Default is `desc` - # - # Example: - # GET /projects/:id/merge_requests - # GET /projects/:id/merge_requests?state=opened - # GET /projects/:id/merge_requests?state=closed - # GET /projects/:id/merge_requests?order_by=created_at - # GET /projects/:id/merge_requests?order_by=updated_at - # GET /projects/:id/merge_requests?sort=desc - # GET /projects/:id/merge_requests?sort=asc - # GET /projects/:id/merge_requests?iid=42 - # + desc 'List merge requests' do + success Entities::MergeRequest + end + params do + optional :state, type: String, values: %w[opened closed merged all], default: 'all', + desc: 'Return opened, closed, merged, or all merge requests' + optional :order_by, type: String, values: %w[created_at updated_at], default: 'created_at', + desc: 'Return merge requests ordered by `created_at` or `updated_at` fields.' + optional :sort, type: String, values: %w[asc desc], default: 'desc', + desc: 'Return merge requests sorted in `asc` or `desc` order.' + optional :iid, type: Array[Integer], desc: 'The IID of the merge requests' + use :pagination + end get ":id/merge_requests" do authorize! :read_merge_request, user_project - merge_requests = user_project.merge_requests.inc_notes_with_associations - unless params[:iid].nil? - merge_requests = filter_by_iid(merge_requests, params[:iid]) - end + merge_requests = user_project.merge_requests.inc_notes_with_associations + merge_requests = filter_by_iid(merge_requests, params[:iid]) if params[:iid].present? merge_requests = - case params["state"] - when "opened" then merge_requests.opened - when "closed" then merge_requests.closed - when "merged" then merge_requests.merged + case params[:state] + when 'opened' then merge_requests.opened + when 'closed' then merge_requests.closed + when 'merged' then merge_requests.merged else merge_requests end - merge_requests = merge_requests.reorder(issuable_order_by => issuable_sort) - present paginate(merge_requests), with: Entities::MergeRequest, current_user: current_user + merge_requests = merge_requests.reorder(params[:order_by] => params[:sort]) + present paginate(merge_requests), with: Entities::MergeRequest, current_user: current_user, project: user_project end - # Create MR - # - # Parameters: - # - # id (required) - The ID of a project - this will be the source of the merge request - # source_branch (required) - The source branch - # target_branch (required) - The target branch - # target_project_id - The target project of the merge request defaults to the :id of the project - # assignee_id - Assignee user ID - # title (required) - Title of MR - # description - Description of MR - # labels (optional) - Labels for MR as a comma-separated list - # milestone_id (optional) - Milestone ID - # - # Example: - # POST /projects/:id/merge_requests - # + desc 'Create a merge request' do + success Entities::MergeRequest + end + params do + requires :title, type: String, desc: 'The title of the merge request' + requires :source_branch, type: String, desc: 'The source branch' + requires :target_branch, type: String, desc: 'The target branch' + optional :target_project_id, type: Integer, + desc: 'The target project of the merge request defaults to the :id of the project' + use :optional_params + end post ":id/merge_requests" do authorize! :create_merge_request, user_project - required_attributes! [:source_branch, :target_branch, :title] - attrs = attributes_for_keys [:source_branch, :target_branch, :assignee_id, :title, :target_project_id, :description, :milestone_id] - # Validate label names in advance - if (errors = validate_label_params(params)).any? - render_api_error!({ labels: errors }, 400) - end - - attrs[:labels] = params[:labels] if params[:labels] + mr_params = declared_params(include_missing: false) + mr_params[:force_remove_source_branch] = mr_params.delete(:remove_source_branch) if mr_params[:remove_source_branch].present? - merge_request = ::MergeRequests::CreateService.new(user_project, current_user, attrs).execute + merge_request = ::MergeRequests::CreateService.new(user_project, current_user, mr_params).execute if merge_request.valid? - present merge_request, with: Entities::MergeRequest, current_user: current_user + present merge_request, with: Entities::MergeRequest, current_user: current_user, project: user_project else handle_merge_request_errors! merge_request.errors end end - # Delete a MR - # - # Parameters: - # id (required) - The ID of the project - # merge_request_id (required) - The MR id + desc 'Delete a merge request' + params do + requires :merge_request_id, type: Integer, desc: 'The ID of a merge request' + end delete ":id/merge_requests/:merge_request_id" do - merge_request = user_project.merge_requests.find_by(id: params[:merge_request_id]) + merge_request = find_project_merge_request(params[:merge_request_id]) authorize!(:destroy_merge_request, merge_request) merge_request.destroy @@ -112,111 +107,81 @@ module API # Routing "merge_request/:merge_request_id/..." is DEPRECATED and WILL BE REMOVED in version 9.0 # Use "merge_requests/:merge_request_id/..." instead. # - [":id/merge_request/:merge_request_id", ":id/merge_requests/:merge_request_id"].each do |path| - # Show MR - # - # Parameters: - # id (required) - The ID of a project - # merge_request_id (required) - The ID of MR - # - # Example: - # GET /projects/:id/merge_requests/:merge_request_id - # + params do + requires :merge_request_id, type: Integer, desc: 'The ID of a merge request' + end + { ":id/merge_request/:merge_request_id" => :deprecated, ":id/merge_requests/:merge_request_id" => :ok }.each do |path, status| + desc 'Get a single merge request' do + if status == :deprecated + detail DEPRECATION_MESSAGE + end + success Entities::MergeRequest + end get path do - merge_request = user_project.merge_requests.find(params[:merge_request_id]) - - authorize! :read_merge_request, merge_request + merge_request = find_merge_request_with_access(params[:merge_request_id]) - present merge_request, with: Entities::MergeRequest, current_user: current_user + present merge_request, with: Entities::MergeRequest, current_user: current_user, project: user_project end - # Show MR commits - # - # Parameters: - # id (required) - The ID of a project - # merge_request_id (required) - The ID of MR - # - # Example: - # GET /projects/:id/merge_requests/:merge_request_id/commits - # + desc 'Get the commits of a merge request' do + success Entities::RepoCommit + end get "#{path}/commits" do - merge_request = user_project.merge_requests. - find(params[:merge_request_id]) - authorize! :read_merge_request, merge_request + merge_request = find_merge_request_with_access(params[:merge_request_id]) + present merge_request.commits, with: Entities::RepoCommit end - # Show MR changes - # - # Parameters: - # id (required) - The ID of a project - # merge_request_id (required) - The ID of MR - # - # Example: - # GET /projects/:id/merge_requests/:merge_request_id/changes - # + desc 'Show the merge request changes' do + success Entities::MergeRequestChanges + end get "#{path}/changes" do - merge_request = user_project.merge_requests. - find(params[:merge_request_id]) - authorize! :read_merge_request, merge_request + merge_request = find_merge_request_with_access(params[:merge_request_id]) + present merge_request, with: Entities::MergeRequestChanges, current_user: current_user end - # Update MR - # - # Parameters: - # id (required) - The ID of a project - # merge_request_id (required) - ID of MR - # target_branch - The target branch - # assignee_id - Assignee user ID - # title - Title of MR - # state_event - Status of MR. (close|reopen|merge) - # description - Description of MR - # labels (optional) - Labels for a MR as a comma-separated list - # milestone_id (optional) - Milestone ID - # Example: - # PUT /projects/:id/merge_requests/:merge_request_id - # + desc 'Update a merge request' do + success Entities::MergeRequest + end + params do + optional :title, type: String, allow_blank: false, desc: 'The title of the merge request' + optional :target_branch, type: String, allow_blank: false, desc: 'The target branch' + optional :state_event, type: String, values: %w[close reopen merge], + desc: 'Status of the merge request' + use :optional_params + at_least_one_of :title, :target_branch, :description, :assignee_id, + :milestone_id, :labels, :state_event, + :remove_source_branch + end put path do - attrs = attributes_for_keys [:target_branch, :assignee_id, :title, :state_event, :description, :milestone_id] - merge_request = user_project.merge_requests.find(params[:merge_request_id]) - authorize! :update_merge_request, merge_request - - # Ensure source_branch is not specified - if params[:source_branch].present? - render_api_error!('Source branch cannot be changed', 400) - end + merge_request = find_merge_request_with_access(params.delete(:merge_request_id), :update_merge_request) - # Validate label names in advance - if (errors = validate_label_params(params)).any? - render_api_error!({ labels: errors }, 400) - end - - attrs[:labels] = params[:labels] if params[:labels] + mr_params = declared_params(include_missing: false) + mr_params[:force_remove_source_branch] = mr_params.delete(:remove_source_branch) if mr_params[:remove_source_branch].present? - merge_request = ::MergeRequests::UpdateService.new(user_project, current_user, attrs).execute(merge_request) + merge_request = ::MergeRequests::UpdateService.new(user_project, current_user, mr_params).execute(merge_request) if merge_request.valid? - present merge_request, with: Entities::MergeRequest, current_user: current_user + present merge_request, with: Entities::MergeRequest, current_user: current_user, project: user_project else handle_merge_request_errors! merge_request.errors end end - # Merge MR - # - # Parameters: - # id (required) - The ID of a project - # merge_request_id (required) - ID of MR - # merge_commit_message (optional) - Custom merge commit message - # should_remove_source_branch (optional) - When true, the source branch will be deleted if possible - # merge_when_build_succeeds (optional) - When true, this MR will be merged when the build succeeds - # sha (optional) - When present, must have the HEAD SHA of the source branch - # Example: - # PUT /projects/:id/merge_requests/:merge_request_id/merge - # + desc 'Merge a merge request' do + success Entities::MergeRequest + end + params do + optional :merge_commit_message, type: String, desc: 'Custom merge commit message' + optional :should_remove_source_branch, type: Boolean, + desc: 'When true, the source branch will be deleted if possible' + optional :merge_when_build_succeeds, type: Boolean, + desc: 'When true, this merge request will be merged when the pipeline succeeds' + optional :sha, type: String, desc: 'When present, must have the HEAD SHA of the source branch' + end put "#{path}/merge" do - merge_request = user_project.merge_requests.find(params[:merge_request_id]) + merge_request = find_project_merge_request(params[:merge_request_id]) # Merge request can not be merged # because user dont have permissions to push into target branch @@ -235,67 +200,53 @@ module API should_remove_source_branch: params[:should_remove_source_branch] } - if to_boolean(params[:merge_when_build_succeeds]) && merge_request.pipeline && merge_request.pipeline.active? - ::MergeRequests::MergeWhenBuildSucceedsService.new(merge_request.target_project, current_user, merge_params). - execute(merge_request) + if params[:merge_when_build_succeeds] && merge_request.head_pipeline && merge_request.head_pipeline.active? + ::MergeRequests::MergeWhenPipelineSucceedsService + .new(merge_request.target_project, current_user, merge_params) + .execute(merge_request) else - ::MergeRequests::MergeService.new(merge_request.target_project, current_user, merge_params). - execute(merge_request) + ::MergeRequests::MergeService + .new(merge_request.target_project, current_user, merge_params) + .execute(merge_request) end - present merge_request, with: Entities::MergeRequest, current_user: current_user + present merge_request, with: Entities::MergeRequest, current_user: current_user, project: user_project end - # Cancel Merge if Merge When build succeeds is enabled - # Parameters: - # id (required) - The ID of a project - # merge_request_id (required) - ID of MR - # + desc 'Cancel merge if "Merge When Pipeline Succeeds" is enabled' do + success Entities::MergeRequest + end post "#{path}/cancel_merge_when_build_succeeds" do - merge_request = user_project.merge_requests.find(params[:merge_request_id]) + merge_request = find_project_merge_request(params[:merge_request_id]) unauthorized! unless merge_request.can_cancel_merge_when_build_succeeds?(current_user) - ::MergeRequest::MergeWhenBuildSucceedsService.new(merge_request.target_project, current_user).cancel(merge_request) + ::MergeRequest::MergeWhenPipelineSucceedsService + .new(merge_request.target_project, current_user) + .cancel(merge_request) end - # Duplicate. DEPRECATED and WILL BE REMOVED in 9.0. - # Use GET "/projects/:id/merge_requests/:merge_request_id/notes" instead - # - # Get a merge request's comments - # - # Parameters: - # id (required) - The ID of a project - # merge_request_id (required) - ID of MR - # Examples: - # GET /projects/:id/merge_requests/:merge_request_id/comments - # + desc 'Get the comments of a merge request' do + detail 'Duplicate. DEPRECATED and WILL BE REMOVED in 9.0' + success Entities::MRNote + end + params do + use :pagination + end get "#{path}/comments" do - merge_request = user_project.merge_requests.find(params[:merge_request_id]) - - authorize! :read_merge_request, merge_request - + merge_request = find_merge_request_with_access(params[:merge_request_id]) present paginate(merge_request.notes.fresh), with: Entities::MRNote end - # Duplicate. DEPRECATED and WILL BE REMOVED in 9.0. - # Use POST "/projects/:id/merge_requests/:merge_request_id/notes" instead - # - # Post comment to merge request - # - # Parameters: - # id (required) - The ID of a project - # merge_request_id (required) - ID of MR - # note (required) - Text of comment - # Examples: - # POST /projects/:id/merge_requests/:merge_request_id/comments - # + desc 'Post a comment to a merge request' do + detail 'Duplicate. DEPRECATED and WILL BE REMOVED in 9.0' + success Entities::MRNote + end + params do + requires :note, type: String, desc: 'The text of the comment' + end post "#{path}/comments" do - required_attributes! [:note] - - merge_request = user_project.merge_requests.find(params[:merge_request_id]) - - authorize! :create_note, merge_request + merge_request = find_merge_request_with_access(params[:merge_request_id], :create_note) opts = { note: params[:note], @@ -312,15 +263,14 @@ module API end end - # List issues that will close on merge - # - # Parameters: - # id (required) - The ID of a project - # merge_request_id (required) - ID of MR - # Examples: - # GET /projects/:id/merge_requests/:merge_request_id/closes_issues + desc 'List issues that will be closed on merge' do + success Entities::MRNote + end + params do + use :pagination + end get "#{path}/closes_issues" do - merge_request = user_project.merge_requests.find(params[:merge_request_id]) + merge_request = find_merge_request_with_access(params[:merge_request_id]) issues = ::Kaminari.paginate_array(merge_request.closes_issues(current_user)) present paginate(issues), with: issue_entity(user_project), current_user: current_user end diff --git a/lib/api/milestones.rb b/lib/api/milestones.rb index 9b73f6826cf..3c373a84ec5 100644 --- a/lib/api/milestones.rb +++ b/lib/api/milestones.rb @@ -1,6 +1,7 @@ module API - # Milestones API class Milestones < Grape::API + include PaginationParams + before { authenticate! } helpers do @@ -11,19 +12,27 @@ module API else milestones end end + + params :optional_params do + optional :description, type: String, desc: 'The description of the milestone' + optional :due_date, type: String, desc: 'The due date of the milestone. The ISO 8601 date format (%Y-%m-%d)' + optional :start_date, type: String, desc: 'The start date of the milestone. The ISO 8601 date format (%Y-%m-%d)' + end end + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do - # Get a list of project milestones - # - # Parameters: - # id (required) - The ID of a project - # state (optional) - Return "active" or "closed" milestones - # Example Request: - # GET /projects/:id/milestones - # GET /projects/:id/milestones?iid=42 - # GET /projects/:id/milestones?state=active - # GET /projects/:id/milestones?state=closed + desc 'Get a list of project milestones' do + success Entities::Milestone + end + params do + optional :state, type: String, values: %w[active closed all], default: 'all', + desc: 'Return "active", "closed", or "all" milestones' + optional :iid, type: Array[Integer], desc: 'The IID of the milestone' + use :pagination + end get ":id/milestones" do authorize! :read_milestone, user_project @@ -34,34 +43,30 @@ module API present paginate(milestones), with: Entities::Milestone end - # Get a single project milestone - # - # Parameters: - # id (required) - The ID of a project - # milestone_id (required) - The ID of a project milestone - # Example Request: - # GET /projects/:id/milestones/:milestone_id + desc 'Get a single project milestone' do + success Entities::Milestone + end + params do + requires :milestone_id, type: Integer, desc: 'The ID of a project milestone' + end get ":id/milestones/:milestone_id" do authorize! :read_milestone, user_project - @milestone = user_project.milestones.find(params[:milestone_id]) - present @milestone, with: Entities::Milestone + milestone = user_project.milestones.find(params[:milestone_id]) + present milestone, with: Entities::Milestone end - # Create a new project milestone - # - # Parameters: - # id (required) - The ID of the project - # title (required) - The title of the milestone - # description (optional) - The description of the milestone - # due_date (optional) - The due date of the milestone - # Example Request: - # POST /projects/:id/milestones + desc 'Create a new project milestone' do + success Entities::Milestone + end + params do + requires :title, type: String, desc: 'The title of the milestone' + use :optional_params + end post ":id/milestones" do authorize! :admin_milestone, user_project - required_attributes! [:title] - attrs = attributes_for_keys [:title, :description, :due_date] - milestone = ::Milestones::CreateService.new(user_project, current_user, attrs).execute + + milestone = ::Milestones::CreateService.new(user_project, current_user, declared_params).execute if milestone.valid? present milestone, with: Entities::Milestone @@ -70,22 +75,23 @@ module API end end - # Update an existing project milestone - # - # Parameters: - # id (required) - The ID of a project - # milestone_id (required) - The ID of a project milestone - # title (optional) - The title of a milestone - # description (optional) - The description of a milestone - # due_date (optional) - The due date of a milestone - # state_event (optional) - The state event of the milestone (close|activate) - # Example Request: - # PUT /projects/:id/milestones/:milestone_id + desc 'Update an existing project milestone' do + success Entities::Milestone + end + params do + requires :milestone_id, type: Integer, desc: 'The ID of a project milestone' + optional :title, type: String, desc: 'The title of the milestone' + optional :state_event, type: String, values: %w[close activate], + desc: 'The state event of the milestone ' + use :optional_params + at_least_one_of :title, :description, :due_date, :state_event + end put ":id/milestones/:milestone_id" do authorize! :admin_milestone, user_project - attrs = attributes_for_keys [:title, :description, :due_date, :state_event] - milestone = user_project.milestones.find(params[:milestone_id]) - milestone = ::Milestones::UpdateService.new(user_project, current_user, attrs).execute(milestone) + milestone = user_project.milestones.find(params.delete(:milestone_id)) + + milestone_params = declared_params(include_missing: false) + milestone = ::Milestones::UpdateService.new(user_project, current_user, milestone_params).execute(milestone) if milestone.valid? present milestone, with: Entities::Milestone @@ -94,25 +100,25 @@ module API end end - # Get all issues for a single project milestone - # - # Parameters: - # id (required) - The ID of a project - # milestone_id (required) - The ID of a project milestone - # Example Request: - # GET /projects/:id/milestones/:milestone_id/issues + desc 'Get all issues for a single project milestone' do + success Entities::Issue + end + params do + requires :milestone_id, type: Integer, desc: 'The ID of a project milestone' + use :pagination + end get ":id/milestones/:milestone_id/issues" do authorize! :read_milestone, user_project - @milestone = user_project.milestones.find(params[:milestone_id]) + milestone = user_project.milestones.find(params[:milestone_id]) finder_params = { project_id: user_project.id, - milestone_title: @milestone.title + milestone_title: milestone.title } issues = IssuesFinder.new(current_user, finder_params).execute - present paginate(issues), with: Entities::Issue, current_user: current_user + present paginate(issues), with: Entities::Issue, current_user: current_user, project: user_project end end end diff --git a/lib/api/namespaces.rb b/lib/api/namespaces.rb index fe981d7b9fa..30761cb9b55 100644 --- a/lib/api/namespaces.rb +++ b/lib/api/namespaces.rb @@ -1,6 +1,7 @@ module API - # namespaces API class Namespaces < Grape::API + include PaginationParams + before { authenticate! } resource :namespaces do @@ -9,6 +10,7 @@ module API end params do optional :search, type: String, desc: "Search query for namespaces" + use :pagination end get do namespaces = current_user.admin ? Namespace.all : current_user.namespaces diff --git a/lib/api/notes.rb b/lib/api/notes.rb index c5c214d4d13..4d2a8f48267 100644 --- a/lib/api/notes.rb +++ b/lib/api/notes.rb @@ -1,27 +1,29 @@ module API - # Notes API class Notes < Grape::API + include PaginationParams + before { authenticate! } NOTEABLE_TYPES = [Issue, MergeRequest, Snippet] + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do NOTEABLE_TYPES.each do |noteable_type| noteables_str = noteable_type.to_s.underscore.pluralize - noteable_id_str = "#{noteable_type.to_s.underscore}_id" - - # Get a list of project +noteable+ notes - # - # Parameters: - # id (required) - The ID of a project - # noteable_id (required) - The ID of an issue or snippet - # Example Request: - # GET /projects/:id/issues/:noteable_id/notes - # GET /projects/:id/snippets/:noteable_id/notes - get ":id/#{noteables_str}/:#{noteable_id_str}/notes" do - @noteable = user_project.send(noteables_str.to_sym).find(params[noteable_id_str.to_sym]) - - if can?(current_user, noteable_read_ability_name(@noteable), @noteable) + + desc 'Get a list of project +noteable+ notes' do + success Entities::Note + end + params do + requires :noteable_id, type: Integer, desc: 'The ID of the noteable' + use :pagination + end + get ":id/#{noteables_str}/:noteable_id/notes" do + noteable = user_project.send(noteables_str.to_sym).find(params[:noteable_id]) + + if can?(current_user, noteable_read_ability_name(noteable), noteable) # We exclude notes that are cross-references and that cannot be viewed # by the current user. By doing this exclusion at this level and not # at the DB query level (which we cannot in that case), the current @@ -31,7 +33,7 @@ module API # paginate() only works with a relation. This could lead to a # mismatch between the pagination headers info and the actual notes # array returned, but this is really a edge-case. - paginate(@noteable.notes). + paginate(noteable.notes). reject { |n| n.cross_reference_not_visible_for?(current_user) } present notes, with: Entities::Note else @@ -39,72 +41,68 @@ module API end end - # Get a single +noteable+ note - # - # Parameters: - # id (required) - The ID of a project - # noteable_id (required) - The ID of an issue or snippet - # note_id (required) - The ID of a note - # Example Request: - # GET /projects/:id/issues/:noteable_id/notes/:note_id - # GET /projects/:id/snippets/:noteable_id/notes/:note_id - get ":id/#{noteables_str}/:#{noteable_id_str}/notes/:note_id" do - @noteable = user_project.send(noteables_str.to_sym).find(params[noteable_id_str.to_sym]) - @note = @noteable.notes.find(params[:note_id]) - can_read_note = can?(current_user, noteable_read_ability_name(@noteable), @noteable) && !@note.cross_reference_not_visible_for?(current_user) + desc 'Get a single +noteable+ note' do + success Entities::Note + end + params do + requires :note_id, type: Integer, desc: 'The ID of a note' + requires :noteable_id, type: Integer, desc: 'The ID of the noteable' + end + get ":id/#{noteables_str}/:noteable_id/notes/:note_id" do + noteable = user_project.send(noteables_str.to_sym).find(params[:noteable_id]) + note = noteable.notes.find(params[:note_id]) + can_read_note = can?(current_user, noteable_read_ability_name(noteable), noteable) && !note.cross_reference_not_visible_for?(current_user) if can_read_note - present @note, with: Entities::Note + present note, with: Entities::Note else not_found!("Note") end end - # Create a new +noteable+ note - # - # Parameters: - # id (required) - The ID of a project - # noteable_id (required) - The ID of an issue or snippet - # body (required) - The content of a note - # created_at (optional) - The date - # Example Request: - # POST /projects/:id/issues/:noteable_id/notes - # POST /projects/:id/snippets/:noteable_id/notes - post ":id/#{noteables_str}/:#{noteable_id_str}/notes" do - required_attributes! [:body] - + desc 'Create a new +noteable+ note' do + success Entities::Note + end + params do + requires :noteable_id, type: Integer, desc: 'The ID of the noteable' + requires :body, type: String, desc: 'The content of a note' + optional :created_at, type: String, desc: 'The creation date of the note' + end + post ":id/#{noteables_str}/:noteable_id/notes" do opts = { - note: params[:body], - noteable_type: noteables_str.classify, - noteable_id: params[noteable_id_str] + note: params[:body], + noteable_type: noteables_str.classify, + noteable_id: params[:noteable_id] } - if params[:created_at] && (current_user.is_admin? || user_project.owner == current_user) - opts[:created_at] = params[:created_at] - end + noteable = user_project.send(noteables_str.to_sym).find(params[:noteable_id]) - note = ::Notes::CreateService.new(user_project, current_user, opts).execute + if can?(current_user, noteable_read_ability_name(noteable), noteable) + if params[:created_at] && (current_user.is_admin? || user_project.owner == current_user) + opts[:created_at] = params[:created_at] + end - if note.valid? - present note, with: Entities::const_get(note.class.name) + note = ::Notes::CreateService.new(user_project, current_user, opts).execute + + if note.valid? + present note, with: Entities::const_get(note.class.name) + else + not_found!("Note #{note.errors.messages}") + end else - not_found!("Note #{note.errors.messages}") + not_found!("Note") end end - # Modify existing +noteable+ note - # - # Parameters: - # id (required) - The ID of a project - # noteable_id (required) - The ID of an issue or snippet - # node_id (required) - The ID of a note - # body (required) - New content of a note - # Example Request: - # PUT /projects/:id/issues/:noteable_id/notes/:note_id - # PUT /projects/:id/snippets/:noteable_id/notes/:node_id - put ":id/#{noteables_str}/:#{noteable_id_str}/notes/:note_id" do - required_attributes! [:body] - + desc 'Update an existing +noteable+ note' do + success Entities::Note + end + params do + requires :noteable_id, type: Integer, desc: 'The ID of the noteable' + requires :note_id, type: Integer, desc: 'The ID of a note' + requires :body, type: String, desc: 'The content of a note' + end + put ":id/#{noteables_str}/:noteable_id/notes/:note_id" do note = user_project.notes.find(params[:note_id]) authorize! :admin_note, note @@ -113,25 +111,23 @@ module API note: params[:body] } - @note = ::Notes::UpdateService.new(user_project, current_user, opts).execute(note) + note = ::Notes::UpdateService.new(user_project, current_user, opts).execute(note) - if @note.valid? - present @note, with: Entities::Note + if note.valid? + present note, with: Entities::Note else render_api_error!("Failed to save note #{note.errors.messages}", 400) end end - # Delete a +noteable+ note - # - # Parameters: - # id (required) - The ID of a project - # noteable_id (required) - The ID of an issue, MR, or snippet - # node_id (required) - The ID of a note - # Example Request: - # DELETE /projects/:id/issues/:noteable_id/notes/:note_id - # DELETE /projects/:id/snippets/:noteable_id/notes/:node_id - delete ":id/#{noteables_str}/:#{noteable_id_str}/notes/:note_id" do + desc 'Delete a +noteable+ note' do + success Entities::Note + end + params do + requires :noteable_id, type: Integer, desc: 'The ID of the noteable' + requires :note_id, type: Integer, desc: 'The ID of a note' + end + delete ":id/#{noteables_str}/:noteable_id/notes/:note_id" do note = user_project.notes.find(params[:note_id]) authorize! :admin_note, note diff --git a/lib/api/notification_settings.rb b/lib/api/notification_settings.rb index a70a7e71073..c5e9b3ad69b 100644 --- a/lib/api/notification_settings.rb +++ b/lib/api/notification_settings.rb @@ -33,10 +33,9 @@ module API begin notification_setting.transaction do new_notification_email = params.delete(:notification_email) - declared_params = declared(params, include_missing: false).to_h current_user.update(notification_email: new_notification_email) if new_notification_email - notification_setting.update(declared_params) + notification_setting.update(declared_params(include_missing: false)) end rescue ArgumentError => e # catch level enum error render_api_error! e.to_s, 400 @@ -81,9 +80,7 @@ module API notification_setting = current_user.notification_settings_for(source) begin - declared_params = declared(params, include_missing: false).to_h - - notification_setting.update(declared_params) + notification_setting.update(declared_params(include_missing: false)) rescue ArgumentError => e # catch level enum error render_api_error! e.to_s, 400 end diff --git a/lib/api/pagination_params.rb b/lib/api/pagination_params.rb new file mode 100644 index 00000000000..8c1e4381a74 --- /dev/null +++ b/lib/api/pagination_params.rb @@ -0,0 +1,24 @@ +module API + # Concern for declare pagination params. + # + # @example + # class CustomApiResource < Grape::API + # include PaginationParams + # + # params do + # use :pagination + # end + # end + module PaginationParams + extend ActiveSupport::Concern + + included do + helpers do + params :pagination do + optional :page, type: Integer, desc: 'Current page number' + optional :per_page, type: Integer, desc: 'Number of items per page' + end + end + end + end +end diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb index 2a0c8e1f2c0..b634b1d0222 100644 --- a/lib/api/pipelines.rb +++ b/lib/api/pipelines.rb @@ -1,5 +1,7 @@ module API class Pipelines < Grape::API + include PaginationParams + before { authenticate! } params do @@ -11,8 +13,7 @@ module API success Entities::Pipeline end params do - optional :page, type: Integer, desc: 'Page number of the current request' - optional :per_page, type: Integer, desc: 'Number of items per page' + use :pagination optional :scope, type: String, values: ['running', 'branches', 'tags'], desc: 'Either running, branches, or tags' end @@ -22,6 +23,27 @@ module API pipelines = PipelinesFinder.new(user_project).execute(scope: params[:scope]) present paginate(pipelines), with: Entities::Pipeline end + + desc 'Create a new pipeline' do + detail 'This feature was introduced in GitLab 8.14' + success Entities::Pipeline + end + params do + requires :ref, type: String, desc: 'Reference' + end + post ':id/pipeline' do + authorize! :create_pipeline, user_project + + new_pipeline = Ci::CreatePipelineService.new(user_project, + current_user, + declared_params(include_missing: false)) + .execute(ignore_skip_ci: true, save_on_errors: false) + if new_pipeline.persisted? + present new_pipeline, with: Entities::Pipeline + else + render_validation_error!(new_pipeline) + end + end desc 'Gets a specific pipeline for the project' do detail 'This feature was introduced in GitLab 8.11' diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb index dd93a85dc54..cb679e6658a 100644 --- a/lib/api/project_hooks.rb +++ b/lib/api/project_hooks.rb @@ -1,114 +1,99 @@ module API - # Projects API class ProjectHooks < Grape::API + include PaginationParams + before { authenticate! } before { authorize_admin_project } + helpers do + params :project_hook_properties do + requires :url, type: String, desc: "The URL to send the request to" + optional :push_events, type: Boolean, desc: "Trigger hook on push events" + optional :issues_events, type: Boolean, desc: "Trigger hook on issues events" + optional :merge_requests_events, type: Boolean, desc: "Trigger hook on merge request events" + optional :tag_push_events, type: Boolean, desc: "Trigger hook on tag push events" + optional :note_events, type: Boolean, desc: "Trigger hook on note(comment) events" + optional :build_events, type: Boolean, desc: "Trigger hook on build events" + optional :pipeline_events, type: Boolean, desc: "Trigger hook on pipeline events" + optional :wiki_page_events, type: Boolean, desc: "Trigger hook on wiki events" + optional :enable_ssl_verification, type: Boolean, desc: "Do SSL verification when triggering the hook" + optional :token, type: String, desc: "Secret token to validate received payloads; this will not be returned in the response" + end + end + + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do - # Get project hooks - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # GET /projects/:id/hooks + desc 'Get project hooks' do + success Entities::ProjectHook + end + params do + use :pagination + end get ":id/hooks" do - @hooks = paginate user_project.hooks - present @hooks, with: Entities::ProjectHook + hooks = paginate user_project.hooks + + present hooks, with: Entities::ProjectHook end - # Get a project hook - # - # Parameters: - # id (required) - The ID of a project - # hook_id (required) - The ID of a project hook - # Example Request: - # GET /projects/:id/hooks/:hook_id + desc 'Get a project hook' do + success Entities::ProjectHook + end + params do + requires :hook_id, type: Integer, desc: 'The ID of a project hook' + end get ":id/hooks/:hook_id" do - @hook = user_project.hooks.find(params[:hook_id]) - present @hook, with: Entities::ProjectHook + hook = user_project.hooks.find(params[:hook_id]) + present hook, with: Entities::ProjectHook end - # Add hook to project - # - # Parameters: - # id (required) - The ID of a project - # url (required) - The hook URL - # Example Request: - # POST /projects/:id/hooks + desc 'Add hook to project' do + success Entities::ProjectHook + end + params do + use :project_hook_properties + end post ":id/hooks" do - required_attributes! [:url] - attrs = attributes_for_keys [ - :url, - :push_events, - :issues_events, - :merge_requests_events, - :tag_push_events, - :note_events, - :build_events, - :pipeline_events, - :wiki_page_events, - :enable_ssl_verification, - :token - ] - @hook = user_project.hooks.new(attrs) + hook = user_project.hooks.new(declared_params(include_missing: false)) - if @hook.save - present @hook, with: Entities::ProjectHook + if hook.save + present hook, with: Entities::ProjectHook else - if @hook.errors[:url].present? - error!("Invalid url given", 422) - end - not_found!("Project hook #{@hook.errors.messages}") + error!("Invalid url given", 422) if hook.errors[:url].present? + + not_found!("Project hook #{hook.errors.messages}") end end - # Update an existing project hook - # - # Parameters: - # id (required) - The ID of a project - # hook_id (required) - The ID of a project hook - # url (required) - The hook URL - # Example Request: - # PUT /projects/:id/hooks/:hook_id + desc 'Update an existing project hook' do + success Entities::ProjectHook + end + params do + requires :hook_id, type: Integer, desc: "The ID of the hook to update" + use :project_hook_properties + end put ":id/hooks/:hook_id" do - @hook = user_project.hooks.find(params[:hook_id]) - required_attributes! [:url] - attrs = attributes_for_keys [ - :url, - :push_events, - :issues_events, - :merge_requests_events, - :tag_push_events, - :note_events, - :build_events, - :pipeline_events, - :wiki_page_events, - :enable_ssl_verification, - :token - ] + hook = user_project.hooks.find(params.delete(:hook_id)) - if @hook.update_attributes attrs - present @hook, with: Entities::ProjectHook + if hook.update_attributes(declared_params(include_missing: false)) + present hook, with: Entities::ProjectHook else - if @hook.errors[:url].present? - error!("Invalid url given", 422) - end - not_found!("Project hook #{@hook.errors.messages}") + error!("Invalid url given", 422) if hook.errors[:url].present? + + not_found!("Project hook #{hook.errors.messages}") end end - # Deletes project hook. This is an idempotent function. - # - # Parameters: - # id (required) - The ID of a project - # hook_id (required) - The ID of hook to delete - # Example Request: - # DELETE /projects/:id/hooks/:hook_id + desc 'Deletes project hook' do + success Entities::ProjectHook + end + params do + requires :hook_id, type: Integer, desc: 'The ID of the hook to delete' + end delete ":id/hooks/:hook_id" do - required_attributes! [:hook_id] - begin - @hook = user_project.hooks.destroy(params[:hook_id]) + present user_project.hooks.destroy(params[:hook_id]), with: Entities::ProjectHook rescue # ProjectHook can raise Error if hook_id not found not_found!("Error deleting hook #{params[:hook_id]}") diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb index ce1bf0d26d2..9d8c5b63685 100644 --- a/lib/api/project_snippets.rb +++ b/lib/api/project_snippets.rb @@ -1,8 +1,12 @@ module API - # Projects API class ProjectSnippets < Grape::API + include PaginationParams + before { authenticate! } + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do helpers do def handle_project_member_errors(errors) @@ -18,111 +22,111 @@ module API end end - # Get a project snippets - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # GET /projects/:id/snippets + desc 'Get all project snippets' do + success Entities::ProjectSnippet + end + params do + use :pagination + end get ":id/snippets" do present paginate(snippets_for_current_user), with: Entities::ProjectSnippet end - # Get a project snippet - # - # Parameters: - # id (required) - The ID of a project - # snippet_id (required) - The ID of a project snippet - # Example Request: - # GET /projects/:id/snippets/:snippet_id + desc 'Get a single project snippet' do + success Entities::ProjectSnippet + end + params do + requires :snippet_id, type: Integer, desc: 'The ID of a project snippet' + end get ":id/snippets/:snippet_id" do - @snippet = snippets_for_current_user.find(params[:snippet_id]) - present @snippet, with: Entities::ProjectSnippet - end - - # Create a new project snippet - # - # Parameters: - # id (required) - The ID of a project - # title (required) - The title of a snippet - # file_name (required) - The name of a snippet file - # code (required) - The content of a snippet - # visibility_level (required) - The snippet's visibility - # Example Request: - # POST /projects/:id/snippets + snippet = snippets_for_current_user.find(params[:snippet_id]) + present snippet, with: Entities::ProjectSnippet + end + + desc 'Create a new project snippet' do + success Entities::ProjectSnippet + end + params do + requires :title, type: String, desc: 'The title of the snippet' + requires :file_name, type: String, desc: 'The file name of the snippet' + requires :code, type: String, desc: 'The content of the snippet' + requires :visibility_level, type: Integer, + values: [Gitlab::VisibilityLevel::PRIVATE, + Gitlab::VisibilityLevel::INTERNAL, + Gitlab::VisibilityLevel::PUBLIC], + desc: 'The visibility level of the snippet' + end post ":id/snippets" do authorize! :create_project_snippet, user_project - required_attributes! [:title, :file_name, :code, :visibility_level] + snippet_params = declared_params + snippet_params[:content] = snippet_params.delete(:code) - attrs = attributes_for_keys [:title, :file_name, :visibility_level] - attrs[:content] = params[:code] if params[:code].present? - @snippet = CreateSnippetService.new(user_project, current_user, - attrs).execute + snippet = CreateSnippetService.new(user_project, current_user, snippet_params).execute - if @snippet.errors.any? - render_validation_error!(@snippet) + if snippet.persisted? + present snippet, with: Entities::ProjectSnippet else - present @snippet, with: Entities::ProjectSnippet + render_validation_error!(snippet) end end - # Update an existing project snippet - # - # Parameters: - # id (required) - The ID of a project - # snippet_id (required) - The ID of a project snippet - # title (optional) - The title of a snippet - # file_name (optional) - The name of a snippet file - # code (optional) - The content of a snippet - # visibility_level (optional) - The snippet's visibility - # Example Request: - # PUT /projects/:id/snippets/:snippet_id + desc 'Update an existing project snippet' do + success Entities::ProjectSnippet + end + params do + requires :snippet_id, type: Integer, desc: 'The ID of a project snippet' + optional :title, type: String, desc: 'The title of the snippet' + optional :file_name, type: String, desc: 'The file name of the snippet' + optional :code, type: String, desc: 'The content of the snippet' + optional :visibility_level, type: Integer, + values: [Gitlab::VisibilityLevel::PRIVATE, + Gitlab::VisibilityLevel::INTERNAL, + Gitlab::VisibilityLevel::PUBLIC], + desc: 'The visibility level of the snippet' + at_least_one_of :title, :file_name, :code, :visibility_level + end put ":id/snippets/:snippet_id" do - @snippet = snippets_for_current_user.find(params[:snippet_id]) - authorize! :update_project_snippet, @snippet + snippet = snippets_for_current_user.find_by(id: params.delete(:snippet_id)) + not_found!('Snippet') unless snippet + + authorize! :update_project_snippet, snippet + + snippet_params = declared_params(include_missing: false) + snippet_params[:content] = snippet_params.delete(:code) if snippet_params[:code].present? - attrs = attributes_for_keys [:title, :file_name, :visibility_level] - attrs[:content] = params[:code] if params[:code].present? + UpdateSnippetService.new(user_project, current_user, snippet, + snippet_params).execute - UpdateSnippetService.new(user_project, current_user, @snippet, - attrs).execute - if @snippet.errors.any? - render_validation_error!(@snippet) + if snippet.persisted? + present snippet, with: Entities::ProjectSnippet else - present @snippet, with: Entities::ProjectSnippet + render_validation_error!(snippet) end end - # Delete a project snippet - # - # Parameters: - # id (required) - The ID of a project - # snippet_id (required) - The ID of a project snippet - # Example Request: - # DELETE /projects/:id/snippets/:snippet_id + desc 'Delete a project snippet' + params do + requires :snippet_id, type: Integer, desc: 'The ID of a project snippet' + end delete ":id/snippets/:snippet_id" do - begin - @snippet = snippets_for_current_user.find(params[:snippet_id]) - authorize! :update_project_snippet, @snippet - @snippet.destroy - rescue - not_found!('Snippet') - end + snippet = snippets_for_current_user.find_by(id: params[:snippet_id]) + not_found!('Snippet') unless snippet + + authorize! :admin_project_snippet, snippet + snippet.destroy end - # Get a raw project snippet - # - # Parameters: - # id (required) - The ID of a project - # snippet_id (required) - The ID of a project snippet - # Example Request: - # GET /projects/:id/snippets/:snippet_id/raw + desc 'Get a raw project snippet' + params do + requires :snippet_id, type: Integer, desc: 'The ID of a project snippet' + end get ":id/snippets/:snippet_id/raw" do - @snippet = snippets_for_current_user.find(params[:snippet_id]) + snippet = snippets_for_current_user.find_by(id: params[:snippet_id]) + not_found!('Snippet') unless snippet env['api.format'] = :txt content_type 'text/plain' - present @snippet.content + present snippet.content end end end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index da16e24d7ea..941f47114a4 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -1,305 +1,313 @@ module API # Projects API class Projects < Grape::API - before { authenticate! } + include PaginationParams + + before { authenticate_non_get! } + + helpers do + params :optional_params do + optional :description, type: String, desc: 'The description of the project' + optional :issues_enabled, type: Boolean, desc: 'Flag indication if the issue tracker is enabled' + optional :merge_requests_enabled, type: Boolean, desc: 'Flag indication if merge requests are enabled' + optional :wiki_enabled, type: Boolean, desc: 'Flag indication if the wiki is enabled' + optional :builds_enabled, type: Boolean, desc: 'Flag indication if builds are enabled' + optional :snippets_enabled, type: Boolean, desc: 'Flag indication if snippets are enabled' + optional :shared_runners_enabled, type: Boolean, desc: 'Flag indication if shared runners are enabled for that project' + optional :container_registry_enabled, type: Boolean, desc: 'Flag indication if the container registry is enabled for that project' + optional :lfs_enabled, type: Boolean, desc: 'Flag indication if Git LFS is enabled for that project' + optional :public, type: Boolean, desc: 'Create a public project. The same as visibility_level = 20.' + optional :visibility_level, type: Integer, values: [ + Gitlab::VisibilityLevel::PRIVATE, + Gitlab::VisibilityLevel::INTERNAL, + Gitlab::VisibilityLevel::PUBLIC ], desc: 'Create a public project. The same as visibility_level = 20.' + optional :public_builds, type: Boolean, desc: 'Perform public builds' + optional :request_access_enabled, type: Boolean, desc: 'Allow users to request member access' + optional :only_allow_merge_if_build_succeeds, type: Boolean, desc: 'Only allow to merge if builds succeed' + optional :only_allow_merge_if_all_discussions_are_resolved, type: Boolean, desc: 'Only allow to merge if all discussions are resolved' + end - resource :projects, requirements: { id: /[^\/]+/ } do - helpers do - def map_public_to_visibility_level(attrs) - publik = attrs.delete(:public) - if publik.present? && !attrs[:visibility_level].present? - publik = to_boolean(publik) - # Since setting the public attribute to private could mean either - # private or internal, use the more conservative option, private. - attrs[:visibility_level] = (publik == true) ? Gitlab::VisibilityLevel::PUBLIC : Gitlab::VisibilityLevel::PRIVATE - end - attrs + def map_public_to_visibility_level(attrs) + publik = attrs.delete(:public) + if !publik.nil? && !attrs[:visibility_level].present? + # Since setting the public attribute to private could mean either + # private or internal, use the more conservative option, private. + attrs[:visibility_level] = (publik == true) ? Gitlab::VisibilityLevel::PUBLIC : Gitlab::VisibilityLevel::PRIVATE end + attrs end + end - # Get a projects list for authenticated user - # - # Example Request: - # GET /projects - get do - projects = current_user.authorized_projects - projects = filter_projects(projects) - projects = paginate projects - entity = params[:simple] ? Entities::BasicProjectDetails : Entities::ProjectWithAccess + resource :projects do + helpers do + params :collection_params do + use :sort_params + use :filter_params + use :pagination - present projects, with: entity, user: current_user + optional :simple, type: Boolean, default: false, + desc: 'Return only the ID, URL, name, and path of each project' + end + + params :sort_params do + optional :order_by, type: String, values: %w[id name path created_at updated_at last_activity_at], + default: 'created_at', desc: 'Return projects ordered by field' + optional :sort, type: String, values: %w[asc desc], default: 'desc', + desc: 'Return projects sorted in ascending and descending order' + end + + params :filter_params do + optional :archived, type: Boolean, default: false, desc: 'Limit by archived status' + optional :visibility, type: String, values: %w[public internal private], + desc: 'Limit by visibility' + optional :search, type: String, desc: 'Return list of authorized projects matching the search criteria' + end + + params :statistics_params do + optional :statistics, type: Boolean, default: false, desc: 'Include project statistics' + end + + params :create_params do + optional :namespace_id, type: Integer, desc: 'Namespace ID for the new project. Default to the user namespace.' + optional :import_url, type: String, desc: 'URL from which the project is imported' + end + + def present_projects(projects, options = {}) + options = options.reverse_merge( + with: Entities::Project, + current_user: current_user, + simple: params[:simple], + ) + + projects = filter_projects(projects) + projects = projects.with_statistics if options[:statistics] + options[:with] = Entities::BasicProjectDetails if options[:simple] + + present paginate(projects), options + end end - # Get a list of visible projects for authenticated user - # - # Example Request: - # GET /projects/visible + desc 'Get a list of visible projects for authenticated user' do + success Entities::BasicProjectDetails + end + params do + use :collection_params + end get '/visible' do - projects = ProjectsFinder.new.execute(current_user) - projects = filter_projects(projects) - projects = paginate projects - entity = params[:simple] ? Entities::BasicProjectDetails : Entities::ProjectWithAccess + entity = current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails + present_projects ProjectsFinder.new.execute(current_user), with: entity + end + + desc 'Get a projects list for authenticated user' do + success Entities::BasicProjectDetails + end + params do + use :collection_params + end + get do + authenticate! - present projects, with: entity, user: current_user + present_projects current_user.authorized_projects, + with: Entities::ProjectWithAccess end - # Get an owned projects list for authenticated user - # - # Example Request: - # GET /projects/owned + desc 'Get an owned projects list for authenticated user' do + success Entities::BasicProjectDetails + end + params do + use :collection_params + use :statistics_params + end get '/owned' do - projects = current_user.owned_projects - projects = filter_projects(projects) - projects = paginate projects - present projects, with: Entities::ProjectWithAccess, user: current_user + authenticate! + + present_projects current_user.owned_projects, + with: Entities::ProjectWithAccess, + statistics: params[:statistics] end - # Gets starred project for the authenticated user - # - # Example Request: - # GET /projects/starred + desc 'Gets starred project for the authenticated user' do + success Entities::BasicProjectDetails + end + params do + use :collection_params + end get '/starred' do - projects = current_user.viewable_starred_projects - projects = filter_projects(projects) - projects = paginate projects - present projects, with: Entities::Project, user: current_user + authenticate! + + present_projects current_user.viewable_starred_projects end - # Get all projects for admin user - # - # Example Request: - # GET /projects/all + desc 'Get all projects for admin user' do + success Entities::BasicProjectDetails + end + params do + use :collection_params + use :statistics_params + end get '/all' do authenticated_as_admin! - projects = Project.all - projects = filter_projects(projects) - projects = paginate projects - present projects, with: Entities::ProjectWithAccess, user: current_user - end - - # Get a single project - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # GET /projects/:id - get ":id" do - present user_project, with: Entities::ProjectWithAccess, user: current_user, - user_can_admin_project: can?(current_user, :admin_project, user_project) + + present_projects Project.all, with: Entities::ProjectWithAccess, statistics: params[:statistics] end - # Get events for a single project - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # GET /projects/:id/events - get ":id/events" do - events = paginate user_project.events.recent - present events, with: Entities::Event - end - - # Create new project - # - # Parameters: - # name (required) - name for new project - # description (optional) - short project description - # issues_enabled (optional) - # merge_requests_enabled (optional) - # builds_enabled (optional) - # wiki_enabled (optional) - # snippets_enabled (optional) - # container_registry_enabled (optional) - # shared_runners_enabled (optional) - # namespace_id (optional) - defaults to user namespace - # public (optional) - if true same as setting visibility_level = 20 - # visibility_level (optional) - 0 by default - # import_url (optional) - # public_builds (optional) - # lfs_enabled (optional) - # request_access_enabled (optional) - Allow users to request member access - # Example Request - # POST /projects + desc 'Search for projects the current user has access to' do + success Entities::Project + end + params do + requires :query, type: String, desc: 'The project name to be searched' + use :sort_params + use :pagination + end + get "/search/:query", requirements: { query: /[^\/]+/ } do + search_service = Search::GlobalService.new(current_user, search: params[:query]).execute + projects = search_service.objects('projects', params[:page]) + projects = projects.reorder(params[:order_by] => params[:sort]) + + present paginate(projects), with: Entities::Project + end + + desc 'Create new project' do + success Entities::Project + end + params do + requires :name, type: String, desc: 'The name of the project' + optional :path, type: String, desc: 'The path of the repository' + use :optional_params + use :create_params + end post do - required_attributes! [:name] - attrs = attributes_for_keys [:builds_enabled, - :container_registry_enabled, - :description, - :import_url, - :issues_enabled, - :lfs_enabled, - :merge_requests_enabled, - :name, - :namespace_id, - :only_allow_merge_if_build_succeeds, - :path, - :public, - :public_builds, - :request_access_enabled, - :shared_runners_enabled, - :snippets_enabled, - :visibility_level, - :wiki_enabled] - attrs = map_public_to_visibility_level(attrs) - @project = ::Projects::CreateService.new(current_user, attrs).execute - if @project.saved? - present @project, with: Entities::Project, - user_can_admin_project: can?(current_user, :admin_project, @project) + attrs = map_public_to_visibility_level(declared_params(include_missing: false)) + project = ::Projects::CreateService.new(current_user, attrs).execute + + if project.saved? + present project, with: Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, project) else - if @project.errors[:limit_reached].present? - error!(@project.errors[:limit_reached], 403) + if project.errors[:limit_reached].present? + error!(project.errors[:limit_reached], 403) end - render_validation_error!(@project) + render_validation_error!(project) end end - # Create new project for a specified user. Only available to admin users. - # - # Parameters: - # user_id (required) - The ID of a user - # name (required) - name for new project - # description (optional) - short project description - # default_branch (optional) - 'master' by default - # issues_enabled (optional) - # merge_requests_enabled (optional) - # builds_enabled (optional) - # wiki_enabled (optional) - # snippets_enabled (optional) - # container_registry_enabled (optional) - # shared_runners_enabled (optional) - # public (optional) - if true same as setting visibility_level = 20 - # visibility_level (optional) - # import_url (optional) - # public_builds (optional) - # lfs_enabled (optional) - # request_access_enabled (optional) - Allow users to request member access - # Example Request - # POST /projects/user/:user_id + desc 'Create new project for a specified user. Only available to admin users.' do + success Entities::Project + end + params do + requires :name, type: String, desc: 'The name of the project' + requires :user_id, type: Integer, desc: 'The ID of a user' + optional :default_branch, type: String, desc: 'The default branch of the project' + use :optional_params + use :create_params + end post "user/:user_id" do authenticated_as_admin! - user = User.find(params[:user_id]) - attrs = attributes_for_keys [:builds_enabled, - :default_branch, - :description, - :import_url, - :issues_enabled, - :lfs_enabled, - :merge_requests_enabled, - :name, - :only_allow_merge_if_build_succeeds, - :public, - :public_builds, - :request_access_enabled, - :shared_runners_enabled, - :snippets_enabled, - :visibility_level, - :wiki_enabled] - attrs = map_public_to_visibility_level(attrs) - @project = ::Projects::CreateService.new(user, attrs).execute - if @project.saved? - present @project, with: Entities::Project, - user_can_admin_project: can?(current_user, :admin_project, @project) + user = User.find_by(id: params.delete(:user_id)) + not_found!('User') unless user + + attrs = map_public_to_visibility_level(declared_params(include_missing: false)) + project = ::Projects::CreateService.new(user, attrs).execute + + if project.saved? + present project, with: Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, project) else - render_validation_error!(@project) + render_validation_error!(project) end end + end + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: { id: /[^\/]+/ } do + desc 'Get a single project' do + success Entities::ProjectWithAccess + end + get ":id" do + entity = current_user ? Entities::ProjectWithAccess : Entities::BasicProjectDetails + present user_project, with: entity, current_user: current_user, + user_can_admin_project: can?(current_user, :admin_project, user_project) + end + + desc 'Get events for a single project' do + success Entities::Event + end + params do + use :pagination + end + get ":id/events" do + present paginate(user_project.events.recent), with: Entities::Event + end - # Fork new project for the current user or provided namespace. - # - # Parameters: - # id (required) - The ID of a project - # namespace (optional) - The ID or name of the namespace that the project will be forked into. - # Example Request - # POST /projects/fork/:id + desc 'Fork new project for the current user or provided namespace.' do + success Entities::Project + end + params do + optional :namespace, type: String, desc: 'The ID or name of the namespace that the project will be forked into' + end post 'fork/:id' do - attrs = {} - namespace_id = params[:namespace] + fork_params = declared_params(include_missing: false) + namespace_id = fork_params[:namespace] if namespace_id.present? - namespace = Namespace.find_by(id: namespace_id) || Namespace.find_by_path_or_name(namespace_id) + fork_params[:namespace] = if namespace_id =~ /^\d+$/ + Namespace.find_by(id: namespace_id) + else + Namespace.find_by_path_or_name(namespace_id) + end - unless namespace && can?(current_user, :create_projects, namespace) + unless fork_params[:namespace] && can?(current_user, :create_projects, fork_params[:namespace]) not_found!('Target Namespace') end - - attrs[:namespace] = namespace end - @forked_project = - ::Projects::ForkService.new(user_project, - current_user, - attrs).execute + forked_project = ::Projects::ForkService.new(user_project, current_user, fork_params).execute - if @forked_project.errors.any? - conflict!(@forked_project.errors.messages) + if forked_project.errors.any? + conflict!(forked_project.errors.messages) else - present @forked_project, with: Entities::Project, - user_can_admin_project: can?(current_user, :admin_project, @forked_project) + present forked_project, with: Entities::Project, + user_can_admin_project: can?(current_user, :admin_project, forked_project) end end - # Update an existing project - # - # Parameters: - # id (required) - the id of a project - # name (optional) - name of a project - # path (optional) - path of a project - # description (optional) - short project description - # issues_enabled (optional) - # merge_requests_enabled (optional) - # builds_enabled (optional) - # wiki_enabled (optional) - # snippets_enabled (optional) - # container_registry_enabled (optional) - # shared_runners_enabled (optional) - # public (optional) - if true same as setting visibility_level = 20 - # visibility_level (optional) - visibility level of a project - # public_builds (optional) - # lfs_enabled (optional) - # Example Request - # PUT /projects/:id + desc 'Update an existing project' do + success Entities::Project + end + params do + optional :name, type: String, desc: 'The name of the project' + optional :default_branch, type: String, desc: 'The default branch of the project' + optional :path, type: String, desc: 'The path of the repository' + use :optional_params + at_least_one_of :name, :description, :issues_enabled, :merge_requests_enabled, + :wiki_enabled, :builds_enabled, :snippets_enabled, + :shared_runners_enabled, :container_registry_enabled, + :lfs_enabled, :public, :visibility_level, :public_builds, + :request_access_enabled, :only_allow_merge_if_build_succeeds, + :only_allow_merge_if_all_discussions_are_resolved, :path, + :default_branch + end put ':id' do - attrs = attributes_for_keys [:builds_enabled, - :container_registry_enabled, - :default_branch, - :description, - :issues_enabled, - :lfs_enabled, - :merge_requests_enabled, - :name, - :only_allow_merge_if_build_succeeds, - :path, - :public, - :public_builds, - :request_access_enabled, - :shared_runners_enabled, - :snippets_enabled, - :visibility_level, - :wiki_enabled] - attrs = map_public_to_visibility_level(attrs) authorize_admin_project + attrs = map_public_to_visibility_level(declared_params(include_missing: false)) authorize! :rename_project, user_project if attrs[:name].present? - if attrs[:visibility_level].present? - authorize! :change_visibility_level, user_project - end + authorize! :change_visibility_level, user_project if attrs[:visibility_level].present? - ::Projects::UpdateService.new(user_project, - current_user, attrs).execute + result = ::Projects::UpdateService.new(user_project, current_user, attrs).execute - if user_project.errors.any? - render_validation_error!(user_project) - else + if result[:status] == :success present user_project, with: Entities::Project, user_can_admin_project: can?(current_user, :admin_project, user_project) + else + render_validation_error!(user_project) end end - # Archive project - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # PUT /projects/:id/archive + desc 'Archive a project' do + success Entities::Project + end post ':id/archive' do authorize!(:archive_project, user_project) @@ -308,12 +316,9 @@ module API present user_project, with: Entities::Project end - # Unarchive project - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # PUT /projects/:id/unarchive + desc 'Unarchive a project' do + success Entities::Project + end post ':id/unarchive' do authorize!(:archive_project, user_project) @@ -322,12 +327,9 @@ module API present user_project, with: Entities::Project end - # Star project - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # POST /projects/:id/star + desc 'Star a project' do + success Entities::Project + end post ':id/star' do if current_user.starred?(user_project) not_modified! @@ -339,12 +341,9 @@ module API end end - # Unstar project - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # DELETE /projects/:id/star + desc 'Unstar a project' do + success Entities::Project + end delete ':id/star' do if current_user.starred?(user_project) current_user.toggle_star(user_project) @@ -356,67 +355,51 @@ module API end end - # Remove project - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # DELETE /projects/:id + desc 'Remove a project' delete ":id" do authorize! :remove_project, user_project ::Projects::DestroyService.new(user_project, current_user, {}).async_execute end - # Mark this project as forked from another - # - # Parameters: - # id: (required) - The ID of the project being marked as a fork - # forked_from_id: (required) - The ID of the project it was forked from - # Example Request: - # POST /projects/:id/fork/:forked_from_id + desc 'Mark this project as forked from another' + params do + requires :forked_from_id, type: String, desc: 'The ID of the project it was forked from' + end post ":id/fork/:forked_from_id" do authenticated_as_admin! - forked_from_project = find_project(params[:forked_from_id]) - unless forked_from_project.nil? - if user_project.forked_from_project.nil? - user_project.create_forked_project_link(forked_to_project_id: user_project.id, forked_from_project_id: forked_from_project.id) - else - render_api_error!("Project already forked", 409) - end + + forked_from_project = find_project!(params[:forked_from_id]) + not_found!("Source Project") unless forked_from_project + + if user_project.forked_from_project.nil? + user_project.create_forked_project_link(forked_to_project_id: user_project.id, forked_from_project_id: forked_from_project.id) else - not_found!("Source Project") + render_api_error!("Project already forked", 409) end end - # Remove a forked_from relationship - # - # Parameters: - # id: (required) - The ID of the project being marked as a fork - # Example Request: - # DELETE /projects/:id/fork + desc 'Remove a forked_from relationship' delete ":id/fork" do authorize! :remove_fork_project, user_project + if user_project.forked? user_project.forked_project_link.destroy + else + not_modified! end end - # Share project with group - # - # Parameters: - # id (required) - The ID of a project - # group_id (required) - The ID of a group - # group_access (required) - Level of permissions for sharing - # expires_at (optional) - Share expiration date - # - # Example Request: - # POST /projects/:id/share + desc 'Share the project with a group' do + success Entities::ProjectGroupLink + end + params do + requires :group_id, type: Integer, desc: 'The ID of a group' + requires :group_access, type: Integer, values: Gitlab::Access.values, desc: 'The group access level' + optional :expires_at, type: Date, desc: 'Share expiration date' + end post ":id/share" do authorize! :admin_project, user_project - required_attributes! [:group_id, :group_access] - attrs = attributes_for_keys [:group_id, :group_access, :expires_at] - - group = Group.find_by_id(attrs[:group_id]) + group = Group.find_by_id(params[:group_id]) unless group && can?(current_user, :read_group, group) not_found!('Group') @@ -426,7 +409,7 @@ module API return render_api_error!("The project sharing with group is disabled", 400) end - link = user_project.project_group_links.new(attrs) + link = user_project.project_group_links.new(declared_params(include_missing: false)) if link.save present link, with: Entities::ProjectGroupLink @@ -435,40 +418,39 @@ module API end end - # Upload a file - # - # Parameters: - # id: (required) - The ID of the project - # file: (required) - The file to be uploaded - post ":id/uploads" do - ::Projects::UploadService.new(user_project, params[:file]).execute + params do + requires :group_id, type: Integer, desc: 'The ID of the group' end + delete ":id/share/:group_id" do + authorize! :admin_project, user_project - # search for projects current_user has access to - # - # Parameters: - # query (required) - A string contained in the project name - # per_page (optional) - number of projects to return per page - # page (optional) - the page to retrieve - # Example Request: - # GET /projects/search/:query - get "/search/:query" do - search_service = Search::GlobalService.new(current_user, search: params[:query]).execute - projects = search_service.objects('projects', params[:page]) - projects = projects.reorder(project_order_by => project_sort) + link = user_project.project_group_links.find_by(group_id: params[:group_id]) + not_found!('Group Link') unless link - present paginate(projects), with: Entities::Project + link.destroy + no_content! end - # Get a users list - # - # Example Request: - # GET /users + desc 'Upload a file' + params do + requires :file, type: File, desc: 'The file to be uploaded' + end + post ":id/uploads" do + ::Projects::UploadService.new(user_project, params[:file]).execute + end + + desc 'Get the users list of a project' do + success Entities::UserBasic + end + params do + optional :search, type: String, desc: 'Return list of users matching the search criteria' + use :pagination + end get ':id/users' do - @users = User.where(id: user_project.team.users.map(&:id)) - @users = @users.search(params[:search]) if params[:search].present? - @users = paginate @users - present @users, with: Entities::UserBasic + users = user_project.team.users + users = users.search(params[:search]) if params[:search].present? + + present paginate(users), with: Entities::UserBasic end end end diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index f55aceed92c..4ca6646a6f1 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -1,11 +1,12 @@ require 'mime/types' module API - # Projects API class Repositories < Grape::API - before { authenticate! } before { authorize! :download_code, user_project } + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do helpers do def handle_project_member_errors(errors) @@ -16,13 +17,14 @@ module API end end - # Get a project repository tree - # - # Parameters: - # id (required) - The ID of a project - # ref_name (optional) - The name of a repository branch or tag, if not given the default branch is used - # Example Request: - # GET /projects/:id/repository/tree + desc 'Get a project repository tree' do + success Entities::RepoTreeObject + end + params do + optional :ref_name, type: String, desc: 'The name of a repository branch or tag, if not given the default branch is used' + optional :path, type: String, desc: 'The path of the tree' + optional :recursive, type: Boolean, default: false, desc: 'Used to get a recursive tree' + end get ':id/repository/tree' do ref = params[:ref_name] || user_project.try(:default_branch) || 'master' path = params[:path] || nil @@ -30,27 +32,20 @@ module API commit = user_project.commit(ref) not_found!('Tree') unless commit - tree = user_project.repository.tree(commit.id, path) + tree = user_project.repository.tree(commit.id, path, recursive: params[:recursive]) present tree.sorted_entries, with: Entities::RepoTreeObject end - # Get a raw file contents - # - # Parameters: - # id (required) - The ID of a project - # sha (required) - The commit or branch name - # filepath (required) - The path to the file to display - # Example Request: - # GET /projects/:id/repository/blobs/:sha + desc 'Get a raw file contents' + params do + requires :sha, type: String, desc: 'The commit, branch name, or tag name' + requires :filepath, type: String, desc: 'The path to the file to display' + end get [ ":id/repository/blobs/:sha", ":id/repository/commits/:sha/blob" ] do - required_attributes! [:filepath] - - ref = params[:sha] - repo = user_project.repository - commit = repo.commit(ref) + commit = repo.commit(params[:sha]) not_found! "Commit" unless commit blob = Gitlab::Git::Blob.find(repo, commit.id, params[:filepath]) @@ -59,20 +54,15 @@ module API send_git_blob repo, blob end - # Get a raw blob contents by blob sha - # - # Parameters: - # id (required) - The ID of a project - # sha (required) - The blob's sha - # Example Request: - # GET /projects/:id/repository/raw_blobs/:sha + desc 'Get a raw blob contents by blob sha' + params do + requires :sha, type: String, desc: 'The commit, branch name, or tag name' + end get ':id/repository/raw_blobs/:sha' do - ref = params[:sha] - repo = user_project.repository begin - blob = Gitlab::Git::Blob.raw(repo, ref) + blob = Gitlab::Git::Blob.raw(repo, params[:sha]) rescue not_found! 'Blob' end @@ -82,17 +72,12 @@ module API send_git_blob repo, blob end - # Get a an archive of the repository - # - # Parameters: - # id (required) - The ID of a project - # sha (optional) - the commit sha to download defaults to the tip of the default branch - # Example Request: - # GET /projects/:id/repository/archive - get ':id/repository/archive', - requirements: { format: Gitlab::Regex.archive_formats_regex } do - authorize! :download_code, user_project - + desc 'Get an archive of the repository' + params do + optional :sha, type: String, desc: 'The commit sha of the archive to be downloaded' + optional :format, type: String, desc: 'The archive format' + end + get ':id/repository/archive', requirements: { format: Gitlab::Regex.archive_formats_regex } do begin send_git_archive user_project.repository, ref: params[:sha], format: params[:format] rescue @@ -100,30 +85,22 @@ module API end end - # Compare two branches, tags or commits - # - # Parameters: - # id (required) - The ID of a project - # from (required) - the commit sha or branch name - # to (required) - the commit sha or branch name - # Example Request: - # GET /projects/:id/repository/compare?from=master&to=feature + desc 'Compare two branches, tags, or commits' do + success Entities::Compare + end + params do + requires :from, type: String, desc: 'The commit, branch name, or tag name to start comparison' + requires :to, type: String, desc: 'The commit, branch name, or tag name to stop comparison' + end get ':id/repository/compare' do - authorize! :download_code, user_project - required_attributes! [:from, :to] compare = Gitlab::Git::Compare.new(user_project.repository.raw_repository, params[:from], params[:to]) present compare, with: Entities::Compare end - # Get repository contributors - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # GET /projects/:id/repository/contributors + desc 'Get repository contributors' do + success Entities::Contributor + end get ':id/repository/contributors' do - authorize! :download_code, user_project - begin present user_project.repository.contributors, with: Entities::Contributor diff --git a/lib/api/runners.rb b/lib/api/runners.rb index ecc8f2fc5a2..4816b5ed1b7 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -1,34 +1,43 @@ module API - # Runners API class Runners < Grape::API + include PaginationParams + before { authenticate! } resource :runners do - # Get runners available for user - # - # Example Request: - # GET /runners + desc 'Get runners available for user' do + success Entities::Runner + end + params do + optional :scope, type: String, values: %w[active paused online], + desc: 'The scope of specific runners to show' + use :pagination + end get do runners = filter_runners(current_user.ci_authorized_runners, params[:scope], without: ['specific', 'shared']) present paginate(runners), with: Entities::Runner end - # Get all runners - shared and specific - # - # Example Request: - # GET /runners/all + desc 'Get all runners - shared and specific' do + success Entities::Runner + end + params do + optional :scope, type: String, values: %w[active paused online specific shared], + desc: 'The scope of specific runners to show' + use :pagination + end get 'all' do authenticated_as_admin! runners = filter_runners(Ci::Runner.all, params[:scope]) present paginate(runners), with: Entities::Runner end - # Get runner's details - # - # Parameters: - # id (required) - The ID of ther runner - # Example Request: - # GET /runners/:id + desc "Get runner's details" do + success Entities::RunnerDetails + end + params do + requires :id, type: Integer, desc: 'The ID of the runner' + end get ':id' do runner = get_runner(params[:id]) authenticate_show_runner!(runner) @@ -36,33 +45,35 @@ module API present runner, with: Entities::RunnerDetails, current_user: current_user end - # Update runner's details - # - # Parameters: - # id (required) - The ID of ther runner - # description (optional) - Runner's description - # active (optional) - Runner's status - # tag_list (optional) - Array of tags for runner - # Example Request: - # PUT /runners/:id + desc "Update runner's details" do + success Entities::RunnerDetails + end + params do + requires :id, type: Integer, desc: 'The ID of the runner' + optional :description, type: String, desc: 'The description of the runner' + optional :active, type: Boolean, desc: 'The state of a runner' + optional :tag_list, type: Array[String], desc: 'The list of tags for a runner' + optional :run_untagged, type: Boolean, desc: 'Flag indicating the runner can execute untagged jobs' + optional :locked, type: Boolean, desc: 'Flag indicating the runner is locked' + at_least_one_of :description, :active, :tag_list, :run_untagged, :locked + end put ':id' do - runner = get_runner(params[:id]) + runner = get_runner(params.delete(:id)) authenticate_update_runner!(runner) - attrs = attributes_for_keys [:description, :active, :tag_list, :run_untagged, :locked] - if runner.update(attrs) + if runner.update(declared_params(include_missing: false)) present runner, with: Entities::RunnerDetails, current_user: current_user else render_validation_error!(runner) end end - # Remove runner - # - # Parameters: - # id (required) - The ID of ther runner - # Example Request: - # DELETE /runners/:id + desc 'Remove a runner' do + success Entities::Runner + end + params do + requires :id, type: Integer, desc: 'The ID of the runner' + end delete ':id' do runner = get_runner(params[:id]) authenticate_delete_runner!(runner) @@ -72,28 +83,32 @@ module API end end + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do before { authorize_admin_project } - # Get runners available for project - # - # Example Request: - # GET /projects/:id/runners + desc 'Get runners available for project' do + success Entities::Runner + end + params do + optional :scope, type: String, values: %w[active paused online specific shared], + desc: 'The scope of specific runners to show' + use :pagination + end get ':id/runners' do runners = filter_runners(Ci::Runner.owned_or_shared(user_project.id), params[:scope]) present paginate(runners), with: Entities::Runner end - # Enable runner for project - # - # Parameters: - # id (required) - The ID of the project - # runner_id (required) - The ID of the runner - # Example Request: - # POST /projects/:id/runners/:runner_id + desc 'Enable a runner for a project' do + success Entities::Runner + end + params do + requires :runner_id, type: Integer, desc: 'The ID of the runner' + end post ':id/runners' do - required_attributes! [:runner_id] - runner = get_runner(params[:runner_id]) authenticate_enable_runner!(runner) @@ -106,13 +121,12 @@ module API end end - # Disable project's runner - # - # Parameters: - # id (required) - The ID of the project - # runner_id (required) - The ID of the runner - # Example Request: - # DELETE /projects/:id/runners/:runner_id + desc "Disable project's runner" do + success Entities::Runner + end + params do + requires :runner_id, type: Integer, desc: 'The ID of the runner' + end delete ':id/runners/:runner_id' do runner_project = user_project.runner_projects.find_by(runner_id: params[:runner_id]) not_found!('Runner') unless runner_project diff --git a/lib/api/services.rb b/lib/api/services.rb index fc8598daa32..a0abec49438 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -1,63 +1,696 @@ module API - # Projects API class Services < Grape::API - before { authenticate! } - before { authorize_admin_project } + services = { + 'asana' => [ + { + required: true, + name: :api_key, + type: String, + desc: 'User API token' + }, + { + required: false, + name: :restrict_to_branch, + type: String, + desc: 'Comma-separated list of branches which will be automatically inspected. Leave blank to include all branches' + } + ], + 'assembla' => [ + { + required: true, + name: :token, + type: String, + desc: 'The authentication token' + }, + { + required: false, + name: :subdomain, + type: String, + desc: 'Subdomain setting' + } + ], + 'bamboo' => [ + { + required: true, + name: :bamboo_url, + type: String, + desc: 'Bamboo root URL like https://bamboo.example.com' + }, + { + required: true, + name: :build_key, + type: String, + desc: 'Bamboo build plan key like' + }, + { + required: true, + name: :username, + type: String, + desc: 'A user with API access, if applicable' + }, + { + required: true, + name: :password, + type: String, + desc: 'Passord of the user' + } + ], + 'bugzilla' => [ + { + required: true, + name: :new_issue_url, + type: String, + desc: 'New issue URL' + }, + { + required: true, + name: :issues_url, + type: String, + desc: 'Issues URL' + }, + { + required: true, + name: :project_url, + type: String, + desc: 'Project URL' + }, + { + required: false, + name: :description, + type: String, + desc: 'Description' + }, + { + required: false, + name: :title, + type: String, + desc: 'Title' + } + ], + 'buildkite' => [ + { + required: true, + name: :token, + type: String, + desc: 'Buildkite project GitLab token' + }, + { + required: true, + name: :project_url, + type: String, + desc: 'The buildkite project URL' + }, + { + required: false, + name: :enable_ssl_verification, + type: Boolean, + desc: 'Enable SSL verification for communication' + } + ], + 'builds-email' => [ + { + required: true, + name: :recipients, + type: String, + desc: 'Comma-separated list of recipient email addresses' + }, + { + required: false, + name: :add_pusher, + type: Boolean, + desc: 'Add pusher to recipients list' + }, + { + required: false, + name: :notify_only_broken_builds, + type: Boolean, + desc: 'Notify only broken builds' + } + ], + 'campfire' => [ + { + required: true, + name: :token, + type: String, + desc: 'Campfire token' + }, + { + required: false, + name: :subdomain, + type: String, + desc: 'Campfire subdomain' + }, + { + required: false, + name: :room, + type: String, + desc: 'Campfire room' + } + ], + 'custom-issue-tracker' => [ + { + required: true, + name: :new_issue_url, + type: String, + desc: 'New issue URL' + }, + { + required: true, + name: :issues_url, + type: String, + desc: 'Issues URL' + }, + { + required: true, + name: :project_url, + type: String, + desc: 'Project URL' + }, + { + required: false, + name: :description, + type: String, + desc: 'Description' + }, + { + required: false, + name: :title, + type: String, + desc: 'Title' + } + ], + 'drone-ci' => [ + { + required: true, + name: :token, + type: String, + desc: 'Drone CI token' + }, + { + required: true, + name: :drone_url, + type: String, + desc: 'Drone CI URL' + }, + { + required: false, + name: :enable_ssl_verification, + type: Boolean, + desc: 'Enable SSL verification for communication' + } + ], + 'emails-on-push' => [ + { + required: true, + name: :recipients, + type: String, + desc: 'Comma-separated list of recipient email addresses' + }, + { + required: false, + name: :disable_diffs, + type: Boolean, + desc: 'Disable code diffs' + }, + { + required: false, + name: :send_from_committer_email, + type: Boolean, + desc: 'Send from committer' + } + ], + 'external-wiki' => [ + { + required: true, + name: :external_wiki_url, + type: String, + desc: 'The URL of the external Wiki' + } + ], + 'flowdock' => [ + { + required: true, + name: :token, + type: String, + desc: 'Flowdock token' + } + ], + 'gemnasium' => [ + { + required: true, + name: :api_key, + type: String, + desc: 'Your personal API key on gemnasium.com' + }, + { + required: true, + name: :token, + type: String, + desc: "The project's slug on gemnasium.com" + } + ], + 'hipchat' => [ + { + required: true, + name: :token, + type: String, + desc: 'The room token' + }, + { + required: false, + name: :room, + type: String, + desc: 'The room name or ID' + }, + { + required: false, + name: :color, + type: String, + desc: 'The room color' + }, + { + required: false, + name: :notify, + type: Boolean, + desc: 'Enable notifications' + }, + { + required: false, + name: :api_version, + type: String, + desc: 'Leave blank for default (v2)' + }, + { + required: false, + name: :server, + type: String, + desc: 'Leave blank for default. https://hipchat.example.com' + } + ], + 'irker' => [ + { + required: true, + name: :recipients, + type: String, + desc: 'Recipients/channels separated by whitespaces' + }, + { + required: false, + name: :default_irc_uri, + type: String, + desc: 'Default: irc://irc.network.net:6697' + }, + { + required: false, + name: :server_host, + type: String, + desc: 'Server host. Default localhost' + }, + { + required: false, + name: :server_port, + type: Integer, + desc: 'Server port. Default 6659' + }, + { + required: false, + name: :colorize_messages, + type: Boolean, + desc: 'Colorize messages' + } + ], + 'jira' => [ + { + required: true, + name: :url, + type: String, + desc: 'The URL to the JIRA project which is being linked to this GitLab project, e.g., https://jira.example.com' + }, + { + required: true, + name: :project_key, + type: String, + desc: 'The short identifier for your JIRA project, all uppercase, e.g., PROJ' + }, + { + required: false, + name: :username, + type: String, + desc: 'The username of the user created to be used with GitLab/JIRA' + }, + { + required: false, + name: :password, + type: String, + desc: 'The password of the user created to be used with GitLab/JIRA' + }, + { + required: false, + name: :jira_issue_transition_id, + type: Integer, + desc: 'The ID of a transition that moves issues to a closed state. You can find this number under the JIRA workflow administration (**Administration > Issues > Workflows**) by selecting **View** under **Operations** of the desired workflow of your project. The ID of each state can be found inside the parenthesis of each transition name under the **Transitions (id)** column ([see screenshot][trans]). By default, this ID is set to `2`' + } + ], + + 'kubernetes' => [ + { + required: true, + name: :namespace, + type: String, + desc: 'The Kubernetes namespace to use' + }, + { + required: true, + name: :api_url, + type: String, + desc: 'The URL to the Kubernetes cluster API, e.g., https://kubernetes.example.com' + }, + { + required: true, + name: :token, + type: String, + desc: 'The service token to authenticate against the Kubernetes cluster with' + }, + { + required: false, + name: :ca_pem, + type: String, + desc: 'A custom certificate authority bundle to verify the Kubernetes cluster with (PEM format)' + }, + ], + 'mattermost-slash-commands' => [ + { + required: true, + name: :token, + type: String, + desc: 'The Mattermost token' + } + ], + 'slack-slash-commands' => [ + { + required: true, + name: :token, + type: String, + desc: 'The Slack token' + } + ], + 'pipelines-email' => [ + { + required: true, + name: :recipients, + type: String, + desc: 'Comma-separated list of recipient email addresses' + }, + { + required: false, + name: :notify_only_broken_builds, + type: Boolean, + desc: 'Notify only broken builds' + } + ], + 'pivotaltracker' => [ + { + required: true, + name: :token, + type: String, + desc: 'The Pivotaltracker token' + }, + { + required: false, + name: :restrict_to_branch, + type: String, + desc: 'Comma-separated list of branches which will be automatically inspected. Leave blank to include all branches.' + } + ], + 'pushover' => [ + { + required: true, + name: :api_key, + type: String, + desc: 'The application key' + }, + { + required: true, + name: :user_key, + type: String, + desc: 'The user key' + }, + { + required: true, + name: :priority, + type: String, + desc: 'The priority' + }, + { + required: true, + name: :device, + type: String, + desc: 'Leave blank for all active devices' + }, + { + required: true, + name: :sound, + type: String, + desc: 'The sound of the notification' + } + ], + 'redmine' => [ + { + required: true, + name: :new_issue_url, + type: String, + desc: 'The new issue URL' + }, + { + required: true, + name: :project_url, + type: String, + desc: 'The project URL' + }, + { + required: true, + name: :issues_url, + type: String, + desc: 'The issues URL' + }, + { + required: false, + name: :description, + type: String, + desc: 'The description of the tracker' + } + ], + 'slack' => [ + { + required: true, + name: :webhook, + type: String, + desc: 'The Slack webhook. e.g. https://hooks.slack.com/services/...' + }, + { + required: false, + name: :new_issue_url, + type: String, + desc: 'The user name' + }, + { + required: false, + name: :channel, + type: String, + desc: 'The channel name' + } + ], + 'mattermost' => [ + { + required: true, + name: :webhook, + type: String, + desc: 'The Mattermost webhook. e.g. http://mattermost_host/hooks/...' + } + ], + 'teamcity' => [ + { + required: true, + name: :teamcity_url, + type: String, + desc: 'TeamCity root URL like https://teamcity.example.com' + }, + { + required: true, + name: :build_type, + type: String, + desc: 'Build configuration ID' + }, + { + required: true, + name: :username, + type: String, + desc: 'A user with permissions to trigger a manual build' + }, + { + required: true, + name: :password, + type: String, + desc: 'The password of the user' + } + ] + } + + service_classes = [ + AsanaService, + AssemblaService, + BambooService, + BugzillaService, + BuildkiteService, + BuildsEmailService, + CampfireService, + CustomIssueTrackerService, + DroneCiService, + EmailsOnPushService, + ExternalWikiService, + FlowdockService, + GemnasiumService, + HipchatService, + IrkerService, + JiraService, + KubernetesService, + MattermostSlashCommandsService, + SlackSlashCommandsService, + PipelinesEmailService, + PivotaltrackerService, + PushoverService, + RedmineService, + SlackService, + MattermostService, + TeamcityService, + ].freeze + + trigger_services = { + 'mattermost-slash-commands' => [ + { + name: :token, + type: String, + desc: 'The Mattermost token' + } + ], + 'slack-slash-commands' => [ + { + name: :token, + type: String, + desc: 'The Slack token' + } + ] + }.freeze resource :projects do - # Set <service_slug> service for project - # - # Example Request: - # - # PUT /projects/:id/services/gitlab-ci - # - put ':id/services/:service_slug' do - if project_service - validators = project_service.class.validators.select do |s| - s.class == ActiveRecord::Validations::PresenceValidator && - s.attributes != [:project_id] + before { authenticate! } + before { authorize_admin_project } + + helpers do + def service_attributes(service) + service.fields.inject([]) do |arr, hash| + arr << hash[:name].to_sym end + end + end - required_attributes! validators.map(&:attributes).flatten.uniq - attrs = attributes_for_keys service_attributes + services.each do |service_slug, settings| + desc "Set #{service_slug} service for project" + params do + service_classes.each do |service| + event_names = service.try(:event_names) || [] + event_names.each do |event_name| + services[service.to_param.tr("_", "-")] << { + required: false, + name: event_name.to_sym, + type: String, + desc: ServicesHelper.service_event_description(event_name) + } + end + end + services.freeze - if project_service.update_attributes(attrs.merge(active: true)) - true + settings.each do |setting| + if setting[:required] + requires setting[:name], type: setting[:type], desc: setting[:desc] + else + optional setting[:name], type: setting[:type], desc: setting[:desc] + end + end + end + put ":id/services/#{service_slug}" do + service = user_project.find_or_initialize_service(service_slug.underscore) + service_params = declared_params(include_missing: false).merge(active: true) + + if service.update_attributes(service_params) + present service, with: Entities::ProjectService, include_passwords: current_user.is_admin? else - not_found! + render_api_error!('400 Bad Request', 400) end end end - # Delete <service_slug> service for project - # - # Example Request: - # - # DELETE /project/:id/services/gitlab-ci - # - delete ':id/services/:service_slug' do - if project_service - attrs = service_attributes.inject({}) do |hash, key| - hash.merge!(key => nil) + desc "Delete a service for project" + params do + requires :service_slug, type: String, values: services.keys, desc: 'The name of the service' + end + delete ":id/services/:service_slug" do + service = user_project.find_or_initialize_service(params[:service_slug].underscore) + + attrs = service_attributes(service).inject({}) do |hash, key| + hash.merge!(key => nil) + end + + if service.update_attributes(attrs.merge(active: false)) + true + else + render_api_error!('400 Bad Request', 400) + end + end + + desc 'Get the service settings for project' do + success Entities::ProjectService + end + params do + requires :service_slug, type: String, values: services.keys, desc: 'The name of the service' + end + get ":id/services/:service_slug" do + service = user_project.find_or_initialize_service(params[:service_slug].underscore) + present service, with: Entities::ProjectService, include_passwords: current_user.is_admin? + end + end + + trigger_services.each do |service_slug, settings| + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects do + desc "Trigger a slash command for #{service_slug}" do + detail 'Added in GitLab 8.13' + end + params do + settings.each do |setting| + requires setting[:name], type: setting[:type], desc: setting[:desc] end + end + post ":id/services/#{service_slug.underscore}/trigger" do + project = find_project(params[:id]) + + # This is not accurate, but done to prevent leakage of the project names + not_found!('Service') unless project + + service = project.find_or_initialize_service(service_slug.underscore) - if project_service.update_attributes(attrs.merge(active: false)) - true + result = service.try(:active?) && service.try(:trigger, params) + + if result + status result[:status] || 200 + present result else - not_found! + not_found!('Service') end end end - - # Get <service_slug> service settings for project - # - # Example Request: - # - # GET /project/:id/services/gitlab-ci - # - get ':id/services/:service_slug' do - present project_service, with: Entities::ProjectService, include_passwords: current_user.is_admin? - end end end end diff --git a/lib/api/session.rb b/lib/api/session.rb index 55ec66a6d67..002ffd1d154 100644 --- a/lib/api/session.rb +++ b/lib/api/session.rb @@ -1,21 +1,20 @@ module API - # Users API class Session < Grape::API - # Login to get token - # - # Parameters: - # login (*required) - user login - # email (*required) - user email - # password (required) - user password - # - # Example Request: - # POST /session + desc 'Login to get token' do + success Entities::UserWithPrivateToken + end + params do + optional :login, type: String, desc: 'The username' + optional :email, type: String, desc: 'The email of the user' + requires :password, type: String, desc: 'The password of the user' + at_least_one_of :login, :email + end post "/session" do user = Gitlab::Auth.find_with_user_password(params[:email] || params[:login], params[:password]) return unauthorized! unless user return render_api_error!('401 Unauthorized. You have 2FA enabled. Please use a personal access token to access the API', 401) if user.two_factor_enabled? - present user, with: Entities::UserLogin + present user, with: Entities::UserWithPrivateToken end end end diff --git a/lib/api/settings.rb b/lib/api/settings.rb index c885fcd7ea3..c5eff16a5de 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -9,23 +9,121 @@ module API end end - # Get current applicaiton settings - # - # Example Request: - # GET /application/settings + desc 'Get the current application settings' do + success Entities::ApplicationSetting + end get "application/settings" do present current_settings, with: Entities::ApplicationSetting end - # Modify applicaiton settings - # - # Example Request: - # PUT /application/settings + desc 'Modify application settings' do + success Entities::ApplicationSetting + end + params do + optional :default_branch_protection, type: Integer, values: [0, 1, 2], desc: 'Determine if developers can push to master' + optional :default_project_visibility, type: Integer, values: Gitlab::VisibilityLevel.values, desc: 'The default project visibility' + optional :default_snippet_visibility, type: Integer, values: Gitlab::VisibilityLevel.values, desc: 'The default snippet visibility' + optional :default_group_visibility, type: Integer, values: Gitlab::VisibilityLevel.values, desc: 'The default group visibility' + optional :restricted_visibility_levels, type: Array[String], desc: 'Selected levels cannot be used by non-admin users for projects or snippets. If the public level is restricted, user profiles are only visible to logged in users.' + optional :import_sources, type: Array[String], values: %w[github bitbucket gitlab google_code fogbugz git gitlab_project], + desc: 'Enabled sources for code import during project creation. OmniAuth must be configured for GitHub, Bitbucket, and GitLab.com' + optional :disabled_oauth_sign_in_sources, type: Array[String], desc: 'Disable certain OAuth sign-in sources' + optional :enabled_git_access_protocol, type: String, values: %w[ssh http nil], desc: 'Allow only the selected protocols to be used for Git access.' + optional :gravatar_enabled, type: Boolean, desc: 'Flag indicating if the Gravatar service is enabled' + optional :default_projects_limit, type: Integer, desc: 'The maximum number of personal projects' + optional :max_attachment_size, type: Integer, desc: 'Maximum attachment size in MB' + optional :session_expire_delay, type: Integer, desc: 'Session duration in minutes. GitLab restart is required to apply changes.' + optional :user_oauth_applications, type: Boolean, desc: 'Allow users to register any application to use GitLab as an OAuth provider' + optional :user_default_external, type: Boolean, desc: 'Newly registered users will by default be external' + optional :signup_enabled, type: Boolean, desc: 'Flag indicating if sign up is enabled' + optional :send_user_confirmation_email, type: Boolean, desc: 'Send confirmation email on sign-up' + optional :domain_whitelist, type: String, desc: 'ONLY users with e-mail addresses that match these domain(s) will be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com' + optional :domain_blacklist_enabled, type: Boolean, desc: 'Enable domain blacklist for sign ups' + given domain_blacklist_enabled: ->(val) { val } do + requires :domain_blacklist, type: String, desc: 'Users with e-mail addresses that match these domain(s) will NOT be able to sign-up. Wildcards allowed. Use separate lines for multiple entries. Ex: domain.com, *.domain.com' + end + optional :after_sign_up_text, type: String, desc: 'Text shown after sign up' + optional :signin_enabled, type: Boolean, desc: 'Flag indicating if sign in is enabled' + optional :require_two_factor_authentication, type: Boolean, desc: 'Require all users to setup Two-factor authentication' + given require_two_factor_authentication: ->(val) { val } do + requires :two_factor_grace_period, type: Integer, desc: 'Amount of time (in hours) that users are allowed to skip forced configuration of two-factor authentication' + end + optional :home_page_url, type: String, desc: 'We will redirect non-logged in users to this page' + optional :after_sign_out_path, type: String, desc: 'We will redirect users to this page after they sign out' + optional :sign_in_text, type: String, desc: 'The sign in text of the GitLab application' + optional :help_page_text, type: String, desc: 'Custom text displayed on the help page' + optional :shared_runners_enabled, type: Boolean, desc: 'Enable shared runners for new projects' + given shared_runners_enabled: ->(val) { val } do + requires :shared_runners_text, type: String, desc: 'Shared runners text ' + end + optional :max_artifacts_size, type: Integer, desc: "Set the maximum file size each build's artifacts can have" + optional :container_registry_token_expire_delay, type: Integer, desc: 'Authorization token duration (minutes)' + optional :metrics_enabled, type: Boolean, desc: 'Enable the InfluxDB metrics' + given metrics_enabled: ->(val) { val } do + requires :metrics_host, type: String, desc: 'The InfluxDB host' + requires :metrics_port, type: Integer, desc: 'The UDP port to use for connecting to InfluxDB' + requires :metrics_pool_size, type: Integer, desc: 'The amount of InfluxDB connections to open' + requires :metrics_timeout, type: Integer, desc: 'The amount of seconds after which an InfluxDB connection will time out' + requires :metrics_method_call_threshold, type: Integer, desc: 'A method call is only tracked when it takes longer to complete than the given amount of milliseconds.' + requires :metrics_sample_interval, type: Integer, desc: 'The sampling interval in seconds' + requires :metrics_packet_size, type: Integer, desc: 'The amount of points to store in a single UDP packet' + end + optional :sidekiq_throttling_enabled, type: Boolean, desc: 'Enable Sidekiq Job Throttling' + given sidekiq_throttling_enabled: ->(val) { val } do + requires :sidekiq_throttling_queus, type: Array[String], desc: 'Choose which queues you wish to throttle' + requires :sidekiq_throttling_factor, type: Float, desc: 'The factor by which the queues should be throttled. A value between 0.0 and 1.0, exclusive.' + end + optional :recaptcha_enabled, type: Boolean, desc: 'Helps prevent bots from creating accounts' + given recaptcha_enabled: ->(val) { val } do + requires :recaptcha_site_key, type: String, desc: 'Generate site key at http://www.google.com/recaptcha' + requires :recaptcha_private_key, type: String, desc: 'Generate private key at http://www.google.com/recaptcha' + end + optional :akismet_enabled, type: Boolean, desc: 'Helps prevent bots from creating issues' + given akismet_enabled: ->(val) { val } do + requires :akismet_api_key, type: String, desc: 'Generate API key at http://www.akismet.com' + end + optional :admin_notification_email, type: String, desc: 'Abuse reports will be sent to this address if it is set. Abuse reports are always available in the admin area.' + optional :sentry_enabled, type: Boolean, desc: 'Sentry is an error reporting and logging tool which is currently not shipped with GitLab, get it here: https://getsentry.com' + given sentry_enabled: ->(val) { val } do + requires :sentry_dsn, type: String, desc: 'Sentry Data Source Name' + end + optional :repository_storage, type: String, desc: 'Storage paths for new projects' + optional :repository_checks_enabled, type: Boolean, desc: "GitLab will periodically run 'git fsck' in all project and wiki repositories to look for silent disk corruption issues." + optional :koding_enabled, type: Boolean, desc: 'Enable Koding' + given koding_enabled: ->(val) { val } do + requires :koding_url, type: String, desc: 'The Koding team URL' + end + optional :plantuml_enabled, type: Boolean, desc: 'Enable PlantUML' + given plantuml_enabled: ->(val) { val } do + requires :plantuml_url, type: String, desc: 'The PlantUML server URL' + end + optional :version_check_enabled, type: Boolean, desc: 'Let GitLab inform you when an update is available.' + optional :email_author_in_body, type: Boolean, desc: 'Some email servers do not support overriding the email sender name. Enable this option to include the name of the author of the issue, merge request or comment in the email body instead.' + optional :html_emails_enabled, type: Boolean, desc: 'By default GitLab sends emails in HTML and plain text formats so mail clients can choose what format to use. Disable this option if you only want to send emails in plain text format.' + optional :housekeeping_enabled, type: Boolean, desc: 'Enable automatic repository housekeeping (git repack, git gc)' + given housekeeping_enabled: ->(val) { val } do + requires :housekeeping_bitmaps_enabled, type: Boolean, desc: "Creating pack file bitmaps makes housekeeping take a little longer but bitmaps should accelerate 'git clone' performance." + requires :housekeeping_incremental_repack_period, type: Integer, desc: "Number of Git pushes after which an incremental 'git repack' is run." + requires :housekeeping_full_repack_period, type: Integer, desc: "Number of Git pushes after which a full 'git repack' is run." + requires :housekeeping_gc_period, type: Integer, desc: "Number of Git pushes after which 'git gc' is run." + end + at_least_one_of :default_branch_protection, :default_project_visibility, :default_snippet_visibility, + :default_group_visibility, :restricted_visibility_levels, :import_sources, + :enabled_git_access_protocol, :gravatar_enabled, :default_projects_limit, + :max_attachment_size, :session_expire_delay, :disabled_oauth_sign_in_sources, + :user_oauth_applications, :user_default_external, :signup_enabled, + :send_user_confirmation_email, :domain_whitelist, :domain_blacklist_enabled, + :after_sign_up_text, :signin_enabled, :require_two_factor_authentication, + :home_page_url, :after_sign_out_path, :sign_in_text, :help_page_text, + :shared_runners_enabled, :max_artifacts_size, :container_registry_token_expire_delay, + :metrics_enabled, :sidekiq_throttling_enabled, :recaptcha_enabled, + :akismet_enabled, :admin_notification_email, :sentry_enabled, + :repository_storage, :repository_checks_enabled, :koding_enabled, :plantuml_enabled, + :version_check_enabled, :email_author_in_body, :html_emails_enabled, + :housekeeping_enabled + end put "application/settings" do - attributes = current_settings.attributes.keys - ["id"] - attrs = attributes_for_keys(attributes) - - if current_settings.update_attributes(attrs) + if current_settings.update_attributes(declared_params(include_missing: false)) present current_settings, with: Entities::ApplicationSetting else render_validation_error!(current_settings) diff --git a/lib/api/sidekiq_metrics.rb b/lib/api/sidekiq_metrics.rb index d3d6827dc54..11f2b40269a 100644 --- a/lib/api/sidekiq_metrics.rb +++ b/lib/api/sidekiq_metrics.rb @@ -39,50 +39,22 @@ module API end end - # Get Sidekiq Queue metrics - # - # Parameters: - # None - # - # Example: - # GET /sidekiq/queue_metrics - # + desc 'Get the Sidekiq queue metrics' get 'sidekiq/queue_metrics' do { queues: queue_metrics } end - # Get Sidekiq Process metrics - # - # Parameters: - # None - # - # Example: - # GET /sidekiq/process_metrics - # + desc 'Get the Sidekiq process metrics' get 'sidekiq/process_metrics' do { processes: process_metrics } end - # Get Sidekiq Job statistics - # - # Parameters: - # None - # - # Example: - # GET /sidekiq/job_stats - # + desc 'Get the Sidekiq job statistics' get 'sidekiq/job_stats' do { jobs: job_stats } end - # Get Sidekiq Compound metrics. Includes all previous metrics - # - # Parameters: - # None - # - # Example: - # GET /sidekiq/compound_metrics - # + desc 'Get the Sidekiq Compound metrics. Includes queue, process, and job statistics' get 'sidekiq/compound_metrics' do { queues: queue_metrics, processes: process_metrics, jobs: job_stats } end diff --git a/lib/api/snippets.rb b/lib/api/snippets.rb new file mode 100644 index 00000000000..e096e636806 --- /dev/null +++ b/lib/api/snippets.rb @@ -0,0 +1,137 @@ +module API + # Snippets API + class Snippets < Grape::API + include PaginationParams + + before { authenticate! } + + resource :snippets do + helpers do + def snippets_for_current_user + SnippetsFinder.new.execute(current_user, filter: :by_user, user: current_user) + end + + def public_snippets + SnippetsFinder.new.execute(current_user, filter: :public) + end + end + + desc 'Get a snippets list for authenticated user' do + detail 'This feature was introduced in GitLab 8.15.' + success Entities::PersonalSnippet + end + params do + use :pagination + end + get do + present paginate(snippets_for_current_user), with: Entities::PersonalSnippet + end + + desc 'List all public snippets current_user has access to' do + detail 'This feature was introduced in GitLab 8.15.' + success Entities::PersonalSnippet + end + params do + use :pagination + end + get 'public' do + present paginate(public_snippets), with: Entities::PersonalSnippet + end + + desc 'Get a single snippet' do + detail 'This feature was introduced in GitLab 8.15.' + success Entities::PersonalSnippet + end + params do + requires :id, type: Integer, desc: 'The ID of a snippet' + end + get ':id' do + snippet = snippets_for_current_user.find(params[:id]) + present snippet, with: Entities::PersonalSnippet + end + + desc 'Create new snippet' do + detail 'This feature was introduced in GitLab 8.15.' + success Entities::PersonalSnippet + end + params do + requires :title, type: String, desc: 'The title of a snippet' + requires :file_name, type: String, desc: 'The name of a snippet file' + requires :content, type: String, desc: 'The content of a snippet' + optional :visibility_level, type: Integer, + values: Gitlab::VisibilityLevel.values, + default: Gitlab::VisibilityLevel::INTERNAL, + desc: 'The visibility level of the snippet' + end + post do + attrs = declared_params(include_missing: false) + snippet = CreateSnippetService.new(nil, current_user, attrs).execute + + if snippet.persisted? + present snippet, with: Entities::PersonalSnippet + else + render_validation_error!(snippet) + end + end + + desc 'Update an existing snippet' do + detail 'This feature was introduced in GitLab 8.15.' + success Entities::PersonalSnippet + end + params do + requires :id, type: Integer, desc: 'The ID of a snippet' + optional :title, type: String, desc: 'The title of a snippet' + optional :file_name, type: String, desc: 'The name of a snippet file' + optional :content, type: String, desc: 'The content of a snippet' + optional :visibility_level, type: Integer, + values: Gitlab::VisibilityLevel.values, + desc: 'The visibility level of the snippet' + at_least_one_of :title, :file_name, :content, :visibility_level + end + put ':id' do + snippet = snippets_for_current_user.find_by(id: params.delete(:id)) + return not_found!('Snippet') unless snippet + authorize! :update_personal_snippet, snippet + + attrs = declared_params(include_missing: false) + + UpdateSnippetService.new(nil, current_user, snippet, attrs).execute + if snippet.persisted? + present snippet, with: Entities::PersonalSnippet + else + render_validation_error!(snippet) + end + end + + desc 'Remove snippet' do + detail 'This feature was introduced in GitLab 8.15.' + success Entities::PersonalSnippet + end + params do + requires :id, type: Integer, desc: 'The ID of a snippet' + end + delete ':id' do + snippet = snippets_for_current_user.find_by(id: params.delete(:id)) + return not_found!('Snippet') unless snippet + authorize! :destroy_personal_snippet, snippet + snippet.destroy + no_content! + end + + desc 'Get a raw snippet' do + detail 'This feature was introduced in GitLab 8.15.' + end + params do + requires :id, type: Integer, desc: 'The ID of a snippet' + end + get ":id/raw" do + snippet = snippets_for_current_user.find_by(id: params.delete(:id)) + return not_found!('Snippet') unless snippet + + env['api.format'] = :txt + content_type 'text/plain' + present snippet.content + end + end + end +end diff --git a/lib/api/subscriptions.rb b/lib/api/subscriptions.rb index c49e2a21b82..e11d7537cc9 100644 --- a/lib/api/subscriptions.rb +++ b/lib/api/subscriptions.rb @@ -3,55 +3,46 @@ module API before { authenticate! } subscribable_types = { - 'merge_request' => proc { |id| user_project.merge_requests.find(id) }, - 'merge_requests' => proc { |id| user_project.merge_requests.find(id) }, + 'merge_request' => proc { |id| find_merge_request_with_access(id, :update_merge_request) }, + 'merge_requests' => proc { |id| find_merge_request_with_access(id, :update_merge_request) }, 'issues' => proc { |id| find_project_issue(id) }, 'labels' => proc { |id| find_project_label(id) }, } + params do + requires :id, type: String, desc: 'The ID of a project' + requires :subscribable_id, type: String, desc: 'The ID of a resource' + end resource :projects do subscribable_types.each do |type, finder| type_singularized = type.singularize - type_id_str = :"#{type_singularized}_id" entity_class = Entities.const_get(type_singularized.camelcase) - # Subscribe to a resource - # - # Parameters: - # id (required) - The ID of a project - # subscribable_id (required) - The ID of a resource - # Example Request: - # POST /projects/:id/labels/:subscribable_id/subscription - # POST /projects/:id/issues/:subscribable_id/subscription - # POST /projects/:id/merge_requests/:subscribable_id/subscription - post ":id/#{type}/:#{type_id_str}/subscription" do - resource = instance_exec(params[type_id_str], &finder) + desc 'Subscribe to a resource' do + success entity_class + end + post ":id/#{type}/:subscribable_id/subscription" do + resource = instance_exec(params[:subscribable_id], &finder) - if resource.subscribed?(current_user) + if resource.subscribed?(current_user, user_project) not_modified! else - resource.subscribe(current_user) - present resource, with: entity_class, current_user: current_user + resource.subscribe(current_user, user_project) + present resource, with: entity_class, current_user: current_user, project: user_project end end - # Unsubscribe from a resource - # - # Parameters: - # id (required) - The ID of a project - # subscribable_id (required) - The ID of a resource - # Example Request: - # DELETE /projects/:id/labels/:subscribable_id/subscription - # DELETE /projects/:id/issues/:subscribable_id/subscription - # DELETE /projects/:id/merge_requests/:subscribable_id/subscription - delete ":id/#{type}/:#{type_id_str}/subscription" do - resource = instance_exec(params[type_id_str], &finder) + desc 'Unsubscribe from a resource' do + success entity_class + end + delete ":id/#{type}/:subscribable_id/subscription" do + resource = instance_exec(params[:subscribable_id], &finder) - if !resource.subscribed?(current_user) + if !resource.subscribed?(current_user, user_project) not_modified! else - resource.unsubscribe(current_user) - present resource, with: entity_class, current_user: current_user + resource.unsubscribe(current_user, user_project) + present resource, with: entity_class, current_user: current_user, project: user_project end end end diff --git a/lib/api/system_hooks.rb b/lib/api/system_hooks.rb index 794e34874f4..708ec8cfe70 100644 --- a/lib/api/system_hooks.rb +++ b/lib/api/system_hooks.rb @@ -12,6 +12,7 @@ module API end get do hooks = SystemHook.all + present hooks, with: Entities::Hook end @@ -19,15 +20,19 @@ module API success Entities::Hook end params do - requires :url, type: String, desc: 'The URL for the system hook' + requires :url, type: String, desc: "The URL to send the request to" + optional :token, type: String, desc: 'The token used to validate payloads' + optional :push_events, type: Boolean, desc: "Trigger hook on push events" + optional :tag_push_events, type: Boolean, desc: "Trigger hook on tag push events" + optional :enable_ssl_verification, type: Boolean, desc: "Do SSL verification when triggering the hook" end post do - hook = SystemHook.new declared(params).to_h + hook = SystemHook.new(declared_params(include_missing: false)) if hook.save present hook, with: Entities::Hook else - not_found! + render_validation_error!(hook) end end diff --git a/lib/api/tags.rb b/lib/api/tags.rb index bf2a199ce21..5b345db3a41 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -1,7 +1,6 @@ module API # Git Tags API class Tags < Grape::API - before { authenticate! } before { authorize! :download_code, user_project } params do @@ -40,10 +39,9 @@ module API end post ':id/repository/tags' do authorize_push_project - create_params = declared(params) result = CreateTagService.new(user_project, current_user). - execute(create_params[:tag_name], create_params[:ref], create_params[:message], create_params[:release_description]) + execute(params[:tag_name], params[:ref], params[:message], params[:release_description]) if result[:status] == :success present result[:tag], diff --git a/lib/api/templates.rb b/lib/api/templates.rb index 8a53d9c0095..e23f99256a5 100644 --- a/lib/api/templates.rb +++ b/lib/api/templates.rb @@ -8,6 +8,10 @@ module API gitlab_ci_ymls: { klass: Gitlab::Template::GitlabCiYmlTemplate, gitlab_version: 8.9 + }, + dockerfiles: { + klass: Gitlab::Template::DockerfileTemplate, + gitlab_version: 8.15 } }.freeze PROJECT_TEMPLATE_REGEX = @@ -51,7 +55,7 @@ module API end params do optional :popular, type: Boolean, desc: 'If passed, returns only popular licenses' - end + end get route do options = { featured: declared(params).popular.present? ? true : nil @@ -69,7 +73,7 @@ module API end params do requires :name, type: String, desc: 'The name of the template' - end + end get route, requirements: { name: /[\w\.-]+/ } do not_found!('License') unless Licensee::License.find(declared(params).name) @@ -78,7 +82,7 @@ module API present template, with: Entities::RepoLicense end end - + GLOBAL_TEMPLATE_TYPES.each do |template_type, properties| klass = properties[:klass] gitlab_version = properties[:gitlab_version] @@ -104,7 +108,7 @@ module API end params do requires :name, type: String, desc: 'The name of the template' - end + end get route do new_template = klass.find(declared(params).name) diff --git a/lib/api/time_tracking_endpoints.rb b/lib/api/time_tracking_endpoints.rb new file mode 100644 index 00000000000..85b5f7d98b8 --- /dev/null +++ b/lib/api/time_tracking_endpoints.rb @@ -0,0 +1,114 @@ +module API + module TimeTrackingEndpoints + extend ActiveSupport::Concern + + included do + helpers do + def issuable_name + declared_params.has_key?(:issue_id) ? 'issue' : 'merge_request' + end + + def issuable_key + "#{issuable_name}_id".to_sym + end + + def update_issuable_key + "update_#{issuable_name}".to_sym + end + + def read_issuable_key + "read_#{issuable_name}".to_sym + end + + def load_issuable + @issuable ||= begin + case issuable_name + when 'issue' + find_project_issue(params.delete(issuable_key)) + when 'merge_request' + find_project_merge_request(params.delete(issuable_key)) + end + end + end + + def update_issuable(attrs) + custom_params = declared_params(include_missing: false) + custom_params.merge!(attrs) + + issuable = update_service.new(user_project, current_user, custom_params).execute(load_issuable) + if issuable.valid? + present issuable, with: Entities::IssuableTimeStats + else + render_validation_error!(issuable) + end + end + + def update_service + issuable_name == 'issue' ? ::Issues::UpdateService : ::MergeRequests::UpdateService + end + end + + issuable_name = name.end_with?('Issues') ? 'issue' : 'merge_request' + issuable_collection_name = issuable_name.pluralize + issuable_key = "#{issuable_name}_id".to_sym + + desc "Set a time estimate for a project #{issuable_name}" + params do + requires issuable_key, type: Integer, desc: "The ID of a project #{issuable_name}" + requires :duration, type: String, desc: 'The duration to be parsed' + end + post ":id/#{issuable_collection_name}/:#{issuable_key}/time_estimate" do + authorize! update_issuable_key, load_issuable + + status :ok + update_issuable(time_estimate: Gitlab::TimeTrackingFormatter.parse(params.delete(:duration))) + end + + desc "Reset the time estimate for a project #{issuable_name}" + params do + requires issuable_key, type: Integer, desc: "The ID of a project #{issuable_name}" + end + post ":id/#{issuable_collection_name}/:#{issuable_key}/reset_time_estimate" do + authorize! update_issuable_key, load_issuable + + status :ok + update_issuable(time_estimate: 0) + end + + desc "Add spent time for a project #{issuable_name}" + params do + requires issuable_key, type: Integer, desc: "The ID of a project #{issuable_name}" + requires :duration, type: String, desc: 'The duration to be parsed' + end + post ":id/#{issuable_collection_name}/:#{issuable_key}/add_spent_time" do + authorize! update_issuable_key, load_issuable + + update_issuable(spend_time: { + duration: Gitlab::TimeTrackingFormatter.parse(params.delete(:duration)), + user: current_user + }) + end + + desc "Reset spent time for a project #{issuable_name}" + params do + requires issuable_key, type: Integer, desc: "The ID of a project #{issuable_name}" + end + post ":id/#{issuable_collection_name}/:#{issuable_key}/reset_spent_time" do + authorize! update_issuable_key, load_issuable + + status :ok + update_issuable(spend_time: { duration: :reset, user: current_user }) + end + + desc "Show time stats for a project #{issuable_name}" + params do + requires issuable_key, type: Integer, desc: "The ID of a project #{issuable_name}" + end + get ":id/#{issuable_collection_name}/:#{issuable_key}/time_stats" do + authorize! read_issuable_key, load_issuable + + present load_issuable, with: Entities::IssuableTimeStats + end + end + end +end diff --git a/lib/api/todos.rb b/lib/api/todos.rb index 832b04a3bb1..9bd077263a7 100644 --- a/lib/api/todos.rb +++ b/lib/api/todos.rb @@ -1,10 +1,11 @@ module API - # Todos API class Todos < Grape::API + include PaginationParams + before { authenticate! } ISSUABLE_TYPES = { - 'merge_requests' => ->(id) { user_project.merge_requests.find(id) }, + 'merge_requests' => ->(id) { find_merge_request_with_access(id) }, 'issues' => ->(id) { find_project_issue(id) } } @@ -44,10 +45,11 @@ module API desc 'Get a todo list' do success Entities::Todo end + params do + use :pagination + end get do - todos = find_todos - - present paginate(todos), with: Entities::Todo, current_user: current_user + present paginate(find_todos), with: Entities::Todo, current_user: current_user end desc 'Mark a todo as done' do diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb index d1d07394e92..87a717ba751 100644 --- a/lib/api/triggers.rb +++ b/lib/api/triggers.rb @@ -1,20 +1,21 @@ module API - # Triggers API class Triggers < Grape::API + include PaginationParams + + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do - # Trigger a GitLab project build - # - # Parameters: - # id (required) - The ID of a CI project - # ref (required) - The name of project's branch or tag - # token (required) - The uniq token of trigger - # variables (optional) - The list of variables to be injected into build - # Example Request: - # POST /projects/:id/trigger/builds - post ":id/trigger/builds" do - required_attributes! [:ref, :token] - - project = Project.find_with_namespace(params[:id]) || Project.find_by(id: params[:id]) + desc 'Trigger a GitLab project build' do + success Entities::TriggerRequest + end + params do + requires :ref, type: String, desc: 'The commit sha or name of a branch or tag' + requires :token, type: String, desc: 'The unique token of trigger' + optional :variables, type: Hash, desc: 'The list of variables to be injected into build' + end + post ":id/(ref/:ref/)trigger/builds" do + project = find_project(params[:id]) trigger = Ci::Trigger.find_by_token(params[:token].to_s) not_found! unless project && trigger unauthorized! unless trigger.project == project @@ -22,10 +23,6 @@ module API # validate variables variables = params[:variables] if variables - unless variables.is_a?(Hash) - render_api_error!('variables needs to be a hash', 400) - end - unless variables.all? { |key, value| key.is_a?(String) && value.is_a?(String) } render_api_error!('variables needs to be a map of key-valued strings', 400) end @@ -44,31 +41,27 @@ module API end end - # Get triggers list - # - # Parameters: - # id (required) - The ID of a project - # page (optional) - The page number for pagination - # per_page (optional) - The value of items per page to show - # Example Request: - # GET /projects/:id/triggers + desc 'Get triggers list' do + success Entities::Trigger + end + params do + use :pagination + end get ':id/triggers' do authenticate! authorize! :admin_build, user_project triggers = user_project.triggers.includes(:trigger_requests) - triggers = paginate(triggers) - present triggers, with: Entities::Trigger + present paginate(triggers), with: Entities::Trigger end - # Get specific trigger of a project - # - # Parameters: - # id (required) - The ID of a project - # token (required) - The `token` of a trigger - # Example Request: - # GET /projects/:id/triggers/:token + desc 'Get specific trigger of a project' do + success Entities::Trigger + end + params do + requires :token, type: String, desc: 'The unique token of trigger' + end get ':id/triggers/:token' do authenticate! authorize! :admin_build, user_project @@ -79,12 +72,9 @@ module API present trigger, with: Entities::Trigger end - # Create trigger - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # POST /projects/:id/triggers + desc 'Create a trigger' do + success Entities::Trigger + end post ':id/triggers' do authenticate! authorize! :admin_build, user_project @@ -94,13 +84,12 @@ module API present trigger, with: Entities::Trigger end - # Delete trigger - # - # Parameters: - # id (required) - The ID of a project - # token (required) - The `token` of a trigger - # Example Request: - # DELETE /projects/:id/triggers/:token + desc 'Delete a trigger' do + success Entities::Trigger + end + params do + requires :token, type: String, desc: 'The unique token of trigger' + end delete ':id/triggers/:token' do authenticate! authorize! :admin_build, user_project diff --git a/lib/api/users.rb b/lib/api/users.rb index c28e07a76b7..11a7368b4c0 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -1,94 +1,109 @@ module API - # Users API class Users < Grape::API - before { authenticate! } + include PaginationParams + + before do + allow_access_with_scope :read_user if request.get? + authenticate! + end resource :users, requirements: { uid: /[0-9]*/, id: /[0-9]*/ } do - # Get a users list - # - # Example Request: - # GET /users - # GET /users?search=Admin - # GET /users?username=root + helpers do + params :optional_attributes do + optional :skype, type: String, desc: 'The Skype username' + optional :linkedin, type: String, desc: 'The LinkedIn username' + optional :twitter, type: String, desc: 'The Twitter username' + optional :website_url, type: String, desc: 'The website of the user' + optional :organization, type: String, desc: 'The organization of the user' + optional :projects_limit, type: Integer, desc: 'The number of projects a user can create' + optional :extern_uid, type: String, desc: 'The external authentication provider UID' + optional :provider, type: String, desc: 'The external provider' + optional :bio, type: String, desc: 'The biography of the user' + optional :location, type: String, desc: 'The location of the user' + optional :admin, type: Boolean, desc: 'Flag indicating the user is an administrator' + optional :can_create_group, type: Boolean, desc: 'Flag indicating the user can create groups' + optional :confirm, type: Boolean, desc: 'Flag indicating the account needs to be confirmed' + optional :external, type: Boolean, desc: 'Flag indicating the user is an external user' + all_or_none_of :extern_uid, :provider + end + end + + desc 'Get the list of users' do + success Entities::UserBasic + end + params do + optional :username, type: String, desc: 'Get a single user with a specific username' + optional :search, type: String, desc: 'Search for a username' + optional :active, type: Boolean, default: false, desc: 'Filters only active users' + optional :external, type: Boolean, default: false, desc: 'Filters only external users' + optional :blocked, type: Boolean, default: false, desc: 'Filters only blocked users' + use :pagination + end get do unless can?(current_user, :read_users_list, nil) render_api_error!("Not authorized.", 403) end if params[:username].present? - @users = User.where(username: params[:username]) + users = User.where(username: params[:username]) else - @users = User.all - @users = @users.active if params[:active].present? - @users = @users.search(params[:search]) if params[:search].present? - @users = paginate @users + users = User.all + users = users.active if params[:active] + users = users.search(params[:search]) if params[:search].present? + users = users.blocked if params[:blocked] + users = users.external if params[:external] && current_user.is_admin? end - if current_user.is_admin? - present @users, with: Entities::UserFull - else - present @users, with: Entities::UserBasic - end + entity = current_user.is_admin? ? Entities::UserPublic : Entities::UserBasic + present paginate(users), with: entity end - # Get a single user - # - # Parameters: - # id (required) - The ID of a user - # Example Request: - # GET /users/:id + desc 'Get a single user' do + success Entities::UserBasic + end + params do + requires :id, type: Integer, desc: 'The ID of the user' + end get ":id" do - @user = User.find(params[:id]) + user = User.find_by(id: params[:id]) + not_found!('User') unless user if current_user && current_user.is_admin? - present @user, with: Entities::UserFull - elsif can?(current_user, :read_user, @user) - present @user, with: Entities::User + present user, with: Entities::UserPublic + elsif can?(current_user, :read_user, user) + present user, with: Entities::User else render_api_error!("User not found.", 404) end end - # Create user. Available only for admin - # - # Parameters: - # email (required) - Email - # password (required) - Password - # name (required) - Name - # username (required) - Name - # skype - Skype ID - # linkedin - Linkedin - # twitter - Twitter account - # website_url - Website url - # organization - Organization - # projects_limit - Number of projects user can create - # extern_uid - External authentication provider UID - # provider - External provider - # bio - Bio - # location - Location of the user - # admin - User is admin - true or false (default) - # can_create_group - User can create groups - true or false - # confirm - Require user confirmation - true (default) or false - # external - Flags the user as external - true or false(default) - # Example Request: - # POST /users + desc 'Create a user. Available only for admins.' do + success Entities::UserPublic + end + params do + requires :email, type: String, desc: 'The email of the user' + requires :password, type: String, desc: 'The password of the new user' + requires :name, type: String, desc: 'The name of the user' + requires :username, type: String, desc: 'The username of the user' + use :optional_attributes + end post do authenticated_as_admin! - required_attributes! [:email, :password, :name, :username] - attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :projects_limit, :username, :bio, :location, :can_create_group, :admin, :confirm, :external, :organization] - admin = attrs.delete(:admin) - confirm = !(attrs.delete(:confirm) =~ /(false|f|no|0)$/i) - user = User.build_user(attrs) - user.admin = admin unless admin.nil? + + # Filter out params which are used later + user_params = declared_params(include_missing: false) + identity_attrs = user_params.slice(:provider, :extern_uid) + confirm = user_params.delete(:confirm) + + user = User.new(user_params.except(:extern_uid, :provider)) user.skip_confirmation! unless confirm - identity_attrs = attributes_for_keys [:provider, :extern_uid] if identity_attrs.any? user.identities.build(identity_attrs) end if user.save - present user, with: Entities::UserFull + present user, with: Entities::UserPublic else conflict!('Email has already been taken') if User. where(email: user.email). @@ -102,46 +117,41 @@ module API end end - # Update user. Available only for admin - # - # Parameters: - # email - Email - # name - Name - # password - Password - # skype - Skype ID - # linkedin - Linkedin - # twitter - Twitter account - # website_url - Website url - # organization - Organization - # projects_limit - Limit projects each user can create - # bio - Bio - # location - Location of the user - # admin - User is admin - true or false (default) - # can_create_group - User can create groups - true or false - # external - Flags the user as external - true or false(default) - # Example Request: - # PUT /users/:id + desc 'Update a user. Available only for admins.' do + success Entities::UserPublic + end + params do + requires :id, type: Integer, desc: 'The ID of the user' + optional :email, type: String, desc: 'The email of the user' + optional :password, type: String, desc: 'The password of the new user' + optional :name, type: String, desc: 'The name of the user' + optional :username, type: String, desc: 'The username of the user' + use :optional_attributes + at_least_one_of :email, :password, :name, :username, :skype, :linkedin, + :twitter, :website_url, :organization, :projects_limit, + :extern_uid, :provider, :bio, :location, :admin, + :can_create_group, :confirm, :external + end put ":id" do authenticated_as_admin! - attrs = attributes_for_keys [:email, :name, :password, :skype, :linkedin, :twitter, :website_url, :projects_limit, :username, :bio, :location, :can_create_group, :admin, :external, :organization] - user = User.find(params[:id]) + user = User.find_by(id: params.delete(:id)) not_found!('User') unless user - admin = attrs.delete(:admin) - user.admin = admin unless admin.nil? - - conflict!('Email has already been taken') if attrs[:email] && - User.where(email: attrs[:email]). + conflict!('Email has already been taken') if params[:email] && + User.where(email: params[:email]). where.not(id: user.id).count > 0 - conflict!('Username has already been taken') if attrs[:username] && - User.where(username: attrs[:username]). + conflict!('Username has already been taken') if params[:username] && + User.where(username: params[:username]). where.not(id: user.id).count > 0 - identity_attrs = attributes_for_keys [:provider, :extern_uid] + user_params = declared_params(include_missing: false) + identity_attrs = user_params.slice(:provider, :extern_uid) + if identity_attrs.any? identity = user.identities.find_by(provider: identity_attrs[:provider]) + if identity identity.update_attributes(identity_attrs) else @@ -150,28 +160,29 @@ module API end end - if user.update_attributes(attrs) - present user, with: Entities::UserFull + if user.update_attributes(user_params.except(:extern_uid, :provider)) + present user, with: Entities::UserPublic else render_validation_error!(user) end end - # Add ssh key to a specified user. Only available to admin users. - # - # Parameters: - # id (required) - The ID of a user - # key (required) - New SSH Key - # title (required) - New SSH Key's title - # Example Request: - # POST /users/:id/keys + desc 'Add an SSH key to a specified user. Available only for admins.' do + success Entities::SSHKey + end + params do + requires :id, type: Integer, desc: 'The ID of the user' + requires :key, type: String, desc: 'The new SSH key' + requires :title, type: String, desc: 'The title of the new SSH key' + end post ":id/keys" do authenticated_as_admin! - required_attributes! [:title, :key] - user = User.find(params[:id]) - attrs = attributes_for_keys [:title, :key] - key = user.keys.new attrs + user = User.find_by(id: params.delete(:id)) + not_found!('User') unless user + + key = user.keys.new(declared_params(include_missing: false)) + if key.save present key, with: Entities::SSHKey else @@ -179,55 +190,55 @@ module API end end - # Get ssh keys of a specified user. Only available to admin users. - # - # Parameters: - # uid (required) - The ID of a user - # Example Request: - # GET /users/:uid/keys - get ':uid/keys' do + desc 'Get the SSH keys of a specified user. Available only for admins.' do + success Entities::SSHKey + end + params do + requires :id, type: Integer, desc: 'The ID of the user' + end + get ':id/keys' do authenticated_as_admin! - user = User.find_by(id: params[:uid]) + + user = User.find_by(id: params[:id]) not_found!('User') unless user present user.keys, with: Entities::SSHKey end - # Delete existing ssh key of a specified user. Only available to admin - # users. - # - # Parameters: - # uid (required) - The ID of a user - # id (required) - SSH Key ID - # Example Request: - # DELETE /users/:uid/keys/:id - delete ':uid/keys/:id' do + desc 'Delete an existing SSH key from a specified user. Available only for admins.' do + success Entities::SSHKey + end + params do + requires :id, type: Integer, desc: 'The ID of the user' + requires :key_id, type: Integer, desc: 'The ID of the SSH key' + end + delete ':id/keys/:key_id' do authenticated_as_admin! - user = User.find_by(id: params[:uid]) + + user = User.find_by(id: params[:id]) not_found!('User') unless user - begin - key = user.keys.find params[:id] - key.destroy - rescue ActiveRecord::RecordNotFound - not_found!('Key') - end + key = user.keys.find_by(id: params[:key_id]) + not_found!('Key') unless key + + present key.destroy, with: Entities::SSHKey end - # Add email to a specified user. Only available to admin users. - # - # Parameters: - # id (required) - The ID of a user - # email (required) - Email address - # Example Request: - # POST /users/:id/emails + desc 'Add an email address to a specified user. Available only for admins.' do + success Entities::Email + end + params do + requires :id, type: Integer, desc: 'The ID of the user' + requires :email, type: String, desc: 'The email of the user' + end post ":id/emails" do authenticated_as_admin! - required_attributes! [:email] - user = User.find(params[:id]) - attrs = attributes_for_keys [:email] - email = user.emails.new attrs + user = User.find_by(id: params.delete(:id)) + not_found!('User') unless user + + email = user.emails.new(declared_params(include_missing: false)) + if email.save NotificationService.new.new_email(email) present email, with: Entities::Email @@ -236,101 +247,95 @@ module API end end - # Get emails of a specified user. Only available to admin users. - # - # Parameters: - # uid (required) - The ID of a user - # Example Request: - # GET /users/:uid/emails - get ':uid/emails' do + desc 'Get the emails addresses of a specified user. Available only for admins.' do + success Entities::Email + end + params do + requires :id, type: Integer, desc: 'The ID of the user' + end + get ':id/emails' do authenticated_as_admin! - user = User.find_by(id: params[:uid]) + user = User.find_by(id: params[:id]) not_found!('User') unless user present user.emails, with: Entities::Email end - # Delete existing email of a specified user. Only available to admin - # users. - # - # Parameters: - # uid (required) - The ID of a user - # id (required) - Email ID - # Example Request: - # DELETE /users/:uid/emails/:id - delete ':uid/emails/:id' do + desc 'Delete an email address of a specified user. Available only for admins.' do + success Entities::Email + end + params do + requires :id, type: Integer, desc: 'The ID of the user' + requires :email_id, type: Integer, desc: 'The ID of the email' + end + delete ':id/emails/:email_id' do authenticated_as_admin! - user = User.find_by(id: params[:uid]) + user = User.find_by(id: params[:id]) not_found!('User') unless user - begin - email = user.emails.find params[:id] - email.destroy + email = user.emails.find_by(id: params[:email_id]) + not_found!('Email') unless email - user.update_secondary_emails! - rescue ActiveRecord::RecordNotFound - not_found!('Email') - end + email.destroy + user.update_secondary_emails! end - # Delete user. Available only for admin - # - # Example Request: - # DELETE /users/:id + desc 'Delete a user. Available only for admins.' do + success Entities::Email + end + params do + requires :id, type: Integer, desc: 'The ID of the user' + end delete ":id" do authenticated_as_admin! user = User.find_by(id: params[:id]) + not_found!('User') unless user - if user - DeleteUserService.new(current_user).execute(user) - else - not_found!('User') - end + DeleteUserService.new(current_user).execute(user) end - # Block user. Available only for admin - # - # Example Request: - # PUT /users/:id/block + desc 'Block a user. Available only for admins.' + params do + requires :id, type: Integer, desc: 'The ID of the user' + end put ':id/block' do authenticated_as_admin! user = User.find_by(id: params[:id]) + not_found!('User') unless user - if !user - not_found!('User') - elsif !user.ldap_blocked? + if !user.ldap_blocked? user.block else forbidden!('LDAP blocked users cannot be modified by the API') end end - # Unblock user. Available only for admin - # - # Example Request: - # PUT /users/:id/unblock + desc 'Unblock a user. Available only for admins.' + params do + requires :id, type: Integer, desc: 'The ID of the user' + end put ':id/unblock' do authenticated_as_admin! user = User.find_by(id: params[:id]) + not_found!('User') unless user - if !user - not_found!('User') - elsif user.ldap_blocked? + if user.ldap_blocked? forbidden!('LDAP blocked users cannot be unblocked by the API') else user.activate end end - desc 'Get contribution events of a specified user' do + desc 'Get the contribution events of a specified user' do detail 'This feature was introduced in GitLab 8.13.' success Entities::Event end params do - requires :id, type: String, desc: 'The user ID' + requires :id, type: Integer, desc: 'The ID of the user' + use :pagination end get ':id/events' do - user = User.find_by(id: declared(params).id) + user = User.find_by(id: params[:id]) not_found!('User') unless user events = user.events. @@ -344,43 +349,43 @@ module API end resource :user do - # Get currently authenticated user - # - # Example Request: - # GET /user + desc 'Get the currently authenticated user' do + success Entities::UserPublic + end get do - present @current_user, with: Entities::UserFull + present current_user, with: sudo? ? Entities::UserWithPrivateToken : Entities::UserPublic end - # Get currently authenticated user's keys - # - # Example Request: - # GET /user/keys + desc "Get the currently authenticated user's SSH keys" do + success Entities::SSHKey + end get "keys" do present current_user.keys, with: Entities::SSHKey end - # Get single key owned by currently authenticated user - # - # Example Request: - # GET /user/keys/:id - get "keys/:id" do - key = current_user.keys.find params[:id] + desc 'Get a single key owned by currently authenticated user' do + success Entities::SSHKey + end + params do + requires :key_id, type: Integer, desc: 'The ID of the SSH key' + end + get "keys/:key_id" do + key = current_user.keys.find_by(id: params[:key_id]) + not_found!('Key') unless key + present key, with: Entities::SSHKey end - # Add new ssh key to currently authenticated user - # - # Parameters: - # key (required) - New SSH Key - # title (required) - New SSH Key's title - # Example Request: - # POST /user/keys + desc 'Add a new SSH key to the currently authenticated user' do + success Entities::SSHKey + end + params do + requires :key, type: String, desc: 'The new SSH key' + requires :title, type: String, desc: 'The title of the new SSH key' + end post "keys" do - required_attributes! [:title, :key] + key = current_user.keys.new(declared_params) - attrs = attributes_for_keys [:title, :key] - key = current_user.keys.new attrs if key.save present key, with: Entities::SSHKey else @@ -388,48 +393,48 @@ module API end end - # Delete existing ssh key of currently authenticated user - # - # Parameters: - # id (required) - SSH Key ID - # Example Request: - # DELETE /user/keys/:id - delete "keys/:id" do - begin - key = current_user.keys.find params[:id] - key.destroy - rescue - end + desc 'Delete an SSH key from the currently authenticated user' do + success Entities::SSHKey + end + params do + requires :key_id, type: Integer, desc: 'The ID of the SSH key' + end + delete "keys/:key_id" do + key = current_user.keys.find_by(id: params[:key_id]) + not_found!('Key') unless key + + present key.destroy, with: Entities::SSHKey end - # Get currently authenticated user's emails - # - # Example Request: - # GET /user/emails + desc "Get the currently authenticated user's email addresses" do + success Entities::Email + end get "emails" do present current_user.emails, with: Entities::Email end - # Get single email owned by currently authenticated user - # - # Example Request: - # GET /user/emails/:id - get "emails/:id" do - email = current_user.emails.find params[:id] + desc 'Get a single email address owned by the currently authenticated user' do + success Entities::Email + end + params do + requires :email_id, type: Integer, desc: 'The ID of the email' + end + get "emails/:email_id" do + email = current_user.emails.find_by(id: params[:email_id]) + not_found!('Email') unless email + present email, with: Entities::Email end - # Add new email to currently authenticated user - # - # Parameters: - # email (required) - Email address - # Example Request: - # POST /user/emails + desc 'Add new email address to the currently authenticated user' do + success Entities::Email + end + params do + requires :email, type: String, desc: 'The new email' + end post "emails" do - required_attributes! [:email] + email = current_user.emails.new(declared_params) - attrs = attributes_for_keys [:email] - email = current_user.emails.new attrs if email.save NotificationService.new.new_email(email) present email, with: Entities::Email @@ -438,20 +443,16 @@ module API end end - # Delete existing email of currently authenticated user - # - # Parameters: - # id (required) - EMail ID - # Example Request: - # DELETE /user/emails/:id - delete "emails/:id" do - begin - email = current_user.emails.find params[:id] - email.destroy + desc 'Delete an email address from the currently authenticated user' + params do + requires :email_id, type: Integer, desc: 'The ID of the email' + end + delete "emails/:email_id" do + email = current_user.emails.find_by(id: params[:email_id]) + not_found!('Email') unless email - current_user.update_secondary_emails! - rescue - end + email.destroy + current_user.update_secondary_emails! end end end diff --git a/lib/api/variables.rb b/lib/api/variables.rb index b9fb3c21dbb..f623b1dfe9f 100644 --- a/lib/api/variables.rb +++ b/lib/api/variables.rb @@ -1,6 +1,8 @@ module API # Projects variables API class Variables < Grape::API + include PaginationParams + before { authenticate! } before { authorize! :admin_build, user_project } @@ -13,8 +15,7 @@ module API success Entities::Variable end params do - optional :page, type: Integer, desc: 'The page number for pagination' - optional :per_page, type: Integer, desc: 'The value of items per page to show' + use :pagination end get ':id/variables' do variables = user_project.variables @@ -29,7 +30,7 @@ module API end get ':id/variables/:key' do key = params[:key] - variable = user_project.variables.find_by(key: key.to_s) + variable = user_project.variables.find_by(key: key) return not_found!('Variable') unless variable diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 0dfffaf0bc6..cefbfdce3bb 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -2,6 +2,7 @@ module Backup class Manager ARCHIVES_TO_BACKUP = %w[uploads builds artifacts lfs registry] FOLDERS_TO_BACKUP = %w[repositories db] + FILE_NAME_SUFFIX = '_gitlab_backup.tar' def pack # Make sure there is a connection @@ -14,7 +15,7 @@ module Backup s[:gitlab_version] = Gitlab::VERSION s[:tar_version] = tar_version s[:skipped] = ENV["SKIP"] - tar_file = "#{s[:backup_created_at].to_i}_gitlab_backup.tar" + tar_file = "#{s[:backup_created_at].strftime('%s_%Y_%m_%d')}#{FILE_NAME_SUFFIX}" Dir.chdir(Gitlab.config.backup.path) do File.open("#{Gitlab.config.backup.path}/backup_information.yml", @@ -82,12 +83,17 @@ module Backup removed = 0 Dir.chdir(Gitlab.config.backup.path) do - file_list = Dir.glob('*_gitlab_backup.tar') - file_list.map! { |f| $1.to_i if f =~ /(\d+)_gitlab_backup.tar/ } - file_list.sort.each do |timestamp| + Dir.glob("*#{FILE_NAME_SUFFIX}").each do |file| + next unless file =~ /(\d+)(?:_\d{4}_\d{2}_\d{2})?_gitlab_backup\.tar/ + + timestamp = $1.to_i + if Time.at(timestamp) < (Time.now - keep_time) - if Kernel.system(*%W(rm #{timestamp}_gitlab_backup.tar)) + begin + FileUtils.rm(file) removed += 1 + rescue => e + $progress.puts "Deleting #{file} failed: #{e.message}".color(:red) end end end @@ -103,41 +109,50 @@ module Backup Dir.chdir(Gitlab.config.backup.path) # check for existing backups in the backup dir - file_list = Dir.glob("*_gitlab_backup.tar").each.map { |f| f.split(/_/).first.to_i } - puts "no backups found" if file_list.count == 0 + file_list = Dir.glob("*#{FILE_NAME_SUFFIX}") + + if file_list.count == 0 + $progress.puts "No backups found in #{Gitlab.config.backup.path}" + $progress.puts "Please make sure that file name ends with #{FILE_NAME_SUFFIX}" + exit 1 + end if file_list.count > 1 && ENV["BACKUP"].nil? - puts "Found more than one backup, please specify which one you want to restore:" - puts "rake gitlab:backup:restore BACKUP=timestamp_of_backup" + $progress.puts 'Found more than one backup, please specify which one you want to restore:' + $progress.puts 'rake gitlab:backup:restore BACKUP=timestamp_of_backup' exit 1 end - tar_file = ENV["BACKUP"].nil? ? File.join("#{file_list.first}_gitlab_backup.tar") : File.join(ENV["BACKUP"] + "_gitlab_backup.tar") + if ENV['BACKUP'].present? + tar_file = "#{ENV['BACKUP']}#{FILE_NAME_SUFFIX}" + else + tar_file = file_list.first + end unless File.exist?(tar_file) - puts "The specified backup doesn't exist!" + $progress.puts "The backup file #{tar_file} does not exist!" exit 1 end - $progress.print "Unpacking backup ... " + $progress.print 'Unpacking backup ... ' unless Kernel.system(*%W(tar -xf #{tar_file})) - puts "unpacking backup failed".color(:red) + $progress.puts 'unpacking backup failed'.color(:red) exit 1 else - $progress.puts "done".color(:green) + $progress.puts 'done'.color(:green) end ENV["VERSION"] = "#{settings[:db_version]}" if settings[:db_version].to_i > 0 # restoring mismatching backups can lead to unexpected problems if settings[:gitlab_version] != Gitlab::VERSION - puts "GitLab version mismatch:".color(:red) - puts " Your current GitLab version (#{Gitlab::VERSION}) differs from the GitLab version in the backup!".color(:red) - puts " Please switch to the following version and try again:".color(:red) - puts " version: #{settings[:gitlab_version]}".color(:red) - puts - puts "Hint: git checkout v#{settings[:gitlab_version]}" + $progress.puts 'GitLab version mismatch:'.color(:red) + $progress.puts " Your current GitLab version (#{Gitlab::VERSION}) differs from the GitLab version in the backup!".color(:red) + $progress.puts ' Please switch to the following version and try again:'.color(:red) + $progress.puts " version: #{settings[:gitlab_version]}".color(:red) + $progress.puts + $progress.puts "Hint: git checkout v#{settings[:gitlab_version]}" exit 1 end end diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 3740d4fb4cd..6d04f68c8f9 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -33,7 +33,7 @@ module Banzai # Returns a String replaced with the return of the block. def self.references_in(text, pattern = object_class.reference_pattern) text.gsub(pattern) do |match| - yield match, $~[object_sym].to_i, $~[:project], $~ + yield match, $~[object_sym].to_i, $~[:project], $~[:namespace], $~ end end @@ -145,8 +145,9 @@ module Banzai # Returns a String with references replaced with links. All links # have `gfm` and `gfm-OBJECT_NAME` class names attached for styling. def object_link_filter(text, pattern, link_content: nil) - references_in(text, pattern) do |match, id, project_ref, matches| - project = project_from_ref_cached(project_ref) + references_in(text, pattern) do |match, id, project_ref, namespace_ref, matches| + project_path = full_project_path(namespace_ref, project_ref) + project = project_from_ref_cached(project_path) if project && object = find_object_cached(project, id) title = object_link_title(object) @@ -217,10 +218,9 @@ module Banzai nodes.each do |node| node.to_html.scan(regex) do - project = $~[:project] || current_project_path + project_path = full_project_path($~[:namespace], $~[:project]) symbol = $~[object_sym] - - refs[project] << symbol if object_class.reference_valid?(symbol) + refs[project_path] << symbol if object_class.reference_valid?(symbol) end end @@ -248,21 +248,32 @@ module Banzai end def projects_relation_for_paths(paths) - Project.where_paths_in(paths).includes(:namespace) + Project.where_full_path_in(paths).includes(:namespace) end # Returns projects for the given paths. def find_projects_for_paths(paths) if RequestStore.active? - to_query = paths - project_refs_cache.keys + cache = project_refs_cache + to_query = paths - cache.keys unless to_query.empty? - projects_relation_for_paths(to_query).each do |project| - get_or_set_cache(project_refs_cache, project.path_with_namespace) { project } + projects = projects_relation_for_paths(to_query) + + found = [] + projects.each do |project| + ref = project.path_with_namespace + get_or_set_cache(cache, ref) { project } + found << ref + end + + not_found = to_query - found + not_found.each do |ref| + get_or_set_cache(cache, ref) { nil } end end - project_refs_cache.slice(*paths).values + cache.slice(*paths).values.compact else projects_relation_for_paths(paths) end @@ -272,8 +283,19 @@ module Banzai @current_project_path ||= project.path_with_namespace end + def current_project_namespace_path + @current_project_namespace_path ||= project.namespace.path + end + private + def full_project_path(namespace, project_ref) + return current_project_path unless project_ref + + namespace_ref = namespace || current_project_namespace_path + "#{namespace_ref}/#{project_ref}" + end + def project_refs_cache RequestStore[:banzai_project_refs] ||= {} end diff --git a/lib/banzai/filter/autolink_filter.rb b/lib/banzai/filter/autolink_filter.rb index 799b83b1069..80c844baecd 100644 --- a/lib/banzai/filter/autolink_filter.rb +++ b/lib/banzai/filter/autolink_filter.rb @@ -71,6 +71,14 @@ module Banzai @doc = parse_html(rinku) end + # Return true if any of the UNSAFE_PROTOCOLS strings are included in the URI scheme + def contains_unsafe?(scheme) + return false unless scheme + + scheme = scheme.strip.downcase + Banzai::Filter::SanitizationFilter::UNSAFE_PROTOCOLS.any? { |protocol| scheme.include?(protocol) } + end + # Autolinks any text matching LINK_PATTERN that Rinku didn't already # replace def text_parse @@ -89,17 +97,27 @@ module Banzai doc end - def autolink_filter(text) - text.gsub(LINK_PATTERN) do |match| - # Remove any trailing HTML entities and store them for appending - # outside the link element. The entity must be marked HTML safe in - # order to be output literally rather than escaped. - match.gsub!(/((?:&[\w#]+;)+)\z/, '') - dropped = ($1 || '').html_safe - - options = link_options.merge(href: match) - content_tag(:a, match, options) + dropped + def autolink_match(match) + # start by stripping out dangerous links + begin + uri = Addressable::URI.parse(match) + return match if contains_unsafe?(uri.scheme) + rescue Addressable::URI::InvalidURIError + return match end + + # Remove any trailing HTML entities and store them for appending + # outside the link element. The entity must be marked HTML safe in + # order to be output literally rather than escaped. + match.gsub!(/((?:&[\w#]+;)+)\z/, '') + dropped = ($1 || '').html_safe + + options = link_options.merge(href: match) + content_tag(:a, match, options) + dropped + end + + def autolink_filter(text) + text.gsub(LINK_PATTERN) { |match| autolink_match(match) } end def link_options diff --git a/lib/banzai/filter/commit_range_reference_filter.rb b/lib/banzai/filter/commit_range_reference_filter.rb index 4358bf45549..eaacb9591b1 100644 --- a/lib/banzai/filter/commit_range_reference_filter.rb +++ b/lib/banzai/filter/commit_range_reference_filter.rb @@ -12,7 +12,7 @@ module Banzai def self.references_in(text, pattern = CommitRange.reference_pattern) text.gsub(pattern) do |match| - yield match, $~[:commit_range], $~[:project], $~ + yield match, $~[:commit_range], $~[:project], $~[:namespace], $~ end end diff --git a/lib/banzai/filter/commit_reference_filter.rb b/lib/banzai/filter/commit_reference_filter.rb index a26dd09c25a..69c06117eda 100644 --- a/lib/banzai/filter/commit_reference_filter.rb +++ b/lib/banzai/filter/commit_reference_filter.rb @@ -12,7 +12,7 @@ module Banzai def self.references_in(text, pattern = Commit.reference_pattern) text.gsub(pattern) do |match| - yield match, $~[:commit], $~[:project], $~ + yield match, $~[:commit], $~[:project], $~[:namespace], $~ end end diff --git a/lib/banzai/filter/external_link_filter.rb b/lib/banzai/filter/external_link_filter.rb index 2f19b59e725..d67d466bce8 100644 --- a/lib/banzai/filter/external_link_filter.rb +++ b/lib/banzai/filter/external_link_filter.rb @@ -10,7 +10,7 @@ module Banzai node.set_attribute('href', href) end - if href =~ /\Ahttp(s)?:\/\// && external_url?(href) + if href =~ %r{\A(https?:)?//[^/]} && external_url?(href) node.set_attribute('rel', 'nofollow noreferrer') node.set_attribute('target', '_blank') end diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb index 9f9a96cdc65..a605dea149e 100644 --- a/lib/banzai/filter/label_reference_filter.rb +++ b/lib/banzai/filter/label_reference_filter.rb @@ -14,16 +14,18 @@ module Banzai def self.references_in(text, pattern = Label.reference_pattern) unescape_html_entities(text).gsub(pattern) do |match| - yield match, $~[:label_id].to_i, $~[:label_name], $~[:project], $~ + yield match, $~[:label_id].to_i, $~[:label_name], $~[:project], $~[:namespace], $~ end end def references_in(text, pattern = Label.reference_pattern) unescape_html_entities(text).gsub(pattern) do |match| - label = find_label($~[:project], $~[:label_id], $~[:label_name]) + namespace, project = $~[:namespace], $~[:project] + project_path = full_project_path(namespace, project) + label = find_label(project_path, $~[:label_id], $~[:label_name]) if label - yield match, label.id, $~[:project], $~ + yield match, label.id, project, namespace, $~ else match end @@ -64,48 +66,12 @@ module Banzai end def object_link_text(object, matches) - if same_group?(object) && namespace_match?(matches) - render_same_project_label(object) - elsif same_project?(object) - render_same_project_label(object) - else - render_cross_project_label(object, matches) - end - end - - def same_group?(object) - object.is_a?(GroupLabel) && object.group == project.group - end - - def namespace_match?(matches) - matches[:project].blank? || matches[:project] == project.path_with_namespace - end - - def same_project?(object) - object.is_a?(ProjectLabel) && object.project == project - end - - def user - context[:current_user] || context[:author] - end - - def project - context[:project] - end - - def render_same_project_label(object) - LabelsHelper.render_colored_label(object) - end - - def render_cross_project_label(object, matches) - source_project = - if matches[:project] - Project.find_with_namespace(matches[:project]) - else - object.project - end + project_path = full_project_path(matches[:namespace], matches[:project]) + project_from_ref = project_from_ref_cached(project_path) + reference = project_from_ref.to_human_reference(project) + label_suffix = " <i>in #{reference}</i>" if reference.present? - LabelsHelper.render_colored_cross_project_label(object, source_project) + LabelsHelper.render_colored_label(object, label_suffix) end def unescape_html_entities(text) diff --git a/lib/banzai/filter/math_filter.rb b/lib/banzai/filter/math_filter.rb new file mode 100644 index 00000000000..b6e784c886b --- /dev/null +++ b/lib/banzai/filter/math_filter.rb @@ -0,0 +1,46 @@ +require 'uri' + +module Banzai + module Filter + # HTML filter that adds class="code math" and removes the dollar sign in $`2+2`$. + # + class MathFilter < HTML::Pipeline::Filter + # Attribute indicating inline or display math. + STYLE_ATTRIBUTE = 'data-math-style'.freeze + + # Class used for tagging elements that should be rendered + TAG_CLASS = 'js-render-math'.freeze + + INLINE_CLASSES = "code math #{TAG_CLASS}".freeze + + DOLLAR_SIGN = '$'.freeze + + def call + doc.css('code').each do |code| + closing = code.next + opening = code.previous + + # We need a sibling before and after. + # They should end and start with $ respectively. + if closing && opening && + closing.text? && opening.text? && + closing.content.first == DOLLAR_SIGN && + opening.content.last == DOLLAR_SIGN + + code[:class] = INLINE_CLASSES + code[STYLE_ATTRIBUTE] = 'inline' + closing.content = closing.content[1..-1] + opening.content = opening.content[0..-2] + end + end + + doc.css('pre.code.math').each do |el| + el[STYLE_ATTRIBUTE] = 'display' + el[:class] += " #{TAG_CLASS}" + end + + doc + end + end + end +end diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb index 58fff496d00..f12014e191f 100644 --- a/lib/banzai/filter/milestone_reference_filter.rb +++ b/lib/banzai/filter/milestone_reference_filter.rb @@ -19,18 +19,20 @@ module Banzai return super(text, pattern) if pattern != Milestone.reference_pattern text.gsub(pattern) do |match| - milestone = find_milestone($~[:project], $~[:milestone_iid], $~[:milestone_name]) + milestone = find_milestone($~[:project], $~[:namespace], $~[:milestone_iid], $~[:milestone_name]) if milestone - yield match, milestone.iid, $~[:project], $~ + yield match, milestone.iid, $~[:project], $~[:namespace], $~ else match end end end - def find_milestone(project_ref, milestone_id, milestone_name) - project = project_from_ref(project_ref) + def find_milestone(project_ref, namespace_ref, milestone_id, milestone_name) + project_path = full_project_path(namespace_ref, project_ref) + project = project_from_ref(project_path) + return unless project milestone_params = milestone_params(milestone_id, milestone_name) @@ -52,11 +54,13 @@ module Banzai end def object_link_text(object, matches) - if context[:project] == object.project - super + milestone_link = escape_once(super) + reference = object.project.to_reference(project) + + if reference.present? + "#{milestone_link} <i>in #{reference}</i>".html_safe else - "#{escape_once(super)} <i>in #{escape_once(object.project.path_with_namespace)}</i>". - html_safe + milestone_link end end diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index 84bfeac8041..ab7af1cad21 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -20,10 +20,10 @@ module Banzai # Examples: # # data_attribute(project: 1, issue: 2) - # # => "data-reference-filter=\"SomeReferenceFilter\" data-project=\"1\" data-issue=\"2\"" + # # => "data-reference-type=\"SomeReferenceFilter\" data-project=\"1\" data-issue=\"2\"" # # data_attribute(project: 3, merge_request: 4) - # # => "data-reference-filter=\"SomeReferenceFilter\" data-project=\"3\" data-merge-request=\"4\"" + # # => "data-reference-type=\"SomeReferenceFilter\" data-project=\"3\" data-merge-request=\"4\"" # # Returns a String def data_attribute(attributes = {}) @@ -31,7 +31,9 @@ module Banzai attributes[:reference_type] ||= self.class.reference_type attributes.delete(:original) if context[:no_original_data] - attributes.map { |key, value| %Q(data-#{key.to_s.dasherize}="#{escape_once(value)}") }.join(" ") + attributes.map do |key, value| + %Q(data-#{key.to_s.dasherize}="#{escape_once(value)}") + end.join(' ') end def escape_once(html) diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb index f09d78be0ce..9e23c8f8c55 100644 --- a/lib/banzai/filter/relative_link_filter.rb +++ b/lib/banzai/filter/relative_link_filter.rb @@ -46,7 +46,7 @@ module Banzai end def rebuild_relative_uri(uri) - file_path = relative_file_path(uri.path) + file_path = relative_file_path(uri) uri.path = [ relative_url_root, @@ -59,8 +59,10 @@ module Banzai uri end - def relative_file_path(path) - nested_path = build_relative_path(path, context[:requested_path]) + def relative_file_path(uri) + path = Addressable::URI.unescape(uri.path) + request_path = Addressable::URI.unescape(context[:requested_path]) + nested_path = build_relative_path(path, request_path) file_exists?(nested_path) ? nested_path : path end @@ -108,11 +110,7 @@ module Banzai end def uri_type(path) - @uri_types[path] ||= begin - unescaped_path = Addressable::URI.unescape(path) - - current_commit.uri_type(unescaped_path) - end + @uri_types[path] ||= current_commit.uri_type(path) end def current_commit diff --git a/lib/banzai/filter/table_of_contents_filter.rb b/lib/banzai/filter/table_of_contents_filter.rb index a4eda6fdf76..8e7084f2543 100644 --- a/lib/banzai/filter/table_of_contents_filter.rb +++ b/lib/banzai/filter/table_of_contents_filter.rb @@ -35,9 +35,11 @@ module Banzai headers[id] += 1 if header_content = node.children.first + # namespace detection will be automatically handled via javascript (see issue #22781) + namespace = "user-content-" href = "#{id}#{uniq}" push_toc(href, text) - header_content.add_previous_sibling(anchor_tag(href)) + header_content.add_previous_sibling(anchor_tag("#{namespace}#{href}", href)) end end @@ -48,8 +50,8 @@ module Banzai private - def anchor_tag(href) - %Q{<a id="#{href}" class="anchor" href="##{href}" aria-hidden="true"></a>} + def anchor_tag(id, href) + %Q{<a id="#{id}" class="anchor" href="##{href}" aria-hidden="true"></a>} end def push_toc(href, text) diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index 5da2d0b008c..5a1f873496c 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -6,6 +6,7 @@ module Banzai Filter::SyntaxHighlightFilter, Filter::SanitizationFilter, + Filter::MathFilter, Filter::UploadLinkFilter, Filter::VideoLinkFilter, Filter::ImageLinkFilter, diff --git a/lib/banzai/reference_parser/base_parser.rb b/lib/banzai/reference_parser/base_parser.rb index f5d110e987b..d8a855ec1fe 100644 --- a/lib/banzai/reference_parser/base_parser.rb +++ b/lib/banzai/reference_parser/base_parser.rb @@ -63,12 +63,7 @@ module Banzai nodes.select do |node| if node.has_attribute?(project_attr) node_id = node.attr(project_attr).to_i - - if project && project.id == node_id - true - else - can?(user, :read_project, projects[node_id]) - end + can_read_reference?(user, projects[node_id]) else true end @@ -226,6 +221,15 @@ module Banzai attr_reader :current_user, :project + # When a feature is disabled or visible only for + # team members we should not allow team members + # see reference comments. + # Override this method on subclasses + # to check if user can read resource + def can_read_reference?(user, ref_project) + raise NotImplementedError + end + def lazy(&block) Gitlab::Lazy.new(&block) end diff --git a/lib/banzai/reference_parser/commit_parser.rb b/lib/banzai/reference_parser/commit_parser.rb index 0fee9d267de..8c54a041cb8 100644 --- a/lib/banzai/reference_parser/commit_parser.rb +++ b/lib/banzai/reference_parser/commit_parser.rb @@ -29,6 +29,12 @@ module Banzai commits end + + private + + def can_read_reference?(user, ref_project) + can?(user, :download_code, ref_project) + end end end end diff --git a/lib/banzai/reference_parser/commit_range_parser.rb b/lib/banzai/reference_parser/commit_range_parser.rb index 69d01f8db15..0878b6afba3 100644 --- a/lib/banzai/reference_parser/commit_range_parser.rb +++ b/lib/banzai/reference_parser/commit_range_parser.rb @@ -33,6 +33,12 @@ module Banzai range.valid_commits? ? range : nil end + + private + + def can_read_reference?(user, ref_project) + can?(user, :download_code, ref_project) + end end end end diff --git a/lib/banzai/reference_parser/external_issue_parser.rb b/lib/banzai/reference_parser/external_issue_parser.rb index a1264db2111..6e7b7669578 100644 --- a/lib/banzai/reference_parser/external_issue_parser.rb +++ b/lib/banzai/reference_parser/external_issue_parser.rb @@ -20,6 +20,12 @@ module Banzai def issue_ids_per_project(nodes) gather_attributes_per_project(nodes, self.class.data_attribute) end + + private + + def can_read_reference?(user, ref_project) + can?(user, :read_issue, ref_project) + end end end end diff --git a/lib/banzai/reference_parser/label_parser.rb b/lib/banzai/reference_parser/label_parser.rb index e5d1eb11d7f..aa76c64ac5f 100644 --- a/lib/banzai/reference_parser/label_parser.rb +++ b/lib/banzai/reference_parser/label_parser.rb @@ -6,6 +6,12 @@ module Banzai def references_relation Label end + + private + + def can_read_reference?(user, ref_project) + can?(user, :read_label, ref_project) + end end end end diff --git a/lib/banzai/reference_parser/merge_request_parser.rb b/lib/banzai/reference_parser/merge_request_parser.rb index c9a9ca79c09..40451947e6c 100644 --- a/lib/banzai/reference_parser/merge_request_parser.rb +++ b/lib/banzai/reference_parser/merge_request_parser.rb @@ -6,6 +6,12 @@ module Banzai def references_relation MergeRequest.includes(:author, :assignee, :target_project) end + + private + + def can_read_reference?(user, ref_project) + can?(user, :read_merge_request, ref_project) + end end end end diff --git a/lib/banzai/reference_parser/milestone_parser.rb b/lib/banzai/reference_parser/milestone_parser.rb index a000ac61e5c..d3968d6b229 100644 --- a/lib/banzai/reference_parser/milestone_parser.rb +++ b/lib/banzai/reference_parser/milestone_parser.rb @@ -6,6 +6,12 @@ module Banzai def references_relation Milestone end + + private + + def can_read_reference?(user, ref_project) + can?(user, :read_milestone, ref_project) + end end end end diff --git a/lib/banzai/reference_parser/snippet_parser.rb b/lib/banzai/reference_parser/snippet_parser.rb index fa71b3c952a..63b592137bb 100644 --- a/lib/banzai/reference_parser/snippet_parser.rb +++ b/lib/banzai/reference_parser/snippet_parser.rb @@ -6,6 +6,12 @@ module Banzai def references_relation Snippet end + + private + + def can_read_reference?(user, ref_project) + can?(user, :read_project_snippet, ref_project) + end end end end diff --git a/lib/banzai/reference_parser/user_parser.rb b/lib/banzai/reference_parser/user_parser.rb index 863f5725d3b..7adaffa19c1 100644 --- a/lib/banzai/reference_parser/user_parser.rb +++ b/lib/banzai/reference_parser/user_parser.rb @@ -30,22 +30,36 @@ module Banzai nodes.each do |node| if node.has_attribute?(group_attr) - node_group = groups[node.attr(group_attr).to_i] - - if node_group && - can?(user, :read_group, node_group) - visible << node - end - # Remaining nodes will be processed by the parent class' - # implementation of this method. + next unless can_read_group_reference?(node, user, groups) + visible << node + elsif can_read_project_reference?(node) + visible << node else remaining << node end end + # If project does not belong to a group + # and does not have the same project id as the current project + # base class will check if user can read the project that contains + # the user reference. visible + super(current_user, remaining) end + # Check if project belongs to a group which + # user can read. + def can_read_group_reference?(node, user, groups) + node_group = groups[node.attr('data-group').to_i] + + node_group && can?(user, :read_group, node_group) + end + + def can_read_project_reference?(node) + node_id = node.attr('data-project').to_i + + project && project.id == node_id + end + def nodes_user_can_reference(current_user, nodes) project_attr = 'data-project' author_attr = 'data-author' @@ -88,6 +102,10 @@ module Banzai collection_objects_for_ids(Project, ids). flat_map { |p| p.team.members.to_a } end + + def can_read_reference?(user, ref_project) + can?(user, :read_project, ref_project) + end end end end diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index ce048a36fa0..f31fb6c3f71 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -46,7 +46,7 @@ module Banzai return html if html.present? html = cacheless_render_field(object, field) - object.update_column(html_field, html) unless object.new_record? || object.destroyed? + update_object(object, html_field, html) unless object.new_record? || object.destroyed? html end @@ -166,5 +166,9 @@ module Banzai return unless cache_key Rails.cache.send(:expanded_key, full_cache_key(cache_key, pipeline_name)) end + + def update_object(object, html_field, html) + object.update_column(html_field, html) + end end end diff --git a/lib/bitbucket/client.rb b/lib/bitbucket/client.rb new file mode 100644 index 00000000000..f8ee7e0f9ae --- /dev/null +++ b/lib/bitbucket/client.rb @@ -0,0 +1,58 @@ +module Bitbucket + class Client + attr_reader :connection + + def initialize(options = {}) + @connection = Connection.new(options) + end + + def issues(repo) + path = "/repositories/#{repo}/issues" + get_collection(path, :issue) + end + + def issue_comments(repo, issue_id) + path = "/repositories/#{repo}/issues/#{issue_id}/comments" + get_collection(path, :comment) + end + + def pull_requests(repo) + path = "/repositories/#{repo}/pullrequests?state=ALL" + get_collection(path, :pull_request) + end + + def pull_request_comments(repo, pull_request) + path = "/repositories/#{repo}/pullrequests/#{pull_request}/comments" + get_collection(path, :pull_request_comment) + end + + def pull_request_diff(repo, pull_request) + path = "/repositories/#{repo}/pullrequests/#{pull_request}/diff" + connection.get(path) + end + + def repo(name) + parsed_response = connection.get("/repositories/#{name}") + Representation::Repo.new(parsed_response) + end + + def repos + path = "/repositories?role=member" + get_collection(path, :repo) + end + + def user + @user ||= begin + parsed_response = connection.get('/user') + Representation::User.new(parsed_response) + end + end + + private + + def get_collection(path, type) + paginator = Paginator.new(connection, path, type) + Collection.new(paginator) + end + end +end diff --git a/lib/bitbucket/collection.rb b/lib/bitbucket/collection.rb new file mode 100644 index 00000000000..3a9379ff680 --- /dev/null +++ b/lib/bitbucket/collection.rb @@ -0,0 +1,21 @@ +module Bitbucket + class Collection < Enumerator + def initialize(paginator) + super() do |yielder| + loop do + paginator.items.each { |item| yielder << item } + end + end + + lazy + end + + def method_missing(method, *args) + return super unless self.respond_to?(method) + + self.send(method, *args) do |item| + block_given? ? yield(item) : item + end + end + end +end diff --git a/lib/bitbucket/connection.rb b/lib/bitbucket/connection.rb new file mode 100644 index 00000000000..7e55cf4deab --- /dev/null +++ b/lib/bitbucket/connection.rb @@ -0,0 +1,69 @@ +module Bitbucket + class Connection + DEFAULT_API_VERSION = '2.0' + DEFAULT_BASE_URI = 'https://api.bitbucket.org/' + DEFAULT_QUERY = {} + + attr_reader :expires_at, :expires_in, :refresh_token, :token + + def initialize(options = {}) + @api_version = options.fetch(:api_version, DEFAULT_API_VERSION) + @base_uri = options.fetch(:base_uri, DEFAULT_BASE_URI) + @default_query = options.fetch(:query, DEFAULT_QUERY) + + @token = options[:token] + @expires_at = options[:expires_at] + @expires_in = options[:expires_in] + @refresh_token = options[:refresh_token] + end + + def get(path, extra_query = {}) + refresh! if expired? + + response = connection.get(build_url(path), params: @default_query.merge(extra_query)) + response.parsed + end + + def expired? + connection.expired? + end + + def refresh! + response = connection.refresh! + + @token = response.token + @expires_at = response.expires_at + @expires_in = response.expires_in + @refresh_token = response.refresh_token + @connection = nil + end + + private + + def client + @client ||= OAuth2::Client.new(provider.app_id, provider.app_secret, options) + end + + def connection + @connection ||= OAuth2::AccessToken.new(client, @token, refresh_token: @refresh_token, expires_at: @expires_at, expires_in: @expires_in) + end + + def build_url(path) + return path if path.starts_with?(root_url) + + "#{root_url}#{path}" + end + + def root_url + @root_url ||= "#{@base_uri}#{@api_version}" + end + + def provider + Gitlab::OAuth::Provider.config_for('bitbucket') + end + + def options + OmniAuth::Strategies::Bitbucket.default_options[:client_options].deep_symbolize_keys + end + end +end diff --git a/lib/bitbucket/error/unauthorized.rb b/lib/bitbucket/error/unauthorized.rb new file mode 100644 index 00000000000..5e2eb57bb0e --- /dev/null +++ b/lib/bitbucket/error/unauthorized.rb @@ -0,0 +1,6 @@ +module Bitbucket + module Error + class Unauthorized < StandardError + end + end +end diff --git a/lib/bitbucket/page.rb b/lib/bitbucket/page.rb new file mode 100644 index 00000000000..2b0a3fe7b1a --- /dev/null +++ b/lib/bitbucket/page.rb @@ -0,0 +1,34 @@ +module Bitbucket + class Page + attr_reader :attrs, :items + + def initialize(raw, type) + @attrs = parse_attrs(raw) + @items = parse_values(raw, representation_class(type)) + end + + def next? + attrs.fetch(:next, false) + end + + def next + attrs.fetch(:next) + end + + private + + def parse_attrs(raw) + raw.slice(*%w(size page pagelen next previous)).symbolize_keys + end + + def parse_values(raw, bitbucket_rep_class) + return [] unless raw['values'] && raw['values'].is_a?(Array) + + bitbucket_rep_class.decorate(raw['values']) + end + + def representation_class(type) + Bitbucket::Representation.const_get(type.to_s.camelize) + end + end +end diff --git a/lib/bitbucket/paginator.rb b/lib/bitbucket/paginator.rb new file mode 100644 index 00000000000..135d0d55674 --- /dev/null +++ b/lib/bitbucket/paginator.rb @@ -0,0 +1,36 @@ +module Bitbucket + class Paginator + PAGE_LENGTH = 50 # The minimum length is 10 and the maximum is 100. + + def initialize(connection, url, type) + @connection = connection + @type = type + @url = url + @page = nil + end + + def items + raise StopIteration unless has_next_page? + + @page = fetch_next_page + @page.items + end + + private + + attr_reader :connection, :page, :url, :type + + def has_next_page? + page.nil? || page.next? + end + + def next_url + page.nil? ? url : page.next + end + + def fetch_next_page + parsed_response = connection.get(next_url, pagelen: PAGE_LENGTH, sort: :created_on) + Page.new(parsed_response, type) + end + end +end diff --git a/lib/bitbucket/representation/base.rb b/lib/bitbucket/representation/base.rb new file mode 100644 index 00000000000..94adaacc9b5 --- /dev/null +++ b/lib/bitbucket/representation/base.rb @@ -0,0 +1,17 @@ +module Bitbucket + module Representation + class Base + def initialize(raw) + @raw = raw + end + + def self.decorate(entries) + entries.map { |entry| new(entry)} + end + + private + + attr_reader :raw + end + end +end diff --git a/lib/bitbucket/representation/comment.rb b/lib/bitbucket/representation/comment.rb new file mode 100644 index 00000000000..4937aa9728f --- /dev/null +++ b/lib/bitbucket/representation/comment.rb @@ -0,0 +1,27 @@ +module Bitbucket + module Representation + class Comment < Representation::Base + def author + user['username'] + end + + def note + raw.fetch('content', {}).fetch('raw', nil) + end + + def created_at + raw['created_on'] + end + + def updated_at + raw['updated_on'] || raw['created_on'] + end + + private + + def user + raw.fetch('user', {}) + end + end + end +end diff --git a/lib/bitbucket/representation/issue.rb b/lib/bitbucket/representation/issue.rb new file mode 100644 index 00000000000..054064395c3 --- /dev/null +++ b/lib/bitbucket/representation/issue.rb @@ -0,0 +1,53 @@ +module Bitbucket + module Representation + class Issue < Representation::Base + CLOSED_STATUS = %w(resolved invalid duplicate wontfix closed).freeze + + def iid + raw['id'] + end + + def kind + raw['kind'] + end + + def author + raw.fetch('reporter', {}).fetch('username', nil) + end + + def description + raw.fetch('content', {}).fetch('raw', nil) + end + + def state + closed? ? 'closed' : 'opened' + end + + def title + raw['title'] + end + + def milestone + raw['milestone']['name'] if raw['milestone'].present? + end + + def created_at + raw['created_on'] + end + + def updated_at + raw['edited_on'] + end + + def to_s + iid + end + + private + + def closed? + CLOSED_STATUS.include?(raw['state']) + end + end + end +end diff --git a/lib/bitbucket/representation/pull_request.rb b/lib/bitbucket/representation/pull_request.rb new file mode 100644 index 00000000000..eebf8093380 --- /dev/null +++ b/lib/bitbucket/representation/pull_request.rb @@ -0,0 +1,65 @@ +module Bitbucket + module Representation + class PullRequest < Representation::Base + def author + raw.fetch('author', {}).fetch('username', nil) + end + + def description + raw['description'] + end + + def iid + raw['id'] + end + + def state + if raw['state'] == 'MERGED' + 'merged' + elsif raw['state'] == 'DECLINED' + 'closed' + else + 'opened' + end + end + + def created_at + raw['created_on'] + end + + def updated_at + raw['updated_on'] + end + + def title + raw['title'] + end + + def source_branch_name + source_branch.fetch('branch', {}).fetch('name', nil) + end + + def source_branch_sha + source_branch.fetch('commit', {}).fetch('hash', nil) + end + + def target_branch_name + target_branch.fetch('branch', {}).fetch('name', nil) + end + + def target_branch_sha + target_branch.fetch('commit', {}).fetch('hash', nil) + end + + private + + def source_branch + raw['source'] + end + + def target_branch + raw['destination'] + end + end + end +end diff --git a/lib/bitbucket/representation/pull_request_comment.rb b/lib/bitbucket/representation/pull_request_comment.rb new file mode 100644 index 00000000000..4f8efe03bae --- /dev/null +++ b/lib/bitbucket/representation/pull_request_comment.rb @@ -0,0 +1,39 @@ +module Bitbucket + module Representation + class PullRequestComment < Comment + def iid + raw['id'] + end + + def file_path + inline.fetch('path') + end + + def old_pos + inline.fetch('from') + end + + def new_pos + inline.fetch('to') + end + + def parent_id + raw.fetch('parent', {}).fetch('id', nil) + end + + def inline? + raw.has_key?('inline') + end + + def has_parent? + raw.has_key?('parent') + end + + private + + def inline + raw.fetch('inline', {}) + end + end + end +end diff --git a/lib/bitbucket/representation/repo.rb b/lib/bitbucket/representation/repo.rb new file mode 100644 index 00000000000..423eff8f2a5 --- /dev/null +++ b/lib/bitbucket/representation/repo.rb @@ -0,0 +1,71 @@ +module Bitbucket + module Representation + class Repo < Representation::Base + attr_reader :owner, :slug + + def initialize(raw) + super(raw) + end + + def owner_and_slug + @owner_and_slug ||= full_name.split('/', 2) + end + + def owner + owner_and_slug.first + end + + def slug + owner_and_slug.last + end + + def clone_url(token = nil) + url = raw['links']['clone'].find { |link| link['name'] == 'https' }.fetch('href') + + if token.present? + clone_url = URI::parse(url) + clone_url.user = "x-token-auth:#{token}" + clone_url.to_s + else + url + end + end + + def description + raw['description'] + end + + def full_name + raw['full_name'] + end + + def issues_enabled? + raw['has_issues'] + end + + def name + raw['name'] + end + + def valid? + raw['scm'] == 'git' + end + + def has_wiki? + raw['has_wiki'] + end + + def visibility_level + if raw['is_private'] + Gitlab::VisibilityLevel::PRIVATE + else + Gitlab::VisibilityLevel::PUBLIC + end + end + + def to_s + full_name + end + end + end +end diff --git a/lib/bitbucket/representation/user.rb b/lib/bitbucket/representation/user.rb new file mode 100644 index 00000000000..ba6b7667b49 --- /dev/null +++ b/lib/bitbucket/representation/user.rb @@ -0,0 +1,9 @@ +module Bitbucket + module Representation + class User < Representation::Base + def username + raw['username'] + end + end + end +end diff --git a/lib/ci/ansi2html.rb b/lib/ci/ansi2html.rb index 229050151d3..c10d3616f31 100644 --- a/lib/ci/ansi2html.rb +++ b/lib/ci/ansi2html.rb @@ -105,7 +105,7 @@ module Ci break elsif s.scan(/</) @out << '<' - elsif s.scan(/\n/) + elsif s.scan(/\r?\n/) @out << '<br>' else @out << s.scan(/./m) diff --git a/lib/ci/api/api.rb b/lib/ci/api/api.rb index a6b9beecded..24bb3649a76 100644 --- a/lib/ci/api/api.rb +++ b/lib/ci/api/api.rb @@ -8,6 +8,16 @@ module Ci rack_response({ 'message' => '404 Not found' }.to_json, 404) end + # Retain 405 error rather than a 500 error for Grape 0.15.0+. + # https://github.com/ruby-grape/grape/blob/a3a28f5b5dfbb2797442e006dbffd750b27f2a76/UPGRADING.md#changes-to-method-not-allowed-routes + rescue_from Grape::Exceptions::MethodNotAllowed do |e| + error! e.message, e.status, e.headers + end + + rescue_from Grape::Exceptions::Base do |e| + error! e.message, e.status, e.headers + end + rescue_from :all do |exception| handle_api_exception(exception) end diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index ed87a2603e8..c4bdef781f7 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -16,6 +16,13 @@ module Ci not_found! unless current_runner.active? update_runner_info + if current_runner.is_runner_queue_value_latest?(params[:last_update]) + header 'X-GitLab-Last-Update', params[:last_update] + return build_not_found! + end + + new_update = current_runner.ensure_runner_queue_value + build = Ci::RegisterBuildService.new.execute(current_runner) if build @@ -26,6 +33,8 @@ module Ci else Gitlab::Metrics.add_event(:build_not_found) + header 'X-GitLab-Last-Update', new_update + build_not_found! end end @@ -41,7 +50,7 @@ module Ci put ":id" do authenticate_runner! build = Ci::Build.where(runner_id: current_runner.id).running.find(params[:id]) - forbidden!('Build has been erased!') if build.erased? + validate_build!(build) update_runner_info @@ -71,9 +80,7 @@ module Ci # PATCH /builds/:id/trace.txt patch ":id/trace.txt" do build = Ci::Build.find_by_id(params[:id]) - not_found! unless build - authenticate_build_token!(build) - forbidden!('Build has been erased!') if build.erased? + authenticate_build!(build) error!('400 Missing header Content-Range', 400) unless request.headers.has_key?('Content-Range') content_range = request.headers['Content-Range'] @@ -104,8 +111,7 @@ module Ci Gitlab::Workhorse.verify_api_request!(headers) not_allowed! unless Gitlab.config.artifacts.enabled build = Ci::Build.find_by_id(params[:id]) - not_found! unless build - authenticate_build_token!(build) + authenticate_build!(build) forbidden!('build is not running') unless build.running? if params[:filesize] @@ -142,10 +148,8 @@ module Ci require_gitlab_workhorse! not_allowed! unless Gitlab.config.artifacts.enabled build = Ci::Build.find_by_id(params[:id]) - not_found! unless build - authenticate_build_token!(build) + authenticate_build!(build) forbidden!('Build is not running!') unless build.running? - forbidden!('Build has been erased!') if build.erased? artifacts_upload_path = ArtifactUploader.artifacts_upload_path artifacts = uploaded_file(:file, artifacts_upload_path) @@ -176,8 +180,7 @@ module Ci # GET /builds/:id/artifacts get ":id/artifacts" do build = Ci::Build.find_by_id(params[:id]) - not_found! unless build - authenticate_build_token!(build) + authenticate_build!(build) artifacts_file = build.artifacts_file unless artifacts_file.file_storage? @@ -202,8 +205,7 @@ module Ci # DELETE /builds/:id/artifacts delete ":id/artifacts" do build = Ci::Build.find_by_id(params[:id]) - not_found! unless build - authenticate_build_token!(build) + authenticate_build!(build) build.erase_artifacts! end diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb index 66c05773b68..792ff628b09 100644 --- a/lib/ci/api/entities.rb +++ b/lib/ci/api/entities.rb @@ -32,6 +32,10 @@ module Ci expose :artifacts_file, using: ArtifactFile, if: ->(build, _) { build.artifacts? } end + class BuildCredentials < Grape::Entity + expose :type, :url, :username, :password + end + class BuildDetails < Build expose :commands expose :repo_url @@ -50,6 +54,8 @@ module Ci expose :variables expose :depends_on_builds, using: Build + + expose :credentials, using: BuildCredentials end class Runner < Grape::Entity diff --git a/lib/ci/api/helpers.rb b/lib/ci/api/helpers.rb index e608f5f6cad..5ff25a3a9b2 100644 --- a/lib/ci/api/helpers.rb +++ b/lib/ci/api/helpers.rb @@ -13,8 +13,19 @@ module Ci forbidden! unless current_runner end - def authenticate_build_token!(build) - forbidden! unless build_token_valid?(build) + def authenticate_build!(build) + validate_build!(build) do + forbidden! unless build_token_valid?(build) + end + end + + def validate_build!(build) + not_found! unless build + + yield if block_given? + + forbidden!('Project has been deleted!') unless build.project + forbidden!('Build has been erased!') if build.erased? end def runner_registration_token_valid? @@ -49,7 +60,7 @@ module Ci end def build_not_found! - if headers['User-Agent'].match(/gitlab-ci-multi-runner \d+\.\d+\.\d+(~beta\.\d+\.g[0-9a-f]+)? /) + if headers['User-Agent'].to_s.match(/gitlab-ci-multi-runner \d+\.\d+\.\d+(~beta\.\d+\.g[0-9a-f]+)? /) no_content! else not_found! diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 3e33c9399e2..7463bd719d5 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -2,7 +2,7 @@ module Ci class GitlabCiYamlProcessor class ValidationError < StandardError; end - include Gitlab::Ci::Config::Node::LegacyValidationHelpers + include Gitlab::Ci::Config::Entry::LegacyValidationHelpers attr_reader :path, :cache, :stages, :jobs @@ -118,7 +118,7 @@ module Ci .merge(job_variables(name)) variables.map do |key, value| - { key: key, value: value, public: true } + { key: key.to_s, value: value, public: true } end end diff --git a/lib/constraints/group_url_constrainer.rb b/lib/constraints/group_url_constrainer.rb index ca39b1961ae..bae4db1ca4d 100644 --- a/lib/constraints/group_url_constrainer.rb +++ b/lib/constraints/group_url_constrainer.rb @@ -1,7 +1,17 @@ -require 'constraints/namespace_url_constrainer' +class GroupUrlConstrainer + def matches?(request) + id = request.params[:id] -class GroupUrlConstrainer < NamespaceUrlConstrainer - def find_resource(id) - Group.find_by_path(id) + return false unless valid?(id) + + Group.find_by_full_path(id).present? + end + + private + + def valid?(id) + id.split('/').all? do |namespace| + NamespaceValidator.valid?(namespace) + end end end diff --git a/lib/constraints/namespace_url_constrainer.rb b/lib/constraints/namespace_url_constrainer.rb deleted file mode 100644 index 91b70143f11..00000000000 --- a/lib/constraints/namespace_url_constrainer.rb +++ /dev/null @@ -1,24 +0,0 @@ -class NamespaceUrlConstrainer - def matches?(request) - id = request.path - id = id.sub(/\A#{relative_url_root}/, '') if relative_url_root - id = id.sub(/\A\/+/, '').split('/').first - id = id.sub(/.atom\z/, '') if id - - if id =~ Gitlab::Regex.namespace_regex - find_resource(id) - end - end - - def find_resource(id) - Namespace.find_by_path(id) - end - - private - - def relative_url_root - if defined?(Gitlab::Application.config.relative_url_root) - Gitlab::Application.config.relative_url_root - end - end -end diff --git a/lib/constraints/project_url_constrainer.rb b/lib/constraints/project_url_constrainer.rb new file mode 100644 index 00000000000..730b05bed97 --- /dev/null +++ b/lib/constraints/project_url_constrainer.rb @@ -0,0 +1,13 @@ +class ProjectUrlConstrainer + def matches?(request) + namespace_path = request.params[:namespace_id] + project_path = request.params[:project_id] || request.params[:id] + full_path = namespace_path + '/' + project_path + + unless ProjectPathValidator.valid?(project_path) + return false + end + + Project.find_with_namespace(full_path).present? + end +end diff --git a/lib/constraints/user_url_constrainer.rb b/lib/constraints/user_url_constrainer.rb index 504a0f5d93e..9ab5bcb12ff 100644 --- a/lib/constraints/user_url_constrainer.rb +++ b/lib/constraints/user_url_constrainer.rb @@ -1,7 +1,5 @@ -require 'constraints/namespace_url_constrainer' - -class UserUrlConstrainer < NamespaceUrlConstrainer - def find_resource(id) - User.find_by('lower(username) = ?', id.downcase) +class UserUrlConstrainer + def matches?(request) + User.find_by_username(request.params[:username]).present? end end diff --git a/lib/email_template_interceptor.rb b/lib/email_template_interceptor.rb new file mode 100644 index 00000000000..63f9f8d7a5a --- /dev/null +++ b/lib/email_template_interceptor.rb @@ -0,0 +1,13 @@ +# Read about interceptors in http://guides.rubyonrails.org/action_mailer_basics.html#intercepting-emails +class EmailTemplateInterceptor + include Gitlab::CurrentSettings + + def self.delivering_email(message) + # Remove HTML part if HTML emails are disabled. + unless current_application_settings.html_emails_enabled + message.parts.delete_if do |part| + part.content_type.start_with?('text/html') + end + end + end +end diff --git a/lib/event_filter.rb b/lib/event_filter.rb index 21f6a9a762b..515095af1c2 100644 --- a/lib/event_filter.rb +++ b/lib/event_filter.rb @@ -14,6 +14,10 @@ class EventFilter 'merged' end + def issue + 'issue' + end + def comments 'comments' end @@ -32,32 +36,20 @@ class EventFilter end def apply_filter(events) - return events unless params.present? - - filter = params.dup - actions = [] + return events if params.blank? || params == EventFilter.all - case filter + case params when EventFilter.push - actions = [Event::PUSHED] + events.where(action: Event::PUSHED) when EventFilter.merged - actions = [Event::MERGED] + events.where(action: Event::MERGED) when EventFilter.comments - actions = [Event::COMMENTED] + events.where(action: Event::COMMENTED) when EventFilter.team - actions = [Event::JOINED, Event::LEFT, Event::EXPIRED] - when EventFilter.all - actions = [ - Event::PUSHED, - Event::MERGED, - Event::COMMENTED, - Event::JOINED, - Event::LEFT, - Event::EXPIRED - ] + events.where(action: [Event::JOINED, Event::LEFT, Event::EXPIRED]) + when EventFilter.issue + events.where(action: [Event::CREATED, Event::UPDATED, Event::CLOSED, Event::REOPENED]) end - - events.where(action: actions) end def options(key) @@ -73,6 +65,10 @@ class EventFilter end def active?(key) - params.include? key + if params.present? + params.include? key + else + key == EventFilter.all + end end end diff --git a/lib/extracts_path.rb b/lib/extracts_path.rb index 9b74364849e..82551f1f222 100644 --- a/lib/extracts_path.rb +++ b/lib/extracts_path.rb @@ -106,7 +106,7 @@ module ExtractsPath # resolved (e.g., when a user inserts an invalid path or ref). def assign_ref_vars # assign allowed options - allowed_options = ["filter_ref", "extended_sha1"] + allowed_options = ["filter_ref"] @options = params.select {|key, value| allowed_options.include?(key) && !value.blank? } @options = HashWithIndifferentAccess.new(@options) @@ -114,17 +114,13 @@ module ExtractsPath @ref, @path = extract_ref(@id) @repo = @project.repository - if @options[:extended_sha1].present? - @commit = @repo.commit(@options[:extended_sha1]) - else - @commit = @repo.commit(@ref) + @commit = @repo.commit(@ref) - if @path.empty? && !@commit && @id.ends_with?('.atom') - @id = @ref = extract_ref_without_atom(@id) - @commit = @repo.commit(@ref) + if @path.empty? && !@commit && @id.ends_with?('.atom') + @id = @ref = extract_ref_without_atom(@id) + @commit = @repo.commit(@ref) - request.format = :atom if @commit - end + request.format = :atom if @commit end raise InvalidPathError unless @commit diff --git a/lib/gitlab/allowable.rb b/lib/gitlab/allowable.rb new file mode 100644 index 00000000000..f48abcc86d5 --- /dev/null +++ b/lib/gitlab/allowable.rb @@ -0,0 +1,7 @@ +module Gitlab + module Allowable + def can?(user, action, subject) + Ability.allowed?(user, action, subject) + end + end +end diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb index 1a22ad9acf5..0618107e2c3 100644 --- a/lib/gitlab/asciidoc.rb +++ b/lib/gitlab/asciidoc.rb @@ -1,4 +1,6 @@ require 'asciidoctor' +require 'asciidoctor/converter/html5' +require "asciidoctor-plantuml" module Gitlab # Parser/renderer for the AsciiDoc format that uses Asciidoctor and filters @@ -6,7 +8,7 @@ module Gitlab module Asciidoc DEFAULT_ADOC_ATTRS = [ 'showtitle', 'idprefix=user-content-', 'idseparator=-', 'env=gitlab', - 'env-gitlab', 'source-highlighter=html-pipeline' + 'env-gitlab', 'source-highlighter=html-pipeline', 'icons=font' ].freeze # Public: Converts the provided Asciidoc markup into HTML. @@ -23,16 +25,51 @@ module Gitlab def self.render(input, context, asciidoc_opts = {}) asciidoc_opts.reverse_merge!( safe: :secure, - backend: :html5, + backend: :gitlab_html5, attributes: [] ) asciidoc_opts[:attributes].unshift(*DEFAULT_ADOC_ATTRS) + plantuml_setup + html = ::Asciidoctor.convert(input, asciidoc_opts) html = Banzai.post_process(html, context) html.html_safe end + + def self.plantuml_setup + Asciidoctor::PlantUml.configure do |conf| + conf.url = ApplicationSetting.current.plantuml_url + conf.svg_enable = ApplicationSetting.current.plantuml_enabled + conf.png_enable = ApplicationSetting.current.plantuml_enabled + conf.txt_enable = false + end + end + + class Html5Converter < Asciidoctor::Converter::Html5Converter + extend Asciidoctor::Converter::Config + + register_for 'gitlab_html5' + + def stem(node) + return super unless node.style.to_sym == :latexmath + + %(<pre#{id_attribute(node)} class="code math js-render-math #{node.role}" data-math-style="display"><code>#{node.content}</code></pre>) + end + + def inline_quoted(node) + return super unless node.type.to_sym == :latexmath + + %(<code#{id_attribute(node)} class="code math js-render-math #{node.role}" data-math-style="inline">#{node.text}</code>) + end + + private + + def id_attribute(node) + node.id ? %( id="#{node.id}") : nil + end + end end end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index aca5d0020cf..8dda65c71ef 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -2,6 +2,10 @@ module Gitlab module Auth class MissingPersonalTokenError < StandardError; end + SCOPES = [:api, :read_user] + DEFAULT_SCOPES = [:api] + OPTIONAL_SCOPES = SCOPES - DEFAULT_SCOPES + class << self def find_for_git_client(login, password, project:, ip:) raise "Must provide an IP for rate limiting" if ip.nil? @@ -88,7 +92,7 @@ module Gitlab def oauth_access_token_check(login, password) if login == "oauth2" && password.present? token = Doorkeeper::AccessToken.by_token(password) - if token && token.accessible? + if valid_oauth_token?(token) user = User.find_by(id: token.resource_owner_id) Gitlab::Auth::Result.new(user, nil, :oauth, read_authentication_abilities) end @@ -97,12 +101,27 @@ module Gitlab def personal_access_token_check(login, password) if login && password - user = User.find_by_personal_access_token(password) + token = PersonalAccessToken.active.find_by_token(password) validation = User.by_login(login) - Gitlab::Auth::Result.new(user, nil, :personal_token, full_authentication_abilities) if user.present? && user == validation + + if valid_personal_access_token?(token, validation) + Gitlab::Auth::Result.new(validation, nil, :personal_token, full_authentication_abilities) + end end end + def valid_oauth_token?(token) + token && token.accessible? && valid_api_token?(token) + end + + def valid_personal_access_token?(token, user) + token && token.user == user && valid_api_token?(token) + end + + def valid_api_token?(token) + AccessTokenValidationService.new(token).include_any_scope?(['api']) + end + def lfs_token_check(login, password) deploy_key_matches = login.match(/\Alfs\+deploy-key-(\d+)\z/) diff --git a/lib/gitlab/auth/result.rb b/lib/gitlab/auth/result.rb index 6be7f690676..39b86c61a18 100644 --- a/lib/gitlab/auth/result.rb +++ b/lib/gitlab/auth/result.rb @@ -9,8 +9,7 @@ module Gitlab def lfs_deploy_token?(for_project) type == :lfs_deploy_token && - actor && - actor.projects.include?(for_project) + actor.try(:has_access_to?, for_project) end def success? diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb index 9cec71a3222..82e194c1af1 100644 --- a/lib/gitlab/backend/shell.rb +++ b/lib/gitlab/backend/shell.rb @@ -127,19 +127,6 @@ module Gitlab 'rm-project', storage, "#{name}.git"]) end - # Gc repository - # - # storage - project storage path - # path - project path with namespace - # - # Ex. - # gc("/path/to/storage", "gitlab/gitlab-ci") - # - def gc(storage, path) - Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'gc', - storage, "#{path}.git"]) - end - # Add new key to gitlab-shell # # Ex. diff --git a/lib/gitlab/badge/build/status.rb b/lib/gitlab/badge/build/status.rb index 50aa45e5406..b762d85b6e5 100644 --- a/lib/gitlab/badge/build/status.rb +++ b/lib/gitlab/badge/build/status.rb @@ -20,8 +20,8 @@ module Gitlab def status @project.pipelines - .where(sha: @sha, ref: @ref) - .status || 'unknown' + .where(sha: @sha) + .latest_status(@ref) || 'unknown' end def metadata diff --git a/lib/gitlab/bitbucket_import.rb b/lib/gitlab/bitbucket_import.rb deleted file mode 100644 index 7298152e7e9..00000000000 --- a/lib/gitlab/bitbucket_import.rb +++ /dev/null @@ -1,6 +0,0 @@ -module Gitlab - module BitbucketImport - mattr_accessor :public_key - @public_key = nil - end -end diff --git a/lib/gitlab/bitbucket_import/client.rb b/lib/gitlab/bitbucket_import/client.rb deleted file mode 100644 index 8d1ad62fae0..00000000000 --- a/lib/gitlab/bitbucket_import/client.rb +++ /dev/null @@ -1,142 +0,0 @@ -module Gitlab - module BitbucketImport - class Client - class Unauthorized < StandardError; end - - attr_reader :consumer, :api - - def self.from_project(project) - import_data_credentials = project.import_data.credentials if project.import_data - if import_data_credentials && import_data_credentials[:bb_session] - token = import_data_credentials[:bb_session][:bitbucket_access_token] - token_secret = import_data_credentials[:bb_session][:bitbucket_access_token_secret] - new(token, token_secret) - else - raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{project.id}" - end - end - - def initialize(access_token = nil, access_token_secret = nil) - @consumer = ::OAuth::Consumer.new( - config.app_id, - config.app_secret, - bitbucket_options - ) - - if access_token && access_token_secret - @api = ::OAuth::AccessToken.new(@consumer, access_token, access_token_secret) - end - end - - def request_token(redirect_uri) - request_token = consumer.get_request_token(oauth_callback: redirect_uri) - - { - oauth_token: request_token.token, - oauth_token_secret: request_token.secret, - oauth_callback_confirmed: request_token.callback_confirmed?.to_s - } - end - - def authorize_url(request_token, redirect_uri) - request_token = ::OAuth::RequestToken.from_hash(consumer, request_token) if request_token.is_a?(Hash) - - if request_token.callback_confirmed? - request_token.authorize_url - else - request_token.authorize_url(oauth_callback: redirect_uri) - end - end - - def get_token(request_token, oauth_verifier, redirect_uri) - request_token = ::OAuth::RequestToken.from_hash(consumer, request_token) if request_token.is_a?(Hash) - - if request_token.callback_confirmed? - request_token.get_access_token(oauth_verifier: oauth_verifier) - else - request_token.get_access_token(oauth_callback: redirect_uri) - end - end - - def user - JSON.parse(get("/api/1.0/user").body) - end - - def issues(project_identifier) - all_issues = [] - offset = 0 - per_page = 50 # Maximum number allowed by Bitbucket - index = 0 - - begin - issues = JSON.parse(get(issue_api_endpoint(project_identifier, per_page, offset)).body) - # Find out how many total issues are present - total = issues["count"] if index == 0 - all_issues.concat(issues["issues"]) - offset += issues["issues"].count - index += 1 - end while all_issues.count < total - - all_issues - end - - def issue_comments(project_identifier, issue_id) - comments = JSON.parse(get("/api/1.0/repositories/#{project_identifier}/issues/#{issue_id}/comments").body) - comments.sort_by { |comment| comment["utc_created_on"] } - end - - def project(project_identifier) - JSON.parse(get("/api/1.0/repositories/#{project_identifier}").body) - end - - def find_deploy_key(project_identifier, key) - JSON.parse(get("/api/1.0/repositories/#{project_identifier}/deploy-keys").body).find do |deploy_key| - deploy_key["key"].chomp == key.chomp - end - end - - def add_deploy_key(project_identifier, key) - deploy_key = find_deploy_key(project_identifier, key) - return if deploy_key - - JSON.parse(api.post("/api/1.0/repositories/#{project_identifier}/deploy-keys", key: key, label: "GitLab import key").body) - end - - def delete_deploy_key(project_identifier, key) - deploy_key = find_deploy_key(project_identifier, key) - return unless deploy_key - - api.delete("/api/1.0/repositories/#{project_identifier}/deploy-keys/#{deploy_key["pk"]}").code == "204" - end - - def projects - JSON.parse(get("/api/1.0/user/repositories").body).select { |repo| repo["scm"] == "git" } - end - - def incompatible_projects - JSON.parse(get("/api/1.0/user/repositories").body).reject { |repo| repo["scm"] == "git" } - end - - private - - def get(url) - response = api.get(url) - raise Unauthorized if (400..499).cover?(response.code.to_i) - - response - end - - def issue_api_endpoint(project_identifier, per_page, offset) - "/api/1.0/repositories/#{project_identifier}/issues?sort=utc_created_on&limit=#{per_page}&start=#{offset}" - end - - def config - Gitlab.config.omniauth.providers.find { |provider| provider.name == "bitbucket" } - end - - def bitbucket_options - OmniAuth::Strategies::Bitbucket.default_options[:client_options].symbolize_keys - end - end - end -end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index f4b5097adb1..44323b47dca 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -1,84 +1,247 @@ module Gitlab module BitbucketImport class Importer - attr_reader :project, :client + include Gitlab::ShellAdapter + + LABELS = [{ title: 'bug', color: '#FF0000' }, + { title: 'enhancement', color: '#428BCA' }, + { title: 'proposal', color: '#69D100' }, + { title: 'task', color: '#7F8C8D' }].freeze + + attr_reader :project, :client, :errors, :users def initialize(project) @project = project - @client = Client.from_project(@project) + @client = Bitbucket::Client.new(project.import_data.credentials) @formatter = Gitlab::ImportFormatter.new + @labels = {} + @errors = [] + @users = {} end def execute - import_issues if has_issues? + import_wiki + import_issues + import_pull_requests + handle_errors true - rescue ActiveRecord::RecordInvalid => e - raise Projects::ImportService::Error.new, e.message - ensure - Gitlab::BitbucketImport::KeyDeleter.new(project).execute end private - def gitlab_user_id(project, bitbucket_id) - if bitbucket_id - user = User.joins(:identities).find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", bitbucket_id.to_s) - (user && user.id) || project.creator_id - else - project.creator_id - end + def handle_errors + return unless errors.any? + + project.update_column(:import_error, { + message: 'The remote data could not be fully imported.', + errors: errors + }.to_json) + end + + def gitlab_user_id(project, username) + find_user_id(username) || project.creator_id end - def identifier - project.import_source + def find_user_id(username) + return nil unless username + + return users[username] if users.key?(username) + + users[username] = User.select(:id) + .joins(:identities) + .find_by("identities.extern_uid = ? AND identities.provider = 'bitbucket'", username) + .try(:id) end - def has_issues? - client.project(identifier)["has_issues"] + def repo + @repo ||= client.repo(project.import_source) end - def import_issues - issues = client.issues(identifier) + def import_wiki + return if project.wiki.repository_exists? - issues.each do |issue| - body = '' - reporter = nil - author = 'Anonymous' + path_with_namespace = "#{project.path_with_namespace}.wiki" + import_url = project.import_url.sub(/\.git\z/, ".git/wiki") + gitlab_shell.import_repository(project.repository_storage_path, path_with_namespace, import_url) + rescue StandardError => e + errors << { type: :wiki, errors: e.message } + end - if issue["reported_by"] && issue["reported_by"]["username"] - reporter = issue["reported_by"]["username"] - author = reporter + def import_issues + return unless repo.issues_enabled? + + create_labels + + client.issues(repo).each do |issue| + begin + description = '' + description += @formatter.author_line(issue.author) unless find_user_id(issue.author) + description += issue.description + + label_name = issue.kind + milestone = issue.milestone ? project.milestones.find_or_create_by(title: issue.milestone) : nil + + gitlab_issue = project.issues.create!( + iid: issue.iid, + title: issue.title, + description: description, + state: issue.state, + author_id: gitlab_user_id(project, issue.author), + milestone: milestone, + created_at: issue.created_at, + updated_at: issue.updated_at + ) + + gitlab_issue.labels << @labels[label_name] + + import_issue_comments(issue, gitlab_issue) if gitlab_issue.persisted? + rescue StandardError => e + errors << { type: :issue, iid: issue.iid, errors: e.message } end + end + end - body = @formatter.author_line(author) - body += issue["content"] + def import_issue_comments(issue, gitlab_issue) + client.issue_comments(repo, issue.iid).each do |comment| + # The note can be blank for issue service messages like "Changed title: ..." + # We would like to import those comments as well but there is no any + # specific parameter that would allow to process them, it's just an empty comment. + # To prevent our importer from just crashing or from creating useless empty comments + # we do this check. + next unless comment.note.present? + + note = '' + note += @formatter.author_line(comment.author) unless find_user_id(comment.author) + note += comment.note + + begin + gitlab_issue.notes.create!( + project: project, + note: note, + author_id: gitlab_user_id(project, comment.author), + created_at: comment.created_at, + updated_at: comment.updated_at + ) + rescue StandardError => e + errors << { type: :issue_comment, iid: issue.iid, errors: e.message } + end + end + end - comments = client.issue_comments(identifier, issue["local_id"]) + def create_labels + LABELS.each do |label| + @labels[label[:title]] = project.labels.create!(label) + end + end - if comments.any? - body += @formatter.comments_header + def import_pull_requests + pull_requests = client.pull_requests(repo) + + pull_requests.each do |pull_request| + begin + description = '' + description += @formatter.author_line(pull_request.author) unless find_user_id(pull_request.author) + description += pull_request.description + + merge_request = project.merge_requests.create( + iid: pull_request.iid, + title: pull_request.title, + description: description, + source_project: project, + source_branch: pull_request.source_branch_name, + source_branch_sha: pull_request.source_branch_sha, + target_project: project, + target_branch: pull_request.target_branch_name, + target_branch_sha: pull_request.target_branch_sha, + state: pull_request.state, + author_id: gitlab_user_id(project, pull_request.author), + assignee_id: nil, + created_at: pull_request.created_at, + updated_at: pull_request.updated_at + ) + + import_pull_request_comments(pull_request, merge_request) if merge_request.persisted? + rescue StandardError => e + errors << { type: :pull_request, iid: pull_request.iid, errors: e.message } end + end + end + + def import_pull_request_comments(pull_request, merge_request) + comments = client.pull_request_comments(repo, pull_request.iid) + + inline_comments, pr_comments = comments.partition(&:inline?) + + import_inline_comments(inline_comments, pull_request, merge_request) + import_standalone_pr_comments(pr_comments, merge_request) + end - comments.each do |comment| - author = 'Anonymous' + def import_inline_comments(inline_comments, pull_request, merge_request) + line_code_map = {} - if comment["author_info"] && comment["author_info"]["username"] - author = comment["author_info"]["username"] - end + children, parents = inline_comments.partition(&:has_parent?) - body += @formatter.comment(author, comment["utc_created_on"], comment["content"]) + # The Bitbucket API returns threaded replies as parent-child + # relationships. We assume that the child can appear in any order in + # the JSON. + parents.each do |comment| + line_code_map[comment.iid] = generate_line_code(comment) + end + + children.each do |comment| + line_code_map[comment.iid] = line_code_map.fetch(comment.parent_id, nil) + end + + inline_comments.each do |comment| + begin + attributes = pull_request_comment_attributes(comment) + attributes.merge!( + position: build_position(merge_request, comment), + line_code: line_code_map.fetch(comment.iid), + type: 'DiffNote') + + merge_request.notes.create!(attributes) + rescue StandardError => e + errors << { type: :pull_request, iid: comment.iid, errors: e.message } end + end + end + + def build_position(merge_request, pr_comment) + params = { + diff_refs: merge_request.diff_refs, + old_path: pr_comment.file_path, + new_path: pr_comment.file_path, + old_line: pr_comment.old_pos, + new_line: pr_comment.new_pos + } - project.issues.create!( - description: body, - title: issue["title"], - state: %w(resolved invalid duplicate wontfix closed).include?(issue["status"]) ? 'closed' : 'opened', - author_id: gitlab_user_id(project, reporter) - ) + Gitlab::Diff::Position.new(params) + end + + def import_standalone_pr_comments(pr_comments, merge_request) + pr_comments.each do |comment| + begin + merge_request.notes.create!(pull_request_comment_attributes(comment)) + rescue StandardError => e + errors << { type: :pull_request, iid: comment.iid, errors: e.message } + end end - rescue ActiveRecord::RecordInvalid => e - raise Projects::ImportService::Error, e.message + end + + def generate_line_code(pr_comment) + Gitlab::Diff::LineCode.generate(pr_comment.file_path, pr_comment.new_pos, pr_comment.old_pos) + end + + def pull_request_comment_attributes(comment) + { + project: project, + note: comment.note, + author_id: gitlab_user_id(project, comment.author), + created_at: comment.created_at, + updated_at: comment.updated_at + } end end end diff --git a/lib/gitlab/bitbucket_import/key_adder.rb b/lib/gitlab/bitbucket_import/key_adder.rb deleted file mode 100644 index 0b63f025d0a..00000000000 --- a/lib/gitlab/bitbucket_import/key_adder.rb +++ /dev/null @@ -1,24 +0,0 @@ -module Gitlab - module BitbucketImport - class KeyAdder - attr_reader :repo, :current_user, :client - - def initialize(repo, current_user, access_params) - @repo, @current_user = repo, current_user - @client = Client.new(access_params[:bitbucket_access_token], - access_params[:bitbucket_access_token_secret]) - end - - def execute - return false unless BitbucketImport.public_key.present? - - project_identifier = "#{repo["owner"]}/#{repo["slug"]}" - client.add_deploy_key(project_identifier, BitbucketImport.public_key) - - true - rescue - false - end - end - end -end diff --git a/lib/gitlab/bitbucket_import/key_deleter.rb b/lib/gitlab/bitbucket_import/key_deleter.rb deleted file mode 100644 index e03c3155b3e..00000000000 --- a/lib/gitlab/bitbucket_import/key_deleter.rb +++ /dev/null @@ -1,23 +0,0 @@ -module Gitlab - module BitbucketImport - class KeyDeleter - attr_reader :project, :current_user, :client - - def initialize(project) - @project = project - @current_user = project.creator - @client = Client.from_project(@project) - end - - def execute - return false unless BitbucketImport.public_key.present? - - client.delete_deploy_key(project.import_source, BitbucketImport.public_key) - - true - rescue - false - end - end - end -end diff --git a/lib/gitlab/bitbucket_import/project_creator.rb b/lib/gitlab/bitbucket_import/project_creator.rb index b90ef0b0fba..d94f70fd1fb 100644 --- a/lib/gitlab/bitbucket_import/project_creator.rb +++ b/lib/gitlab/bitbucket_import/project_creator.rb @@ -1,10 +1,11 @@ module Gitlab module BitbucketImport class ProjectCreator - attr_reader :repo, :namespace, :current_user, :session_data + attr_reader :repo, :name, :namespace, :current_user, :session_data - def initialize(repo, namespace, current_user, session_data) + def initialize(repo, name, namespace, current_user, session_data) @repo = repo + @name = name @namespace = namespace @current_user = current_user @session_data = session_data @@ -13,17 +14,24 @@ module Gitlab def execute ::Projects::CreateService.new( current_user, - name: repo["name"], - path: repo["slug"], - description: repo["description"], + name: name, + path: name, + description: repo.description, namespace_id: namespace.id, - visibility_level: repo["is_private"] ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::VisibilityLevel::PUBLIC, - import_type: "bitbucket", - import_source: "#{repo["owner"]}/#{repo["slug"]}", - import_url: "ssh://git@bitbucket.org/#{repo["owner"]}/#{repo["slug"]}.git", - import_data: { credentials: { bb_session: session_data } } + visibility_level: repo.visibility_level, + import_type: 'bitbucket', + import_source: repo.full_name, + import_url: repo.clone_url(session_data[:token]), + import_data: { credentials: session_data }, + skip_wiki: skip_wiki ).execute end + + private + + def skip_wiki + repo.has_wiki? + end end end end diff --git a/lib/gitlab/chat_commands/base_command.rb b/lib/gitlab/chat_commands/base_command.rb new file mode 100644 index 00000000000..4fe53ce93a9 --- /dev/null +++ b/lib/gitlab/chat_commands/base_command.rb @@ -0,0 +1,51 @@ +module Gitlab + module ChatCommands + class BaseCommand + QUERY_LIMIT = 5 + + def self.match(_text) + raise NotImplementedError + end + + def self.help_message + raise NotImplementedError + end + + def self.available?(_project) + raise NotImplementedError + end + + def self.allowed?(_user, _ability) + true + end + + def self.can?(object, action, subject) + Ability.allowed?(object, action, subject) + end + + def execute(_) + raise NotImplementedError + end + + def collection + raise NotImplementedError + end + + attr_accessor :project, :current_user, :params + + def initialize(project, user, params = {}) + @project, @current_user, @params = project, user, params.dup + end + + private + + def find_by_iid(iid) + collection.find_by(iid: iid) + end + + def presenter + Gitlab::ChatCommands::Presenter.new + end + end + end +end diff --git a/lib/gitlab/chat_commands/command.rb b/lib/gitlab/chat_commands/command.rb new file mode 100644 index 00000000000..145086755e4 --- /dev/null +++ b/lib/gitlab/chat_commands/command.rb @@ -0,0 +1,63 @@ +module Gitlab + module ChatCommands + class Command < BaseCommand + COMMANDS = [ + Gitlab::ChatCommands::IssueShow, + Gitlab::ChatCommands::IssueCreate, + Gitlab::ChatCommands::IssueSearch, + Gitlab::ChatCommands::Deploy, + ].freeze + + def execute + command, match = match_command + + if command + if command.allowed?(project, current_user) + present command.new(project, current_user, params).execute(match) + else + access_denied + end + else + help(help_messages) + end + end + + def match_command + match = nil + service = available_commands.find do |klass| + match = klass.match(command) + end + + [service, match] + end + + private + + def help_messages + available_commands.map(&:help_message) + end + + def available_commands + COMMANDS.select do |klass| + klass.available?(project) + end + end + + def command + params[:text] + end + + def help(messages) + presenter.help(messages, params[:command]) + end + + def access_denied + presenter.access_denied + end + + def present(resource) + presenter.present(resource) + end + end + end +end diff --git a/lib/gitlab/chat_commands/deploy.rb b/lib/gitlab/chat_commands/deploy.rb new file mode 100644 index 00000000000..7127d2f6d04 --- /dev/null +++ b/lib/gitlab/chat_commands/deploy.rb @@ -0,0 +1,58 @@ +module Gitlab + module ChatCommands + class Deploy < BaseCommand + include Gitlab::Routing.url_helpers + + def self.match(text) + /\Adeploy\s+(?<from>\S+.*)\s+to+\s+(?<to>\S+.*)\z/.match(text) + end + + def self.help_message + 'deploy <environment> to <target-environment>' + end + + def self.available?(project) + project.builds_enabled? + end + + def self.allowed?(project, user) + can?(user, :create_deployment, project) + end + + def execute(match) + from = match[:from] + to = match[:to] + + actions = find_actions(from, to) + return unless actions.present? + + if actions.one? + play!(from, to, actions.first) + else + Result.new(:error, 'Too many actions defined') + end + end + + private + + def play!(from, to, action) + new_action = action.play(current_user) + + Result.new(:success, "Deployment from #{from} to #{to} started. Follow the progress: #{url(new_action)}.") + end + + def find_actions(from, to) + environment = project.environments.find_by(name: from) + return unless environment + + environment.actions_for(to).select(&:starts_environment?) + end + + def url(subject) + polymorphic_url( + [subject.project.namespace.becomes(Namespace), subject.project, subject] + ) + end + end + end +end diff --git a/lib/gitlab/chat_commands/issue_command.rb b/lib/gitlab/chat_commands/issue_command.rb new file mode 100644 index 00000000000..84de3e44c70 --- /dev/null +++ b/lib/gitlab/chat_commands/issue_command.rb @@ -0,0 +1,13 @@ +module Gitlab + module ChatCommands + class IssueCommand < BaseCommand + def self.available?(project) + project.issues_enabled? && project.default_issues_tracker? + end + + def collection + IssuesFinder.new(current_user, project_id: project.id).execute + end + end + end +end diff --git a/lib/gitlab/chat_commands/issue_create.rb b/lib/gitlab/chat_commands/issue_create.rb new file mode 100644 index 00000000000..cefb6775db8 --- /dev/null +++ b/lib/gitlab/chat_commands/issue_create.rb @@ -0,0 +1,26 @@ +module Gitlab + module ChatCommands + class IssueCreate < IssueCommand + def self.match(text) + # we can not match \n with the dot by passing the m modifier as than + # the title and description are not seperated + /\Aissue\s+(new|create)\s+(?<title>[^\n]*)\n*(?<description>(.|\n)*)/.match(text) + end + + def self.help_message + 'issue new <title> *`⇧ Shift`*+*`↵ Enter`* <description>' + end + + def self.allowed?(project, user) + can?(user, :create_issue, project) + end + + def execute(match) + title = match[:title] + description = match[:description].to_s.rstrip + + Issues::CreateService.new(project, current_user, title: title, description: description).execute + end + end + end +end diff --git a/lib/gitlab/chat_commands/issue_search.rb b/lib/gitlab/chat_commands/issue_search.rb new file mode 100644 index 00000000000..51bf80c800b --- /dev/null +++ b/lib/gitlab/chat_commands/issue_search.rb @@ -0,0 +1,17 @@ +module Gitlab + module ChatCommands + class IssueSearch < IssueCommand + def self.match(text) + /\Aissue\s+search\s+(?<query>.*)/.match(text) + end + + def self.help_message + "issue search <your query>" + end + + def execute(match) + collection.search(match[:query]).limit(QUERY_LIMIT) + end + end + end +end diff --git a/lib/gitlab/chat_commands/issue_show.rb b/lib/gitlab/chat_commands/issue_show.rb new file mode 100644 index 00000000000..2a45d49cf6b --- /dev/null +++ b/lib/gitlab/chat_commands/issue_show.rb @@ -0,0 +1,17 @@ +module Gitlab + module ChatCommands + class IssueShow < IssueCommand + def self.match(text) + /\Aissue\s+show\s+#{Issue.reference_prefix}?(?<iid>\d+)/.match(text) + end + + def self.help_message + "issue show <id>" + end + + def execute(match) + find_by_iid(match[:iid]) + end + end + end +end diff --git a/lib/gitlab/chat_commands/presenter.rb b/lib/gitlab/chat_commands/presenter.rb new file mode 100644 index 00000000000..8930a21f406 --- /dev/null +++ b/lib/gitlab/chat_commands/presenter.rb @@ -0,0 +1,131 @@ +module Gitlab + module ChatCommands + class Presenter + include Gitlab::Routing + + def authorize_chat_name(url) + message = if url + ":wave: Hi there! Before I do anything for you, please [connect your GitLab account](#{url})." + else + ":sweat_smile: Couldn't identify you, nor can I autorize you!" + end + + ephemeral_response(message) + end + + def help(commands, trigger) + if commands.none? + ephemeral_response("No commands configured") + else + commands.map! { |command| "#{trigger} #{command}" } + message = header_with_list("Available commands", commands) + + ephemeral_response(message) + end + end + + def present(subject) + return not_found unless subject + + if subject.is_a?(Gitlab::ChatCommands::Result) + show_result(subject) + elsif subject.respond_to?(:count) + if subject.none? + not_found + elsif subject.one? + single_resource(subject.first) + else + multiple_resources(subject) + end + else + single_resource(subject) + end + end + + def access_denied + ephemeral_response("Whoops! That action is not allowed. This incident will be [reported](https://xkcd.com/838/).") + end + + private + + def show_result(result) + case result.type + when :success + in_channel_response(result.message) + else + ephemeral_response(result.message) + end + end + + def not_found + ephemeral_response("404 not found! GitLab couldn't find what you were looking for! :boom:") + end + + def single_resource(resource) + return error(resource) if resource.errors.any? || !resource.persisted? + + message = "#{title(resource)}:" + message << "\n\n#{resource.description}" if resource.try(:description) + + in_channel_response(message) + end + + def multiple_resources(resources) + titles = resources.map { |resource| title(resource) } + + message = header_with_list("Multiple results were found:", titles) + + ephemeral_response(message) + end + + def error(resource) + message = header_with_list("The action was not successful, because:", resource.errors.messages) + + ephemeral_response(message) + end + + def title(resource) + reference = resource.try(:to_reference) || resource.try(:id) + title = resource.try(:title) || resource.try(:name) + + "[#{reference} #{title}](#{url(resource)})" + end + + def header_with_list(header, items) + message = [header] + + items.each do |item| + message << "- #{item}" + end + + message.join("\n") + end + + def url(resource) + url_for( + [ + resource.project.namespace.becomes(Namespace), + resource.project, + resource + ] + ) + end + + def ephemeral_response(message) + { + response_type: :ephemeral, + text: message, + status: 200 + } + end + + def in_channel_response(message) + { + response_type: :in_channel, + text: message, + status: 200 + } + end + end + end +end diff --git a/lib/gitlab/chat_commands/result.rb b/lib/gitlab/chat_commands/result.rb new file mode 100644 index 00000000000..324d7ef43a3 --- /dev/null +++ b/lib/gitlab/chat_commands/result.rb @@ -0,0 +1,5 @@ +module Gitlab + module ChatCommands + Result = Struct.new(:type, :message) + end +end diff --git a/lib/gitlab/chat_name_token.rb b/lib/gitlab/chat_name_token.rb new file mode 100644 index 00000000000..1b081aa9b1d --- /dev/null +++ b/lib/gitlab/chat_name_token.rb @@ -0,0 +1,45 @@ +require 'json' + +module Gitlab + class ChatNameToken + attr_reader :token + + TOKEN_LENGTH = 50 + EXPIRY_TIME = 10.minutes + + def initialize(token = new_token) + @token = token + end + + def get + Gitlab::Redis.with do |redis| + data = redis.get(redis_key) + JSON.parse(data, symbolize_names: true) if data + end + end + + def store!(params) + Gitlab::Redis.with do |redis| + params = params.to_json + redis.set(redis_key, params, ex: EXPIRY_TIME) + token + end + end + + def delete + Gitlab::Redis.with do |redis| + redis.del(redis_key) + end + end + + private + + def new_token + Devise.friendly_token(TOKEN_LENGTH) + end + + def redis_key + "gitlab:chat_names:#{token}" + end + end +end diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb index cb1065223d4..273118135a9 100644 --- a/lib/gitlab/checks/change_access.rb +++ b/lib/gitlab/checks/change_access.rb @@ -1,13 +1,16 @@ module Gitlab module Checks class ChangeAccess - attr_reader :user_access, :project + attr_reader :user_access, :project, :skip_authorization - def initialize(change, user_access:, project:) + def initialize( + change, user_access:, project:, env: {}, skip_authorization: false) @oldrev, @newrev, @ref = change.values_at(:oldrev, :newrev, :ref) @branch_name = Gitlab::Git.branch_name(@ref) @user_access = user_access @project = project + @env = env + @skip_authorization = skip_authorization end def exec @@ -23,12 +26,13 @@ module Gitlab protected def protected_branch_checks + return if skip_authorization return unless @branch_name return unless project.protected_branch?(@branch_name) - if forced_push? && user_access.cannot_do_action?(:force_push_code_to_protected_branches) + if forced_push? return "You are not allowed to force push code to a protected branch on this project." - elsif Gitlab::Git.blank_ref?(@newrev) && user_access.cannot_do_action?(:remove_protected_branches) + elsif Gitlab::Git.blank_ref?(@newrev) return "You are not allowed to delete protected branches from this project." end @@ -48,6 +52,8 @@ module Gitlab end def tag_checks + return if skip_authorization + tag_ref = Gitlab::Git.tag_name(@ref) if tag_ref && protected_tag?(tag_ref) && user_access.cannot_do_action?(:admin_project) @@ -56,6 +62,8 @@ module Gitlab end def push_checks + return if skip_authorization + if user_access.cannot_do_action?(:push_code) "You are not allowed to push code to this project." end @@ -68,7 +76,7 @@ module Gitlab end def forced_push? - Gitlab::Checks::ForcePush.force_push?(@project, @oldrev, @newrev) + Gitlab::Checks::ForcePush.force_push?(@project, @oldrev, @newrev, env: @env) end def matching_merge_request? diff --git a/lib/gitlab/checks/force_push.rb b/lib/gitlab/checks/force_push.rb index 5fe86553bd0..de0c9049ebf 100644 --- a/lib/gitlab/checks/force_push.rb +++ b/lib/gitlab/checks/force_push.rb @@ -1,15 +1,20 @@ module Gitlab module Checks class ForcePush - def self.force_push?(project, oldrev, newrev) + def self.force_push?(project, oldrev, newrev, env: {}) return false if project.empty_repo? # Created or deleted branch if Gitlab::Git.blank_ref?(oldrev) || Gitlab::Git.blank_ref?(newrev) false else - missed_ref, _ = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} --git-dir=#{project.repository.path_to_repo} rev-list --max-count=1 #{oldrev} ^#{newrev})) - missed_ref.present? + missed_ref, exit_status = Gitlab::Git::RevList.new(oldrev, newrev, project: project, env: env).execute + + if exit_status == 0 + missed_ref.present? + else + raise "Got a non-zero exit code while calling out to `git rev-list` in the force-push check." + end end end end diff --git a/lib/gitlab/ci/build/credentials/base.rb b/lib/gitlab/ci/build/credentials/base.rb new file mode 100644 index 00000000000..29a7a27c963 --- /dev/null +++ b/lib/gitlab/ci/build/credentials/base.rb @@ -0,0 +1,13 @@ +module Gitlab + module Ci + module Build + module Credentials + class Base + def type + self.class.name.demodulize.underscore + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/credentials/factory.rb b/lib/gitlab/ci/build/credentials/factory.rb new file mode 100644 index 00000000000..2423aa8857d --- /dev/null +++ b/lib/gitlab/ci/build/credentials/factory.rb @@ -0,0 +1,27 @@ +module Gitlab + module Ci + module Build + module Credentials + class Factory + def initialize(build) + @build = build + end + + def create! + credentials.select(&:valid?) + end + + private + + def credentials + providers.map { |provider| provider.new(@build) } + end + + def providers + [Registry] + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/credentials/registry.rb b/lib/gitlab/ci/build/credentials/registry.rb new file mode 100644 index 00000000000..55eafcaed10 --- /dev/null +++ b/lib/gitlab/ci/build/credentials/registry.rb @@ -0,0 +1,24 @@ +module Gitlab + module Ci + module Build + module Credentials + class Registry < Base + attr_reader :username, :password + + def initialize(build) + @username = 'gitlab-ci-token' + @password = build.token + end + + def url + Gitlab.config.registry.host_port + end + + def valid? + Gitlab.config.registry.enabled + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index bbfa6cf7d05..f7ff7ea212e 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -4,16 +4,10 @@ module Gitlab # Base GitLab CI Configuration facade # class Config - ## - # Temporary delegations that should be removed after refactoring - # - delegate :before_script, :image, :services, :after_script, :variables, - :stages, :cache, :jobs, to: :@global - def initialize(config) @config = Loader.new(config).load! - @global = Node::Global.new(@config) + @global = Entry::Global.new(@config) @global.compose! end @@ -28,6 +22,41 @@ module Gitlab def to_hash @config end + + ## + # Temporary method that should be removed after refactoring + # + def before_script + @global.before_script_value + end + + def image + @global.image_value + end + + def services + @global.services_value + end + + def after_script + @global.after_script_value + end + + def variables + @global.variables_value + end + + def stages + @global.stages_value + end + + def cache + @global.cache_value + end + + def jobs + @global.jobs_value + end end end end diff --git a/lib/gitlab/ci/config/node/artifacts.rb b/lib/gitlab/ci/config/entry/artifacts.rb index 844bd2fe998..b756b0d4555 100644 --- a/lib/gitlab/ci/config/node/artifacts.rb +++ b/lib/gitlab/ci/config/entry/artifacts.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Entry that represents a configuration of job artifacts. # - class Artifacts < Entry + class Artifacts < Node include Validatable include Attributable diff --git a/lib/gitlab/ci/config/node/attributable.rb b/lib/gitlab/ci/config/entry/attributable.rb index 221b666f9f6..1c8b55ee4c4 100644 --- a/lib/gitlab/ci/config/node/attributable.rb +++ b/lib/gitlab/ci/config/entry/attributable.rb @@ -1,7 +1,7 @@ module Gitlab module Ci class Config - module Node + module Entry module Attributable extend ActiveSupport::Concern diff --git a/lib/gitlab/ci/config/node/boolean.rb b/lib/gitlab/ci/config/entry/boolean.rb index 84b03ee7832..f3357f85b99 100644 --- a/lib/gitlab/ci/config/node/boolean.rb +++ b/lib/gitlab/ci/config/entry/boolean.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Entry that represents a boolean value. # - class Boolean < Entry + class Boolean < Node include Validatable validations do diff --git a/lib/gitlab/ci/config/node/cache.rb b/lib/gitlab/ci/config/entry/cache.rb index b4bda2841ac..7653cab668b 100644 --- a/lib/gitlab/ci/config/node/cache.rb +++ b/lib/gitlab/ci/config/entry/cache.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Entry that represents a cache configuration # - class Cache < Entry + class Cache < Node include Configurable ALLOWED_KEYS = %i[key untracked paths] @@ -14,13 +14,13 @@ module Gitlab validates :config, allowed_keys: ALLOWED_KEYS end - node :key, Node::Key, + entry :key, Entry::Key, description: 'Cache key used to define a cache affinity.' - node :untracked, Node::Boolean, + entry :untracked, Entry::Boolean, description: 'Cache all untracked files.' - node :paths, Node::Paths, + entry :paths, Entry::Paths, description: 'Specify which paths should be cached across builds.' end end diff --git a/lib/gitlab/ci/config/node/commands.rb b/lib/gitlab/ci/config/entry/commands.rb index d7657ae314b..65d19db249c 100644 --- a/lib/gitlab/ci/config/node/commands.rb +++ b/lib/gitlab/ci/config/entry/commands.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Entry that represents a job script. # - class Commands < Entry + class Commands < Node include Validatable validations do diff --git a/lib/gitlab/ci/config/node/configurable.rb b/lib/gitlab/ci/config/entry/configurable.rb index 6b7ab2fdaf2..833ae4a0ff3 100644 --- a/lib/gitlab/ci/config/node/configurable.rb +++ b/lib/gitlab/ci/config/entry/configurable.rb @@ -1,7 +1,7 @@ module Gitlab module Ci class Config - module Node + module Entry ## # This mixin is responsible for adding DSL, which purpose is to # simplifly process of adding child nodes. @@ -48,8 +48,8 @@ module Gitlab private # rubocop:disable Lint/UselessAccessModifier - def node(key, node, metadata) - factory = Node::Factory.new(node) + def entry(key, entry, metadata) + factory = Entry::Factory.new(entry) .with(description: metadata[:description]) (@nodes ||= {}).merge!(key.to_sym => factory) @@ -66,8 +66,6 @@ module Gitlab @entries[symbol].value end - - alias_method symbol.to_sym, "#{symbol}_value".to_sym end end end diff --git a/lib/gitlab/ci/config/node/environment.rb b/lib/gitlab/ci/config/entry/environment.rb index 9a95ef43628..f7c530c7d9f 100644 --- a/lib/gitlab/ci/config/node/environment.rb +++ b/lib/gitlab/ci/config/entry/environment.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Entry that represents an environment. # - class Environment < Entry + class Environment < Node include Validatable ALLOWED_KEYS = %i[name url action on_stop] @@ -33,7 +33,6 @@ module Gitlab validates :url, length: { maximum: 255 }, - addressable_url: true, allow_nil: true validates :action, diff --git a/lib/gitlab/ci/config/node/factory.rb b/lib/gitlab/ci/config/entry/factory.rb index 5387f29ad59..9f5e393d191 100644 --- a/lib/gitlab/ci/config/node/factory.rb +++ b/lib/gitlab/ci/config/entry/factory.rb @@ -1,15 +1,15 @@ module Gitlab module Ci class Config - module Node + module Entry ## - # Factory class responsible for fabricating node entry objects. + # Factory class responsible for fabricating entry objects. # class Factory class InvalidFactory < StandardError; end - def initialize(node) - @node = node + def initialize(entry) + @entry = entry @metadata = {} @attributes = {} end @@ -37,11 +37,11 @@ module Gitlab # See issue #18775. # if @value.nil? - Node::Unspecified.new( + Entry::Unspecified.new( fabricate_unspecified ) else - fabricate(@node, @value) + fabricate(@entry, @value) end end @@ -49,21 +49,21 @@ module Gitlab def fabricate_unspecified ## - # If node has a default value we fabricate concrete node + # If entry has a default value we fabricate concrete node # with default value. # - if @node.default.nil? - fabricate(Node::Undefined) + if @entry.default.nil? + fabricate(Entry::Undefined) else - fabricate(@node, @node.default) + fabricate(@entry, @entry.default) end end - def fabricate(node, value = nil) - node.new(value, @metadata).tap do |entry| - entry.key = @attributes[:key] - entry.parent = @attributes[:parent] - entry.description = @attributes[:description] + def fabricate(entry, value = nil) + entry.new(value, @metadata).tap do |node| + node.key = @attributes[:key] + node.parent = @attributes[:parent] + node.description = @attributes[:description] end end end diff --git a/lib/gitlab/ci/config/node/global.rb b/lib/gitlab/ci/config/entry/global.rb index 2a2943c9288..a4ec8f0ff2f 100644 --- a/lib/gitlab/ci/config/node/global.rb +++ b/lib/gitlab/ci/config/entry/global.rb @@ -1,36 +1,36 @@ module Gitlab module Ci class Config - module Node + module Entry ## - # This class represents a global entry - root node for entire + # This class represents a global entry - root Entry for entire # GitLab CI Configuration file. # - class Global < Entry + class Global < Node include Configurable - node :before_script, Node::Script, + entry :before_script, Entry::Script, description: 'Script that will be executed before each job.' - node :image, Node::Image, + entry :image, Entry::Image, description: 'Docker image that will be used to execute jobs.' - node :services, Node::Services, + entry :services, Entry::Services, description: 'Docker images that will be linked to the container.' - node :after_script, Node::Script, + entry :after_script, Entry::Script, description: 'Script that will be executed after each job.' - node :variables, Node::Variables, + entry :variables, Entry::Variables, description: 'Environment variables that will be used.' - node :stages, Node::Stages, + entry :stages, Entry::Stages, description: 'Configuration of stages for this pipeline.' - node :types, Node::Stages, + entry :types, Entry::Stages, description: 'Deprecated: stages for this pipeline.' - node :cache, Node::Cache, + entry :cache, Entry::Cache, description: 'Configure caching between build jobs.' helpers :before_script, :image, :services, :after_script, @@ -46,7 +46,7 @@ module Gitlab private def compose_jobs! - factory = Node::Factory.new(Node::Jobs) + factory = Entry::Factory.new(Entry::Jobs) .value(@config.except(*self.class.nodes.keys)) .with(key: :jobs, parent: self, description: 'Jobs definition for this pipeline') diff --git a/lib/gitlab/ci/config/node/hidden.rb b/lib/gitlab/ci/config/entry/hidden.rb index fe4ee8a7fc6..6fc3aa385bc 100644 --- a/lib/gitlab/ci/config/node/hidden.rb +++ b/lib/gitlab/ci/config/entry/hidden.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## - # Entry that represents a hidden CI/CD job. + # Entry that represents a hidden CI/CD key. # - class Hidden < Entry + class Hidden < Node include Validatable validations do diff --git a/lib/gitlab/ci/config/node/image.rb b/lib/gitlab/ci/config/entry/image.rb index 5d3c7c5eab0..b5050257688 100644 --- a/lib/gitlab/ci/config/node/image.rb +++ b/lib/gitlab/ci/config/entry/image.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Entry that represents a Docker image. # - class Image < Entry + class Image < Node include Validatable validations do diff --git a/lib/gitlab/ci/config/node/job.rb b/lib/gitlab/ci/config/entry/job.rb index 603334d6793..a55362f0b6b 100644 --- a/lib/gitlab/ci/config/node/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Entry that represents a concrete CI/CD job. # - class Job < Entry + class Job < Node include Configurable include Attributable @@ -13,12 +13,10 @@ module Gitlab type stage when artifacts cache dependencies before_script after_script variables environment] - attributes :tags, :allow_failure, :when, :dependencies - validations do validates :config, allowed_keys: ALLOWED_KEYS - validates :config, presence: true + validates :script, presence: true validates :name, presence: true validates :name, type: Symbol @@ -34,49 +32,51 @@ module Gitlab end end - node :before_script, Node::Script, + entry :before_script, Entry::Script, description: 'Global before script overridden in this job.' - node :script, Node::Commands, + entry :script, Entry::Commands, description: 'Commands that will be executed in this job.' - node :stage, Node::Stage, + entry :stage, Entry::Stage, description: 'Pipeline stage this job will be executed into.' - node :type, Node::Stage, + entry :type, Entry::Stage, description: 'Deprecated: stage this job will be executed into.' - node :after_script, Node::Script, + entry :after_script, Entry::Script, description: 'Commands that will be executed when finishing job.' - node :cache, Node::Cache, + entry :cache, Entry::Cache, description: 'Cache definition for this job.' - node :image, Node::Image, + entry :image, Entry::Image, description: 'Image that will be used to execute this job.' - node :services, Node::Services, + entry :services, Entry::Services, description: 'Services that will be used to execute this job.' - node :only, Node::Trigger, + entry :only, Entry::Trigger, description: 'Refs policy this job will be executed for.' - node :except, Node::Trigger, + entry :except, Entry::Trigger, description: 'Refs policy this job will be executed for.' - node :variables, Node::Variables, + entry :variables, Entry::Variables, description: 'Environment variables available for this job.' - node :artifacts, Node::Artifacts, + entry :artifacts, Entry::Artifacts, description: 'Artifacts configuration for this job.' - node :environment, Node::Environment, + entry :environment, Entry::Environment, description: 'Environment configuration for this job.' helpers :before_script, :script, :stage, :type, :after_script, :cache, :image, :services, :only, :except, :variables, :artifacts, :commands, :environment + attributes :script, :tags, :allow_failure, :when, :dependencies + def compose!(deps = nil) super do if type_defined? && !stage_defined? @@ -108,7 +108,7 @@ module Gitlab self.class.nodes.each_key do |key| global_entry = deps[key] - job_entry = @entries[key] + job_entry = self[key] if global_entry.specified? && !job_entry.specified? @entries[key] = global_entry @@ -118,20 +118,20 @@ module Gitlab def to_hash { name: name, - before_script: before_script, - script: script, + before_script: before_script_value, + script: script_value, commands: commands, - image: image, - services: services, - stage: stage, - cache: cache, - only: only, - except: except, - variables: variables_defined? ? variables : nil, - environment: environment_defined? ? environment : nil, - environment_name: environment_defined? ? environment[:name] : nil, - artifacts: artifacts, - after_script: after_script } + image: image_value, + services: services_value, + stage: stage_value, + cache: cache_value, + only: only_value, + except: except_value, + variables: variables_defined? ? variables_value : nil, + environment: environment_defined? ? environment_value : nil, + environment_name: environment_defined? ? environment_value[:name] : nil, + artifacts: artifacts_value, + after_script: after_script_value } end end end diff --git a/lib/gitlab/ci/config/node/jobs.rb b/lib/gitlab/ci/config/entry/jobs.rb index d10e80d1a7d..5671a09480b 100644 --- a/lib/gitlab/ci/config/node/jobs.rb +++ b/lib/gitlab/ci/config/entry/jobs.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Entry that represents a set of jobs. # - class Jobs < Entry + class Jobs < Node include Validatable validations do @@ -29,9 +29,9 @@ module Gitlab def compose!(deps = nil) super do @config.each do |name, config| - node = hidden?(name) ? Node::Hidden : Node::Job + node = hidden?(name) ? Entry::Hidden : Entry::Job - factory = Node::Factory.new(node) + factory = Entry::Factory.new(node) .value(config || {}) .metadata(name: name) .with(key: name, parent: self, diff --git a/lib/gitlab/ci/config/node/key.rb b/lib/gitlab/ci/config/entry/key.rb index f8b461ca098..0e4c9fe6edc 100644 --- a/lib/gitlab/ci/config/node/key.rb +++ b/lib/gitlab/ci/config/entry/key.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Entry that represents a key. # - class Key < Entry + class Key < Node include Validatable validations do diff --git a/lib/gitlab/ci/config/node/legacy_validation_helpers.rb b/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb index 0c291efe6a5..f01975aab5c 100644 --- a/lib/gitlab/ci/config/node/legacy_validation_helpers.rb +++ b/lib/gitlab/ci/config/entry/legacy_validation_helpers.rb @@ -1,7 +1,7 @@ module Gitlab module Ci class Config - module Node + module Entry module LegacyValidationHelpers private diff --git a/lib/gitlab/ci/config/node/entry.rb b/lib/gitlab/ci/config/entry/node.rb index 8717eabf81e..5eef2868cd6 100644 --- a/lib/gitlab/ci/config/node/entry.rb +++ b/lib/gitlab/ci/config/entry/node.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Base abstract class for each configuration entry node. # - class Entry + class Node class InvalidError < StandardError; end attr_reader :config, :metadata @@ -21,7 +21,7 @@ module Gitlab end def [](key) - @entries[key] || Node::Undefined.new + @entries[key] || Entry::Undefined.new end def compose!(deps = nil) diff --git a/lib/gitlab/ci/config/node/paths.rb b/lib/gitlab/ci/config/entry/paths.rb index 3c6d3a52966..68dad161149 100644 --- a/lib/gitlab/ci/config/node/paths.rb +++ b/lib/gitlab/ci/config/entry/paths.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Entry that represents an array of paths. # - class Paths < Entry + class Paths < Node include Validatable validations do diff --git a/lib/gitlab/ci/config/node/script.rb b/lib/gitlab/ci/config/entry/script.rb index 39328f0fade..29ecd9995ca 100644 --- a/lib/gitlab/ci/config/node/script.rb +++ b/lib/gitlab/ci/config/entry/script.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Entry that represents a script. # - class Script < Entry + class Script < Node include Validatable validations do diff --git a/lib/gitlab/ci/config/node/services.rb b/lib/gitlab/ci/config/entry/services.rb index 481e2b66adc..84f8ab780f5 100644 --- a/lib/gitlab/ci/config/node/services.rb +++ b/lib/gitlab/ci/config/entry/services.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Entry that represents a configuration of Docker services. # - class Services < Entry + class Services < Node include Validatable validations do diff --git a/lib/gitlab/ci/config/node/stage.rb b/lib/gitlab/ci/config/entry/stage.rb index cbc97641f5a..b7afaba1de8 100644 --- a/lib/gitlab/ci/config/node/stage.rb +++ b/lib/gitlab/ci/config/entry/stage.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Entry that represents a stage for a job. # - class Stage < Entry + class Stage < Node include Validatable validations do diff --git a/lib/gitlab/ci/config/node/stages.rb b/lib/gitlab/ci/config/entry/stages.rb index b1fe45357ff..ec187bd3732 100644 --- a/lib/gitlab/ci/config/node/stages.rb +++ b/lib/gitlab/ci/config/entry/stages.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Entry that represents a configuration for pipeline stages. # - class Stages < Entry + class Stages < Node include Validatable validations do diff --git a/lib/gitlab/ci/config/node/trigger.rb b/lib/gitlab/ci/config/entry/trigger.rb index d8b31975088..28b0a9ffe01 100644 --- a/lib/gitlab/ci/config/node/trigger.rb +++ b/lib/gitlab/ci/config/entry/trigger.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Entry that represents a trigger policy for the job. # - class Trigger < Entry + class Trigger < Node include Validatable validations do diff --git a/lib/gitlab/ci/config/node/undefined.rb b/lib/gitlab/ci/config/entry/undefined.rb index 33e78023539..b33b8238230 100644 --- a/lib/gitlab/ci/config/node/undefined.rb +++ b/lib/gitlab/ci/config/entry/undefined.rb @@ -1,13 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## - # This class represents an undefined node. + # This class represents an undefined entry. # - # Implements the Null Object pattern. - # - class Undefined < Entry + class Undefined < Node def initialize(*) super(nil) end diff --git a/lib/gitlab/ci/config/node/unspecified.rb b/lib/gitlab/ci/config/entry/unspecified.rb index a7d1f6131b8..fbb2551e870 100644 --- a/lib/gitlab/ci/config/node/unspecified.rb +++ b/lib/gitlab/ci/config/entry/unspecified.rb @@ -1,9 +1,9 @@ module Gitlab module Ci class Config - module Node + module Entry ## - # This class represents an unspecified entry node. + # This class represents an unspecified entry. # # It decorates original entry adding method that indicates it is # unspecified. diff --git a/lib/gitlab/ci/config/node/validatable.rb b/lib/gitlab/ci/config/entry/validatable.rb index 085e6e988d1..f7f1b111571 100644 --- a/lib/gitlab/ci/config/node/validatable.rb +++ b/lib/gitlab/ci/config/entry/validatable.rb @@ -1,13 +1,13 @@ module Gitlab module Ci class Config - module Node + module Entry module Validatable extend ActiveSupport::Concern class_methods do def validator - @validator ||= Class.new(Node::Validator).tap do |validator| + @validator ||= Class.new(Entry::Validator).tap do |validator| if defined?(@validations) @validations.each { |rules| validator.class_eval(&rules) } end diff --git a/lib/gitlab/ci/config/node/validator.rb b/lib/gitlab/ci/config/entry/validator.rb index 43c7e102b50..55343005fe3 100644 --- a/lib/gitlab/ci/config/node/validator.rb +++ b/lib/gitlab/ci/config/entry/validator.rb @@ -1,14 +1,14 @@ module Gitlab module Ci class Config - module Node + module Entry class Validator < SimpleDelegator include ActiveModel::Validations - include Node::Validators + include Entry::Validators - def initialize(node) - super(node) - @node = node + def initialize(entry) + super(entry) + @entry = entry end def messages @@ -30,7 +30,7 @@ module Gitlab def key_name if key.blank? - @node.class.name.demodulize.underscore.humanize + @entry.class.name.demodulize.underscore.humanize else key end diff --git a/lib/gitlab/ci/config/node/validators.rb b/lib/gitlab/ci/config/entry/validators.rb index e20908ad3cb..8632dd0e233 100644 --- a/lib/gitlab/ci/config/node/validators.rb +++ b/lib/gitlab/ci/config/entry/validators.rb @@ -1,7 +1,7 @@ module Gitlab module Ci class Config - module Node + module Entry module Validators class AllowedKeysValidator < ActiveModel::EachValidator def validate_each(record, attribute, value) diff --git a/lib/gitlab/ci/config/node/variables.rb b/lib/gitlab/ci/config/entry/variables.rb index 5f813f81f55..c3b0e651c3a 100644 --- a/lib/gitlab/ci/config/node/variables.rb +++ b/lib/gitlab/ci/config/entry/variables.rb @@ -1,11 +1,11 @@ module Gitlab module Ci class Config - module Node + module Entry ## # Entry that represents environment variables. # - class Variables < Entry + class Variables < Node include Validatable validations do diff --git a/lib/gitlab/ci/status/build/cancelable.rb b/lib/gitlab/ci/status/build/cancelable.rb new file mode 100644 index 00000000000..a979fe7d573 --- /dev/null +++ b/lib/gitlab/ci/status/build/cancelable.rb @@ -0,0 +1,37 @@ +module Gitlab + module Ci + module Status + module Build + class Cancelable < SimpleDelegator + include Status::Extended + + def has_action? + can?(user, :update_build, subject) + end + + def action_icon + 'ban' + end + + def action_path + cancel_namespace_project_build_path(subject.project.namespace, + subject.project, + subject) + end + + def action_method + :post + end + + def action_title + 'Cancel' + end + + def self.matches?(build, user) + build.cancelable? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/common.rb b/lib/gitlab/ci/status/build/common.rb new file mode 100644 index 00000000000..3fec2c5d4db --- /dev/null +++ b/lib/gitlab/ci/status/build/common.rb @@ -0,0 +1,19 @@ +module Gitlab + module Ci + module Status + module Build + module Common + def has_details? + can?(user, :read_build, subject) + end + + def details_path + namespace_project_build_path(subject.project.namespace, + subject.project, + subject) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/factory.rb b/lib/gitlab/ci/status/build/factory.rb new file mode 100644 index 00000000000..38ac6edc9f1 --- /dev/null +++ b/lib/gitlab/ci/status/build/factory.rb @@ -0,0 +1,21 @@ +module Gitlab + module Ci + module Status + module Build + class Factory < Status::Factory + def self.extended_statuses + [[Status::Build::Cancelable, + Status::Build::Retryable], + [Status::Build::FailedAllowed, + Status::Build::Play, + Status::Build::Stop]] + end + + def self.common_helpers + Status::Build::Common + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/failed_allowed.rb b/lib/gitlab/ci/status/build/failed_allowed.rb new file mode 100644 index 00000000000..807afe24bd5 --- /dev/null +++ b/lib/gitlab/ci/status/build/failed_allowed.rb @@ -0,0 +1,27 @@ +module Gitlab + module Ci + module Status + module Build + class FailedAllowed < SimpleDelegator + include Status::Extended + + def label + 'failed (allowed to fail)' + end + + def icon + 'icon_status_warning' + end + + def group + 'failed_with_warnings' + end + + def self.matches?(build, user) + build.failed? && build.allow_failure? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/play.rb b/lib/gitlab/ci/status/build/play.rb new file mode 100644 index 00000000000..1bf949c96dd --- /dev/null +++ b/lib/gitlab/ci/status/build/play.rb @@ -0,0 +1,57 @@ +module Gitlab + module Ci + module Status + module Build + class Play < SimpleDelegator + include Status::Extended + + def text + 'manual' + end + + def label + 'manual play action' + end + + def icon + 'icon_status_manual' + end + + def group + 'manual' + end + + def has_action? + can?(user, :update_build, subject) + end + + def action_icon + 'play' + end + + def action_title + 'Play' + end + + def action_class + 'ci-play-icon' + end + + def action_path + play_namespace_project_build_path(subject.project.namespace, + subject.project, + subject) + end + + def action_method + :post + end + + def self.matches?(build, user) + build.playable? && !build.stops_environment? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/retryable.rb b/lib/gitlab/ci/status/build/retryable.rb new file mode 100644 index 00000000000..8e38d6a8523 --- /dev/null +++ b/lib/gitlab/ci/status/build/retryable.rb @@ -0,0 +1,37 @@ +module Gitlab + module Ci + module Status + module Build + class Retryable < SimpleDelegator + include Status::Extended + + def has_action? + can?(user, :update_build, subject) + end + + def action_icon + 'refresh' + end + + def action_title + 'Retry' + end + + def action_path + retry_namespace_project_build_path(subject.project.namespace, + subject.project, + subject) + end + + def action_method + :post + end + + def self.matches?(build, user) + build.retryable? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/stop.rb b/lib/gitlab/ci/status/build/stop.rb new file mode 100644 index 00000000000..e1dfdb76d41 --- /dev/null +++ b/lib/gitlab/ci/status/build/stop.rb @@ -0,0 +1,53 @@ +module Gitlab + module Ci + module Status + module Build + class Stop < SimpleDelegator + include Status::Extended + + def text + 'manual' + end + + def label + 'manual stop action' + end + + def icon + 'icon_status_manual' + end + + def group + 'manual' + end + + def has_action? + can?(user, :update_build, subject) + end + + def action_icon + 'stop' + end + + def action_title + 'Stop' + end + + def action_path + play_namespace_project_build_path(subject.project.namespace, + subject.project, + subject) + end + + def action_method + :post + end + + def self.matches?(build, user) + build.playable? && build.stops_environment? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/canceled.rb b/lib/gitlab/ci/status/canceled.rb new file mode 100644 index 00000000000..dd6d99e9075 --- /dev/null +++ b/lib/gitlab/ci/status/canceled.rb @@ -0,0 +1,19 @@ +module Gitlab + module Ci + module Status + class Canceled < Status::Core + def text + 'canceled' + end + + def label + 'canceled' + end + + def icon + 'icon_status_canceled' + end + end + end + end +end diff --git a/lib/gitlab/ci/status/core.rb b/lib/gitlab/ci/status/core.rb new file mode 100644 index 00000000000..73b6ab5a635 --- /dev/null +++ b/lib/gitlab/ci/status/core.rb @@ -0,0 +1,62 @@ +module Gitlab + module Ci + module Status + # Base abstract class fore core status + # + class Core + include Gitlab::Routing + include Gitlab::Allowable + + attr_reader :subject, :user + + def initialize(subject, user) + @subject = subject + @user = user + end + + def icon + raise NotImplementedError + end + + def label + raise NotImplementedError + end + + def group + self.class.name.demodulize.underscore + end + + def has_details? + false + end + + def details_path + raise NotImplementedError + end + + def has_action? + false + end + + def action_icon + raise NotImplementedError + end + + def action_class + end + + def action_path + raise NotImplementedError + end + + def action_method + raise NotImplementedError + end + + def action_title + raise NotImplementedError + end + end + end + end +end diff --git a/lib/gitlab/ci/status/created.rb b/lib/gitlab/ci/status/created.rb new file mode 100644 index 00000000000..6596d7e01ca --- /dev/null +++ b/lib/gitlab/ci/status/created.rb @@ -0,0 +1,19 @@ +module Gitlab + module Ci + module Status + class Created < Status::Core + def text + 'created' + end + + def label + 'created' + end + + def icon + 'icon_status_created' + end + end + end + end +end diff --git a/lib/gitlab/ci/status/extended.rb b/lib/gitlab/ci/status/extended.rb new file mode 100644 index 00000000000..d367c9bda69 --- /dev/null +++ b/lib/gitlab/ci/status/extended.rb @@ -0,0 +1,15 @@ +module Gitlab + module Ci + module Status + module Extended + extend ActiveSupport::Concern + + class_methods do + def matches?(_subject, _user) + raise NotImplementedError + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/external/common.rb b/lib/gitlab/ci/status/external/common.rb new file mode 100644 index 00000000000..4969a350862 --- /dev/null +++ b/lib/gitlab/ci/status/external/common.rb @@ -0,0 +1,22 @@ +module Gitlab + module Ci + module Status + module External + module Common + def has_details? + subject.target_url.present? && + can?(user, :read_commit_status, subject) + end + + def details_path + subject.target_url + end + + def has_action? + false + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/external/factory.rb b/lib/gitlab/ci/status/external/factory.rb new file mode 100644 index 00000000000..07b15bd8d97 --- /dev/null +++ b/lib/gitlab/ci/status/external/factory.rb @@ -0,0 +1,13 @@ +module Gitlab + module Ci + module Status + module External + class Factory < Status::Factory + def self.common_helpers + Status::External::Common + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/factory.rb b/lib/gitlab/ci/status/factory.rb new file mode 100644 index 00000000000..15836c699c7 --- /dev/null +++ b/lib/gitlab/ci/status/factory.rb @@ -0,0 +1,52 @@ +module Gitlab + module Ci + module Status + class Factory + def initialize(subject, user) + @subject = subject + @user = user + @status = subject.status || HasStatus::DEFAULT_STATUS + end + + def fabricate! + if extended_statuses.none? + core_status + else + compound_extended_status + end + end + + def core_status + Gitlab::Ci::Status + .const_get(@status.capitalize) + .new(@subject, @user) + .extend(self.class.common_helpers) + end + + def compound_extended_status + extended_statuses.inject(core_status) do |status, extended| + extended.new(status) + end + end + + def extended_statuses + return @extended_statuses if defined?(@extended_statuses) + + groups = self.class.extended_statuses.map do |group| + Array(group).find { |status| status.matches?(@subject, @user) } + end + + @extended_statuses = groups.flatten.compact + end + + def self.extended_statuses + [] + end + + def self.common_helpers + Module.new + end + end + end + end +end diff --git a/lib/gitlab/ci/status/failed.rb b/lib/gitlab/ci/status/failed.rb new file mode 100644 index 00000000000..c5b5e3203ad --- /dev/null +++ b/lib/gitlab/ci/status/failed.rb @@ -0,0 +1,19 @@ +module Gitlab + module Ci + module Status + class Failed < Status::Core + def text + 'failed' + end + + def label + 'failed' + end + + def icon + 'icon_status_failed' + end + end + end + end +end diff --git a/lib/gitlab/ci/status/pending.rb b/lib/gitlab/ci/status/pending.rb new file mode 100644 index 00000000000..d30f35a59a2 --- /dev/null +++ b/lib/gitlab/ci/status/pending.rb @@ -0,0 +1,19 @@ +module Gitlab + module Ci + module Status + class Pending < Status::Core + def text + 'pending' + end + + def label + 'pending' + end + + def icon + 'icon_status_pending' + end + end + end + end +end diff --git a/lib/gitlab/ci/status/pipeline/common.rb b/lib/gitlab/ci/status/pipeline/common.rb new file mode 100644 index 00000000000..76bfd18bf40 --- /dev/null +++ b/lib/gitlab/ci/status/pipeline/common.rb @@ -0,0 +1,23 @@ +module Gitlab + module Ci + module Status + module Pipeline + module Common + def has_details? + can?(user, :read_pipeline, subject) + end + + def details_path + namespace_project_pipeline_path(subject.project.namespace, + subject.project, + subject) + end + + def has_action? + false + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/pipeline/factory.rb b/lib/gitlab/ci/status/pipeline/factory.rb new file mode 100644 index 00000000000..13c8343b12a --- /dev/null +++ b/lib/gitlab/ci/status/pipeline/factory.rb @@ -0,0 +1,17 @@ +module Gitlab + module Ci + module Status + module Pipeline + class Factory < Status::Factory + def self.extended_statuses + [Status::SuccessWarning] + end + + def self.common_helpers + Status::Pipeline::Common + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/running.rb b/lib/gitlab/ci/status/running.rb new file mode 100644 index 00000000000..2aba3c373c7 --- /dev/null +++ b/lib/gitlab/ci/status/running.rb @@ -0,0 +1,19 @@ +module Gitlab + module Ci + module Status + class Running < Status::Core + def text + 'running' + end + + def label + 'running' + end + + def icon + 'icon_status_running' + end + end + end + end +end diff --git a/lib/gitlab/ci/status/skipped.rb b/lib/gitlab/ci/status/skipped.rb new file mode 100644 index 00000000000..16282aefd03 --- /dev/null +++ b/lib/gitlab/ci/status/skipped.rb @@ -0,0 +1,19 @@ +module Gitlab + module Ci + module Status + class Skipped < Status::Core + def text + 'skipped' + end + + def label + 'skipped' + end + + def icon + 'icon_status_skipped' + end + end + end + end +end diff --git a/lib/gitlab/ci/status/stage/common.rb b/lib/gitlab/ci/status/stage/common.rb new file mode 100644 index 00000000000..7852f492e1d --- /dev/null +++ b/lib/gitlab/ci/status/stage/common.rb @@ -0,0 +1,24 @@ +module Gitlab + module Ci + module Status + module Stage + module Common + def has_details? + can?(user, :read_pipeline, subject.pipeline) + end + + def details_path + namespace_project_pipeline_path(subject.project.namespace, + subject.project, + subject.pipeline, + anchor: subject.name) + end + + def has_action? + false + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/stage/factory.rb b/lib/gitlab/ci/status/stage/factory.rb new file mode 100644 index 00000000000..4c37f084d07 --- /dev/null +++ b/lib/gitlab/ci/status/stage/factory.rb @@ -0,0 +1,17 @@ +module Gitlab + module Ci + module Status + module Stage + class Factory < Status::Factory + def self.extended_statuses + [Status::SuccessWarning] + end + + def self.common_helpers + Status::Stage::Common + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/success.rb b/lib/gitlab/ci/status/success.rb new file mode 100644 index 00000000000..c09c5f006e3 --- /dev/null +++ b/lib/gitlab/ci/status/success.rb @@ -0,0 +1,19 @@ +module Gitlab + module Ci + module Status + class Success < Status::Core + def text + 'passed' + end + + def label + 'passed' + end + + def icon + 'icon_status_success' + end + end + end + end +end diff --git a/lib/gitlab/ci/status/success_warning.rb b/lib/gitlab/ci/status/success_warning.rb new file mode 100644 index 00000000000..d4cdab6957a --- /dev/null +++ b/lib/gitlab/ci/status/success_warning.rb @@ -0,0 +1,33 @@ +module Gitlab + module Ci + module Status + ## + # Extended status used when pipeline or stage passed conditionally. + # This means that failed jobs that are allowed to fail were present. + # + class SuccessWarning < SimpleDelegator + include Status::Extended + + def text + 'passed' + end + + def label + 'passed with warnings' + end + + def icon + 'icon_status_warning' + end + + def group + 'success_with_warnings' + end + + def self.matches?(subject, user) + subject.success? && subject.has_warnings? + end + end + end + end +end diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index b164f5a2eea..7e3d5647b39 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -1,45 +1,44 @@ module Gitlab class ContributionsCalendar - attr_reader :activity_dates, :projects, :user + attr_reader :contributor + attr_reader :current_user + attr_reader :projects - def initialize(projects, user) - @projects = projects - @user = user + def initialize(contributor, current_user = nil) + @contributor = contributor + @current_user = current_user + @projects = ContributedProjectsFinder.new(contributor).execute(current_user) end def activity_dates return @activity_dates if @activity_dates.present? - @activity_dates = {} + # Can't use Event.contributions here because we need to check 3 different + # project_features for the (currently) 3 different contribution types date_from = 1.year.ago + repo_events = event_counts(date_from, :repository). + having(action: Event::PUSHED) + issue_events = event_counts(date_from, :issues). + having(action: [Event::CREATED, Event::CLOSED], target_type: "Issue") + mr_events = event_counts(date_from, :merge_requests). + having(action: [Event::MERGED, Event::CREATED, Event::CLOSED], target_type: "MergeRequest") - events = Event.reorder(nil).contributions.where(author_id: user.id). - where("created_at > ?", date_from).where(project_id: projects). - group('date(created_at)'). - select('date(created_at) as date, count(id) as total_amount'). - map(&:attributes) + union = Gitlab::SQL::Union.new([repo_events, issue_events, mr_events]) + events = Event.find_by_sql(union.to_sql).map(&:attributes) - activity_dates = (1.year.ago.to_date..Date.today).to_a - - activity_dates.each do |date| - day_events = events.find { |day_events| day_events["date"] == date } - - if day_events - @activity_dates[date] = day_events["total_amount"] - end + @activity_events = events.each_with_object(Hash.new {|h, k| h[k] = 0 }) do |event, activities| + activities[event["date"]] += event["total_amount"] end - - @activity_dates end def events_by_date(date) - events = Event.contributions.where(author_id: user.id). - where("created_at > ? AND created_at < ?", date.beginning_of_day, date.end_of_day). + events = Event.contributions.where(author_id: contributor.id). + where(created_at: date.beginning_of_day..date.end_of_day). where(project_id: projects) - events.select do |event| - event.push? || event.issue? || event.merge_request? - end + # Use visible_to_user? instead of the complicated logic in activity_dates + # because we're only viewing the events for a single day. + events.select {|event| event.visible_to_user?(current_user) } end def starting_year @@ -49,5 +48,30 @@ module Gitlab def starting_month Date.today.month end + + private + + def event_counts(date_from, feature) + t = Event.arel_table + + # re-running the contributed projects query in each union is expensive, so + # use IN(project_ids...) instead. It's the intersection of two users so + # the list will be (relatively) short + @contributed_project_ids ||= projects.uniq.pluck(:id) + authed_projects = Project.where(id: @contributed_project_ids). + with_feature_available_for_user(feature, current_user). + reorder(nil). + select(:id) + + conditions = t[:created_at].gteq(date_from.beginning_of_day). + and(t[:created_at].lteq(Date.today.end_of_day)). + and(t[:author_id].eq(contributor.id)) + + Event.reorder(nil). + select(t[:project_id], t[:target_type], t[:action], 'date(created_at) AS date', 'count(id) as total_amount'). + group(t[:project_id], t[:target_type], t[:action], 'date(created_at)'). + where(conditions). + having(t[:project_id].in(Arel::Nodes::SqlLiteral.new(authed_projects.to_sql))) + end end end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index ef9160d6437..2ff27e46d64 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -23,6 +23,10 @@ module Gitlab settings || fake_application_settings end + def sidekiq_throttling_enabled? + current_application_settings.sidekiq_throttling_enabled? + end + def fake_application_settings OpenStruct.new( default_projects_limit: Settings.gitlab['default_projects_limit'], @@ -31,6 +35,7 @@ module Gitlab signin_enabled: Settings.gitlab['signin_enabled'], gravatar_enabled: Settings.gravatar['enabled'], koding_enabled: false, + plantuml_enabled: false, sign_in_text: nil, after_sign_up_text: nil, help_page_text: nil, @@ -41,7 +46,7 @@ module Gitlab default_project_visibility: Settings.gitlab.default_projects_features['visibility_level'], default_snippet_visibility: Settings.gitlab.default_projects_features['visibility_level'], domain_whitelist: Settings.gitlab['domain_whitelist'], - import_sources: %w[github bitbucket gitlab google_code fogbugz git gitlab_project], + import_sources: %w[gitea github bitbucket gitlab google_code fogbugz git gitlab_project], shared_runners_enabled: Settings.gitlab_ci['shared_runners_enabled'], max_artifacts_size: Settings.artifacts['max_size'], require_two_factor_authentication: false, @@ -50,6 +55,7 @@ module Gitlab repository_checks_enabled: true, container_registry_token_expire_delay: 5, user_default_external: false, + sidekiq_throttling_enabled: false, ) end diff --git a/lib/gitlab/cycle_analytics/base_event_fetcher.rb b/lib/gitlab/cycle_analytics/base_event_fetcher.rb new file mode 100644 index 00000000000..0d8791d396b --- /dev/null +++ b/lib/gitlab/cycle_analytics/base_event_fetcher.rb @@ -0,0 +1,65 @@ +module Gitlab + module CycleAnalytics + class BaseEventFetcher + include BaseQuery + + attr_reader :projections, :query, :stage, :order + + def initialize(project:, stage:, options:) + @project = project + @stage = stage + @options = options + end + + def fetch + update_author! + + event_result.map do |event| + serialize(event) if has_permission?(event['id']) + end.compact + end + + def order + @order || default_order + end + + private + + def update_author! + return unless event_result.any? && event_result.first['author_id'] + + Updater.update!(event_result, from: 'author_id', to: 'author', klass: User) + end + + def event_result + @event_result ||= ActiveRecord::Base.connection.exec_query(events_query.to_sql).to_a + end + + def events_query + diff_fn = subtract_datetimes_diff(base_query, @options[:start_time_attrs], @options[:end_time_attrs]) + + base_query.project(extract_diff_epoch(diff_fn).as('total_time'), *projections).order(order.desc) + end + + def default_order + [@options[:start_time_attrs]].flatten.first + end + + def serialize(_event) + raise NotImplementedError.new("Expected #{self.name} to implement serialize(event)") + end + + def has_permission?(id) + allowed_ids.nil? || allowed_ids.include?(id.to_i) + end + + def allowed_ids + nil + end + + def event_result_ids + event_result.map { |event| event['id'] } + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/base_query.rb b/lib/gitlab/cycle_analytics/base_query.rb new file mode 100644 index 00000000000..d560dca45c8 --- /dev/null +++ b/lib/gitlab/cycle_analytics/base_query.rb @@ -0,0 +1,31 @@ +module Gitlab + module CycleAnalytics + module BaseQuery + include MetricsTables + include Gitlab::Database::Median + include Gitlab::Database::DateTime + + private + + def base_query + @base_query ||= stage_query + end + + def stage_query + query = mr_closing_issues_table.join(issue_table).on(issue_table[:id].eq(mr_closing_issues_table[:issue_id])). + join(issue_metrics_table).on(issue_table[:id].eq(issue_metrics_table[:issue_id])). + where(issue_table[:project_id].eq(@project.id)). + where(issue_table[:deleted_at].eq(nil)). + where(issue_table[:created_at].gteq(@options[:from])) + + # Load merge_requests + query = query.join(mr_table, Arel::Nodes::OuterJoin). + on(mr_table[:id].eq(mr_closing_issues_table[:merge_request_id])). + join(mr_metrics_table). + on(mr_table[:id].eq(mr_metrics_table[:merge_request_id])) + + query + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/base_stage.rb b/lib/gitlab/cycle_analytics/base_stage.rb new file mode 100644 index 00000000000..74bbcdcb3dd --- /dev/null +++ b/lib/gitlab/cycle_analytics/base_stage.rb @@ -0,0 +1,54 @@ +module Gitlab + module CycleAnalytics + class BaseStage + include BaseQuery + + def initialize(project:, options:) + @project = project + @options = options + end + + def events + event_fetcher.fetch + end + + def as_json + AnalyticsStageSerializer.new.represent(self).as_json + end + + def title + name.to_s.capitalize + end + + def median + cte_table = Arel::Table.new("cte_table_for_#{name}") + + # Build a `SELECT` query. We find the first of the `end_time_attrs` that isn't `NULL` (call this end_time). + # Next, we find the first of the start_time_attrs that isn't `NULL` (call this start_time). + # We compute the (end_time - start_time) interval, and give it an alias based on the current + # cycle analytics stage. + interval_query = Arel::Nodes::As.new( + cte_table, + subtract_datetimes(base_query.dup, start_time_attrs, end_time_attrs, name.to_s)) + + median_datetime(cte_table, interval_query, name) + end + + def name + raise NotImplementedError.new("Expected #{self.name} to implement name") + end + + private + + def event_fetcher + @event_fetcher ||= Gitlab::CycleAnalytics::EventFetcher[name].new(project: @project, + stage: name, + options: event_options) + end + + def event_options + @options.merge(start_time_attrs: start_time_attrs, end_time_attrs: end_time_attrs) + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/code_event_fetcher.rb b/lib/gitlab/cycle_analytics/code_event_fetcher.rb new file mode 100644 index 00000000000..5245b9ca8fc --- /dev/null +++ b/lib/gitlab/cycle_analytics/code_event_fetcher.rb @@ -0,0 +1,25 @@ +module Gitlab + module CycleAnalytics + class CodeEventFetcher < BaseEventFetcher + include MergeRequestAllowed + + def initialize(*args) + @projections = [mr_table[:title], + mr_table[:iid], + mr_table[:id], + mr_table[:created_at], + mr_table[:state], + mr_table[:author_id]] + @order = mr_table[:created_at] + + super(*args) + end + + private + + def serialize(event) + AnalyticsMergeRequestSerializer.new(project: @project).represent(event).as_json + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/code_stage.rb b/lib/gitlab/cycle_analytics/code_stage.rb new file mode 100644 index 00000000000..d1bc2055ba8 --- /dev/null +++ b/lib/gitlab/cycle_analytics/code_stage.rb @@ -0,0 +1,21 @@ +module Gitlab + module CycleAnalytics + class CodeStage < BaseStage + def start_time_attrs + @start_time_attrs ||= issue_metrics_table[:first_mentioned_in_commit_at] + end + + def end_time_attrs + @end_time_attrs ||= mr_table[:created_at] + end + + def name + :code + end + + def description + "Time until first merge request" + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/event_fetcher.rb b/lib/gitlab/cycle_analytics/event_fetcher.rb new file mode 100644 index 00000000000..50e126cf00b --- /dev/null +++ b/lib/gitlab/cycle_analytics/event_fetcher.rb @@ -0,0 +1,9 @@ +module Gitlab + module CycleAnalytics + module EventFetcher + def self.[](stage_name) + CycleAnalytics.const_get("#{stage_name.to_s.camelize}EventFetcher") + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/issue_allowed.rb b/lib/gitlab/cycle_analytics/issue_allowed.rb new file mode 100644 index 00000000000..a7652a70641 --- /dev/null +++ b/lib/gitlab/cycle_analytics/issue_allowed.rb @@ -0,0 +1,9 @@ +module Gitlab + module CycleAnalytics + module IssueAllowed + def allowed_ids + @allowed_ids ||= IssuesFinder.new(@options[:current_user], project_id: @project.id).execute.where(id: event_result_ids).pluck(:id) + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/issue_event_fetcher.rb b/lib/gitlab/cycle_analytics/issue_event_fetcher.rb new file mode 100644 index 00000000000..0d8da99455e --- /dev/null +++ b/lib/gitlab/cycle_analytics/issue_event_fetcher.rb @@ -0,0 +1,23 @@ +module Gitlab + module CycleAnalytics + class IssueEventFetcher < BaseEventFetcher + include IssueAllowed + + def initialize(*args) + @projections = [issue_table[:title], + issue_table[:iid], + issue_table[:id], + issue_table[:created_at], + issue_table[:author_id]] + + super(*args) + end + + private + + def serialize(event) + AnalyticsIssueSerializer.new(project: @project).represent(event).as_json + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/issue_stage.rb b/lib/gitlab/cycle_analytics/issue_stage.rb new file mode 100644 index 00000000000..d2068fbc38f --- /dev/null +++ b/lib/gitlab/cycle_analytics/issue_stage.rb @@ -0,0 +1,22 @@ +module Gitlab + module CycleAnalytics + class IssueStage < BaseStage + def start_time_attrs + @start_time_attrs ||= issue_table[:created_at] + end + + def end_time_attrs + @end_time_attrs ||= [issue_metrics_table[:first_associated_with_milestone_at], + issue_metrics_table[:first_added_to_board_at]] + end + + def name + :issue + end + + def description + "Time before an issue gets scheduled" + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/merge_request_allowed.rb b/lib/gitlab/cycle_analytics/merge_request_allowed.rb new file mode 100644 index 00000000000..28f6db44759 --- /dev/null +++ b/lib/gitlab/cycle_analytics/merge_request_allowed.rb @@ -0,0 +1,9 @@ +module Gitlab + module CycleAnalytics + module MergeRequestAllowed + def allowed_ids + @allowed_ids ||= MergeRequestsFinder.new(@options[:current_user], project_id: @project.id).execute.where(id: event_result_ids).pluck(:id) + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/metrics_tables.rb b/lib/gitlab/cycle_analytics/metrics_tables.rb new file mode 100644 index 00000000000..9d25ef078e8 --- /dev/null +++ b/lib/gitlab/cycle_analytics/metrics_tables.rb @@ -0,0 +1,37 @@ +module Gitlab + module CycleAnalytics + module MetricsTables + def mr_metrics_table + MergeRequest::Metrics.arel_table + end + + def mr_table + MergeRequest.arel_table + end + + def mr_diff_table + MergeRequestDiff.arel_table + end + + def mr_closing_issues_table + MergeRequestsClosingIssues.arel_table + end + + def issue_table + Issue.arel_table + end + + def issue_metrics_table + Issue::Metrics.arel_table + end + + def user_table + User.arel_table + end + + def build_table + ::CommitStatus.arel_table + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/permissions.rb b/lib/gitlab/cycle_analytics/permissions.rb new file mode 100644 index 00000000000..bef3b95ff1b --- /dev/null +++ b/lib/gitlab/cycle_analytics/permissions.rb @@ -0,0 +1,44 @@ +module Gitlab + module CycleAnalytics + class Permissions + STAGE_PERMISSIONS = { + issue: :read_issue, + code: :read_merge_request, + test: :read_build, + review: :read_merge_request, + staging: :read_build, + production: :read_issue, + }.freeze + + def self.get(*args) + new(*args).get + end + + def initialize(user:, project:) + @user = user + @project = project + @stage_permission_hash = {} + end + + def get + ::CycleAnalytics::STAGES.each do |stage| + @stage_permission_hash[stage] = authorized_stage?(stage) + end + + @stage_permission_hash + end + + private + + def authorized_stage?(stage) + return false unless authorize_project(:read_cycle_analytics) + + STAGE_PERMISSIONS[stage] ? authorize_project(STAGE_PERMISSIONS[stage]) : true + end + + def authorize_project(permission) + Ability.allowed?(@user, permission, @project) + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb new file mode 100644 index 00000000000..88a8710dbe6 --- /dev/null +++ b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb @@ -0,0 +1,44 @@ +module Gitlab + module CycleAnalytics + class PlanEventFetcher < BaseEventFetcher + def initialize(*args) + @projections = [mr_diff_table[:st_commits].as('commits'), + issue_metrics_table[:first_mentioned_in_commit_at]] + + super(*args) + end + + def events_query + base_query.join(mr_diff_table).on(mr_diff_table[:merge_request_id].eq(mr_table[:id])) + + super + end + + private + + def serialize(event) + st_commit = first_time_reference_commit(event.delete('commits'), event) + + return unless st_commit + + serialize_commit(event, st_commit, query) + end + + def first_time_reference_commit(commits, event) + return nil if commits.blank? + + YAML.load(commits).find do |commit| + next unless commit[:committed_date] && event['first_mentioned_in_commit_at'] + + commit[:committed_date].to_i == DateTime.parse(event['first_mentioned_in_commit_at'].to_s).to_i + end + end + + def serialize_commit(event, st_commit, query) + commit = Commit.new(Gitlab::Git::Commit.new(st_commit), @project) + + AnalyticsCommitSerializer.new(project: @project, total_time: event['total_time']).represent(commit).as_json + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/plan_stage.rb b/lib/gitlab/cycle_analytics/plan_stage.rb new file mode 100644 index 00000000000..3b4dfc6a30e --- /dev/null +++ b/lib/gitlab/cycle_analytics/plan_stage.rb @@ -0,0 +1,22 @@ +module Gitlab + module CycleAnalytics + class PlanStage < BaseStage + def start_time_attrs + @start_time_attrs ||= [issue_metrics_table[:first_associated_with_milestone_at], + issue_metrics_table[:first_added_to_board_at]] + end + + def end_time_attrs + @end_time_attrs ||= issue_metrics_table[:first_mentioned_in_commit_at] + end + + def name + :plan + end + + def description + "Time before an issue starts implementation" + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/production_event_fetcher.rb b/lib/gitlab/cycle_analytics/production_event_fetcher.rb new file mode 100644 index 00000000000..0fa2e87f673 --- /dev/null +++ b/lib/gitlab/cycle_analytics/production_event_fetcher.rb @@ -0,0 +1,6 @@ +module Gitlab + module CycleAnalytics + class ProductionEventFetcher < IssueEventFetcher + end + end +end diff --git a/lib/gitlab/cycle_analytics/production_helper.rb b/lib/gitlab/cycle_analytics/production_helper.rb new file mode 100644 index 00000000000..d693443bfa4 --- /dev/null +++ b/lib/gitlab/cycle_analytics/production_helper.rb @@ -0,0 +1,9 @@ +module Gitlab + module CycleAnalytics + module ProductionHelper + def stage_query + super.where(mr_metrics_table[:first_deployed_to_production_at].gteq(@options[:from])) + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/production_stage.rb b/lib/gitlab/cycle_analytics/production_stage.rb new file mode 100644 index 00000000000..2a6bcc80116 --- /dev/null +++ b/lib/gitlab/cycle_analytics/production_stage.rb @@ -0,0 +1,28 @@ +module Gitlab + module CycleAnalytics + class ProductionStage < BaseStage + include ProductionHelper + + def start_time_attrs + @start_time_attrs ||= issue_table[:created_at] + end + + def end_time_attrs + @end_time_attrs ||= mr_metrics_table[:first_deployed_to_production_at] + end + + def name + :production + end + + def description + "From issue creation until deploy to production" + end + + def query + # Limit to merge requests that have been deployed to production after `@from` + query.where(mr_metrics_table[:first_deployed_to_production_at].gteq(@from)) + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/review_event_fetcher.rb b/lib/gitlab/cycle_analytics/review_event_fetcher.rb new file mode 100644 index 00000000000..4df0bd06393 --- /dev/null +++ b/lib/gitlab/cycle_analytics/review_event_fetcher.rb @@ -0,0 +1,22 @@ +module Gitlab + module CycleAnalytics + class ReviewEventFetcher < BaseEventFetcher + include MergeRequestAllowed + + def initialize(*args) + @projections = [mr_table[:title], + mr_table[:iid], + mr_table[:id], + mr_table[:created_at], + mr_table[:state], + mr_table[:author_id]] + + super(*args) + end + + def serialize(event) + AnalyticsMergeRequestSerializer.new(project: @project).represent(event).as_json + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/review_stage.rb b/lib/gitlab/cycle_analytics/review_stage.rb new file mode 100644 index 00000000000..fbaa3010d81 --- /dev/null +++ b/lib/gitlab/cycle_analytics/review_stage.rb @@ -0,0 +1,21 @@ +module Gitlab + module CycleAnalytics + class ReviewStage < BaseStage + def start_time_attrs + @start_time_attrs ||= mr_table[:created_at] + end + + def end_time_attrs + @end_time_attrs ||= mr_metrics_table[:merged_at] + end + + def name + :review + end + + def description + "Time between merge request creation and merge/close" + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/stage.rb b/lib/gitlab/cycle_analytics/stage.rb new file mode 100644 index 00000000000..28e0455df59 --- /dev/null +++ b/lib/gitlab/cycle_analytics/stage.rb @@ -0,0 +1,9 @@ +module Gitlab + module CycleAnalytics + module Stage + def self.[](stage_name) + CycleAnalytics.const_get("#{stage_name.to_s.camelize}Stage") + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/stage_summary.rb b/lib/gitlab/cycle_analytics/stage_summary.rb new file mode 100644 index 00000000000..b34baf5b081 --- /dev/null +++ b/lib/gitlab/cycle_analytics/stage_summary.rb @@ -0,0 +1,23 @@ +module Gitlab + module CycleAnalytics + class StageSummary + def initialize(project, from:, current_user:) + @project = project + @from = from + @current_user = current_user + end + + def data + [serialize(Summary::Issue.new(project: @project, from: @from, current_user: @current_user)), + serialize(Summary::Commit.new(project: @project, from: @from)), + serialize(Summary::Deploy.new(project: @project, from: @from))] + end + + private + + def serialize(summary_object) + AnalyticsSummarySerializer.new.represent(summary_object).as_json + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/staging_event_fetcher.rb b/lib/gitlab/cycle_analytics/staging_event_fetcher.rb new file mode 100644 index 00000000000..a34731a5fcd --- /dev/null +++ b/lib/gitlab/cycle_analytics/staging_event_fetcher.rb @@ -0,0 +1,30 @@ +module Gitlab + module CycleAnalytics + class StagingEventFetcher < BaseEventFetcher + def initialize(*args) + @projections = [build_table[:id]] + @order = build_table[:created_at] + + super(*args) + end + + def fetch + Updater.update!(event_result, from: 'id', to: 'build', klass: ::Ci::Build) + + super + end + + def events_query + base_query.join(build_table).on(mr_metrics_table[:pipeline_id].eq(build_table[:commit_id])) + + super + end + + private + + def serialize(event) + AnalyticsBuildSerializer.new.represent(event['build']).as_json + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/staging_stage.rb b/lib/gitlab/cycle_analytics/staging_stage.rb new file mode 100644 index 00000000000..945909a4d62 --- /dev/null +++ b/lib/gitlab/cycle_analytics/staging_stage.rb @@ -0,0 +1,22 @@ +module Gitlab + module CycleAnalytics + class StagingStage < BaseStage + include ProductionHelper + def start_time_attrs + @start_time_attrs ||= mr_metrics_table[:merged_at] + end + + def end_time_attrs + @end_time_attrs ||= mr_metrics_table[:first_deployed_to_production_at] + end + + def name + :staging + end + + def description + "From merge request merge until deploy to production" + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/summary/base.rb b/lib/gitlab/cycle_analytics/summary/base.rb new file mode 100644 index 00000000000..43fa3795e5c --- /dev/null +++ b/lib/gitlab/cycle_analytics/summary/base.rb @@ -0,0 +1,20 @@ +module Gitlab + module CycleAnalytics + module Summary + class Base + def initialize(project:, from:) + @project = project + @from = from + end + + def title + self.class.name.demodulize + end + + def value + raise NotImplementedError.new("Expected #{self.name} to implement value") + end + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/summary/commit.rb b/lib/gitlab/cycle_analytics/summary/commit.rb new file mode 100644 index 00000000000..7b8faa4d854 --- /dev/null +++ b/lib/gitlab/cycle_analytics/summary/commit.rb @@ -0,0 +1,39 @@ +module Gitlab + module CycleAnalytics + module Summary + class Commit < Base + def value + @value ||= count_commits + end + + private + + # Don't use the `Gitlab::Git::Repository#log` method, because it enforces + # a limit. Since we need a commit count, we _can't_ enforce a limit, so + # the easiest way forward is to replicate the relevant portions of the + # `log` function here. + def count_commits + return unless ref + + repository = @project.repository.raw_repository + sha = @project.repository.commit(ref).sha + + cmd = %W(git --git-dir=#{repository.path} log) + cmd << '--format=%H' + cmd << "--after=#{@from.iso8601}" + cmd << sha + + output, status = Gitlab::Popen.popen(cmd) + + raise IOError, output unless status.zero? + + output.lines.count + end + + def ref + @ref ||= @project.default_branch.presence + end + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/summary/deploy.rb b/lib/gitlab/cycle_analytics/summary/deploy.rb new file mode 100644 index 00000000000..06032e9200e --- /dev/null +++ b/lib/gitlab/cycle_analytics/summary/deploy.rb @@ -0,0 +1,11 @@ +module Gitlab + module CycleAnalytics + module Summary + class Deploy < Base + def value + @value ||= @project.deployments.where("created_at > ?", @from).count + end + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/summary/issue.rb b/lib/gitlab/cycle_analytics/summary/issue.rb new file mode 100644 index 00000000000..008468f24b9 --- /dev/null +++ b/lib/gitlab/cycle_analytics/summary/issue.rb @@ -0,0 +1,21 @@ +module Gitlab + module CycleAnalytics + module Summary + class Issue < Base + def initialize(project:, from:, current_user:) + @project = project + @from = from + @current_user = current_user + end + + def title + 'New Issue' + end + + def value + @value ||= IssuesFinder.new(@current_user, project_id: @project.id).execute.created_after(@from).count + end + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/test_event_fetcher.rb b/lib/gitlab/cycle_analytics/test_event_fetcher.rb new file mode 100644 index 00000000000..a2589c6601a --- /dev/null +++ b/lib/gitlab/cycle_analytics/test_event_fetcher.rb @@ -0,0 +1,6 @@ +module Gitlab + module CycleAnalytics + class TestEventFetcher < StagingEventFetcher + end + end +end diff --git a/lib/gitlab/cycle_analytics/test_stage.rb b/lib/gitlab/cycle_analytics/test_stage.rb new file mode 100644 index 00000000000..0079d56e0e4 --- /dev/null +++ b/lib/gitlab/cycle_analytics/test_stage.rb @@ -0,0 +1,29 @@ +module Gitlab + module CycleAnalytics + class TestStage < BaseStage + def start_time_attrs + @start_time_attrs ||= mr_metrics_table[:latest_build_started_at] + end + + def end_time_attrs + @end_time_attrs ||= mr_metrics_table[:latest_build_finished_at] + end + + def name + :test + end + + def description + "Total test time for all commits/merges" + end + + def stage_query + if @options[:branch] + super.where(build_table[:ref].eq(@options[:branch])) + else + super + end + end + end + end +end diff --git a/lib/gitlab/cycle_analytics/updater.rb b/lib/gitlab/cycle_analytics/updater.rb new file mode 100644 index 00000000000..953268ebd46 --- /dev/null +++ b/lib/gitlab/cycle_analytics/updater.rb @@ -0,0 +1,30 @@ +module Gitlab + module CycleAnalytics + class Updater + def self.update!(*args) + new(*args).update! + end + + def initialize(event_result, from:, to:, klass:) + @event_result = event_result + @klass = klass + @from = from + @to = to + end + + def update! + @event_result.each do |event| + event[@to] = items[event.delete(@from).to_i].first + end + end + + def result_ids + @event_result.map { |event| event[@from] } + end + + def items + @items ||= @klass.find(result_ids).group_by { |item| item['id'] } + end + end + end +end diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb index 06a783ebc1c..e50e54b6e99 100644 --- a/lib/gitlab/data_builder/pipeline.rb +++ b/lib/gitlab/data_builder/pipeline.rb @@ -22,7 +22,7 @@ module Gitlab sha: pipeline.sha, before_sha: pipeline.before_sha, status: pipeline.status, - stages: pipeline.stages, + stages: pipeline.stages_name, created_at: pipeline.created_at, finished_at: pipeline.finished_at, duration: pipeline.duration diff --git a/lib/gitlab/database/date_time.rb b/lib/gitlab/database/date_time.rb index b6a89f715fd..25e56998038 100644 --- a/lib/gitlab/database/date_time.rb +++ b/lib/gitlab/database/date_time.rb @@ -7,21 +7,25 @@ module Gitlab # # Note: For MySQL, the interval is returned in seconds. # For PostgreSQL, the interval is returned as an INTERVAL type. - def subtract_datetimes(query_so_far, end_time_attrs, start_time_attrs, as) - diff_fn = if Gitlab::Database.postgresql? - Arel::Nodes::Subtraction.new( - Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(end_time_attrs)), - Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(start_time_attrs))) - elsif Gitlab::Database.mysql? - Arel::Nodes::NamedFunction.new( - "TIMESTAMPDIFF", - [Arel.sql('second'), - Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(start_time_attrs)), - Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(end_time_attrs))]) - end + def subtract_datetimes(query_so_far, start_time_attrs, end_time_attrs, as) + diff_fn = subtract_datetimes_diff(query_so_far, start_time_attrs, end_time_attrs) query_so_far.project(diff_fn.as(as)) end + + def subtract_datetimes_diff(query_so_far, start_time_attrs, end_time_attrs) + if Gitlab::Database.postgresql? + Arel::Nodes::Subtraction.new( + Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(end_time_attrs)), + Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(start_time_attrs))) + elsif Gitlab::Database.mysql? + Arel::Nodes::NamedFunction.new( + "TIMESTAMPDIFF", + [Arel.sql('second'), + Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(start_time_attrs)), + Arel::Nodes::NamedFunction.new("COALESCE", Array.wrap(end_time_attrs))]) + end + end end end end diff --git a/lib/gitlab/database/median.rb b/lib/gitlab/database/median.rb index 1444d25ebc7..08607c27c09 100644 --- a/lib/gitlab/database/median.rb +++ b/lib/gitlab/database/median.rb @@ -103,6 +103,11 @@ module Gitlab Arel.sql(%Q{EXTRACT(EPOCH FROM "#{arel_attribute.relation.name}"."#{arel_attribute.name}")}) end + def extract_diff_epoch(diff) + return diff unless Gitlab::Database.postgresql? + + Arel.sql(%Q{EXTRACT(EPOCH FROM (#{diff.to_sql}))}) + end # Need to cast '0' to an INTERVAL before we can check if the interval is positive def zero_interval Arel::Nodes::NamedFunction.new("CAST", [Arel.sql("'0' AS INTERVAL")]) diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index ce85e5e0123..c6bf25b5874 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -55,6 +55,12 @@ module Gitlab repository.commit(deleted_file ? old_ref : new_ref) end + def old_content_commit + return unless diff_refs + + repository.commit(old_ref) + end + def old_ref diff_refs.try(:base_sha) end @@ -111,13 +117,10 @@ module Gitlab diff_lines.count(&:removed?) end - def old_blob(commit = content_commit) + def old_blob(commit = old_content_commit) return unless commit - parent_id = commit.parent_id - return unless parent_id - - repository.blob_at(parent_id, old_path) + repository.blob_at(commit.id, old_path) end def blob(commit = content_commit) @@ -126,7 +129,7 @@ module Gitlab repository.blob_at(commit.id, file_path) end - def cache_key + def file_identifier "#{file_path}-#{new_file}-#{deleted_file}-#{renamed_file}" end end diff --git a/lib/gitlab/diff/file_collection/merge_request_diff.rb b/lib/gitlab/diff/file_collection/merge_request_diff.rb index dc4d47c878b..329d12f13d1 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff.rb @@ -20,7 +20,7 @@ module Gitlab # Extracted method to highlight in the same iteration to the diff_collection. def decorate_diff!(diff) diff_file = super - cache_highlight!(diff_file) if cacheable? + cache_highlight!(diff_file) if cacheable?(diff_file) diff_file end @@ -39,7 +39,7 @@ module Gitlab # hashes that represent serialized diff lines. # def cache_highlight!(diff_file) - item_key = diff_file.cache_key + item_key = diff_file.file_identifier if highlight_cache[item_key] highlight_diff_file_from_cache!(diff_file, highlight_cache[item_key]) @@ -60,8 +60,11 @@ module Gitlab Rails.cache.write(cache_key, highlight_cache) if @highlight_cache_was_empty end - def cacheable? - @merge_request_diff.present? + def cacheable?(diff_file) + @merge_request_diff.present? && + diff_file.blob && + diff_file.blob.text? && + @project.repository.diffable?(diff_file.blob) end def cache_key diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index b1a6d5fe0f6..c8e36d8ff4a 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -2,39 +2,38 @@ module Gitlab # Checks if a set of migrations requires downtime or not. class EeCompatCheck + CE_REPO = 'https://gitlab.com/gitlab-org/gitlab-ce.git'.freeze EE_REPO = 'https://gitlab.com/gitlab-org/gitlab-ee.git'.freeze + CHECK_DIR = Rails.root.join('ee_compat_check') + MAX_FETCH_DEPTH = 500 + IGNORED_FILES_REGEX = /(VERSION|CHANGELOG\.md:\d+)/.freeze - attr_reader :ce_branch, :check_dir, :ce_repo + attr_reader :repo_dir, :patches_dir, :ce_repo, :ce_branch - def initialize(branch:, check_dir:, ce_repo: nil) + def initialize(branch:, ce_repo: CE_REPO) + @repo_dir = CHECK_DIR.join('repo') + @patches_dir = CHECK_DIR.join('patches') @ce_branch = branch - @check_dir = check_dir - @ce_repo = ce_repo || 'https://gitlab.com/gitlab-org/gitlab-ce.git' + @ce_repo = ce_repo end def check ensure_ee_repo - delete_patches + ensure_patches_dir generate_patch(ce_branch, ce_patch_full_path) - Dir.chdir(check_dir) do - step("In the #{check_dir} directory") - - step("Pulling latest master", %w[git pull --ff-only origin master]) + Dir.chdir(repo_dir) do + step("In the #{repo_dir} directory") status = catch(:halt_check) do ce_branch_compat_check! - - delete_ee_branch_locally - + delete_ee_branch_locally! ee_branch_presence_check! - ee_branch_compat_check! end - delete_ee_branch_locally - delete_patches + delete_ee_branch_locally! if status.nil? true @@ -47,20 +46,43 @@ module Gitlab private def ensure_ee_repo - if Dir.exist?(check_dir) - step("#{check_dir} already exists") + if Dir.exist?(repo_dir) + step("#{repo_dir} already exists") else - cmd = %W[git clone --branch master --single-branch --depth 1 #{EE_REPO} #{check_dir}] - step("Cloning #{EE_REPO} into #{check_dir}", cmd) + cmd = %W[git clone --branch master --single-branch --depth 200 #{EE_REPO} #{repo_dir}] + step("Cloning #{EE_REPO} into #{repo_dir}", cmd) end end - def ce_branch_compat_check! - cmd = %W[git apply --check #{ce_patch_full_path}] - status = step("Checking if #{ce_patch_name} applies cleanly to EE/master", cmd) + def ensure_patches_dir + FileUtils.mkdir_p(patches_dir) + end + + def generate_patch(branch, patch_path) + FileUtils.rm(patch_path, force: true) + + depth = 0 + loop do + depth += 50 + cmd = %W[git fetch --depth #{depth} origin --prune +refs/heads/master:refs/remotes/origin/master] + Gitlab::Popen.popen(cmd) + _, status = Gitlab::Popen.popen(%w[git merge-base FETCH_HEAD HEAD]) + + raise "#{branch} is too far behind master, please rebase it!" if depth >= MAX_FETCH_DEPTH + break if status.zero? + end - if status.zero? - puts ce_applies_cleanly_msg(ce_branch) + step("Generating the patch against master in #{patch_path}") + output, status = Gitlab::Popen.popen(%w[git format-patch FETCH_HEAD --stdout]) + throw(:halt_check, :ko) unless status.zero? + + File.write(patch_path, output) + throw(:halt_check, :ko) unless File.exist?(patch_path) + end + + def ce_branch_compat_check! + if check_patch(ce_patch_full_path).zero? + puts applies_cleanly_msg(ce_branch) throw(:halt_check) end end @@ -80,10 +102,8 @@ module Gitlab step("Checking out origin/#{ee_branch}", %W[git checkout -b #{ee_branch} FETCH_HEAD]) generate_patch(ee_branch, ee_patch_full_path) - cmd = %W[git apply --check #{ee_patch_full_path}] - status = step("Checking if #{ee_patch_name} applies cleanly to EE/master", cmd) - unless status.zero? + unless check_patch(ee_patch_full_path).zero? puts puts ee_branch_doesnt_apply_cleanly_msg @@ -91,50 +111,49 @@ module Gitlab end puts - puts ee_applies_cleanly_msg + puts applies_cleanly_msg(ee_branch) end - def generate_patch(branch, filepath) - FileUtils.rm(filepath, force: true) - - depth = 0 - loop do - depth += 10 - step("Fetching origin/master", %W[git fetch origin master --depth=#{depth}]) - status = step("Finding merge base with master", %W[git merge-base FETCH_HEAD #{branch}]) + def check_patch(patch_path) + step("Checking out master", %w[git checkout master]) + step("Reseting to latest master", %w[git reset --hard origin/master]) - break if status.zero? || depth > 500 - end + step("Checking if #{patch_path} applies cleanly to EE/master") + output, status = Gitlab::Popen.popen(%W[git apply --check #{patch_path}]) - raise "#{branch} is too far behind master, please rebase it!" if depth > 500 + unless status.zero? + failed_files = output.lines.reduce([]) do |memo, line| + if line.start_with?('error: patch failed:') + file = line.sub(/\Aerror: patch failed: /, '') + memo << file unless file =~ IGNORED_FILES_REGEX + end + memo + end - step("Generating the patch against master") - output, status = Gitlab::Popen.popen(%w[git format-patch FETCH_HEAD --stdout]) - throw(:halt_check, :ko) unless status.zero? + if failed_files.empty? + status = 0 + else + puts "\nConflicting files:" + failed_files.each do |file| + puts " - #{file}" + end + end + end - File.write(filepath, output) - throw(:halt_check, :ko) unless File.exist?(filepath) + status end - def delete_ee_branch_locally + def delete_ee_branch_locally! command(%w[git checkout master]) step("Deleting the local #{ee_branch} branch", %W[git branch -D #{ee_branch}]) end - def delete_patches - step("Deleting #{ce_patch_full_path}") - FileUtils.rm(ce_patch_full_path, force: true) - - step("Deleting #{ee_patch_full_path}") - FileUtils.rm(ee_patch_full_path, force: true) - end - def ce_patch_name - @ce_patch_name ||= "#{ce_branch}.patch" + @ce_patch_name ||= patch_name_from_branch(ce_branch) end def ce_patch_full_path - @ce_patch_full_path ||= File.expand_path(ce_patch_name, check_dir) + @ce_patch_full_path ||= patches_dir.join(ce_patch_name) end def ee_branch @@ -142,19 +161,26 @@ module Gitlab end def ee_patch_name - @ee_patch_name ||= "#{ee_branch}.patch" + @ee_patch_name ||= patch_name_from_branch(ee_branch) end def ee_patch_full_path - @ee_patch_full_path ||= File.expand_path(ee_patch_name, check_dir) + @ee_patch_full_path ||= patches_dir.join(ee_patch_name) + end + + def patch_name_from_branch(branch_name) + branch_name.parameterize << '.patch' end def step(desc, cmd = nil) puts "\n=> #{desc}\n" if cmd + start = Time.now puts "\n$ #{cmd.join(' ')}" - command(cmd) + status = command(cmd) + puts "\nFinished in #{Time.now - start} seconds" + status end end @@ -165,12 +191,12 @@ module Gitlab status end - def ce_applies_cleanly_msg(ce_branch) + def applies_cleanly_msg(branch) <<-MSG.strip_heredoc ================================================================= 🎉 Congratulations!! 🎉 - The #{ce_branch} branch applies cleanly to EE/master! + The #{branch} branch applies cleanly to EE/master! Much ❤️!! =================================================================\n @@ -211,7 +237,7 @@ module Gitlab # In the EE repo $ git fetch origin - $ git checkout -b #{ee_branch} FETCH_HEAD + $ git checkout -b #{ee_branch} origin/master $ git fetch #{ce_repo} #{ce_branch} $ git cherry-pick SHA # Repeat for all the commits you want to pick @@ -245,17 +271,5 @@ module Gitlab =================================================================\n MSG end - - def ee_applies_cleanly_msg - <<-MSG.strip_heredoc - ================================================================= - 🎉 Congratulations!! 🎉 - - The #{ee_branch} branch applies cleanly to EE/master! - - Much ❤️!! - =================================================================\n - MSG - end end end diff --git a/lib/gitlab/email/handler.rb b/lib/gitlab/email/handler.rb index 5cf9d5ebe28..bd2f5d3615e 100644 --- a/lib/gitlab/email/handler.rb +++ b/lib/gitlab/email/handler.rb @@ -1,11 +1,11 @@ require 'gitlab/email/handler/create_note_handler' require 'gitlab/email/handler/create_issue_handler' +require 'gitlab/email/handler/unsubscribe_handler' module Gitlab module Email module Handler - # The `CreateIssueHandler` feature is disabled for the time being. - HANDLERS = [CreateNoteHandler] + HANDLERS = [UnsubscribeHandler, CreateNoteHandler, CreateIssueHandler] def self.for(mail, mail_key) HANDLERS.find do |klass| diff --git a/lib/gitlab/email/handler/base_handler.rb b/lib/gitlab/email/handler/base_handler.rb index 7cccf465334..3f6ace0311a 100644 --- a/lib/gitlab/email/handler/base_handler.rb +++ b/lib/gitlab/email/handler/base_handler.rb @@ -9,52 +9,13 @@ module Gitlab @mail_key = mail_key end - def message - @message ||= process_message - end - - def author + def can_execute? raise NotImplementedError end - def project + def execute raise NotImplementedError end - - private - - def validate_permission!(permission) - raise UserNotFoundError unless author - raise UserBlockedError if author.blocked? - raise ProjectNotFound unless author.can?(:read_project, project) - raise UserNotAuthorizedError unless author.can?(permission, project) - end - - def process_message - message = ReplyParser.new(mail).execute.strip - add_attachments(message) - end - - def add_attachments(reply) - attachments = Email::AttachmentUploader.new(mail).execute(project) - - reply + attachments.map do |link| - "\n\n#{link[:markdown]}" - end.join - end - - def verify_record!(record:, invalid_exception:, record_name:) - return if record.persisted? - return if record.errors.key?(:commands_only) - - error_title = "The #{record_name} could not be created for the following reasons:" - - msg = error_title + record.errors.full_messages.map do |error| - "\n\n- #{error}" - end.join - - raise invalid_exception, msg - end end end end diff --git a/lib/gitlab/email/handler/create_issue_handler.rb b/lib/gitlab/email/handler/create_issue_handler.rb index 4e6566af8ab..127fae159d5 100644 --- a/lib/gitlab/email/handler/create_issue_handler.rb +++ b/lib/gitlab/email/handler/create_issue_handler.rb @@ -5,16 +5,17 @@ module Gitlab module Email module Handler class CreateIssueHandler < BaseHandler - attr_reader :project_path, :authentication_token + include ReplyProcessing + attr_reader :project_path, :incoming_email_token def initialize(mail, mail_key) super(mail, mail_key) - @project_path, @authentication_token = + @project_path, @incoming_email_token = mail_key && mail_key.split('+', 2) end def can_handle? - !authentication_token.nil? + !incoming_email_token.nil? end def execute @@ -29,7 +30,7 @@ module Gitlab end def author - @author ||= User.find_by(authentication_token: authentication_token) + @author ||= User.find_by(incoming_email_token: incoming_email_token) end def project diff --git a/lib/gitlab/email/handler/create_note_handler.rb b/lib/gitlab/email/handler/create_note_handler.rb index 447c7a6a6b9..d87ba427f4b 100644 --- a/lib/gitlab/email/handler/create_note_handler.rb +++ b/lib/gitlab/email/handler/create_note_handler.rb @@ -1,10 +1,13 @@ require 'gitlab/email/handler/base_handler' +require 'gitlab/email/handler/reply_processing' module Gitlab module Email module Handler class CreateNoteHandler < BaseHandler + include ReplyProcessing + def can_handle? mail_key =~ /\A\w+\z/ end @@ -24,6 +27,8 @@ module Gitlab record_name: 'comment') end + private + def author sent_notification.recipient end @@ -36,8 +41,6 @@ module Gitlab @sent_notification ||= SentNotification.for(mail_key) end - private - def create_note Notes::CreateService.new( project, diff --git a/lib/gitlab/email/handler/reply_processing.rb b/lib/gitlab/email/handler/reply_processing.rb new file mode 100644 index 00000000000..32c5caf93e8 --- /dev/null +++ b/lib/gitlab/email/handler/reply_processing.rb @@ -0,0 +1,54 @@ +module Gitlab + module Email + module Handler + module ReplyProcessing + private + + def author + raise NotImplementedError + end + + def project + raise NotImplementedError + end + + def message + @message ||= process_message + end + + def process_message + message = ReplyParser.new(mail).execute.strip + add_attachments(message) + end + + def add_attachments(reply) + attachments = Email::AttachmentUploader.new(mail).execute(project) + + reply + attachments.map do |link| + "\n\n#{link[:markdown]}" + end.join + end + + def validate_permission!(permission) + raise UserNotFoundError unless author + raise UserBlockedError if author.blocked? + raise ProjectNotFound unless author.can?(:read_project, project) + raise UserNotAuthorizedError unless author.can?(permission, project) + end + + def verify_record!(record:, invalid_exception:, record_name:) + return if record.persisted? + return if record.errors.key?(:commands_only) + + error_title = "The #{record_name} could not be created for the following reasons:" + + msg = error_title + record.errors.full_messages.map do |error| + "\n\n- #{error}" + end.join + + raise invalid_exception, msg + end + end + end + end +end diff --git a/lib/gitlab/email/handler/unsubscribe_handler.rb b/lib/gitlab/email/handler/unsubscribe_handler.rb new file mode 100644 index 00000000000..97d7a8d65ff --- /dev/null +++ b/lib/gitlab/email/handler/unsubscribe_handler.rb @@ -0,0 +1,32 @@ +require 'gitlab/email/handler/base_handler' + +module Gitlab + module Email + module Handler + class UnsubscribeHandler < BaseHandler + def can_handle? + mail_key =~ /\A\w+#{Regexp.escape(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX)}\z/ + end + + def execute + raise SentNotificationNotFoundError unless sent_notification + return unless sent_notification.unsubscribable? + + noteable = sent_notification.noteable + raise NoteableNotFoundError unless noteable + noteable.unsubscribe(sent_notification.recipient) + end + + private + + def sent_notification + @sent_notification ||= SentNotification.for(reply_key) + end + + def reply_key + mail_key.sub(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX, '') + end + end + end + end +end diff --git a/lib/gitlab/email/html_parser.rb b/lib/gitlab/email/html_parser.rb new file mode 100644 index 00000000000..a4ca62bfc41 --- /dev/null +++ b/lib/gitlab/email/html_parser.rb @@ -0,0 +1,34 @@ +module Gitlab + module Email + class HTMLParser + def self.parse_reply(raw_body) + new(raw_body).filtered_text + end + + attr_reader :raw_body + def initialize(raw_body) + @raw_body = raw_body + end + + def document + @document ||= Nokogiri::HTML.parse(raw_body) + end + + def filter_replies! + document.xpath('//blockquote').each(&:remove) + document.xpath('//table').each(&:remove) + end + + def filtered_html + @filtered_html ||= begin + filter_replies! + document.inner_html + end + end + + def filtered_text + @filtered_text ||= Html2Text.convert(filtered_html) + end + end + end +end diff --git a/lib/gitlab/email/reply_parser.rb b/lib/gitlab/email/reply_parser.rb index 3411eb1d9ce..8c8dd1b9cef 100644 --- a/lib/gitlab/email/reply_parser.rb +++ b/lib/gitlab/email/reply_parser.rb @@ -13,9 +13,17 @@ module Gitlab encoding = body.encoding - body = discourse_email_trimmer(body) + body = EmailReplyTrimmer.trim(body) - body = EmailReplyParser.parse_reply(body) + return '' unless body + + # not using /\s+$/ here because that deletes empty lines + body = body.gsub(/[ \t]$/, '') + + # NOTE: We currently don't support empty quotes. + # EmailReplyTrimmer allows this as a special case, + # so we detect it manually here. + return "" if body.lines.all? { |l| l.strip.empty? || l.start_with?('>') } body.force_encoding(encoding).encode("UTF-8") end @@ -23,19 +31,26 @@ module Gitlab private def select_body(message) - text = message.text_part if message.multipart? - text ||= message if message.content_type !~ /text\/html/ + if message.multipart? + part = message.text_part || message.html_part || message + else + part = message + end - return "" unless text + decoded = fix_charset(part) - text = fix_charset(text) + return "" unless decoded # Certain trigger phrases that means we didn't parse correctly - if text =~ /(Content\-Type\:|multipart\/alternative|text\/plain)/ + if decoded =~ /(Content\-Type\:|multipart\/alternative|text\/plain)/ return "" end - text + if (part.content_type || '').include? 'text/html' + HTMLParser.parse_reply(decoded) + else + decoded + end end # Force encoding to UTF-8 on a Mail::Message or Mail::Part @@ -50,30 +65,6 @@ module Gitlab rescue nil end - - REPLYING_HEADER_LABELS = %w(From Sent To Subject Reply To Cc Bcc Date) - REPLYING_HEADER_REGEX = Regexp.union(REPLYING_HEADER_LABELS.map { |label| "#{label}:" }) - - def discourse_email_trimmer(body) - lines = body.scrub.lines.to_a - range_end = 0 - - lines.each_with_index do |l, idx| - # This one might be controversial but so many reply lines have years, times and end with a colon. - # Let's try it and see how well it works. - break if (l =~ /\d{4}/ && l =~ /\d:\d\d/ && l =~ /\:$/) || - (l =~ /On \w+ \d+,? \d+,?.*wrote:/) - - # Headers on subsequent lines - break if (0..2).all? { |off| lines[idx + off] =~ REPLYING_HEADER_REGEX } - # Headers on the same line - break if REPLYING_HEADER_LABELS.count { |label| l.include?(label) } >= 3 - - range_end = idx - end - - lines[0..range_end].join.strip - end end end end diff --git a/lib/gitlab/production_logger.rb b/lib/gitlab/environment_logger.rb index 89ce7144b1b..407cc572656 100644 --- a/lib/gitlab/production_logger.rb +++ b/lib/gitlab/environment_logger.rb @@ -1,7 +1,7 @@ module Gitlab - class ProductionLogger < Gitlab::Logger + class EnvironmentLogger < Gitlab::Logger def self.file_name_noext - 'production' + Rails.env end end end diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index 7e8f35e9298..2dd42704396 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -1,66 +1,52 @@ +require 'securerandom' + module Gitlab # This class implements an 'exclusive lease'. We call it a 'lease' # because it has a set expiry time. We call it 'exclusive' because only # one caller may obtain a lease for a given key at a time. The # implementation is intended to work across GitLab processes and across - # servers. It is a 'cheap' alternative to using SQL queries and updates: + # servers. It is a cheap alternative to using SQL queries and updates: # you do not need to change the SQL schema to start using # ExclusiveLease. # - # It is important to choose the timeout wisely. If the timeout is very - # high (1 hour) then the throughput of your operation gets very low (at - # most once an hour). If the timeout is lower than how long your - # operation may take then you cannot count on exclusivity. For example, - # if the timeout is 10 seconds and you do an operation which may take 20 - # seconds then two overlapping operations may hold a lease for the same - # key at the same time. - # - # This class has no 'cancel' method. I originally decided against adding - # it because it would add complexity and a false sense of security. The - # complexity: instead of setting '1' we would have to set a UUID, and to - # delete it we would have to execute Lua on the Redis server to only - # delete the key if the value was our own UUID. Otherwise there is a - # chance that when you intend to cancel your lease you actually delete - # someone else's. The false sense of security: you cannot design your - # system to rely too much on the lease being cancelled after use because - # the calling (Ruby) process may crash or be killed. You _cannot_ count - # on begin/ensure blocks to cancel a lease, because the 'ensure' does - # not always run. Think of 'kill -9' from the Unicorn master for - # instance. - # - # If you find that leases are getting in your way, ask yourself: would - # it be enough to lower the lease timeout? Another thing that might be - # appropriate is to only use a lease for bulk/automated operations, and - # to ignore the lease when you get a single 'manual' user request (a - # button click). - # class ExclusiveLease + LUA_CANCEL_SCRIPT = <<-EOS + local key, uuid = KEYS[1], ARGV[1] + if redis.call("get", key) == uuid then + redis.call("del", key) + end + EOS + + def self.cancel(key, uuid) + Gitlab::Redis.with do |redis| + redis.eval(LUA_CANCEL_SCRIPT, keys: [redis_key(key)], argv: [uuid]) + end + end + + def self.redis_key(key) + "gitlab:exclusive_lease:#{key}" + end + def initialize(key, timeout:) - @key, @timeout = key, timeout + @redis_key = self.class.redis_key(key) + @timeout = timeout + @uuid = SecureRandom.uuid end - # Try to obtain the lease. Return true on success, + # Try to obtain the lease. Return lease UUID on success, # false if the lease is already taken. def try_obtain # Performing a single SET is atomic Gitlab::Redis.with do |redis| - !!redis.set(redis_key, '1', nx: true, ex: @timeout) + redis.set(@redis_key, @uuid, nx: true, ex: @timeout) && @uuid end end # Returns true if the key for this lease is set. def exists? Gitlab::Redis.with do |redis| - redis.exists(redis_key) + redis.exists(@redis_key) end end - - # No #cancel method. See comments above! - - private - - def redis_key - "gitlab:exclusive_lease:#{@key}" - end end end diff --git a/lib/gitlab/file_detector.rb b/lib/gitlab/file_detector.rb new file mode 100644 index 00000000000..1d93a67dc56 --- /dev/null +++ b/lib/gitlab/file_detector.rb @@ -0,0 +1,63 @@ +require 'set' + +module Gitlab + # Module that can be used to detect if a path points to a special file such as + # a README or a CONTRIBUTING file. + module FileDetector + PATTERNS = { + readme: /\Areadme/i, + changelog: /\A(changelog|history|changes|news)/i, + license: /\A(licen[sc]e|copying)(\..+|\z)/i, + contributing: /\Acontributing/i, + version: 'version', + gitignore: '.gitignore', + koding: '.koding.yml', + gitlab_ci: '.gitlab-ci.yml', + avatar: /\Alogo\.(png|jpg|gif)\z/ + } + + # Returns an Array of file types based on the given paths. + # + # This method can be used to check if a list of file paths (e.g. of changed + # files) involve any special files such as a README or a LICENSE file. + # + # Example: + # + # types_in_paths(%w{README.md foo/bar.txt}) # => [:readme] + def self.types_in_paths(paths) + types = Set.new + + paths.each do |path| + type = type_of(path) + + types << type if type + end + + types.to_a + end + + # Returns the type of a file path, or nil if none could be detected. + # + # Returned types are Symbols such as `:readme`, `:version`, etc. + # + # Example: + # + # type_of('README.md') # => :readme + # type_of('VERSION') # => :version + def self.type_of(path) + name = File.basename(path) + + PATTERNS.each do |type, search| + did_match = if search.is_a?(Regexp) + name =~ search + else + name.casecmp(search) == 0 + end + + return type if did_match + end + + nil + end + end +end diff --git a/lib/gitlab/gfm/reference_rewriter.rb b/lib/gitlab/gfm/reference_rewriter.rb index a7c596dced0..b984492d369 100644 --- a/lib/gitlab/gfm/reference_rewriter.rb +++ b/lib/gitlab/gfm/reference_rewriter.rb @@ -76,7 +76,7 @@ module Gitlab if referable.respond_to?(:project) referable.to_reference(target_project) else - referable.to_reference(@source_project, target_project) + referable.to_reference(@source_project, target_project: target_project) end end diff --git a/lib/gitlab/gfm/uploads_rewriter.rb b/lib/gitlab/gfm/uploads_rewriter.rb index abc8c8c55e6..8fab5489616 100644 --- a/lib/gitlab/gfm/uploads_rewriter.rb +++ b/lib/gitlab/gfm/uploads_rewriter.rb @@ -1,3 +1,5 @@ +require 'fileutils' + module Gitlab module Gfm ## @@ -22,7 +24,9 @@ module Gitlab return markdown unless file.try(:exists?) new_uploader = FileUploader.new(target_project) - new_uploader.store!(file) + with_link_in_tmp_dir(file.file) do |open_tmp_file| + new_uploader.store!(open_tmp_file) + end new_uploader.to_markdown end end @@ -46,6 +50,19 @@ module Gitlab uploader.retrieve_from_store!(file) uploader.file end + + # Because the uploaders use 'move_to_store' we must have a temporary + # file that is allowed to be (re)moved. + def with_link_in_tmp_dir(file) + dir = Dir.mktmpdir('UploadsRewriter', File.dirname(file)) + # The filename matters to Carrierwave so we make sure to preserve it + tmp_file = File.join(dir, File.basename(file)) + File.link(file, tmp_file) + # Open the file to placate Carrierwave + File.open(tmp_file) { |open_file| yield open_file } + ensure + FileUtils.rm_rf(dir) + end end end end diff --git a/lib/gitlab/git/attributes.rb b/lib/gitlab/git/attributes.rb new file mode 100644 index 00000000000..42140ecc993 --- /dev/null +++ b/lib/gitlab/git/attributes.rb @@ -0,0 +1,131 @@ +module Gitlab + module Git + # Class for parsing Git attribute files and extracting the attributes for + # file patterns. + # + # Unlike Rugged this parser only needs a single IO call (a call to `open`), + # vastly reducing the time spent in extracting attributes. + # + # This class _only_ supports parsing the attributes file located at + # `$GIT_DIR/info/attributes` as GitLab doesn't use any other files + # (`.gitattributes` is copied to this particular path). + # + # Basic usage: + # + # attributes = Gitlab::Git::Attributes.new(some_repo.path) + # + # attributes.attributes('README.md') # => { "eol" => "lf } + class Attributes + # path - The path to the Git repository. + def initialize(path) + @path = File.expand_path(path) + @patterns = nil + end + + # Returns all the Git attributes for the given path. + # + # path - A path to a file for which to get the attributes. + # + # Returns a Hash. + def attributes(path) + full_path = File.join(@path, path) + + patterns.each do |pattern, attrs| + return attrs if File.fnmatch?(pattern, full_path) + end + + {} + end + + # Returns a Hash containing the file patterns and their attributes. + def patterns + @patterns ||= parse_file + end + + # Parses an attribute string. + # + # These strings can be in the following formats: + # + # text # => { "text" => true } + # -text # => { "text" => false } + # key=value # => { "key" => "value" } + # + # string - The string to parse. + # + # Returns a Hash containing the attributes and their values. + def parse_attributes(string) + values = {} + dash = '-' + equal = '=' + binary = 'binary' + + string.split(/\s+/).each do |chunk| + # Data such as "foo = bar" should be treated as "foo" and "bar" being + # separate boolean attributes. + next if chunk == equal + + key = chunk + + # Input: "-foo" + if chunk.start_with?(dash) + key = chunk.byteslice(1, chunk.length - 1) + value = false + + # Input: "foo=bar" + elsif chunk.include?(equal) + key, value = chunk.split(equal, 2) + + # Input: "foo" + else + value = true + end + + values[key] = value + + # When the "binary" option is set the "diff" option should be set to + # the inverse. If "diff" is later set it should overwrite the + # automatically set value. + values['diff'] = false if key == binary && value + end + + values + end + + # Iterates over every line in the attributes file. + def each_line + full_path = File.join(@path, 'info/attributes') + + return unless File.exist?(full_path) + + File.open(full_path, 'r') do |handle| + handle.each_line do |line| + break unless line.valid_encoding? + + yield line.strip + end + end + end + + private + + # Parses the Git attributes file. + def parse_file + pairs = [] + comment = '#' + + each_line do |line| + next if line.start_with?(comment) || line.empty? + + pattern, attrs = line.split(/\s+/, 2) + + parsed = attrs ? parse_attributes(attrs) : {} + + pairs << [File.join(@path, pattern), parsed] + end + + # Newer entries take precedence over older entries. + pairs.reverse.to_h + end + end + end +end diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb new file mode 100644 index 00000000000..58193391926 --- /dev/null +++ b/lib/gitlab/git/blame.rb @@ -0,0 +1,75 @@ +module Gitlab + module Git + class Blame + include Gitlab::Git::EncodingHelper + + attr_reader :lines, :blames + + def initialize(repository, sha, path) + @repo = repository + @sha = sha + @path = path + @lines = [] + @blames = load_blame + end + + def each + @blames.each do |blame| + yield( + Gitlab::Git::Commit.new(blame.commit), + blame.line + ) + end + end + + private + + def load_blame + cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{@repo.path} blame -p #{@sha} -- #{@path}) + # Read in binary mode to ensure ASCII-8BIT + raw_output = IO.popen(cmd, 'rb') {|io| io.read } + output = encode_utf8(raw_output) + process_raw_blame output + end + + def process_raw_blame(output) + lines, final = [], [] + info, commits = {}, {} + + # process the output + output.split("\n").each do |line| + if line[0, 1] == "\t" + lines << line[1, line.size] + elsif m = /^(\w{40}) (\d+) (\d+)/.match(line) + commit_id, old_lineno, lineno = m[1], m[2].to_i, m[3].to_i + commits[commit_id] = nil unless commits.key?(commit_id) + info[lineno] = [commit_id, old_lineno] + end + end + + # load all commits in single call + commits.keys.each do |key| + commits[key] = @repo.lookup(key) + end + + # get it together + info.sort.each do |lineno, (commit_id, old_lineno)| + commit = commits[commit_id] + final << BlameLine.new(lineno, old_lineno, commit, lines[lineno - 1]) + end + + @lines = final + end + end + + class BlameLine + attr_accessor :lineno, :oldlineno, :commit, :line + def initialize(lineno, oldlineno, commit, line) + @lineno = lineno + @oldlineno = oldlineno + @commit = commit + @line = line + end + end + end +end diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb new file mode 100644 index 00000000000..b742d9e1e4b --- /dev/null +++ b/lib/gitlab/git/blob.rb @@ -0,0 +1,330 @@ +module Gitlab + module Git + class Blob + include Linguist::BlobHelper + include Gitlab::Git::EncodingHelper + + # This number is the maximum amount of data that we want to display to + # the user. We load as much as we can for encoding detection + # (Linguist) and LFS pointer parsing. All other cases where we need full + # blob data should use load_all_data!. + MAX_DATA_DISPLAY_SIZE = 10485760 + + attr_accessor :name, :path, :size, :data, :mode, :id, :commit_id, :loaded_size, :binary + + class << self + def find(repository, sha, path) + commit = repository.lookup(sha) + root_tree = commit.tree + + blob_entry = find_entry_by_path(repository, root_tree.oid, path) + + return nil unless blob_entry + + if blob_entry[:type] == :commit + submodule_blob(blob_entry, path, sha) + else + blob = repository.lookup(blob_entry[:oid]) + + if blob + new( + id: blob.oid, + name: blob_entry[:name], + size: blob.size, + data: blob.content(MAX_DATA_DISPLAY_SIZE), + mode: blob_entry[:filemode].to_s(8), + path: path, + commit_id: sha, + binary: blob.binary? + ) + end + end + end + + def raw(repository, sha) + blob = repository.lookup(sha) + + new( + id: blob.oid, + size: blob.size, + data: blob.content(MAX_DATA_DISPLAY_SIZE), + binary: blob.binary? + ) + end + + # Recursive search of blob id by path + # + # Ex. + # blog/ # oid: 1a + # app/ # oid: 2a + # models/ # oid: 3a + # file.rb # oid: 4a + # + # + # Blob.find_entry_by_path(repo, '1a', 'app/file.rb') # => '4a' + # + def find_entry_by_path(repository, root_id, path) + root_tree = repository.lookup(root_id) + # Strip leading slashes + path[/^\/*/] = '' + path_arr = path.split('/') + + entry = root_tree.find do |entry| + entry[:name] == path_arr[0] + end + + return nil unless entry + + if path_arr.size > 1 + return nil unless entry[:type] == :tree + path_arr.shift + find_entry_by_path(repository, entry[:oid], path_arr.join('/')) + else + [:blob, :commit].include?(entry[:type]) ? entry : nil + end + end + + def submodule_blob(blob_entry, path, sha) + new( + id: blob_entry[:oid], + name: blob_entry[:name], + data: '', + path: path, + commit_id: sha, + ) + end + + # Commit file in repository and return commit sha + # + # options should contain next structure: + # file: { + # content: 'Lorem ipsum...', + # path: 'documents/story.txt', + # update: true + # }, + # author: { + # email: 'user@example.com', + # name: 'Test User', + # time: Time.now + # }, + # committer: { + # email: 'user@example.com', + # name: 'Test User', + # time: Time.now + # }, + # commit: { + # message: 'Wow such commit', + # branch: 'master', + # update_ref: false + # } + # + # rubocop:disable Metrics/AbcSize + # rubocop:disable Metrics/CyclomaticComplexity + # rubocop:disable Metrics/PerceivedComplexity + def commit(repository, options, action = :add) + file = options[:file] + update = file[:update].nil? ? true : file[:update] + author = options[:author] + committer = options[:committer] + commit = options[:commit] + repo = repository.rugged + ref = commit[:branch] + update_ref = commit[:update_ref].nil? ? true : commit[:update_ref] + parents = [] + mode = 0o100644 + + unless ref.start_with?('refs/') + ref = 'refs/heads/' + ref + end + + path_name = Gitlab::Git::PathHelper.normalize_path(file[:path]) + # Abort if any invalid characters remain (e.g. ../foo) + raise Gitlab::Git::Repository::InvalidBlobName.new("Invalid path") if path_name.each_filename.to_a.include?('..') + + filename = path_name.to_s + index = repo.index + + unless repo.empty? + rugged_ref = repo.references[ref] + raise Gitlab::Git::Repository::InvalidRef.new("Invalid branch name") unless rugged_ref + last_commit = rugged_ref.target + index.read_tree(last_commit.tree) + parents = [last_commit] + end + + if action == :remove + index.remove(filename) + else + file_entry = index.get(filename) + + if action == :rename + old_path_name = Gitlab::Git::PathHelper.normalize_path(file[:previous_path]) + old_filename = old_path_name.to_s + file_entry = index.get(old_filename) + index.remove(old_filename) unless file_entry.blank? + end + + if file_entry + raise Gitlab::Git::Repository::InvalidBlobName.new("Filename already exists; update not allowed") unless update + + # Preserve the current file mode if one is available + mode = file_entry[:mode] if file_entry[:mode] + end + + content = file[:content] + detect = CharlockHolmes::EncodingDetector.new.detect(content) if content + + unless detect && detect[:type] == :binary + # When writing to the repo directly as we are doing here, + # the `core.autocrlf` config isn't taken into account. + content.gsub!("\r\n", "\n") if repository.autocrlf + end + + oid = repo.write(content, :blob) + index.add(path: filename, oid: oid, mode: mode) + end + + opts = {} + opts[:tree] = index.write_tree(repo) + opts[:author] = author + opts[:committer] = committer + opts[:message] = commit[:message] + opts[:parents] = parents + opts[:update_ref] = ref if update_ref + + Rugged::Commit.create(repo, opts) + end + # rubocop:enable Metrics/AbcSize + # rubocop:enable Metrics/CyclomaticComplexity + # rubocop:enable Metrics/PerceivedComplexity + + # Remove file from repository and return commit sha + # + # options should contain next structure: + # file: { + # path: 'documents/story.txt' + # }, + # author: { + # email: 'user@example.com', + # name: 'Test User', + # time: Time.now + # }, + # committer: { + # email: 'user@example.com', + # name: 'Test User', + # time: Time.now + # }, + # commit: { + # message: 'Remove FILENAME', + # branch: 'master' + # } + # + def remove(repository, options) + commit(repository, options, :remove) + end + + # Rename file from repository and return commit sha + # + # options should contain next structure: + # file: { + # previous_path: 'documents/old_story.txt' + # path: 'documents/story.txt' + # content: 'Lorem ipsum...', + # update: true + # }, + # author: { + # email: 'user@example.com', + # name: 'Test User', + # time: Time.now + # }, + # committer: { + # email: 'user@example.com', + # name: 'Test User', + # time: Time.now + # }, + # commit: { + # message: 'Rename FILENAME', + # branch: 'master' + # } + # + def rename(repository, options) + commit(repository, options, :rename) + end + end + + def initialize(options) + %w(id name path size data mode commit_id binary).each do |key| + self.send("#{key}=", options[key.to_sym]) + end + + @loaded_all_data = false + # Retain the actual size before it is encoded + @loaded_size = @data.bytesize if @data + end + + def binary? + @binary.nil? ? super : @binary == true + end + + def empty? + !data || data == '' + end + + def data + encode! @data + end + + # Load all blob data (not just the first MAX_DATA_DISPLAY_SIZE bytes) into + # memory as a Ruby string. + def load_all_data!(repository) + return if @data == '' # don't mess with submodule blobs + return @data if @loaded_all_data + + @loaded_all_data = true + @data = repository.lookup(id).content + @loaded_size = @data.bytesize + end + + def name + encode! @name + end + + # Valid LFS object pointer is a text file consisting of + # version + # oid + # size + # see https://github.com/github/git-lfs/blob/v1.1.0/docs/spec.md#the-pointer + def lfs_pointer? + has_lfs_version_key? && lfs_oid.present? && lfs_size.present? + end + + def lfs_oid + if has_lfs_version_key? + oid = data.match(/(?<=sha256:)([0-9a-f]{64})/) + return oid[1] if oid + end + + nil + end + + def lfs_size + if has_lfs_version_key? + size = data.match(/(?<=size )([0-9]+)/) + return size[1] if size + end + + nil + end + + def truncated? + size && (size > loaded_size) + end + + private + + def has_lfs_version_key? + !empty? && text? && data.start_with?("version https://git-lfs.github.com/spec") + end + end + end +end diff --git a/lib/gitlab/git/blob_snippet.rb b/lib/gitlab/git/blob_snippet.rb new file mode 100644 index 00000000000..e98de57fc22 --- /dev/null +++ b/lib/gitlab/git/blob_snippet.rb @@ -0,0 +1,32 @@ +module Gitlab + module Git + class BlobSnippet + include Linguist::BlobHelper + + attr_accessor :ref + attr_accessor :lines + attr_accessor :filename + attr_accessor :startline + + def initialize(ref, lines, startline, filename) + @ref, @lines, @startline, @filename = ref, lines, startline, filename + end + + def data + lines.join("\n") if lines + end + + def name + filename + end + + def size + data.length + end + + def mode + nil + end + end + end +end diff --git a/lib/gitlab/git/branch.rb b/lib/gitlab/git/branch.rb new file mode 100644 index 00000000000..586380da94a --- /dev/null +++ b/lib/gitlab/git/branch.rb @@ -0,0 +1,6 @@ +module Gitlab + module Git + class Branch < Ref + end + end +end diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb new file mode 100644 index 00000000000..d785516ebdd --- /dev/null +++ b/lib/gitlab/git/commit.rb @@ -0,0 +1,310 @@ +# Gitlab::Git::Commit is a wrapper around native Rugged::Commit object +module Gitlab + module Git + class Commit + include Gitlab::Git::EncodingHelper + + attr_accessor :raw_commit, :head, :refs + + SERIALIZE_KEYS = [ + :id, :message, :parent_ids, + :authored_date, :author_name, :author_email, + :committed_date, :committer_name, :committer_email + ].freeze + + attr_accessor *SERIALIZE_KEYS # rubocop:disable Lint/AmbiguousOperator + + def ==(other) + return false unless other.is_a?(Gitlab::Git::Commit) + + methods = [:message, :parent_ids, :authored_date, :author_name, + :author_email, :committed_date, :committer_name, + :committer_email] + + methods.all? do |method| + send(method) == other.send(method) + end + end + + class << self + # Get commits collection + # + # Ex. + # Commit.where( + # repo: repo, + # ref: 'master', + # path: 'app/models', + # limit: 10, + # offset: 5, + # ) + # + def where(options) + repo = options.delete(:repo) + raise 'Gitlab::Git::Repository is required' unless repo.respond_to?(:log) + + repo.log(options).map { |c| decorate(c) } + end + + # Get single commit + # + # Ex. + # Commit.find(repo, '29eda46b') + # + # Commit.find(repo, 'master') + # + def find(repo, commit_id = "HEAD") + return decorate(commit_id) if commit_id.is_a?(Rugged::Commit) + + obj = if commit_id.is_a?(String) + repo.rev_parse_target(commit_id) + else + Gitlab::Git::Ref.dereference_object(commit_id) + end + + return nil unless obj.is_a?(Rugged::Commit) + + decorate(obj) + rescue Rugged::ReferenceError, Rugged::InvalidError, Rugged::ObjectError, Gitlab::Git::Repository::NoRepository + nil + end + + # Get last commit for HEAD + # + # Ex. + # Commit.last(repo) + # + def last(repo) + find(repo) + end + + # Get last commit for specified path and ref + # + # Ex. + # Commit.last_for_path(repo, '29eda46b', 'app/models') + # + # Commit.last_for_path(repo, 'master', 'Gemfile') + # + def last_for_path(repo, ref, path = nil) + where( + repo: repo, + ref: ref, + path: path, + limit: 1 + ).first + end + + # Get commits between two revspecs + # See also #repository.commits_between + # + # Ex. + # Commit.between(repo, '29eda46b', 'master') + # + def between(repo, base, head) + repo.commits_between(base, head).map do |commit| + decorate(commit) + end + rescue Rugged::ReferenceError + [] + end + + # Delegate Repository#find_commits + def find_all(repo, options = {}) + repo.find_commits(options) + end + + def decorate(commit, ref = nil) + Gitlab::Git::Commit.new(commit, ref) + end + + # Returns a diff object for the changes introduced by +rugged_commit+. + # If +rugged_commit+ doesn't have a parent, then the diff is between + # this commit and an empty repo. See Repository#diff for the keys + # allowed in the +options+ hash. + def diff_from_parent(rugged_commit, options = {}) + options ||= {} + break_rewrites = options[:break_rewrites] + actual_options = Gitlab::Git::Diff.filter_diff_options(options) + + diff = if rugged_commit.parents.empty? + rugged_commit.diff(actual_options.merge(reverse: true)) + else + rugged_commit.parents[0].diff(rugged_commit, actual_options) + end + + diff.find_similar!(break_rewrites: break_rewrites) + diff + end + end + + def initialize(raw_commit, head = nil) + raise "Nil as raw commit passed" unless raw_commit + + if raw_commit.is_a?(Hash) + init_from_hash(raw_commit) + elsif raw_commit.is_a?(Rugged::Commit) + init_from_rugged(raw_commit) + else + raise "Invalid raw commit type: #{raw_commit.class}" + end + + @head = head + end + + def sha + id + end + + def short_id(length = 10) + id.to_s[0..length] + end + + def safe_message + @safe_message ||= message + end + + def created_at + committed_date + end + + # Was this commit committed by a different person than the original author? + def different_committer? + author_name != committer_name || author_email != committer_email + end + + def parent_id + parent_ids.first + end + + # Shows the diff between the commit's parent and the commit. + # + # Cuts out the header and stats from #to_patch and returns only the diff. + def to_diff(options = {}) + diff_from_parent(options).patch + end + + # Returns a diff object for the changes from this commit's first parent. + # If there is no parent, then the diff is between this commit and an + # empty repo. See Repository#diff for keys allowed in the +options+ + # hash. + def diff_from_parent(options = {}) + Commit.diff_from_parent(raw_commit, options) + end + + def has_zero_stats? + stats.total.zero? + rescue + true + end + + def no_commit_message + "--no commit message" + end + + def to_hash + serialize_keys.map.with_object({}) do |key, hash| + hash[key] = send(key) + end + end + + def date + committed_date + end + + def diffs(options = {}) + Gitlab::Git::DiffCollection.new(diff_from_parent(options), options) + end + + def parents + raw_commit.parents.map { |c| Gitlab::Git::Commit.new(c) } + end + + def tree + raw_commit.tree + end + + def stats + Gitlab::Git::CommitStats.new(self) + end + + def to_patch(options = {}) + begin + raw_commit.to_mbox(options) + rescue Rugged::InvalidError => ex + if ex.message =~ /Commit \w+ is a merge commit/ + 'Patch format is not currently supported for merge commits.' + end + end + end + + # Get a collection of Rugged::Reference objects for this commit. + # + # Ex. + # commit.ref(repo) + # + def refs(repo) + repo.refs_hash[id] + end + + # Get ref names collection + # + # Ex. + # commit.ref_names(repo) + # + def ref_names(repo) + refs(repo).map do |ref| + ref.name.sub(%r{^refs/(heads|remotes|tags)/}, "") + end + end + + def message + encode! @message + end + + def author_name + encode! @author_name + end + + def author_email + encode! @author_email + end + + def committer_name + encode! @committer_name + end + + def committer_email + encode! @committer_email + end + + private + + def init_from_hash(hash) + raw_commit = hash.symbolize_keys + + serialize_keys.each do |key| + send("#{key}=", raw_commit[key]) + end + end + + def init_from_rugged(commit) + author = commit.author + committer = commit.committer + + @raw_commit = commit + @id = commit.oid + @message = commit.message + @authored_date = author[:time] + @committed_date = committer[:time] + @author_name = author[:name] + @author_email = author[:email] + @committer_name = committer[:name] + @committer_email = committer[:email] + @parent_ids = commit.parents.map(&:oid) + end + + def serialize_keys + SERIALIZE_KEYS + end + end + end +end diff --git a/lib/gitlab/git/commit_stats.rb b/lib/gitlab/git/commit_stats.rb new file mode 100644 index 00000000000..e9118bbed0e --- /dev/null +++ b/lib/gitlab/git/commit_stats.rb @@ -0,0 +1,26 @@ +# Gitlab::Git::CommitStats counts the additions, deletions, and total changes +# in a commit. +module Gitlab + module Git + class CommitStats + attr_reader :id, :additions, :deletions, :total + + # Instantiate a CommitStats object + def initialize(commit) + @id = commit.id + @additions = 0 + @deletions = 0 + @total = 0 + + diff = commit.diff_from_parent + + diff.each_patch do |p| + # TODO: Use the new Rugged convenience methods when they're released + @additions += p.stat[0] + @deletions += p.stat[1] + @total += p.changes + end + end + end + end +end diff --git a/lib/gitlab/git/compare.rb b/lib/gitlab/git/compare.rb new file mode 100644 index 00000000000..696a2acd5e3 --- /dev/null +++ b/lib/gitlab/git/compare.rb @@ -0,0 +1,43 @@ +module Gitlab + module Git + class Compare + attr_reader :head, :base, :straight + + def initialize(repository, base, head, straight = false) + @repository = repository + @straight = straight + + unless base && head + @commits = [] + return + end + + @base = Gitlab::Git::Commit.find(repository, base.try(:strip)) + @head = Gitlab::Git::Commit.find(repository, head.try(:strip)) + + @commits = [] unless @base && @head + @commits = [] if same + end + + def same + @base && @head && @base.id == @head.id + end + + def commits + return @commits if defined?(@commits) + + @commits = Gitlab::Git::Commit.between(@repository, @base.id, @head.id) + end + + def diffs(options = {}) + unless @head && @base + return Gitlab::Git::DiffCollection.new([]) + end + + paths = options.delete(:paths) || [] + options[:straight] = @straight + Gitlab::Git::Diff.between(@repository, @head.id, @base.id, options, *paths) + end + end + end +end diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb new file mode 100644 index 00000000000..d6b3b5705a9 --- /dev/null +++ b/lib/gitlab/git/diff.rb @@ -0,0 +1,322 @@ +# Gitlab::Git::Diff is a wrapper around native Rugged::Diff object +module Gitlab + module Git + class Diff + class TimeoutError < StandardError; end + include Gitlab::Git::EncodingHelper + + # Diff properties + attr_accessor :old_path, :new_path, :a_mode, :b_mode, :diff + + # Stats properties + attr_accessor :new_file, :renamed_file, :deleted_file + + attr_accessor :too_large + + # The maximum size of a diff to display. + DIFF_SIZE_LIMIT = 102400 # 100 KB + + # The maximum size before a diff is collapsed. + DIFF_COLLAPSE_LIMIT = 10240 # 10 KB + + class << self + def between(repo, head, base, options = {}, *paths) + straight = options.delete(:straight) || false + + common_commit = if straight + base + else + # Only show what is new in the source branch + # compared to the target branch, not the other way + # around. The linex below with merge_base is + # equivalent to diff with three dots (git diff + # branch1...branch2) From the git documentation: + # "git diff A...B" is equivalent to "git diff + # $(git-merge-base A B) B" + repo.merge_base_commit(head, base) + end + + options ||= {} + actual_options = filter_diff_options(options) + repo.diff(common_commit, head, actual_options, *paths) + end + + # Return a copy of the +options+ hash containing only keys that can be + # passed to Rugged. Allowed options are: + # + # :max_size :: + # An integer specifying the maximum byte size of a file before a it + # will be treated as binary. The default value is 512MB. + # + # :context_lines :: + # The number of unchanged lines that define the boundary of a hunk + # (and to display before and after the actual changes). The default is + # 3. + # + # :interhunk_lines :: + # The maximum number of unchanged lines between hunk boundaries before + # the hunks will be merged into a one. The default is 0. + # + # :old_prefix :: + # The virtual "directory" to prefix to old filenames in hunk headers. + # The default is "a". + # + # :new_prefix :: + # The virtual "directory" to prefix to new filenames in hunk headers. + # The default is "b". + # + # :reverse :: + # If true, the sides of the diff will be reversed. + # + # :force_text :: + # If true, all files will be treated as text, disabling binary + # attributes & detection. + # + # :ignore_whitespace :: + # If true, all whitespace will be ignored. + # + # :ignore_whitespace_change :: + # If true, changes in amount of whitespace will be ignored. + # + # :ignore_whitespace_eol :: + # If true, whitespace at end of line will be ignored. + # + # :ignore_submodules :: + # if true, submodules will be excluded from the diff completely. + # + # :patience :: + # If true, the "patience diff" algorithm will be used (currenlty + # unimplemented). + # + # :include_ignored :: + # If true, ignored files will be included in the diff. + # + # :include_untracked :: + # If true, untracked files will be included in the diff. + # + # :include_unmodified :: + # If true, unmodified files will be included in the diff. + # + # :recurse_untracked_dirs :: + # Even if +:include_untracked+ is true, untracked directories will + # only be marked with a single entry in the diff. If this flag is set + # to true, all files under ignored directories will be included in the + # diff, too. + # + # :disable_pathspec_match :: + # If true, the given +*paths+ will be applied as exact matches, + # instead of as fnmatch patterns. + # + # :deltas_are_icase :: + # If true, filename comparisons will be made with case-insensitivity. + # + # :include_untracked_content :: + # if true, untracked content will be contained in the the diff patch + # text. + # + # :skip_binary_check :: + # If true, diff deltas will be generated without spending time on + # binary detection. This is useful to improve performance in cases + # where the actual file content difference is not needed. + # + # :include_typechange :: + # If true, type changes for files will not be interpreted as deletion + # of the "old file" and addition of the "new file", but will generate + # typechange records. + # + # :include_typechange_trees :: + # Even if +:include_typechange+ is true, blob -> tree changes will + # still usually be handled as a deletion of the blob. If this flag is + # set to true, blob -> tree changes will be marked as typechanges. + # + # :ignore_filemode :: + # If true, file mode changes will be ignored. + # + # :recurse_ignored_dirs :: + # Even if +:include_ignored+ is true, ignored directories will only be + # marked with a single entry in the diff. If this flag is set to true, + # all files under ignored directories will be included in the diff, + # too. + def filter_diff_options(options, default_options = {}) + allowed_options = [:max_size, :context_lines, :interhunk_lines, + :old_prefix, :new_prefix, :reverse, :force_text, + :ignore_whitespace, :ignore_whitespace_change, + :ignore_whitespace_eol, :ignore_submodules, + :patience, :include_ignored, :include_untracked, + :include_unmodified, :recurse_untracked_dirs, + :disable_pathspec_match, :deltas_are_icase, + :include_untracked_content, :skip_binary_check, + :include_typechange, :include_typechange_trees, + :ignore_filemode, :recurse_ignored_dirs, :paths, + :max_files, :max_lines, :all_diffs, :no_collapse] + + if default_options + actual_defaults = default_options.dup + actual_defaults.keep_if do |key| + allowed_options.include?(key) + end + else + actual_defaults = {} + end + + if options + filtered_opts = options.dup + filtered_opts.keep_if do |key| + allowed_options.include?(key) + end + filtered_opts = actual_defaults.merge(filtered_opts) + else + filtered_opts = actual_defaults + end + + filtered_opts + end + end + + def initialize(raw_diff, collapse: false) + case raw_diff + when Hash + init_from_hash(raw_diff, collapse: collapse) + when Rugged::Patch, Rugged::Diff::Delta + init_from_rugged(raw_diff, collapse: collapse) + when nil + raise "Nil as raw diff passed" + else + raise "Invalid raw diff type: #{raw_diff.class}" + end + end + + def serialize_keys + @serialize_keys ||= %i(diff new_path old_path a_mode b_mode new_file renamed_file deleted_file too_large) + end + + def to_hash + hash = {} + + keys = serialize_keys + + keys.each do |key| + hash[key] = send(key) + end + + hash + end + + def submodule? + a_mode == '160000' || b_mode == '160000' + end + + def line_count + @line_count ||= Util.count_lines(@diff) + end + + def too_large? + if @too_large.nil? + @too_large = @diff.bytesize >= DIFF_SIZE_LIMIT + else + @too_large + end + end + + def collapsible? + @diff.bytesize >= DIFF_COLLAPSE_LIMIT + end + + def prune_large_diff! + @diff = '' + @line_count = 0 + @too_large = true + end + + def collapsed? + return @collapsed if defined?(@collapsed) + false + end + + def prune_collapsed_diff! + @diff = '' + @line_count = 0 + @collapsed = true + end + + private + + def init_from_rugged(rugged, collapse: false) + if rugged.is_a?(Rugged::Patch) + init_from_rugged_patch(rugged, collapse: collapse) + d = rugged.delta + else + d = rugged + end + + @new_path = encode!(d.new_file[:path]) + @old_path = encode!(d.old_file[:path]) + @a_mode = d.old_file[:mode].to_s(8) + @b_mode = d.new_file[:mode].to_s(8) + @new_file = d.added? + @renamed_file = d.renamed? + @deleted_file = d.deleted? + end + + def init_from_rugged_patch(patch, collapse: false) + # Don't bother initializing diffs that are too large. If a diff is + # binary we're not going to display anything so we skip the size check. + return if !patch.delta.binary? && prune_large_patch(patch, collapse) + + @diff = encode!(strip_diff_headers(patch.to_s)) + end + + def init_from_hash(hash, collapse: false) + raw_diff = hash.symbolize_keys + + serialize_keys.each do |key| + send(:"#{key}=", raw_diff[key.to_sym]) + end + + prune_large_diff! if too_large? + prune_collapsed_diff! if collapse && collapsible? + end + + # If the patch surpasses any of the diff limits it calls the appropiate + # prune method and returns true. Otherwise returns false. + def prune_large_patch(patch, collapse) + size = 0 + + patch.each_hunk do |hunk| + hunk.each_line do |line| + size += line.content.bytesize + + if size >= DIFF_SIZE_LIMIT + prune_large_diff! + return true + end + end + end + + if collapse && size >= DIFF_COLLAPSE_LIMIT + prune_collapsed_diff! + return true + end + + false + end + + # Strip out the information at the beginning of the patch's text to match + # Grit's output + def strip_diff_headers(diff_text) + # Delete everything up to the first line that starts with '---' or + # 'Binary' + diff_text.sub!(/\A.*?^(---|Binary)/m, '\1') + + if diff_text.start_with?('---', 'Binary') + diff_text + else + # If the diff_text did not contain a line starting with '---' or + # 'Binary', return the empty string. No idea why; we are just + # preserving behavior from before the refactor. + '' + end + end + end + end +end diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb new file mode 100644 index 00000000000..65e06f5065d --- /dev/null +++ b/lib/gitlab/git/diff_collection.rb @@ -0,0 +1,129 @@ +module Gitlab + module Git + class DiffCollection + include Enumerable + + DEFAULT_LIMITS = { max_files: 100, max_lines: 5000 }.freeze + + def initialize(iterator, options = {}) + @iterator = iterator + @max_files = options.fetch(:max_files, DEFAULT_LIMITS[:max_files]) + @max_lines = options.fetch(:max_lines, DEFAULT_LIMITS[:max_lines]) + @max_bytes = @max_files * 5120 # Average 5 KB per file + @safe_max_files = [@max_files, DEFAULT_LIMITS[:max_files]].min + @safe_max_lines = [@max_lines, DEFAULT_LIMITS[:max_lines]].min + @safe_max_bytes = @safe_max_files * 5120 # Average 5 KB per file + @all_diffs = !!options.fetch(:all_diffs, false) + @no_collapse = !!options.fetch(:no_collapse, true) + @deltas_only = !!options.fetch(:deltas_only, false) + + @line_count = 0 + @byte_count = 0 + @overflow = false + @array = Array.new + end + + def each(&block) + if @populated + # @iterator.each is slower than just iterating the array in place + @array.each(&block) + elsif @deltas_only + each_delta(&block) + else + each_patch(&block) + end + end + + def empty? + !@iterator.any? + end + + def overflow? + populate! + !!@overflow + end + + def size + @size ||= count # forces a loop using each method + end + + def real_size + populate! + + if @overflow + "#{size}+" + else + size.to_s + end + end + + def decorate! + collection = each_with_index do |element, i| + @array[i] = yield(element) + end + @populated = true + collection + end + + private + + def populate! + return if @populated + + each { nil } # force a loop through all diffs + @populated = true + nil + end + + def over_safe_limits?(files) + files >= @safe_max_files || @line_count > @safe_max_lines || @byte_count >= @safe_max_bytes + end + + def each_delta + @iterator.each_delta.with_index do |delta, i| + diff = Gitlab::Git::Diff.new(delta) + + yield @array[i] = diff + end + end + + def each_patch + @iterator.each_with_index do |raw, i| + # First yield cached Diff instances from @array + if @array[i] + yield @array[i] + next + end + + # We have exhausted @array, time to create new Diff instances or stop. + break if @overflow + + if !@all_diffs && i >= @max_files + @overflow = true + break + end + + collapse = !@all_diffs && !@no_collapse + + diff = Gitlab::Git::Diff.new(raw, collapse: collapse) + + if collapse && over_safe_limits?(i) + diff.prune_collapsed_diff! + end + + @line_count += diff.line_count + @byte_count += diff.diff.bytesize + + if !@all_diffs && (@line_count >= @max_lines || @byte_count >= @max_bytes) + # This last Diff instance pushes us over the lines limit. We stop and + # discard it. + @overflow = true + break + end + + yield @array[i] = diff + end + end + end + end +end diff --git a/lib/gitlab/git/encoding_helper.rb b/lib/gitlab/git/encoding_helper.rb new file mode 100644 index 00000000000..e57d228e688 --- /dev/null +++ b/lib/gitlab/git/encoding_helper.rb @@ -0,0 +1,58 @@ +module Gitlab + module Git + module EncodingHelper + extend self + + # This threshold is carefully tweaked to prevent usage of encodings detected + # by CharlockHolmes with low confidence. If CharlockHolmes confidence is low, + # we're better off sticking with utf8 encoding. + # Reason: git diff can return strings with invalid utf8 byte sequences if it + # truncates a diff in the middle of a multibyte character. In this case + # CharlockHolmes will try to guess the encoding and will likely suggest an + # obscure encoding with low confidence. + # There is a lot more info with this merge request: + # https://gitlab.com/gitlab-org/gitlab_git/merge_requests/77#note_4754193 + ENCODING_CONFIDENCE_THRESHOLD = 40 + + def encode!(message) + return nil unless message.respond_to? :force_encoding + + # if message is utf-8 encoding, just return it + message.force_encoding("UTF-8") + return message if message.valid_encoding? + + # return message if message type is binary + detect = CharlockHolmes::EncodingDetector.detect(message) + return message.force_encoding("BINARY") if detect && detect[:type] == :binary + + # force detected encoding if we have sufficient confidence. + if detect && detect[:encoding] && detect[:confidence] > ENCODING_CONFIDENCE_THRESHOLD + message.force_encoding(detect[:encoding]) + end + + # encode and clean the bad chars + message.replace clean(message) + rescue + encoding = detect ? detect[:encoding] : "unknown" + "--broken encoding: #{encoding}" + end + + def encode_utf8(message) + detect = CharlockHolmes::EncodingDetector.detect(message) + if detect + CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8') + else + clean(message) + end + end + + private + + def clean(message) + message.encode("UTF-16BE", undef: :replace, invalid: :replace, replace: "") + .encode("UTF-8") + .gsub("\0".encode("UTF-8"), "") + end + end + end +end diff --git a/lib/gitlab/git/path_helper.rb b/lib/gitlab/git/path_helper.rb new file mode 100644 index 00000000000..0148cd8df05 --- /dev/null +++ b/lib/gitlab/git/path_helper.rb @@ -0,0 +1,16 @@ +module Gitlab + module Git + class PathHelper + class << self + def normalize_path(filename) + # Strip all leading slashes so that //foo -> foo + filename[/^\/*/] = '' + + # Expand relative paths (e.g. foo/../bar) + filename = Pathname.new(filename) + filename.relative_path_from(Pathname.new('')) + end + end + end + end +end diff --git a/lib/gitlab/git/popen.rb b/lib/gitlab/git/popen.rb new file mode 100644 index 00000000000..df9ca3ee5ac --- /dev/null +++ b/lib/gitlab/git/popen.rb @@ -0,0 +1,26 @@ +require 'open3' + +module Gitlab + module Git + module Popen + def popen(cmd, path) + unless cmd.is_a?(Array) + raise "System commands must be given as an array of strings" + end + + vars = { "PWD" => path } + options = { chdir: path } + + @cmd_output = "" + @cmd_status = 0 + Open3.popen3(vars, *cmd, options) do |stdin, stdout, stderr, wait_thr| + @cmd_output << stdout.read + @cmd_output << stderr.read + @cmd_status = wait_thr.value.exitstatus + end + + [@cmd_output, @cmd_status] + end + end + end +end diff --git a/lib/gitlab/git/ref.rb b/lib/gitlab/git/ref.rb new file mode 100644 index 00000000000..37ef6836742 --- /dev/null +++ b/lib/gitlab/git/ref.rb @@ -0,0 +1,49 @@ +module Gitlab + module Git + class Ref + include Gitlab::Git::EncodingHelper + + # Branch or tag name + # without "refs/tags|heads" prefix + attr_reader :name + + # Target sha. + # Usually it is commit sha but in case + # when tag reference on other tag it can be tag sha + attr_reader :target + + # Dereferenced target + # Commit object to which the Ref points to + attr_reader :dereferenced_target + + # Extract branch name from full ref path + # + # Ex. + # Ref.extract_branch_name('refs/heads/master') #=> 'master' + def self.extract_branch_name(str) + str.gsub(/\Arefs\/heads\//, '') + end + + def self.dereference_object(object) + object = object.target while object.is_a?(Rugged::Tag::Annotation) + + object + end + + def initialize(repository, name, target) + encode! name + @name = name.gsub(/\Arefs\/(tags|heads)\//, '') + @dereferenced_target = Gitlab::Git::Commit.find(repository, target) + @target = if target.respond_to?(:oid) + target.oid + elsif target.respond_to?(:name) + target.name + elsif target.is_a? String + target + else + nil + end + end + end + end +end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb new file mode 100644 index 00000000000..7068e68a855 --- /dev/null +++ b/lib/gitlab/git/repository.rb @@ -0,0 +1,1251 @@ +# Gitlab::Git::Repository is a wrapper around native Rugged::Repository object +require 'forwardable' +require 'tempfile' +require 'forwardable' +require "rubygems/package" + +module Gitlab + module Git + class Repository + extend Forwardable + include Gitlab::Git::Popen + + SEARCH_CONTEXT_LINES = 3 + + class NoRepository < StandardError; end + class InvalidBlobName < StandardError; end + class InvalidRef < StandardError; end + + # Full path to repo + attr_reader :path + + # Directory name of repo + attr_reader :name + + # Rugged repo object + attr_reader :rugged + + # 'path' must be the path to a _bare_ git repository, e.g. + # /path/to/my-repo.git + def initialize(path) + @path = path + @name = path.split("/").last + @attributes = Gitlab::Git::Attributes.new(path) + end + + # Default branch in the repository + def root_ref + @root_ref ||= discover_default_branch + end + + # Alias to old method for compatibility + def raw + rugged + end + + def rugged + @rugged ||= Rugged::Repository.new(path) + rescue Rugged::RepositoryError, Rugged::OSError + raise NoRepository.new('no repository for such path') + end + + # Returns an Array of branch names + # sorted by name ASC + def branch_names + branches.map(&:name) + end + + # Returns an Array of Branches + def branches + rugged.branches.map do |rugged_ref| + begin + Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target) + rescue Rugged::ReferenceError + # Omit invalid branch + end + end.compact.sort_by(&:name) + end + + def reload_rugged + @rugged = nil + end + + # Directly find a branch with a simple name (e.g. master) + # + # force_reload causes a new Rugged repository to be instantiated + # + # This is to work around a bug in libgit2 that causes in-memory refs to + # be stale/invalid when packed-refs is changed. + # See https://gitlab.com/gitlab-org/gitlab-ce/issues/15392#note_14538333 + def find_branch(name, force_reload = false) + reload_rugged if force_reload + + rugged_ref = rugged.branches[name] + Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target) if rugged_ref + end + + def local_branches + rugged.branches.each(:local).map do |branch| + Gitlab::Git::Branch.new(self, branch.name, branch.target) + end + end + + # Returns the number of valid branches + def branch_count + rugged.branches.count do |ref| + begin + ref.name && ref.target # ensures the branch is valid + + true + rescue Rugged::ReferenceError + false + end + end + end + + # Returns an Array of tag names + def tag_names + rugged.tags.map { |t| t.name } + end + + # Returns an Array of Tags + def tags + rugged.references.each("refs/tags/*").map do |ref| + message = nil + + if ref.target.is_a?(Rugged::Tag::Annotation) + tag_message = ref.target.message + + if tag_message.respond_to?(:chomp) + message = tag_message.chomp + end + end + + Gitlab::Git::Tag.new(self, ref.name, ref.target, message) + end.sort_by(&:name) + end + + # Returns true if the given tag exists + # + # name - The name of the tag as a String. + def tag_exists?(name) + !!rugged.tags[name] + end + + # Returns true if the given branch exists + # + # name - The name of the branch as a String. + def branch_exists?(name) + rugged.branches.exists?(name) + + # If the branch name is invalid (e.g. ".foo") Rugged will raise an error. + # Whatever code calls this method shouldn't have to deal with that so + # instead we just return `false` (which is true since a branch doesn't + # exist when it has an invalid name). + rescue Rugged::ReferenceError + false + end + + # Returns an Array of branch and tag names + def ref_names + branch_names + tag_names + end + + # Deprecated. Will be removed in 5.2 + def heads + rugged.references.each("refs/heads/*").map do |head| + Gitlab::Git::Ref.new(self, head.name, head.target) + end.sort_by(&:name) + end + + def has_commits? + !empty? + end + + def empty? + rugged.empty? + end + + def bare? + rugged.bare? + end + + def repo_exists? + !!rugged + end + + # Discovers the default branch based on the repository's available branches + # + # - If no branches are present, returns nil + # - If one branch is present, returns its name + # - If two or more branches are present, returns current HEAD or master or first branch + def discover_default_branch + names = branch_names + + return if names.empty? + + return names[0] if names.length == 1 + + if rugged_head + extracted_name = Ref.extract_branch_name(rugged_head.name) + + return extracted_name if names.include?(extracted_name) + end + + if names.include?('master') + 'master' + else + names[0] + end + end + + def rugged_head + rugged.head + rescue Rugged::ReferenceError + nil + end + + def archive_metadata(ref, storage_path, format = "tar.gz") + ref ||= root_ref + commit = Gitlab::Git::Commit.find(self, ref) + return {} if commit.nil? + + project_name = self.name.chomp('.git') + prefix = "#{project_name}-#{ref}-#{commit.id}" + + { + 'RepoPath' => path, + 'ArchivePrefix' => prefix, + 'ArchivePath' => archive_file_path(prefix, storage_path, format), + 'CommitId' => commit.id, + } + end + + def archive_file_path(name, storage_path, format = "tar.gz") + # Build file path + return nil unless name + + extension = + case format + when "tar.bz2", "tbz", "tbz2", "tb2", "bz2" + "tar.bz2" + when "tar" + "tar" + when "zip" + "zip" + else + # everything else should fall back to tar.gz + "tar.gz" + end + + file_name = "#{name}.#{extension}" + File.join(storage_path, self.name, file_name) + end + + # Return repo size in megabytes + def size + size = popen(%w(du -sk), path).first.strip.to_i + (size.to_f / 1024).round(2) + end + + # Returns an array of BlobSnippets for files at the specified +ref+ that + # contain the +query+ string. + def search_files(query, ref = nil) + greps = [] + ref ||= root_ref + + populated_index(ref).each do |entry| + # Discard submodules + next if submodule?(entry) + + blob = Gitlab::Git::Blob.raw(self, entry[:oid]) + + # Skip binary files + next if blob.data.encoding == Encoding::ASCII_8BIT + + blob.load_all_data!(self) + greps += build_greps(blob.data, query, ref, entry[:path]) + end + + greps + end + + # Use the Rugged Walker API to build an array of commits. + # + # Usage. + # repo.log( + # ref: 'master', + # path: 'app/models', + # limit: 10, + # offset: 5, + # after: Time.new(2016, 4, 21, 14, 32, 10) + # ) + # + def log(options) + default_options = { + limit: 10, + offset: 0, + path: nil, + follow: false, + skip_merges: false, + disable_walk: false, + after: nil, + before: nil + } + + options = default_options.merge(options) + options[:limit] ||= 0 + options[:offset] ||= 0 + actual_ref = options[:ref] || root_ref + begin + sha = sha_from_ref(actual_ref) + rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError + # Return an empty array if the ref wasn't found + return [] + end + + if log_using_shell?(options) + log_by_shell(sha, options) + else + log_by_walk(sha, options) + end + end + + def log_using_shell?(options) + options[:path].present? || + options[:disable_walk] || + options[:skip_merges] || + options[:after] || + options[:before] + end + + def log_by_walk(sha, options) + walk_options = { + show: sha, + sort: Rugged::SORT_DATE, + limit: options[:limit], + offset: options[:offset] + } + Rugged::Walker.walk(rugged, walk_options).to_a + end + + def log_by_shell(sha, options) + cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path} log) + cmd += %W(-n #{options[:limit].to_i}) + cmd += %w(--format=%H) + cmd += %W(--skip=#{options[:offset].to_i}) + cmd += %w(--follow) if options[:follow] + cmd += %w(--no-merges) if options[:skip_merges] + cmd += %W(--after=#{options[:after].iso8601}) if options[:after] + cmd += %W(--before=#{options[:before].iso8601}) if options[:before] + cmd += [sha] + cmd += %W(-- #{options[:path]}) if options[:path].present? + + raw_output = IO.popen(cmd) {|io| io.read } + + log = raw_output.lines.map do |c| + Rugged::Commit.new(rugged, c.strip) + end + + log.is_a?(Array) ? log : [] + end + + def sha_from_ref(ref) + rev_parse_target(ref).oid + end + + # Return the object that +revspec+ points to. If +revspec+ is an + # annotated tag, then return the tag's target instead. + def rev_parse_target(revspec) + obj = rugged.rev_parse(revspec) + Ref.dereference_object(obj) + end + + # Return a collection of Rugged::Commits between the two revspec arguments. + # See http://git-scm.com/docs/git-rev-parse.html#_specifying_revisions for + # a detailed list of valid arguments. + def commits_between(from, to) + walker = Rugged::Walker.new(rugged) + walker.sorting(Rugged::SORT_DATE | Rugged::SORT_REVERSE) + + sha_from = sha_from_ref(from) + sha_to = sha_from_ref(to) + + walker.push(sha_to) + walker.hide(sha_from) + + commits = walker.to_a + walker.reset + + commits + end + + # Counts the amount of commits between `from` and `to`. + def count_commits_between(from, to) + commits_between(from, to).size + end + + # Returns the SHA of the most recent common ancestor of +from+ and +to+ + def merge_base_commit(from, to) + rugged.merge_base(from, to) + end + + # Return an array of Diff objects that represent the diff + # between +from+ and +to+. See Diff::filter_diff_options for the allowed + # diff options. The +options+ hash can also include :break_rewrites to + # split larger rewrites into delete/add pairs. + def diff(from, to, options = {}, *paths) + Gitlab::Git::DiffCollection.new(diff_patches(from, to, options, *paths), options) + end + + # Returns commits collection + # + # Ex. + # repo.find_commits( + # ref: 'master', + # max_count: 10, + # skip: 5, + # order: :date + # ) + # + # +options+ is a Hash of optional arguments to git + # :ref is the ref from which to begin (SHA1 or name) + # :contains is the commit contained by the refs from which to begin (SHA1 or name) + # :max_count is the maximum number of commits to fetch + # :skip is the number of commits to skip + # :order is the commits order and allowed value is :date(default) or :topo + # + def find_commits(options = {}) + actual_options = options.dup + + allowed_options = [:ref, :max_count, :skip, :contains, :order] + + actual_options.keep_if do |key| + allowed_options.include?(key) + end + + default_options = { skip: 0 } + actual_options = default_options.merge(actual_options) + + walker = Rugged::Walker.new(rugged) + + if actual_options[:ref] + walker.push(rugged.rev_parse_oid(actual_options[:ref])) + elsif actual_options[:contains] + branches_contains(actual_options[:contains]).each do |branch| + walker.push(branch.target_id) + end + else + rugged.references.each("refs/heads/*") do |ref| + walker.push(ref.target_id) + end + end + + if actual_options[:order] == :topo + walker.sorting(Rugged::SORT_TOPO) + else + walker.sorting(Rugged::SORT_DATE) + end + + commits = [] + offset = actual_options[:skip] + limit = actual_options[:max_count] + walker.each(offset: offset, limit: limit) do |commit| + gitlab_commit = Gitlab::Git::Commit.decorate(commit) + commits.push(gitlab_commit) + end + + walker.reset + + commits + rescue Rugged::OdbError + [] + end + + # Returns branch names collection that contains the special commit(SHA1 + # or name) + # + # Ex. + # repo.branch_names_contains('master') + # + def branch_names_contains(commit) + branches_contains(commit).map { |c| c.name } + end + + # Returns branch collection that contains the special commit(SHA1 or name) + # + # Ex. + # repo.branch_names_contains('master') + # + def branches_contains(commit) + commit_obj = rugged.rev_parse(commit) + parent = commit_obj.parents.first unless commit_obj.parents.empty? + + walker = Rugged::Walker.new(rugged) + + rugged.branches.select do |branch| + walker.push(branch.target_id) + walker.hide(parent) if parent + result = walker.any? { |c| c.oid == commit_obj.oid } + walker.reset + + result + end + end + + # Get refs hash which key is SHA1 + # and value is a Rugged::Reference + def refs_hash + # Initialize only when first call + if @refs_hash.nil? + @refs_hash = Hash.new { |h, k| h[k] = [] } + + rugged.references.each do |r| + # Symbolic/remote references may not have an OID; skip over them + target_oid = r.target.try(:oid) + if target_oid + sha = rev_parse_target(target_oid).oid + @refs_hash[sha] << r + end + end + end + @refs_hash + end + + # Lookup for rugged object by oid or ref name + def lookup(oid_or_ref_name) + rugged.rev_parse(oid_or_ref_name) + end + + # Return hash with submodules info for this repository + # + # Ex. + # { + # "rack" => { + # "id" => "c67be4624545b4263184c4a0e8f887efd0a66320", + # "path" => "rack", + # "url" => "git://github.com/chneukirchen/rack.git" + # }, + # "encoding" => { + # "id" => .... + # } + # } + # + def submodules(ref) + commit = rev_parse_target(ref) + return {} unless commit + + begin + content = blob_content(commit, ".gitmodules") + rescue InvalidBlobName + return {} + end + + parse_gitmodules(commit, content) + end + + # Return total commits count accessible from passed ref + def commit_count(ref) + walker = Rugged::Walker.new(rugged) + walker.sorting(Rugged::SORT_TOPO | Rugged::SORT_REVERSE) + oid = rugged.rev_parse_oid(ref) + walker.push(oid) + walker.count + end + + # Sets HEAD to the commit specified by +ref+; +ref+ can be a branch or + # tag name or a commit SHA. Valid +reset_type+ values are: + # + # [:soft] + # the head will be moved to the commit. + # [:mixed] + # will trigger a +:soft+ reset, plus the index will be replaced + # with the content of the commit tree. + # [:hard] + # will trigger a +:mixed+ reset and the working directory will be + # replaced with the content of the index. (Untracked and ignored files + # will be left alone) + def reset(ref, reset_type) + rugged.reset(ref, reset_type) + end + + # Mimic the `git clean` command and recursively delete untracked files. + # Valid keys that can be passed in the +options+ hash are: + # + # :d - Remove untracked directories + # :f - Remove untracked directories that are managed by a different + # repository + # :x - Remove ignored files + # + # The value in +options+ must evaluate to true for an option to take + # effect. + # + # Examples: + # + # repo.clean(d: true, f: true) # Enable the -d and -f options + # + # repo.clean(d: false, x: true) # -x is enabled, -d is not + def clean(options = {}) + strategies = [:remove_untracked] + strategies.push(:force) if options[:f] + strategies.push(:remove_ignored) if options[:x] + + # TODO: implement this method + end + + # Check out the specified ref. Valid options are: + # + # :b - Create a new branch at +start_point+ and set HEAD to the new + # branch. + # + # * These options are passed to the Rugged::Repository#checkout method: + # + # :progress :: + # A callback that will be executed for checkout progress notifications. + # Up to 3 parameters are passed on each execution: + # + # - The path to the last updated file (or +nil+ on the very first + # invocation). + # - The number of completed checkout steps. + # - The number of total checkout steps to be performed. + # + # :notify :: + # A callback that will be executed for each checkout notification + # types specified with +:notify_flags+. Up to 5 parameters are passed + # on each execution: + # + # - An array containing the +:notify_flags+ that caused the callback + # execution. + # - The path of the current file. + # - A hash describing the baseline blob (or +nil+ if it does not + # exist). + # - A hash describing the target blob (or +nil+ if it does not exist). + # - A hash describing the workdir blob (or +nil+ if it does not + # exist). + # + # :strategy :: + # A single symbol or an array of symbols representing the strategies + # to use when performing the checkout. Possible values are: + # + # :none :: + # Perform a dry run (default). + # + # :safe :: + # Allow safe updates that cannot overwrite uncommitted data. + # + # :safe_create :: + # Allow safe updates plus creation of missing files. + # + # :force :: + # Allow all updates to force working directory to look like index. + # + # :allow_conflicts :: + # Allow checkout to make safe updates even if conflicts are found. + # + # :remove_untracked :: + # Remove untracked files not in index (that are not ignored). + # + # :remove_ignored :: + # Remove ignored files not in index. + # + # :update_only :: + # Only update existing files, don't create new ones. + # + # :dont_update_index :: + # Normally checkout updates index entries as it goes; this stops + # that. + # + # :no_refresh :: + # Don't refresh index/config/etc before doing checkout. + # + # :disable_pathspec_match :: + # Treat pathspec as simple list of exact match file paths. + # + # :skip_locked_directories :: + # Ignore directories in use, they will be left empty. + # + # :skip_unmerged :: + # Allow checkout to skip unmerged files (NOT IMPLEMENTED). + # + # :use_ours :: + # For unmerged files, checkout stage 2 from index (NOT IMPLEMENTED). + # + # :use_theirs :: + # For unmerged files, checkout stage 3 from index (NOT IMPLEMENTED). + # + # :update_submodules :: + # Recursively checkout submodules with same options (NOT + # IMPLEMENTED). + # + # :update_submodules_if_changed :: + # Recursively checkout submodules if HEAD moved in super repo (NOT + # IMPLEMENTED). + # + # :disable_filters :: + # If +true+, filters like CRLF line conversion will be disabled. + # + # :dir_mode :: + # Mode for newly created directories. Default: +0755+. + # + # :file_mode :: + # Mode for newly created files. Default: +0755+ or +0644+. + # + # :file_open_flags :: + # Mode for opening files. Default: + # <code>IO::CREAT | IO::TRUNC | IO::WRONLY</code>. + # + # :notify_flags :: + # A single symbol or an array of symbols representing the cases in + # which the +:notify+ callback should be invoked. Possible values are: + # + # :none :: + # Do not invoke the +:notify+ callback (default). + # + # :conflict :: + # Invoke the callback for conflicting paths. + # + # :dirty :: + # Invoke the callback for "dirty" files, i.e. those that do not need + # an update but no longer match the baseline. + # + # :updated :: + # Invoke the callback for any file that was changed. + # + # :untracked :: + # Invoke the callback for untracked files. + # + # :ignored :: + # Invoke the callback for ignored files. + # + # :all :: + # Invoke the callback for all these cases. + # + # :paths :: + # A glob string or an array of glob strings specifying which paths + # should be taken into account for the checkout operation. +nil+ will + # match all files. Default: +nil+. + # + # :baseline :: + # A Rugged::Tree that represents the current, expected contents of the + # workdir. Default: +HEAD+. + # + # :target_directory :: + # A path to an alternative workdir directory in which the checkout + # should be performed. + def checkout(ref, options = {}, start_point = "HEAD") + if options[:b] + rugged.branches.create(ref, start_point) + options.delete(:b) + end + default_options = { strategy: [:recreate_missing, :safe] } + rugged.checkout(ref, default_options.merge(options)) + end + + # Delete the specified branch from the repository + def delete_branch(branch_name) + rugged.branches.delete(branch_name) + end + + # Create a new branch named **ref+ based on **stat_point+, HEAD by default + # + # Examples: + # create_branch("feature") + # create_branch("other-feature", "master") + def create_branch(ref, start_point = "HEAD") + rugged_ref = rugged.branches.create(ref, start_point) + Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target) + rescue Rugged::ReferenceError => e + raise InvalidRef.new("Branch #{ref} already exists") if e.to_s =~ /'refs\/heads\/#{ref}'/ + raise InvalidRef.new("Invalid reference #{start_point}") + end + + # Return an array of this repository's remote names + def remote_names + rugged.remotes.each_name.to_a + end + + # Delete the specified remote from this repository. + def remote_delete(remote_name) + rugged.remotes.delete(remote_name) + end + + # Add a new remote to this repository. Returns a Rugged::Remote object + def remote_add(remote_name, url) + rugged.remotes.create(remote_name, url) + end + + # Update the specified remote using the values in the +options+ hash + # + # Example + # repo.update_remote("origin", url: "path/to/repo") + def remote_update(remote_name, options = {}) + # TODO: Implement other remote options + rugged.remotes.set_url(remote_name, options[:url]) if options[:url] + end + + # Fetch the specified remote + def fetch(remote_name) + rugged.remotes[remote_name].fetch + end + + # Push +*refspecs+ to the remote identified by +remote_name+. + def push(remote_name, *refspecs) + rugged.remotes[remote_name].push(refspecs) + end + + # Merge the +source_name+ branch into the +target_name+ branch. This is + # equivalent to `git merge --no_ff +source_name+`, since a merge commit + # is always created. + def merge(source_name, target_name, options = {}) + our_commit = rugged.branches[target_name].target + their_commit = rugged.branches[source_name].target + + raise "Invalid merge target" if our_commit.nil? + raise "Invalid merge source" if their_commit.nil? + + merge_index = rugged.merge_commits(our_commit, their_commit) + return false if merge_index.conflicts? + + actual_options = options.merge( + parents: [our_commit, their_commit], + tree: merge_index.write_tree(rugged), + update_ref: "refs/heads/#{target_name}" + ) + Rugged::Commit.create(rugged, actual_options) + end + + def commits_since(from_date) + walker = Rugged::Walker.new(rugged) + walker.sorting(Rugged::SORT_DATE | Rugged::SORT_REVERSE) + + rugged.references.each("refs/heads/*") do |ref| + walker.push(ref.target_id) + end + + commits = [] + walker.each do |commit| + break if commit.author[:time].to_date < from_date + commits.push(commit) + end + + commits + end + + AUTOCRLF_VALUES = { + "true" => true, + "false" => false, + "input" => :input + }.freeze + + def autocrlf + AUTOCRLF_VALUES[rugged.config['core.autocrlf']] + end + + def autocrlf=(value) + rugged.config['core.autocrlf'] = AUTOCRLF_VALUES.invert[value] + end + + # Create a new directory with a .gitkeep file. Creates + # all required nested directories (i.e. mkdir -p behavior) + # + # options should contain next structure: + # author: { + # email: 'user@example.com', + # name: 'Test User', + # time: Time.now + # }, + # committer: { + # email: 'user@example.com', + # name: 'Test User', + # time: Time.now + # }, + # commit: { + # message: 'Wow such commit', + # branch: 'master', + # update_ref: false + # } + def mkdir(path, options = {}) + # Check if this directory exists; if it does, then don't bother + # adding .gitkeep file. + ref = options[:commit][:branch] + path = Gitlab::Git::PathHelper.normalize_path(path).to_s + rugged_ref = rugged.ref(ref) + + raise InvalidRef.new("Invalid ref") if rugged_ref.nil? + + target_commit = rugged_ref.target + + raise InvalidRef.new("Invalid target commit") if target_commit.nil? + + entry = tree_entry(target_commit, path) + + if entry + if entry[:type] == :blob + raise InvalidBlobName.new("Directory already exists as a file") + else + raise InvalidBlobName.new("Directory already exists") + end + end + + options[:file] = { + content: '', + path: "#{path}/.gitkeep", + update: true + } + + Gitlab::Git::Blob.commit(self, options) + end + + # Returns result like "git ls-files" , recursive and full file path + # + # Ex. + # repo.ls_files('master') + # + def ls_files(ref) + actual_ref = ref || root_ref + + begin + sha_from_ref(actual_ref) + rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError + # Return an empty array if the ref wasn't found + return [] + end + + cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path} ls-tree) + cmd += %w(-r) + cmd += %w(--full-tree) + cmd += %w(--full-name) + cmd += %W(-- #{actual_ref}) + + raw_output = IO.popen(cmd, &:read).split("\n").map do |f| + stuff, path = f.split("\t") + _mode, type, _sha = stuff.split(" ") + path if type == "blob" + # Contain only blob type + end + + raw_output.compact + end + + def copy_gitattributes(ref) + begin + commit = lookup(ref) + rescue Rugged::ReferenceError + raise InvalidRef.new("Ref #{ref} is invalid") + end + + # Create the paths + info_dir_path = File.join(path, 'info') + info_attributes_path = File.join(info_dir_path, 'attributes') + + begin + # Retrieve the contents of the blob + gitattributes_content = blob_content(commit, '.gitattributes') + rescue InvalidBlobName + # No .gitattributes found. Should now remove any info/attributes and return + File.delete(info_attributes_path) if File.exist?(info_attributes_path) + return + end + + # Create the info directory if needed + Dir.mkdir(info_dir_path) unless File.directory?(info_dir_path) + + # Write the contents of the .gitattributes file to info/attributes + # Use binary mode to prevent Rails from converting ASCII-8BIT to UTF-8 + File.open(info_attributes_path, "wb") do |file| + file.write(gitattributes_content) + end + end + + # Checks if the blob should be diffable according to its attributes + def diffable?(blob) + attributes(blob.path).fetch('diff') { blob.text? } + end + + # Returns the Git attributes for the given file path. + # + # See `Gitlab::Git::Attributes` for more information. + def attributes(path) + @attributes.attributes(path) + end + + private + + # Get the content of a blob for a given commit. If the blob is a commit + # (for submodules) then return the blob's OID. + def blob_content(commit, blob_name) + blob_entry = tree_entry(commit, blob_name) + + unless blob_entry + raise InvalidBlobName.new("Invalid blob name: #{blob_name}") + end + + case blob_entry[:type] + when :commit + blob_entry[:oid] + when :tree + raise InvalidBlobName.new("#{blob_name} is a tree, not a blob") + when :blob + rugged.lookup(blob_entry[:oid]).content + end + end + + # Parses the contents of a .gitmodules file and returns a hash of + # submodule information. + def parse_gitmodules(commit, content) + results = {} + + current = "" + content.split("\n").each do |txt| + if txt =~ /^\s*\[/ + current = txt.match(/(?<=").*(?=")/)[0] + results[current] = {} + else + next unless results[current] + match_data = txt.match(/(\w+)\s*=\s*(.*)/) + next unless match_data + target = match_data[2].chomp + results[current][match_data[1]] = target + + if match_data[1] == "path" + begin + results[current]["id"] = blob_content(commit, target) + rescue InvalidBlobName + results.delete(current) + end + end + end + end + + results + end + + # Returns true if +commit+ introduced changes to +path+, using commit + # trees to make that determination. Uses the history simplification + # rules that `git log` uses by default, where a commit is omitted if it + # is TREESAME to any parent. + # + # If the +follow+ option is true and the file specified by +path+ was + # renamed, then the path value is set to the old path. + def commit_touches_path?(commit, path, follow, walker) + entry = tree_entry(commit, path) + + if commit.parents.empty? + # This is the root commit, return true if it has +path+ in its tree + return !entry.nil? + end + + num_treesame = 0 + commit.parents.each do |parent| + parent_entry = tree_entry(parent, path) + + # Only follow the first TREESAME parent for merge commits + if num_treesame > 0 + walker.hide(parent) + next + end + + if entry.nil? && parent_entry.nil? + num_treesame += 1 + elsif entry && parent_entry && entry[:oid] == parent_entry[:oid] + num_treesame += 1 + end + end + + case num_treesame + when 0 + detect_rename(commit, commit.parents.first, path) if follow + true + else false + end + end + + # Find the entry for +path+ in the tree for +commit+ + def tree_entry(commit, path) + pathname = Pathname.new(path) + first = true + tmp_entry = nil + + pathname.each_filename do |dir| + if first + tmp_entry = commit.tree[dir] + first = false + elsif tmp_entry.nil? + return nil + else + tmp_entry = rugged.lookup(tmp_entry[:oid]) + return nil unless tmp_entry.type == :tree + tmp_entry = tmp_entry[dir] + end + end + + tmp_entry + end + + # Compare +commit+ and +parent+ for +path+. If +path+ is a file and was + # renamed in +commit+, then set +path+ to the old filename. + def detect_rename(commit, parent, path) + diff = parent.diff(commit, paths: [path], disable_pathspec_match: true) + + # If +path+ is a filename, not a directory, then we should only have + # one delta. We don't need to follow renames for directories. + return nil if diff.each_delta.count > 1 + + delta = diff.each_delta.first + if delta.added? + full_diff = parent.diff(commit) + full_diff.find_similar! + + full_diff.each_delta do |full_delta| + if full_delta.renamed? && path == full_delta.new_file[:path] + # Look for the old path in ancestors + path.replace(full_delta.old_file[:path]) + end + end + end + end + + def archive_to_file(treeish = 'master', filename = 'archive.tar.gz', format = nil, compress_cmd = %w(gzip -n)) + git_archive_cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path} archive) + + # Put files into a directory before archiving + prefix = "#{archive_name(treeish)}/" + git_archive_cmd << "--prefix=#{prefix}" + + # Format defaults to tar + git_archive_cmd << "--format=#{format}" if format + + git_archive_cmd += %W(-- #{treeish}) + + open(filename, 'w') do |file| + # Create a pipe to act as the '|' in 'git archive ... | gzip' + pipe_rd, pipe_wr = IO.pipe + + # Get the compression process ready to accept data from the read end + # of the pipe + compress_pid = spawn(*nice(compress_cmd), in: pipe_rd, out: file) + # The read end belongs to the compression process now; we should + # close our file descriptor for it. + pipe_rd.close + + # Start 'git archive' and tell it to write into the write end of the + # pipe. + git_archive_pid = spawn(*nice(git_archive_cmd), out: pipe_wr) + # The write end belongs to 'git archive' now; close it. + pipe_wr.close + + # When 'git archive' and the compression process are finished, we are + # done. + Process.waitpid(git_archive_pid) + raise "#{git_archive_cmd.join(' ')} failed" unless $?.success? + Process.waitpid(compress_pid) + raise "#{compress_cmd.join(' ')} failed" unless $?.success? + end + end + + def nice(cmd) + nice_cmd = %w(nice -n 20) + unless unsupported_platform? + nice_cmd += %w(ionice -c 2 -n 7) + end + nice_cmd + cmd + end + + def unsupported_platform? + %w[darwin freebsd solaris].map { |platform| RUBY_PLATFORM.include?(platform) }.any? + end + + # Returns true if the index entry has the special file mode that denotes + # a submodule. + def submodule?(index_entry) + index_entry[:mode] == 57344 + end + + # Return a Rugged::Index that has read from the tree at +ref_name+ + def populated_index(ref_name) + commit = rev_parse_target(ref_name) + index = rugged.index + index.read_tree(commit.tree) + index + end + + # Return an array of BlobSnippets for lines in +file_contents+ that match + # +query+ + def build_greps(file_contents, query, ref, filename) + # The file_contents string is potentially huge so we make sure to loop + # through it one line at a time. This gives Ruby the chance to GC lines + # we are not interested in. + # + # We need to do a little extra work because we are not looking for just + # the lines that matches the query, but also for the context + # (surrounding lines). We will use Enumerable#each_cons to efficiently + # loop through the lines while keeping surrounding lines on hand. + # + # First, we turn "foo\nbar\nbaz" into + # [ + # [nil, -3], [nil, -2], [nil, -1], + # ['foo', 0], ['bar', 1], ['baz', 3], + # [nil, 4], [nil, 5], [nil, 6] + # ] + lines_with_index = Enumerator.new do |yielder| + # Yield fake 'before' lines for the first line of file_contents + (-SEARCH_CONTEXT_LINES..-1).each do |i| + yielder.yield [nil, i] + end + + # Yield the actual file contents + count = 0 + file_contents.each_line do |line| + line.chomp! + yielder.yield [line, count] + count += 1 + end + + # Yield fake 'after' lines for the last line of file_contents + (count + 1..count + SEARCH_CONTEXT_LINES).each do |i| + yielder.yield [nil, i] + end + end + + greps = [] + + # Loop through consecutive blocks of lines with indexes + lines_with_index.each_cons(2 * SEARCH_CONTEXT_LINES + 1) do |line_block| + # Get the 'middle' line and index from the block + line, _ = line_block[SEARCH_CONTEXT_LINES] + + next unless line && line.match(/#{Regexp.escape(query)}/i) + + # Yay, 'line' contains a match! + # Get an array with just the context lines (no indexes) + match_with_context = line_block.map(&:first) + # Remove 'nil' lines in case we are close to the first or last line + match_with_context.compact! + + # Get the line number (1-indexed) of the first context line + first_context_line_number = line_block[0][1] + 1 + + greps << Gitlab::Git::BlobSnippet.new( + ref, + match_with_context, + first_context_line_number, + filename + ) + end + + greps + end + + # Return the Rugged patches for the diff between +from+ and +to+. + def diff_patches(from, to, options = {}, *paths) + options ||= {} + break_rewrites = options[:break_rewrites] + actual_options = Gitlab::Git::Diff.filter_diff_options(options.merge(paths: paths)) + + diff = rugged.diff(from, to, actual_options) + diff.find_similar!(break_rewrites: break_rewrites) + diff.each_patch + end + end + end +end diff --git a/lib/gitlab/git/rev_list.rb b/lib/gitlab/git/rev_list.rb new file mode 100644 index 00000000000..79dd0cf7df2 --- /dev/null +++ b/lib/gitlab/git/rev_list.rb @@ -0,0 +1,42 @@ +module Gitlab + module Git + class RevList + attr_reader :project, :env + + ALLOWED_VARIABLES = %w[GIT_OBJECT_DIRECTORY GIT_ALTERNATE_OBJECT_DIRECTORIES].freeze + + def initialize(oldrev, newrev, project:, env: nil) + @project = project + @env = env.presence || {} + @args = [Gitlab.config.git.bin_path, + "--git-dir=#{project.repository.path_to_repo}", + "rev-list", + "--max-count=1", + oldrev, + "^#{newrev}"] + end + + def execute + Gitlab::Popen.popen(@args, nil, parse_environment_variables) + end + + def valid? + environment_variables.all? do |(name, value)| + value.to_s.start_with?(project.repository.path_to_repo) + end + end + + private + + def parse_environment_variables + return {} unless valid? + + environment_variables + end + + def environment_variables + @environment_variables ||= env.slice(*ALLOWED_VARIABLES).compact + end + end + end +end diff --git a/lib/gitlab/git/tag.rb b/lib/gitlab/git/tag.rb new file mode 100644 index 00000000000..b5342c3d310 --- /dev/null +++ b/lib/gitlab/git/tag.rb @@ -0,0 +1,17 @@ +module Gitlab + module Git + class Tag < Ref + attr_reader :object_sha + + def initialize(repository, name, target, message = nil) + super(repository, name, target) + + @message = message + end + + def message + encode! @message + end + end + end +end diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb new file mode 100644 index 00000000000..f7450e8b58f --- /dev/null +++ b/lib/gitlab/git/tree.rb @@ -0,0 +1,104 @@ +module Gitlab + module Git + class Tree + include Gitlab::Git::EncodingHelper + + attr_accessor :id, :root_id, :name, :path, :type, + :mode, :commit_id, :submodule_url + + class << self + # Get list of tree objects + # for repository based on commit sha and path + # Uses rugged for raw objects + def where(repository, sha, path = nil) + path = nil if path == '' || path == '/' + + commit = repository.lookup(sha) + root_tree = commit.tree + + tree = if path + id = find_id_by_path(repository, root_tree.oid, path) + if id + repository.lookup(id) + else + [] + end + else + root_tree + end + + tree.map do |entry| + new( + id: entry[:oid], + root_id: root_tree.oid, + name: entry[:name], + type: entry[:type], + mode: entry[:filemode], + path: path ? File.join(path, entry[:name]) : entry[:name], + commit_id: sha, + ) + end + end + + # Recursive search of tree id for path + # + # Ex. + # blog/ # oid: 1a + # app/ # oid: 2a + # models/ # oid: 3a + # views/ # oid: 4a + # + # + # Tree.find_id_by_path(repo, '1a', 'app/models') # => '3a' + # + def find_id_by_path(repository, root_id, path) + root_tree = repository.lookup(root_id) + path_arr = path.split('/') + + entry = root_tree.find do |entry| + entry[:name] == path_arr[0] && entry[:type] == :tree + end + + return nil unless entry + + if path_arr.size > 1 + path_arr.shift + find_id_by_path(repository, entry[:oid], path_arr.join('/')) + else + entry[:oid] + end + end + end + + def initialize(options) + %w(id root_id name path type mode commit_id).each do |key| + self.send("#{key}=", options[key.to_sym]) + end + end + + def name + encode! @name + end + + def dir? + type == :tree + end + + def file? + type == :blob + end + + def submodule? + type == :commit + end + + def readme? + name =~ /^readme/i + end + + def contributing? + name =~ /^contributing/i + end + end + end +end diff --git a/lib/gitlab/git/util.rb b/lib/gitlab/git/util.rb new file mode 100644 index 00000000000..7973da2e8f8 --- /dev/null +++ b/lib/gitlab/git/util.rb @@ -0,0 +1,18 @@ +module Gitlab + module Git + module Util + LINE_SEP = "\n".freeze + + def self.count_lines(string) + case string[-1] + when nil + 0 + when LINE_SEP + string.count(LINE_SEP) + else + string.count(LINE_SEP) + 1 + end + end + end + end +end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 799794c0171..7e1484613f2 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -2,132 +2,178 @@ # class return an instance of `GitlabAccessStatus` module Gitlab class GitAccess + UnauthorizedError = Class.new(StandardError) + + ERROR_MESSAGES = { + upload: 'You are not allowed to upload code for this project.', + download: 'You are not allowed to download code from this project.', + deploy_key_upload: + 'This deploy key does not have write access to this project.', + no_repo: 'A repository for this project does not exist yet.' + } + DOWNLOAD_COMMANDS = %w{ git-upload-pack git-upload-archive } PUSH_COMMANDS = %w{ git-receive-pack } + ALL_COMMANDS = DOWNLOAD_COMMANDS + PUSH_COMMANDS attr_reader :actor, :project, :protocol, :user_access, :authentication_abilities - def initialize(actor, project, protocol, authentication_abilities:) + def initialize(actor, project, protocol, authentication_abilities:, env: {}) @actor = actor @project = project @protocol = protocol @authentication_abilities = authentication_abilities @user_access = UserAccess.new(user, project: project) + @env = env end def check(cmd, changes) - return build_status_object(false, "Git access over #{protocol.upcase} is not allowed") unless protocol_allowed? - - unless actor - return build_status_object(false, "No user or key was provided.") - end - - if user && !user_access.allowed? - return build_status_object(false, "Your account has been blocked.") - end - - unless project && (user_access.can_read_project? || deploy_key_can_read_project?) - return build_status_object(false, 'The project you were looking for could not be found.') - end + check_protocol! + check_active_user! + check_project_accessibility! + check_command_existence!(cmd) + check_repository_existence! case cmd when *DOWNLOAD_COMMANDS - download_access_check + check_download_access! when *PUSH_COMMANDS - push_access_check(changes) - else - build_status_object(false, "The command you're trying to execute is not allowed.") + check_push_access!(changes) end + + build_status_object(true) + rescue UnauthorizedError => ex + build_status_object(false, ex.message) end - def download_access_check - if user - user_download_access_check - elsif deploy_key - build_status_object(true) - else - raise 'Wrong actor' + def guest_can_download_code? + Guest.can?(:download_code, project) + end + + def user_can_download_code? + authentication_abilities.include?(:download_code) && user_access.can_do_action?(:download_code) + end + + def build_can_download_code? + authentication_abilities.include?(:build_download_code) && user_access.can_do_action?(:build_download_code) + end + + def protocol_allowed? + Gitlab::ProtocolAccess.allowed?(protocol) + end + + private + + def check_protocol! + unless protocol_allowed? + raise UnauthorizedError, "Git access over #{protocol.upcase} is not allowed" end end - def push_access_check(changes) - if user - user_push_access_check(changes) - elsif deploy_key - build_status_object(false, "Deploy keys are not allowed to push code.") - else - raise 'Wrong actor' + def check_active_user! + return if deploy_key? + + if user && !user_access.allowed? + raise UnauthorizedError, "Your account has been blocked." end end - def user_download_access_check - unless user_can_download_code? || build_can_download_code? - return build_status_object(false, "You are not allowed to download code from this project.") + def check_project_accessibility! + if project.blank? || !can_read_project? + raise UnauthorizedError, 'The project you were looking for could not be found.' end + end - build_status_object(true) + def check_command_existence!(cmd) + unless ALL_COMMANDS.include?(cmd) + raise UnauthorizedError, "The command you're trying to execute is not allowed." + end end - def user_can_download_code? - authentication_abilities.include?(:download_code) && user_access.can_do_action?(:download_code) + def check_repository_existence! + unless project.repository.exists? + raise UnauthorizedError, ERROR_MESSAGES[:no_repo] + end end - def build_can_download_code? - authentication_abilities.include?(:build_download_code) && user_access.can_do_action?(:build_download_code) + def check_download_access! + return if deploy_key? + + passed = user_can_download_code? || + build_can_download_code? || + guest_can_download_code? + + unless passed + raise UnauthorizedError, ERROR_MESSAGES[:download] + end end - def user_push_access_check(changes) - unless authentication_abilities.include?(:push_code) - return build_status_object(false, "You are not allowed to upload code for this project.") + def check_push_access!(changes) + if deploy_key + check_deploy_key_push_access! + elsif user + check_user_push_access! + else + raise UnauthorizedError, ERROR_MESSAGES[:upload] end - if changes.blank? - return build_status_object(true) + return if changes.blank? # Allow access. + + check_change_access!(changes) + end + + def check_user_push_access! + unless authentication_abilities.include?(:push_code) + raise UnauthorizedError, ERROR_MESSAGES[:upload] end + end - unless project.repository.exists? - return build_status_object(false, "A repository for this project does not exist yet.") + def check_deploy_key_push_access! + unless deploy_key.can_push_to?(project) + raise UnauthorizedError, ERROR_MESSAGES[:deploy_key_upload] end + end + def check_change_access!(changes) changes_list = Gitlab::ChangesList.new(changes) # Iterate over all changes to find if user allowed all of them to be applied changes_list.each do |change| - status = change_access_check(change) + status = check_single_change_access(change) unless status.allowed? # If user does not have access to make at least one change - cancel all push - return status + raise UnauthorizedError, status.message end end - - build_status_object(true) - end - - def change_access_check(change) - Checks::ChangeAccess.new(change, user_access: user_access, project: project).exec end - def protocol_allowed? - Gitlab::ProtocolAccess.allowed?(protocol) + def check_single_change_access(change) + Checks::ChangeAccess.new( + change, + user_access: user_access, + project: project, + env: @env, + skip_authorization: deploy_key?).exec end - private - def matching_merge_request?(newrev, branch_name) Checks::MatchingMergeRequest.new(newrev, branch_name, project).match? end def deploy_key - actor if actor.is_a?(DeployKey) + actor if deploy_key? end - def deploy_key_can_read_project? + def deploy_key? + actor.is_a?(DeployKey) + end + + def can_read_project? if deploy_key - return true if project.public? - deploy_key.projects.include?(project) - else - false - end + deploy_key.has_access_to?(project) + elsif user + user.can?(:read_project, project) + end || Guest.can?(:read_project, project) end protected diff --git a/lib/gitlab/git_access_wiki.rb b/lib/gitlab/git_access_wiki.rb index f71d3575909..67eaa5e088d 100644 --- a/lib/gitlab/git_access_wiki.rb +++ b/lib/gitlab/git_access_wiki.rb @@ -1,6 +1,14 @@ module Gitlab class GitAccessWiki < GitAccess - def change_access_check(change) + def guest_can_download_code? + Guest.can?(:download_wiki_code, project) + end + + def user_can_download_code? + authentication_abilities.include?(:download_code) && user_access.can_do_action?(:download_wiki_code) + end + + def check_single_change_access(change) if user_access.can_do_action?(:create_wiki) build_status_object(true) else diff --git a/lib/gitlab/github_import/base_formatter.rb b/lib/gitlab/github_import/base_formatter.rb index 6dbae64a9fe..95dba9a327b 100644 --- a/lib/gitlab/github_import/base_formatter.rb +++ b/lib/gitlab/github_import/base_formatter.rb @@ -15,6 +15,10 @@ module Gitlab end end + def url + raw_data.url || '' + end + private def gitlab_user_id(github_id) diff --git a/lib/gitlab/github_import/branch_formatter.rb b/lib/gitlab/github_import/branch_formatter.rb index 4750675ae9d..0a8d05b5fe1 100644 --- a/lib/gitlab/github_import/branch_formatter.rb +++ b/lib/gitlab/github_import/branch_formatter.rb @@ -8,7 +8,7 @@ module Gitlab end def valid? - repo.present? + sha.present? && ref.present? end private diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 85df6547a67..ba869faa92e 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -4,10 +4,12 @@ module Gitlab GITHUB_SAFE_REMAINING_REQUESTS = 100 GITHUB_SAFE_SLEEP_TIME = 500 - attr_reader :access_token + attr_reader :access_token, :host, :api_version - def initialize(access_token) + def initialize(access_token, host: nil, api_version: 'v3') @access_token = access_token + @host = host.to_s.sub(%r{/+\z}, '') + @api_version = api_version if access_token ::Octokit.auto_paginate = false @@ -17,7 +19,7 @@ module Gitlab def api @api ||= ::Octokit::Client.new( access_token: access_token, - api_endpoint: github_options[:site], + api_endpoint: api_endpoint, # If there is no config, we're connecting to github.com and we # should verify ssl. connection_options: { @@ -64,6 +66,14 @@ module Gitlab private + def api_endpoint + if host.present? && api_version.present? + "#{host}/api/#{api_version}" + else + github_options[:site] + end + end + def config Gitlab.config.omniauth.providers.find { |provider| provider.name == "github" } end diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index ecc28799737..ec1318ab33c 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -3,7 +3,7 @@ module Gitlab class Importer include Gitlab::ShellAdapter - attr_reader :client, :errors, :project, :repo, :repo_url + attr_reader :errors, :project, :repo, :repo_url def initialize(project) @project = project @@ -11,23 +11,52 @@ module Gitlab @repo_url = project.import_url @errors = [] @labels = {} + end + + def client + return @client if defined?(@client) + unless credentials + raise Projects::ImportService::Error, + "Unable to find project import data credentials for project ID: #{@project.id}" + end - if credentials - @client = Client.new(credentials[:user]) - else - raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{@project.id}" + opts = {} + # Gitea plan to be GitHub compliant + if project.gitea_import? + uri = URI.parse(project.import_url) + host = "#{uri.scheme}://#{uri.host}:#{uri.port}#{uri.path}".sub(%r{/?[\w-]+/[\w-]+\.git\z}, '') + opts = { + host: host, + api_version: 'v1' + } end + + @client = Client.new(credentials[:user], opts) end def execute + # The ordering of importing is important here due to the way GitHub structures their data + # 1. Labels are required by other items while not having a dependency on anything else + # so need to be first + # 2. Pull requests must come before issues. Every pull request is also an issue but not + # all issues are pull requests. Only the issue entity has labels defined in GitHub. GitLab + # doesn't structure data like this so we need to make sure that we've created the MRs + # before we attempt to add the labels defined in the GitHub issue for the related, already + # imported, pull request import_labels import_milestones - import_issues import_pull_requests + import_issues import_comments(:issues) import_comments(:pull_requests) import_wiki - import_releases + + # Gitea doesn't have a Release API yet + # See https://github.com/go-gitea/gitea/issues/330 + unless project.gitea_import? + import_releases + end + handle_errors true @@ -36,7 +65,9 @@ module Gitlab private def credentials - @credentials ||= project.import_data.credentials if project.import_data + return @credentials if defined?(@credentials) + + @credentials = project.import_data ? project.import_data.credentials : nil end def handle_errors @@ -52,22 +83,25 @@ module Gitlab fetch_resources(:labels, repo, per_page: 100) do |labels| labels.each do |raw| begin - label = LabelFormatter.new(project, raw).create! - @labels[label.title] = label.id + gh_label = LabelFormatter.new(project, raw) + gh_label.create! rescue => e - errors << { type: :label, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + errors << { type: :label, url: Gitlab::UrlSanitizer.sanitize(gh_label.url), errors: e.message } end end end + + cache_labels! end def import_milestones fetch_resources(:milestones, repo, state: :all, per_page: 100) do |milestones| milestones.each do |raw| begin - MilestoneFormatter.new(project, raw).create! + gh_milestone = MilestoneFormatter.new(project, raw) + gh_milestone.create! rescue => e - errors << { type: :milestone, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + errors << { type: :milestone, url: Gitlab::UrlSanitizer.sanitize(gh_milestone.url), errors: e.message } end end end @@ -78,13 +112,17 @@ module Gitlab issues.each do |raw| gh_issue = IssueFormatter.new(project, raw) - if gh_issue.valid? - begin - issue = gh_issue.create! - apply_labels(issue, raw) - rescue => e - errors << { type: :issue, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } - end + begin + issuable = + if gh_issue.pull_request? + MergeRequest.find_by_iid(gh_issue.number) + else + gh_issue.create! + end + + apply_labels(issuable, raw) + rescue => e + errors << { type: :issue, url: Gitlab::UrlSanitizer.sanitize(gh_issue.url), errors: e.message } end end end @@ -93,19 +131,23 @@ module Gitlab def import_pull_requests fetch_resources(:pull_requests, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |pull_requests| pull_requests.each do |raw| - pull_request = PullRequestFormatter.new(project, raw) - next unless pull_request.valid? + gh_pull_request = PullRequestFormatter.new(project, raw) + next unless gh_pull_request.valid? begin - restore_source_branch(pull_request) unless pull_request.source_branch_exists? - restore_target_branch(pull_request) unless pull_request.target_branch_exists? + restore_source_branch(gh_pull_request) unless gh_pull_request.source_branch_exists? + restore_target_branch(gh_pull_request) unless gh_pull_request.target_branch_exists? + + merge_request = gh_pull_request.create! - merge_request = pull_request.create! - apply_labels(merge_request, raw) + # Gitea doesn't return PR in the Issue API endpoint, so labels must be assigned at this stage + if project.gitea_import? + apply_labels(merge_request, raw) + end rescue => e - errors << { type: :pull_request, url: Gitlab::UrlSanitizer.sanitize(pull_request.url), errors: e.message } + errors << { type: :pull_request, url: Gitlab::UrlSanitizer.sanitize(gh_pull_request.url), errors: e.message } ensure - clean_up_restored_branches(pull_request) + clean_up_restored_branches(gh_pull_request) end end end @@ -132,21 +174,14 @@ module Gitlab remove_branch(pull_request.target_branch_name) unless pull_request.target_branch_exists? end - def apply_labels(issuable, raw_issuable) - # GH returns labels for issues but not for pull requests! - labels = if issuable.is_a?(MergeRequest) - client.labels_for_issue(repo, raw_issuable.number) - else - raw_issuable.labels - end + def apply_labels(issuable, raw) + return unless raw.labels.count > 0 - if labels.count > 0 - label_ids = labels - .map { |attrs| @labels[attrs.name] } - .compact + label_ids = raw.labels + .map { |attrs| @labels[attrs.name] } + .compact - issuable.update_attribute(:label_ids, label_ids) - end + issuable.update_attribute(:label_ids, label_ids) end def import_comments(issuable_type) @@ -228,12 +263,18 @@ module Gitlab gh_release = ReleaseFormatter.new(project, raw) gh_release.create! if gh_release.valid? rescue => e - errors << { type: :release, url: Gitlab::UrlSanitizer.sanitize(raw.url), errors: e.message } + errors << { type: :release, url: Gitlab::UrlSanitizer.sanitize(gh_release.url), errors: e.message } end end end end + def cache_labels! + project.labels.select(:id, :title).find_each do |label| + @labels[label.title] = label.id + end + end + def fetch_resources(resource_type, *opts) return if imported?(resource_type) diff --git a/lib/gitlab/github_import/issuable_formatter.rb b/lib/gitlab/github_import/issuable_formatter.rb new file mode 100644 index 00000000000..256f360efc7 --- /dev/null +++ b/lib/gitlab/github_import/issuable_formatter.rb @@ -0,0 +1,60 @@ +module Gitlab + module GithubImport + class IssuableFormatter < BaseFormatter + def project_association + raise NotImplementedError + end + + def number + raw_data.number + end + + def find_condition + { iid: number } + end + + private + + def state + raw_data.state == 'closed' ? 'closed' : 'opened' + end + + def assigned? + raw_data.assignee.present? + end + + def assignee_id + if assigned? + gitlab_user_id(raw_data.assignee.id) + end + end + + def author + raw_data.user.login + end + + def author_id + gitlab_author_id || project.creator_id + end + + def body + raw_data.body || "" + end + + def description + if gitlab_author_id + body + else + formatter.author_line(author) + body + end + end + + def milestone + if raw_data.milestone.present? + milestone = MilestoneFormatter.new(project, raw_data.milestone) + project.milestones.find_by(milestone.find_condition) + end + end + end + end +end diff --git a/lib/gitlab/github_import/issue_formatter.rb b/lib/gitlab/github_import/issue_formatter.rb index 8c32ac59fc5..6f5ac4dac0d 100644 --- a/lib/gitlab/github_import/issue_formatter.rb +++ b/lib/gitlab/github_import/issue_formatter.rb @@ -1,6 +1,6 @@ module Gitlab module GithubImport - class IssueFormatter < BaseFormatter + class IssueFormatter < IssuableFormatter def attributes { iid: number, @@ -24,58 +24,8 @@ module Gitlab :issues end - def find_condition - { iid: number } - end - - def number - raw_data.number - end - - def valid? - raw_data.pull_request.nil? - end - - private - - def assigned? - raw_data.assignee.present? - end - - def assignee_id - if assigned? - gitlab_user_id(raw_data.assignee.id) - end - end - - def author - raw_data.user.login - end - - def author_id - gitlab_author_id || project.creator_id - end - - def body - raw_data.body || "" - end - - def description - if gitlab_author_id - body - else - formatter.author_line(author) + body - end - end - - def milestone - if raw_data.milestone.present? - project.milestones.find_by(iid: raw_data.milestone.number) - end - end - - def state - raw_data.state == 'closed' ? 'closed' : 'opened' + def pull_request? + raw_data.pull_request.present? end end end diff --git a/lib/gitlab/github_import/milestone_formatter.rb b/lib/gitlab/github_import/milestone_formatter.rb index 401dd962521..dd782eff059 100644 --- a/lib/gitlab/github_import/milestone_formatter.rb +++ b/lib/gitlab/github_import/milestone_formatter.rb @@ -3,7 +3,7 @@ module Gitlab class MilestoneFormatter < BaseFormatter def attributes { - iid: raw_data.number, + iid: number, project: project, title: raw_data.title, description: raw_data.description, @@ -19,7 +19,15 @@ module Gitlab end def find_condition - { iid: raw_data.number } + { iid: number } + end + + def number + if project.gitea_import? + raw_data.id + else + raw_data.number + end end private diff --git a/lib/gitlab/github_import/project_creator.rb b/lib/gitlab/github_import/project_creator.rb index a2410068845..3f635be22ba 100644 --- a/lib/gitlab/github_import/project_creator.rb +++ b/lib/gitlab/github_import/project_creator.rb @@ -1,14 +1,15 @@ module Gitlab module GithubImport class ProjectCreator - attr_reader :repo, :name, :namespace, :current_user, :session_data + attr_reader :repo, :name, :namespace, :current_user, :session_data, :type - def initialize(repo, name, namespace, current_user, session_data) + def initialize(repo, name, namespace, current_user, session_data, type: 'github') @repo = repo @name = name @namespace = namespace @current_user = current_user @session_data = session_data + @type = type end def execute @@ -19,7 +20,7 @@ module Gitlab description: repo.description, namespace_id: namespace.id, visibility_level: visibility_level, - import_type: "github", + import_type: type, import_source: repo.full_name, import_url: import_url, skip_wiki: skip_wiki @@ -29,7 +30,7 @@ module Gitlab private def import_url - repo.clone_url.sub('https://', "https://#{session_data[:github_access_token]}@") + repo.clone_url.sub('://', "://#{session_data[:github_access_token]}@") end def visibility_level diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb index b9a227fb11a..4ea0200e89b 100644 --- a/lib/gitlab/github_import/pull_request_formatter.rb +++ b/lib/gitlab/github_import/pull_request_formatter.rb @@ -1,6 +1,6 @@ module Gitlab module GithubImport - class PullRequestFormatter < BaseFormatter + class PullRequestFormatter < IssuableFormatter delegate :exists?, :project, :ref, :repo, :sha, to: :source_branch, prefix: true delegate :exists?, :project, :ref, :repo, :sha, to: :target_branch, prefix: true @@ -28,14 +28,6 @@ module Gitlab :merge_requests end - def find_condition - { iid: number } - end - - def number - raw_data.number - end - def valid? source_branch.valid? && target_branch.valid? end @@ -60,57 +52,15 @@ module Gitlab end end - def url - raw_data.url - end - private - def assigned? - raw_data.assignee.present? - end - - def assignee_id - if assigned? - gitlab_user_id(raw_data.assignee.id) - end - end - - def author - raw_data.user.login - end - - def author_id - gitlab_author_id || project.creator_id - end - - def body - raw_data.body || "" - end - - def description - if gitlab_author_id - body + def state + if raw_data.state == 'closed' && raw_data.merged_at.present? + 'merged' else - formatter.author_line(author) + body + super end end - - def milestone - if raw_data.milestone.present? - project.milestones.find_by(iid: raw_data.milestone.number) - end - end - - def state - @state ||= if raw_data.state == 'closed' && raw_data.merged_at.present? - 'merged' - elsif raw_data.state == 'closed' - 'closed' - else - 'opened' - end - end end end end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 2c21804fe7a..b8a5ac907a4 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -8,9 +8,12 @@ module Gitlab gon.shortcuts_path = help_page_path('shortcuts') gon.user_color_scheme = Gitlab::ColorSchemes.for_user(current_user).css_class gon.award_menu_url = emojis_path + gon.katex_css_url = ActionController::Base.helpers.asset_path('katex.css') + gon.katex_js_url = ActionController::Base.helpers.asset_path('katex.js') if current_user gon.current_user_id = current_user.id + gon.current_username = current_user.username end end end diff --git a/lib/gitlab/identifier.rb b/lib/gitlab/identifier.rb index f8809db21aa..94678b6ec40 100644 --- a/lib/gitlab/identifier.rb +++ b/lib/gitlab/identifier.rb @@ -21,10 +21,8 @@ module Gitlab return if !commit || !commit.author_email - email = commit.author_email - - identify_with_cache(:email, email) do - User.find_by(email: email) + identify_with_cache(:email, commit.author_email) do + commit.author end end diff --git a/lib/gitlab/import_export/attribute_cleaner.rb b/lib/gitlab/import_export/attribute_cleaner.rb index f755a404693..34169319b26 100644 --- a/lib/gitlab/import_export/attribute_cleaner.rb +++ b/lib/gitlab/import_export/attribute_cleaner.rb @@ -3,10 +3,25 @@ module Gitlab class AttributeCleaner ALLOWED_REFERENCES = RelationFactory::PROJECT_REFERENCES + RelationFactory::USER_REFERENCES + ['group_id'] - def self.clean!(relation_hash:) - relation_hash.reject! do |key, _value| - key.end_with?('_id') && !ALLOWED_REFERENCES.include?(key) - end + def self.clean(*args) + new(*args).clean + end + + def initialize(relation_hash:, relation_class:) + @relation_hash = relation_hash + @relation_class = relation_class + end + + def clean + @relation_hash.reject do |key, _value| + prohibited_key?(key) || !@relation_class.attribute_method?(key) + end.except('id') + end + + private + + def prohibited_key?(key) + key.end_with?('_id') && !ALLOWED_REFERENCES.include?(key) end end end diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index 113895ba22c..ffd17118c91 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -43,6 +43,14 @@ module Gitlab raise Projects::ImportService::Error.new("Unable to decompress #{@archive_file} into #{@shared.export_path}") unless result + remove_symlinks! + end + + def remove_symlinks! + Dir["#{@shared.export_path}/**/*"].each do |path| + FileUtils.rm(path) if File.lstat(path).symlink? + end + true end end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index e6ecd118609..08ad3274b38 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -6,6 +6,7 @@ project_tree: - :events - issues: - :events + - :timelogs - notes: - :author - :events @@ -27,6 +28,7 @@ project_tree: - :events - :merge_request_diff - :events + - :timelogs - label_links: - label: :priorities diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index 36c4cf6efa0..2405b94db50 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -1,13 +1,10 @@ module Gitlab module ImportExport class MembersMapper - attr_reader :missing_author_ids - def initialize(exported_members:, user:, project:) - @exported_members = exported_members + @exported_members = user.admin? ? exported_members : [] @user = user @project = project - @missing_author_ids = [] # This needs to run first, as second call would be from #map # which means project members already exist. @@ -39,7 +36,6 @@ module Gitlab def missing_keys_tracking_hash Hash.new do |_, key| - @missing_author_ids << key default_user_id end end @@ -55,11 +51,16 @@ module Gitlab end def member_hash(member) - member.except('id').merge(source_id: @project.id, importing: true) + parsed_hash(member).merge('source_id' => @project.id, 'importing' => true) + end + + def parsed_hash(member) + Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: member.deep_stringify_keys, + relation_class: ProjectMember) end def find_project_user_query(member) - user_arel[:username].eq(member['user']['username']).or(user_arel[:email].eq(member['user']['email'])) + user_arel[:email].eq(member['user']['email']).or(user_arel[:username].eq(member['user']['username'])) end def user_arel diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 7cdba880a93..cda6ddf0443 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -9,8 +9,14 @@ module Gitlab end def restore - json = IO.read(@path) - @tree_hash = ActiveSupport::JSON.decode(json) + begin + json = IO.read(@path) + @tree_hash = ActiveSupport::JSON.decode(json) + rescue => e + Rails.logger.error("Import/Export error: #{e.message}") + raise Gitlab::ImportExport::Error.new('Incorrect JSON format') + end + @project_members = @tree_hash.delete('project_members') ActiveRecord::Base.no_touching do @@ -114,7 +120,7 @@ module Gitlab members_mapper: members_mapper, user: @user, project_id: restored_project.id) - end + end.compact relation_hash_list.is_a?(Array) ? relation_array : relation_array.first end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index dc630e76411..19e43cce768 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -14,7 +14,7 @@ module Gitlab priorities: :label_priorities, label: :project_label }.freeze - USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id].freeze + USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id created_by_id merge_user_id resolved_by_id].freeze PROJECT_REFERENCES = %w[project_id source_project_id gl_project_id target_project_id].freeze @@ -22,7 +22,7 @@ module Gitlab IMPORTED_OBJECT_MAX_RETRIES = 5.freeze - EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels project_label group_label].freeze + EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels group_label group_labels].freeze def self.create(*args) new(*args).create @@ -30,7 +30,7 @@ module Gitlab def initialize(relation_sym:, relation_hash:, members_mapper:, user:, project_id:) @relation_name = OVERRIDES[relation_sym] || relation_sym - @relation_hash = relation_hash.except('id', 'noteable_id').merge('project_id' => project_id) + @relation_hash = relation_hash.except('noteable_id').merge('project_id' => project_id) @members_mapper = members_mapper @user = user @imported_object_retries = 0 @@ -40,6 +40,8 @@ module Gitlab # the relation_hash, updating references with new object IDs, mapping users using # the "members_mapper" object, also updating notes if required. def create + return nil if unknown_service? + setup_models generate_imported_object @@ -78,17 +80,13 @@ module Gitlab # is left. def set_note_author old_author_id = @relation_hash['author_id'] - - # Users with admin access can map users - @relation_hash['author_id'] = admin_user? ? @members_mapper.map[old_author_id] : @members_mapper.default_user_id - author = @relation_hash.delete('author') - update_note_for_missing_author(author['name']) if missing_author?(old_author_id) + update_note_for_missing_author(author['name']) unless has_author?(old_author_id) end - def missing_author?(old_author_id) - !admin_user? || @members_mapper.missing_author_ids.include?(old_author_id) + def has_author?(old_author_id) + admin_user? && @members_mapper.map.keys.include?(old_author_id) end def missing_author_note(updated_at, author_name) @@ -99,6 +97,8 @@ module Gitlab def generate_imported_object if BUILD_MODELS.include?(@relation_name) # call #trace= method after assigning the other attributes trace = @relation_hash.delete('trace') + @relation_hash.delete('token') + imported_object do |object| object.trace = trace object.commit_id = nil @@ -172,11 +172,8 @@ module Gitlab end def parsed_relation_hash - @parsed_relation_hash ||= begin - Gitlab::ImportExport::AttributeCleaner.clean!(relation_hash: @relation_hash) - - @relation_hash.reject { |k, _v| !relation_class.attribute_method?(k) } - end + @parsed_relation_hash ||= Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: @relation_hash, + relation_class: relation_class) end def set_st_diffs @@ -188,7 +185,7 @@ module Gitlab # Otherwise always create the record, skipping the extra SELECT clause. @existing_or_new_object ||= begin if EXISTING_OBJECT_CHECK.include?(@relation_name) - attribute_hash = attribute_hash_for(['events', 'priorities']) + attribute_hash = attribute_hash_for(['events']) existing_object.assign_attributes(attribute_hash) if attribute_hash.any? @@ -209,15 +206,38 @@ module Gitlab def existing_object @existing_object ||= begin - finder_attributes = @relation_name == :group_label ? %w[title group_id] : %w[title project_id] - finder_hash = parsed_relation_hash.slice(*finder_attributes) - existing_object = relation_class.find_or_create_by(finder_hash) + existing_object = find_or_create_object! + # Done in two steps, as MySQL behaves differently than PostgreSQL using # the +find_or_create_by+ method and does not return the ID the second time. existing_object.update!(parsed_relation_hash) existing_object end end + + def unknown_service? + @relation_name == :services && parsed_relation_hash['type'] && + !Object.const_defined?(parsed_relation_hash['type']) + end + + def find_or_create_object! + finder_attributes = @relation_name == :group_label ? %w[title group_id] : %w[title project_id] + finder_hash = parsed_relation_hash.slice(*finder_attributes) + + if label? + label = relation_class.find_or_initialize_by(finder_hash) + parsed_relation_hash.delete('priorities') if label.persisted? + + label.save! + label + else + relation_class.find_or_create_by(finder_hash) + end + end + + def label? + @relation_name.to_s.include?('label') + end end end end diff --git a/lib/gitlab/import_export/version_checker.rb b/lib/gitlab/import_export/version_checker.rb index fc08082fc86..bd3c3ee3b2f 100644 --- a/lib/gitlab/import_export/version_checker.rb +++ b/lib/gitlab/import_export/version_checker.rb @@ -24,12 +24,19 @@ module Gitlab end def verify_version!(version) - if Gem::Version.new(version) != Gem::Version.new(Gitlab::ImportExport.version) + if different_version?(version) raise Gitlab::ImportExport::Error.new("Import version mismatch: Required #{Gitlab::ImportExport.version} but was #{version}") else true end end + + def different_version?(version) + Gem::Version.new(version) != Gem::Version.new(Gitlab::ImportExport.version) + rescue => e + Rails.logger.error("Import/Export error: #{e.message}") + raise Gitlab::ImportExport::Error.new('Incorrect VERSION format') + end end end end diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index 94261b7eeed..45958710c13 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -7,21 +7,38 @@ module Gitlab module ImportSources extend CurrentSettings + ImportSource = Struct.new(:name, :title, :importer) + + ImportTable = [ + ImportSource.new('github', 'GitHub', Gitlab::GithubImport::Importer), + ImportSource.new('bitbucket', 'Bitbucket', Gitlab::BitbucketImport::Importer), + ImportSource.new('gitlab', 'GitLab.com', Gitlab::GitlabImport::Importer), + ImportSource.new('google_code', 'Google Code', Gitlab::GoogleCodeImport::Importer), + ImportSource.new('fogbugz', 'FogBugz', Gitlab::FogbugzImport::Importer), + ImportSource.new('git', 'Repo by URL', nil), + ImportSource.new('gitlab_project', 'GitLab export', Gitlab::ImportExport::Importer), + ImportSource.new('gitea', 'Gitea', Gitlab::GithubImport::Importer) + ].freeze + class << self + def options + @options ||= Hash[ImportTable.map { |importer| [importer.title, importer.name] }] + end + def values - options.values + @values ||= ImportTable.map(&:name) end - def options - { - 'GitHub' => 'github', - 'Bitbucket' => 'bitbucket', - 'GitLab.com' => 'gitlab', - 'Google Code' => 'google_code', - 'FogBugz' => 'fogbugz', - 'Repo by URL' => 'git', - 'GitLab export' => 'gitlab_project' - } + def importer_names + @importer_names ||= ImportTable.select(&:importer).map(&:name) + end + + def importer(name) + ImportTable.find { |import_source| import_source.name == name }.importer + end + + def title(name) + options.key(name) end end end diff --git a/lib/gitlab/incoming_email.rb b/lib/gitlab/incoming_email.rb index d7be50bd437..b91012d6405 100644 --- a/lib/gitlab/incoming_email.rb +++ b/lib/gitlab/incoming_email.rb @@ -1,5 +1,8 @@ module Gitlab module IncomingEmail + UNSUBSCRIBE_SUFFIX = '+unsubscribe'.freeze + WILDCARD_PLACEHOLDER = '%{key}'.freeze + class << self FALLBACK_MESSAGE_ID_REGEX = /\Areply\-(.+)@#{Gitlab.config.gitlab.host}\Z/.freeze @@ -7,8 +10,20 @@ module Gitlab config.enabled && config.address end + def supports_wildcard? + config.address && config.address.include?(WILDCARD_PLACEHOLDER) + end + + def supports_issue_creation? + enabled? && supports_wildcard? + end + def reply_address(key) - config.address.gsub('%{key}', key) + config.address.sub(WILDCARD_PLACEHOLDER, key) + end + + def unsubscribe_address(key) + config.address.sub(WILDCARD_PLACEHOLDER, "#{key}#{UNSUBSCRIBE_SUFFIX}") end def key_from_address(address) @@ -39,7 +54,7 @@ module Gitlab return nil unless wildcard_address regex = Regexp.escape(wildcard_address) - regex = regex.gsub(Regexp.escape('%{key}'), "(.+)") + regex = regex.sub(Regexp.escape(WILDCARD_PLACEHOLDER), '(.+)') Regexp.new(regex).freeze end end diff --git a/lib/gitlab/kubernetes.rb b/lib/gitlab/kubernetes.rb new file mode 100644 index 00000000000..288771c1c12 --- /dev/null +++ b/lib/gitlab/kubernetes.rb @@ -0,0 +1,80 @@ +module Gitlab + # Helper methods to do with Kubernetes network services & resources + module Kubernetes + # This is the comand that is run to start a terminal session. Kubernetes + # expects `command=foo&command=bar, not `command[]=foo&command[]=bar` + EXEC_COMMAND = URI.encode_www_form( + ['sh', '-c', 'bash || sh'].map { |value| ['command', value] } + ) + + # Filters an array of pods (as returned by the kubernetes API) by their labels + def filter_pods(pods, labels = {}) + pods.select do |pod| + metadata = pod.fetch("metadata", {}) + pod_labels = metadata.fetch("labels", nil) + next unless pod_labels + + labels.all? { |k, v| pod_labels[k.to_s] == v } + end + end + + # Converts a pod (as returned by the kubernetes API) into a terminal + def terminals_for_pod(api_url, namespace, pod) + metadata = pod.fetch("metadata", {}) + status = pod.fetch("status", {}) + spec = pod.fetch("spec", {}) + + containers = spec["containers"] + pod_name = metadata["name"] + phase = status["phase"] + + return unless containers.present? && pod_name.present? && phase == "Running" + + created_at = DateTime.parse(metadata["creationTimestamp"]) rescue nil + + containers.map do |container| + { + selectors: { pod: pod_name, container: container["name"] }, + url: container_exec_url(api_url, namespace, pod_name, container["name"]), + subprotocols: ['channel.k8s.io'], + headers: Hash.new { |h, k| h[k] = [] }, + created_at: created_at, + } + end + end + + def add_terminal_auth(terminal, token, ca_pem = nil) + terminal[:headers]['Authorization'] << "Bearer #{token}" + terminal[:ca_pem] = ca_pem if ca_pem.present? + terminal + end + + def container_exec_url(api_url, namespace, pod_name, container_name) + url = URI.parse(api_url) + url.path = [ + url.path.sub(%r{/+\z}, ''), + 'api', 'v1', + 'namespaces', ERB::Util.url_encode(namespace), + 'pods', ERB::Util.url_encode(pod_name), + 'exec' + ].join('/') + + url.query = { + container: container_name, + tty: true, + stdin: true, + stdout: true, + stderr: true, + }.to_query + '&' + EXEC_COMMAND + + case url.scheme + when 'http' + url.scheme = 'ws' + when 'https' + url.scheme = 'wss' + end + + url.to_s + end + end +end diff --git a/lib/gitlab/ldap/access.rb b/lib/gitlab/ldap/access.rb index 7e06bd2b0fb..54a5b1d31cd 100644 --- a/lib/gitlab/ldap/access.rb +++ b/lib/gitlab/ldap/access.rb @@ -34,21 +34,21 @@ module Gitlab def allowed? if ldap_user unless ldap_config.active_directory - user.activate if user.ldap_blocked? + unblock_user(user, 'is available again') if user.ldap_blocked? return true end # Block user in GitLab if he/she was blocked in AD if Gitlab::LDAP::Person.disabled_via_active_directory?(user.ldap_identity.extern_uid, adapter) - user.ldap_block + block_user(user, 'is disabled in Active Directory') false else - user.activate if user.ldap_blocked? + unblock_user(user, 'is not disabled anymore') if user.ldap_blocked? true end else # Block the user if they no longer exist in LDAP/AD - user.ldap_block + block_user(user, 'does not exist anymore') false end end @@ -64,6 +64,24 @@ module Gitlab def ldap_user @ldap_user ||= Gitlab::LDAP::Person.find_by_dn(user.ldap_identity.extern_uid, adapter) end + + def block_user(user, reason) + user.ldap_block + + Gitlab::AppLogger.info( + "LDAP account \"#{user.ldap_identity.extern_uid}\" #{reason}, " \ + "blocking Gitlab user \"#{user.name}\" (#{user.email})" + ) + end + + def unblock_user(user, reason) + user.activate + + Gitlab::AppLogger.info( + "LDAP account \"#{user.ldap_identity.extern_uid}\" #{reason}, " \ + "unblocking Gitlab user \"#{user.name}\" (#{user.email})" + ) + end end end end diff --git a/lib/gitlab/ldap/adapter.rb b/lib/gitlab/ldap/adapter.rb index 8b38cfaefb6..7b05290e5cc 100644 --- a/lib/gitlab/ldap/adapter.rb +++ b/lib/gitlab/ldap/adapter.rb @@ -89,9 +89,7 @@ module Gitlab end def user_filter(filter = nil) - if config.user_filter.present? - user_filter = Net::LDAP::Filter.construct(config.user_filter) - end + user_filter = config.constructed_user_filter if config.user_filter.present? if user_filter && filter Net::LDAP::Filter.join(filter, user_filter) diff --git a/lib/gitlab/ldap/auth_hash.rb b/lib/gitlab/ldap/auth_hash.rb index bf4dd9542d5..95378e5a769 100644 --- a/lib/gitlab/ldap/auth_hash.rb +++ b/lib/gitlab/ldap/auth_hash.rb @@ -25,7 +25,7 @@ module Gitlab end def get_raw(key) - auth_hash.extra[:raw_info][key] + auth_hash.extra[:raw_info][key] if auth_hash.extra end def ldap_config diff --git a/lib/gitlab/ldap/authentication.rb b/lib/gitlab/ldap/authentication.rb index bad683c6511..4745311402c 100644 --- a/lib/gitlab/ldap/authentication.rb +++ b/lib/gitlab/ldap/authentication.rb @@ -54,11 +54,9 @@ module Gitlab # Apply LDAP user filter if present if config.user_filter.present? - filter = Net::LDAP::Filter.join( - filter, - Net::LDAP::Filter.construct(config.user_filter) - ) + filter = Net::LDAP::Filter.join(filter, config.constructed_user_filter) end + filter end diff --git a/lib/gitlab/ldap/config.rb b/lib/gitlab/ldap/config.rb index f9bb5775323..28129198438 100644 --- a/lib/gitlab/ldap/config.rb +++ b/lib/gitlab/ldap/config.rb @@ -13,7 +13,7 @@ module Gitlab end def self.providers - servers.map {|server| server['provider_name'] } + servers.map { |server| server['provider_name'] } end def self.valid_provider?(provider) @@ -38,13 +38,31 @@ module Gitlab end def adapter_options - { - host: options['host'], - port: options['port'], - encryption: encryption - }.tap do |options| - options.merge!(auth_options) if has_auth? + opts = base_options.merge( + encryption: encryption, + ) + + opts.merge!(auth_options) if has_auth? + + opts + end + + def omniauth_options + opts = base_options.merge( + base: base, + method: options['method'], + filter: omniauth_user_filter, + name_proc: name_proc + ) + + if has_auth? + opts.merge!( + bind_dn: options['bind_dn'], + password: options['password'] + ) end + + opts end def base @@ -68,6 +86,10 @@ module Gitlab options['user_filter'] end + def constructed_user_filter + @constructed_user_filter ||= Net::LDAP::Filter.construct(user_filter) + end + def group_base options['group_base'] end @@ -85,15 +107,48 @@ module Gitlab end def attributes - options['attributes'] + default_attributes.merge(options['attributes']) end def timeout options['timeout'].to_i end + def has_auth? + options['password'] || options['bind_dn'] + end + + def allow_username_or_email_login + options['allow_username_or_email_login'] + end + + def name_proc + if allow_username_or_email_login + Proc.new { |name| name.gsub(/@.*\z/, '') } + else + Proc.new { |name| name } + end + end + + def default_attributes + { + 'username' => %w(uid userid sAMAccountName), + 'email' => %w(mail email userPrincipalName), + 'name' => 'cn', + 'first_name' => 'givenName', + 'last_name' => 'sn' + } + end + protected + def base_options + { + host: options['host'], + port: options['port'] + } + end + def base_config Gitlab.config.ldap end @@ -123,8 +178,14 @@ module Gitlab } end - def has_auth? - options['password'] || options['bind_dn'] + def omniauth_user_filter + uid_filter = Net::LDAP::Filter.eq(uid, '%{username}') + + if user_filter.present? + Net::LDAP::Filter.join(uid_filter, constructed_user_filter).to_s + else + uid_filter.to_s + end end end end diff --git a/lib/gitlab/ldap/person.rb b/lib/gitlab/ldap/person.rb index b81f3e8e8f5..7084fd1767d 100644 --- a/lib/gitlab/ldap/person.rb +++ b/lib/gitlab/ldap/person.rb @@ -28,7 +28,7 @@ module Gitlab end def name - entry.cn.first + attribute_value(:name).first end def uid @@ -40,7 +40,7 @@ module Gitlab end def email - entry.try(:mail) + attribute_value(:email) end def dn @@ -56,6 +56,19 @@ module Gitlab def config @config ||= Gitlab::LDAP::Config.new(provider) end + + # Using the LDAP attributes configuration, find and return the first + # attribute with a value. For example, by default, when given 'email', + # this method looks for 'mail', 'email' and 'userPrincipalName' and + # returns the first with a value. + def attribute_value(attribute) + attributes = Array(config.attributes[attribute.to_s]) + selected_attr = attributes.find { |attr| entry.respond_to?(attr) } + + return nil unless selected_attr + + entry.public_send(selected_attr) + end end end end diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb index a5220d92312..3503fac40e8 100644 --- a/lib/gitlab/mail_room.rb +++ b/lib/gitlab/mail_room.rb @@ -31,6 +31,7 @@ module Gitlab config[:ssl] = false if config[:ssl].nil? config[:start_tls] = false if config[:start_tls].nil? config[:mailbox] = 'inbox' if config[:mailbox].nil? + config[:idle_timeout] = 60 if config[:idle_timeout].nil? if config[:enabled] && config[:address] gitlab_redis = Gitlab::Redis.new(rails_env) diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb index 01c96a6fe96..47f88727fc8 100644 --- a/lib/gitlab/metrics/rack_middleware.rb +++ b/lib/gitlab/metrics/rack_middleware.rb @@ -70,8 +70,19 @@ module Gitlab def tag_endpoint(trans, env) endpoint = env[ENDPOINT_KEY] - path = endpoint_paths_cache[endpoint.route.route_method][endpoint.route.route_path] - trans.action = "Grape##{endpoint.route.route_method} #{path}" + + begin + route = endpoint.route + rescue + # endpoint.route is calling env[Grape::Env::GRAPE_ROUTING_ARGS][:route_info] + # but env[Grape::Env::GRAPE_ROUTING_ARGS] is nil in the case of a 405 response + # so we're rescuing exceptions and bailing out + end + + if route + path = endpoint_paths_cache[route.request_method][route.path] + trans.action = "Grape##{route.request_method} #{path}" + end end private diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb new file mode 100644 index 00000000000..dd99f9bb7d7 --- /dev/null +++ b/lib/gitlab/middleware/multipart.rb @@ -0,0 +1,103 @@ +# Gitlab::Middleware::Multipart - a Rack::Multipart replacement +# +# Rack::Multipart leaves behind tempfiles in /tmp and uses valuable Ruby +# process time to copy files around. This alternative solution uses +# gitlab-workhorse to clean up the tempfiles and puts the tempfiles in a +# location where copying should not be needed. +# +# When gitlab-workhorse finds files in a multipart MIME body it sends +# a signed message via a request header. This message lists the names of +# the multipart entries that gitlab-workhorse filtered out of the +# multipart structure and saved to tempfiles. Workhorse adds new entries +# in the multipart structure with paths to the tempfiles. +# +# The job of this Rack middleware is to detect and decode the message +# from workhorse. If present, it walks the Rack 'params' hash for the +# current request, opens the respective tempfiles, and inserts the open +# Ruby File objects in the params hash where Rack::Multipart would have +# put them. The goal is that application code deeper down can keep +# working the way it did with Rack::Multipart without changes. +# +# CAVEAT: the code that modifies the params hash is a bit complex. It is +# conceivable that certain Rack params structures will not be modified +# correctly. We are not aware of such bugs at this time though. +# + +module Gitlab + module Middleware + class Multipart + RACK_ENV_KEY = 'HTTP_GITLAB_WORKHORSE_MULTIPART_FIELDS' + + class Handler + def initialize(env, message) + @request = Rack::Request.new(env) + @rewritten_fields = message['rewritten_fields'] + @open_files = [] + end + + def with_open_files + @rewritten_fields.each do |field, tmp_path| + parsed_field = Rack::Utils.parse_nested_query(field) + raise "unexpected field: #{field.inspect}" unless parsed_field.count == 1 + + key, value = parsed_field.first + if value.nil? + value = open_file(tmp_path) + @open_files << value + else + value = decorate_params_value(value, @request.params[key], tmp_path) + end + @request.update_param(key, value) + end + + yield + ensure + @open_files.each(&:close) + end + + # This function calls itself recursively + def decorate_params_value(path_hash, value_hash, tmp_path) + unless path_hash.is_a?(Hash) && path_hash.count == 1 + raise "invalid path: #{path_hash.inspect}" + end + path_key, path_value = path_hash.first + + unless value_hash.is_a?(Hash) && value_hash[path_key] + raise "invalid value hash: #{value_hash.inspect}" + end + + case path_value + when nil + value_hash[path_key] = open_file(tmp_path) + @open_files << value_hash[path_key] + value_hash + when Hash + decorate_params_value(path_value, value_hash[path_key], tmp_path) + value_hash + else + raise "unexpected path value: #{path_value.inspect}" + end + end + + def open_file(path) + ::UploadedFile.new(path, File.basename(path), 'application/octet-stream') + end + end + + def initialize(app) + @app = app + end + + def call(env) + encoded_message = env.delete(RACK_ENV_KEY) + return @app.call(env) if encoded_message.blank? + + message = Gitlab::Workhorse.decode_jwt(encoded_message)[0] + + Handler.new(env, message).with_open_files do + @app.call(env) + end + end + end + end +end diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index 0a91d3918d5..96ed20af918 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -39,7 +39,7 @@ module Gitlab log.info "(#{provider}) saving user #{auth_hash.email} from login with extern_uid => #{auth_hash.uid}" gl_user rescue ActiveRecord::RecordInvalid => e - log.info "(#{provider}) Error saving user: #{gl_user.errors.full_messages}" + log.info "(#{provider}) Error saving user #{auth_hash.uid} (#{auth_hash.email}): #{gl_user.errors.full_messages}" return self, e.record.errors end @@ -102,6 +102,8 @@ module Gitlab Gitlab::LDAP::Config.providers.each do |provider| adapter = Gitlab::LDAP::Adapter.new(provider) @ldap_person = Gitlab::LDAP::Person.find_by_uid(auth_hash.uid, adapter) + # The `uid` might actually be a DN. Try it next. + @ldap_person ||= Gitlab::LDAP::Person.find_by_dn(auth_hash.uid, adapter) break if @ldap_person end @ldap_person diff --git a/lib/gitlab/popen.rb b/lib/gitlab/popen.rb index cc74bb29087..4bc5cda8cb5 100644 --- a/lib/gitlab/popen.rb +++ b/lib/gitlab/popen.rb @@ -5,13 +5,13 @@ module Gitlab module Popen extend self - def popen(cmd, path = nil) + def popen(cmd, path = nil, vars = {}) unless cmd.is_a?(Array) raise "System commands must be given as an array of strings" end path ||= Dir.pwd - vars = { "PWD" => path } + vars['PWD'] = path options = { chdir: path } unless File.directory?(path) diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 24733435a5a..6bdf3db9cb8 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -5,11 +5,7 @@ module Gitlab def initialize(current_user, project, query, repository_ref = nil) @current_user = current_user @project = project - @repository_ref = if repository_ref.present? - repository_ref - else - nil - end + @repository_ref = repository_ref.presence || project.default_branch @query = query end @@ -44,36 +40,81 @@ module Gitlab @commits_count ||= commits.count end + def self.parse_search_result(result) + ref = nil + filename = nil + basename = nil + startline = 0 + + result.each_line.each_with_index do |line, index| + if line =~ /^.*:.*:\d+:/ + ref, filename, startline = line.split(':') + startline = startline.to_i - index + extname = Regexp.escape(File.extname(filename)) + basename = filename.sub(/#{extname}$/, '') + break + end + end + + data = "" + + result.each_line do |line| + data << line.sub(ref, '').sub(filename, '').sub(/^:-\d+-/, '').sub(/^::\d+:/, '') + end + + OpenStruct.new( + filename: filename, + basename: basename, + ref: ref, + startline: startline, + data: data + ) + end + private def blobs - if project.empty_repo? || query.blank? - [] - else - project.repository.search_files(query, repository_ref) + @blobs ||= begin + blobs = project.repository.search_files_by_content(query, repository_ref).first(100) + found_file_names = Set.new + + results = blobs.map do |blob| + blob = self.class.parse_search_result(blob) + found_file_names << blob.filename + + [blob.filename, blob] + end + + project.repository.search_files_by_name(query, repository_ref).first(100).each do |filename| + results << [filename, nil] unless found_file_names.include?(filename) + end + + results.sort_by(&:first) end end def wiki_blobs - if project.wiki_enabled? && query.present? - project_wiki = ProjectWiki.new(project) + @wiki_blobs ||= begin + if project.wiki_enabled? && query.present? + project_wiki = ProjectWiki.new(project) - unless project_wiki.empty? - project_wiki.search_files(query) + unless project_wiki.empty? + project_wiki.search_files(query) + else + [] + end else [] end - else - [] end end def notes - project.notes.user.search(query, as_user: @current_user).order('updated_at DESC') + @notes ||= NotesFinder.new(project, @current_user, search: query).execute.user.order('updated_at DESC') end def commits - project.repository.find_commits_by_message(query) + @commits ||= project.repository.find_commits_by_message(query) end def project_ids_relation diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb index 9226da2d6b1..9384102acec 100644 --- a/lib/gitlab/redis.rb +++ b/lib/gitlab/redis.rb @@ -42,7 +42,7 @@ module Gitlab return @_raw_config if defined?(@_raw_config) begin - @_raw_config = File.read(CONFIG_FILE).freeze + @_raw_config = ERB.new(File.read(CONFIG_FILE)).result.freeze rescue Errno::ENOENT @_raw_config = false end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 0d30e1bb92e..a3fa7c1331a 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -2,12 +2,25 @@ module Gitlab module Regex extend self - NAMESPACE_REGEX_STR = '(?:[a-zA-Z0-9_\.][a-zA-Z0-9_\-\.]*[a-zA-Z0-9_\-]|[a-zA-Z0-9_])(?<!\.git|\.atom)'.freeze + # The namespace regex is used in Javascript to validate usernames in the "Register" form. However, Javascript + # does not support the negative lookbehind assertion (?<!) that disallows usernames ending in `.git` and `.atom`. + # Since this is a non-trivial problem to solve in Javascript (heavily complicate the regex, modify view code to + # allow non-regex validatiions, etc), `NAMESPACE_REGEX_STR_SIMPLE` serves as a Javascript-compatible version of + # `NAMESPACE_REGEX_STR`, with the negative lookbehind assertion removed. This means that the client-side validation + # will pass for usernames ending in `.atom` and `.git`, but will be caught by the server-side validation. + PATH_REGEX_STR = '[a-zA-Z0-9_\.][a-zA-Z0-9_\-\.]*'.freeze + NAMESPACE_REGEX_STR_SIMPLE = PATH_REGEX_STR + '[a-zA-Z0-9_\-]|[a-zA-Z0-9_]'.freeze + NAMESPACE_REGEX_STR = '(?:' + NAMESPACE_REGEX_STR_SIMPLE + ')(?<!\.git|\.atom)'.freeze + PROJECT_REGEX_STR = PATH_REGEX_STR + '(?<!\.git|\.atom)'.freeze def namespace_regex @namespace_regex ||= /\A#{NAMESPACE_REGEX_STR}\z/.freeze end + def namespace_route_regex + @namespace_route_regex ||= /#{NAMESPACE_REGEX_STR}/.freeze + end + def namespace_regex_message "can contain only letters, digits, '_', '-' and '.'. " \ "Cannot start with '-' or end in '.', '.git' or '.atom'." \ @@ -22,16 +35,24 @@ module Gitlab end def project_name_regex - @project_name_regex ||= /\A[\p{Alnum}_][\p{Alnum}\p{Pd}_\. ]*\z/.freeze + @project_name_regex ||= /\A[\p{Alnum}\u{00A9}-\u{1f9c0}_][\p{Alnum}\p{Pd}\u{00A9}-\u{1f9c0}_\. ]*\z/.freeze end def project_name_regex_message - "can contain only letters, digits, '_', '.', dash and space. " \ - "It must start with letter, digit or '_'." + "can contain only letters, digits, emojis, '_', '.', dash, space. " \ + "It must start with letter, digit, emoji or '_'." end def project_path_regex - @project_path_regex ||= /\A[a-zA-Z0-9_.][a-zA-Z0-9_\-\.]*(?<!\.git|\.atom)\z/.freeze + @project_path_regex ||= /\A#{PROJECT_REGEX_STR}\z/.freeze + end + + def project_route_regex + @project_route_regex ||= /#{PROJECT_REGEX_STR}/.freeze + end + + def project_git_route_regex + @project_route_git_regex ||= /#{PATH_REGEX_STR}\.git/.freeze end def project_path_regex_message @@ -40,15 +61,15 @@ module Gitlab end def file_name_regex - @file_name_regex ||= /\A[a-zA-Z0-9_\-\.\@]*\z/.freeze + @file_name_regex ||= /\A[[[:alnum:]]_\-\.\@\+]*\z/.freeze end def file_name_regex_message - "can contain only letters, digits, '_', '-', '@' and '.'." + "can contain only letters, digits, '_', '-', '@', '+' and '.'." end def file_path_regex - @file_path_regex ||= /\A[a-zA-Z0-9_\-\.\/\@]*\z/.freeze + @file_path_regex ||= /\A[[[:alnum:]]_\-\.\/\@]*\z/.freeze end def file_path_regex_message @@ -102,5 +123,22 @@ module Gitlab def environment_name_regex_message "can contain only letters, digits, '-', '_', '/', '$', '{', '}', '.' and spaces" end + + def kubernetes_namespace_regex + /\A[a-z0-9]([-a-z0-9]*[a-z0-9])?\z/ + end + + def kubernetes_namespace_regex_message + "can contain only letters, digits or '-', and cannot start or end with '-'" + end + + def environment_slug_regex + @environment_slug_regex ||= /\A[a-z]([a-z0-9-]*[a-z0-9])?\z/.freeze + end + + def environment_slug_regex_message + "can contain only lowercase letters, digits, and '-'. " \ + "Must start with a letter, and cannot end with '-'" + end end end diff --git a/lib/gitlab/routing.rb b/lib/gitlab/routing.rb index 5132177de51..632e2d87500 100644 --- a/lib/gitlab/routing.rb +++ b/lib/gitlab/routing.rb @@ -1,5 +1,11 @@ module Gitlab module Routing + extend ActiveSupport::Concern + + included do + include Gitlab::Routing.url_helpers + end + # Returns the URL helpers Module. # # This method caches the output as Rails' "url_helpers" method creates an diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 2690938fe82..35212992698 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -50,7 +50,7 @@ module Gitlab end def issues - issues = Issue.visible_to_user(current_user).where(project_id: project_ids_relation) + issues = IssuesFinder.new(current_user).execute.where(project_id: project_ids_relation) if query =~ /#(\d+)\z/ issues = issues.where(iid: $1) @@ -68,7 +68,7 @@ module Gitlab end def merge_requests - merge_requests = MergeRequest.in_projects(project_ids_relation) + merge_requests = MergeRequestsFinder.new(current_user).execute.in_projects(project_ids_relation) if query =~ /[#!](\d+)\z/ merge_requests = merge_requests.where(iid: $1) else diff --git a/lib/gitlab/serialize/ci/variables.rb b/lib/gitlab/serialize/ci/variables.rb new file mode 100644 index 00000000000..3a9443bfcd9 --- /dev/null +++ b/lib/gitlab/serialize/ci/variables.rb @@ -0,0 +1,27 @@ +module Gitlab + module Serialize + module Ci + # This serializer could make sure our YAML variables' keys and values + # are always strings. This is more for legacy build data because + # from now on we convert them into strings before saving to database. + module Variables + extend self + + def load(string) + return unless string + + object = YAML.safe_load(string, [Symbol]) + + object.map do |variable| + variable[:key] = variable[:key].to_s + variable + end + end + + def dump(object) + YAML.dump(object) + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_throttler.rb b/lib/gitlab/sidekiq_throttler.rb new file mode 100644 index 00000000000..d4d39a888e7 --- /dev/null +++ b/lib/gitlab/sidekiq_throttler.rb @@ -0,0 +1,23 @@ +module Gitlab + class SidekiqThrottler + class << self + def execute! + if Gitlab::CurrentSettings.sidekiq_throttling_enabled? + Gitlab::CurrentSettings.current_application_settings.sidekiq_throttling_queues.each do |queue| + Sidekiq::Queue[queue].limit = queue_limit + end + end + end + + private + + def queue_limit + @queue_limit ||= + begin + factor = Gitlab::CurrentSettings.current_application_settings.sidekiq_throttling_factor + (factor * Sidekiq.options[:concurrency]).ceil + end + end + end + end +end diff --git a/lib/gitlab/sql/union.rb b/lib/gitlab/sql/union.rb index 1cd89b3a9c4..222021e8802 100644 --- a/lib/gitlab/sql/union.rb +++ b/lib/gitlab/sql/union.rb @@ -22,9 +22,7 @@ module Gitlab # By using "unprepared_statements" we remove the usage of placeholders # (thus fixing this problem), at a slight performance cost. fragments = ActiveRecord::Base.connection.unprepared_statement do - @relations.map do |rel| - rel.reorder(nil).to_sql - end + @relations.map { |rel| rel.reorder(nil).to_sql }.reject(&:blank?) end fragments.join("\nUNION\n") diff --git a/lib/gitlab/template/dockerfile_template.rb b/lib/gitlab/template/dockerfile_template.rb new file mode 100644 index 00000000000..d5d3e045a42 --- /dev/null +++ b/lib/gitlab/template/dockerfile_template.rb @@ -0,0 +1,30 @@ +module Gitlab + module Template + class DockerfileTemplate < BaseTemplate + def content + explanation = "# This file is a template, and might need editing before it works on your project." + [explanation, super].join("\n") + end + + class << self + def extension + 'Dockerfile' + end + + def categories + { + "General" => '' + } + end + + def base_dir + Rails.root.join('vendor/dockerfile') + end + + def finder(project = nil) + Gitlab::Template::Finders::GlobalTemplateFinder.new(self.base_dir, self.extension, self.categories) + end + end + end + end +end diff --git a/lib/gitlab/template/gitlab_ci_yml_template.rb b/lib/gitlab/template/gitlab_ci_yml_template.rb index 8d1a1ed54c9..9d2ecee9756 100644 --- a/lib/gitlab/template/gitlab_ci_yml_template.rb +++ b/lib/gitlab/template/gitlab_ci_yml_template.rb @@ -13,8 +13,9 @@ module Gitlab def categories { - "General" => '', - "Pages" => 'Pages' + 'General' => '', + 'Pages' => 'Pages', + 'Auto deploy' => 'autodeploy' } end @@ -25,6 +26,11 @@ module Gitlab def finder(project = nil) Gitlab::Template::Finders::GlobalTemplateFinder.new(self.base_dir, self.extension, self.categories) end + + def dropdown_names(context) + categories = context == 'autodeploy' ? ['Auto deploy'] : ['General', 'Pages'] + super().slice(*categories) + end end end end diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb index d4020af76f9..19ab76ae80f 100644 --- a/lib/gitlab/themes.rb +++ b/lib/gitlab/themes.rb @@ -15,7 +15,7 @@ module Gitlab Theme.new(1, 'Graphite', 'ui_graphite'), Theme.new(2, 'Charcoal', 'ui_charcoal'), Theme.new(3, 'Green', 'ui_green'), - Theme.new(4, 'Gray', 'ui_gray'), + Theme.new(4, 'Black', 'ui_black'), Theme.new(5, 'Violet', 'ui_violet'), Theme.new(6, 'Blue', 'ui_blue') ].freeze diff --git a/lib/gitlab/time_tracking_formatter.rb b/lib/gitlab/time_tracking_formatter.rb new file mode 100644 index 00000000000..d615c24149a --- /dev/null +++ b/lib/gitlab/time_tracking_formatter.rb @@ -0,0 +1,34 @@ +module Gitlab + module TimeTrackingFormatter + extend self + + def parse(string) + with_custom_config do + string.sub!(/\A-/, '') + + seconds = ChronicDuration.parse(string, default_unit: 'hours') rescue nil + seconds *= -1 if seconds && Regexp.last_match + seconds + end + end + + def output(seconds) + with_custom_config do + ChronicDuration.output(seconds, format: :short, limit_to_hours: false, weeks: true) rescue nil + end + end + + def with_custom_config + # We may want to configure it through project settings in a future version. + ChronicDuration.hours_per_day = 8 + ChronicDuration.days_per_week = 5 + + result = yield + + ChronicDuration.hours_per_day = 24 + ChronicDuration.days_per_week = 7 + + result + end + end +end diff --git a/lib/gitlab/update_path_error.rb b/lib/gitlab/update_path_error.rb new file mode 100644 index 00000000000..ce14cc887d0 --- /dev/null +++ b/lib/gitlab/update_path_error.rb @@ -0,0 +1,3 @@ +module Gitlab + class UpdatePathError < StandardError; end +end diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index 99d0c28e749..ccb456bcc94 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -24,6 +24,8 @@ module Gitlab wiki_page_url when ProjectSnippet project_snippet_url(object) + when Snippet + personal_snippet_url(object) else raise NotImplementedError.new("No URL builder defined for #{object.class}") end diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb index 9858d2e7d83..6ce9b229294 100644 --- a/lib/gitlab/user_access.rb +++ b/lib/gitlab/user_access.rb @@ -8,6 +8,8 @@ module Gitlab end def can_do_action?(action) + return false if no_user_or_blocked? + @permission_cache ||= {} @permission_cache[action] ||= user.can?(action, project) end @@ -17,7 +19,7 @@ module Gitlab end def allowed? - return false if user.blank? || user.blocked? + return false if no_user_or_blocked? if user.requires_ldap_check? && user.try_obtain_ldap_lease return false unless Gitlab::LDAP::Access.allowed?(user) @@ -27,20 +29,22 @@ module Gitlab end def can_push_to_branch?(ref) - return false unless user + return false if no_user_or_blocked? if project.protected_branch?(ref) return true if project.empty_repo? && project.user_can_push_to_empty_repo?(user) access_levels = project.protected_branches.matching(ref).map(&:push_access_levels).flatten - access_levels.any? { |access_level| access_level.check_access(user) } + has_access = access_levels.any? { |access_level| access_level.check_access(user) } + + has_access || !project.repository.branch_exists?(ref) && can_merge_to_branch?(ref) else user.can?(:push_code, project) end end def can_merge_to_branch?(ref) - return false unless user + return false if no_user_or_blocked? if project.protected_branch?(ref) access_levels = project.protected_branches.matching(ref).map(&:merge_access_levels).flatten @@ -51,9 +55,15 @@ module Gitlab end def can_read_project? - return false unless user + return false if no_user_or_blocked? user.can?(:read_project, project) end + + private + + def no_user_or_blocked? + user.nil? || user.blocked? + end end end diff --git a/lib/gitlab/view/presenter/base.rb b/lib/gitlab/view/presenter/base.rb new file mode 100644 index 00000000000..83c8ba5c1cf --- /dev/null +++ b/lib/gitlab/view/presenter/base.rb @@ -0,0 +1,28 @@ +module Gitlab + module View + module Presenter + module Base + extend ActiveSupport::Concern + + include Gitlab::Routing + include Gitlab::Allowable + + attr_reader :subject + + def can?(user, action, overriden_subject = nil) + super(user, action, overriden_subject || subject) + end + + class_methods do + def presenter? + true + end + + def presents(name) + define_method(name) { subject } + end + end + end + end + end +end diff --git a/lib/gitlab/view/presenter/delegated.rb b/lib/gitlab/view/presenter/delegated.rb new file mode 100644 index 00000000000..f4d330c590e --- /dev/null +++ b/lib/gitlab/view/presenter/delegated.rb @@ -0,0 +1,19 @@ +module Gitlab + module View + module Presenter + class Delegated < SimpleDelegator + include Gitlab::View::Presenter::Base + + def initialize(subject, **attributes) + @subject = subject + + attributes.each do |key, value| + define_singleton_method(key) { value } + end + + super(subject) + end + end + end + end +end diff --git a/lib/gitlab/view/presenter/factory.rb b/lib/gitlab/view/presenter/factory.rb new file mode 100644 index 00000000000..d172d61e2c9 --- /dev/null +++ b/lib/gitlab/view/presenter/factory.rb @@ -0,0 +1,24 @@ +module Gitlab + module View + module Presenter + class Factory + def initialize(subject, **attributes) + @subject = subject + @attributes = attributes + end + + def fabricate! + presenter_class.new(subject, attributes) + end + + private + + attr_reader :subject, :attributes + + def presenter_class + "#{subject.class.name}Presenter".constantize + end + end + end + end +end diff --git a/lib/gitlab/view/presenter/simple.rb b/lib/gitlab/view/presenter/simple.rb new file mode 100644 index 00000000000..b7653a0f3cc --- /dev/null +++ b/lib/gitlab/view/presenter/simple.rb @@ -0,0 +1,17 @@ +module Gitlab + module View + module Presenter + class Simple + include Gitlab::View::Presenter::Base + + def initialize(subject, **attributes) + @subject = subject + + attributes.each do |key, value| + define_singleton_method(key) { value } + end + end + end + end + end +end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 594439a5d4b..a3b502ffd6a 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -15,10 +15,17 @@ module Gitlab class << self def git_http_ok(repository, user) - { + params = { GL_ID: Gitlab::GlId.gl_id(user), RepoPath: repository.path_to_repo, } + + params.merge!( + GitalySocketPath: Gitlab.config.gitaly.socket_path, + GitalyResourcePath: "/projects/#{repository.project.id}/git-http/info-refs", + ) if Gitlab.config.gitaly.socket_path.present? + + params end def lfs_upload_ok(oid, size) @@ -95,6 +102,19 @@ module Gitlab ] end + def terminal_websocket(terminal) + details = { + 'Terminal' => { + 'Subprotocols' => terminal[:subprotocols], + 'Url' => terminal[:url], + 'Header' => terminal[:headers] + } + } + details['Terminal']['CAPem'] = terminal[:ca_pem] if terminal.has_key?(:ca_pem) + + details + end + def version path = Rails.root.join(VERSION_FILE) path.readable? ? path.read.chomp : 'unknown' @@ -117,8 +137,12 @@ module Gitlab end def verify_api_request!(request_headers) + decode_jwt(request_headers[INTERNAL_API_REQUEST_HEADER]) + end + + def decode_jwt(encoded_message) JWT.decode( - request_headers[INTERNAL_API_REQUEST_HEADER], + encoded_message, secret, true, { iss: 'gitlab-workhorse', verify_iss: true, algorithm: 'HS256' }, diff --git a/lib/mattermost/client.rb b/lib/mattermost/client.rb new file mode 100644 index 00000000000..ec2903b7ec6 --- /dev/null +++ b/lib/mattermost/client.rb @@ -0,0 +1,41 @@ +module Mattermost + class ClientError < Mattermost::Error; end + + class Client + attr_reader :user + + def initialize(user) + @user = user + end + + private + + def with_session(&blk) + Mattermost::Session.new(user).with_session(&blk) + end + + def json_get(path, options = {}) + with_session do |session| + json_response session.get(path, options) + end + end + + def json_post(path, options = {}) + with_session do |session| + json_response session.post(path, options) + end + end + + def json_response(response) + json_response = JSON.parse(response.body) + + unless response.success? + raise Mattermost::ClientError.new(json_response['message'] || 'Undefined error') + end + + json_response + rescue JSON::JSONError + raise Mattermost::ClientError.new('Cannot parse response') + end + end +end diff --git a/lib/mattermost/command.rb b/lib/mattermost/command.rb new file mode 100644 index 00000000000..d1e4bb0eccf --- /dev/null +++ b/lib/mattermost/command.rb @@ -0,0 +1,10 @@ +module Mattermost + class Command < Client + def create(params) + response = json_post("/api/v3/teams/#{params[:team_id]}/commands/create", + body: params.to_json) + + response['token'] + end + end +end diff --git a/lib/mattermost/error.rb b/lib/mattermost/error.rb new file mode 100644 index 00000000000..014df175be0 --- /dev/null +++ b/lib/mattermost/error.rb @@ -0,0 +1,3 @@ +module Mattermost + class Error < StandardError; end +end diff --git a/lib/mattermost/session.rb b/lib/mattermost/session.rb new file mode 100644 index 00000000000..377cb7b1021 --- /dev/null +++ b/lib/mattermost/session.rb @@ -0,0 +1,160 @@ +module Mattermost + class NoSessionError < Mattermost::Error + def message + 'No session could be set up, is Mattermost configured with Single Sign On?' + end + end + + class ConnectionError < Mattermost::Error; end + + # This class' prime objective is to obtain a session token on a Mattermost + # instance with SSO configured where this GitLab instance is the provider. + # + # The process depends on OAuth, but skips a step in the authentication cycle. + # For example, usually a user would click the 'login in GitLab' button on + # Mattermost, which would yield a 302 status code and redirects you to GitLab + # to approve the use of your account on Mattermost. Which would trigger a + # callback so Mattermost knows this request is approved and gets the required + # data to create the user account etc. + # + # This class however skips the button click, and also the approval phase to + # speed up the process and keep it without manual action and get a session + # going. + class Session + include Doorkeeper::Helpers::Controller + include HTTParty + + LEASE_TIMEOUT = 60 + + base_uri Settings.mattermost.host + + attr_accessor :current_resource_owner, :token + + def initialize(current_user) + @current_resource_owner = current_user + end + + def with_session + with_lease do + raise Mattermost::NoSessionError unless create + + begin + yield self + rescue Errno::ECONNREFUSED + raise Mattermost::NoSessionError + ensure + destroy + end + end + end + + # Next methods are needed for Doorkeeper + def pre_auth + @pre_auth ||= Doorkeeper::OAuth::PreAuthorization.new( + Doorkeeper.configuration, server.client_via_uid, params) + end + + def authorization + @authorization ||= strategy.request + end + + def strategy + @strategy ||= server.authorization_request(pre_auth.response_type) + end + + def request + @request ||= OpenStruct.new(parameters: params) + end + + def params + Rack::Utils.parse_query(oauth_uri.query).symbolize_keys + end + + def get(path, options = {}) + handle_exceptions do + self.class.get(path, options.merge(headers: @headers)) + end + end + + def post(path, options = {}) + handle_exceptions do + self.class.post(path, options.merge(headers: @headers)) + end + end + + private + + def create + return unless oauth_uri + return unless token_uri + + @token = request_token + @headers = { + Authorization: "Bearer #{@token}" + } + + @token + end + + def destroy + post('/api/v3/users/logout') + end + + def oauth_uri + return @oauth_uri if defined?(@oauth_uri) + + @oauth_uri = nil + + response = get("/api/v3/oauth/gitlab/login", follow_redirects: false) + return unless 300 <= response.code && response.code < 400 + + redirect_uri = response.headers['location'] + return unless redirect_uri + + @oauth_uri = URI.parse(redirect_uri) + end + + def token_uri + @token_uri ||= + if oauth_uri + authorization.authorize.redirect_uri if pre_auth.authorizable? + end + end + + def request_token + response = get(token_uri, follow_redirects: false) + + if 200 <= response.code && response.code < 400 + response.headers['token'] + end + end + + def with_lease + lease_uuid = lease_try_obtain + raise NoSessionError unless lease_uuid + + begin + yield + ensure + Gitlab::ExclusiveLease.cancel(lease_key, lease_uuid) + end + end + + def lease_key + "mattermost:session" + end + + def lease_try_obtain + lease = ::Gitlab::ExclusiveLease.new(lease_key, timeout: LEASE_TIMEOUT) + lease.try_obtain + end + + def handle_exceptions + yield + rescue HTTParty::Error => e + raise Mattermost::ConnectionError.new(e.message) + rescue Errno::ECONNREFUSED + raise Mattermost::ConnectionError.new(e.message) + end + end +end diff --git a/lib/mattermost/team.rb b/lib/mattermost/team.rb new file mode 100644 index 00000000000..784eca6ab5a --- /dev/null +++ b/lib/mattermost/team.rb @@ -0,0 +1,7 @@ +module Mattermost + class Team < Client + def all + json_get('/api/v3/teams/all') + end + end +end diff --git a/lib/omniauth/strategies/bitbucket.rb b/lib/omniauth/strategies/bitbucket.rb new file mode 100644 index 00000000000..5a7d67c2390 --- /dev/null +++ b/lib/omniauth/strategies/bitbucket.rb @@ -0,0 +1,41 @@ +require 'omniauth-oauth2' + +module OmniAuth + module Strategies + class Bitbucket < OmniAuth::Strategies::OAuth2 + option :name, 'bitbucket' + + option :client_options, { + site: 'https://bitbucket.org', + authorize_url: 'https://bitbucket.org/site/oauth2/authorize', + token_url: 'https://bitbucket.org/site/oauth2/access_token' + } + + uid do + raw_info['username'] + end + + info do + { + name: raw_info['display_name'], + avatar: raw_info['links']['avatar']['href'], + email: primary_email + } + end + + def raw_info + @raw_info ||= access_token.get('api/2.0/user').parsed + end + + def primary_email + primary = emails.find { |i| i['is_primary'] && i['is_confirmed'] } + primary && primary['email'] || nil + end + + def emails + email_response = access_token.get('api/2.0/user/emails').parsed + @emails ||= email_response && email_response['values'] || nil + end + end + end +end diff --git a/lib/rouge/lexers/math.rb b/lib/rouge/lexers/math.rb new file mode 100644 index 00000000000..80784adfd76 --- /dev/null +++ b/lib/rouge/lexers/math.rb @@ -0,0 +1,21 @@ +module Rouge + module Lexers + class Math < Lexer + title "A passthrough lexer used for LaTeX input" + desc "A boring lexer that doesn't highlight anything" + + tag 'math' + mimetypes 'text/plain' + + default_options token: 'Text' + + def token + @token ||= Token[option :token] + end + + def stream_tokens(string, &b) + yield self.token, string + end + end + end +end diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab index d521de28e8a..2f7c34a3f31 100644 --- a/lib/support/nginx/gitlab +++ b/lib/support/nginx/gitlab @@ -20,6 +20,11 @@ upstream gitlab-workhorse { server unix:/home/git/gitlab/tmp/sockets/gitlab-workhorse.socket fail_timeout=0; } +map $http_upgrade $connection_upgrade_gitlab { + default upgrade; + '' close; +} + ## Normal HTTP host server { ## Either remove "default_server" from the listen line below, @@ -53,6 +58,8 @@ server { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade_gitlab; proxy_pass http://gitlab-workhorse; } diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl index bf014b56cf6..5661394058d 100644 --- a/lib/support/nginx/gitlab-ssl +++ b/lib/support/nginx/gitlab-ssl @@ -24,6 +24,11 @@ upstream gitlab-workhorse { server unix:/home/git/gitlab/tmp/sockets/gitlab-workhorse.socket fail_timeout=0; } +map $http_upgrade $connection_upgrade_gitlab_ssl { + default upgrade; + '' close; +} + ## Redirects all HTTP traffic to the HTTPS host server { ## Either remove "default_server" from the listen line below, @@ -98,6 +103,9 @@ server { proxy_set_header X-Forwarded-Ssl on; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade_gitlab_ssl; + proxy_pass http://gitlab-workhorse; } diff --git a/lib/tasks/dev.rake b/lib/tasks/dev.rake index 6f27972c4e4..5e94fba97bf 100644 --- a/lib/tasks/dev.rake +++ b/lib/tasks/dev.rake @@ -7,9 +7,4 @@ namespace :dev do Rake::Task["gitlab:setup"].invoke Rake::Task["gitlab:shell:setup"].invoke end - - desc 'GitLab | Start/restart foreman and watch for changes' - task :foreman => :environment do - sh 'rerun --dir app,config,lib -- foreman start' - end end diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 2ae48a970ce..35c4194e87c 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -760,7 +760,7 @@ namespace :gitlab do end namespace :ldap do - task :check, [:limit] => :environment do |t, args| + task :check, [:limit] => :environment do |_, args| # Only show up to 100 results because LDAP directories can be very big. # This setting only affects the `rake gitlab:check` script. args.with_defaults(limit: 100) @@ -768,7 +768,7 @@ namespace :gitlab do start_checking "LDAP" if Gitlab::LDAP::Config.enabled? - print_users(args.limit) + check_ldap(args.limit) else puts 'LDAP is disabled in config/gitlab.yml' end @@ -776,21 +776,42 @@ namespace :gitlab do finished_checking "LDAP" end - def print_users(limit) - puts "LDAP users with access to your GitLab server (only showing the first #{limit} results)" - + def check_ldap(limit) servers = Gitlab::LDAP::Config.providers servers.each do |server| puts "Server: #{server}" - Gitlab::LDAP::Adapter.open(server) do |adapter| - users = adapter.users(adapter.config.uid, '*', limit) - users.each do |user| - puts "\tDN: #{user.dn}\t #{adapter.config.uid}: #{user.uid}" + + begin + Gitlab::LDAP::Adapter.open(server) do |adapter| + check_ldap_auth(adapter) + + puts "LDAP users with access to your GitLab server (only showing the first #{limit} results)" + + users = adapter.users(adapter.config.uid, '*', limit) + users.each do |user| + puts "\tDN: #{user.dn}\t #{adapter.config.uid}: #{user.uid}" + end end + rescue Net::LDAP::ConnectionRefusedError, Errno::ECONNREFUSED => e + puts "Could not connect to the LDAP server: #{e.message}".color(:red) end end end + + def check_ldap_auth(adapter) + auth = adapter.config.has_auth? + + if auth && adapter.ldap.bind + message = 'Success'.color(:green) + elsif auth + message = 'Failed. Check `bind_dn` and `password` configuration values'.color(:red) + else + message = 'Anonymous. No `bind_dn` or `password` configured'.color(:yellow) + end + + puts "LDAP authentication... #{message}" + end end namespace :repo do diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index b7cbdc6cd78..4a696a52b4d 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -91,5 +91,28 @@ namespace :gitlab do puts "To block these users run this command with BLOCK=true".color(:yellow) end end + + # This is a rake task which removes faulty refs. These refs where only + # created in the 8.13.RC cycle, and fixed in the stable builds which were + # released. So likely this should only be run once on gitlab.com + # Faulty refs are moved so they are kept around, else some features break. + desc 'GitLab | Cleanup | Remove faulty deployment refs' + task move_faulty_deployment_refs: :environment do + projects = Project.where(id: Deployment.select(:project_id).distinct) + + projects.find_each do |project| + rugged = project.repository.rugged + + max_iid = project.deployments.maximum(:iid) + + rugged.references.each('refs/environments/**/*') do |ref| + id = ref.name.split('/').last.to_i + next unless id > max_iid + + project.deployments.find(id).create_ref + rugged.references.delete(ref) + end + end + end end end diff --git a/lib/tasks/gitlab/dev.rake b/lib/tasks/gitlab/dev.rake index 5ee99dfc810..7db0779def8 100644 --- a/lib/tasks/gitlab/dev.rake +++ b/lib/tasks/gitlab/dev.rake @@ -1,18 +1,19 @@ namespace :gitlab do namespace :dev do desc 'Checks if the branch would apply cleanly to EE' - task ee_compat_check: :environment do - return if defined?(Gitlab::License) - return unless ENV['CI'] + task :ee_compat_check, [:branch] => :environment do |_, args| + opts = + if ENV['CI'] + { branch: ENV['CI_BUILD_REF_NAME'] } + else + unless args[:branch] + puts "Must specify a branch as an argument".color(:red) + exit 1 + end + args + end - success = - Gitlab::EeCompatCheck.new( - branch: ENV['CI_BUILD_REF_NAME'], - check_dir: File.expand_path('ee-compat-check', __dir__), - ce_repo: ENV['CI_BUILD_REPO'] - ).check - - if success + if Gitlab::EeCompatCheck.new(opts || {}).check exit 0 else exit 1 diff --git a/lib/tasks/gitlab/generate_docs.rake b/lib/tasks/gitlab/generate_docs.rake deleted file mode 100644 index f6448c38e10..00000000000 --- a/lib/tasks/gitlab/generate_docs.rake +++ /dev/null @@ -1,7 +0,0 @@ -namespace :gitlab do - desc "GitLab | Generate sdocs for project" - task generate_docs: :environment do - system(*%W(bundle exec sdoc -o doc/code app lib)) - end -end - diff --git a/lib/tasks/gitlab/git.rake b/lib/tasks/gitlab/git.rake index f9834a4dae8..a67c1fe1f27 100644 --- a/lib/tasks/gitlab/git.rake +++ b/lib/tasks/gitlab/git.rake @@ -3,7 +3,7 @@ namespace :gitlab do desc "GitLab | Git | Repack" task repack: :environment do - failures = perform_git_cmd(%W(git repack -a --quiet), "Repacking repo") + failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} repack -a --quiet), "Repacking repo") if failures.empty? puts "Done".color(:green) else @@ -13,17 +13,17 @@ namespace :gitlab do desc "GitLab | Git | Run garbage collection on all repos" task gc: :environment do - failures = perform_git_cmd(%W(git gc --auto --quiet), "Garbage Collecting") + failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} gc --auto --quiet), "Garbage Collecting") if failures.empty? puts "Done".color(:green) else output_failures(failures) end end - + desc "GitLab | Git | Prune all repos" task prune: :environment do - failures = perform_git_cmd(%W(git prune), "Git Prune") + failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} prune), "Git Prune") if failures.empty? puts "Done".color(:green) else diff --git a/lib/tasks/gitlab/helpers.rake b/lib/tasks/gitlab/helpers.rake new file mode 100644 index 00000000000..dd2d5861481 --- /dev/null +++ b/lib/tasks/gitlab/helpers.rake @@ -0,0 +1,8 @@ +require 'tasks/gitlab/task_helpers' + +# Prevent StateMachine warnings from outputting during a cron task +StateMachines::Machine.ignore_method_conflicts = true if ENV['CRON'] + +namespace :gitlab do + include Gitlab::TaskHelpers +end diff --git a/lib/tasks/gitlab/import.rake b/lib/tasks/gitlab/import.rake index dbdd4e977e8..a2eca74a3c8 100644 --- a/lib/tasks/gitlab/import.rake +++ b/lib/tasks/gitlab/import.rake @@ -63,8 +63,7 @@ namespace :gitlab do if project.persisted? puts " * Created #{project.name} (#{repo_path})".color(:green) - project.update_repository_size - project.update_commit_count + ProjectCacheWorker.perform(project.id) else puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red) puts " Errors: #{project.errors.messages}".color(:red) diff --git a/lib/tasks/gitlab/ldap.rake b/lib/tasks/gitlab/ldap.rake new file mode 100644 index 00000000000..c66a2a263dc --- /dev/null +++ b/lib/tasks/gitlab/ldap.rake @@ -0,0 +1,40 @@ +namespace :gitlab do + namespace :ldap do + desc 'GitLab | LDAP | Rename provider' + task :rename_provider, [:old_provider, :new_provider] => :environment do |_, args| + old_provider = args[:old_provider] || + prompt('What is the old provider? Ex. \'ldapmain\': '.color(:blue)) + new_provider = args[:new_provider] || + prompt('What is the new provider ID? Ex. \'ldapcustom\': '.color(:blue)) + puts '' # Add some separation in the output + + identities = Identity.where(provider: old_provider) + identity_count = identities.count + + if identities.empty? + puts "Found no user identities with '#{old_provider}' provider." + puts 'Please check the provider name and try again.' + exit 1 + end + + plural_id_count = ActionController::Base.helpers.pluralize(identity_count, 'user') + + unless ENV['force'] == 'yes' + puts "#{plural_id_count} with provider '#{old_provider}' will be updated to '#{new_provider}'" + puts 'If the new provider is incorrect, users will be unable to sign in' + ask_to_continue + puts '' + end + + updated_count = identities.update_all(provider: new_provider) + + if updated_count == identity_count + puts 'User identities were successfully updated'.color(:green) + else + plural_updated_count = ActionController::Base.helpers.pluralize(updated_count, 'user') + puts 'Some user identities could not be updated'.color(:red) + puts "Successfully updated #{plural_updated_count} out of #{plural_id_count} total" + end + end + end +end diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index 58761a129d4..5a09cd7ce41 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -5,42 +5,23 @@ namespace :gitlab do warn_user_is_not_gitlab default_version = Gitlab::Shell.version_required - default_version_tag = 'v' + default_version - args.with_defaults(tag: default_version_tag, repo: "https://gitlab.com/gitlab-org/gitlab-shell.git") + default_version_tag = "v#{default_version}" + args.with_defaults(tag: default_version_tag, repo: 'https://gitlab.com/gitlab-org/gitlab-shell.git') - user = Gitlab.config.gitlab.user - home_dir = Rails.env.test? ? Rails.root.join('tmp/tests') : Gitlab.config.gitlab.user_home gitlab_url = Gitlab.config.gitlab.url # gitlab-shell requires a / at the end of the url gitlab_url += '/' unless gitlab_url.end_with?('/') target_dir = Gitlab.config.gitlab_shell.path - # Clone if needed - if File.directory?(target_dir) - Dir.chdir(target_dir) do - system(*%W(Gitlab.config.git.bin_path} fetch --tags --quiet)) - system(*%W(Gitlab.config.git.bin_path} checkout --quiet #{default_version_tag})) - end - else - system(*%W(#{Gitlab.config.git.bin_path} clone -- #{args.repo} #{target_dir})) - end + checkout_or_clone_tag(tag: default_version_tag, repo: args.repo, target_dir: target_dir) # Make sure we're on the right tag Dir.chdir(target_dir) do - # First try to checkout without fetching - # to avoid stalling tests if the Internet is down. - reseted = reset_to_commit(args) - - unless reseted - system(*%W(#{Gitlab.config.git.bin_path} fetch origin)) - reset_to_commit(args) - end - config = { - user: user, + user: Gitlab.config.gitlab.user, gitlab_url: gitlab_url, http_settings: {self_signed_cert: false}.stringify_keys, - auth_file: File.join(home_dir, ".ssh", "authorized_keys"), + auth_file: File.join(user_home, ".ssh", "authorized_keys"), redis: { bin: %x{which redis-cli}.chomp, namespace: "resque:gitlab" @@ -74,7 +55,7 @@ namespace :gitlab do # be an issue since it is more than likely that there are no "normal" # user accounts on a gitlab server). The alternative is for the admin to # install a ruby (1.9.3+) in the global path. - File.open(File.join(home_dir, ".ssh", "environment"), "w+") do |f| + File.open(File.join(user_home, ".ssh", "environment"), "w+") do |f| f.puts "PATH=#{ENV['PATH']}" end @@ -142,15 +123,4 @@ namespace :gitlab do puts "Quitting...".color(:red) exit 1 end - - def reset_to_commit(args) - tag, status = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} describe -- #{args.tag})) - - unless status.zero? - tag, status = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} describe -- origin/#{args.tag})) - end - - tag = tag.strip - system(*%W(#{Gitlab.config.git.bin_path} reset --hard #{tag})) - end end diff --git a/lib/tasks/gitlab/task_helpers.rake b/lib/tasks/gitlab/task_helpers.rake deleted file mode 100644 index 74be413423a..00000000000 --- a/lib/tasks/gitlab/task_helpers.rake +++ /dev/null @@ -1,140 +0,0 @@ -module Gitlab - class TaskAbortedByUserError < StandardError; end -end - -require 'rainbow/ext/string' - -# Prevent StateMachine warnings from outputting during a cron task -StateMachines::Machine.ignore_method_conflicts = true if ENV['CRON'] - -namespace :gitlab do - - # Ask if the user wants to continue - # - # Returns "yes" the user chose to continue - # Raises Gitlab::TaskAbortedByUserError if the user chose *not* to continue - def ask_to_continue - answer = prompt("Do you want to continue (yes/no)? ".color(:blue), %w{yes no}) - raise Gitlab::TaskAbortedByUserError unless answer == "yes" - end - - # Check which OS is running - # - # It will primarily use lsb_relase to determine the OS. - # It has fallbacks to Debian, SuSE, OS X and systems running systemd. - def os_name - os_name = run_command(%W(lsb_release -irs)) - os_name ||= if File.readable?('/etc/system-release') - File.read('/etc/system-release') - end - os_name ||= if File.readable?('/etc/debian_version') - debian_version = File.read('/etc/debian_version') - "Debian #{debian_version}" - end - os_name ||= if File.readable?('/etc/SuSE-release') - File.read('/etc/SuSE-release') - end - os_name ||= if os_x_version = run_command(%W(sw_vers -productVersion)) - "Mac OS X #{os_x_version}" - end - os_name ||= if File.readable?('/etc/os-release') - File.read('/etc/os-release').match(/PRETTY_NAME=\"(.+)\"/)[1] - end - os_name.try(:squish!) - end - - # Prompt the user to input something - # - # message - the message to display before input - # choices - array of strings of acceptable answers or nil for any answer - # - # Returns the user's answer - def prompt(message, choices = nil) - begin - print(message) - answer = STDIN.gets.chomp - end while choices.present? && !choices.include?(answer) - answer - end - - # Runs the given command and matches the output against the given pattern - # - # Returns nil if nothing matched - # Returns the MatchData if the pattern matched - # - # see also #run_command - # see also String#match - def run_and_match(command, regexp) - run_command(command).try(:match, regexp) - end - - # Runs the given command - # - # Returns nil if the command was not found - # Returns the output of the command otherwise - # - # see also #run_and_match - def run_command(command) - output, _ = Gitlab::Popen.popen(command) - output - rescue Errno::ENOENT - '' # if the command does not exist, return an empty string - end - - def uid_for(user_name) - run_command(%W(id -u #{user_name})).chomp.to_i - end - - def gid_for(group_name) - begin - Etc.getgrnam(group_name).gid - rescue ArgumentError # no group - "group #{group_name} doesn't exist" - end - end - - def warn_user_is_not_gitlab - unless @warned_user_not_gitlab - gitlab_user = Gitlab.config.gitlab.user - current_user = run_command(%W(whoami)).chomp - unless current_user == gitlab_user - puts " Warning ".color(:black).background(:yellow) - puts " You are running as user #{current_user.color(:magenta)}, we hope you know what you are doing." - puts " Things may work\/fail for the wrong reasons." - puts " For correct results you should run this as user #{gitlab_user.color(:magenta)}." - puts "" - end - @warned_user_not_gitlab = true - end - end - - # Tries to configure git itself - # - # Returns true if all subcommands were successfull (according to their exit code) - # Returns false if any or all subcommands failed. - def auto_fix_git_config(options) - if !@warned_user_not_gitlab - command_success = options.map do |name, value| - system(*%W(#{Gitlab.config.git.bin_path} config --global #{name} #{value})) - end - - command_success.all? - else - false - end - end - - def all_repos - Gitlab.config.repositories.storages.each do |name, path| - IO.popen(%W(find #{path} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find| - find.each_line do |path| - yield path.chomp - end - end - end - end - - def repository_storage_paths_args - Gitlab.config.repositories.storages.values - end -end diff --git a/lib/tasks/gitlab/task_helpers.rb b/lib/tasks/gitlab/task_helpers.rb new file mode 100644 index 00000000000..e128738b5f8 --- /dev/null +++ b/lib/tasks/gitlab/task_helpers.rb @@ -0,0 +1,190 @@ +require 'rainbow/ext/string' + +module Gitlab + TaskFailedError = Class.new(StandardError) + TaskAbortedByUserError = Class.new(StandardError) + + module TaskHelpers + # Ask if the user wants to continue + # + # Returns "yes" the user chose to continue + # Raises Gitlab::TaskAbortedByUserError if the user chose *not* to continue + def ask_to_continue + answer = prompt("Do you want to continue (yes/no)? ".color(:blue), %w{yes no}) + raise Gitlab::TaskAbortedByUserError unless answer == "yes" + end + + # Check which OS is running + # + # It will primarily use lsb_relase to determine the OS. + # It has fallbacks to Debian, SuSE, OS X and systems running systemd. + def os_name + os_name = run_command(%W(lsb_release -irs)) + os_name ||= if File.readable?('/etc/system-release') + File.read('/etc/system-release') + end + os_name ||= if File.readable?('/etc/debian_version') + debian_version = File.read('/etc/debian_version') + "Debian #{debian_version}" + end + os_name ||= if File.readable?('/etc/SuSE-release') + File.read('/etc/SuSE-release') + end + os_name ||= if os_x_version = run_command(%W(sw_vers -productVersion)) + "Mac OS X #{os_x_version}" + end + os_name ||= if File.readable?('/etc/os-release') + File.read('/etc/os-release').match(/PRETTY_NAME=\"(.+)\"/)[1] + end + os_name.try(:squish!) + end + + # Prompt the user to input something + # + # message - the message to display before input + # choices - array of strings of acceptable answers or nil for any answer + # + # Returns the user's answer + def prompt(message, choices = nil) + begin + print(message) + answer = STDIN.gets.chomp + end while choices.present? && !choices.include?(answer) + answer + end + + # Runs the given command and matches the output against the given pattern + # + # Returns nil if nothing matched + # Returns the MatchData if the pattern matched + # + # see also #run_command + # see also String#match + def run_and_match(command, regexp) + run_command(command).try(:match, regexp) + end + + # Runs the given command + # + # Returns '' if the command was not found + # Returns the output of the command otherwise + # + # see also #run_and_match + def run_command(command) + output, _ = Gitlab::Popen.popen(command) + output + rescue Errno::ENOENT + '' # if the command does not exist, return an empty string + end + + # Runs the given command and raises a Gitlab::TaskFailedError exception if + # the command does not exit with 0 + # + # Returns the output of the command otherwise + def run_command!(command) + output, status = Gitlab::Popen.popen(command) + + raise Gitlab::TaskFailedError unless status.zero? + + output + end + + def uid_for(user_name) + run_command(%W(id -u #{user_name})).chomp.to_i + end + + def gid_for(group_name) + begin + Etc.getgrnam(group_name).gid + rescue ArgumentError # no group + "group #{group_name} doesn't exist" + end + end + + def warn_user_is_not_gitlab + unless @warned_user_not_gitlab + gitlab_user = Gitlab.config.gitlab.user + current_user = run_command(%W(whoami)).chomp + unless current_user == gitlab_user + puts " Warning ".color(:black).background(:yellow) + puts " You are running as user #{current_user.color(:magenta)}, we hope you know what you are doing." + puts " Things may work\/fail for the wrong reasons." + puts " For correct results you should run this as user #{gitlab_user.color(:magenta)}." + puts "" + end + @warned_user_not_gitlab = true + end + end + + # Tries to configure git itself + # + # Returns true if all subcommands were successfull (according to their exit code) + # Returns false if any or all subcommands failed. + def auto_fix_git_config(options) + if !@warned_user_not_gitlab + command_success = options.map do |name, value| + system(*%W(#{Gitlab.config.git.bin_path} config --global #{name} #{value})) + end + + command_success.all? + else + false + end + end + + def all_repos + Gitlab.config.repositories.storages.each do |name, path| + IO.popen(%W(find #{path} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find| + find.each_line do |path| + yield path.chomp + end + end + end + end + + def repository_storage_paths_args + Gitlab.config.repositories.storages.values + end + + def user_home + Rails.env.test? ? Rails.root.join('tmp/tests') : Gitlab.config.gitlab.user_home + end + + def checkout_or_clone_tag(tag:, repo:, target_dir:) + if Dir.exist?(target_dir) + checkout_tag(tag, target_dir) + else + clone_repo(repo, target_dir) + end + + reset_to_tag(tag, target_dir) + end + + def clone_repo(repo, target_dir) + run_command!(%W[#{Gitlab.config.git.bin_path} clone -- #{repo} #{target_dir}]) + end + + def checkout_tag(tag, target_dir) + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} fetch --tags --quiet]) + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} checkout --quiet #{tag}]) + end + + def reset_to_tag(tag_wanted, target_dir) + tag = + begin + # First try to checkout without fetching + # to avoid stalling tests if the Internet is down. + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} describe -- #{tag_wanted}]) + rescue Gitlab::TaskFailedError + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} fetch origin]) + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} describe -- origin/#{tag_wanted}]) + end + + if tag + run_command!(%W[#{Gitlab.config.git.bin_path} -C #{target_dir} reset --hard #{tag.strip}]) + else + raise Gitlab::TaskFailedError + end + end + end +end diff --git a/lib/tasks/gitlab/update_commit_count.rake b/lib/tasks/gitlab/update_commit_count.rake deleted file mode 100644 index 3bd10b0208b..00000000000 --- a/lib/tasks/gitlab/update_commit_count.rake +++ /dev/null @@ -1,20 +0,0 @@ -namespace :gitlab do - desc "GitLab | Update commit count for projects" - task update_commit_count: :environment do - projects = Project.where(commit_count: 0) - puts "#{projects.size} projects need to be updated. This might take a while." - ask_to_continue unless ENV['force'] == 'yes' - - projects.find_each(batch_size: 100) do |project| - print "#{project.name_with_namespace.color(:yellow)} ... " - - unless project.repo_exists? - puts "skipping, because the repo is empty".color(:magenta) - next - end - - project.update_commit_count - puts project.commit_count.to_s.color(:green) - end - end -end diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake index 4f76dad7286..b77a5bb62d1 100644 --- a/lib/tasks/gitlab/update_templates.rake +++ b/lib/tasks/gitlab/update_templates.rake @@ -44,7 +44,7 @@ namespace :gitlab do ), Template.new( "https://gitlab.com/gitlab-org/gitlab-ci-yml.git", - /(\.{1,2}|LICENSE|Pages|\.gitlab-ci.yml)\z/ + /(\.{1,2}|LICENSE|Pages|autodeploy|\.gitlab-ci.yml)\z/ ) ] diff --git a/lib/tasks/gitlab/workhorse.rake b/lib/tasks/gitlab/workhorse.rake new file mode 100644 index 00000000000..baea94bf8ca --- /dev/null +++ b/lib/tasks/gitlab/workhorse.rake @@ -0,0 +1,23 @@ +namespace :gitlab do + namespace :workhorse do + desc "GitLab | Install or upgrade gitlab-workhorse" + task :install, [:dir] => :environment do |t, args| + warn_user_is_not_gitlab + unless args.dir.present? + abort %(Please specify the directory where you want to install gitlab-workhorse:\n rake "gitlab:workhorse:install[/home/git/gitlab-workhorse]") + end + + tag = "v#{Gitlab::Workhorse.version}" + repo = 'https://gitlab.com/gitlab-org/gitlab-workhorse.git' + + checkout_or_clone_tag(tag: tag, repo: repo, target_dir: args.dir) + + _, status = Gitlab::Popen.popen(%w[which gmake]) + command = status.zero? ? 'gmake' : 'make' + + Dir.chdir(args.dir) do + run_command!([command]) + end + end + end +end diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake index 141a0b74ec0..f5caca3ddbf 100644 --- a/lib/tasks/migrate/setup_postgresql.rake +++ b/lib/tasks/migrate/setup_postgresql.rake @@ -1,8 +1,12 @@ +require Rails.root.join('lib/gitlab/database') +require Rails.root.join('lib/gitlab/database/migration_helpers') require Rails.root.join('db/migrate/20151007120511_namespaces_projects_path_lower_indexes') require Rails.root.join('db/migrate/20151008110232_add_users_lower_username_email_indexes') +require Rails.root.join('db/migrate/20161212142807_add_lower_path_index_to_routes') desc 'GitLab | Sets up PostgreSQL' task setup_postgresql: :environment do NamespacesProjectsPathLowerIndexes.new.up AddUsersLowerUsernameEmailIndexes.new.up + AddLowerPathIndexToRoutes.new.up end |