diff options
Diffstat (limited to 'lib')
80 files changed, 2473 insertions, 536 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index 3e53ab693ab..1bf20f76ad6 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -5,10 +5,13 @@ module API version %w(v3 v4), using: :path version 'v3', using: :path do + helpers ::API::V3::Helpers + mount ::API::V3::AwardEmoji mount ::API::V3::Boards mount ::API::V3::Branches mount ::API::V3::BroadcastMessages + mount ::API::V3::Builds mount ::API::V3::Commits mount ::API::V3::DeployKeys mount ::API::V3::Environments @@ -20,6 +23,7 @@ module API mount ::API::V3::MergeRequestDiffs mount ::API::V3::MergeRequests mount ::API::V3::Notes + mount ::API::V3::Pipelines mount ::API::V3::ProjectHooks mount ::API::V3::Milestones mount ::API::V3::Projects @@ -59,6 +63,10 @@ module API error! e.message, e.status, e.headers end + rescue_from Gitlab::Auth::TooManyIps do |e| + rack_response({ 'message' => '403 Forbidden' }.to_json, 403) + end + rescue_from :all do |exception| handle_api_exception(exception) end @@ -76,7 +84,6 @@ module API mount ::API::Boards mount ::API::Branches mount ::API::BroadcastMessages - mount ::API::Builds mount ::API::Commits mount ::API::CommitStatuses mount ::API::DeployKeys @@ -86,6 +93,7 @@ module API mount ::API::Groups mount ::API::Internal mount ::API::Issues + mount ::API::Jobs mount ::API::Keys mount ::API::Labels mount ::API::Lint diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb index 07a1bcdbe18..f9e0c2c4e16 100644 --- a/lib/api/award_emoji.rb +++ b/lib/api/award_emoji.rb @@ -3,12 +3,16 @@ module API include PaginationParams before { authenticate! } - AWARDABLES = %w[issue merge_request snippet].freeze + AWARDABLES = [ + { type: 'issue', find_by: :iid }, + { type: 'merge_request', find_by: :iid }, + { type: 'snippet', find_by: :id } + ].freeze resource :projects do - AWARDABLES.each do |awardable_type| - awardable_string = awardable_type.pluralize - awardable_id_string = "#{awardable_type}_id" + AWARDABLES.each do |awardable_params| + awardable_string = awardable_params[:type].pluralize + awardable_id_string = "#{awardable_params[:type]}_#{awardable_params[:find_by]}" params do requires :id, type: String, desc: 'The ID of a project' @@ -104,10 +108,10 @@ module API note_id = params.delete(:note_id) awardable.notes.find(note_id) - elsif params.include?(:issue_id) - user_project.issues.find(params[:issue_id]) - elsif params.include?(:merge_request_id) - user_project.merge_requests.find(params[:merge_request_id]) + elsif params.include?(:issue_iid) + user_project.issues.find_by!(iid: params[:issue_iid]) + elsif params.include?(:merge_request_iid) + user_project.merge_requests.find_by!(iid: params[:merge_request_iid]) else user_project.snippets.find(params[:snippet_id]) end diff --git a/lib/api/commits.rb b/lib/api/commits.rb index fd03e92264d..42401abfe0f 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -18,22 +18,34 @@ module API optional :ref_name, type: String, desc: 'The name of a repository branch or tag, if not given the default branch is used' optional :since, type: DateTime, desc: 'Only commits after or on this date will be returned' optional :until, type: DateTime, desc: 'Only commits before or on this date will be returned' - optional :page, type: Integer, default: 0, desc: 'The page for pagination' - optional :per_page, type: Integer, default: 20, desc: 'The number of results per page' optional :path, type: String, desc: 'The file path' + use :pagination end get ":id/repository/commits" do - ref = params[:ref_name] || user_project.try(:default_branch) || 'master' - offset = params[:page] * params[:per_page] + path = params[:path] + before = params[:until] + after = params[:since] + ref = params[:ref_name] || user_project.try(:default_branch) || 'master' + offset = (params[:page] - 1) * params[:per_page] commits = user_project.repository.commits(ref, - path: params[:path], + path: path, limit: params[:per_page], offset: offset, - after: params[:since], - before: params[:until]) + before: before, + after: after) + + commit_count = + if path || before || after + user_project.repository.count_commits(ref: ref, path: path, before: before, after: after) + else + # Cacheable commit count. + user_project.repository.commit_count_for_ref(ref) + end + + paginated_commits = Kaminari.paginate_array(commits, total_count: commit_count) - present commits, with: Entities::RepoCommit + present paginate(paginated_commits), with: Entities::RepoCommit end desc 'Commit multiple file changes as one commit' do @@ -127,7 +139,7 @@ module API commit_params = { commit: commit, - create_merge_request: false, + start_branch: params[:branch], target_branch: params[:branch] } diff --git a/lib/api/deployments.rb b/lib/api/deployments.rb index c5feb49b22f..2f1ad12c38c 100644 --- a/lib/api/deployments.rb +++ b/lib/api/deployments.rb @@ -1,5 +1,5 @@ module API - # Deployments RESTfull API endpoints + # Deployments RESTful API endpoints class Deployments < Grape::API include PaginationParams diff --git a/lib/api/entities.rb b/lib/api/entities.rb index d2d21f5d03a..0a12ee72d49 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -49,7 +49,8 @@ module API class ProjectHook < Hook expose :project_id, :issues_events, :merge_requests_events - expose :note_events, :build_events, :pipeline_events, :wiki_page_events + expose :note_events, :pipeline_events, :wiki_page_events + expose :build_events, as: :job_events end class BasicProjectDetails < Grape::Entity @@ -80,7 +81,7 @@ module API expose(:issues_enabled) { |project, options| project.feature_available?(:issues, options[:current_user]) } expose(:merge_requests_enabled) { |project, options| project.feature_available?(:merge_requests, options[:current_user]) } expose(:wiki_enabled) { |project, options| project.feature_available?(:wiki, options[:current_user]) } - expose(:builds_enabled) { |project, options| project.feature_available?(:builds, options[:current_user]) } + expose(:jobs_enabled) { |project, options| project.feature_available?(:builds, options[:current_user]) } expose(:snippets_enabled) { |project, options| project.feature_available?(:snippets, options[:current_user]) } expose :created_at, :last_activity_at @@ -93,7 +94,7 @@ module API expose :star_count, :forks_count expose :open_issues_count, if: lambda { |project, options| project.feature_available?(:issues, options[:current_user]) && project.default_issues_tracker? } expose :runners_token, if: lambda { |_project, options| options[:user_can_admin_project] } - expose :public_builds + expose :public_builds, as: :public_jobs expose :shared_with_groups do |project, options| SharedGroup.represent(project.project_group_links.all, options) end @@ -109,7 +110,7 @@ module API expose :storage_size expose :repository_size expose :lfs_objects_size - expose :build_artifacts_size + expose :build_artifacts_size, as: :job_artifacts_size end class Member < UserBasic @@ -144,7 +145,7 @@ module API expose :storage_size expose :repository_size expose :lfs_objects_size - expose :build_artifacts_size + expose :build_artifacts_size, as: :job_artifacts_size end end end @@ -249,14 +250,11 @@ module API expose :start_date end - class Issue < ProjectEntity + class IssueBasic < ProjectEntity expose :label_names, as: :labels expose :milestone, using: Entities::Milestone expose :assignee, :author, using: Entities::UserBasic - expose :subscribed do |issue, options| - issue.subscribed?(options[:current_user], options[:project] || issue.project) - end expose :user_notes_count expose :upvotes, :downvotes expose :due_date @@ -267,6 +265,12 @@ module API end end + class Issue < IssueBasic + expose :subscribed do |issue, options| + issue.subscribed?(options[:current_user], options[:project] || issue.project) + end + end + class IssuableTimeStats < Grape::Entity expose :time_estimate expose :total_time_spent @@ -279,7 +283,7 @@ module API expose :id end - class MergeRequest < ProjectEntity + class MergeRequestBasic < ProjectEntity expose :target_branch, :source_branch expose :upvotes, :downvotes expose :author, :assignee, using: Entities::UserBasic @@ -291,9 +295,6 @@ module API expose :merge_status expose :diff_head_sha, as: :sha expose :merge_commit_sha - expose :subscribed do |merge_request, options| - merge_request.subscribed?(options[:current_user], options[:project]) - end expose :user_notes_count expose :should_remove_source_branch?, as: :should_remove_source_branch expose :force_remove_source_branch?, as: :force_remove_source_branch @@ -303,6 +304,12 @@ module API end end + class MergeRequest < MergeRequestBasic + expose :subscribed do |merge_request, options| + merge_request.subscribed?(options[:current_user], options[:project]) + end + end + class MergeRequestChanges < MergeRequest expose :diffs, as: :changes, using: Entities::RepoDiff do |compare, _| compare.raw_diffs(all_diffs: true).to_a @@ -448,7 +455,8 @@ module API class ProjectService < Grape::Entity expose :id, :title, :created_at, :updated_at, :active expose :push_events, :issues_events, :merge_requests_events - expose :tag_push_events, :note_events, :build_events, :pipeline_events + expose :tag_push_events, :note_events, :pipeline_events + expose :build_events, as: :job_events # Expose serialized properties expose :properties do |service, options| field_names = service.fields. @@ -592,10 +600,6 @@ module API end end - class TriggerRequest < Grape::Entity - expose :id, :variables - end - class Runner < Grape::Entity expose :id expose :description @@ -624,7 +628,7 @@ module API expose :id, :token end - class BuildArtifactFile < Grape::Entity + class JobArtifactFile < Grape::Entity expose :filename, :size end @@ -632,18 +636,21 @@ module API expose :id, :sha, :ref, :status end - class Build < Grape::Entity + class Job < Grape::Entity expose :id, :status, :stage, :name, :ref, :tag, :coverage expose :created_at, :started_at, :finished_at expose :user, with: User - expose :artifacts_file, using: BuildArtifactFile, if: -> (build, opts) { build.artifacts? } + expose :artifacts_file, using: JobArtifactFile, if: -> (job, opts) { job.artifacts? } expose :commit, with: RepoCommit expose :runner, with: Runner expose :pipeline, with: PipelineBasic end class Trigger < Grape::Entity - expose :token, :created_at, :updated_at, :deleted_at, :last_used + expose :id + expose :token, :description + expose :created_at, :updated_at, :deleted_at, :last_used + expose :owner, using: Entities::UserBasic end class Variable < Grape::Entity @@ -664,14 +671,14 @@ module API end class Environment < EnvironmentBasic - expose :project, using: Entities::Project + expose :project, using: Entities::BasicProjectDetails end class Deployment < Grape::Entity expose :id, :iid, :ref, :sha, :created_at expose :user, using: Entities::UserBasic expose :environment, using: Entities::EnvironmentBasic - expose :deployable, using: Entities::Build + expose :deployable, using: Entities::Job end class RepoLicense < Grape::Entity @@ -698,5 +705,99 @@ module API expose :id, :message, :starts_at, :ends_at, :color, :font expose :active?, as: :active end + + class PersonalAccessToken < Grape::Entity + expose :id, :name, :revoked, :created_at, :scopes + expose :active?, as: :active + expose :expires_at do |personal_access_token| + personal_access_token.expires_at ? personal_access_token.expires_at.strftime("%Y-%m-%d") : nil + end + end + + class PersonalAccessTokenWithToken < PersonalAccessToken + expose :token + end + + class ImpersonationToken < PersonalAccessTokenWithToken + expose :impersonation + end + + module JobRequest + class JobInfo < Grape::Entity + expose :name, :stage + expose :project_id, :project_name + end + + class GitInfo < Grape::Entity + expose :repo_url, :ref, :sha, :before_sha + expose :ref_type do |model| + if model.tag + 'tag' + else + 'branch' + end + end + end + + class RunnerInfo < Grape::Entity + expose :timeout + end + + class Step < Grape::Entity + expose :name, :script, :timeout, :when, :allow_failure + end + + class Image < Grape::Entity + expose :name + end + + class Artifacts < Grape::Entity + expose :name, :untracked, :paths, :when, :expire_in + end + + class Cache < Grape::Entity + expose :key, :untracked, :paths + end + + class Credentials < Grape::Entity + expose :type, :url, :username, :password + end + + class ArtifactFile < Grape::Entity + expose :filename, :size + end + + class Dependency < Grape::Entity + expose :id, :name + expose :artifacts_file, using: ArtifactFile, if: ->(job, _) { job.artifacts? } + end + + class Response < Grape::Entity + expose :id + expose :token + expose :allow_git_fetch + + expose :job_info, using: JobInfo do |model| + model + end + + expose :git_info, using: GitInfo do |model| + model + end + + expose :runner_info, using: RunnerInfo do |model| + model + end + + expose :variables + expose :steps, using: Step + expose :image, using: Image + expose :services, using: Image + expose :artifacts, using: Artifacts + expose :cache, using: Cache + expose :credentials, using: Credentials + expose :depends_on_builds, as: :dependencies, using: Dependency + end + end end end diff --git a/lib/api/files.rb b/lib/api/files.rb index 9c4e43d77cc..bb8f5c3076d 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -14,6 +14,19 @@ module API } end + def assign_file_vars! + authorize! :download_code, user_project + + @commit = user_project.commit(params[:ref]) + not_found!('Commit') unless @commit + + @repo = user_project.repository + @blob = @repo.blob_at(@commit.sha, params[:file_path]) + + not_found!('File') unless @blob + @blob.load_all_data!(@repo) + end + def commit_response(attrs) { file_path: attrs[:file_path], @@ -22,7 +35,7 @@ module API end params :simple_file_params do - requires :file_path, type: String, desc: 'The path to new file. Ex. lib/class.rb' + requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' requires :branch, type: String, desc: 'The name of branch' requires :commit_message, type: String, desc: 'Commit Message' optional :author_email, type: String, desc: 'The email of the author' @@ -40,34 +53,35 @@ module API requires :id, type: String, desc: 'The project ID' end resource :projects do - desc 'Get a file from repository' + desc 'Get raw file contents from the repository' params do - requires :file_path, type: String, desc: 'The path to the file. Ex. lib/class.rb' - requires :ref, type: String, desc: 'The name of branch, tag, or commit' + requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' + requires :ref, type: String, desc: 'The name of branch, tag commit' end - get ":id/repository/files" do - authorize! :download_code, user_project - - commit = user_project.commit(params[:ref]) - not_found!('Commit') unless commit + get ":id/repository/files/:file_path/raw" do + assign_file_vars! - repo = user_project.repository - blob = repo.blob_at(commit.sha, params[:file_path]) - not_found!('File') unless blob + send_git_blob @repo, @blob + end - blob.load_all_data!(repo) - status(200) + desc 'Get a file from the repository' + params do + requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' + requires :ref, type: String, desc: 'The name of branch, tag or commit' + end + get ":id/repository/files/:file_path", requirements: { file_path: /.+/ } do + assign_file_vars! { - file_name: blob.name, - file_path: blob.path, - size: blob.size, + file_name: @blob.name, + file_path: @blob.path, + size: @blob.size, encoding: "base64", - content: Base64.strict_encode64(blob.data), + content: Base64.strict_encode64(@blob.data), ref: params[:ref], - blob_id: blob.id, - commit_id: commit.id, - last_commit_id: repo.last_commit_id_for_path(commit.sha, params[:file_path]) + blob_id: @blob.id, + commit_id: @commit.id, + last_commit_id: @repo.last_commit_id_for_path(@commit.sha, params[:file_path]) } end @@ -75,7 +89,7 @@ module API params do use :extended_file_params end - post ":id/repository/files" do + post ":id/repository/files/:file_path", requirements: { file_path: /.+/ } do authorize! :push_code, user_project file_params = declared_params(include_missing: false) @@ -93,7 +107,7 @@ module API params do use :extended_file_params end - put ":id/repository/files" do + put ":id/repository/files/:file_path", requirements: { file_path: /.+/ } do authorize! :push_code, user_project file_params = declared_params(include_missing: false) @@ -112,7 +126,7 @@ module API params do use :simple_file_params end - delete ":id/repository/files" do + delete ":id/repository/files/:file_path", requirements: { file_path: /.+/ } do authorize! :push_code, user_project file_params = declared_params(include_missing: false) diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 4600abc7dc7..a9b364da9e1 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -82,16 +82,16 @@ module API label || not_found!('Label') end - def find_project_issue(id) - IssuesFinder.new(current_user, project_id: user_project.id).find(id) + def find_project_issue(iid) + IssuesFinder.new(current_user, project_id: user_project.id).find_by!(iid: iid) end - def find_project_merge_request(id) - MergeRequestsFinder.new(current_user, project_id: user_project.id).find(id) + def find_project_merge_request(iid) + MergeRequestsFinder.new(current_user, project_id: user_project.id).find_by!(iid: iid) end - def find_merge_request_with_access(id, access_level = :read_merge_request) - merge_request = user_project.merge_requests.find(id) + def find_merge_request_with_access(iid, access_level = :read_merge_request) + merge_request = user_project.merge_requests.find_by!(iid: iid) authorize! access_level, merge_request merge_request end @@ -252,6 +252,10 @@ module API # project helpers def filter_projects(projects) + if params[:membership] + projects = projects.merge(current_user.authorized_projects) + end + if params[:owned] projects = projects.merge(current_user.owned_projects) end @@ -332,16 +336,17 @@ module API def initial_current_user return @initial_current_user if defined?(@initial_current_user) + Gitlab::Auth::UniqueIpsLimiter.limit_user! do + @initial_current_user ||= find_user_by_private_token(scopes: @scopes) + @initial_current_user ||= doorkeeper_guard(scopes: @scopes) + @initial_current_user ||= find_user_from_warden - @initial_current_user ||= find_user_by_private_token(scopes: @scopes) - @initial_current_user ||= doorkeeper_guard(scopes: @scopes) - @initial_current_user ||= find_user_from_warden + unless @initial_current_user && Gitlab::UserAccess.new(@initial_current_user).allowed? + @initial_current_user = nil + end - unless @initial_current_user && Gitlab::UserAccess.new(@initial_current_user).allowed? - @initial_current_user = nil + @initial_current_user end - - @initial_current_user end def sudo! @@ -384,14 +389,6 @@ module API header(*Gitlab::Workhorse.send_git_archive(repository, ref: ref, format: format)) end - def issue_entity(project) - if project.has_external_issue_tracker? - Entities::ExternalIssue - else - Entities::Issue - end - end - # The Grape Error Middleware only has access to env but no params. We workaround this by # defining a method that returns the right value. def define_params_for_grape_middleware diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index 080a6274957..2135a787b11 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -9,11 +9,11 @@ module API # In addition, they may have a '.git' extension and multiple namespaces # # Transform all these cases to 'namespace/project' - def clean_project_path(project_path, storage_paths = Repository.storages.values) + def clean_project_path(project_path, storages = Gitlab.config.repositories.storages.values) project_path = project_path.sub(/\.git\z/, '') - storage_paths.each do |storage_path| - storage_path = File.expand_path(storage_path) + storages.each do |storage| + storage_path = File.expand_path(storage['path']) if project_path.start_with?(storage_path) project_path = project_path.sub(storage_path, '') diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb index 119ca81b883..ec2bcaed929 100644 --- a/lib/api/helpers/runner.rb +++ b/lib/api/helpers/runner.rb @@ -1,6 +1,10 @@ module API module Helpers module Runner + JOB_TOKEN_HEADER = 'HTTP_JOB_TOKEN'.freeze + JOB_TOKEN_PARAM = :token + UPDATE_RUNNER_EVERY = 10 * 60 + def runner_registration_token_valid? ActiveSupport::SecurityUtils.variable_size_secure_compare(params[:token], current_application_settings.runners_registration_token) @@ -18,6 +22,56 @@ module API def current_runner @runner ||= ::Ci::Runner.find_by_token(params[:token].to_s) end + + def update_runner_info + return unless update_runner? + + current_runner.contacted_at = Time.now + current_runner.assign_attributes(get_runner_version_from_params) + current_runner.save if current_runner.changed? + end + + def update_runner? + # Use a random threshold to prevent beating DB updates. + # It generates a distribution between [40m, 80m]. + # + contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY) + + current_runner.contacted_at.nil? || + (Time.now - current_runner.contacted_at) >= contacted_at_max_age + end + + def job_not_found! + if headers['User-Agent'].to_s =~ /gitlab(-ci-multi)?-runner \d+\.\d+\.\d+(~beta\.\d+\.g[0-9a-f]+)? / + no_content! + else + not_found! + end + end + + def validate_job!(job) + not_found! unless job + + yield if block_given? + + forbidden!('Project has been deleted!') unless job.project + forbidden!('Job has been erased!') if job.erased? + end + + def authenticate_job!(job) + validate_job!(job) do + forbidden! unless job_token_valid?(job) + end + end + + def job_token_valid?(job) + token = (params[JOB_TOKEN_PARAM] || env[JOB_TOKEN_HEADER]).to_s + token && job.valid_token?(token) + end + + def max_artifacts_size + current_application_settings.max_artifacts_size.megabytes.to_i + end end end end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 1d6d0b05750..4a9f2b26fb2 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -41,7 +41,7 @@ module API resource :issues do desc "Get currently authenticated user's issues" do - success Entities::Issue + success Entities::IssueBasic end params do optional :state, type: String, values: %w[opened closed all], default: 'all', @@ -51,7 +51,7 @@ module API get do issues = find_issues(scope: 'authored') - present paginate(issues), with: Entities::Issue, current_user: current_user + present paginate(issues), with: Entities::IssueBasic, current_user: current_user end end @@ -60,7 +60,7 @@ module API end resource :groups do desc 'Get a list of group issues' do - success Entities::Issue + success Entities::IssueBasic end params do optional :state, type: String, values: %w[opened closed all], default: 'opened', @@ -72,7 +72,7 @@ module API issues = find_issues(group_id: group.id, state: params[:state] || 'opened') - present paginate(issues), with: Entities::Issue, current_user: current_user + present paginate(issues), with: Entities::IssueBasic, current_user: current_user end end @@ -83,7 +83,7 @@ module API include TimeTrackingEndpoints desc 'Get a list of project issues' do - success Entities::Issue + success Entities::IssueBasic end params do optional :state, type: String, values: %w[opened closed all], default: 'all', @@ -95,17 +95,17 @@ module API issues = find_issues(project_id: project.id) - present paginate(issues), with: Entities::Issue, current_user: current_user, project: user_project + present paginate(issues), with: Entities::IssueBasic, current_user: current_user, project: user_project end desc 'Get a single project issue' do success Entities::Issue end params do - requires :issue_id, type: Integer, desc: 'The ID of a project issue' + requires :issue_iid, type: Integer, desc: 'The internal ID of a project issue' end - get ":id/issues/:issue_id" do - issue = find_project_issue(params[:issue_id]) + get ":id/issues/:issue_iid" do + issue = find_project_issue(params[:issue_iid]) present issue, with: Entities::Issue, current_user: current_user, project: user_project end @@ -152,7 +152,7 @@ module API success Entities::Issue end params do - requires :issue_id, type: Integer, desc: 'The ID of a project issue' + requires :issue_iid, type: Integer, desc: 'The internal ID of a project issue' optional :title, type: String, desc: 'The title of an issue' optional :updated_at, type: DateTime, desc: 'Date time when the issue was updated. Available only for admins and project owners.' @@ -161,8 +161,8 @@ module API at_least_one_of :title, :description, :assignee_id, :milestone_id, :labels, :created_at, :due_date, :confidential, :state_event end - put ':id/issues/:issue_id' do - issue = user_project.issues.find(params.delete(:issue_id)) + put ':id/issues/:issue_iid' do + issue = user_project.issues.find_by!(iid: params.delete(:issue_iid)) authorize! :update_issue, issue # Setting created_at time only allowed for admins and project owners @@ -189,11 +189,11 @@ module API success Entities::Issue end params do - requires :issue_id, type: Integer, desc: 'The ID of a project issue' + requires :issue_iid, type: Integer, desc: 'The internal ID of a project issue' requires :to_project_id, type: Integer, desc: 'The ID of the new project' end - post ':id/issues/:issue_id/move' do - issue = user_project.issues.find_by(id: params[:issue_id]) + post ':id/issues/:issue_iid/move' do + issue = user_project.issues.find_by(iid: params[:issue_iid]) not_found!('Issue') unless issue new_project = Project.find_by(id: params[:to_project_id]) @@ -209,10 +209,10 @@ module API desc 'Delete a project issue' params do - requires :issue_id, type: Integer, desc: 'The ID of a project issue' + requires :issue_iid, type: Integer, desc: 'The internal ID of a project issue' end - delete ":id/issues/:issue_id" do - issue = user_project.issues.find_by(id: params[:issue_id]) + delete ":id/issues/:issue_iid" do + issue = user_project.issues.find_by(iid: params[:issue_iid]) not_found!('Issue') unless issue authorize!(:destroy_issue, issue) diff --git a/lib/api/builds.rb b/lib/api/jobs.rb index 5b76913fe45..44118522abe 100644 --- a/lib/api/builds.rb +++ b/lib/api/jobs.rb @@ -1,5 +1,5 @@ module API - class Builds < Grape::API + class Jobs < Grape::API include PaginationParams before { authenticate! } @@ -13,10 +13,13 @@ module API optional :scope, types: [String, Array[String]], desc: 'The scope of builds to show', values: ::CommitStatus::AVAILABLE_STATUSES, coerce_with: ->(scope) { - if scope.is_a?(String) + case scope + when String [scope] - elsif scope.is_a?(Hashie::Mash) + when Hashie::Mash scope.values + when Hashie::Array + scope else ['unknown'] end @@ -24,79 +27,72 @@ module API end end - desc 'Get a project builds' do - success Entities::Build + desc 'Get a projects jobs' do + success Entities::Job end params do use :optional_scope use :pagination end - get ':id/builds' do + get ':id/jobs' do builds = user_project.builds.order('id DESC') builds = filter_builds(builds, params[:scope]) - present paginate(builds), with: Entities::Build, - user_can_download_artifacts: can?(current_user, :read_build, user_project) + present paginate(builds), with: Entities::Job end - desc 'Get builds for a specific commit of a project' do - success Entities::Build + desc 'Get pipeline jobs' do + success Entities::Job end params do - requires :sha, type: String, desc: 'The SHA id of a commit' + requires :pipeline_id, type: Integer, desc: 'The pipeline ID' use :optional_scope use :pagination end - get ':id/repository/commits/:sha/builds' do - authorize_read_builds! - - return not_found! unless user_project.commit(params[:sha]) - - pipelines = user_project.pipelines.where(sha: params[:sha]) - builds = user_project.builds.where(pipeline: pipelines).order('id DESC') + get ':id/pipelines/:pipeline_id/jobs' do + pipeline = user_project.pipelines.find(params[:pipeline_id]) + builds = pipeline.builds builds = filter_builds(builds, params[:scope]) - present paginate(builds), with: Entities::Build, - user_can_download_artifacts: can?(current_user, :read_build, user_project) + present paginate(builds), with: Entities::Job end - desc 'Get a specific build of a project' do - success Entities::Build + desc 'Get a specific job of a project' do + success Entities::Job end params do - requires :build_id, type: Integer, desc: 'The ID of a build' + requires :job_id, type: Integer, desc: 'The ID of a job' end - get ':id/builds/:build_id' do + get ':id/jobs/:job_id' do authorize_read_builds! - build = get_build!(params[:build_id]) + build = get_build!(params[:job_id]) - present build, with: Entities::Build, - user_can_download_artifacts: can?(current_user, :read_build, user_project) + present build, with: Entities::Job end - desc 'Download the artifacts file from build' do + desc 'Download the artifacts file from a job' do detail 'This feature was introduced in GitLab 8.5' end params do - requires :build_id, type: Integer, desc: 'The ID of a build' + requires :job_id, type: Integer, desc: 'The ID of a job' end - get ':id/builds/:build_id/artifacts' do + get ':id/jobs/:job_id/artifacts' do authorize_read_builds! - build = get_build!(params[:build_id]) + build = get_build!(params[:job_id]) present_artifacts!(build.artifacts_file) end - desc 'Download the artifacts file from build' do + desc 'Download the artifacts file from a job' do detail 'This feature was introduced in GitLab 8.10' end params do requires :ref_name, type: String, desc: 'The ref from repository' - requires :job, type: String, desc: 'The name for the build' + requires :job, type: String, desc: 'The name for the job' end - get ':id/builds/artifacts/:ref_name/download', + get ':id/jobs/artifacts/:ref_name/download', requirements: { ref_name: /.+/ } do authorize_read_builds! @@ -109,14 +105,14 @@ module API # TODO: We should use `present_file!` and leave this implementation for backward compatibility (when build trace # is saved in the DB instead of file). But before that, we need to consider how to replace the value of # `runners_token` with some mask (like `xxxxxx`) when sending trace file directly by workhorse. - desc 'Get a trace of a specific build of a project' + desc 'Get a trace of a specific job of a project' params do - requires :build_id, type: Integer, desc: 'The ID of a build' + requires :job_id, type: Integer, desc: 'The ID of a job' end - get ':id/builds/:build_id/trace' do + get ':id/jobs/:job_id/trace' do authorize_read_builds! - build = get_build!(params[:build_id]) + build = get_build!(params[:job_id]) header 'Content-Disposition', "infile; filename=\"#{build.id}.log\"" content_type 'text/plain' @@ -126,96 +122,91 @@ module API body trace end - desc 'Cancel a specific build of a project' do - success Entities::Build + desc 'Cancel a specific job of a project' do + success Entities::Job end params do - requires :build_id, type: Integer, desc: 'The ID of a build' + requires :job_id, type: Integer, desc: 'The ID of a job' end - post ':id/builds/:build_id/cancel' do + post ':id/jobs/:job_id/cancel' do authorize_update_builds! - build = get_build!(params[:build_id]) + build = get_build!(params[:job_id]) build.cancel - present build, with: Entities::Build, - user_can_download_artifacts: can?(current_user, :read_build, user_project) + present build, with: Entities::Job end desc 'Retry a specific build of a project' do - success Entities::Build + success Entities::Job end params do - requires :build_id, type: Integer, desc: 'The ID of a build' + requires :job_id, type: Integer, desc: 'The ID of a build' end - post ':id/builds/:build_id/retry' do + post ':id/jobs/:job_id/retry' do authorize_update_builds! - build = get_build!(params[:build_id]) - return forbidden!('Build is not retryable') unless build.retryable? + build = get_build!(params[:job_id]) + return forbidden!('Job is not retryable') unless build.retryable? build = Ci::Build.retry(build, current_user) - present build, with: Entities::Build, - user_can_download_artifacts: can?(current_user, :read_build, user_project) + present build, with: Entities::Job end - desc 'Erase build (remove artifacts and build trace)' do - success Entities::Build + desc 'Erase job (remove artifacts and the trace)' do + success Entities::Job end params do - requires :build_id, type: Integer, desc: 'The ID of a build' + requires :job_id, type: Integer, desc: 'The ID of a build' end - post ':id/builds/:build_id/erase' do + post ':id/jobs/:job_id/erase' do authorize_update_builds! - build = get_build!(params[:build_id]) - return forbidden!('Build is not erasable!') unless build.erasable? + build = get_build!(params[:job_id]) + return forbidden!('Job is not erasable!') unless build.erasable? build.erase(erased_by: current_user) - present build, with: Entities::Build, - user_can_download_artifacts: can?(current_user, :download_build_artifacts, user_project) + present build, with: Entities::Job end desc 'Keep the artifacts to prevent them from being deleted' do - success Entities::Build + success Entities::Job end params do - requires :build_id, type: Integer, desc: 'The ID of a build' + requires :job_id, type: Integer, desc: 'The ID of a job' end - post ':id/builds/:build_id/artifacts/keep' do + post ':id/jobs/:job_id/artifacts/keep' do authorize_update_builds! - build = get_build!(params[:build_id]) + build = get_build!(params[:job_id]) return not_found!(build) unless build.artifacts? build.keep_artifacts! status 200 - present build, with: Entities::Build, - user_can_download_artifacts: can?(current_user, :read_build, user_project) + present build, with: Entities::Job end - desc 'Trigger a manual build' do - success Entities::Build + desc 'Trigger a manual job' do + success Entities::Job detail 'This feature was added in GitLab 8.11' end params do - requires :build_id, type: Integer, desc: 'The ID of a Build' + requires :job_id, type: Integer, desc: 'The ID of a Job' end - post ":id/builds/:build_id/play" do + post ":id/jobs/:job_id/play" do authorize_read_builds! - build = get_build!(params[:build_id]) + build = get_build!(params[:job_id]) bad_request!("Unplayable Job") unless build.playable? build.play(current_user) status 200 - present build, with: Entities::Build, - user_can_download_artifacts: can?(current_user, :read_build, user_project) + present build, with: Entities::Job end end diff --git a/lib/api/merge_request_diffs.rb b/lib/api/merge_request_diffs.rb index 4901a7cfea6..a59e39cca26 100644 --- a/lib/api/merge_request_diffs.rb +++ b/lib/api/merge_request_diffs.rb @@ -13,11 +13,11 @@ module API params do requires :id, type: String, desc: 'The ID of a project' - requires :merge_request_id, type: Integer, desc: 'The ID of a merge request' + requires :merge_request_iid, type: Integer, desc: 'The IID of a merge request' use :pagination end - get ":id/merge_requests/:merge_request_id/versions" do - merge_request = find_merge_request_with_access(params[:merge_request_id]) + get ":id/merge_requests/:merge_request_iid/versions" do + merge_request = find_merge_request_with_access(params[:merge_request_iid]) present paginate(merge_request.merge_request_diffs), with: Entities::MergeRequestDiff end @@ -29,12 +29,12 @@ module API params do requires :id, type: String, desc: 'The ID of a project' - requires :merge_request_id, type: Integer, desc: 'The ID of a merge request' + requires :merge_request_iid, type: Integer, desc: 'The IID of a merge request' requires :version_id, type: Integer, desc: 'The ID of a merge request diff version' end - get ":id/merge_requests/:merge_request_id/versions/:version_id" do - merge_request = find_merge_request_with_access(params[:merge_request_id]) + get ":id/merge_requests/:merge_request_iid/versions/:version_id" do + merge_request = find_merge_request_with_access(params[:merge_request_iid]) present merge_request.merge_request_diffs.find(params[:version_id]), with: Entities::MergeRequestDiffFull end diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 4638a66811d..7a03955a045 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -25,6 +25,14 @@ module API render_api_error!(errors, 400) end + def issue_entity(project) + if project.has_external_issue_tracker? + Entities::ExternalIssue + else + Entities::IssueBasic + end + end + params :optional_params do optional :description, type: String, desc: 'The description of the merge request' optional :assignee_id, type: Integer, desc: 'The ID of a user to assign the merge request' @@ -35,7 +43,7 @@ module API end desc 'List merge requests' do - success Entities::MergeRequest + success Entities::MergeRequestBasic end params do optional :state, type: String, values: %w[opened closed merged all], default: 'all', @@ -62,7 +70,7 @@ module API end merge_requests = merge_requests.reorder(params[:order_by] => params[:sort]) - present paginate(merge_requests), with: Entities::MergeRequest, current_user: current_user, project: user_project + present paginate(merge_requests), with: Entities::MergeRequestBasic, current_user: current_user, project: user_project end desc 'Create a merge request' do @@ -93,23 +101,23 @@ module API desc 'Delete a merge request' params do - requires :merge_request_id, type: Integer, desc: 'The ID of a merge request' + requires :merge_request_iid, type: Integer, desc: 'The IID of a merge request' end - delete ":id/merge_requests/:merge_request_id" do - merge_request = find_project_merge_request(params[:merge_request_id]) + delete ":id/merge_requests/:merge_request_iid" do + merge_request = find_project_merge_request(params[:merge_request_iid]) authorize!(:destroy_merge_request, merge_request) merge_request.destroy end params do - requires :merge_request_id, type: Integer, desc: 'The ID of a merge request' + requires :merge_request_iid, type: Integer, desc: 'The IID of a merge request' end desc 'Get a single merge request' do success Entities::MergeRequest end - get ':id/merge_requests/:merge_request_id' do - merge_request = find_merge_request_with_access(params[:merge_request_id]) + get ':id/merge_requests/:merge_request_iid' do + merge_request = find_merge_request_with_access(params[:merge_request_iid]) present merge_request, with: Entities::MergeRequest, current_user: current_user, project: user_project end @@ -117,8 +125,8 @@ module API desc 'Get the commits of a merge request' do success Entities::RepoCommit end - get ':id/merge_requests/:merge_request_id/commits' do - merge_request = find_merge_request_with_access(params[:merge_request_id]) + get ':id/merge_requests/:merge_request_iid/commits' do + merge_request = find_merge_request_with_access(params[:merge_request_iid]) commits = ::Kaminari.paginate_array(merge_request.commits) present paginate(commits), with: Entities::RepoCommit @@ -127,8 +135,8 @@ module API desc 'Show the merge request changes' do success Entities::MergeRequestChanges end - get ':id/merge_requests/:merge_request_id/changes' do - merge_request = find_merge_request_with_access(params[:merge_request_id]) + get ':id/merge_requests/:merge_request_iid/changes' do + merge_request = find_merge_request_with_access(params[:merge_request_iid]) present merge_request, with: Entities::MergeRequestChanges, current_user: current_user end @@ -146,8 +154,8 @@ module API :milestone_id, :labels, :state_event, :remove_source_branch end - put ':id/merge_requests/:merge_request_id' do - merge_request = find_merge_request_with_access(params.delete(:merge_request_id), :update_merge_request) + put ':id/merge_requests/:merge_request_iid' do + merge_request = find_merge_request_with_access(params.delete(:merge_request_iid), :update_merge_request) mr_params = declared_params(include_missing: false) mr_params[:force_remove_source_branch] = mr_params.delete(:remove_source_branch) if mr_params[:remove_source_branch].present? @@ -172,8 +180,8 @@ module API desc: 'When true, this merge request will be merged when the pipeline succeeds' optional :sha, type: String, desc: 'When present, must have the HEAD SHA of the source branch' end - put ':id/merge_requests/:merge_request_id/merge' do - merge_request = find_project_merge_request(params[:merge_request_id]) + put ':id/merge_requests/:merge_request_iid/merge' do + merge_request = find_project_merge_request(params[:merge_request_iid]) # Merge request can not be merged # because user dont have permissions to push into target branch @@ -208,8 +216,8 @@ module API desc 'Cancel merge if "Merge When Pipeline Succeeds" is enabled' do success Entities::MergeRequest end - post ':id/merge_requests/:merge_request_id/cancel_merge_when_pipeline_succeeds' do - merge_request = find_project_merge_request(params[:merge_request_id]) + post ':id/merge_requests/:merge_request_iid/cancel_merge_when_pipeline_succeeds' do + merge_request = find_project_merge_request(params[:merge_request_iid]) unauthorized! unless merge_request.can_cancel_merge_when_pipeline_succeeds?(current_user) @@ -224,8 +232,8 @@ module API params do use :pagination end - get ':id/merge_requests/:merge_request_id/comments' do - merge_request = find_merge_request_with_access(params[:merge_request_id]) + get ':id/merge_requests/:merge_request_iid/comments' do + merge_request = find_merge_request_with_access(params[:merge_request_iid]) present paginate(merge_request.notes.fresh), with: Entities::MRNote end @@ -235,8 +243,8 @@ module API params do requires :note, type: String, desc: 'The text of the comment' end - post ':id/merge_requests/:merge_request_id/comments' do - merge_request = find_merge_request_with_access(params[:merge_request_id], :create_note) + post ':id/merge_requests/:merge_request_iid/comments' do + merge_request = find_merge_request_with_access(params[:merge_request_iid], :create_note) opts = { note: params[:note], @@ -259,8 +267,8 @@ module API params do use :pagination end - get ':id/merge_requests/:merge_request_id/closes_issues' do - merge_request = find_merge_request_with_access(params[:merge_request_id]) + get ':id/merge_requests/:merge_request_iid/closes_issues' do + merge_request = find_merge_request_with_access(params[:merge_request_iid]) issues = ::Kaminari.paginate_array(merge_request.closes_issues(current_user)) present paginate(issues), with: issue_entity(user_project), current_user: current_user end diff --git a/lib/api/milestones.rb b/lib/api/milestones.rb index bd74174c655..e7f7edd95c7 100644 --- a/lib/api/milestones.rb +++ b/lib/api/milestones.rb @@ -103,7 +103,7 @@ module API end desc 'Get all issues for a single project milestone' do - success Entities::Issue + success Entities::IssueBasic end params do requires :milestone_id, type: Integer, desc: 'The ID of a project milestone' @@ -120,12 +120,12 @@ module API } issues = IssuesFinder.new(current_user, finder_params).execute - present paginate(issues), with: Entities::Issue, current_user: current_user, project: user_project + present paginate(issues), with: Entities::IssueBasic, current_user: current_user, project: user_project end desc 'Get all merge requests for a single project milestone' do detail 'This feature was introduced in GitLab 9.' - success Entities::MergeRequest + success Entities::MergeRequestBasic end params do requires :milestone_id, type: Integer, desc: 'The ID of a project milestone' @@ -142,7 +142,10 @@ module API } merge_requests = MergeRequestsFinder.new(current_user, finder_params).execute - present paginate(merge_requests), with: Entities::MergeRequest, current_user: current_user, project: user_project + present paginate(merge_requests), + with: Entities::MergeRequestBasic, + current_user: current_user, + project: user_project end end end diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb index 3afc1e385fe..0721b975ba4 100644 --- a/lib/api/pipelines.rb +++ b/lib/api/pipelines.rb @@ -10,7 +10,7 @@ module API resource :projects do desc 'Get all Pipelines of the project' do detail 'This feature was introduced in GitLab 8.11.' - success Entities::Pipeline + success Entities::PipelineBasic end params do use :pagination @@ -21,7 +21,7 @@ module API authorize! :read_pipeline, user_project pipelines = PipelinesFinder.new(user_project).execute(scope: params[:scope]) - present paginate(pipelines), with: Entities::Pipeline + present paginate(pipelines), with: Entities::PipelineBasic end desc 'Create a new pipeline' do diff --git a/lib/api/projects.rb b/lib/api/projects.rb index f302496c12b..63a4cdd5954 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -46,9 +46,10 @@ module API optional :archived, type: Boolean, default: false, desc: 'Limit by archived status' optional :visibility, type: String, values: Gitlab::VisibilityLevel.string_values, desc: 'Limit by visibility' - optional :search, type: String, desc: 'Return list of authorized projects matching the search criteria' + optional :search, type: String, desc: 'Return list of projects matching the search criteria' optional :owned, type: Boolean, default: false, desc: 'Limit by owned by authenticated user' optional :starred, type: Boolean, default: false, desc: 'Limit by starred status' + optional :membership, type: Boolean, default: false, desc: 'Limit by projects that the current user is a member of' end params :statistics_params do diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index 36166780149..531ef5a63ea 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -17,19 +17,34 @@ module API end not_found! end + + def assign_blob_vars! + authorize! :download_code, user_project + + @repo = user_project.repository + + begin + @blob = Gitlab::Git::Blob.raw(@repo, params[:sha]) + @blob.load_all_data!(@repo) + rescue + not_found! 'Blob' + end + + not_found! 'Blob' unless @blob + end end desc 'Get a project repository tree' do success Entities::RepoTreeObject end params do - optional :ref_name, type: String, desc: 'The name of a repository branch or tag, if not given the default branch is used' + optional :ref, type: String, desc: 'The name of a repository branch or tag, if not given the default branch is used' optional :path, type: String, desc: 'The path of the tree' optional :recursive, type: Boolean, default: false, desc: 'Used to get a recursive tree' use :pagination end get ':id/repository/tree' do - ref = params[:ref_name] || user_project.try(:default_branch) || 'master' + ref = params[:ref] || user_project.try(:default_branch) || 'master' path = params[:path] || nil commit = user_project.commit(ref) @@ -40,39 +55,29 @@ module API present paginate(entries), with: Entities::RepoTreeObject end - desc 'Get a raw file contents' + desc 'Get raw blob contents from the repository' params do requires :sha, type: String, desc: 'The commit, branch name, or tag name' - requires :filepath, type: String, desc: 'The path to the file to display' end - get [":id/repository/blobs/:sha", ":id/repository/commits/:sha/blob"] do - repo = user_project.repository - - commit = repo.commit(params[:sha]) - not_found! "Commit" unless commit + get ':id/repository/blobs/:sha/raw' do + assign_blob_vars! - blob = Gitlab::Git::Blob.find(repo, commit.id, params[:filepath]) - not_found! "File" unless blob - - send_git_blob repo, blob + send_git_blob @repo, @blob end - desc 'Get a raw blob contents by blob sha' + desc 'Get a blob from the repository' params do requires :sha, type: String, desc: 'The commit, branch name, or tag name' end - get ':id/repository/raw_blobs/:sha' do - repo = user_project.repository - - begin - blob = Gitlab::Git::Blob.raw(repo, params[:sha]) - rescue - not_found! 'Blob' - end - - not_found! 'Blob' unless blob + get ':id/repository/blobs/:sha' do + assign_blob_vars! - send_git_blob repo, blob + { + size: @blob.size, + encoding: "base64", + content: Base64.strict_encode64(@blob.data), + sha: @blob.id + } end desc 'Get an archive of the repository' diff --git a/lib/api/runner.rb b/lib/api/runner.rb index 47858f1866b..c700d2ef4a1 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -48,5 +48,203 @@ module API Ci::Runner.find_by_token(params[:token]).destroy end end + + resource :jobs do + desc 'Request a job' do + success Entities::JobRequest::Response + end + params do + requires :token, type: String, desc: %q(Runner's authentication token) + optional :last_update, type: String, desc: %q(Runner's queue last_update token) + optional :info, type: Hash, desc: %q(Runner's metadata) + end + post '/request' do + authenticate_runner! + not_found! unless current_runner.active? + update_runner_info + + if current_runner.is_runner_queue_value_latest?(params[:last_update]) + header 'X-GitLab-Last-Update', params[:last_update] + Gitlab::Metrics.add_event(:build_not_found_cached) + return job_not_found! + end + + new_update = current_runner.ensure_runner_queue_value + result = ::Ci::RegisterJobService.new(current_runner).execute + + if result.valid? + if result.build + Gitlab::Metrics.add_event(:build_found, + project: result.build.project.path_with_namespace) + present result.build, with: Entities::JobRequest::Response + else + Gitlab::Metrics.add_event(:build_not_found) + header 'X-GitLab-Last-Update', new_update + job_not_found! + end + else + # We received build that is invalid due to concurrency conflict + Gitlab::Metrics.add_event(:build_invalid) + conflict! + end + end + + desc 'Updates a job' do + http_codes [[200, 'Job was updated'], [403, 'Forbidden']] + end + params do + requires :token, type: String, desc: %q(Runners's authentication token) + requires :id, type: Integer, desc: %q(Job's ID) + optional :trace, type: String, desc: %q(Job's full trace) + optional :state, type: String, desc: %q(Job's status: success, failed) + end + put '/:id' do + job = Ci::Build.find_by_id(params[:id]) + authenticate_job!(job) + + job.update_attributes(trace: params[:trace]) if params[:trace] + + Gitlab::Metrics.add_event(:update_build, + project: job.project.path_with_namespace) + + case params[:state].to_s + when 'success' + job.success + when 'failed' + job.drop + end + end + + desc 'Appends a patch to the job trace' do + http_codes [[202, 'Trace was patched'], + [400, 'Missing Content-Range header'], + [403, 'Forbidden'], + [416, 'Range not satisfiable']] + end + params do + requires :id, type: Integer, desc: %q(Job's ID) + optional :token, type: String, desc: %q(Job's authentication token) + end + patch '/:id/trace' do + job = Ci::Build.find_by_id(params[:id]) + authenticate_job!(job) + + error!('400 Missing header Content-Range', 400) unless request.headers.has_key?('Content-Range') + content_range = request.headers['Content-Range'] + content_range = content_range.split('-') + + current_length = job.trace_length + unless current_length == content_range[0].to_i + return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{current_length}" }) + end + + job.append_trace(request.body.read, content_range[0].to_i) + + status 202 + header 'Job-Status', job.status + header 'Range', "0-#{job.trace_length}" + end + + desc 'Authorize artifacts uploading for job' do + http_codes [[200, 'Upload allowed'], + [403, 'Forbidden'], + [405, 'Artifacts support not enabled'], + [413, 'File too large']] + end + params do + requires :id, type: Integer, desc: %q(Job's ID) + optional :token, type: String, desc: %q(Job's authentication token) + optional :filesize, type: Integer, desc: %q(Artifacts filesize) + end + post '/:id/artifacts/authorize' do + not_allowed! unless Gitlab.config.artifacts.enabled + require_gitlab_workhorse! + Gitlab::Workhorse.verify_api_request!(headers) + + job = Ci::Build.find_by_id(params[:id]) + authenticate_job!(job) + forbidden!('Job is not running') unless job.running? + + if params[:filesize] + file_size = params[:filesize].to_i + file_to_large! unless file_size < max_artifacts_size + end + + status 200 + content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE + Gitlab::Workhorse.artifact_upload_ok + end + + desc 'Upload artifacts for job' do + success Entities::JobRequest::Response + http_codes [[201, 'Artifact uploaded'], + [400, 'Bad request'], + [403, 'Forbidden'], + [405, 'Artifacts support not enabled'], + [413, 'File too large']] + end + params do + requires :id, type: Integer, desc: %q(Job's ID) + optional :token, type: String, desc: %q(Job's authentication token) + optional :expire_in, type: String, desc: %q(Specify when artifacts should expire) + optional :file, type: File, desc: %q(Artifact's file) + optional 'file.path', type: String, desc: %q(path to locally stored body (generated by Workhorse)) + optional 'file.name', type: String, desc: %q(real filename as send in Content-Disposition (generated by Workhorse)) + optional 'file.type', type: String, desc: %q(real content type as send in Content-Type (generated by Workhorse)) + optional 'metadata.path', type: String, desc: %q(path to locally stored body (generated by Workhorse)) + optional 'metadata.name', type: String, desc: %q(filename (generated by Workhorse)) + end + post '/:id/artifacts' do + not_allowed! unless Gitlab.config.artifacts.enabled + require_gitlab_workhorse! + + job = Ci::Build.find_by_id(params[:id]) + authenticate_job!(job) + forbidden!('Job is not running!') unless job.running? + + artifacts_upload_path = ArtifactUploader.artifacts_upload_path + artifacts = uploaded_file(:file, artifacts_upload_path) + metadata = uploaded_file(:metadata, artifacts_upload_path) + + bad_request!('Missing artifacts file!') unless artifacts + file_to_large! unless artifacts.size < max_artifacts_size + + job.artifacts_file = artifacts + job.artifacts_metadata = metadata + job.artifacts_expire_in = params['expire_in'] || + Gitlab::CurrentSettings.current_application_settings.default_artifacts_expire_in + + if job.save + present job, with: Entities::JobRequest::Response + else + render_validation_error!(job) + end + end + + desc 'Download the artifacts file for job' do + http_codes [[200, 'Upload allowed'], + [403, 'Forbidden'], + [404, 'Artifact not found']] + end + params do + requires :id, type: Integer, desc: %q(Job's ID) + optional :token, type: String, desc: %q(Job's authentication token) + end + get '/:id/artifacts' do + job = Ci::Build.find_by_id(params[:id]) + authenticate_job!(job) + + artifacts_file = job.artifacts_file + unless artifacts_file.file_storage? + return redirect_to job.artifacts_file.url + end + + unless artifacts_file.exists? + not_found! + end + + present_file!(artifacts_file.path, artifacts_file.filename) + end + end end end diff --git a/lib/api/services.rb b/lib/api/services.rb index 79a5f27dc4d..5aa2f5eba7b 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -122,9 +122,9 @@ module API }, { required: false, - name: :notify_only_broken_builds, + name: :notify_only_broken_jobs, type: Boolean, - desc: 'Notify only broken builds' + desc: 'Notify only broken jobs' } ], 'campfire' => [ @@ -403,9 +403,9 @@ module API }, { required: false, - name: :notify_only_broken_builds, + name: :notify_only_broken_jobs, type: Boolean, - desc: 'Notify only broken builds' + desc: 'Notify only broken jobs' } ], 'pivotaltracker' => [ @@ -422,6 +422,14 @@ module API desc: 'Comma-separated list of branches which will be automatically inspected. Leave blank to include all branches.' } ], + 'prometheus' => [ + { + required: true, + name: :api_url, + type: String, + desc: 'Prometheus API Base URL, like http://prometheus.example.com/' + } + ], 'pushover' => [ { required: true, @@ -558,6 +566,7 @@ module API SlackSlashCommandsService, PipelinesEmailService, PivotaltrackerService, + PrometheusService, PushoverService, RedmineService, SlackService, @@ -611,7 +620,7 @@ module API desc "Set #{service_slug} service for project" params do service_classes.each do |service| - event_names = service.try(:event_names) || [] + event_names = service.try(:event_names) || next event_names.each do |event_name| services[service.to_param.tr("_", "-")] << { required: false, diff --git a/lib/api/time_tracking_endpoints.rb b/lib/api/time_tracking_endpoints.rb index 85b5f7d98b8..05b4b490e27 100644 --- a/lib/api/time_tracking_endpoints.rb +++ b/lib/api/time_tracking_endpoints.rb @@ -5,11 +5,11 @@ module API included do helpers do def issuable_name - declared_params.has_key?(:issue_id) ? 'issue' : 'merge_request' + declared_params.has_key?(:issue_iid) ? 'issue' : 'merge_request' end def issuable_key - "#{issuable_name}_id".to_sym + "#{issuable_name}_iid".to_sym end def update_issuable_key @@ -50,7 +50,7 @@ module API issuable_name = name.end_with?('Issues') ? 'issue' : 'merge_request' issuable_collection_name = issuable_name.pluralize - issuable_key = "#{issuable_name}_id".to_sym + issuable_key = "#{issuable_name}_iid".to_sym desc "Set a time estimate for a project #{issuable_name}" params do diff --git a/lib/api/todos.rb b/lib/api/todos.rb index e59030428da..d9b8837a5bb 100644 --- a/lib/api/todos.rb +++ b/lib/api/todos.rb @@ -5,8 +5,8 @@ module API before { authenticate! } ISSUABLE_TYPES = { - 'merge_requests' => ->(id) { find_merge_request_with_access(id) }, - 'issues' => ->(id) { find_project_issue(id) } + 'merge_requests' => ->(iid) { find_merge_request_with_access(iid) }, + 'issues' => ->(iid) { find_project_issue(iid) } }.freeze params do @@ -14,13 +14,13 @@ module API end resource :projects do ISSUABLE_TYPES.each do |type, finder| - type_id_str = "#{type.singularize}_id".to_sym + type_id_str = "#{type.singularize}_iid".to_sym desc 'Create a todo on an issuable' do success Entities::Todo end params do - requires type_id_str, type: Integer, desc: 'The ID of an issuable' + requires type_id_str, type: Integer, desc: 'The IID of an issuable' end post ":id/#{type}/:#{type_id_str}/todo" do issuable = instance_exec(params[type_id_str], &finder) diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb index b7c9c5f2b7f..119e9024712 100644 --- a/lib/api/triggers.rb +++ b/lib/api/triggers.rb @@ -6,15 +6,15 @@ module API requires :id, type: String, desc: 'The ID of a project' end resource :projects do - desc 'Trigger a GitLab project build' do - success Entities::TriggerRequest + desc 'Trigger a GitLab project pipeline' do + success Entities::Pipeline end params do requires :ref, type: String, desc: 'The commit sha or name of a branch or tag' requires :token, type: String, desc: 'The unique token of trigger' optional :variables, type: Hash, desc: 'The list of variables to be injected into build' end - post ":id/(ref/:ref/)trigger/builds" do + post ":id/(ref/:ref/)trigger/pipeline" do project = find_project(params[:id]) trigger = Ci::Trigger.find_by_token(params[:token].to_s) not_found! unless project && trigger @@ -29,9 +29,9 @@ module API # create request and trigger builds trigger_request = Ci::CreateTriggerRequestService.new.execute(project, trigger, params[:ref].to_s, variables) if trigger_request - present trigger_request, with: Entities::TriggerRequest + present trigger_request.pipeline, with: Entities::Pipeline else - errors = 'No builds created' + errors = 'No pipeline created' render_api_error!(errors, 400) end end @@ -55,13 +55,13 @@ module API success Entities::Trigger end params do - requires :token, type: String, desc: 'The unique token of trigger' + requires :trigger_id, type: Integer, desc: 'The trigger ID' end - get ':id/triggers/:token' do + get ':id/triggers/:trigger_id' do authenticate! authorize! :admin_build, user_project - trigger = user_project.triggers.find_by(token: params[:token].to_s) + trigger = user_project.triggers.find(params.delete(:trigger_id)) return not_found!('Trigger') unless trigger present trigger, with: Entities::Trigger @@ -70,26 +70,76 @@ module API desc 'Create a trigger' do success Entities::Trigger end + params do + requires :description, type: String, desc: 'The trigger description' + end post ':id/triggers' do authenticate! authorize! :admin_build, user_project - trigger = user_project.triggers.create + trigger = user_project.triggers.create( + declared_params(include_missing: false).merge(owner: current_user)) - present trigger, with: Entities::Trigger + if trigger.valid? + present trigger, with: Entities::Trigger + else + render_validation_error!(trigger) + end + end + + desc 'Update a trigger' do + success Entities::Trigger + end + params do + requires :trigger_id, type: Integer, desc: 'The trigger ID' + optional :description, type: String, desc: 'The trigger description' + end + put ':id/triggers/:trigger_id' do + authenticate! + authorize! :admin_build, user_project + + trigger = user_project.triggers.find(params.delete(:trigger_id)) + return not_found!('Trigger') unless trigger + + if trigger.update(declared_params(include_missing: false)) + present trigger, with: Entities::Trigger + else + render_validation_error!(trigger) + end + end + + desc 'Take ownership of trigger' do + success Entities::Trigger + end + params do + requires :trigger_id, type: Integer, desc: 'The trigger ID' + end + post ':id/triggers/:trigger_id/take_ownership' do + authenticate! + authorize! :admin_build, user_project + + trigger = user_project.triggers.find(params.delete(:trigger_id)) + return not_found!('Trigger') unless trigger + + if trigger.update(owner: current_user) + status :ok + present trigger, with: Entities::Trigger + else + render_validation_error!(trigger) + end end desc 'Delete a trigger' do success Entities::Trigger end params do - requires :token, type: String, desc: 'The unique token of trigger' + requires :trigger_id, type: Integer, desc: 'The trigger ID' end - delete ':id/triggers/:token' do + delete ':id/triggers/:trigger_id' do authenticate! authorize! :admin_build, user_project - trigger = user_project.triggers.find_by(token: params[:token].to_s) + trigger = user_project.triggers.find(params.delete(:trigger_id)) return not_found!('Trigger') unless trigger trigger.destroy diff --git a/lib/api/users.rb b/lib/api/users.rb index 7bb4b76f830..549003f576a 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -9,6 +9,11 @@ module API resource :users, requirements: { uid: /[0-9]*/, id: /[0-9]*/ } do helpers do + def find_user(params) + id = params[:user_id] || params[:id] + User.find_by(id: id) || not_found!('User') + end + params :optional_attributes do optional :skype, type: String, desc: 'The Skype username' optional :linkedin, type: String, desc: 'The LinkedIn username' @@ -362,6 +367,76 @@ module API present paginate(events), with: Entities::Event end + + params do + requires :user_id, type: Integer, desc: 'The ID of the user' + end + segment ':user_id' do + resource :impersonation_tokens do + helpers do + def finder(options = {}) + user = find_user(params) + PersonalAccessTokensFinder.new({ user: user, impersonation: true }.merge(options)) + end + + def find_impersonation_token + finder.find_by(id: declared_params[:impersonation_token_id]) || not_found!('Impersonation Token') + end + end + + before { authenticated_as_admin! } + + desc 'Retrieve impersonation tokens. Available only for admins.' do + detail 'This feature was introduced in GitLab 9.0' + success Entities::ImpersonationToken + end + params do + use :pagination + optional :state, type: String, default: 'all', values: %w[all active inactive], desc: 'Filters (all|active|inactive) impersonation_tokens' + end + get { present paginate(finder(declared_params(include_missing: false)).execute), with: Entities::ImpersonationToken } + + desc 'Create a impersonation token. Available only for admins.' do + detail 'This feature was introduced in GitLab 9.0' + success Entities::ImpersonationToken + end + params do + requires :name, type: String, desc: 'The name of the impersonation token' + optional :expires_at, type: Date, desc: 'The expiration date in the format YEAR-MONTH-DAY of the impersonation token' + optional :scopes, type: Array, desc: 'The array of scopes of the impersonation token' + end + post do + impersonation_token = finder.build(declared_params(include_missing: false)) + + if impersonation_token.save + present impersonation_token, with: Entities::ImpersonationToken + else + render_validation_error!(impersonation_token) + end + end + + desc 'Retrieve impersonation token. Available only for admins.' do + detail 'This feature was introduced in GitLab 9.0' + success Entities::ImpersonationToken + end + params do + requires :impersonation_token_id, type: Integer, desc: 'The ID of the impersonation token' + end + get ':impersonation_token_id' do + present find_impersonation_token, with: Entities::ImpersonationToken + end + + desc 'Revoke a impersonation token. Available only for admins.' do + detail 'This feature was introduced in GitLab 9.0' + end + params do + requires :impersonation_token_id, type: Integer, desc: 'The ID of the impersonation token' + end + delete ':impersonation_token_id' do + find_impersonation_token.revoke! + end + end + end end resource :user do diff --git a/lib/api/v3/award_emoji.rb b/lib/api/v3/award_emoji.rb index 1e35283631f..cf9e1551f60 100644 --- a/lib/api/v3/award_emoji.rb +++ b/lib/api/v3/award_emoji.rb @@ -16,11 +16,64 @@ module API requires :"#{awardable_id_string}", type: Integer, desc: "The ID of an Issue, Merge Request or Snippet" end - [":id/#{awardable_string}/:#{awardable_id_string}/award_emoji", - ":id/#{awardable_string}/:#{awardable_id_string}/notes/:note_id/award_emoji"].each do |endpoint| + [ + ":id/#{awardable_string}/:#{awardable_id_string}/award_emoji", + ":id/#{awardable_string}/:#{awardable_id_string}/notes/:note_id/award_emoji" + ].each do |endpoint| + + desc 'Get a list of project +awardable+ award emoji' do + detail 'This feature was introduced in 8.9' + success Entities::AwardEmoji + end + params do + use :pagination + end + get endpoint do + if can_read_awardable? + awards = awardable.award_emoji + present paginate(awards), with: Entities::AwardEmoji + else + not_found!("Award Emoji") + end + end + + desc 'Get a specific award emoji' do + detail 'This feature was introduced in 8.9' + success Entities::AwardEmoji + end + params do + requires :award_id, type: Integer, desc: 'The ID of the award' + end + get "#{endpoint}/:award_id" do + if can_read_awardable? + present awardable.award_emoji.find(params[:award_id]), with: Entities::AwardEmoji + else + not_found!("Award Emoji") + end + end + + desc 'Award a new Emoji' do + detail 'This feature was introduced in 8.9' + success Entities::AwardEmoji + end + params do + requires :name, type: String, desc: 'The name of a award_emoji (without colons)' + end + post endpoint do + not_found!('Award Emoji') unless can_read_awardable? && can_award_awardable? + + award = awardable.create_award_emoji(params[:name], current_user) + + if award.persisted? + present award, with: Entities::AwardEmoji + else + not_found!("Award Emoji #{award.errors.messages}") + end + end + desc 'Delete a +awardables+ award emoji' do detail 'This feature was introduced in 8.9' - success ::API::Entities::AwardEmoji + success Entities::AwardEmoji end params do requires :award_id, type: Integer, desc: 'The ID of an award emoji' @@ -30,13 +83,22 @@ module API unauthorized! unless award.user == current_user || current_user.admin? - present award.destroy, with: ::API::Entities::AwardEmoji + award.destroy + present award, with: Entities::AwardEmoji end end end end helpers do + def can_read_awardable? + can?(current_user, read_ability(awardable), awardable) + end + + def can_award_awardable? + awardable.user_can_award?(current_user, params[:name]) + end + def awardable @awardable ||= begin @@ -53,6 +115,15 @@ module API end end end + + def read_ability(awardable) + case awardable + when Note + read_ability(awardable.noteable) + else + :"read_#{awardable.class.to_s.underscore}" + end + end end end end diff --git a/lib/api/v3/builds.rb b/lib/api/v3/builds.rb new file mode 100644 index 00000000000..6f97102c6ef --- /dev/null +++ b/lib/api/v3/builds.rb @@ -0,0 +1,255 @@ +module API + module V3 + class Builds < Grape::API + include PaginationParams + + before { authenticate! } + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects do + helpers do + params :optional_scope do + optional :scope, types: [String, Array[String]], desc: 'The scope of builds to show', + values: %w(pending running failed success canceled skipped), + coerce_with: ->(scope) { + if scope.is_a?(String) + [scope] + elsif scope.is_a?(Hashie::Mash) + scope.values + else + ['unknown'] + end + } + end + end + + desc 'Get a project builds' do + success ::API::V3::Entities::Build + end + params do + use :optional_scope + use :pagination + end + get ':id/builds' do + builds = user_project.builds.order('id DESC') + builds = filter_builds(builds, params[:scope]) + + present paginate(builds), with: ::API::V3::Entities::Build + end + + desc 'Get builds for a specific commit of a project' do + success ::API::V3::Entities::Build + end + params do + requires :sha, type: String, desc: 'The SHA id of a commit' + use :optional_scope + use :pagination + end + get ':id/repository/commits/:sha/builds' do + authorize_read_builds! + + return not_found! unless user_project.commit(params[:sha]) + + pipelines = user_project.pipelines.where(sha: params[:sha]) + builds = user_project.builds.where(pipeline: pipelines).order('id DESC') + builds = filter_builds(builds, params[:scope]) + + present paginate(builds), with: ::API::V3::Entities::Build + end + + desc 'Get a specific build of a project' do + success ::API::V3::Entities::Build + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end + get ':id/builds/:build_id' do + authorize_read_builds! + + build = get_build!(params[:build_id]) + + present build, with: ::API::V3::Entities::Build + end + + desc 'Download the artifacts file from build' do + detail 'This feature was introduced in GitLab 8.5' + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end + get ':id/builds/:build_id/artifacts' do + authorize_read_builds! + + build = get_build!(params[:build_id]) + + present_artifacts!(build.artifacts_file) + end + + desc 'Download the artifacts file from build' do + detail 'This feature was introduced in GitLab 8.10' + end + params do + requires :ref_name, type: String, desc: 'The ref from repository' + requires :job, type: String, desc: 'The name for the build' + end + get ':id/builds/artifacts/:ref_name/download', + requirements: { ref_name: /.+/ } do + authorize_read_builds! + + builds = user_project.latest_successful_builds_for(params[:ref_name]) + latest_build = builds.find_by!(name: params[:job]) + + present_artifacts!(latest_build.artifacts_file) + end + + # TODO: We should use `present_file!` and leave this implementation for backward compatibility (when build trace + # is saved in the DB instead of file). But before that, we need to consider how to replace the value of + # `runners_token` with some mask (like `xxxxxx`) when sending trace file directly by workhorse. + desc 'Get a trace of a specific build of a project' + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end + get ':id/builds/:build_id/trace' do + authorize_read_builds! + + build = get_build!(params[:build_id]) + + header 'Content-Disposition', "infile; filename=\"#{build.id}.log\"" + content_type 'text/plain' + env['api.format'] = :binary + + trace = build.trace + body trace + end + + desc 'Cancel a specific build of a project' do + success ::API::V3::Entities::Build + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end + post ':id/builds/:build_id/cancel' do + authorize_update_builds! + + build = get_build!(params[:build_id]) + + build.cancel + + present build, with: ::API::V3::Entities::Build + end + + desc 'Retry a specific build of a project' do + success ::API::V3::Entities::Build + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end + post ':id/builds/:build_id/retry' do + authorize_update_builds! + + build = get_build!(params[:build_id]) + return forbidden!('Build is not retryable') unless build.retryable? + + build = Ci::Build.retry(build, current_user) + + present build, with: ::API::V3::Entities::Build + end + + desc 'Erase build (remove artifacts and build trace)' do + success ::API::V3::Entities::Build + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end + post ':id/builds/:build_id/erase' do + authorize_update_builds! + + build = get_build!(params[:build_id]) + return forbidden!('Build is not erasable!') unless build.erasable? + + build.erase(erased_by: current_user) + present build, with: ::API::V3::Entities::Build + end + + desc 'Keep the artifacts to prevent them from being deleted' do + success ::API::V3::Entities::Build + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end + post ':id/builds/:build_id/artifacts/keep' do + authorize_update_builds! + + build = get_build!(params[:build_id]) + return not_found!(build) unless build.artifacts? + + build.keep_artifacts! + + status 200 + present build, with: ::API::V3::Entities::Build + end + + desc 'Trigger a manual build' do + success ::API::V3::Entities::Build + detail 'This feature was added in GitLab 8.11' + end + params do + requires :build_id, type: Integer, desc: 'The ID of a Build' + end + post ":id/builds/:build_id/play" do + authorize_read_builds! + + build = get_build!(params[:build_id]) + + bad_request!("Unplayable Job") unless build.playable? + + build.play(current_user) + + status 200 + present build, with: ::API::V3::Entities::Build + end + end + + helpers do + def get_build(id) + user_project.builds.find_by(id: id.to_i) + end + + def get_build!(id) + get_build(id) || not_found! + end + + def present_artifacts!(artifacts_file) + if !artifacts_file.file_storage? + redirect_to(build.artifacts_file.url) + elsif artifacts_file.exists? + present_file!(artifacts_file.path, artifacts_file.filename) + else + not_found! + end + end + + def filter_builds(builds, scope) + return builds if scope.nil? || scope.empty? + + available_statuses = ::CommitStatus::AVAILABLE_STATUSES + + unknown = scope - available_statuses + render_api_error!('Scope contains invalid value(s)', 400) unless unknown.empty? + + builds.where(status: available_statuses && scope) + end + + def authorize_read_builds! + authorize! :read_build, user_project + end + + def authorize_update_builds! + authorize! :update_build, user_project + end + end + end + end +end diff --git a/lib/api/v3/commits.rb b/lib/api/v3/commits.rb index 506204b3517..d254d247042 100644 --- a/lib/api/v3/commits.rb +++ b/lib/api/v3/commits.rb @@ -130,9 +130,7 @@ module API commit_params = { commit: commit, - create_merge_request: false, - source_project: user_project, - source_branch: commit.cherry_pick_branch_name, + start_branch: params[:branch], target_branch: params[:branch] } diff --git a/lib/api/v3/deployments.rb b/lib/api/v3/deployments.rb new file mode 100644 index 00000000000..95114ad1fe1 --- /dev/null +++ b/lib/api/v3/deployments.rb @@ -0,0 +1,43 @@ +module API + module V3 + # Deployments RESTful API endpoints + class Deployments < Grape::API + include PaginationParams + + before { authenticate! } + + params do + requires :id, type: String, desc: 'The project ID' + end + resource :projects do + desc 'Get all deployments of the project' do + detail 'This feature was introduced in GitLab 8.11.' + success ::API::V3::Deployments + end + params do + use :pagination + end + get ':id/deployments' do + authorize! :read_deployment, user_project + + present paginate(user_project.deployments), with: ::API::V3::Deployments + end + + desc 'Gets a specific deployment' do + detail 'This feature was introduced in GitLab 8.11.' + success ::API::V3::Deployments + end + params do + requires :deployment_id, type: Integer, desc: 'The deployment ID' + end + get ':id/deployments/:deployment_id' do + authorize! :read_deployment, user_project + + deployment = user_project.deployments.find(params[:deployment_id]) + + present deployment, with: ::API::V3::Deployments + end + end + end + end +end diff --git a/lib/api/v3/entities.rb b/lib/api/v3/entities.rb index 270d99a2348..832b4bdeb4f 100644 --- a/lib/api/v3/entities.rb +++ b/lib/api/v3/entities.rb @@ -81,7 +81,7 @@ module API expose :request_access_enabled expose :only_allow_merge_if_all_discussions_are_resolved - expose :statistics, using: 'API::Entities::ProjectStatistics', if: :statistics + expose :statistics, using: '::API::V3::Entities::ProjectStatistics', if: :statistics end class ProjectWithAccess < Project @@ -186,6 +186,68 @@ module API class Environment < ::API::Entities::EnvironmentBasic expose :project, using: Entities::Project end + + class Trigger < Grape::Entity + expose :token, :created_at, :updated_at, :deleted_at, :last_used + expose :owner, using: ::API::Entities::UserBasic + end + + class TriggerRequest < Grape::Entity + expose :id, :variables + end + + class Build < Grape::Entity + expose :id, :status, :stage, :name, :ref, :tag, :coverage + expose :created_at, :started_at, :finished_at + expose :user, with: ::API::Entities::User + expose :artifacts_file, using: ::API::Entities::JobArtifactFile, if: -> (build, opts) { build.artifacts? } + expose :commit, with: ::API::Entities::RepoCommit + expose :runner, with: ::API::Entities::Runner + expose :pipeline, with: ::API::Entities::PipelineBasic + end + + class BuildArtifactFile < Grape::Entity + expose :filename, :size + end + + class Deployment < Grape::Entity + expose :id, :iid, :ref, :sha, :created_at + expose :user, using: ::API::Entities::UserBasic + expose :environment, using: ::API::Entities::EnvironmentBasic + expose :deployable, using: Entities::Build + end + + class MergeRequestChanges < MergeRequest + expose :diffs, as: :changes, using: ::API::Entities::RepoDiff do |compare, _| + compare.raw_diffs(all_diffs: true).to_a + end + end + + class ProjectStatistics < Grape::Entity + expose :commit_count + expose :storage_size + expose :repository_size + expose :lfs_objects_size + expose :build_artifacts_size + end + + class ProjectService < Grape::Entity + expose :id, :title, :created_at, :updated_at, :active + expose :push_events, :issues_events, :merge_requests_events + expose :tag_push_events, :note_events, :build_events, :pipeline_events + # Expose serialized properties + expose :properties do |service, options| + field_names = service.fields. + select { |field| options[:include_passwords] || field[:type] != 'password' }. + map { |field| field[:name] } + service.properties.slice(*field_names) + end + end + + class ProjectHook < ::API::Entities::Hook + expose :project_id, :issues_events, :merge_requests_events + expose :note_events, :build_events, :pipeline_events, :wiki_page_events + end end end end diff --git a/lib/api/v3/helpers.rb b/lib/api/v3/helpers.rb new file mode 100644 index 00000000000..0f234d4cdad --- /dev/null +++ b/lib/api/v3/helpers.rb @@ -0,0 +1,19 @@ +module API + module V3 + module Helpers + def find_project_issue(id) + IssuesFinder.new(current_user, project_id: user_project.id).find(id) + end + + def find_project_merge_request(id) + MergeRequestsFinder.new(current_user, project_id: user_project.id).find(id) + end + + def find_merge_request_with_access(id, access_level = :read_merge_request) + merge_request = user_project.merge_requests.find(id) + authorize! access_level, merge_request + merge_request + end + end + end +end diff --git a/lib/api/v3/merge_request_diffs.rb b/lib/api/v3/merge_request_diffs.rb new file mode 100644 index 00000000000..a462803e26c --- /dev/null +++ b/lib/api/v3/merge_request_diffs.rb @@ -0,0 +1,43 @@ +module API + module V3 + # MergeRequestDiff API + class MergeRequestDiffs < Grape::API + before { authenticate! } + + resource :projects do + desc 'Get a list of merge request diff versions' do + detail 'This feature was introduced in GitLab 8.12.' + success ::API::Entities::MergeRequestDiff + end + + params do + requires :id, type: String, desc: 'The ID of a project' + requires :merge_request_id, type: Integer, desc: 'The ID of a merge request' + end + + get ":id/merge_requests/:merge_request_id/versions" do + merge_request = find_merge_request_with_access(params[:merge_request_id]) + + present merge_request.merge_request_diffs, with: ::API::Entities::MergeRequestDiff + end + + desc 'Get a single merge request diff version' do + detail 'This feature was introduced in GitLab 8.12.' + success ::API::Entities::MergeRequestDiffFull + end + + params do + requires :id, type: String, desc: 'The ID of a project' + requires :merge_request_id, type: Integer, desc: 'The ID of a merge request' + requires :version_id, type: Integer, desc: 'The ID of a merge request diff version' + end + + get ":id/merge_requests/:merge_request_id/versions/:version_id" do + merge_request = find_merge_request_with_access(params[:merge_request_id]) + + present merge_request.merge_request_diffs.find(params[:version_id]), with: ::API::Entities::MergeRequestDiffFull + end + end + end + end +end diff --git a/lib/api/v3/merge_requests.rb b/lib/api/v3/merge_requests.rb index 654e818e1b5..7dbd4691a94 100644 --- a/lib/api/v3/merge_requests.rb +++ b/lib/api/v3/merge_requests.rb @@ -28,6 +28,14 @@ module API render_api_error!(errors, 400) end + def issue_entity(project) + if project.has_external_issue_tracker? + ::API::Entities::ExternalIssue + else + ::API::Entities::Issue + end + end + params :optional_params do optional :description, type: String, desc: 'The description of the merge request' optional :assignee_id, type: Integer, desc: 'The ID of a user to assign the merge request' diff --git a/lib/api/v3/milestones.rb b/lib/api/v3/milestones.rb index bbc29c40ee2..2a850a08a8a 100644 --- a/lib/api/v3/milestones.rb +++ b/lib/api/v3/milestones.rb @@ -37,6 +37,27 @@ module API present paginate(milestones), with: ::API::Entities::Milestone end + + desc 'Get all issues for a single project milestone' do + success ::API::Entities::Issue + end + params do + requires :milestone_id, type: Integer, desc: 'The ID of a project milestone' + use :pagination + end + get ':id/milestones/:milestone_id/issues' do + authorize! :read_milestone, user_project + + milestone = user_project.milestones.find(params[:milestone_id]) + + finder_params = { + project_id: user_project.id, + milestone_title: milestone.title + } + + issues = IssuesFinder.new(current_user, finder_params).execute + present paginate(issues), with: ::API::Entities::Issue, current_user: current_user, project: user_project + end end end end diff --git a/lib/api/v3/pipelines.rb b/lib/api/v3/pipelines.rb new file mode 100644 index 00000000000..2c26a5f7d35 --- /dev/null +++ b/lib/api/v3/pipelines.rb @@ -0,0 +1,36 @@ +module API + module V3 + class Pipelines < Grape::API + include PaginationParams + + before { authenticate! } + + params do + requires :id, type: String, desc: 'The project ID' + end + resource :projects do + desc 'Get all Pipelines of the project' do + detail 'This feature was introduced in GitLab 8.11.' + success ::API::Entities::Pipeline + end + params do + use :pagination + optional :scope, type: String, values: %w(running branches tags), + desc: 'Either running, branches, or tags' + end + get ':id/pipelines' do + authorize! :read_pipeline, user_project + + pipelines = PipelinesFinder.new(user_project).execute(scope: params[:scope]) + present paginate(pipelines), with: ::API::Entities::Pipeline + end + end + + helpers do + def pipeline + @pipeline ||= user_project.pipelines.find(params[:pipeline_id]) + end + end + end + end +end diff --git a/lib/api/v3/project_hooks.rb b/lib/api/v3/project_hooks.rb new file mode 100644 index 00000000000..861b991b8e1 --- /dev/null +++ b/lib/api/v3/project_hooks.rb @@ -0,0 +1,106 @@ +module API + module V3 + class ProjectHooks < Grape::API + include PaginationParams + + before { authenticate! } + before { authorize_admin_project } + + helpers do + params :project_hook_properties do + requires :url, type: String, desc: "The URL to send the request to" + optional :push_events, type: Boolean, desc: "Trigger hook on push events" + optional :issues_events, type: Boolean, desc: "Trigger hook on issues events" + optional :merge_requests_events, type: Boolean, desc: "Trigger hook on merge request events" + optional :tag_push_events, type: Boolean, desc: "Trigger hook on tag push events" + optional :note_events, type: Boolean, desc: "Trigger hook on note(comment) events" + optional :build_events, type: Boolean, desc: "Trigger hook on build events" + optional :pipeline_events, type: Boolean, desc: "Trigger hook on pipeline events" + optional :wiki_page_events, type: Boolean, desc: "Trigger hook on wiki events" + optional :enable_ssl_verification, type: Boolean, desc: "Do SSL verification when triggering the hook" + optional :token, type: String, desc: "Secret token to validate received payloads; this will not be returned in the response" + end + end + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects do + desc 'Get project hooks' do + success ::API::V3::Entities::ProjectHook + end + params do + use :pagination + end + get ":id/hooks" do + hooks = paginate user_project.hooks + + present hooks, with: ::API::V3::Entities::ProjectHook + end + + desc 'Get a project hook' do + success ::API::V3::Entities::ProjectHook + end + params do + requires :hook_id, type: Integer, desc: 'The ID of a project hook' + end + get ":id/hooks/:hook_id" do + hook = user_project.hooks.find(params[:hook_id]) + present hook, with: ::API::V3::Entities::ProjectHook + end + + desc 'Add hook to project' do + success ::API::V3::Entities::ProjectHook + end + params do + use :project_hook_properties + end + post ":id/hooks" do + hook = user_project.hooks.new(declared_params(include_missing: false)) + + if hook.save + present hook, with: ::API::V3::Entities::ProjectHook + else + error!("Invalid url given", 422) if hook.errors[:url].present? + + not_found!("Project hook #{hook.errors.messages}") + end + end + + desc 'Update an existing project hook' do + success ::API::V3::Entities::ProjectHook + end + params do + requires :hook_id, type: Integer, desc: "The ID of the hook to update" + use :project_hook_properties + end + put ":id/hooks/:hook_id" do + hook = user_project.hooks.find(params.delete(:hook_id)) + + if hook.update_attributes(declared_params(include_missing: false)) + present hook, with: ::API::V3::Entities::ProjectHook + else + error!("Invalid url given", 422) if hook.errors[:url].present? + + not_found!("Project hook #{hook.errors.messages}") + end + end + + desc 'Deletes project hook' do + success ::API::V3::Entities::ProjectHook + end + params do + requires :hook_id, type: Integer, desc: 'The ID of the hook to delete' + end + delete ":id/hooks/:hook_id" do + begin + present user_project.hooks.destroy(params[:hook_id]), with: ::API::V3::Entities::ProjectHook + rescue + # ProjectHook can raise Error if hook_id not found + not_found!("Error deleting hook #{params[:hook_id]}") + end + end + end + end + end +end diff --git a/lib/api/v3/repositories.rb b/lib/api/v3/repositories.rb index 3549ea225ef..44584e2eb70 100644 --- a/lib/api/v3/repositories.rb +++ b/lib/api/v3/repositories.rb @@ -38,6 +38,60 @@ module API present tree.sorted_entries, with: ::API::Entities::RepoTreeObject end + desc 'Get a raw file contents' + params do + requires :sha, type: String, desc: 'The commit, branch name, or tag name' + requires :filepath, type: String, desc: 'The path to the file to display' + end + get [":id/repository/blobs/:sha", ":id/repository/commits/:sha/blob"] do + repo = user_project.repository + commit = repo.commit(params[:sha]) + not_found! "Commit" unless commit + blob = Gitlab::Git::Blob.find(repo, commit.id, params[:filepath]) + not_found! "File" unless blob + send_git_blob repo, blob + end + + desc 'Get a raw blob contents by blob sha' + params do + requires :sha, type: String, desc: 'The commit, branch name, or tag name' + end + get ':id/repository/raw_blobs/:sha' do + repo = user_project.repository + begin + blob = Gitlab::Git::Blob.raw(repo, params[:sha]) + rescue + not_found! 'Blob' + end + not_found! 'Blob' unless blob + send_git_blob repo, blob + end + + desc 'Get an archive of the repository' + params do + optional :sha, type: String, desc: 'The commit sha of the archive to be downloaded' + optional :format, type: String, desc: 'The archive format' + end + get ':id/repository/archive', requirements: { format: Gitlab::Regex.archive_formats_regex } do + begin + send_git_archive user_project.repository, ref: params[:sha], format: params[:format] + rescue + not_found!('File') + end + end + + desc 'Compare two branches, tags, or commits' do + success ::API::Entities::Compare + end + params do + requires :from, type: String, desc: 'The commit, branch name, or tag name to start comparison' + requires :to, type: String, desc: 'The commit, branch name, or tag name to stop comparison' + end + get ':id/repository/compare' do + compare = Gitlab::Git::Compare.new(user_project.repository.raw_repository, params[:from], params[:to]) + present compare, with: ::API::Entities::Compare + end + desc 'Get repository contributors' do success ::API::Entities::Contributor end diff --git a/lib/api/v3/services.rb b/lib/api/v3/services.rb index af0a058f69b..d77185ffe5a 100644 --- a/lib/api/v3/services.rb +++ b/lib/api/v3/services.rb @@ -537,6 +537,23 @@ module API ] } + trigger_services = { + 'mattermost-slash-commands' => [ + { + name: :token, + type: String, + desc: 'The Mattermost token' + } + ], + 'slack-slash-commands' => [ + { + name: :token, + type: String, + desc: 'The Slack token' + } + ] + }.freeze + resource :projects do before { authenticate! } before { authorize_admin_project } @@ -567,6 +584,57 @@ module API render_api_error!('400 Bad Request', 400) end end + + desc 'Get the service settings for project' do + success Entities::ProjectService + end + params do + requires :service_slug, type: String, values: services.keys, desc: 'The name of the service' + end + get ":id/services/:service_slug" do + service = user_project.find_or_initialize_service(params[:service_slug].underscore) + present service, with: Entities::ProjectService, include_passwords: current_user.is_admin? + end + end + + trigger_services.each do |service_slug, settings| + helpers do + def chat_command_service(project, service_slug, params) + project.services.active.where(template: false).find do |service| + service.try(:token) == params[:token] && service.to_param == service_slug.underscore + end + end + end + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects do + desc "Trigger a slash command for #{service_slug}" do + detail 'Added in GitLab 8.13' + end + params do + settings.each do |setting| + requires setting[:name], type: setting[:type], desc: setting[:desc] + end + end + post ":id/services/#{service_slug.underscore}/trigger" do + project = find_project(params[:id]) + + # This is not accurate, but done to prevent leakage of the project names + not_found!('Service') unless project + + service = chat_command_service(project, service_slug, params) + result = service.try(:trigger, params) + + if result + status result[:status] || 200 + present result + else + not_found!('Service') + end + end + end end end end diff --git a/lib/api/v3/time_tracking_endpoints.rb b/lib/api/v3/time_tracking_endpoints.rb new file mode 100644 index 00000000000..81ae4e8137d --- /dev/null +++ b/lib/api/v3/time_tracking_endpoints.rb @@ -0,0 +1,116 @@ +module API + module V3 + module TimeTrackingEndpoints + extend ActiveSupport::Concern + + included do + helpers do + def issuable_name + declared_params.has_key?(:issue_id) ? 'issue' : 'merge_request' + end + + def issuable_key + "#{issuable_name}_id".to_sym + end + + def update_issuable_key + "update_#{issuable_name}".to_sym + end + + def read_issuable_key + "read_#{issuable_name}".to_sym + end + + def load_issuable + @issuable ||= begin + case issuable_name + when 'issue' + find_project_issue(params.delete(issuable_key)) + when 'merge_request' + find_project_merge_request(params.delete(issuable_key)) + end + end + end + + def update_issuable(attrs) + custom_params = declared_params(include_missing: false) + custom_params.merge!(attrs) + + issuable = update_service.new(user_project, current_user, custom_params).execute(load_issuable) + if issuable.valid? + present issuable, with: ::API::Entities::IssuableTimeStats + else + render_validation_error!(issuable) + end + end + + def update_service + issuable_name == 'issue' ? ::Issues::UpdateService : ::MergeRequests::UpdateService + end + end + + issuable_name = name.end_with?('Issues') ? 'issue' : 'merge_request' + issuable_collection_name = issuable_name.pluralize + issuable_key = "#{issuable_name}_id".to_sym + + desc "Set a time estimate for a project #{issuable_name}" + params do + requires issuable_key, type: Integer, desc: "The ID of a project #{issuable_name}" + requires :duration, type: String, desc: 'The duration to be parsed' + end + post ":id/#{issuable_collection_name}/:#{issuable_key}/time_estimate" do + authorize! update_issuable_key, load_issuable + + status :ok + update_issuable(time_estimate: Gitlab::TimeTrackingFormatter.parse(params.delete(:duration))) + end + + desc "Reset the time estimate for a project #{issuable_name}" + params do + requires issuable_key, type: Integer, desc: "The ID of a project #{issuable_name}" + end + post ":id/#{issuable_collection_name}/:#{issuable_key}/reset_time_estimate" do + authorize! update_issuable_key, load_issuable + + status :ok + update_issuable(time_estimate: 0) + end + + desc "Add spent time for a project #{issuable_name}" + params do + requires issuable_key, type: Integer, desc: "The ID of a project #{issuable_name}" + requires :duration, type: String, desc: 'The duration to be parsed' + end + post ":id/#{issuable_collection_name}/:#{issuable_key}/add_spent_time" do + authorize! update_issuable_key, load_issuable + + update_issuable(spend_time: { + duration: Gitlab::TimeTrackingFormatter.parse(params.delete(:duration)), + user: current_user + }) + end + + desc "Reset spent time for a project #{issuable_name}" + params do + requires issuable_key, type: Integer, desc: "The ID of a project #{issuable_name}" + end + post ":id/#{issuable_collection_name}/:#{issuable_key}/reset_spent_time" do + authorize! update_issuable_key, load_issuable + + status :ok + update_issuable(spend_time: { duration: :reset, user: current_user }) + end + + desc "Show time stats for a project #{issuable_name}" + params do + requires issuable_key, type: Integer, desc: "The ID of a project #{issuable_name}" + end + get ":id/#{issuable_collection_name}/:#{issuable_key}/time_stats" do + authorize! read_issuable_key, load_issuable + + present load_issuable, with: ::API::Entities::IssuableTimeStats + end + end + end + end +end diff --git a/lib/api/v3/triggers.rb b/lib/api/v3/triggers.rb index 4051d4bca8d..1dfdb6a5956 100644 --- a/lib/api/v3/triggers.rb +++ b/lib/api/v3/triggers.rb @@ -7,8 +7,81 @@ module API requires :id, type: String, desc: 'The ID of a project' end resource :projects do + desc 'Trigger a GitLab project build' do + success ::API::V3::Entities::TriggerRequest + end + params do + requires :ref, type: String, desc: 'The commit sha or name of a branch or tag' + requires :token, type: String, desc: 'The unique token of trigger' + optional :variables, type: Hash, desc: 'The list of variables to be injected into build' + end + post ":id/(ref/:ref/)trigger/builds" do + project = find_project(params[:id]) + trigger = Ci::Trigger.find_by_token(params[:token].to_s) + not_found! unless project && trigger + unauthorized! unless trigger.project == project + + # validate variables + variables = params[:variables].to_h + unless variables.all? { |key, value| key.is_a?(String) && value.is_a?(String) } + render_api_error!('variables needs to be a map of key-valued strings', 400) + end + + # create request and trigger builds + trigger_request = Ci::CreateTriggerRequestService.new.execute(project, trigger, params[:ref].to_s, variables) + if trigger_request + present trigger_request, with: ::API::V3::Entities::TriggerRequest + else + errors = 'No builds created' + render_api_error!(errors, 400) + end + end + + desc 'Get triggers list' do + success ::API::V3::Entities::Trigger + end + params do + use :pagination + end + get ':id/triggers' do + authenticate! + authorize! :admin_build, user_project + + triggers = user_project.triggers.includes(:trigger_requests) + + present paginate(triggers), with: ::API::V3::Entities::Trigger + end + + desc 'Get specific trigger of a project' do + success ::API::V3::Entities::Trigger + end + params do + requires :token, type: String, desc: 'The unique token of trigger' + end + get ':id/triggers/:token' do + authenticate! + authorize! :admin_build, user_project + + trigger = user_project.triggers.find_by(token: params[:token].to_s) + return not_found!('Trigger') unless trigger + + present trigger, with: ::API::V3::Entities::Trigger + end + + desc 'Create a trigger' do + success ::API::V3::Entities::Trigger + end + post ':id/triggers' do + authenticate! + authorize! :admin_build, user_project + + trigger = user_project.triggers.create + + present trigger, with: ::API::V3::Entities::Trigger + end + desc 'Delete a trigger' do - success ::API::Entities::Trigger + success ::API::V3::Entities::Trigger end params do requires :token, type: String, desc: 'The unique token of trigger' @@ -22,7 +95,7 @@ module API trigger.destroy - present trigger, with: ::API::Entities::Trigger + present trigger, with: ::API::V3::Entities::Trigger end end end diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 5cc164a6325..7b4476fa4db 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -51,7 +51,8 @@ module Backup if directory.files.create(key: tar_file, body: File.open(tar_file), public: false, multipart_chunk_size: Gitlab.config.backup.upload.multipart_chunk_size, - encryption: Gitlab.config.backup.upload.encryption) + encryption: Gitlab.config.backup.upload.encryption, + storage_class: Gitlab.config.backup.upload.storage_class) $progress.puts "done".color(:green) else puts "uploading backup to #{remote_directory} failed".color(:red) diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index d16d5ba4960..cd745d35e7c 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -68,7 +68,8 @@ module Backup end def restore - Gitlab.config.repositories.storages.each do |name, path| + Gitlab.config.repositories.storages.each do |name, repository_storage| + path = repository_storage['path'] next unless File.exist?(path) # Move repos dir to 'repositories.old' dir @@ -180,9 +181,8 @@ module Backup return unless Dir.exist?(path) dir_entries = Dir.entries(path) - %w[annex custom_hooks].each do |entry| - yield(entry) if dir_entries.include?(entry) - end + + yield('custom_hooks') if dir_entries.include?('custom_hooks') end def prepare @@ -200,7 +200,7 @@ module Backup private def repository_storage_paths_args - Gitlab.config.repositories.storages.values + Gitlab.config.repositories.storages.values.map { |rs| rs['path'] } end end end diff --git a/lib/banzai/filter/emoji_filter.rb b/lib/banzai/filter/emoji_filter.rb index a8c1ca0c60a..d6138816e70 100644 --- a/lib/banzai/filter/emoji_filter.rb +++ b/lib/banzai/filter/emoji_filter.rb @@ -17,8 +17,8 @@ module Banzai next unless content.include?(':') || node.text.match(emoji_unicode_pattern) - html = emoji_name_image_filter(content) - html = emoji_unicode_image_filter(html) + html = emoji_unicode_element_unicode_filter(content) + html = emoji_name_element_unicode_filter(html) next if html == content @@ -27,33 +27,30 @@ module Banzai doc end - # Replace :emoji: with corresponding images. + # Replace :emoji: with corresponding gl-emoji unicode. # # text - String text to replace :emoji: in. # - # Returns a String with :emoji: replaced with images. - def emoji_name_image_filter(text) + # Returns a String with :emoji: replaced with gl-emoji unicode. + def emoji_name_element_unicode_filter(text) text.gsub(emoji_pattern) do |match| name = $1 - emoji_image_tag(name, emoji_url(name)) + Gitlab::Emoji.gl_emoji_tag(name) end end - # Replace unicode emoji with corresponding images if they exist. + # Replace unicode emoji with corresponding gl-emoji unicode. # # text - String text to replace unicode emoji in. # - # Returns a String with unicode emoji replaced with images. - def emoji_unicode_image_filter(text) + # Returns a String with unicode emoji replaced with gl-emoji unicode. + def emoji_unicode_element_unicode_filter(text) text.gsub(emoji_unicode_pattern) do |moji| - emoji_image_tag(Gitlab::Emoji.emojis_by_moji[moji]['name'], emoji_unicode_url(moji)) + emoji_info = Gitlab::Emoji.emojis_by_moji[moji] + Gitlab::Emoji.gl_emoji_tag(emoji_info['name']) end end - def emoji_image_tag(emoji_name, emoji_url) - "<img class='emoji' title=':#{emoji_name}:' alt=':#{emoji_name}:' src='#{emoji_url}' height='20' width='20' align='absmiddle' />" - end - # Build a regexp that matches all valid :emoji: names. def self.emoji_pattern @emoji_pattern ||= /:(#{Gitlab::Emoji.emojis_names.map { |name| Regexp.escape(name) }.join('|')}):/ @@ -66,52 +63,13 @@ module Banzai private - def emoji_url(name) - emoji_path = emoji_filename(name) - - if context[:asset_host] - # Asset host is specified. - url_to_image(emoji_path) - elsif context[:asset_root] - # Gitlab url is specified - File.join(context[:asset_root], url_to_image(emoji_path)) - else - # All other cases - url_to_image(emoji_path) - end - end - - def emoji_unicode_url(moji) - emoji_unicode_path = emoji_unicode_filename(moji) - - if context[:asset_host] - url_to_image(emoji_unicode_path) - elsif context[:asset_root] - File.join(context[:asset_root], url_to_image(emoji_unicode_path)) - else - url_to_image(emoji_unicode_path) - end - end - - def url_to_image(image) - ActionController::Base.helpers.url_to_image(image) - end - def emoji_pattern self.class.emoji_pattern end - def emoji_filename(name) - "#{Gitlab::Emoji.emoji_filename(name)}.png" - end - def emoji_unicode_pattern self.class.emoji_unicode_pattern end - - def emoji_unicode_filename(name) - "#{Gitlab::Emoji.emoji_unicode_filename(name)}.png" - end end end end diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index af1e575fc89..d5f9e252f62 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -35,6 +35,10 @@ module Banzai # Allow span elements whitelist[:elements].push('span') + # Allow html5 details/summary elements + whitelist[:elements].push('details') + whitelist[:elements].push('summary') + # Allow abbr elements with title attribute whitelist[:elements].push('abbr') whitelist[:attributes]['abbr'] = %w(title) diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index b51e76d93f2..746e76a1b1f 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -24,7 +24,7 @@ module Ci new_update = current_runner.ensure_runner_queue_value - result = Ci::RegisterBuildService.new(current_runner).execute + result = Ci::RegisterJobService.new(current_runner).execute if result.valid? if result.build diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index e390919ae1d..15a461a16dd 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -58,7 +58,7 @@ module Ci commands: job[:commands], tag_list: job[:tags] || [], name: job[:name].to_s, - allow_failure: job[:allow_failure] || false, + allow_failure: job[:ignore], when: job[:when] || 'on_success', environment: job[:environment_name], coverage_regex: job[:coverage], diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 0a5abc92190..eee5601b0ed 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -2,9 +2,17 @@ module Gitlab module Auth MissingPersonalTokenError = Class.new(StandardError) - SCOPES = [:api, :read_user].freeze + # Scopes used for GitLab API access + API_SCOPES = [:api, :read_user].freeze + + # Scopes used for OpenID Connect + OPENID_SCOPES = [:openid].freeze + + # Default scopes for OAuth applications that don't define their own DEFAULT_SCOPES = [:api].freeze - OPTIONAL_SCOPES = SCOPES - DEFAULT_SCOPES + + # Other available scopes + OPTIONAL_SCOPES = (API_SCOPES + OPENID_SCOPES - DEFAULT_SCOPES).freeze class << self def find_for_git_client(login, password, project:, ip:) @@ -18,27 +26,30 @@ module Gitlab build_access_token_check(login, password) || lfs_token_check(login, password) || oauth_access_token_check(login, password) || - personal_access_token_check(login, password) || user_with_password_for_git(login, password) || + personal_access_token_check(password) || Gitlab::Auth::Result.new rate_limit!(ip, success: result.success?, login: login) + Gitlab::Auth::UniqueIpsLimiter.limit_user!(result.actor) result end def find_with_user_password(login, password) - user = User.by_login(login) + Gitlab::Auth::UniqueIpsLimiter.limit_user! do + user = User.by_login(login) - # If no user is found, or it's an LDAP server, try LDAP. - # LDAP users are only authenticated via LDAP - if user.nil? || user.ldap_user? - # Second chance - try LDAP authentication - return nil unless Gitlab::LDAP::Config.enabled? + # If no user is found, or it's an LDAP server, try LDAP. + # LDAP users are only authenticated via LDAP + if user.nil? || user.ldap_user? + # Second chance - try LDAP authentication + return nil unless Gitlab::LDAP::Config.enabled? - Gitlab::LDAP::Authentication.login(login, password) - else - user if user.valid_password?(password) + Gitlab::LDAP::Authentication.login(login, password) + else + user if user.active? && user.valid_password?(password) + end end end @@ -102,14 +113,13 @@ module Gitlab end end - def personal_access_token_check(login, password) - if login && password - token = PersonalAccessToken.active.find_by_token(password) - validation = User.by_login(login) + def personal_access_token_check(password) + return unless password.present? - if valid_personal_access_token?(token, validation) - Gitlab::Auth::Result.new(validation, nil, :personal_token, full_authentication_abilities) - end + token = PersonalAccessTokensFinder.new(state: 'active').find_by(token: password) + + if token && valid_api_token?(token) + Gitlab::Auth::Result.new(token.user, nil, :personal_token, full_authentication_abilities) end end @@ -117,10 +127,6 @@ module Gitlab token && token.accessible? && valid_api_token?(token) end - def valid_personal_access_token?(token, user) - token && token.user == user && valid_api_token?(token) - end - def valid_api_token?(token) AccessTokenValidationService.new(token).include_any_scope?(['api']) end diff --git a/lib/gitlab/auth/too_many_ips.rb b/lib/gitlab/auth/too_many_ips.rb new file mode 100644 index 00000000000..ed862791551 --- /dev/null +++ b/lib/gitlab/auth/too_many_ips.rb @@ -0,0 +1,17 @@ +module Gitlab + module Auth + class TooManyIps < StandardError + attr_reader :user_id, :ip, :unique_ips_count + + def initialize(user_id, ip, unique_ips_count) + @user_id = user_id + @ip = ip + @unique_ips_count = unique_ips_count + end + + def message + "User #{user_id} from IP: #{ip} tried logging from too many ips: #{unique_ips_count}" + end + end + end +end diff --git a/lib/gitlab/auth/unique_ips_limiter.rb b/lib/gitlab/auth/unique_ips_limiter.rb new file mode 100644 index 00000000000..bf2239ca150 --- /dev/null +++ b/lib/gitlab/auth/unique_ips_limiter.rb @@ -0,0 +1,43 @@ +module Gitlab + module Auth + class UniqueIpsLimiter + USER_UNIQUE_IPS_PREFIX = 'user_unique_ips'.freeze + + class << self + def limit_user_id!(user_id) + if config.unique_ips_limit_enabled + ip = RequestContext.client_ip + unique_ips = update_and_return_ips_count(user_id, ip) + + raise TooManyIps.new(user_id, ip, unique_ips) if unique_ips > config.unique_ips_limit_per_user + end + end + + def limit_user!(user = nil) + user ||= yield if block_given? + limit_user_id!(user.id) unless user.nil? + user + end + + def config + Gitlab::CurrentSettings.current_application_settings + end + + def update_and_return_ips_count(user_id, ip) + time = Time.now.utc.to_i + key = "#{USER_UNIQUE_IPS_PREFIX}:#{user_id}" + + Gitlab::Redis.with do |redis| + unique_ips_count = nil + redis.multi do |r| + r.zadd(key, time, ip) + r.zremrangebyscore(key, 0, time - config.unique_ips_limit_time_window) + unique_ips_count = r.zcard(key) + end + unique_ips_count.value + end + end + end + end + end +end diff --git a/lib/gitlab/award_emoji.rb b/lib/gitlab/award_emoji.rb deleted file mode 100644 index 7555326d384..00000000000 --- a/lib/gitlab/award_emoji.rb +++ /dev/null @@ -1,84 +0,0 @@ -module Gitlab - class AwardEmoji - CATEGORIES = { - objects: "Objects", - travel: "Travel", - symbols: "Symbols", - nature: "Nature", - people: "People", - activity: "Activity", - flags: "Flags", - food: "Food" - }.with_indifferent_access - - def self.normalize_emoji_name(name) - aliases[name] || name - end - - def self.emoji_by_category - unless @emoji_by_category - @emoji_by_category = Hash.new { |h, key| h[key] = [] } - - emojis.each do |emoji_name, data| - data["name"] = emoji_name - - # Skip Fitzpatrick(tone) modifiers - next if data["category"] == "modifier" - - category = data["category"] - - @emoji_by_category[category] << data - end - - @emoji_by_category = @emoji_by_category.sort.to_h - end - - @emoji_by_category - end - - def self.emojis - @emojis ||= - begin - json_path = File.join(Rails.root, 'fixtures', 'emojis', 'index.json' ) - JSON.parse(File.read(json_path)) - end - end - - def self.aliases - @aliases ||= - begin - json_path = File.join(Rails.root, 'fixtures', 'emojis', 'aliases.json') - JSON.parse(File.read(json_path)) - end - end - - # Returns an Array of Emoji names and their asset URLs. - def self.urls - @urls ||= begin - path = File.join(Rails.root, 'fixtures', 'emojis', 'digests.json') - # Construct the full asset path ourselves because - # ActionView::Helpers::AssetUrlHelper.asset_url is slow for hundreds - # of entries since it has to do a lot of extra work (e.g. regexps). - prefix = Gitlab::Application.config.assets.prefix - digest = Gitlab::Application.config.assets.digest - base = - if defined?(Gitlab::Application.config.relative_url_root) && Gitlab::Application.config.relative_url_root - Gitlab::Application.config.relative_url_root - else - '' - end - - JSON.parse(File.read(path)).map do |hash| - fname = - if digest - "#{hash['unicode']}-#{hash['digest']}" - else - hash['unicode'] - end - - { name: hash['name'], path: File.join(base, prefix, "#{fname}.png") } - end - end - end - end -end diff --git a/lib/gitlab/ci/build/image.rb b/lib/gitlab/ci/build/image.rb new file mode 100644 index 00000000000..c62aeb60fa9 --- /dev/null +++ b/lib/gitlab/ci/build/image.rb @@ -0,0 +1,33 @@ +module Gitlab + module Ci + module Build + class Image + attr_reader :name + + class << self + def from_image(job) + image = Gitlab::Ci::Build::Image.new(job.options[:image]) + return unless image.valid? + image + end + + def from_services(job) + services = job.options[:services].to_a.map do |service| + Gitlab::Ci::Build::Image.new(service) + end + + services.select(&:valid?).compact + end + end + + def initialize(image) + @name = image + end + + def valid? + @name.present? + end + end + end + end +end diff --git a/lib/gitlab/ci/build/step.rb b/lib/gitlab/ci/build/step.rb new file mode 100644 index 00000000000..1877429ac46 --- /dev/null +++ b/lib/gitlab/ci/build/step.rb @@ -0,0 +1,46 @@ +module Gitlab + module Ci + module Build + class Step + WHEN_ON_FAILURE = 'on_failure'.freeze + WHEN_ON_SUCCESS = 'on_success'.freeze + WHEN_ALWAYS = 'always'.freeze + + attr_reader :name + attr_writer :script + attr_accessor :timeout, :when, :allow_failure + + class << self + def from_commands(job) + self.new(:script).tap do |step| + step.script = job.commands + step.timeout = job.timeout + step.when = WHEN_ON_SUCCESS + end + end + + def from_after_script(job) + after_script = job.options[:after_script] + return unless after_script + + self.new(:after_script).tap do |step| + step.script = after_script + step.timeout = job.timeout + step.when = WHEN_ALWAYS + step.allow_failure = true + end + end + end + + def initialize(name) + @name = name + @allow_failure = false + end + + def script + @script.split("\n") + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/cache.rb b/lib/gitlab/ci/config/entry/cache.rb index 066643ccfcc..f074df9c7a1 100644 --- a/lib/gitlab/ci/config/entry/cache.rb +++ b/lib/gitlab/ci/config/entry/cache.rb @@ -22,6 +22,12 @@ module Gitlab entry :paths, Entry::Paths, description: 'Specify which paths should be cached across builds.' + + helpers :key + + def value + super.merge(key: key_value) + end end end end diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 7f7662f2776..176301bcca1 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -104,6 +104,14 @@ module Gitlab (before_script_value.to_a + script_value.to_a).join("\n") end + def manual_action? + self.when == 'manual' + end + + def ignored? + allow_failure.nil? ? manual_action? : allow_failure + end + private def inherit!(deps) @@ -135,7 +143,8 @@ module Gitlab environment_name: environment_defined? ? environment_value[:name] : nil, coverage: coverage_defined? ? coverage_value : nil, artifacts: artifacts_value, - after_script: after_script_value } + after_script: after_script_value, + ignore: ignored? } end end end diff --git a/lib/gitlab/ci/config/entry/key.rb b/lib/gitlab/ci/config/entry/key.rb index 0e4c9fe6edc..f27ad0a7759 100644 --- a/lib/gitlab/ci/config/entry/key.rb +++ b/lib/gitlab/ci/config/entry/key.rb @@ -11,6 +11,10 @@ module Gitlab validations do validates :config, key: true end + + def self.default + 'default' + end end end end diff --git a/lib/gitlab/ci/config/entry/node.rb b/lib/gitlab/ci/config/entry/node.rb index 55a5447ab51..a6a914d79c1 100644 --- a/lib/gitlab/ci/config/entry/node.rb +++ b/lib/gitlab/ci/config/entry/node.rb @@ -70,6 +70,12 @@ module Gitlab true end + def inspect + val = leaf? ? config : descendants + unspecified = specified? ? '' : '(unspecified) ' + "#<#{self.class.name} #{unspecified}{#{key}: #{val.inspect}}>" + end + def self.default end diff --git a/lib/gitlab/ci/config/entry/undefined.rb b/lib/gitlab/ci/config/entry/undefined.rb index b33b8238230..1171ac10f22 100644 --- a/lib/gitlab/ci/config/entry/undefined.rb +++ b/lib/gitlab/ci/config/entry/undefined.rb @@ -29,6 +29,10 @@ module Gitlab def relevant? false end + + def inspect + "#<#{self.class.name}>" + end end end end diff --git a/lib/gitlab/ci/status/build/play.rb b/lib/gitlab/ci/status/build/play.rb index 0f4b7b24cef..3495b8d0448 100644 --- a/lib/gitlab/ci/status/build/play.rb +++ b/lib/gitlab/ci/status/build/play.rb @@ -5,22 +5,10 @@ module Gitlab class Play < SimpleDelegator include Status::Extended - def text - 'manual' - end - def label 'manual play action' end - def icon - 'icon_status_manual' - end - - def group - 'manual' - end - def has_action? can?(user, :update_build, subject) end diff --git a/lib/gitlab/ci/status/build/stop.rb b/lib/gitlab/ci/status/build/stop.rb index 90401cad0d2..e8530f2aaae 100644 --- a/lib/gitlab/ci/status/build/stop.rb +++ b/lib/gitlab/ci/status/build/stop.rb @@ -5,22 +5,10 @@ module Gitlab class Stop < SimpleDelegator include Status::Extended - def text - 'manual' - end - def label 'manual stop action' end - def icon - 'icon_status_manual' - end - - def group - 'manual' - end - def has_action? can?(user, :update_build, subject) end diff --git a/lib/gitlab/ci/status/manual.rb b/lib/gitlab/ci/status/manual.rb new file mode 100644 index 00000000000..5f28521901d --- /dev/null +++ b/lib/gitlab/ci/status/manual.rb @@ -0,0 +1,19 @@ +module Gitlab + module Ci + module Status + class Manual < Status::Core + def text + 'manual' + end + + def label + 'manual action' + end + + def icon + 'icon_status_manual' + end + end + end + end +end diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb index e50e54b6e99..182a30fd74d 100644 --- a/lib/gitlab/data_builder/pipeline.rb +++ b/lib/gitlab/data_builder/pipeline.rb @@ -39,7 +39,7 @@ module Gitlab started_at: build.started_at, finished_at: build.finished_at, when: build.when, - manual: build.manual?, + manual: build.action?, user: build.user.try(:hook_attrs), runner: build.runner && runner_hook_attrs(build.runner), artifacts_file: { diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index d160cadc2d0..f3f417c1a63 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -24,7 +24,7 @@ module Gitlab def self.nulls_last_order(field, direction = 'ASC') order = "#{field} #{direction}" - if Gitlab::Database.postgresql? + if postgresql? order << ' NULLS LAST' else # `field IS NULL` will be `0` for non-NULL columns and `1` for NULL @@ -38,7 +38,7 @@ module Gitlab def self.nulls_first_order(field, direction = 'ASC') order = "#{field} #{direction}" - if Gitlab::Database.postgresql? + if postgresql? order << ' NULLS FIRST' else # `field IS NULL` will be `0` for non-NULL columns and `1` for NULL @@ -50,7 +50,7 @@ module Gitlab end def self.random - Gitlab::Database.postgresql? ? "RANDOM()" : "RAND()" + postgresql? ? "RANDOM()" : "RAND()" end def true_value diff --git a/lib/gitlab/emoji.rb b/lib/gitlab/emoji.rb index bbbca8acc40..42703545c4f 100644 --- a/lib/gitlab/emoji.rb +++ b/lib/gitlab/emoji.rb @@ -1,7 +1,7 @@ module Gitlab module Emoji extend self - + def emojis Gemojione.index.instance_variable_get(:@emoji_by_name) end @@ -18,6 +18,10 @@ module Gitlab emojis.keys end + def emojis_aliases + @emoji_aliases ||= JSON.parse(File.read(Rails.root.join('fixtures', 'emojis', 'aliases.json'))) + end + def emoji_filename(name) emojis[name]["unicode"] end @@ -25,5 +29,42 @@ module Gitlab def emoji_unicode_filename(moji) emojis_by_moji[moji]["unicode"] end + + def emoji_unicode_version(name) + @emoji_unicode_versions_by_name ||= JSON.parse(File.read(Rails.root.join('node_modules', 'emoji-unicode-version', 'emoji-unicode-version-map.json'))) + @emoji_unicode_versions_by_name[name] + end + + def normalize_emoji_name(name) + emojis_aliases[name] || name + end + + def emoji_image_tag(name, src) + "<img class='emoji' title=':#{name}:' alt=':#{name}:' src='#{src}' height='20' width='20' align='absmiddle' />" + end + + # CSS sprite fallback takes precedence over image fallback + def gl_emoji_tag(name, image: false, sprite: false, force_fallback: false) + emoji_name = emojis_aliases[name] || name + emoji_info = emojis[emoji_name] + emoji_fallback_image_source = ActionController::Base.helpers.url_to_image("emoji/#{emoji_info['name']}.png") + emoji_fallback_sprite_class = "emoji-#{emoji_name}" + + data = { + name: emoji_name, + unicode_version: emoji_unicode_version(emoji_name) + } + data[:fallback_src] = emoji_fallback_image_source if image + data[:fallback_sprite_class] = emoji_fallback_sprite_class if sprite + ActionController::Base.helpers.content_tag 'gl-emoji', + class: ("emoji-icon #{emoji_fallback_sprite_class}" if force_fallback && sprite), + data: data do + if force_fallback && !sprite + emoji_image_tag(emoji_name, emoji_fallback_image_source) + else + emoji_info['moji'] + end + end + end end end diff --git a/lib/gitlab/etag_caching/middleware.rb b/lib/gitlab/etag_caching/middleware.rb new file mode 100644 index 00000000000..0f24f9bbfde --- /dev/null +++ b/lib/gitlab/etag_caching/middleware.rb @@ -0,0 +1,66 @@ +module Gitlab + module EtagCaching + class Middleware + RESERVED_WORDS = ProjectPathValidator::RESERVED.map { |word| "/#{word}/" }.join('|') + ROUTE_REGEXP = Regexp.union( + %r(^(?!.*(#{RESERVED_WORDS})).*/noteable/issue/\d+/notes\z) + ) + + def initialize(app) + @app = app + end + + def call(env) + return @app.call(env) unless enabled_for_current_route?(env) + Gitlab::Metrics.add_event(:etag_caching_middleware_used) + + etag, cached_value_present = get_etag(env) + if_none_match = env['HTTP_IF_NONE_MATCH'] + + if if_none_match == etag + Gitlab::Metrics.add_event(:etag_caching_cache_hit) + [304, { 'ETag' => etag }, ['']] + else + track_cache_miss(if_none_match, cached_value_present) + + status, headers, body = @app.call(env) + headers['ETag'] = etag + [status, headers, body] + end + end + + private + + def enabled_for_current_route?(env) + ROUTE_REGEXP.match(env['PATH_INFO']) + end + + def get_etag(env) + cache_key = env['PATH_INFO'] + store = Store.new + current_value = store.get(cache_key) + cached_value_present = current_value.present? + + unless cached_value_present + current_value = store.touch(cache_key, only_if_missing: true) + end + + [weak_etag_format(current_value), cached_value_present] + end + + def weak_etag_format(value) + %Q{W/"#{value}"} + end + + def track_cache_miss(if_none_match, cached_value_present) + if if_none_match.blank? + Gitlab::Metrics.add_event(:etag_caching_header_missing) + elsif !cached_value_present + Gitlab::Metrics.add_event(:etag_caching_key_not_found) + else + Gitlab::Metrics.add_event(:etag_caching_resource_changed) + end + end + end + end +end diff --git a/lib/gitlab/etag_caching/store.rb b/lib/gitlab/etag_caching/store.rb new file mode 100644 index 00000000000..9532e432f78 --- /dev/null +++ b/lib/gitlab/etag_caching/store.rb @@ -0,0 +1,32 @@ +module Gitlab + module EtagCaching + class Store + EXPIRY_TIME = 10.minutes + REDIS_NAMESPACE = 'etag:'.freeze + + def get(key) + Gitlab::Redis.with { |redis| redis.get(redis_key(key)) } + end + + def touch(key, only_if_missing: false) + etag = generate_etag + + Gitlab::Redis.with do |redis| + redis.set(redis_key(key), etag, ex: EXPIRY_TIME, nx: only_if_missing) + end + + etag + end + + private + + def generate_etag + SecureRandom.hex + end + + def redis_key(key) + "#{REDIS_NAMESPACE}#{key}" + end + end + end +end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 6540730ca7a..228ef7bb7a9 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -354,6 +354,18 @@ module Gitlab lines.map! { |c| Rugged::Commit.new(rugged, c.strip) } end + def count_commits(options) + cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} rev-list] + cmd << "--after=#{options[:after].iso8601}" if options[:after] + cmd << "--before=#{options[:before].iso8601}" if options[:before] + cmd += %W[--count #{options[:ref]}] + cmd += %W[-- #{options[:path]}] if options[:path].present? + + raw_output = IO.popen(cmd) { |io| io.read } + + raw_output.to_i + end + def sha_from_ref(ref) rev_parse_target(ref).oid end diff --git a/lib/gitlab/github_import/branch_formatter.rb b/lib/gitlab/github_import/branch_formatter.rb index 0a8d05b5fe1..5d29e698b27 100644 --- a/lib/gitlab/github_import/branch_formatter.rb +++ b/lib/gitlab/github_import/branch_formatter.rb @@ -18,7 +18,7 @@ module Gitlab end def commit_exists? - project.repository.commit(sha).present? + project.repository.branch_names_contains(sha).include?(ref) end def short_id diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 9c384069661..6c275a8d5de 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -4,16 +4,17 @@ module Gitlab gon.api_version = 'v3' # v4 Is not officially released yet, therefore can't be considered as "frozen" gon.default_avatar_url = URI.join(Gitlab.config.gitlab.url, ActionController::Base.helpers.image_path('no_avatar.png')).to_s gon.max_file_size = current_application_settings.max_attachment_size + gon.asset_host = ActionController::Base.asset_host gon.relative_url_root = Gitlab.config.gitlab.relative_url_root gon.shortcuts_path = help_page_path('shortcuts') gon.user_color_scheme = Gitlab::ColorSchemes.for_user(current_user).css_class - gon.award_menu_url = emojis_path gon.katex_css_url = ActionController::Base.helpers.asset_path('katex.css') gon.katex_js_url = ActionController::Base.helpers.asset_path('katex.js') if current_user gon.current_user_id = current_user.id gon.current_username = current_user.username + gon.current_user_fullname = current_user.name end end end diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb index 5764ab15652..6023fa1820f 100644 --- a/lib/gitlab/middleware/go.rb +++ b/lib/gitlab/middleware/go.rb @@ -30,21 +30,69 @@ module Gitlab end def go_body(request) - base_url = Gitlab.config.gitlab.url - # Go subpackages may be in the form of namespace/project/path1/path2/../pathN - # We can just ignore the paths and leave the namespace/project - path_info = request.env["PATH_INFO"] - path_info.sub!(/^\//, '') - project_path = path_info.split('/').first(2).join('/') - request_url = URI.join(base_url, project_path) - domain_path = strip_url(request_url.to_s) + project_url = URI.join(Gitlab.config.gitlab.url, project_path(request)) + import_prefix = strip_url(project_url.to_s) - "<!DOCTYPE html><html><head><meta content='#{domain_path} git #{request_url}.git' name='go-import'></head></html>\n" + "<!DOCTYPE html><html><head><meta content='#{import_prefix} git #{project_url}.git' name='go-import'></head></html>\n" end def strip_url(url) url.gsub(/\Ahttps?:\/\//, '') end + + def project_path(request) + path_info = request.env["PATH_INFO"] + path_info.sub!(/^\//, '') + + # Go subpackages may be in the form of `namespace/project/path1/path2/../pathN`. + # In a traditional project with a single namespace, this would denote repo + # `namespace/project` with subpath `path1/path2/../pathN`, but with nested + # groups, this could also be `namespace/project/path1` with subpath + # `path2/../pathN`, for example. + + # We find all potential project paths out of the path segments + path_segments = path_info.split('/') + simple_project_path = path_segments.first(2).join('/') + + # If the path is at most 2 segments long, it is a simple `namespace/project` path and we're done + return simple_project_path if path_segments.length <= 2 + + project_paths = [] + begin + project_paths << path_segments.join('/') + path_segments.pop + end while path_segments.length >= 2 + + # We see if a project exists with any of these potential paths + project = project_for_paths(project_paths, request) + + if project + # If a project is found and the user has access, we return the full project path + project.full_path + else + # If not, we return the first two components as if it were a simple `namespace/project` path, + # so that we don't reveal the existence of a nested project the user doesn't have access to. + # This means that for an unauthenticated request to `group/subgroup/project/subpackage` + # for a private `group/subgroup/project` with subpackage path `subpackage`, GitLab will respond + # as if the user is looking for project `group/subgroup`, with subpackage path `project/subpackage`. + # Since `go get` doesn't authenticate by default, this means that + # `go get gitlab.com/group/subgroup/project/subpackage` will not work for private projects. + # `go get gitlab.com/group/subgroup/project.git/subpackage` will work, since Go is smart enough + # to figure that out. `import 'gitlab.com/...'` behaves the same as `go get`. + simple_project_path + end + end + + def project_for_paths(paths, request) + project = Project.where_full_path_in(paths).first + return unless Ability.allowed?(current_user(request), :read_project, project) + + project + end + + def current_user(request) + request.env['warden']&.authenticate + end end end end diff --git a/lib/gitlab/prometheus.rb b/lib/gitlab/prometheus.rb new file mode 100644 index 00000000000..62239779454 --- /dev/null +++ b/lib/gitlab/prometheus.rb @@ -0,0 +1,70 @@ +module Gitlab + PrometheusError = Class.new(StandardError) + + # Helper methods to interact with Prometheus network services & resources + class Prometheus + attr_reader :api_url + + def initialize(api_url:) + @api_url = api_url + end + + def ping + json_api_get('query', query: '1') + end + + def query(query) + get_result('vector') do + json_api_get('query', query: query) + end + end + + def query_range(query, start: 8.hours.ago) + get_result('matrix') do + json_api_get('query_range', + query: query, + start: start.to_f, + end: Time.now.utc.to_f, + step: 1.minute.to_i) + end + end + + private + + def json_api_get(type, args = {}) + get(join_api_url(type, args)) + rescue Errno::ECONNREFUSED + raise PrometheusError, 'Connection refused' + end + + def join_api_url(type, args = {}) + url = URI.parse(api_url) + rescue URI::Error + raise PrometheusError, "Invalid API URL: #{api_url}" + else + url.path = [url.path.sub(%r{/+\z}, ''), 'api', 'v1', type].join('/') + url.query = args.to_query + + url.to_s + end + + def get(url) + handle_response(HTTParty.get(url)) + end + + def handle_response(response) + if response.code == 200 && response['status'] == 'success' + response['data'] || {} + elsif response.code == 400 + raise PrometheusError, response['error'] || 'Bad data received' + else + raise PrometheusError, "#{response.code} - #{response.body}" + end + end + + def get_result(expected_type) + data = yield + data['result'] if data['resultType'] == expected_type + end + end +end diff --git a/lib/gitlab/request_context.rb b/lib/gitlab/request_context.rb new file mode 100644 index 00000000000..fef536ecb0b --- /dev/null +++ b/lib/gitlab/request_context.rb @@ -0,0 +1,21 @@ +module Gitlab + class RequestContext + class << self + def client_ip + RequestStore[:client_ip] + end + end + + def initialize(app) + @app = app + end + + def call(env) + req = Rack::Request.new(env) + + RequestStore[:client_ip] = req.ip + + @app.call(env) + end + end +end diff --git a/lib/gitlab/sidekiq_status.rb b/lib/gitlab/sidekiq_status.rb index aadc401ff8d..11e5f1b645c 100644 --- a/lib/gitlab/sidekiq_status.rb +++ b/lib/gitlab/sidekiq_status.rb @@ -44,19 +44,42 @@ module Gitlab # Returns true if all the given job have been completed. # - # jids - The Sidekiq job IDs to check. + # job_ids - The Sidekiq job IDs to check. # # Returns true or false. - def self.all_completed?(jids) - keys = jids.map { |jid| key_for(jid) } + def self.all_completed?(job_ids) + self.num_running(job_ids).zero? + end + + # Returns the number of jobs that are running. + # + # job_ids - The Sidekiq job IDs to check. + def self.num_running(job_ids) + responses = self.job_status(job_ids) - responses = Sidekiq.redis do |redis| + responses.select(&:present?).count + end + + # Returns the number of jobs that have completed. + # + # job_ids - The Sidekiq job IDs to check. + def self.num_completed(job_ids) + job_ids.size - self.num_running(job_ids) + end + + # Returns the job status for each of the given job IDs. + # + # job_ids - The Sidekiq job IDs to check. + # + # Returns an array of true or false indicating job completion. + def self.job_status(job_ids) + keys = job_ids.map { |jid| key_for(jid) } + + Sidekiq.redis do |redis| redis.pipelined do keys.each { |key| redis.exists(key) } end end - - responses.all? { |value| !value } end def self.key_for(jid) diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 3ff9f9eb5e7..eae1a0abf06 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -8,6 +8,7 @@ module Gitlab VERSION_FILE = 'GITLAB_WORKHORSE_VERSION'.freeze INTERNAL_API_CONTENT_TYPE = 'application/vnd.gitlab-workhorse+json'.freeze INTERNAL_API_REQUEST_HEADER = 'Gitlab-Workhorse-Api-Request'.freeze + NOTIFICATION_CHANNEL = 'workhorse:notifications'.freeze # Supposedly the effective key size for HMAC-SHA256 is 256 bits, i.e. 32 # bytes https://tools.ietf.org/html/rfc4868#section-2.6 @@ -154,6 +155,18 @@ module Gitlab Rails.root.join('.gitlab_workhorse_secret') end + def set_key_and_notify(key, value, expire: nil, overwrite: true) + Gitlab::Redis.with do |redis| + result = redis.set(key, value, ex: expire, nx: !overwrite) + if result + redis.publish(NOTIFICATION_CHANNEL, "#{key}=#{value}") + value + else + redis.get(key) + end + end + end + protected def encode(hash) diff --git a/lib/mattermost/client.rb b/lib/mattermost/client.rb index ad6df246091..3d60618006c 100644 --- a/lib/mattermost/client.rb +++ b/lib/mattermost/client.rb @@ -26,7 +26,7 @@ module Mattermost def session_get(path, options = {}) with_session do |session| - get(session, path, options) + get(session, path, options) end end diff --git a/lib/mattermost/session.rb b/lib/mattermost/session.rb index 5388966605d..688a79c0441 100644 --- a/lib/mattermost/session.rb +++ b/lib/mattermost/session.rb @@ -153,7 +153,7 @@ module Mattermost yield rescue HTTParty::Error => e raise Mattermost::ConnectionError.new(e.message) - rescue Errno::ECONNREFUSED + rescue Errno::ECONNREFUSED => e raise Mattermost::ConnectionError.new(e.message) end end diff --git a/lib/mattermost/team.rb b/lib/mattermost/team.rb index 09dfd082b3a..2cdbbdece16 100644 --- a/lib/mattermost/team.rb +++ b/lib/mattermost/team.rb @@ -1,7 +1,18 @@ module Mattermost class Team < Client + # Returns **all** teams for an admin def all - session_get('/api/v3/teams/all') + session_get('/api/v3/teams/all').values + end + + # Creates a team on the linked Mattermost instance, the team admin will be the + # `current_user` passed to the Mattermost::Client instance + def create(name:, display_name:, type:) + session_post('/api/v3/teams/create', body: { + name: name, + display_name: display_name, + type: type + }.to_json) end end end diff --git a/lib/tasks/gemojione.rake b/lib/tasks/gemojione.rake index 993112aee3b..1f93b5a4dd2 100644 --- a/lib/tasks/gemojione.rake +++ b/lib/tasks/gemojione.rake @@ -5,29 +5,29 @@ namespace :gemojione do require 'json' dir = Gemojione.images_path - digests = [] - aliases = Hash.new { |hash, key| hash[key] = [] } - aliases_path = File.join(Rails.root, 'fixtures', 'emojis', 'aliases.json') - - JSON.parse(File.read(aliases_path)).each do |alias_name, real_name| - aliases[real_name] << alias_name - end - - Gitlab::AwardEmoji.emojis.map do |name, emoji_hash| - fpath = File.join(dir, "#{emoji_hash['unicode']}.png") - digest = Digest::SHA256.file(fpath).hexdigest - - digests << { name: name, unicode: emoji_hash['unicode'], digest: digest } + resultant_emoji_map = {} + + Gitlab::Emoji.emojis.each do |name, emoji_hash| + # Ignore aliases + unless Gitlab::Emoji.emojis_aliases.key?(name) + fpath = File.join(dir, "#{emoji_hash['unicode']}.png") + hash_digest = Digest::SHA256.file(fpath).hexdigest + + entry = { + category: emoji_hash['category'], + moji: emoji_hash['moji'], + unicodeVersion: Gitlab::Emoji.emoji_unicode_version(name), + digest: hash_digest, + } - aliases[name].each do |alias_name| - digests << { name: alias_name, unicode: emoji_hash['unicode'], digest: digest } + resultant_emoji_map[name] = entry end end out = File.join(Rails.root, 'fixtures', 'emojis', 'digests.json') File.open(out, 'w') do |handle| - handle.write(JSON.pretty_generate(digests)) + handle.write(JSON.pretty_generate(resultant_emoji_map)) end end @@ -55,21 +55,40 @@ namespace :gemojione do SPRITESHEET_WIDTH = 860 SPRITESHEET_HEIGHT = 840 + # Setup a map to rename image files + emoji_unicode_string_to_name_map = {} + Gitlab::Emoji.emojis.each do |name, emoji_hash| + # Ignore aliases + unless Gitlab::Emoji.emojis_aliases.key?(name) + emoji_unicode_string_to_name_map[emoji_hash['unicode']] = name + end + end + + # Copy the Gemojione assets to the temporary folder for renaming + emoji_dir = "app/assets/images/emoji" + FileUtils.rm_rf(emoji_dir) + FileUtils.mkdir_p(emoji_dir, mode: 0700) + FileUtils.cp_r(File.join(Gemojione.images_path, '.'), emoji_dir) + Dir[File.join(emoji_dir, "**/*.png")].each do |png| + image_path = png + rename_to_named_emoji_image!(emoji_unicode_string_to_name_map, image_path) + end + Dir.mktmpdir do |tmpdir| - # Copy the Gemojione assets to the temporary folder for resizing - FileUtils.cp_r(Gemojione.images_path, tmpdir) + FileUtils.cp_r(File.join(emoji_dir, '.'), tmpdir) Dir.chdir(tmpdir) do Dir["**/*.png"].each do |png| - resize!(File.join(tmpdir, png), SIZE) + tmp_image_path = File.join(tmpdir, png) + resize!(tmp_image_path, SIZE) end end - style_path = Rails.root.join(*%w(app assets stylesheets pages emojis.scss)) + style_path = Rails.root.join(*%w(app assets stylesheets framework emoji-sprites.scss)) # Combine the resized assets into a packed sprite and re-generate the SCSS SpriteFactory.cssurl = "image-url('$IMAGE')" - SpriteFactory.run!(File.join(tmpdir, 'png'), { + SpriteFactory.run!(tmpdir, { output_style: style_path, output_image: "app/assets/images/emoji.png", selector: '.emoji-', @@ -83,7 +102,7 @@ namespace :gemojione do # let's simplify it system(%Q(sed -i '' "s/width: #{SIZE}px; height: #{SIZE}px; background: image-url('emoji.png')/background-position:/" #{style_path})) system(%Q(sed -i '' "s/ no-repeat//" #{style_path})) - system(%Q(sed -i '' "s/ 0px/ 0/" #{style_path})) + system(%Q(sed -i '' "s/ 0px/ 0/g" #{style_path})) # Append a generic rule that applies to all Emojis File.open(style_path, 'a') do |f| @@ -92,6 +111,8 @@ namespace :gemojione do .emoji-icon { background-image: image-url('emoji.png'); background-repeat: no-repeat; + color: transparent; + text-indent: -99em; height: #{SIZE}px; width: #{SIZE}px; @@ -112,16 +133,17 @@ namespace :gemojione do # Now do it again but for Retina Dir.mktmpdir do |tmpdir| # Copy the Gemojione assets to the temporary folder for resizing - FileUtils.cp_r(Gemojione.images_path, tmpdir) + FileUtils.cp_r(File.join(emoji_dir, '.'), tmpdir) Dir.chdir(tmpdir) do Dir["**/*.png"].each do |png| - resize!(File.join(tmpdir, png), RETINA) + tmp_image_path = File.join(tmpdir, png) + resize!(tmp_image_path, RETINA) end end # Combine the resized assets into a packed sprite and re-generate the SCSS - SpriteFactory.run!(File.join(tmpdir), { + SpriteFactory.run!(tmpdir, { output_image: "app/assets/images/emoji@2x.png", style: false, nocomments: true, @@ -155,4 +177,20 @@ namespace :gemojione do image.write(image_path) { self.quality = 100 } image.destroy! end + + EMOJI_IMAGE_PATH_RE = /(.*?)(([0-9a-f]-?)+)\.png$/i + def rename_to_named_emoji_image!(emoji_unicode_string_to_name_map, image_path) + # Rename file from unicode to emoji name + matches = EMOJI_IMAGE_PATH_RE.match(image_path) + preceding_path = matches[1] + unicode_string = matches[2] + name = emoji_unicode_string_to_name_map[unicode_string] + if name + new_png_path = File.join(preceding_path, "#{name}.png") + FileUtils.mv(image_path, new_png_path) + new_png_path + else + puts "Warning: emoji_unicode_string_to_name_map missing entry for #{unicode_string}. Full path: #{image_path}" + end + end end diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 38edd49b6ed..a6f8c4ced5d 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -354,7 +354,8 @@ namespace :gitlab do def check_repo_base_exists puts "Repo base directory exists?" - Gitlab.config.repositories.storages.each do |name, repo_base_path| + Gitlab.config.repositories.storages.each do |name, repository_storage| + repo_base_path = repository_storage['path'] print "#{name}... " if File.exist?(repo_base_path) @@ -378,7 +379,8 @@ namespace :gitlab do def check_repo_base_is_not_symlink puts "Repo storage directories are symlinks?" - Gitlab.config.repositories.storages.each do |name, repo_base_path| + Gitlab.config.repositories.storages.each do |name, repository_storage| + repo_base_path = repository_storage['path'] print "#{name}... " unless File.exist?(repo_base_path) @@ -401,7 +403,8 @@ namespace :gitlab do def check_repo_base_permissions puts "Repo paths access is drwxrws---?" - Gitlab.config.repositories.storages.each do |name, repo_base_path| + Gitlab.config.repositories.storages.each do |name, repository_storage| + repo_base_path = repository_storage['path'] print "#{name}... " unless File.exist?(repo_base_path) @@ -431,7 +434,8 @@ namespace :gitlab do gitlab_shell_owner_group = Gitlab.config.gitlab_shell.owner_group puts "Repo paths owned by #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group}?" - Gitlab.config.repositories.storages.each do |name, repo_base_path| + Gitlab.config.repositories.storages.each do |name, repository_storage| + repo_base_path = repository_storage['path'] print "#{name}... " unless File.exist?(repo_base_path) @@ -810,8 +814,8 @@ namespace :gitlab do namespace :repo do desc "GitLab | Check the integrity of the repositories managed by GitLab" task check: :environment do - Gitlab.config.repositories.storages.each do |name, path| - namespace_dirs = Dir.glob(File.join(path, '*')) + Gitlab.config.repositories.storages.each do |name, repository_storage| + namespace_dirs = Dir.glob(File.join(repository_storage['path'], '*')) namespace_dirs.each do |namespace_dir| repo_dirs = Dir.glob(File.join(namespace_dir, '*')) diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index daf7382dd02..f76bef5f4bf 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -6,7 +6,8 @@ namespace :gitlab do remove_flag = ENV['REMOVE'] namespaces = Namespace.pluck(:path) - Gitlab.config.repositories.storages.each do |name, git_base_path| + Gitlab.config.repositories.storages.each do |name, repository_storage| + git_base_path = repository_storage['path'] all_dirs = Dir.glob(git_base_path + '/*') puts git_base_path.color(:yellow) @@ -47,7 +48,8 @@ namespace :gitlab do warn_user_is_not_gitlab move_suffix = "+orphaned+#{Time.now.to_i}" - Gitlab.config.repositories.storages.each do |name, repo_root| + Gitlab.config.repositories.storages.each do |name, repository_storage| + repo_root = repository_storage['path'] # Look for global repos (legacy, depth 1) and normal repos (depth 2) IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find| find.each_line do |path| diff --git a/lib/tasks/gitlab/import.rake b/lib/tasks/gitlab/import.rake index 66e7b7685f7..48bd9139ce8 100644 --- a/lib/tasks/gitlab/import.rake +++ b/lib/tasks/gitlab/import.rake @@ -11,7 +11,8 @@ namespace :gitlab do # desc "GitLab | Import bare repositories from repositories -> storages into GitLab project instance" task repos: :environment do - Gitlab.config.repositories.storages.each do |name, git_base_path| + Gitlab.config.repositories.storages.each_value do |repository_storage| + git_base_path = repository_storage['path'] repos_to_import = Dir.glob(git_base_path + '/**/*.git') repos_to_import.each do |repo_path| diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake index ae78fe64eb8..a2a2db487b7 100644 --- a/lib/tasks/gitlab/info.rake +++ b/lib/tasks/gitlab/info.rake @@ -14,6 +14,8 @@ namespace :gitlab do rake_version = run_and_match(%w(rake --version), /[\d\.]+/).try(:to_s) # check redis version redis_version = run_and_match(%w(redis-cli --version), /redis-cli (\d+\.\d+\.\d+)/).to_a + # check Git version + git_version = run_and_match([Gitlab.config.git.bin_path, '--version'], /git version ([\d\.]+)/).to_a puts "" puts "System information".color(:yellow) @@ -26,6 +28,7 @@ namespace :gitlab do puts "Bundler Version:#{bunder_version || "unknown".color(:red)}" puts "Rake Version:\t#{rake_version || "unknown".color(:red)}" puts "Redis Version:\t#{redis_version[1] || "unknown".color(:red)}" + puts "Git Version:\t#{git_version[1] || "unknown".color(:red)}" puts "Sidekiq Version:#{Sidekiq::VERSION}" # check database adapter @@ -62,8 +65,8 @@ namespace :gitlab do puts "GitLab Shell".color(:yellow) puts "Version:\t#{gitlab_shell_version || "unknown".color(:red)}" puts "Repository storage paths:" - Gitlab.config.repositories.storages.each do |name, path| - puts "- #{name}: \t#{path}" + Gitlab.config.repositories.storages.each do |name, repository_storage| + puts "- #{name}: \t#{repository_storage['path']}" end puts "Hooks:\t\t#{Gitlab.config.gitlab_shell.hooks_path}" puts "Git:\t\t#{Gitlab.config.git.bin_path}" diff --git a/lib/tasks/gitlab/task_helpers.rb b/lib/tasks/gitlab/task_helpers.rb index 2a999ad6959..bb755ae689b 100644 --- a/lib/tasks/gitlab/task_helpers.rb +++ b/lib/tasks/gitlab/task_helpers.rb @@ -130,8 +130,8 @@ module Gitlab end def all_repos - Gitlab.config.repositories.storages.each do |name, path| - IO.popen(%W(find #{path} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find| + Gitlab.config.repositories.storages.each_value do |repository_storage| + IO.popen(%W(find #{repository_storage['path']} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find| find.each_line do |path| yield path.chomp end @@ -140,7 +140,7 @@ module Gitlab end def repository_storage_paths_args - Gitlab.config.repositories.storages.values + Gitlab.config.repositories.storages.values.map { |rs| rs['path'] } end def user_home |