diff options
Diffstat (limited to 'lib')
36 files changed, 606 insertions, 199 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index f3f64244589..e953f3d2eca 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -146,6 +146,7 @@ module API mount ::API::Repositories mount ::API::Runner mount ::API::Runners + mount ::API::Search mount ::API::Services mount ::API::Settings mount ::API::SidekiqMetrics diff --git a/lib/api/entities.rb b/lib/api/entities.rb index e13463ec66b..7838de13c56 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -314,24 +314,20 @@ module API end end - class ProjectSnippet < Grape::Entity + class Snippet < Grape::Entity expose :id, :title, :file_name, :description expose :author, using: Entities::UserBasic expose :updated_at, :created_at - - expose :web_url do |snippet, options| + expose :project_id + expose :web_url do |snippet| Gitlab::UrlBuilder.build(snippet) end end - class PersonalSnippet < Grape::Entity - expose :id, :title, :file_name, :description - expose :author, using: Entities::UserBasic - expose :updated_at, :created_at + class ProjectSnippet < Snippet + end - expose :web_url do |snippet| - Gitlab::UrlBuilder.build(snippet) - end + class PersonalSnippet < Snippet expose :raw_url do |snippet| Gitlab::UrlBuilder.build(snippet) + "/raw" end @@ -1168,5 +1164,14 @@ module API class ApplicationWithSecret < Application expose :secret end + + class Blob < Grape::Entity + expose :basename + expose :data + expose :filename + expose :id + expose :ref + expose :startline + end end end diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index eb67de81a0d..cd59da6fc70 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -60,8 +60,20 @@ module API false end + def project_path + project&.path || project_path_match[:project_path] + end + + def namespace_path + project&.namespace&.full_path || project_path_match[:namespace_path] + end + private + def project_path_match + @project_path_match ||= params[:project].match(Gitlab::PathRegex.full_project_git_path_regex) || {} + end + # rubocop:disable Gitlab/ModuleWithInstanceVariables def set_project if params[:gl_repository] diff --git a/lib/api/helpers/pagination.rb b/lib/api/helpers/pagination.rb index bb70370ba77..09805049169 100644 --- a/lib/api/helpers/pagination.rb +++ b/lib/api/helpers/pagination.rb @@ -12,13 +12,16 @@ module API private def add_pagination_headers(paginated_data) - header 'X-Total', paginated_data.total_count.to_s - header 'X-Total-Pages', total_pages(paginated_data).to_s header 'X-Per-Page', paginated_data.limit_value.to_s header 'X-Page', paginated_data.current_page.to_s header 'X-Next-Page', paginated_data.next_page.to_s header 'X-Prev-Page', paginated_data.prev_page.to_s header 'Link', pagination_links(paginated_data) + + return if data_without_counts?(paginated_data) + + header 'X-Total', paginated_data.total_count.to_s + header 'X-Total-Pages', total_pages(paginated_data).to_s end def pagination_links(paginated_data) @@ -37,8 +40,10 @@ module API request_params[:page] = 1 links << %(<#{request_url}?#{request_params.to_query}>; rel="first") - request_params[:page] = total_pages(paginated_data) - links << %(<#{request_url}?#{request_params.to_query}>; rel="last") + unless data_without_counts?(paginated_data) + request_params[:page] = total_pages(paginated_data) + links << %(<#{request_url}?#{request_params.to_query}>; rel="last") + end links.join(', ') end @@ -55,6 +60,10 @@ module API relation end + + def data_without_counts?(paginated_data) + paginated_data.is_a?(Kaminari::PaginatableWithoutCount) + end end end end diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb index 3d0d1287407..fbe30192a16 100644 --- a/lib/api/helpers/runner.rb +++ b/lib/api/helpers/runner.rb @@ -3,7 +3,6 @@ module API module Runner JOB_TOKEN_HEADER = 'HTTP_JOB_TOKEN'.freeze JOB_TOKEN_PARAM = :token - UPDATE_RUNNER_EVERY = 10 * 60 def runner_registration_token_valid? ActiveSupport::SecurityUtils.variable_size_secure_compare(params[:token], @@ -18,30 +17,14 @@ module API def authenticate_runner! forbidden! unless current_runner + + current_runner.update_cached_info(get_runner_version_from_params) end def current_runner @runner ||= ::Ci::Runner.find_by_token(params[:token].to_s) end - def update_runner_info - return unless update_runner? - - current_runner.contacted_at = Time.now - current_runner.assign_attributes(get_runner_version_from_params) - current_runner.save if current_runner.changed? - end - - def update_runner? - # Use a random threshold to prevent beating DB updates. - # It generates a distribution between [40m, 80m]. - # - contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY) - - current_runner.contacted_at.nil? || - (Time.now - current_runner.contacted_at) >= contacted_at_max_age - end - def validate_job!(job) not_found! unless job diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 063f0d6599c..9285fb90cdc 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -42,11 +42,14 @@ module API end access_checker_klass = wiki? ? Gitlab::GitAccessWiki : Gitlab::GitAccess - access_checker = access_checker_klass - .new(actor, project, protocol, authentication_abilities: ssh_authentication_abilities, redirected_path: redirected_path) + access_checker = access_checker_klass.new(actor, project, + protocol, authentication_abilities: ssh_authentication_abilities, + namespace_path: namespace_path, project_path: project_path, + redirected_path: redirected_path) begin access_checker.check(params[:action], params[:changes]) + @project ||= access_checker.project rescue Gitlab::GitAccess::UnauthorizedError, Gitlab::GitAccess::NotFoundError => e return { status: false, message: e.message } end @@ -207,8 +210,11 @@ module API # A user is not guaranteed to be returned; an orphaned write deploy # key could be used if user - redirect_message = Gitlab::Checks::ProjectMoved.fetch_redirect_message(user.id, project.id) + redirect_message = Gitlab::Checks::ProjectMoved.fetch_message(user.id, project.id) + project_created_message = Gitlab::Checks::ProjectCreated.fetch_message(user.id, project.id) + output[:redirected_message] = redirect_message if redirect_message + output[:project_created_message] = project_created_message if project_created_message end output diff --git a/lib/api/runner.rb b/lib/api/runner.rb index 1f80646a2ea..5469cba69a6 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -78,7 +78,6 @@ module API post '/request' do authenticate_runner! no_content! unless current_runner.active? - update_runner_info if current_runner.runner_queue_value_latest?(params[:last_update]) header 'X-GitLab-Last-Update', params[:last_update] diff --git a/lib/api/search.rb b/lib/api/search.rb new file mode 100644 index 00000000000..9f08fd96a3b --- /dev/null +++ b/lib/api/search.rb @@ -0,0 +1,115 @@ +module API + class Search < Grape::API + include PaginationParams + + before { authenticate! } + + helpers do + SCOPE_ENTITY = { + merge_requests: Entities::MergeRequestBasic, + issues: Entities::IssueBasic, + projects: Entities::BasicProjectDetails, + milestones: Entities::Milestone, + notes: Entities::Note, + commits: Entities::Commit, + blobs: Entities::Blob, + wiki_blobs: Entities::Blob, + snippet_titles: Entities::Snippet, + snippet_blobs: Entities::Snippet + }.freeze + + def search(additional_params = {}) + search_params = { + scope: params[:scope], + search: params[:search], + snippets: snippets?, + page: params[:page], + per_page: params[:per_page] + }.merge(additional_params) + + results = SearchService.new(current_user, search_params).search_objects + + process_results(results) + end + + def process_results(results) + case params[:scope] + when 'wiki_blobs' + paginate(results).map { |blob| Gitlab::ProjectSearchResults.parse_search_result(blob) } + when 'blobs' + paginate(results).map { |blob| blob[1] } + else + paginate(results) + end + end + + def snippets? + %w(snippet_blobs snippet_titles).include?(params[:scope]).to_s + end + + def entity + SCOPE_ENTITY[params[:scope].to_sym] + end + end + + resource :search do + desc 'Search on GitLab' do + detail 'This feature was introduced in GitLab 10.5.' + end + params do + requires :search, type: String, desc: 'The expression it should be searched for' + requires :scope, + type: String, + desc: 'The scope of search, available scopes: + projects, issues, merge_requests, milestones, snippet_titles, snippet_blobs', + values: %w(projects issues merge_requests milestones snippet_titles snippet_blobs) + use :pagination + end + get do + present search, with: entity + end + end + + resource :groups, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + desc 'Search on GitLab' do + detail 'This feature was introduced in GitLab 10.5.' + end + params do + requires :id, type: String, desc: 'The ID of a group' + requires :search, type: String, desc: 'The expression it should be searched for' + requires :scope, + type: String, + desc: 'The scope of search, available scopes: + projects, issues, merge_requests, milestones', + values: %w(projects issues merge_requests milestones) + use :pagination + end + get ':id/-/search' do + find_group!(params[:id]) + + present search(group_id: params[:id]), with: entity + end + end + + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + desc 'Search on GitLab' do + detail 'This feature was introduced in GitLab 10.5.' + end + params do + requires :id, type: String, desc: 'The ID of a project' + requires :search, type: String, desc: 'The expression it should be searched for' + requires :scope, + type: String, + desc: 'The scope of search, available scopes: + issues, merge_requests, milestones, notes, wiki_blobs, commits, blobs', + values: %w(issues merge_requests milestones notes wiki_blobs commits blobs) + use :pagination + end + get ':id/-/search' do + find_project!(params[:id]) + + present search(project_id: params[:id]), with: entity + end + end + end +end diff --git a/lib/api/v3/projects.rb b/lib/api/v3/projects.rb index c856ba99f09..7d8b1f369fe 100644 --- a/lib/api/v3/projects.rb +++ b/lib/api/v3/projects.rb @@ -174,7 +174,7 @@ module API use :pagination end get "/search/:query", requirements: { query: %r{[^/]+} } do - search_service = Search::GlobalService.new(current_user, search: params[:query]).execute + search_service = ::Search::GlobalService.new(current_user, search: params[:query]).execute projects = search_service.objects('projects', params[:page], false) projects = projects.reorder(params[:order_by] => params[:sort]) diff --git a/lib/constraints/user_url_constrainer.rb b/lib/constraints/user_url_constrainer.rb index b7633aa7cbb..3b3ed1c6ddb 100644 --- a/lib/constraints/user_url_constrainer.rb +++ b/lib/constraints/user_url_constrainer.rb @@ -2,7 +2,7 @@ class UserUrlConstrainer def matches?(request) full_path = request.params[:username] - return false unless UserPathValidator.valid_path?(full_path) + return false unless NamespacePathValidator.valid_path?(full_path) User.find_by_full_path(full_path, follow_redirects: request.get?).present? end diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb index 945d70e7a24..d75e73dac10 100644 --- a/lib/gitlab/checks/change_access.rb +++ b/lib/gitlab/checks/change_access.rb @@ -31,13 +31,14 @@ module Gitlab @protocol = protocol end - def exec + def exec(skip_commits_check: false) return true if skip_authorization push_checks branch_checks tag_checks lfs_objects_exist_check + commits_check unless skip_commits_check true end @@ -117,6 +118,24 @@ module Gitlab end end + def commits_check + return if deletion? || newrev.nil? + + # n+1: https://gitlab.com/gitlab-org/gitlab-ee/issues/3593 + ::Gitlab::GitalyClient.allow_n_plus_1_calls do + commits.each do |commit| + commit_check.validate(commit, validations_for_commit(commit)) + end + end + + commit_check.validate_file_paths + end + + # Method overwritten in EE to inject custom validations + def validations_for_commit(_) + [] + end + private def updated_from_web? @@ -150,6 +169,14 @@ module Gitlab raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:lfs_objects_missing] end end + + def commit_check + @commit_check ||= Gitlab::Checks::CommitCheck.new(project, user_access.user, newrev, oldrev) + end + + def commits + project.repository.new_commits(newrev) + end end end end diff --git a/lib/gitlab/checks/commit_check.rb b/lib/gitlab/checks/commit_check.rb new file mode 100644 index 00000000000..ae0cd142378 --- /dev/null +++ b/lib/gitlab/checks/commit_check.rb @@ -0,0 +1,61 @@ +module Gitlab + module Checks + class CommitCheck + include Gitlab::Utils::StrongMemoize + + attr_reader :project, :user, :newrev, :oldrev + + def initialize(project, user, newrev, oldrev) + @project = project + @user = user + @newrev = user + @oldrev = user + @file_paths = [] + end + + def validate(commit, validations) + return if validations.empty? && path_validations.empty? + + commit.raw_deltas.each do |diff| + @file_paths << (diff.new_path || diff.old_path) + + validations.each do |validation| + if error = validation.call(diff) + raise ::Gitlab::GitAccess::UnauthorizedError, error + end + end + end + end + + def validate_file_paths + path_validations.each do |validation| + if error = validation.call(@file_paths) + raise ::Gitlab::GitAccess::UnauthorizedError, error + end + end + end + + private + + def validate_lfs_file_locks? + strong_memoize(:validate_lfs_file_locks) do + project.lfs_enabled? && project.lfs_file_locks.any? && newrev && oldrev + end + end + + def lfs_file_locks_validation + lambda do |paths| + lfs_lock = project.lfs_file_locks.where(path: paths).where.not(user_id: user.id).first + + if lfs_lock + return "The path '#{lfs_lock.path}' is locked in Git LFS by #{lfs_lock.user.name}" + end + end + end + + def path_validations + validate_lfs_file_locks? ? [lfs_file_locks_validation] : [] + end + end + end +end diff --git a/lib/gitlab/checks/post_push_message.rb b/lib/gitlab/checks/post_push_message.rb new file mode 100644 index 00000000000..473c0385b34 --- /dev/null +++ b/lib/gitlab/checks/post_push_message.rb @@ -0,0 +1,46 @@ +module Gitlab + module Checks + class PostPushMessage + def initialize(project, user, protocol) + @project = project + @user = user + @protocol = protocol + end + + def self.fetch_message(user_id, project_id) + key = message_key(user_id, project_id) + + Gitlab::Redis::SharedState.with do |redis| + message = redis.get(key) + redis.del(key) + message + end + end + + def add_message + return unless user.present? && project.present? + + Gitlab::Redis::SharedState.with do |redis| + key = self.class.message_key(user.id, project.id) + redis.setex(key, 5.minutes, message) + end + end + + def message + raise NotImplementedError + end + + protected + + attr_reader :project, :user, :protocol + + def self.message_key(user_id, project_id) + raise NotImplementedError + end + + def url_to_repo + protocol == 'ssh' ? project.ssh_url_to_repo : project.http_url_to_repo + end + end + end +end diff --git a/lib/gitlab/checks/project_created.rb b/lib/gitlab/checks/project_created.rb new file mode 100644 index 00000000000..cec270d6a58 --- /dev/null +++ b/lib/gitlab/checks/project_created.rb @@ -0,0 +1,31 @@ +module Gitlab + module Checks + class ProjectCreated < PostPushMessage + PROJECT_CREATED = "project_created".freeze + + def message + <<~MESSAGE + + The private project #{project.full_path} was successfully created. + + To configure the remote, run: + git remote add origin #{url_to_repo} + + To view the project, visit: + #{project_url} + + MESSAGE + end + + private + + def self.message_key(user_id, project_id) + "#{PROJECT_CREATED}:#{user_id}:#{project_id}" + end + + def project_url + Gitlab::Routing.url_helpers.project_url(project) + end + end + end +end diff --git a/lib/gitlab/checks/project_moved.rb b/lib/gitlab/checks/project_moved.rb index dfb2f4d4054..3263790a876 100644 --- a/lib/gitlab/checks/project_moved.rb +++ b/lib/gitlab/checks/project_moved.rb @@ -1,38 +1,16 @@ module Gitlab module Checks - class ProjectMoved + class ProjectMoved < PostPushMessage REDIRECT_NAMESPACE = "redirect_namespace".freeze - def initialize(project, user, redirected_path, protocol) - @project = project - @user = user + def initialize(project, user, protocol, redirected_path) @redirected_path = redirected_path - @protocol = protocol - end - - def self.fetch_redirect_message(user_id, project_id) - redirect_key = redirect_message_key(user_id, project_id) - Gitlab::Redis::SharedState.with do |redis| - message = redis.get(redirect_key) - redis.del(redirect_key) - message - end - end - - def add_redirect_message - # Don't bother with sending a redirect message for anonymous clones - # because they never see it via the `/internal/post_receive` endpoint - return unless user.present? && project.present? - - Gitlab::Redis::SharedState.with do |redis| - key = self.class.redirect_message_key(user.id, project.id) - redis.setex(key, 5.minutes, redirect_message) - end + super(project, user, protocol) end - def redirect_message(rejected: false) - <<~MESSAGE.strip_heredoc + def message(rejected: false) + <<~MESSAGE Project '#{redirected_path}' was moved to '#{project.full_path}'. Please update your Git remote: @@ -47,17 +25,17 @@ module Gitlab private - attr_reader :project, :redirected_path, :protocol, :user + attr_reader :redirected_path - def self.redirect_message_key(user_id, project_id) + def self.message_key(user_id, project_id) "#{REDIRECT_NAMESPACE}:#{user_id}:#{project_id}" end def remote_url_message(rejected) if rejected - "git remote set-url origin #{url} and try again." + "git remote set-url origin #{url_to_repo} and try again." else - "git remote set-url origin #{url}" + "git remote set-url origin #{url_to_repo}" end end diff --git a/lib/gitlab/ci/config/loader.rb b/lib/gitlab/ci/config/loader.rb index e7d9f6a7761..141d2714cb6 100644 --- a/lib/gitlab/ci/config/loader.rb +++ b/lib/gitlab/ci/config/loader.rb @@ -6,6 +6,8 @@ module Gitlab def initialize(config) @config = YAML.safe_load(config, [Symbol], [], true) + rescue Psych::Exception => e + raise FormatError, e.message end def valid? diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 0bd78b03448..a7285ac8f9d 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -85,7 +85,7 @@ module Gitlab begin Gitlab::Ci::YamlProcessor.new(content) nil - rescue ValidationError, Psych::SyntaxError => e + rescue ValidationError => e e.message end end diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 4828301dbb9..b2fca2c16de 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -53,11 +53,7 @@ module Gitlab def batch(repository, blob_references, blob_size_limit: MAX_DATA_DISPLAY_SIZE) Gitlab::GitalyClient.migrate(:list_blobs_by_sha_path) do |is_enabled| if is_enabled - Gitlab::GitalyClient.allow_n_plus_1_calls do - blob_references.map do |sha, path| - find_by_gitaly(repository, sha, path, limit: blob_size_limit) - end - end + repository.gitaly_blob_client.get_blobs(blob_references, blob_size_limit).to_a else blob_references.map do |sha, path| find_by_rugged(repository, sha, path, limit: blob_size_limit) diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 768617e2cae..d95561fe1b2 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -402,15 +402,6 @@ module Gitlab end end - # Get a collection of Rugged::Reference objects for this commit. - # - # Ex. - # commit.ref(repo) - # - def refs(repo) - repo.refs_hash[id] - end - # Get ref names collection # # Ex. @@ -418,7 +409,7 @@ module Gitlab # def ref_names(repo) refs(repo).map do |ref| - ref.name.sub(%r{^refs/(heads|remotes|tags)/}, "") + ref.sub(%r{^refs/(heads|remotes|tags)/}, "") end end @@ -553,6 +544,15 @@ module Gitlab date: Google::Protobuf::Timestamp.new(seconds: author_or_committer[:time].to_i) ) end + + # Get a collection of Gitlab::Git::Ref objects for this commit. + # + # Ex. + # commit.ref(repo) + # + def refs(repo) + repo.refs_hash[id] + end end end end diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb index e29a1f7afa1..24f027d8da4 100644 --- a/lib/gitlab/git/hook.rb +++ b/lib/gitlab/git/hook.rb @@ -82,14 +82,20 @@ module Gitlab end def call_update_hook(gl_id, gl_username, oldrev, newrev, ref) - Dir.chdir(repo_path) do - env = { - 'GL_ID' => gl_id, - 'GL_USERNAME' => gl_username - } - stdout, stderr, status = Open3.capture3(env, path, ref, oldrev, newrev) - [status.success?, (stderr.presence || stdout).gsub(/\R/, "<br>").html_safe] - end + env = { + 'GL_ID' => gl_id, + 'GL_USERNAME' => gl_username, + 'PWD' => repo_path + } + + options = { + chdir: repo_path + } + + args = [ref, oldrev, newrev] + + stdout, stderr, status = Open3.capture3(env, path, *args, options) + [status.success?, (stderr.presence || stdout).gsub(/\R/, "<br>").html_safe] end def retrieve_error_message(stderr, stdout) diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index d7510061def..6761fb0937a 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -631,21 +631,18 @@ module Gitlab end end - # Get refs hash which key is SHA1 - # and value is a Rugged::Reference + # Get refs hash which key is is the commit id + # and value is a Gitlab::Git::Tag or Gitlab::Git::Branch + # Note that both inherit from Gitlab::Git::Ref def refs_hash - # Initialize only when first call - if @refs_hash.nil? - @refs_hash = Hash.new { |h, k| h[k] = [] } - - rugged.references.each do |r| - # Symbolic/remote references may not have an OID; skip over them - target_oid = r.target.try(:oid) - if target_oid - sha = rev_parse_target(target_oid).oid - @refs_hash[sha] << r - end - end + return @refs_hash if @refs_hash + + @refs_hash = Hash.new { |h, k| h[k] = [] } + + (tags + branches).each do |ref| + next unless ref.target && ref.name + + @refs_hash[ref.dereferenced_target.id] << ref.name end @refs_hash diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index 39040d56971..ac12271a87e 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -25,9 +25,8 @@ module Gitlab @repository.exists? end - # Disabled because of https://gitlab.com/gitlab-org/gitaly/merge_requests/539 def write_page(name, format, content, commit_details) - @repository.gitaly_migrate(:wiki_write_page, status: Gitlab::GitalyClient::MigrationStatus::DISABLED) do |is_enabled| + @repository.gitaly_migrate(:wiki_write_page) do |is_enabled| if is_enabled gitaly_write_page(name, format, content, commit_details) gollum_wiki.clear_cache @@ -48,9 +47,8 @@ module Gitlab end end - # Disable because of https://gitlab.com/gitlab-org/gitlab-ce/issues/42094 def update_page(page_path, title, format, content, commit_details) - @repository.gitaly_migrate(:wiki_update_page, status: Gitlab::GitalyClient::MigrationStatus::DISABLED) do |is_enabled| + @repository.gitaly_migrate(:wiki_update_page) do |is_enabled| if is_enabled gitaly_update_page(page_path, title, format, content, commit_details) gollum_wiki.clear_cache diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 56f6febe86d..8ec3386184a 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -2,15 +2,19 @@ # class return an instance of `GitlabAccessStatus` module Gitlab class GitAccess + include Gitlab::Utils::StrongMemoize + UnauthorizedError = Class.new(StandardError) NotFoundError = Class.new(StandardError) + ProjectCreationError = Class.new(StandardError) ProjectMovedError = Class.new(NotFoundError) ERROR_MESSAGES = { upload: 'You are not allowed to upload code for this project.', download: 'You are not allowed to download code from this project.', - deploy_key_upload: - 'This deploy key does not have write access to this project.', + auth_upload: 'You are not allowed to upload code.', + auth_download: 'You are not allowed to download code.', + deploy_key_upload: 'This deploy key does not have write access to this project.', no_repo: 'A repository for this project does not exist yet.', project_not_found: 'The project you were looking for could not be found.', account_blocked: 'Your account has been blocked.', @@ -25,24 +29,31 @@ module Gitlab PUSH_COMMANDS = %w{ git-receive-pack }.freeze ALL_COMMANDS = DOWNLOAD_COMMANDS + PUSH_COMMANDS - attr_reader :actor, :project, :protocol, :authentication_abilities, :redirected_path + attr_reader :actor, :project, :protocol, :authentication_abilities, :namespace_path, :project_path, :redirected_path - def initialize(actor, project, protocol, authentication_abilities:, redirected_path: nil) + def initialize(actor, project, protocol, authentication_abilities:, namespace_path: nil, project_path: nil, redirected_path: nil) @actor = actor @project = project @protocol = protocol - @redirected_path = redirected_path @authentication_abilities = authentication_abilities + @namespace_path = namespace_path + @project_path = project_path + @redirected_path = redirected_path end def check(cmd, changes) check_protocol! check_valid_actor! check_active_user! - check_project_accessibility! - check_project_moved! + check_authentication_abilities!(cmd) check_command_disabled!(cmd) check_command_existence!(cmd) + check_db_accessibility!(cmd) + + ensure_project_on_push!(cmd, changes) + + check_project_accessibility! + check_project_moved! check_repository_existence! case cmd @@ -95,6 +106,19 @@ module Gitlab end end + def check_authentication_abilities!(cmd) + case cmd + when *DOWNLOAD_COMMANDS + unless authentication_abilities.include?(:download_code) || authentication_abilities.include?(:build_download_code) + raise UnauthorizedError, ERROR_MESSAGES[:auth_download] + end + when *PUSH_COMMANDS + unless authentication_abilities.include?(:push_code) + raise UnauthorizedError, ERROR_MESSAGES[:auth_upload] + end + end + end + def check_project_accessibility! if project.blank? || !can_read_project? raise NotFoundError, ERROR_MESSAGES[:project_not_found] @@ -104,12 +128,12 @@ module Gitlab def check_project_moved! return if redirected_path.nil? - project_moved = Checks::ProjectMoved.new(project, user, redirected_path, protocol) + project_moved = Checks::ProjectMoved.new(project, user, protocol, redirected_path) if project_moved.permanent_redirect? - project_moved.add_redirect_message + project_moved.add_message else - raise ProjectMovedError, project_moved.redirect_message(rejected: true) + raise ProjectMovedError, project_moved.message(rejected: true) end end @@ -139,6 +163,40 @@ module Gitlab end end + def check_db_accessibility!(cmd) + return unless receive_pack?(cmd) + + if Gitlab::Database.read_only? + raise UnauthorizedError, push_to_read_only_message + end + end + + def ensure_project_on_push!(cmd, changes) + return if project || deploy_key? + return unless receive_pack?(cmd) && changes == '_any' && authentication_abilities.include?(:push_code) + + namespace = Namespace.find_by_full_path(namespace_path) + + return unless user&.can?(:create_projects, namespace) + + project_params = { + path: project_path, + namespace_id: namespace.id, + visibility_level: Gitlab::VisibilityLevel::PRIVATE + } + + project = Projects::CreateService.new(user, project_params).execute + + unless project.saved? + raise ProjectCreationError, "Could not create project: #{project.errors.full_messages.join(', ')}" + end + + @project = project + user_access.project = @project + + Checks::ProjectCreated.new(project, user, protocol).add_message + end + def check_repository_existence! unless project.repository.exists? raise UnauthorizedError, ERROR_MESSAGES[:no_repo] @@ -146,9 +204,8 @@ module Gitlab end def check_download_access! - return if deploy_key? - - passed = user_can_download_code? || + passed = deploy_key? || + user_can_download_code? || build_can_download_code? || guest_can_download_code? @@ -162,35 +219,21 @@ module Gitlab raise UnauthorizedError, ERROR_MESSAGES[:read_only] end - if Gitlab::Database.read_only? - raise UnauthorizedError, push_to_read_only_message - end - if deploy_key - check_deploy_key_push_access! + unless deploy_key.can_push_to?(project) + raise UnauthorizedError, ERROR_MESSAGES[:deploy_key_upload] + end elsif user - check_user_push_access! + # User access is verified in check_change_access! else raise UnauthorizedError, ERROR_MESSAGES[:upload] end - return if changes.blank? # Allow access. + return if changes.blank? # Allow access this is needed for EE. check_change_access!(changes) end - def check_user_push_access! - unless authentication_abilities.include?(:push_code) - raise UnauthorizedError, ERROR_MESSAGES[:upload] - end - end - - def check_deploy_key_push_access! - unless deploy_key.can_push_to?(project) - raise UnauthorizedError, ERROR_MESSAGES[:deploy_key_upload] - end - end - def check_change_access!(changes) changes_list = Gitlab::ChangesList.new(changes) diff --git a/lib/gitlab/gitaly_client/blob_service.rb b/lib/gitlab/gitaly_client/blob_service.rb index d70a1a7665e..dfa0fa43b0f 100644 --- a/lib/gitlab/gitaly_client/blob_service.rb +++ b/lib/gitlab/gitaly_client/blob_service.rb @@ -1,6 +1,8 @@ module Gitlab module GitalyClient class BlobService + include Gitlab::EncodingHelper + def initialize(repository) @gitaly_repo = repository.gitaly_repository end @@ -54,6 +56,30 @@ module Gitlab end end end + + def get_blobs(revision_paths, limit = -1) + return [] if revision_paths.empty? + + revision_paths.map! do |rev, path| + Gitaly::GetBlobsRequest::RevisionPath.new(revision: rev, path: encode_binary(path)) + end + + request = Gitaly::GetBlobsRequest.new( + repository: @gitaly_repo, + revision_paths: revision_paths, + limit: limit + ) + + response = GitalyClient.call( + @gitaly_repo.storage_name, + :blob_service, + :get_blobs, + request, + timeout: GitalyClient.default_timeout + ) + + GitalyClient::BlobsStitcher.new(response) + end end end end diff --git a/lib/gitlab/gitaly_client/blobs_stitcher.rb b/lib/gitlab/gitaly_client/blobs_stitcher.rb new file mode 100644 index 00000000000..5ca592ff812 --- /dev/null +++ b/lib/gitlab/gitaly_client/blobs_stitcher.rb @@ -0,0 +1,47 @@ +module Gitlab + module GitalyClient + class BlobsStitcher + include Enumerable + + def initialize(rpc_response) + @rpc_response = rpc_response + end + + def each + current_blob_data = nil + + @rpc_response.each do |msg| + begin + if msg.oid.blank? && msg.data.blank? + next + elsif msg.oid.present? + yield new_blob(current_blob_data) if current_blob_data + + current_blob_data = msg.to_h.slice(:oid, :path, :size, :revision, :mode) + current_blob_data[:data] = msg.data.dup + else + current_blob_data[:data] << msg.data + end + end + end + + yield new_blob(current_blob_data) if current_blob_data + end + + private + + def new_blob(blob_data) + Gitlab::Git::Blob.new( + id: blob_data[:oid], + mode: blob_data[:mode].to_s(8), + name: File.basename(blob_data[:path]), + path: blob_data[:path], + size: blob_data[:size], + commit_id: blob_data[:revision], + data: blob_data[:data], + binary: Gitlab::Git::Blob.binary?(blob_data[:data]) + ) + end + end + end +end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 5767f06b0ce..269a048cf5d 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -222,14 +222,25 @@ module Gitlab end def find_commit(revision) - request = Gitaly::FindCommitRequest.new( - repository: @gitaly_repo, - revision: encode_binary(revision) - ) - - response = GitalyClient.call(@repository.storage, :commit_service, :find_commit, request, timeout: GitalyClient.medium_timeout) - - response.commit + if RequestStore.active? + # We don't use RequeStstore.fetch(key) { ... } directly because `revision` + # can be a branch name, so we can't use it as a key as it could point + # to another commit later on (happens a lot in tests). + key = { + storage: @gitaly_repo.storage_name, + relative_path: @gitaly_repo.relative_path, + commit_id: revision + } + return RequestStore[key] if RequestStore.exist?(key) + + commit = call_find_commit(revision) + return unless commit + + key[:commit_id] = commit.id + RequestStore[key] = commit + else + call_find_commit(revision) + end end def patch(revision) @@ -346,6 +357,17 @@ module Gitlab def encode_repeated(a) Google::Protobuf::RepeatedField.new(:bytes, a.map { |s| encode_binary(s) } ) end + + def call_find_commit(revision) + request = Gitaly::FindCommitRequest.new( + repository: @gitaly_repo, + revision: encode_binary(revision) + ) + + response = GitalyClient.call(@repository.storage, :commit_service, :find_commit, request, timeout: GitalyClient.medium_timeout) + + response.commit + end end end end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 2daed10f678..9f404003125 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -27,6 +27,8 @@ project_tree: - :releases - project_members: - :user + - lfs_file_locks: + - :user - merge_requests: - notes: - :author diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index e40a001d20c..a3e1c66c19f 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -178,7 +178,7 @@ module Gitlab valid_username = ::Namespace.clean_path(username) uniquify = Uniquify.new - valid_username = uniquify.string(valid_username) { |s| !UserPathValidator.valid_path?(s) } + valid_username = uniquify.string(valid_username) { |s| !NamespacePathValidator.valid_path?(s) } name = auth_hash.name name = valid_username if name.strip.empty? diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index 7e5dfd33502..4dc38aae61e 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -171,24 +171,16 @@ module Gitlab @project_git_route_regex ||= /#{project_route_regex}\.git/.freeze end - def root_namespace_path_regex - @root_namespace_path_regex ||= %r{\A#{root_namespace_route_regex}/\z} - end - def full_namespace_path_regex @full_namespace_path_regex ||= %r{\A#{full_namespace_route_regex}/\z} end - def project_path_regex - @project_path_regex ||= %r{\A#{project_route_regex}/\z} - end - def full_project_path_regex @full_project_path_regex ||= %r{\A#{full_namespace_route_regex}/#{project_route_regex}/\z} end - def full_namespace_format_regex - @namespace_format_regex ||= /A#{FULL_NAMESPACE_FORMAT_REGEX}\z/.freeze + def full_project_git_path_regex + @full_project_git_path_regex ||= %r{\A\/?(?<namespace_path>#{full_namespace_route_regex})\/(?<project_path>#{project_route_regex})\.git\z} end def namespace_format_regex diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 4823f703ba4..9e2fa07a205 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -2,11 +2,12 @@ module Gitlab class ProjectSearchResults < SearchResults attr_reader :project, :repository_ref - def initialize(current_user, project, query, repository_ref = nil) + def initialize(current_user, project, query, repository_ref = nil, per_page: 20) @current_user = current_user @project = project @repository_ref = repository_ref.presence || project.default_branch @query = query + @per_page = per_page end def objects(scope, page = nil) diff --git a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb index 69d055c901c..294a6ae34ca 100644 --- a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb +++ b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb @@ -4,7 +4,7 @@ module Gitlab class AdditionalMetricsDeploymentQuery < BaseQuery include QueryAdditionalMetrics - def query(deployment_id) + def query(environment_id, deployment_id) Deployment.find_by(id: deployment_id).try do |deployment| query_metrics( common_query_context( diff --git a/lib/gitlab/prometheus/queries/deployment_query.rb b/lib/gitlab/prometheus/queries/deployment_query.rb index 170f483540e..6e6da593178 100644 --- a/lib/gitlab/prometheus/queries/deployment_query.rb +++ b/lib/gitlab/prometheus/queries/deployment_query.rb @@ -2,7 +2,7 @@ module Gitlab module Prometheus module Queries class DeploymentQuery < BaseQuery - def query(deployment_id) + def query(environment_id, deployment_id) Deployment.find_by(id: deployment_id).try do |deployment| environment_slug = deployment.environment.slug diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb index aa94614bf18..10527972663 100644 --- a/lib/gitlab/prometheus_client.rb +++ b/lib/gitlab/prometheus_client.rb @@ -3,10 +3,10 @@ module Gitlab # Helper methods to interact with Prometheus network services & resources class PrometheusClient - attr_reader :api_url + attr_reader :rest_client, :headers - def initialize(api_url:) - @api_url = api_url + def initialize(rest_client) + @rest_client = rest_client end def ping @@ -40,37 +40,40 @@ module Gitlab private def json_api_get(type, args = {}) - get(join_api_url(type, args)) + path = ['api', 'v1', type].join('/') + get(path, args) + rescue JSON::ParserError + raise PrometheusError, 'Parsing response failed' rescue Errno::ECONNREFUSED raise PrometheusError, 'Connection refused' end - def join_api_url(type, args = {}) - url = URI.parse(api_url) - rescue URI::Error - raise PrometheusError, "Invalid API URL: #{api_url}" - else - url.path = [url.path.sub(%r{/+\z}, ''), 'api', 'v1', type].join('/') - url.query = args.to_query - - url.to_s - end - - def get(url) - handle_response(HTTParty.get(url)) + def get(path, args) + response = rest_client[path].get(params: args) + handle_response(response) rescue SocketError - raise PrometheusError, "Can't connect to #{url}" + raise PrometheusError, "Can't connect to #{rest_client.url}" rescue OpenSSL::SSL::SSLError - raise PrometheusError, "#{url} contains invalid SSL data" - rescue HTTParty::Error + raise PrometheusError, "#{rest_client.url} contains invalid SSL data" + rescue RestClient::ExceptionWithResponse => ex + handle_exception_response(ex.response) + rescue RestClient::Exception raise PrometheusError, "Network connection error" end def handle_response(response) - if response.code == 200 && response['status'] == 'success' - response['data'] || {} - elsif response.code == 400 - raise PrometheusError, response['error'] || 'Bad data received' + json_data = JSON.parse(response.body) + if response.code == 200 && json_data['status'] == 'success' + json_data['data'] || {} + else + raise PrometheusError, "#{response.code} - #{response.body}" + end + end + + def handle_exception_response(response) + if response.code == 400 + json_data = JSON.parse(response.body) + raise PrometheusError, json_data['error'] || 'Bad data received' else raise PrometheusError, "#{response.code} - #{response.body}" end diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 7362514167f..5ad219179f3 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -10,6 +10,7 @@ module Gitlab @ref = opts.fetch(:ref, nil) @startline = opts.fetch(:startline, nil) @data = opts.fetch(:data, nil) + @per_page = opts.fetch(:per_page, 20) end def path @@ -21,7 +22,7 @@ module Gitlab end end - attr_reader :current_user, :query + attr_reader :current_user, :query, :per_page # Limit search results by passed projects # It allows us to search only for projects user has access to @@ -33,11 +34,12 @@ module Gitlab # query attr_reader :default_project_filter - def initialize(current_user, limit_projects, query, default_project_filter: false) + def initialize(current_user, limit_projects, query, default_project_filter: false, per_page: 20) @current_user = current_user @limit_projects = limit_projects || Project.all @query = query @default_project_filter = default_project_filter + @per_page = per_page end def objects(scope, page = nil, without_count = true) @@ -153,10 +155,6 @@ module Gitlab 'projects' end - def per_page - 20 - end - def project_ids_relation limit_projects.select(:id).reorder(nil) end diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb index 2bfb7caefd9..b89ae2505c9 100644 --- a/lib/gitlab/sidekiq_middleware/memory_killer.rb +++ b/lib/gitlab/sidekiq_middleware/memory_killer.rb @@ -45,7 +45,7 @@ module Gitlab private def get_rss - output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{pid})) + output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{pid}), Rails.root.to_s) return 0 unless status.zero? output.to_i diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb index f357488ac61..15eb1c41213 100644 --- a/lib/gitlab/user_access.rb +++ b/lib/gitlab/user_access.rb @@ -6,7 +6,8 @@ module Gitlab [user&.id, project&.id] end - attr_reader :user, :project + attr_reader :user + attr_accessor :project def initialize(user, project: nil) @user = user |