diff options
| author | Jose Ivan Vargas <jvargas@gitlab.com> | 2018-02-12 12:48:31 -0600 |
|---|---|---|
| committer | Jose Ivan Vargas <jvargas@gitlab.com> | 2018-02-12 12:48:31 -0600 |
| commit | c833a09d9156e26851746250460ddd2f091e73cd (patch) | |
| tree | 01791c2a8461b53c797b3d5b91858f98b07eb477 /lib | |
| parent | 46ae03628de47d1bef2683a3a5fe4963b3df7d52 (diff) | |
| parent | 0a22ff267b2d3d787d3da44d927caac7bd442832 (diff) | |
| download | gitlab-ce-c833a09d9156e26851746250460ddd2f091e73cd.tar.gz | |
Merge branch 'master' into jivl-update-katex
Diffstat (limited to 'lib')
58 files changed, 874 insertions, 306 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index f3f64244589..e953f3d2eca 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -146,6 +146,7 @@ module API mount ::API::Repositories mount ::API::Runner mount ::API::Runners + mount ::API::Search mount ::API::Services mount ::API::Settings mount ::API::SidekiqMetrics diff --git a/lib/api/entities.rb b/lib/api/entities.rb index e13463ec66b..7838de13c56 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -314,24 +314,20 @@ module API end end - class ProjectSnippet < Grape::Entity + class Snippet < Grape::Entity expose :id, :title, :file_name, :description expose :author, using: Entities::UserBasic expose :updated_at, :created_at - - expose :web_url do |snippet, options| + expose :project_id + expose :web_url do |snippet| Gitlab::UrlBuilder.build(snippet) end end - class PersonalSnippet < Grape::Entity - expose :id, :title, :file_name, :description - expose :author, using: Entities::UserBasic - expose :updated_at, :created_at + class ProjectSnippet < Snippet + end - expose :web_url do |snippet| - Gitlab::UrlBuilder.build(snippet) - end + class PersonalSnippet < Snippet expose :raw_url do |snippet| Gitlab::UrlBuilder.build(snippet) + "/raw" end @@ -1168,5 +1164,14 @@ module API class ApplicationWithSecret < Application expose :secret end + + class Blob < Grape::Entity + expose :basename + expose :data + expose :filename + expose :id + expose :ref + expose :startline + end end end diff --git a/lib/api/helpers/internal_helpers.rb b/lib/api/helpers/internal_helpers.rb index eb67de81a0d..cd59da6fc70 100644 --- a/lib/api/helpers/internal_helpers.rb +++ b/lib/api/helpers/internal_helpers.rb @@ -60,8 +60,20 @@ module API false end + def project_path + project&.path || project_path_match[:project_path] + end + + def namespace_path + project&.namespace&.full_path || project_path_match[:namespace_path] + end + private + def project_path_match + @project_path_match ||= params[:project].match(Gitlab::PathRegex.full_project_git_path_regex) || {} + end + # rubocop:disable Gitlab/ModuleWithInstanceVariables def set_project if params[:gl_repository] diff --git a/lib/api/helpers/pagination.rb b/lib/api/helpers/pagination.rb index bb70370ba77..09805049169 100644 --- a/lib/api/helpers/pagination.rb +++ b/lib/api/helpers/pagination.rb @@ -12,13 +12,16 @@ module API private def add_pagination_headers(paginated_data) - header 'X-Total', paginated_data.total_count.to_s - header 'X-Total-Pages', total_pages(paginated_data).to_s header 'X-Per-Page', paginated_data.limit_value.to_s header 'X-Page', paginated_data.current_page.to_s header 'X-Next-Page', paginated_data.next_page.to_s header 'X-Prev-Page', paginated_data.prev_page.to_s header 'Link', pagination_links(paginated_data) + + return if data_without_counts?(paginated_data) + + header 'X-Total', paginated_data.total_count.to_s + header 'X-Total-Pages', total_pages(paginated_data).to_s end def pagination_links(paginated_data) @@ -37,8 +40,10 @@ module API request_params[:page] = 1 links << %(<#{request_url}?#{request_params.to_query}>; rel="first") - request_params[:page] = total_pages(paginated_data) - links << %(<#{request_url}?#{request_params.to_query}>; rel="last") + unless data_without_counts?(paginated_data) + request_params[:page] = total_pages(paginated_data) + links << %(<#{request_url}?#{request_params.to_query}>; rel="last") + end links.join(', ') end @@ -55,6 +60,10 @@ module API relation end + + def data_without_counts?(paginated_data) + paginated_data.is_a?(Kaminari::PaginatableWithoutCount) + end end end end diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb index 3d0d1287407..fbe30192a16 100644 --- a/lib/api/helpers/runner.rb +++ b/lib/api/helpers/runner.rb @@ -3,7 +3,6 @@ module API module Runner JOB_TOKEN_HEADER = 'HTTP_JOB_TOKEN'.freeze JOB_TOKEN_PARAM = :token - UPDATE_RUNNER_EVERY = 10 * 60 def runner_registration_token_valid? ActiveSupport::SecurityUtils.variable_size_secure_compare(params[:token], @@ -18,30 +17,14 @@ module API def authenticate_runner! forbidden! unless current_runner + + current_runner.update_cached_info(get_runner_version_from_params) end def current_runner @runner ||= ::Ci::Runner.find_by_token(params[:token].to_s) end - def update_runner_info - return unless update_runner? - - current_runner.contacted_at = Time.now - current_runner.assign_attributes(get_runner_version_from_params) - current_runner.save if current_runner.changed? - end - - def update_runner? - # Use a random threshold to prevent beating DB updates. - # It generates a distribution between [40m, 80m]. - # - contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY) - - current_runner.contacted_at.nil? || - (Time.now - current_runner.contacted_at) >= contacted_at_max_age - end - def validate_job!(job) not_found! unless job diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 063f0d6599c..b3660e4a1d0 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -13,7 +13,7 @@ module API # key_id - ssh key id for Git over SSH # user_id - user id for Git over HTTP # protocol - Git access protocol being used, e.g. HTTP or SSH - # project - project path with namespace + # project - project full_path (not path on disk) # action - git action (git-upload-pack or git-receive-pack) # changes - changes as "oldrev newrev ref", see Gitlab::ChangesList post "/allowed" do @@ -42,11 +42,14 @@ module API end access_checker_klass = wiki? ? Gitlab::GitAccessWiki : Gitlab::GitAccess - access_checker = access_checker_klass - .new(actor, project, protocol, authentication_abilities: ssh_authentication_abilities, redirected_path: redirected_path) + access_checker = access_checker_klass.new(actor, project, + protocol, authentication_abilities: ssh_authentication_abilities, + namespace_path: namespace_path, project_path: project_path, + redirected_path: redirected_path) begin access_checker.check(params[:action], params[:changes]) + @project ||= access_checker.project rescue Gitlab::GitAccess::UnauthorizedError, Gitlab::GitAccess::NotFoundError => e return { status: false, message: e.message } end @@ -207,8 +210,11 @@ module API # A user is not guaranteed to be returned; an orphaned write deploy # key could be used if user - redirect_message = Gitlab::Checks::ProjectMoved.fetch_redirect_message(user.id, project.id) + redirect_message = Gitlab::Checks::ProjectMoved.fetch_message(user.id, project.id) + project_created_message = Gitlab::Checks::ProjectCreated.fetch_message(user.id, project.id) + output[:redirected_message] = redirect_message if redirect_message + output[:project_created_message] = project_created_message if project_created_message end output diff --git a/lib/api/runner.rb b/lib/api/runner.rb index 1f80646a2ea..5469cba69a6 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -78,7 +78,6 @@ module API post '/request' do authenticate_runner! no_content! unless current_runner.active? - update_runner_info if current_runner.runner_queue_value_latest?(params[:last_update]) header 'X-GitLab-Last-Update', params[:last_update] diff --git a/lib/api/search.rb b/lib/api/search.rb new file mode 100644 index 00000000000..b9982e03bb3 --- /dev/null +++ b/lib/api/search.rb @@ -0,0 +1,115 @@ +module API + class Search < Grape::API + include PaginationParams + + before { authenticate! } + + helpers do + SCOPE_ENTITY = { + merge_requests: Entities::MergeRequestBasic, + issues: Entities::IssueBasic, + projects: Entities::BasicProjectDetails, + milestones: Entities::Milestone, + notes: Entities::Note, + commits: Entities::Commit, + blobs: Entities::Blob, + wiki_blobs: Entities::Blob, + snippet_titles: Entities::Snippet, + snippet_blobs: Entities::Snippet + }.freeze + + def search(additional_params = {}) + search_params = { + scope: params[:scope], + search: params[:search], + snippets: snippets?, + page: params[:page], + per_page: params[:per_page] + }.merge(additional_params) + + results = SearchService.new(current_user, search_params).search_objects + + process_results(results) + end + + def process_results(results) + case params[:scope] + when 'wiki_blobs' + paginate(results).map { |blob| Gitlab::ProjectSearchResults.parse_search_result(blob) } + when 'blobs' + paginate(results).map { |blob| blob[1] } + else + paginate(results) + end + end + + def snippets? + %w(snippet_blobs snippet_titles).include?(params[:scope]).to_s + end + + def entity + SCOPE_ENTITY[params[:scope].to_sym] + end + end + + resource :search do + desc 'Search on GitLab' do + detail 'This feature was introduced in GitLab 10.5.' + end + params do + requires :search, type: String, desc: 'The expression it should be searched for' + requires :scope, + type: String, + desc: 'The scope of search, available scopes: + projects, issues, merge_requests, milestones, snippet_titles, snippet_blobs', + values: %w(projects issues merge_requests milestones snippet_titles snippet_blobs) + use :pagination + end + get do + present search, with: entity + end + end + + resource :groups, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + desc 'Search on GitLab' do + detail 'This feature was introduced in GitLab 10.5.' + end + params do + requires :id, type: String, desc: 'The ID of a group' + requires :search, type: String, desc: 'The expression it should be searched for' + requires :scope, + type: String, + desc: 'The scope of search, available scopes: + projects, issues, merge_requests, milestones', + values: %w(projects issues merge_requests milestones) + use :pagination + end + get ':id/-/search' do + group = find_group!(params[:id]) + + present search(group_id: group.id), with: entity + end + end + + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + desc 'Search on GitLab' do + detail 'This feature was introduced in GitLab 10.5.' + end + params do + requires :id, type: String, desc: 'The ID of a project' + requires :search, type: String, desc: 'The expression it should be searched for' + requires :scope, + type: String, + desc: 'The scope of search, available scopes: + issues, merge_requests, milestones, notes, wiki_blobs, commits, blobs', + values: %w(issues merge_requests milestones notes wiki_blobs commits blobs) + use :pagination + end + get ':id/-/search' do + project = find_project!(params[:id]) + + present search(project_id: project.id), with: entity + end + end + end +end diff --git a/lib/api/todos.rb b/lib/api/todos.rb index ffccfebe752..c6dbcf84e3a 100644 --- a/lib/api/todos.rb +++ b/lib/api/todos.rb @@ -60,7 +60,7 @@ module API end post ':id/mark_as_done' do TodoService.new.mark_todos_as_done_by_ids(params[:id], current_user) - todo = Todo.find(params[:id]) + todo = current_user.todos.find(params[:id]) present todo, with: Entities::Todo, current_user: current_user end diff --git a/lib/api/users.rb b/lib/api/users.rb index c7c2aa280d5..3cc12724b8a 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -18,6 +18,14 @@ module API User.find_by(id: id) || not_found!('User') end + def reorder_users(users) + if params[:order_by] && params[:sort] + users.reorder(params[:order_by] => params[:sort]) + else + users + end + end + params :optional_attributes do optional :skype, type: String, desc: 'The Skype username' optional :linkedin, type: String, desc: 'The LinkedIn username' @@ -35,6 +43,13 @@ module API optional :avatar, type: File, desc: 'Avatar image for user' all_or_none_of :extern_uid, :provider end + + params :sort_params do + optional :order_by, type: String, values: %w[id name username created_at updated_at], + default: 'id', desc: 'Return users ordered by a field' + optional :sort, type: String, values: %w[asc desc], default: 'desc', + desc: 'Return users sorted in ascending and descending order' + end end desc 'Get the list of users' do @@ -53,16 +68,18 @@ module API optional :created_before, type: DateTime, desc: 'Return users created before the specified time' all_or_none_of :extern_uid, :provider + use :sort_params use :pagination end get do authenticated_as_admin! if params[:external].present? || (params[:extern_uid].present? && params[:provider].present?) unless current_user&.admin? - params.except!(:created_after, :created_before) + params.except!(:created_after, :created_before, :order_by, :sort) end users = UsersFinder.new(current_user, params).execute + users = reorder_users(users) authorized = can?(current_user, :read_users_list) diff --git a/lib/api/v3/projects.rb b/lib/api/v3/projects.rb index c856ba99f09..7d8b1f369fe 100644 --- a/lib/api/v3/projects.rb +++ b/lib/api/v3/projects.rb @@ -174,7 +174,7 @@ module API use :pagination end get "/search/:query", requirements: { query: %r{[^/]+} } do - search_service = Search::GlobalService.new(current_user, search: params[:query]).execute + search_service = ::Search::GlobalService.new(current_user, search: params[:query]).execute projects = search_service.objects('projects', params[:page], false) projects = projects.reorder(params[:order_by] => params[:sort]) diff --git a/lib/api/v3/todos.rb b/lib/api/v3/todos.rb index 2f2cf259987..3e2c61f6dbd 100644 --- a/lib/api/v3/todos.rb +++ b/lib/api/v3/todos.rb @@ -12,7 +12,7 @@ module API end delete ':id' do TodoService.new.mark_todos_as_done_by_ids(params[:id], current_user) - todo = Todo.find(params[:id]) + todo = current_user.todos.find(params[:id]) present todo, with: ::API::Entities::Todo, current_user: current_user end diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index a79a0154846..0ac7e231b5b 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -14,23 +14,33 @@ module Banzai end def highlight_node(node) - code = node.text css_classes = 'code highlight js-syntax-highlight' - language = node.attr('lang') + lang = node.attr('lang') + retried = false - if use_rouge?(language) - lexer = lexer_for(language) + if use_rouge?(lang) + lexer = lexer_for(lang) language = lexer.tag + else + lexer = Rouge::Lexers::PlainText.new + language = lang + end + + begin + code = Rouge::Formatters::HTMLGitlab.format(lex(lexer, node.text), tag: language) + css_classes << " #{language}" if language + rescue + # Gracefully handle syntax highlighter bugs/errors to ensure users can + # still access an issue/comment/etc. First, retry with the plain text + # filter. If that fails, then just skip this entirely, but that would + # be a pretty bad upstream bug. + return if retried - begin - code = Rouge::Formatters::HTMLGitlab.format(lex(lexer, code), tag: language) - css_classes << " #{language}" - rescue - # Gracefully handle syntax highlighter bugs/errors to ensure - # users can still access an issue/comment/etc. + language = nil + lexer = Rouge::Lexers::PlainText.new + retried = true - language = nil - end + retry end highlighted = %(<pre class="#{css_classes}" lang="#{language}" v-pre="true"><code>#{code}</code></pre>) diff --git a/lib/carrier_wave_string_file.rb b/lib/carrier_wave_string_file.rb new file mode 100644 index 00000000000..6c848902e4a --- /dev/null +++ b/lib/carrier_wave_string_file.rb @@ -0,0 +1,5 @@ +class CarrierWaveStringFile < StringIO + def original_filename + "" + end +end diff --git a/lib/constraints/user_url_constrainer.rb b/lib/constraints/user_url_constrainer.rb index b7633aa7cbb..3b3ed1c6ddb 100644 --- a/lib/constraints/user_url_constrainer.rb +++ b/lib/constraints/user_url_constrainer.rb @@ -2,7 +2,7 @@ class UserUrlConstrainer def matches?(request) full_path = request.params[:username] - return false unless UserPathValidator.valid_path?(full_path) + return false unless NamespacePathValidator.valid_path?(full_path) User.find_by_full_path(full_path, follow_redirects: request.get?).present? end diff --git a/lib/gitlab/background_migration/create_fork_network_memberships_range.rb b/lib/gitlab/background_migration/create_fork_network_memberships_range.rb index 03b17b319fa..1b4a9e8a194 100644 --- a/lib/gitlab/background_migration/create_fork_network_memberships_range.rb +++ b/lib/gitlab/background_migration/create_fork_network_memberships_range.rb @@ -14,6 +14,14 @@ module Gitlab def perform(start_id, end_id) log("Creating memberships for forks: #{start_id} - #{end_id}") + insert_members(start_id, end_id) + + if missing_members?(start_id, end_id) + BackgroundMigrationWorker.perform_in(RESCHEDULE_DELAY, "CreateForkNetworkMembershipsRange", [start_id, end_id]) + end + end + + def insert_members(start_id, end_id) ActiveRecord::Base.connection.execute <<~INSERT_MEMBERS INSERT INTO fork_network_members (fork_network_id, project_id, forked_from_project_id) @@ -33,10 +41,9 @@ module Gitlab WHERE existing_members.project_id = forked_project_links.forked_to_project_id ) INSERT_MEMBERS - - if missing_members?(start_id, end_id) - BackgroundMigrationWorker.perform_in(RESCHEDULE_DELAY, "CreateForkNetworkMembershipsRange", [start_id, end_id]) - end + rescue ActiveRecord::RecordNotUnique => e + # `fork_network_member` was created concurrently in another migration + log(e.message) end def missing_members?(start_id, end_id) diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb index 945d70e7a24..d75e73dac10 100644 --- a/lib/gitlab/checks/change_access.rb +++ b/lib/gitlab/checks/change_access.rb @@ -31,13 +31,14 @@ module Gitlab @protocol = protocol end - def exec + def exec(skip_commits_check: false) return true if skip_authorization push_checks branch_checks tag_checks lfs_objects_exist_check + commits_check unless skip_commits_check true end @@ -117,6 +118,24 @@ module Gitlab end end + def commits_check + return if deletion? || newrev.nil? + + # n+1: https://gitlab.com/gitlab-org/gitlab-ee/issues/3593 + ::Gitlab::GitalyClient.allow_n_plus_1_calls do + commits.each do |commit| + commit_check.validate(commit, validations_for_commit(commit)) + end + end + + commit_check.validate_file_paths + end + + # Method overwritten in EE to inject custom validations + def validations_for_commit(_) + [] + end + private def updated_from_web? @@ -150,6 +169,14 @@ module Gitlab raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:lfs_objects_missing] end end + + def commit_check + @commit_check ||= Gitlab::Checks::CommitCheck.new(project, user_access.user, newrev, oldrev) + end + + def commits + project.repository.new_commits(newrev) + end end end end diff --git a/lib/gitlab/checks/commit_check.rb b/lib/gitlab/checks/commit_check.rb new file mode 100644 index 00000000000..ae0cd142378 --- /dev/null +++ b/lib/gitlab/checks/commit_check.rb @@ -0,0 +1,61 @@ +module Gitlab + module Checks + class CommitCheck + include Gitlab::Utils::StrongMemoize + + attr_reader :project, :user, :newrev, :oldrev + + def initialize(project, user, newrev, oldrev) + @project = project + @user = user + @newrev = user + @oldrev = user + @file_paths = [] + end + + def validate(commit, validations) + return if validations.empty? && path_validations.empty? + + commit.raw_deltas.each do |diff| + @file_paths << (diff.new_path || diff.old_path) + + validations.each do |validation| + if error = validation.call(diff) + raise ::Gitlab::GitAccess::UnauthorizedError, error + end + end + end + end + + def validate_file_paths + path_validations.each do |validation| + if error = validation.call(@file_paths) + raise ::Gitlab::GitAccess::UnauthorizedError, error + end + end + end + + private + + def validate_lfs_file_locks? + strong_memoize(:validate_lfs_file_locks) do + project.lfs_enabled? && project.lfs_file_locks.any? && newrev && oldrev + end + end + + def lfs_file_locks_validation + lambda do |paths| + lfs_lock = project.lfs_file_locks.where(path: paths).where.not(user_id: user.id).first + + if lfs_lock + return "The path '#{lfs_lock.path}' is locked in Git LFS by #{lfs_lock.user.name}" + end + end + end + + def path_validations + validate_lfs_file_locks? ? [lfs_file_locks_validation] : [] + end + end + end +end diff --git a/lib/gitlab/checks/force_push.rb b/lib/gitlab/checks/force_push.rb index dc5d285ea65..c9c3050cfc2 100644 --- a/lib/gitlab/checks/force_push.rb +++ b/lib/gitlab/checks/force_push.rb @@ -15,8 +15,8 @@ module Gitlab .ancestor?(oldrev, newrev) else Gitlab::Git::RevList.new( - path_to_repo: project.repository.path_to_repo, - oldrev: oldrev, newrev: newrev).missed_ref.present? + project.repository.raw, oldrev: oldrev, newrev: newrev + ).missed_ref.present? end end end diff --git a/lib/gitlab/checks/post_push_message.rb b/lib/gitlab/checks/post_push_message.rb new file mode 100644 index 00000000000..473c0385b34 --- /dev/null +++ b/lib/gitlab/checks/post_push_message.rb @@ -0,0 +1,46 @@ +module Gitlab + module Checks + class PostPushMessage + def initialize(project, user, protocol) + @project = project + @user = user + @protocol = protocol + end + + def self.fetch_message(user_id, project_id) + key = message_key(user_id, project_id) + + Gitlab::Redis::SharedState.with do |redis| + message = redis.get(key) + redis.del(key) + message + end + end + + def add_message + return unless user.present? && project.present? + + Gitlab::Redis::SharedState.with do |redis| + key = self.class.message_key(user.id, project.id) + redis.setex(key, 5.minutes, message) + end + end + + def message + raise NotImplementedError + end + + protected + + attr_reader :project, :user, :protocol + + def self.message_key(user_id, project_id) + raise NotImplementedError + end + + def url_to_repo + protocol == 'ssh' ? project.ssh_url_to_repo : project.http_url_to_repo + end + end + end +end diff --git a/lib/gitlab/checks/project_created.rb b/lib/gitlab/checks/project_created.rb new file mode 100644 index 00000000000..cec270d6a58 --- /dev/null +++ b/lib/gitlab/checks/project_created.rb @@ -0,0 +1,31 @@ +module Gitlab + module Checks + class ProjectCreated < PostPushMessage + PROJECT_CREATED = "project_created".freeze + + def message + <<~MESSAGE + + The private project #{project.full_path} was successfully created. + + To configure the remote, run: + git remote add origin #{url_to_repo} + + To view the project, visit: + #{project_url} + + MESSAGE + end + + private + + def self.message_key(user_id, project_id) + "#{PROJECT_CREATED}:#{user_id}:#{project_id}" + end + + def project_url + Gitlab::Routing.url_helpers.project_url(project) + end + end + end +end diff --git a/lib/gitlab/checks/project_moved.rb b/lib/gitlab/checks/project_moved.rb index dfb2f4d4054..3263790a876 100644 --- a/lib/gitlab/checks/project_moved.rb +++ b/lib/gitlab/checks/project_moved.rb @@ -1,38 +1,16 @@ module Gitlab module Checks - class ProjectMoved + class ProjectMoved < PostPushMessage REDIRECT_NAMESPACE = "redirect_namespace".freeze - def initialize(project, user, redirected_path, protocol) - @project = project - @user = user + def initialize(project, user, protocol, redirected_path) @redirected_path = redirected_path - @protocol = protocol - end - - def self.fetch_redirect_message(user_id, project_id) - redirect_key = redirect_message_key(user_id, project_id) - Gitlab::Redis::SharedState.with do |redis| - message = redis.get(redirect_key) - redis.del(redirect_key) - message - end - end - - def add_redirect_message - # Don't bother with sending a redirect message for anonymous clones - # because they never see it via the `/internal/post_receive` endpoint - return unless user.present? && project.present? - - Gitlab::Redis::SharedState.with do |redis| - key = self.class.redirect_message_key(user.id, project.id) - redis.setex(key, 5.minutes, redirect_message) - end + super(project, user, protocol) end - def redirect_message(rejected: false) - <<~MESSAGE.strip_heredoc + def message(rejected: false) + <<~MESSAGE Project '#{redirected_path}' was moved to '#{project.full_path}'. Please update your Git remote: @@ -47,17 +25,17 @@ module Gitlab private - attr_reader :project, :redirected_path, :protocol, :user + attr_reader :redirected_path - def self.redirect_message_key(user_id, project_id) + def self.message_key(user_id, project_id) "#{REDIRECT_NAMESPACE}:#{user_id}:#{project_id}" end def remote_url_message(rejected) if rejected - "git remote set-url origin #{url} and try again." + "git remote set-url origin #{url_to_repo} and try again." else - "git remote set-url origin #{url}" + "git remote set-url origin #{url_to_repo}" end end diff --git a/lib/gitlab/ci/config/loader.rb b/lib/gitlab/ci/config/loader.rb index e7d9f6a7761..141d2714cb6 100644 --- a/lib/gitlab/ci/config/loader.rb +++ b/lib/gitlab/ci/config/loader.rb @@ -6,6 +6,8 @@ module Gitlab def initialize(config) @config = YAML.safe_load(config, [Symbol], [], true) + rescue Psych::Exception => e + raise FormatError, e.message end def valid? diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index baf55b1fa07..f2e5124c8a8 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -52,12 +52,14 @@ module Gitlab end def exist? - current_path.present? || old_trace.present? + trace_artifact&.exists? || current_path.present? || old_trace.present? end def read stream = Gitlab::Ci::Trace::Stream.new do - if current_path + if trace_artifact + trace_artifact.open + elsif current_path File.open(current_path, "rb") elsif old_trace StringIO.new(old_trace) @@ -82,6 +84,8 @@ module Gitlab end def erase! + trace_artifact&.destroy + paths.each do |trace_path| FileUtils.rm(trace_path, force: true) end @@ -137,6 +141,10 @@ module Gitlab "#{job.id}.log" ) if job.project&.ci_id end + + def trace_artifact + job.job_artifacts_trace + end end end end diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 0bd78b03448..a7285ac8f9d 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -85,7 +85,7 @@ module Gitlab begin Gitlab::Ci::YamlProcessor.new(content) nil - rescue ValidationError, Psych::SyntaxError => e + rescue ValidationError => e e.message end end diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index c0edcabc6fd..6659efa0961 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -28,9 +28,9 @@ module Gitlab # encode and clean the bad chars message.replace clean(message) - rescue ArgumentError - return nil - rescue + rescue ArgumentError => e + return unless e.message.include?('unknown encoding name') + encoding = detect ? detect[:encoding] : "unknown" "--broken encoding: #{encoding}" end diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 4828301dbb9..b2fca2c16de 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -53,11 +53,7 @@ module Gitlab def batch(repository, blob_references, blob_size_limit: MAX_DATA_DISPLAY_SIZE) Gitlab::GitalyClient.migrate(:list_blobs_by_sha_path) do |is_enabled| if is_enabled - Gitlab::GitalyClient.allow_n_plus_1_calls do - blob_references.map do |sha, path| - find_by_gitaly(repository, sha, path, limit: blob_size_limit) - end - end + repository.gitaly_blob_client.get_blobs(blob_references, blob_size_limit).to_a else blob_references.map do |sha, path| find_by_rugged(repository, sha, path, limit: blob_size_limit) diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 768617e2cae..d95561fe1b2 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -402,15 +402,6 @@ module Gitlab end end - # Get a collection of Rugged::Reference objects for this commit. - # - # Ex. - # commit.ref(repo) - # - def refs(repo) - repo.refs_hash[id] - end - # Get ref names collection # # Ex. @@ -418,7 +409,7 @@ module Gitlab # def ref_names(repo) refs(repo).map do |ref| - ref.name.sub(%r{^refs/(heads|remotes|tags)/}, "") + ref.sub(%r{^refs/(heads|remotes|tags)/}, "") end end @@ -553,6 +544,15 @@ module Gitlab date: Google::Protobuf::Timestamp.new(seconds: author_or_committer[:time].to_i) ) end + + # Get a collection of Gitlab::Git::Ref objects for this commit. + # + # Ex. + # commit.ref(repo) + # + def refs(repo) + repo.refs_hash[id] + end end end end diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb index e29a1f7afa1..24f027d8da4 100644 --- a/lib/gitlab/git/hook.rb +++ b/lib/gitlab/git/hook.rb @@ -82,14 +82,20 @@ module Gitlab end def call_update_hook(gl_id, gl_username, oldrev, newrev, ref) - Dir.chdir(repo_path) do - env = { - 'GL_ID' => gl_id, - 'GL_USERNAME' => gl_username - } - stdout, stderr, status = Open3.capture3(env, path, ref, oldrev, newrev) - [status.success?, (stderr.presence || stdout).gsub(/\R/, "<br>").html_safe] - end + env = { + 'GL_ID' => gl_id, + 'GL_USERNAME' => gl_username, + 'PWD' => repo_path + } + + options = { + chdir: repo_path + } + + args = [ref, oldrev, newrev] + + stdout, stderr, status = Open3.capture3(env, path, *args, options) + [status.success?, (stderr.presence || stdout).gsub(/\R/, "<br>").html_safe] end def retrieve_error_message(stderr, stdout) diff --git a/lib/gitlab/git/lfs_changes.rb b/lib/gitlab/git/lfs_changes.rb index 732dd5d998a..48434047fce 100644 --- a/lib/gitlab/git/lfs_changes.rb +++ b/lib/gitlab/git/lfs_changes.rb @@ -25,8 +25,7 @@ module Gitlab private def rev_list - ::Gitlab::Git::RevList.new(path_to_repo: @repository.path_to_repo, - newrev: @newrev) + Gitlab::Git::RevList.new(@repository, newrev: @newrev) end end end diff --git a/lib/gitlab/git/lfs_pointer_file.rb b/lib/gitlab/git/lfs_pointer_file.rb new file mode 100644 index 00000000000..da12ed7d125 --- /dev/null +++ b/lib/gitlab/git/lfs_pointer_file.rb @@ -0,0 +1,25 @@ +module Gitlab + module Git + class LfsPointerFile + def initialize(data) + @data = data + end + + def pointer + @pointer ||= <<~FILE + version https://git-lfs.github.com/spec/v1 + oid sha256:#{sha256} + size #{size} + FILE + end + + def size + @size ||= @data.bytesize + end + + def sha256 + @sha256 ||= Digest::SHA256.hexdigest(@data) + end + end + end +end diff --git a/lib/gitlab/git/popen.rb b/lib/gitlab/git/popen.rb index e0bd2bbe47b..c1767046ff0 100644 --- a/lib/gitlab/git/popen.rb +++ b/lib/gitlab/git/popen.rb @@ -25,7 +25,7 @@ module Gitlab stdin.close if lazy_block - return lazy_block.call(stdout.lazy) + return [lazy_block.call(stdout.lazy), 0] else cmd_output << stdout.read end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index ab1362a3bb0..5f014e43c6f 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -128,6 +128,10 @@ module Gitlab raise NoRepository.new('no repository for such path') end + def cleanup + @rugged&.close + end + def circuit_breaker @circuit_breaker ||= Gitlab::Git::Storage::CircuitBreaker.for_storage(storage) end @@ -627,21 +631,18 @@ module Gitlab end end - # Get refs hash which key is SHA1 - # and value is a Rugged::Reference + # Get refs hash which key is is the commit id + # and value is a Gitlab::Git::Tag or Gitlab::Git::Branch + # Note that both inherit from Gitlab::Git::Ref def refs_hash - # Initialize only when first call - if @refs_hash.nil? - @refs_hash = Hash.new { |h, k| h[k] = [] } - - rugged.references.each do |r| - # Symbolic/remote references may not have an OID; skip over them - target_oid = r.target.try(:oid) - if target_oid - sha = rev_parse_target(target_oid).oid - @refs_hash[sha] << r - end - end + return @refs_hash if @refs_hash + + @refs_hash = Hash.new { |h, k| h[k] = [] } + + (tags + branches).each do |ref| + next unless ref.target && ref.name + + @refs_hash[ref.dereferenced_target.id] << ref.name end @refs_hash @@ -1427,6 +1428,26 @@ module Gitlab end end + def rev_list(including: [], excluding: [], objects: false, &block) + args = ['rev-list'] + + args.push(*rev_list_param(including)) + + exclude_param = *rev_list_param(excluding) + if exclude_param.any? + args.push('--not') + args.push(*exclude_param) + end + + args.push('--objects') if objects + + run_git!(args, lazy_block: block) + end + + def missed_ref(oldrev, newrev) + run_git!(['rev-list', '--max-count=1', oldrev, "^#{newrev}"]) + end + private def local_write_ref(ref_path, ref, old_ref: nil, shell: true) @@ -1475,7 +1496,7 @@ module Gitlab Rails.logger.error "Unable to create #{ref_path} reference for repository #{path}: #{ex}" end - def run_git(args, chdir: path, env: {}, nice: false, &block) + def run_git(args, chdir: path, env: {}, nice: false, lazy_block: nil, &block) cmd = [Gitlab.config.git.bin_path, *args] cmd.unshift("nice") if nice @@ -1485,12 +1506,12 @@ module Gitlab end circuit_breaker.perform do - popen(cmd, chdir, env, &block) + popen(cmd, chdir, env, lazy_block: lazy_block, &block) end end - def run_git!(args, chdir: path, env: {}, nice: false, &block) - output, status = run_git(args, chdir: chdir, env: env, nice: nice, &block) + def run_git!(args, chdir: path, env: {}, nice: false, lazy_block: nil, &block) + output, status = run_git(args, chdir: chdir, env: env, nice: nice, lazy_block: lazy_block, &block) raise GitError, output unless status.zero? @@ -1593,17 +1614,14 @@ module Gitlab # Gitaly note: JV: Trying to get rid of the 'filter' option so we can implement this with 'git'. def branches_filter(filter: nil, sort_by: nil) - # n+1: https://gitlab.com/gitlab-org/gitlab-ce/issues/37464 - branches = Gitlab::GitalyClient.allow_n_plus_1_calls do - rugged.branches.each(filter).map do |rugged_ref| - begin - target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) - Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) - rescue Rugged::ReferenceError - # Omit invalid branch - end - end.compact - end + branches = rugged.branches.each(filter).map do |rugged_ref| + begin + target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) + Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) + rescue Rugged::ReferenceError + # Omit invalid branch + end + end.compact sort_branches(branches, sort_by) end @@ -2372,6 +2390,10 @@ module Gitlab rescue Rugged::ReferenceError 0 end + + def rev_list_param(spec) + spec == :all ? ['--all'] : spec + end end end end diff --git a/lib/gitlab/git/rev_list.rb b/lib/gitlab/git/rev_list.rb index f8b2e7e0e21..38c3a55f96f 100644 --- a/lib/gitlab/git/rev_list.rb +++ b/lib/gitlab/git/rev_list.rb @@ -5,17 +5,17 @@ module Gitlab class RevList include Gitlab::Git::Popen - attr_reader :oldrev, :newrev, :path_to_repo + attr_reader :oldrev, :newrev, :repository - def initialize(path_to_repo:, newrev:, oldrev: nil) + def initialize(repository, newrev:, oldrev: nil) @oldrev = oldrev @newrev = newrev - @path_to_repo = path_to_repo + @repository = repository end # This method returns an array of new commit references def new_refs - execute([*base_args, newrev, '--not', '--all']) + repository.rev_list(including: newrev, excluding: :all).split("\n") end # Finds newly added objects @@ -28,66 +28,39 @@ module Gitlab # When given a block it will yield objects as a lazy enumerator so # the caller can limit work done instead of processing megabytes of data def new_objects(require_path: nil, not_in: nil, &lazy_block) - args = [*base_args, newrev, *not_in_refs(not_in), '--objects'] + opts = { + including: newrev, + excluding: not_in.nil? ? :all : not_in, + require_path: require_path + } - get_objects(args, require_path: require_path, &lazy_block) + get_objects(opts, &lazy_block) end def all_objects(require_path: nil, &lazy_block) - args = [*base_args, '--all', '--objects'] - - get_objects(args, require_path: require_path, &lazy_block) + get_objects(including: :all, require_path: require_path, &lazy_block) end # This methods returns an array of missed references # # Should become obsolete after https://gitlab.com/gitlab-org/gitaly/issues/348. def missed_ref - execute([*base_args, '--max-count=1', oldrev, "^#{newrev}"]) + repository.missed_ref(oldrev, newrev).split("\n") end private - def not_in_refs(references) - return ['--not', '--all'] unless references - return [] if references.empty? - - references.prepend('--not') - end - def execute(args) - output, status = popen(args, nil, Gitlab::Git::Env.to_env_hash) - - unless status.zero? - raise "Got a non-zero exit code while calling out `#{args.join(' ')}`: #{output}" - end - - output.split("\n") - end - - def lazy_execute(args, &lazy_block) - popen(args, nil, Gitlab::Git::Env.to_env_hash, lazy_block: lazy_block) - end - - def base_args - [ - Gitlab.config.git.bin_path, - "--git-dir=#{path_to_repo}", - 'rev-list' - ] + repository.rev_list(args).split("\n") end - def get_objects(args, require_path: nil) - if block_given? - lazy_execute(args) do |lazy_output| - objects = objects_from_output(lazy_output, require_path: require_path) + def get_objects(including: [], excluding: [], require_path: nil) + opts = { including: including, excluding: excluding, objects: true } - yield(objects) - end - else - object_output = execute(args) + repository.rev_list(opts) do |lazy_output| + objects = objects_from_output(lazy_output, require_path: require_path) - objects_from_output(object_output, require_path: require_path) + yield(objects) end end diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index daa17fc72cf..ac12271a87e 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -25,9 +25,8 @@ module Gitlab @repository.exists? end - # Disabled because of https://gitlab.com/gitlab-org/gitaly/merge_requests/539 def write_page(name, format, content, commit_details) - @repository.gitaly_migrate(:wiki_write_page, status: Gitlab::GitalyClient::MigrationStatus::DISABLED) do |is_enabled| + @repository.gitaly_migrate(:wiki_write_page) do |is_enabled| if is_enabled gitaly_write_page(name, format, content, commit_details) gollum_wiki.clear_cache @@ -48,9 +47,8 @@ module Gitlab end end - # Disable because of https://gitlab.com/gitlab-org/gitlab-ce/issues/42094 def update_page(page_path, title, format, content, commit_details) - @repository.gitaly_migrate(:wiki_update_page, status: Gitlab::GitalyClient::MigrationStatus::DISABLED) do |is_enabled| + @repository.gitaly_migrate(:wiki_update_page) do |is_enabled| if is_enabled gitaly_update_page(page_path, title, format, content, commit_details) gollum_wiki.clear_cache @@ -96,11 +94,23 @@ module Gitlab # :per_page - The number of items per page. # :limit - Total number of items to return. def page_versions(page_path, options = {}) - current_page = gollum_page_by_path(page_path) + @repository.gitaly_migrate(:wiki_page_versions) do |is_enabled| + if is_enabled + versions = gitaly_wiki_client.page_versions(page_path, options) + + # Gitaly uses gollum-lib to get the versions. Gollum defaults to 20 + # per page, but also fetches 20 if `limit` or `per_page` < 20. + # Slicing returns an array with the expected number of items. + slice_bound = options[:limit] || options[:per_page] || Gollum::Page.per_page + versions[0..slice_bound] + else + current_page = gollum_page_by_path(page_path) - commits_from_page(current_page, options).map do |gitlab_git_commit| - gollum_page = gollum_wiki.page(current_page.title, gitlab_git_commit.id) - Gitlab::Git::WikiPageVersion.new(gitlab_git_commit, gollum_page&.format) + commits_from_page(current_page, options).map do |gitlab_git_commit| + gollum_page = gollum_wiki.page(current_page.title, gitlab_git_commit.id) + Gitlab::Git::WikiPageVersion.new(gitlab_git_commit, gollum_page&.format) + end + end end end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 56f6febe86d..8ec3386184a 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -2,15 +2,19 @@ # class return an instance of `GitlabAccessStatus` module Gitlab class GitAccess + include Gitlab::Utils::StrongMemoize + UnauthorizedError = Class.new(StandardError) NotFoundError = Class.new(StandardError) + ProjectCreationError = Class.new(StandardError) ProjectMovedError = Class.new(NotFoundError) ERROR_MESSAGES = { upload: 'You are not allowed to upload code for this project.', download: 'You are not allowed to download code from this project.', - deploy_key_upload: - 'This deploy key does not have write access to this project.', + auth_upload: 'You are not allowed to upload code.', + auth_download: 'You are not allowed to download code.', + deploy_key_upload: 'This deploy key does not have write access to this project.', no_repo: 'A repository for this project does not exist yet.', project_not_found: 'The project you were looking for could not be found.', account_blocked: 'Your account has been blocked.', @@ -25,24 +29,31 @@ module Gitlab PUSH_COMMANDS = %w{ git-receive-pack }.freeze ALL_COMMANDS = DOWNLOAD_COMMANDS + PUSH_COMMANDS - attr_reader :actor, :project, :protocol, :authentication_abilities, :redirected_path + attr_reader :actor, :project, :protocol, :authentication_abilities, :namespace_path, :project_path, :redirected_path - def initialize(actor, project, protocol, authentication_abilities:, redirected_path: nil) + def initialize(actor, project, protocol, authentication_abilities:, namespace_path: nil, project_path: nil, redirected_path: nil) @actor = actor @project = project @protocol = protocol - @redirected_path = redirected_path @authentication_abilities = authentication_abilities + @namespace_path = namespace_path + @project_path = project_path + @redirected_path = redirected_path end def check(cmd, changes) check_protocol! check_valid_actor! check_active_user! - check_project_accessibility! - check_project_moved! + check_authentication_abilities!(cmd) check_command_disabled!(cmd) check_command_existence!(cmd) + check_db_accessibility!(cmd) + + ensure_project_on_push!(cmd, changes) + + check_project_accessibility! + check_project_moved! check_repository_existence! case cmd @@ -95,6 +106,19 @@ module Gitlab end end + def check_authentication_abilities!(cmd) + case cmd + when *DOWNLOAD_COMMANDS + unless authentication_abilities.include?(:download_code) || authentication_abilities.include?(:build_download_code) + raise UnauthorizedError, ERROR_MESSAGES[:auth_download] + end + when *PUSH_COMMANDS + unless authentication_abilities.include?(:push_code) + raise UnauthorizedError, ERROR_MESSAGES[:auth_upload] + end + end + end + def check_project_accessibility! if project.blank? || !can_read_project? raise NotFoundError, ERROR_MESSAGES[:project_not_found] @@ -104,12 +128,12 @@ module Gitlab def check_project_moved! return if redirected_path.nil? - project_moved = Checks::ProjectMoved.new(project, user, redirected_path, protocol) + project_moved = Checks::ProjectMoved.new(project, user, protocol, redirected_path) if project_moved.permanent_redirect? - project_moved.add_redirect_message + project_moved.add_message else - raise ProjectMovedError, project_moved.redirect_message(rejected: true) + raise ProjectMovedError, project_moved.message(rejected: true) end end @@ -139,6 +163,40 @@ module Gitlab end end + def check_db_accessibility!(cmd) + return unless receive_pack?(cmd) + + if Gitlab::Database.read_only? + raise UnauthorizedError, push_to_read_only_message + end + end + + def ensure_project_on_push!(cmd, changes) + return if project || deploy_key? + return unless receive_pack?(cmd) && changes == '_any' && authentication_abilities.include?(:push_code) + + namespace = Namespace.find_by_full_path(namespace_path) + + return unless user&.can?(:create_projects, namespace) + + project_params = { + path: project_path, + namespace_id: namespace.id, + visibility_level: Gitlab::VisibilityLevel::PRIVATE + } + + project = Projects::CreateService.new(user, project_params).execute + + unless project.saved? + raise ProjectCreationError, "Could not create project: #{project.errors.full_messages.join(', ')}" + end + + @project = project + user_access.project = @project + + Checks::ProjectCreated.new(project, user, protocol).add_message + end + def check_repository_existence! unless project.repository.exists? raise UnauthorizedError, ERROR_MESSAGES[:no_repo] @@ -146,9 +204,8 @@ module Gitlab end def check_download_access! - return if deploy_key? - - passed = user_can_download_code? || + passed = deploy_key? || + user_can_download_code? || build_can_download_code? || guest_can_download_code? @@ -162,35 +219,21 @@ module Gitlab raise UnauthorizedError, ERROR_MESSAGES[:read_only] end - if Gitlab::Database.read_only? - raise UnauthorizedError, push_to_read_only_message - end - if deploy_key - check_deploy_key_push_access! + unless deploy_key.can_push_to?(project) + raise UnauthorizedError, ERROR_MESSAGES[:deploy_key_upload] + end elsif user - check_user_push_access! + # User access is verified in check_change_access! else raise UnauthorizedError, ERROR_MESSAGES[:upload] end - return if changes.blank? # Allow access. + return if changes.blank? # Allow access this is needed for EE. check_change_access!(changes) end - def check_user_push_access! - unless authentication_abilities.include?(:push_code) - raise UnauthorizedError, ERROR_MESSAGES[:upload] - end - end - - def check_deploy_key_push_access! - unless deploy_key.can_push_to?(project) - raise UnauthorizedError, ERROR_MESSAGES[:deploy_key_upload] - end - end - def check_change_access!(changes) changes_list = Gitlab::ChangesList.new(changes) diff --git a/lib/gitlab/gitaly_client/blob_service.rb b/lib/gitlab/gitaly_client/blob_service.rb index d70a1a7665e..dfa0fa43b0f 100644 --- a/lib/gitlab/gitaly_client/blob_service.rb +++ b/lib/gitlab/gitaly_client/blob_service.rb @@ -1,6 +1,8 @@ module Gitlab module GitalyClient class BlobService + include Gitlab::EncodingHelper + def initialize(repository) @gitaly_repo = repository.gitaly_repository end @@ -54,6 +56,30 @@ module Gitlab end end end + + def get_blobs(revision_paths, limit = -1) + return [] if revision_paths.empty? + + revision_paths.map! do |rev, path| + Gitaly::GetBlobsRequest::RevisionPath.new(revision: rev, path: encode_binary(path)) + end + + request = Gitaly::GetBlobsRequest.new( + repository: @gitaly_repo, + revision_paths: revision_paths, + limit: limit + ) + + response = GitalyClient.call( + @gitaly_repo.storage_name, + :blob_service, + :get_blobs, + request, + timeout: GitalyClient.default_timeout + ) + + GitalyClient::BlobsStitcher.new(response) + end end end end diff --git a/lib/gitlab/gitaly_client/blobs_stitcher.rb b/lib/gitlab/gitaly_client/blobs_stitcher.rb new file mode 100644 index 00000000000..5ca592ff812 --- /dev/null +++ b/lib/gitlab/gitaly_client/blobs_stitcher.rb @@ -0,0 +1,47 @@ +module Gitlab + module GitalyClient + class BlobsStitcher + include Enumerable + + def initialize(rpc_response) + @rpc_response = rpc_response + end + + def each + current_blob_data = nil + + @rpc_response.each do |msg| + begin + if msg.oid.blank? && msg.data.blank? + next + elsif msg.oid.present? + yield new_blob(current_blob_data) if current_blob_data + + current_blob_data = msg.to_h.slice(:oid, :path, :size, :revision, :mode) + current_blob_data[:data] = msg.data.dup + else + current_blob_data[:data] << msg.data + end + end + end + + yield new_blob(current_blob_data) if current_blob_data + end + + private + + def new_blob(blob_data) + Gitlab::Git::Blob.new( + id: blob_data[:oid], + mode: blob_data[:mode].to_s(8), + name: File.basename(blob_data[:path]), + path: blob_data[:path], + size: blob_data[:size], + commit_id: blob_data[:revision], + data: blob_data[:data], + binary: Gitlab::Git::Blob.binary?(blob_data[:data]) + ) + end + end + end +end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 5767f06b0ce..269a048cf5d 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -222,14 +222,25 @@ module Gitlab end def find_commit(revision) - request = Gitaly::FindCommitRequest.new( - repository: @gitaly_repo, - revision: encode_binary(revision) - ) - - response = GitalyClient.call(@repository.storage, :commit_service, :find_commit, request, timeout: GitalyClient.medium_timeout) - - response.commit + if RequestStore.active? + # We don't use RequeStstore.fetch(key) { ... } directly because `revision` + # can be a branch name, so we can't use it as a key as it could point + # to another commit later on (happens a lot in tests). + key = { + storage: @gitaly_repo.storage_name, + relative_path: @gitaly_repo.relative_path, + commit_id: revision + } + return RequestStore[key] if RequestStore.exist?(key) + + commit = call_find_commit(revision) + return unless commit + + key[:commit_id] = commit.id + RequestStore[key] = commit + else + call_find_commit(revision) + end end def patch(revision) @@ -346,6 +357,17 @@ module Gitlab def encode_repeated(a) Google::Protobuf::RepeatedField.new(:bytes, a.map { |s| encode_binary(s) } ) end + + def call_find_commit(revision) + request = Gitaly::FindCommitRequest.new( + repository: @gitaly_repo, + revision: encode_binary(revision) + ) + + response = GitalyClient.call(@repository.storage, :commit_service, :find_commit, request, timeout: GitalyClient.medium_timeout) + + response.commit + end end end end diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb index 8e87a8cc36f..0d8dd5cb8f4 100644 --- a/lib/gitlab/gitaly_client/wiki_service.rb +++ b/lib/gitlab/gitaly_client/wiki_service.rb @@ -101,6 +101,30 @@ module Gitlab pages end + # options: + # :page - The Integer page number. + # :per_page - The number of items per page. + # :limit - Total number of items to return. + def page_versions(page_path, options) + request = Gitaly::WikiGetPageVersionsRequest.new( + repository: @gitaly_repo, + page_path: encode_binary(page_path), + page: options[:page] || 1, + per_page: options[:per_page] || Gollum::Page.per_page + ) + + stream = GitalyClient.call(@repository.storage, :wiki_service, :wiki_get_page_versions, request) + + versions = [] + stream.each do |message| + message.versions.each do |version| + versions << new_wiki_page_version(version) + end + end + + versions + end + def find_file(name, revision) request = Gitaly::WikiFindFileRequest.new( repository: @gitaly_repo, @@ -141,7 +165,7 @@ module Gitlab private - # If a block is given and the yielded value is true, iteration will be + # If a block is given and the yielded value is truthy, iteration will be # stopped early at that point; else the iterator is consumed entirely. # The iterator is traversed with `next` to allow resuming the iteration. def wiki_page_from_iterator(iterator) @@ -158,10 +182,7 @@ module Gitlab else wiki_page = GitalyClient::WikiPage.new(page.to_h) - version = Gitlab::Git::WikiPageVersion.new( - Gitlab::Git::Commit.decorate(@repository, page.version.commit), - page.version.format - ) + version = new_wiki_page_version(page.version) end end @@ -170,6 +191,13 @@ module Gitlab [wiki_page, version] end + def new_wiki_page_version(version) + Gitlab::Git::WikiPageVersion.new( + Gitlab::Git::Commit.decorate(@repository, version.commit), + version.format + ) + end + def gitaly_commit_details(commit_details) Gitaly::WikiCommitDetails.new( name: encode_binary(commit_details.name), diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 2daed10f678..9f404003125 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -27,6 +27,8 @@ project_tree: - :releases - project_members: - :user + - lfs_file_locks: + - :user - merge_requests: - notes: - :author diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index c14646b0611..a00795f553e 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -50,9 +50,10 @@ module Gitlab end def wiki_restorer - Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: wiki_repo_path, + Gitlab::ImportExport::WikiRestorer.new(path_to_bundle: wiki_repo_path, shared: @shared, - project: ProjectWiki.new(project_tree.restored_project)) + project: ProjectWiki.new(project_tree.restored_project), + wiki_enabled: @project.wiki_enabled?) end def uploads_restorer diff --git a/lib/gitlab/import_export/wiki_restorer.rb b/lib/gitlab/import_export/wiki_restorer.rb new file mode 100644 index 00000000000..f33bfb332ab --- /dev/null +++ b/lib/gitlab/import_export/wiki_restorer.rb @@ -0,0 +1,23 @@ +module Gitlab + module ImportExport + class WikiRestorer < RepoRestorer + def initialize(project:, shared:, path_to_bundle:, wiki_enabled:) + super(project: project, shared: shared, path_to_bundle: path_to_bundle) + + @wiki_enabled = wiki_enabled + end + + def restore + @project.wiki if create_empty_wiki? + + super + end + + private + + def create_empty_wiki? + !File.exist?(@path_to_bundle) && @wiki_enabled + end + end + end +end diff --git a/lib/gitlab/ldap/auth_hash.rb b/lib/gitlab/ldap/auth_hash.rb index 1bd0965679a..96171dc26c4 100644 --- a/lib/gitlab/ldap/auth_hash.rb +++ b/lib/gitlab/ldap/auth_hash.rb @@ -7,6 +7,12 @@ module Gitlab @uid ||= Gitlab::LDAP::Person.normalize_dn(super) end + def username + super.tap do |username| + username.downcase! if ldap_config.lowercase_usernames + end + end + private def get_info(key) diff --git a/lib/gitlab/ldap/config.rb b/lib/gitlab/ldap/config.rb index cde60addcf7..47b3fce3e7a 100644 --- a/lib/gitlab/ldap/config.rb +++ b/lib/gitlab/ldap/config.rb @@ -139,6 +139,10 @@ module Gitlab options['allow_username_or_email_login'] end + def lowercase_usernames + options['lowercase_usernames'] + end + def name_proc if allow_username_or_email_login proc { |name| name.gsub(/@.*\z/, '') } diff --git a/lib/gitlab/ldap/person.rb b/lib/gitlab/ldap/person.rb index e81cec6ba1a..b91757c2a4b 100644 --- a/lib/gitlab/ldap/person.rb +++ b/lib/gitlab/ldap/person.rb @@ -82,7 +82,9 @@ module Gitlab # be returned. We need only one for username. # Ex. `uid` returns only one value but `mail` may # return an array of multiple email addresses. - [username].flatten.first + [username].flatten.first.tap do |username| + username.downcase! if config.lowercase_usernames + end end def email diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index e40a001d20c..a3e1c66c19f 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -178,7 +178,7 @@ module Gitlab valid_username = ::Namespace.clean_path(username) uniquify = Uniquify.new - valid_username = uniquify.string(valid_username) { |s| !UserPathValidator.valid_path?(s) } + valid_username = uniquify.string(valid_username) { |s| !NamespacePathValidator.valid_path?(s) } name = auth_hash.name name = valid_username if name.strip.empty? diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index 7e5dfd33502..4dc38aae61e 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -171,24 +171,16 @@ module Gitlab @project_git_route_regex ||= /#{project_route_regex}\.git/.freeze end - def root_namespace_path_regex - @root_namespace_path_regex ||= %r{\A#{root_namespace_route_regex}/\z} - end - def full_namespace_path_regex @full_namespace_path_regex ||= %r{\A#{full_namespace_route_regex}/\z} end - def project_path_regex - @project_path_regex ||= %r{\A#{project_route_regex}/\z} - end - def full_project_path_regex @full_project_path_regex ||= %r{\A#{full_namespace_route_regex}/#{project_route_regex}/\z} end - def full_namespace_format_regex - @namespace_format_regex ||= /A#{FULL_NAMESPACE_FORMAT_REGEX}\z/.freeze + def full_project_git_path_regex + @full_project_git_path_regex ||= %r{\A\/?(?<namespace_path>#{full_namespace_route_regex})\/(?<project_path>#{project_route_regex})\.git\z} end def namespace_format_regex diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 4823f703ba4..9e2fa07a205 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -2,11 +2,12 @@ module Gitlab class ProjectSearchResults < SearchResults attr_reader :project, :repository_ref - def initialize(current_user, project, query, repository_ref = nil) + def initialize(current_user, project, query, repository_ref = nil, per_page: 20) @current_user = current_user @project = project @repository_ref = repository_ref.presence || project.default_branch @query = query + @per_page = per_page end def objects(scope, page = nil) diff --git a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb index 69d055c901c..294a6ae34ca 100644 --- a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb +++ b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb @@ -4,7 +4,7 @@ module Gitlab class AdditionalMetricsDeploymentQuery < BaseQuery include QueryAdditionalMetrics - def query(deployment_id) + def query(environment_id, deployment_id) Deployment.find_by(id: deployment_id).try do |deployment| query_metrics( common_query_context( diff --git a/lib/gitlab/prometheus/queries/deployment_query.rb b/lib/gitlab/prometheus/queries/deployment_query.rb index 170f483540e..6e6da593178 100644 --- a/lib/gitlab/prometheus/queries/deployment_query.rb +++ b/lib/gitlab/prometheus/queries/deployment_query.rb @@ -2,7 +2,7 @@ module Gitlab module Prometheus module Queries class DeploymentQuery < BaseQuery - def query(deployment_id) + def query(environment_id, deployment_id) Deployment.find_by(id: deployment_id).try do |deployment| environment_slug = deployment.environment.slug diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb index aa94614bf18..10527972663 100644 --- a/lib/gitlab/prometheus_client.rb +++ b/lib/gitlab/prometheus_client.rb @@ -3,10 +3,10 @@ module Gitlab # Helper methods to interact with Prometheus network services & resources class PrometheusClient - attr_reader :api_url + attr_reader :rest_client, :headers - def initialize(api_url:) - @api_url = api_url + def initialize(rest_client) + @rest_client = rest_client end def ping @@ -40,37 +40,40 @@ module Gitlab private def json_api_get(type, args = {}) - get(join_api_url(type, args)) + path = ['api', 'v1', type].join('/') + get(path, args) + rescue JSON::ParserError + raise PrometheusError, 'Parsing response failed' rescue Errno::ECONNREFUSED raise PrometheusError, 'Connection refused' end - def join_api_url(type, args = {}) - url = URI.parse(api_url) - rescue URI::Error - raise PrometheusError, "Invalid API URL: #{api_url}" - else - url.path = [url.path.sub(%r{/+\z}, ''), 'api', 'v1', type].join('/') - url.query = args.to_query - - url.to_s - end - - def get(url) - handle_response(HTTParty.get(url)) + def get(path, args) + response = rest_client[path].get(params: args) + handle_response(response) rescue SocketError - raise PrometheusError, "Can't connect to #{url}" + raise PrometheusError, "Can't connect to #{rest_client.url}" rescue OpenSSL::SSL::SSLError - raise PrometheusError, "#{url} contains invalid SSL data" - rescue HTTParty::Error + raise PrometheusError, "#{rest_client.url} contains invalid SSL data" + rescue RestClient::ExceptionWithResponse => ex + handle_exception_response(ex.response) + rescue RestClient::Exception raise PrometheusError, "Network connection error" end def handle_response(response) - if response.code == 200 && response['status'] == 'success' - response['data'] || {} - elsif response.code == 400 - raise PrometheusError, response['error'] || 'Bad data received' + json_data = JSON.parse(response.body) + if response.code == 200 && json_data['status'] == 'success' + json_data['data'] || {} + else + raise PrometheusError, "#{response.code} - #{response.body}" + end + end + + def handle_exception_response(response) + if response.code == 400 + json_data = JSON.parse(response.body) + raise PrometheusError, json_data['error'] || 'Bad data received' else raise PrometheusError, "#{response.code} - #{response.body}" end diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 7362514167f..5ad219179f3 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -10,6 +10,7 @@ module Gitlab @ref = opts.fetch(:ref, nil) @startline = opts.fetch(:startline, nil) @data = opts.fetch(:data, nil) + @per_page = opts.fetch(:per_page, 20) end def path @@ -21,7 +22,7 @@ module Gitlab end end - attr_reader :current_user, :query + attr_reader :current_user, :query, :per_page # Limit search results by passed projects # It allows us to search only for projects user has access to @@ -33,11 +34,12 @@ module Gitlab # query attr_reader :default_project_filter - def initialize(current_user, limit_projects, query, default_project_filter: false) + def initialize(current_user, limit_projects, query, default_project_filter: false, per_page: 20) @current_user = current_user @limit_projects = limit_projects || Project.all @query = query @default_project_filter = default_project_filter + @per_page = per_page end def objects(scope, page = nil, without_count = true) @@ -153,10 +155,6 @@ module Gitlab 'projects' end - def per_page - 20 - end - def project_ids_relation limit_projects.select(:id).reorder(nil) end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index f4a41dc3eda..4ba44e0feef 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -294,7 +294,8 @@ module Gitlab # add_namespace("/path/to/storage", "gitlab") # def add_namespace(storage, name) - Gitlab::GitalyClient.migrate(:add_namespace) do |enabled| + Gitlab::GitalyClient.migrate(:add_namespace, + status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled| if enabled gitaly_namespace_client(storage).add(name) else @@ -315,7 +316,8 @@ module Gitlab # rm_namespace("/path/to/storage", "gitlab") # def rm_namespace(storage, name) - Gitlab::GitalyClient.migrate(:remove_namespace) do |enabled| + Gitlab::GitalyClient.migrate(:remove_namespace, + status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled| if enabled gitaly_namespace_client(storage).remove(name) else @@ -333,7 +335,8 @@ module Gitlab # mv_namespace("/path/to/storage", "gitlab", "gitlabhq") # def mv_namespace(storage, old_name, new_name) - Gitlab::GitalyClient.migrate(:rename_namespace) do |enabled| + Gitlab::GitalyClient.migrate(:rename_namespace, + status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled| if enabled gitaly_namespace_client(storage).rename(old_name, new_name) else @@ -368,7 +371,8 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def exists?(storage, dir_name) - Gitlab::GitalyClient.migrate(:namespace_exists) do |enabled| + Gitlab::GitalyClient.migrate(:namespace_exists, + status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled| if enabled gitaly_namespace_client(storage).exists?(dir_name) else diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb index 2bfb7caefd9..b89ae2505c9 100644 --- a/lib/gitlab/sidekiq_middleware/memory_killer.rb +++ b/lib/gitlab/sidekiq_middleware/memory_killer.rb @@ -45,7 +45,7 @@ module Gitlab private def get_rss - output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{pid})) + output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{pid}), Rails.root.to_s) return 0 unless status.zero? output.to_i diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb index f357488ac61..15eb1c41213 100644 --- a/lib/gitlab/user_access.rb +++ b/lib/gitlab/user_access.rb @@ -6,7 +6,8 @@ module Gitlab [user&.id, project&.id] end - attr_reader :user, :project + attr_reader :user + attr_accessor :project def initialize(user, project: nil) @user = user diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index b3f8b0d174d..823df67ea39 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -161,6 +161,18 @@ module Gitlab ] end + def send_url(url, allow_redirects: false) + params = { + 'URL' => url, + 'AllowRedirects' => allow_redirects + } + + [ + SEND_DATA_HEADER, + "send-url:#{encode(params)}" + ] + end + def terminal_websocket(terminal) details = { 'Terminal' => { diff --git a/lib/tasks/gemojione.rake b/lib/tasks/gemojione.rake index c2d3a6b6950..c6942d22926 100644 --- a/lib/tasks/gemojione.rake +++ b/lib/tasks/gemojione.rake @@ -115,7 +115,7 @@ namespace :gemojione do end end - style_path = Rails.root.join(*%w(app assets stylesheets framework emoji-sprites.scss)) + style_path = Rails.root.join(*%w(app assets stylesheets framework emoji_sprites.scss)) # Combine the resized assets into a packed sprite and re-generate the SCSS SpriteFactory.cssurl = "image-url('$IMAGE')" |