diff options
Diffstat (limited to 'lib')
120 files changed, 1899 insertions, 574 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index 8094597d238..ae161efb358 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -13,7 +13,8 @@ module API formatter: Gitlab::GrapeLogging::Formatters::LogrageWithTimestamp.new, include: [ GrapeLogging::Loggers::FilterParameters.new, - GrapeLogging::Loggers::ClientEnv.new + GrapeLogging::Loggers::ClientEnv.new, + Gitlab::GrapeLogging::Loggers::UserLogger.new ] allow_access_with_scope :api @@ -119,6 +120,7 @@ module API mount ::API::Features mount ::API::Files mount ::API::Groups + mount ::API::GroupMilestones mount ::API::Internal mount ::API::Issues mount ::API::Jobs @@ -129,8 +131,6 @@ module API mount ::API::Members mount ::API::MergeRequestDiffs mount ::API::MergeRequests - mount ::API::ProjectMilestones - mount ::API::GroupMilestones mount ::API::Namespaces mount ::API::Notes mount ::API::NotificationSettings @@ -139,6 +139,7 @@ module API mount ::API::PipelineSchedules mount ::API::ProjectHooks mount ::API::Projects + mount ::API::ProjectMilestones mount ::API::ProjectSnippets mount ::API::ProtectedBranches mount ::API::Repositories diff --git a/lib/api/boards.rb b/lib/api/boards.rb index 366b0dc9a6f..6c706b2b4e1 100644 --- a/lib/api/boards.rb +++ b/lib/api/boards.rb @@ -1,45 +1,46 @@ module API class Boards < Grape::API + include BoardsResponses include PaginationParams before { authenticate! } + helpers do + def board_parent + user_project + end + end + params do requires :id, type: String, desc: 'The ID of a project' end resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do - desc 'Get all project boards' do - detail 'This feature was introduced in 8.13' - success Entities::Board - end - params do - use :pagination - end - get ':id/boards' do - authorize!(:read_board, user_project) - present paginate(user_project.boards), with: Entities::Board + segment ':id/boards' do + desc 'Get all project boards' do + detail 'This feature was introduced in 8.13' + success Entities::Board + end + params do + use :pagination + end + get '/' do + authorize!(:read_board, user_project) + present paginate(board_parent.boards), with: Entities::Board + end + + desc 'Find a project board' do + detail 'This feature was introduced in 10.4' + success Entities::Board + end + get '/:board_id' do + present board, with: Entities::Board + end end params do requires :board_id, type: Integer, desc: 'The ID of a board' end segment ':id/boards/:board_id' do - helpers do - def project_board - board = user_project.boards.first - - if params[:board_id] == board.id - board - else - not_found!('Board') - end - end - - def board_lists - project_board.lists.destroyable - end - end - desc 'Get the lists of a project board' do detail 'Does not include `done` list. This feature was introduced in 8.13' success Entities::List @@ -72,22 +73,13 @@ module API requires :label_id, type: Integer, desc: 'The ID of an existing label' end post '/lists' do - unless available_labels.exists?(params[:label_id]) + unless available_labels_for(user_project).exists?(params[:label_id]) render_api_error!({ error: 'Label not found!' }, 400) end authorize!(:admin_list, user_project) - service = ::Boards::Lists::CreateService.new(user_project, current_user, - { label_id: params[:label_id] }) - - list = service.execute(project_board) - - if list.valid? - present list, with: Entities::List - else - render_validation_error!(list) - end + create_list end desc 'Moves a board list to a new position' do @@ -99,18 +91,11 @@ module API requires :position, type: Integer, desc: 'The position of the list' end put '/lists/:list_id' do - list = project_board.lists.movable.find(params[:list_id]) + list = board_lists.find(params[:list_id]) authorize!(:admin_list, user_project) - service = ::Boards::Lists::MoveService.new(user_project, current_user, - { position: params[:position] }) - - if service.execute(list) - present list, with: Entities::List - else - render_api_error!({ error: "List could not be moved!" }, 400) - end + move_list(list) end desc 'Delete a board list' do @@ -124,12 +109,7 @@ module API authorize!(:admin_list, user_project) list = board_lists.find(params[:list_id]) - destroy_conditionally!(list) do |list| - service = ::Boards::Lists::DestroyService.new(user_project, current_user) - unless service.execute(list) - render_api_error!({ error: 'List could not be deleted!' }, 400) - end - end + destroy_list(list) end end end diff --git a/lib/api/boards_responses.rb b/lib/api/boards_responses.rb new file mode 100644 index 00000000000..ead0943a74d --- /dev/null +++ b/lib/api/boards_responses.rb @@ -0,0 +1,50 @@ +module API + module BoardsResponses + extend ActiveSupport::Concern + + included do + helpers do + def board + board_parent.boards.find(params[:board_id]) + end + + def board_lists + board.lists.destroyable + end + + def create_list + create_list_service = + ::Boards::Lists::CreateService.new(board_parent, current_user, { label_id: params[:label_id] }) + + list = create_list_service.execute(board) + + if list.valid? + present list, with: Entities::List + else + render_validation_error!(list) + end + end + + def move_list(list) + move_list_service = + ::Boards::Lists::MoveService.new(board_parent, current_user, { position: params[:position].to_i }) + + if move_list_service.execute(list) + present list, with: Entities::List + else + render_api_error!({ error: "List could not be moved!" }, 400) + end + end + + def destroy_list(list) + destroy_conditionally!(list) do |list| + service = ::Boards::Lists::DestroyService.new(board_parent, current_user) + unless service.execute(list) + render_api_error!({ error: 'List could not be deleted!' }, 400) + end + end + end + end + end + end +end diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 38e05074353..d8fd6a6eb06 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -82,13 +82,14 @@ module API end params do requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' + optional :stats, type: Boolean, default: true, desc: 'Include commit stats' end get ':id/repository/commits/:sha', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do commit = user_project.commit(params[:sha]) not_found! 'Commit' unless commit - present commit, with: Entities::CommitDetail + present commit, with: Entities::CommitDetail, stats: params[:stats] end desc 'Get the diff for a specific commit of a project' do diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 4ad4a1f7867..c4ef2c74658 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -278,7 +278,7 @@ module API end class CommitDetail < Commit - expose :stats, using: Entities::CommitStats + expose :stats, using: Entities::CommitStats, if: :stats expose :status expose :last_pipeline, using: 'API::Entities::PipelineBasic' end @@ -791,6 +791,8 @@ module API class Board < Grape::Entity expose :id + expose :project, using: Entities::BasicProjectDetails + expose :lists, using: Entities::List do |board| board.lists.destroyable end @@ -862,6 +864,8 @@ module API expose :active expose :is_shared expose :name + expose :online?, as: :online + expose :status end class RunnerDetails < Runner @@ -914,7 +918,7 @@ module API class Trigger < Grape::Entity expose :id expose :token, :description - expose :created_at, :updated_at, :deleted_at, :last_used + expose :created_at, :updated_at, :last_used expose :owner, using: Entities::UserBasic end @@ -1133,6 +1137,7 @@ module API class PagesDomainBasic < Grape::Entity expose :domain expose :url + expose :project_id expose :certificate, as: :certificate_expiration, if: ->(pages_domain, _) { pages_domain.certificate? }, diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 9ba15893f55..6134ad2bfc7 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -5,6 +5,7 @@ module API SUDO_HEADER = "HTTP_SUDO".freeze SUDO_PARAM = :sudo + API_USER_ENV = 'gitlab.api.user'.freeze def declared_params(options = {}) options = { include_parent_namespaces: false }.merge(options) @@ -25,6 +26,7 @@ module API check_unmodified_since!(last_updated) status 204 + if block_given? yield resource else @@ -48,10 +50,16 @@ module API validate_access_token!(scopes: scopes_registered_for_endpoint) unless sudo? + save_current_user_in_env(@current_user) if @current_user + @current_user end # rubocop:enable Gitlab/ModuleWithInstanceVariables + def save_current_user_in_env(user) + env[API_USER_ENV] = { user_id: user.id, username: user.username } + end + def sudo? initial_current_user != current_user end @@ -69,13 +77,20 @@ module API end def wiki_page - page = user_project.wiki.find_page(params[:slug]) + page = ProjectWiki.new(user_project, current_user).find_page(params[:slug]) page || not_found!('Wiki Page') end - def available_labels - @available_labels ||= LabelsFinder.new(current_user, project_id: user_project.id).execute + def available_labels_for(label_parent) + search_params = + if label_parent.is_a?(Project) + { project_id: label_parent.id } + else + { group_id: label_parent.id, only_group_labels: true } + end + + LabelsFinder.new(current_user, search_params).execute end def find_user(id) @@ -141,7 +156,9 @@ module API end def find_project_label(id) - label = available_labels.find_by_id(id) || available_labels.find_by_title(id) + labels = available_labels_for(user_project) + label = labels.find_by_id(id) || labels.find_by_title(id) + label || not_found!('Label') end diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 79b302aae70..063f0d6599c 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -82,6 +82,18 @@ module API end # + # Get a ssh key using the fingerprint + # + get "/authorized_keys" do + fingerprint = params.fetch(:fingerprint) do + Gitlab::InsecureKeyFingerprint.new(params.fetch(:key)).fingerprint + end + key = Key.find_by(fingerprint: fingerprint) + not_found!("Key") if key.nil? + present key, with: Entities::SSHKey + end + + # # Discover user by ssh key or user id # get "/discover" do @@ -91,6 +103,7 @@ module API elsif params[:user_id] user = User.find_by(id: params[:user_id]) end + present user, with: Entities::UserSafe end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index b29c5848aef..c99fe3ab5b3 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -175,6 +175,7 @@ module API issue = ::Issues::CreateService.new(user_project, current_user, issue_params.merge(request: request, api: true)).execute + if issue.spam? render_api_error!({ error: 'Spam detected' }, 400) end @@ -277,6 +278,19 @@ module API present paginate(merge_requests), with: Entities::MergeRequestBasic, current_user: current_user, project: user_project end + desc 'List participants for an issue' do + success Entities::UserBasic + end + params do + requires :issue_iid, type: Integer, desc: 'The internal ID of a project issue' + end + get ':id/issues/:issue_iid/participants' do + issue = find_project_issue(params[:issue_iid]) + participants = ::Kaminari.paginate_array(issue.participants) + + present paginate(participants), with: Entities::UserBasic, current_user: current_user, project: user_project + end + desc 'Get the user agent details for an issue' do success Entities::UserAgentDetail end diff --git a/lib/api/labels.rb b/lib/api/labels.rb index e41a1720ac1..81eaf56e48e 100644 --- a/lib/api/labels.rb +++ b/lib/api/labels.rb @@ -15,7 +15,7 @@ module API use :pagination end get ':id/labels' do - present paginate(available_labels), with: Entities::Label, current_user: current_user, project: user_project + present paginate(available_labels_for(user_project)), with: Entities::Label, current_user: current_user, project: user_project end desc 'Create a new label' do @@ -30,7 +30,7 @@ module API post ':id/labels' do authorize! :admin_label, user_project - label = available_labels.find_by(title: params[:name]) + label = available_labels_for(user_project).find_by(title: params[:name]) conflict!('Label already exists') if label priority = params.delete(:priority) diff --git a/lib/api/members.rb b/lib/api/members.rb index 22e4bdead41..5446f6b54b1 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -59,7 +59,9 @@ module API member = source.add_user(params[:user_id], params[:access_level], current_user: current_user, expires_at: params[:expires_at]) - if member.persisted? && member.valid? + if !member + not_allowed! # This currently can only be reached in EE + elsif member.persisted? && member.valid? present member.user, with: Entities::Member, member: member else render_validation_error!(member) diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 02f2b75ab9d..420aaf1c964 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -24,6 +24,13 @@ module API .preload(:notes, :author, :assignee, :milestone, :latest_merge_request_diff, :labels, :timelogs) end + def merge_request_pipelines_with_access + authorize! :read_pipeline, user_project + + mr = find_merge_request_with_access(params[:merge_request_iid]) + mr.all_pipelines + end + params :merge_requests_params do optional :state, type: String, values: %w[opened closed merged all], default: 'all', desc: 'Return opened, closed, merged, or all merge requests' @@ -185,6 +192,16 @@ module API present merge_request, with: Entities::MergeRequest, current_user: current_user, project: user_project end + desc 'Get the participants of a merge request' do + success Entities::UserBasic + end + get ':id/merge_requests/:merge_request_iid/participants' do + merge_request = find_merge_request_with_access(params[:merge_request_iid]) + participants = ::Kaminari.paginate_array(merge_request.participants) + + present paginate(participants), with: Entities::UserBasic + end + desc 'Get the commits of a merge request' do success Entities::Commit end @@ -204,6 +221,15 @@ module API present merge_request, with: Entities::MergeRequestChanges, current_user: current_user end + desc 'Get the merge request pipelines' do + success Entities::PipelineBasic + end + get ':id/merge_requests/:merge_request_iid/pipelines' do + pipelines = merge_request_pipelines_with_access + + present paginate(pipelines), with: Entities::PipelineBasic + end + desc 'Update a merge request' do success Entities::MergeRequest end diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb index 74b3376a1f3..675c963bae2 100644 --- a/lib/api/pipelines.rb +++ b/lib/api/pipelines.rb @@ -48,6 +48,7 @@ module API current_user, declared_params(include_missing: false)) .execute(:api, ignore_skip_ci: true, save_on_errors: false) + if new_pipeline.persisted? present new_pipeline, with: Entities::Pipeline else diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb index 2ccda1c1aa1..5bed58c2d63 100644 --- a/lib/api/project_snippets.rb +++ b/lib/api/project_snippets.rb @@ -13,6 +13,7 @@ module API if errors[:project_access].any? error!(errors[:project_access], 422) end + not_found! end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index fa222bf2b1c..653126e79ea 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -154,6 +154,7 @@ module API if project.errors[:limit_reached].present? error!(project.errors[:limit_reached], 403) end + render_validation_error!(project) end end diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index 4f36bbd760f..9638c53a1df 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -15,6 +15,7 @@ module API if errors[:project_access].any? error!(errors[:project_access], 422) end + not_found! end diff --git a/lib/api/v3/commits.rb b/lib/api/v3/commits.rb index 0ef26aa696a..4f6ea8f502e 100644 --- a/lib/api/v3/commits.rb +++ b/lib/api/v3/commits.rb @@ -71,13 +71,14 @@ module API end params do requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' + optional :stats, type: Boolean, default: true, desc: 'Include commit stats' end get ":id/repository/commits/:sha", requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do commit = user_project.commit(params[:sha]) not_found! "Commit" unless commit - present commit, with: ::API::Entities::CommitDetail + present commit, with: ::API::Entities::CommitDetail, stats: params[:stats] end desc 'Get the diff for a specific commit of a project' do diff --git a/lib/api/v3/entities.rb b/lib/api/v3/entities.rb index c17b6f45ed8..64758dae7d3 100644 --- a/lib/api/v3/entities.rb +++ b/lib/api/v3/entities.rb @@ -207,7 +207,7 @@ module API end class Trigger < Grape::Entity - expose :token, :created_at, :updated_at, :deleted_at, :last_used + expose :token, :created_at, :updated_at, :last_used expose :owner, using: ::API::Entities::UserBasic end diff --git a/lib/api/v3/labels.rb b/lib/api/v3/labels.rb index bd5eb2175e8..4157462ec2a 100644 --- a/lib/api/v3/labels.rb +++ b/lib/api/v3/labels.rb @@ -11,7 +11,7 @@ module API success ::API::Entities::Label end get ':id/labels' do - present available_labels, with: ::API::Entities::Label, current_user: current_user, project: user_project + present available_labels_for(user_project), with: ::API::Entities::Label, current_user: current_user, project: user_project end desc 'Delete an existing label' do diff --git a/lib/api/v3/members.rb b/lib/api/v3/members.rb index 684860b553e..de226e4e573 100644 --- a/lib/api/v3/members.rb +++ b/lib/api/v3/members.rb @@ -67,6 +67,7 @@ module API unless member member = source.add_user(params[:user_id], params[:access_level], current_user: current_user, expires_at: params[:expires_at]) end + if member.persisted? && member.valid? present member.user, with: ::API::Entities::Member, member: member else diff --git a/lib/api/v3/merge_requests.rb b/lib/api/v3/merge_requests.rb index 1d6d823f32b..0a24fea52a3 100644 --- a/lib/api/v3/merge_requests.rb +++ b/lib/api/v3/merge_requests.rb @@ -126,6 +126,7 @@ module API if status == :deprecated detail DEPRECATION_MESSAGE end + success ::API::V3::Entities::MergeRequest end get path do diff --git a/lib/api/v3/project_snippets.rb b/lib/api/v3/project_snippets.rb index c41fee32610..6ba425ba8c7 100644 --- a/lib/api/v3/project_snippets.rb +++ b/lib/api/v3/project_snippets.rb @@ -14,6 +14,7 @@ module API if errors[:project_access].any? error!(errors[:project_access], 422) end + not_found! end diff --git a/lib/api/v3/projects.rb b/lib/api/v3/projects.rb index 7c260b8d910..446f804124b 100644 --- a/lib/api/v3/projects.rb +++ b/lib/api/v3/projects.rb @@ -41,6 +41,7 @@ module API # private or internal, use the more conservative option, private. attrs[:visibility_level] = (publik == true) ? Gitlab::VisibilityLevel::PUBLIC : Gitlab::VisibilityLevel::PRIVATE end + attrs end @@ -201,6 +202,7 @@ module API if project.errors[:limit_reached].present? error!(project.errors[:limit_reached], 403) end + render_validation_error!(project) end end diff --git a/lib/api/v3/repositories.rb b/lib/api/v3/repositories.rb index f9a47101e27..5b54734bb45 100644 --- a/lib/api/v3/repositories.rb +++ b/lib/api/v3/repositories.rb @@ -14,6 +14,7 @@ module API if errors[:project_access].any? error!(errors[:project_access], 422) end + not_found! end end diff --git a/lib/api/v3/snippets.rb b/lib/api/v3/snippets.rb index 126ec72248e..85613c8ed84 100644 --- a/lib/api/v3/snippets.rb +++ b/lib/api/v3/snippets.rb @@ -97,6 +97,7 @@ module API attrs = declared_params(include_missing: false) UpdateSnippetService.new(nil, current_user, snippet, attrs).execute + if snippet.persisted? present snippet, with: ::API::Entities::PersonalSnippet else diff --git a/lib/backup/database.rb b/lib/backup/database.rb index d97e5d98229..5e6828de597 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -31,6 +31,7 @@ module Backup pgsql_args << "-n" pgsql_args << Gitlab.config.backup.pg_schema end + spawn('pg_dump', *pgsql_args, config['database'], out: compress_wr) end compress_wr.close diff --git a/lib/backup/files.rb b/lib/backup/files.rb index 30a91647b77..287d591e88d 100644 --- a/lib/backup/files.rb +++ b/lib/backup/files.rb @@ -18,7 +18,7 @@ module Backup FileUtils.rm_f(backup_tarball) if ENV['STRATEGY'] == 'copy' - cmd = %W(cp -a #{app_files_dir} #{Gitlab.config.backup.path}) + cmd = %W(rsync -a --exclude=lost+found #{app_files_dir} #{Gitlab.config.backup.path}) output, status = Gitlab::Popen.popen(cmd) unless status.zero? @@ -26,10 +26,10 @@ module Backup abort 'Backup failed' end - run_pipeline!([%W(tar -C #{@backup_files_dir} -cf - .), %w(gzip -c -1)], out: [backup_tarball, 'w', 0600]) + run_pipeline!([%W(tar --exclude=lost+found -C #{@backup_files_dir} -cf - .), %w(gzip -c -1)], out: [backup_tarball, 'w', 0600]) FileUtils.rm_rf(@backup_files_dir) else - run_pipeline!([%W(tar -C #{app_files_dir} -cf - .), %w(gzip -c -1)], out: [backup_tarball, 'w', 0600]) + run_pipeline!([%W(tar --exclude=lost+found -C #{app_files_dir} -cf - .), %w(gzip -c -1)], out: [backup_tarball, 'w', 0600]) end end diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 2a04c03919d..6715159a1aa 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -47,6 +47,7 @@ module Backup if File.exist?(path_to_wiki_repo) progress.print " * #{display_repo_path(wiki)} ... " + if empty_repo?(wiki) progress.puts " [SKIPPED]".color(:cyan) else diff --git a/lib/banzai/filter/mermaid_filter.rb b/lib/banzai/filter/mermaid_filter.rb index b545b947a2c..65c131e08d9 100644 --- a/lib/banzai/filter/mermaid_filter.rb +++ b/lib/banzai/filter/mermaid_filter.rb @@ -2,16 +2,7 @@ module Banzai module Filter class MermaidFilter < HTML::Pipeline::Filter def call - doc.css('pre[lang="mermaid"]').add_class('mermaid') - doc.css('pre[lang="mermaid"]').add_class('js-render-mermaid') - - # The `<code></code>` blocks are added in the lib/banzai/filter/syntax_highlight_filter.rb - # We want to keep context and consistency, so we the blocks are added for all filters. - # Details: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/15107/diffs?diff_id=7962900#note_45495859 - doc.css('pre[lang="mermaid"]').each do |pre| - document = pre.at('code') - document.replace(document.content) - end + doc.css('pre[lang="mermaid"] > code').add_class('js-render-mermaid') doc end diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb index 5c197afd782..f6169b2c85d 100644 --- a/lib/banzai/filter/relative_link_filter.rb +++ b/lib/banzai/filter/relative_link_filter.rb @@ -50,15 +50,22 @@ module Banzai end def process_link_to_upload_attr(html_attr) - uri_parts = [html_attr.value] + path_parts = [html_attr.value] if group - uri_parts.unshift(relative_url_root, 'groups', group.full_path, '-') + path_parts.unshift(relative_url_root, 'groups', group.full_path, '-') elsif project - uri_parts.unshift(relative_url_root, project.full_path) + path_parts.unshift(relative_url_root, project.full_path) end - html_attr.value = File.join(*uri_parts) + path = File.join(*path_parts) + + html_attr.value = + if context[:only_path] + path + else + URI.join(Gitlab.config.gitlab.base_url, path).to_s + end end def process_link_to_repository_attr(html_attr) diff --git a/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb b/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb new file mode 100644 index 00000000000..7bffffec94d --- /dev/null +++ b/lib/gitlab/background_migration/add_merge_request_diff_commits_count.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation +# rubocop:disable Metrics/LineLength + +module Gitlab + module BackgroundMigration + class AddMergeRequestDiffCommitsCount + class MergeRequestDiff < ActiveRecord::Base + self.table_name = 'merge_request_diffs' + end + + def perform(start_id, stop_id) + Rails.logger.info("Setting commits_count for merge request diffs: #{start_id} - #{stop_id}") + + update = ' + commits_count = ( + SELECT count(*) + FROM merge_request_diff_commits + WHERE merge_request_diffs.id = merge_request_diff_commits.merge_request_diff_id + )'.squish + + MergeRequestDiff.where(id: start_id..stop_id).update_all(update) + end + end + end +end diff --git a/lib/gitlab/background_migration/cleanup_concurrent_type_change.rb b/lib/gitlab/background_migration/cleanup_concurrent_type_change.rb new file mode 100644 index 00000000000..de622f657b2 --- /dev/null +++ b/lib/gitlab/background_migration/cleanup_concurrent_type_change.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Background migration for cleaning up a concurrent column rename. + class CleanupConcurrentTypeChange + include Database::MigrationHelpers + + RESCHEDULE_DELAY = 10.minutes + + # table - The name of the table the migration is performed for. + # old_column - The name of the old (to drop) column. + # new_column - The name of the new column. + def perform(table, old_column, new_column) + return unless column_exists?(:issues, new_column) + + rows_to_migrate = define_model_for(table) + .where(new_column => nil) + .where + .not(old_column => nil) + + if rows_to_migrate.any? + BackgroundMigrationWorker.perform_in( + RESCHEDULE_DELAY, + 'CleanupConcurrentTypeChange', + [table, old_column, new_column] + ) + else + cleanup_concurrent_column_type_change(table, old_column) + end + end + + # These methods are necessary so we can re-use the migration helpers in + # this class. + def connection + ActiveRecord::Base.connection + end + + def method_missing(name, *args, &block) + connection.__send__(name, *args, &block) # rubocop: disable GitlabSecurity/PublicSend + end + + def respond_to_missing?(*args) + connection.respond_to?(*args) || super + end + + def define_model_for(table) + Class.new(ActiveRecord::Base) do + self.table_name = table + end + end + end + end +end diff --git a/lib/gitlab/background_migration/copy_column.rb b/lib/gitlab/background_migration/copy_column.rb new file mode 100644 index 00000000000..a2cb215c230 --- /dev/null +++ b/lib/gitlab/background_migration/copy_column.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # CopyColumn is a simple (reusable) background migration that can be used to + # update the value of a column based on the value of another column in the + # same table. + # + # For this background migration to work the table that is migrated _has_ to + # have an `id` column as the primary key. + class CopyColumn + # table - The name of the table that contains the columns. + # copy_from - The column containing the data to copy. + # copy_to - The column to copy the data to. + # start_id - The start ID of the range of rows to update. + # end_id - The end ID of the range of rows to update. + def perform(table, copy_from, copy_to, start_id, end_id) + return unless connection.column_exists?(table, copy_to) + + quoted_table = connection.quote_table_name(table) + quoted_copy_from = connection.quote_column_name(copy_from) + quoted_copy_to = connection.quote_column_name(copy_to) + + # We're using raw SQL here since this job may be frequently executed. As + # a result dynamically defining models would lead to many unnecessary + # schema information queries. + connection.execute <<-SQL.strip_heredoc + UPDATE #{quoted_table} + SET #{quoted_copy_to} = #{quoted_copy_from} + WHERE id BETWEEN #{start_id} AND #{end_id} + SQL + end + + def connection + ActiveRecord::Base.connection + end + end + end +end diff --git a/lib/gitlab/background_migration/delete_conflicting_redirect_routes_range.rb b/lib/gitlab/background_migration/delete_conflicting_redirect_routes_range.rb index a1af045a71f..21b626dde56 100644 --- a/lib/gitlab/background_migration/delete_conflicting_redirect_routes_range.rb +++ b/lib/gitlab/background_migration/delete_conflicting_redirect_routes_range.rb @@ -1,44 +1,12 @@ # frozen_string_literal: true -# rubocop:disable Metrics/LineLength # rubocop:disable Style/Documentation module Gitlab module BackgroundMigration class DeleteConflictingRedirectRoutesRange - class Route < ActiveRecord::Base - self.table_name = 'routes' - end - - class RedirectRoute < ActiveRecord::Base - self.table_name = 'redirect_routes' - end - - # start_id - The start ID of the range of events to process - # end_id - The end ID of the range to process. def perform(start_id, end_id) - return unless migrate? - - conflicts = RedirectRoute.where(routes_match_redirects_clause(start_id, end_id)) - num_rows = conflicts.delete_all - - Rails.logger.info("Gitlab::BackgroundMigration::DeleteConflictingRedirectRoutesRange [#{start_id}, #{end_id}] - Deleted #{num_rows} redirect routes that were conflicting with routes.") - end - - def migrate? - Route.table_exists? && RedirectRoute.table_exists? - end - - def routes_match_redirects_clause(start_id, end_id) - <<~ROUTES_MATCH_REDIRECTS - EXISTS ( - SELECT 1 FROM routes - WHERE ( - LOWER(redirect_routes.path) = LOWER(routes.path) - OR LOWER(redirect_routes.path) LIKE LOWER(CONCAT(routes.path, '/%')) - ) - AND routes.id BETWEEN #{start_id} AND #{end_id} - ) - ROUTES_MATCH_REDIRECTS + # No-op. + # See https://gitlab.com/gitlab-com/infrastructure/issues/3460#note_53223252 end end end diff --git a/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb b/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb index 84ac00f1a5c..7088aa0860a 100644 --- a/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb +++ b/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb @@ -128,8 +128,14 @@ module Gitlab end def process_event(event) - replicate_event(event) - create_push_event_payload(event) if event.push_event? + ActiveRecord::Base.transaction do + replicate_event(event) + create_push_event_payload(event) if event.push_event? + end + rescue ActiveRecord::InvalidForeignKey => e + # A foreign key error means the associated event was removed. In this + # case we'll just skip migrating the event. + Rails.logger.error("Unable to migrate event #{event.id}: #{e}") end def replicate_event(event) @@ -137,9 +143,6 @@ module Gitlab .with_indifferent_access.except(:title, :data) EventForMigration.create!(new_attributes) - rescue ActiveRecord::InvalidForeignKey - # A foreign key error means the associated event was removed. In this - # case we'll just skip migrating the event. end def create_push_event_payload(event) @@ -156,9 +159,6 @@ module Gitlab ref: event.trimmed_ref_name, commit_title: event.commit_title ) - rescue ActiveRecord::InvalidForeignKey - # A foreign key error means the associated event was removed. In this - # case we'll just skip migrating the event. end def find_events(start_id, end_id) diff --git a/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data.rb b/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data.rb new file mode 100644 index 00000000000..8a901a9bf39 --- /dev/null +++ b/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data.rb @@ -0,0 +1,135 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/LineLength +# rubocop:disable Metrics/MethodLength +# rubocop:disable Metrics/ClassLength +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class PopulateMergeRequestMetricsWithEventsData + def perform(min_merge_request_id, max_merge_request_id) + insert_metrics_for_range(min_merge_request_id, max_merge_request_id) + update_metrics_with_events_data(min_merge_request_id, max_merge_request_id) + end + + # Inserts merge_request_metrics records for merge_requests without it for + # a given merge request batch. + def insert_metrics_for_range(min, max) + metrics_not_exists_clause = + <<-SQL.strip_heredoc + NOT EXISTS (SELECT 1 FROM merge_request_metrics + WHERE merge_request_metrics.merge_request_id = merge_requests.id) + SQL + + MergeRequest.where(metrics_not_exists_clause).where(id: min..max).each_batch do |batch| + select_sql = batch.select(:id, :created_at, :updated_at).to_sql + + execute("INSERT INTO merge_request_metrics (merge_request_id, created_at, updated_at) #{select_sql}") + end + end + + def update_metrics_with_events_data(min, max) + if Gitlab::Database.postgresql? + # Uses WITH syntax in order to update merged and closed events with a single UPDATE. + # WITH is not supported by MySQL. + update_events_for_range(min, max) + else + update_merged_events_for_range(min, max) + update_closed_events_for_range(min, max) + end + end + + private + + # Updates merge_request_metrics latest_closed_at, latest_closed_by_id and merged_by_id + # based on the latest event records on events table for a given merge request batch. + def update_events_for_range(min, max) + sql = <<-SQL.strip_heredoc + WITH events_for_update AS ( + SELECT DISTINCT ON (target_id, action) target_id, action, author_id, updated_at + FROM events + WHERE target_id BETWEEN #{min} AND #{max} + AND target_type = 'MergeRequest' + AND action IN (#{Event::CLOSED},#{Event::MERGED}) + ORDER BY target_id, action, id DESC + ) + UPDATE merge_request_metrics met + SET latest_closed_at = latest_closed.updated_at, + latest_closed_by_id = latest_closed.author_id, + merged_by_id = latest_merged.author_id + FROM (SELECT * FROM events_for_update WHERE action = #{Event::CLOSED}) AS latest_closed + FULL OUTER JOIN + (SELECT * FROM events_for_update WHERE action = #{Event::MERGED}) AS latest_merged + USING (target_id) + WHERE target_id = merge_request_id; + SQL + + execute(sql) + end + + # Updates merge_request_metrics latest_closed_at, latest_closed_by_id based on the latest closed + # records on events table for a given merge request batch. + def update_closed_events_for_range(min, max) + sql = + <<-SQL.strip_heredoc + UPDATE merge_request_metrics metrics, + (#{select_events(min, max, Event::CLOSED)}) closed_events + SET metrics.latest_closed_by_id = closed_events.author_id, + metrics.latest_closed_at = closed_events.updated_at #{where_matches_closed_events}; + SQL + + execute(sql) + end + + # Updates merge_request_metrics merged_by_id based on the latest merged + # records on events table for a given merge request batch. + def update_merged_events_for_range(min, max) + sql = + <<-SQL.strip_heredoc + UPDATE merge_request_metrics metrics, + (#{select_events(min, max, Event::MERGED)}) merged_events + SET metrics.merged_by_id = merged_events.author_id #{where_matches_merged_events}; + SQL + + execute(sql) + end + + def execute(sql) + @connection ||= ActiveRecord::Base.connection + @connection.execute(sql) + end + + def select_events(min, max, action) + select_max_event_id = <<-SQL.strip_heredoc + SELECT max(id) + FROM events + WHERE action = #{action} + AND target_type = 'MergeRequest' + AND target_id BETWEEN #{min} AND #{max} + GROUP BY target_id + SQL + + <<-SQL.strip_heredoc + SELECT author_id, updated_at, target_id + FROM events + WHERE id IN(#{select_max_event_id}) + SQL + end + + def where_matches_closed_events + <<-SQL.strip_heredoc + WHERE metrics.merge_request_id = closed_events.target_id + AND metrics.latest_closed_at IS NULL + AND metrics.latest_closed_by_id IS NULL + SQL + end + + def where_matches_merged_events + <<-SQL.strip_heredoc + WHERE metrics.merge_request_id = merged_events.target_id + AND metrics.merged_by_id IS NULL + SQL + end + end + end +end diff --git a/lib/gitlab/bare_repository_import/importer.rb b/lib/gitlab/bare_repository_import/importer.rb index 298409d8b5a..884a3de8f62 100644 --- a/lib/gitlab/bare_repository_import/importer.rb +++ b/lib/gitlab/bare_repository_import/importer.rb @@ -14,7 +14,7 @@ module Gitlab repos_to_import.each do |repo_path| bare_repo = Gitlab::BareRepositoryImport::Repository.new(import_path, repo_path) - if bare_repo.hashed? || bare_repo.wiki? + unless bare_repo.processable? log " * Skipping repo #{bare_repo.repo_path}".color(:yellow) next @@ -55,12 +55,16 @@ module Gitlab name: project_name, path: project_name, skip_disk_validation: true, - import_type: 'gitlab_project', + skip_wiki: bare_repo.wiki_exists?, + import_type: 'bare_repository', namespace_id: group&.id).execute if project.persisted? && mv_repo(project) log " * Created #{project.name} (#{project_full_path})".color(:green) + project.write_repository_config + project.repository.create_hooks + ProjectCacheWorker.perform_async(project.id) else log " * Failed trying to create #{project.name} (#{project_full_path})".color(:red) diff --git a/lib/gitlab/bare_repository_import/repository.rb b/lib/gitlab/bare_repository_import/repository.rb index fa7891c8dcc..85b79362196 100644 --- a/lib/gitlab/bare_repository_import/repository.rb +++ b/lib/gitlab/bare_repository_import/repository.rb @@ -6,39 +6,56 @@ module Gitlab def initialize(root_path, repo_path) @root_path = root_path @repo_path = repo_path - @root_path << '/' unless root_path.ends_with?('/') + full_path = + if hashed? && !wiki? + repository.config.get('gitlab.fullpath') + else + repo_relative_path + end + # Split path into 'all/the/namespaces' and 'project_name' - @group_path, _, @project_name = repo_relative_path.rpartition('/') + @group_path, _, @project_name = full_path.to_s.rpartition('/') end def wiki_exists? File.exist?(wiki_path) end - def wiki? - @wiki ||= repo_path.end_with?('.wiki.git') - end - def wiki_path @wiki_path ||= repo_path.sub(/\.git$/, '.wiki.git') end - def hashed? - @hashed ||= group_path.start_with?('@hashed') - end - def project_full_path @project_full_path ||= "#{group_path}/#{project_name}" end + def processable? + return false if wiki? + return false if hashed? && (group_path.blank? || project_name.blank?) + + true + end + private + def wiki? + @wiki ||= repo_path.end_with?('.wiki.git') + end + + def hashed? + @hashed ||= repo_relative_path.include?('@hashed') + end + def repo_relative_path # Remove root path and `.git` at the end repo_path[@root_path.size...-4] end + + def repository + @repository ||= Rugged::Repository.new(repo_path) + end end end end diff --git a/lib/gitlab/checks/project_moved.rb b/lib/gitlab/checks/project_moved.rb index 3a1c0a3455e..dfb2f4d4054 100644 --- a/lib/gitlab/checks/project_moved.rb +++ b/lib/gitlab/checks/project_moved.rb @@ -21,6 +21,10 @@ module Gitlab end def add_redirect_message + # Don't bother with sending a redirect message for anonymous clones + # because they never see it via the `/internal/post_receive` endpoint + return unless user.present? && project.present? + Gitlab::Redis::SharedState.with do |redis| key = self.class.redirect_message_key(user.id, project.id) redis.setex(key, 5.minutes, redirect_message) diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb index 72b75791bbb..35eadf6fa93 100644 --- a/lib/gitlab/ci/ansi2html.rb +++ b/lib/gitlab/ci/ansi2html.rb @@ -148,6 +148,7 @@ module Gitlab stream.seek(@offset) append = @offset > 0 end + start_offset = @offset open_new_tag @@ -155,6 +156,7 @@ module Gitlab stream.each_line do |line| s = StringScanner.new(line) until s.eos? + if s.scan(Gitlab::Regex.build_trace_section_regex) handle_section(s) elsif s.scan(/\e([@-_])(.*?)([@-~])/) @@ -168,6 +170,7 @@ module Gitlab else @out << s.scan(/./m) end + @offset += s.matched_size end end @@ -234,10 +237,12 @@ module Gitlab # Most terminals show bold colored text in the light color variant # Let's mimic that here if @style_mask & STYLE_SWITCHES[:bold] != 0 - fg_color.sub!(/fg-(\w{2,}+)/, 'fg-l-\1') + fg_color.sub!(/fg-([a-z]{2,}+)/, 'fg-l-\1') end + css_classes << fg_color end + css_classes << @bg_color unless @bg_color.nil? STYLE_SWITCHES.each do |css_class, flag| diff --git a/lib/gitlab/conflict/file_collection.rb b/lib/gitlab/conflict/file_collection.rb index 76aee5a3deb..0a3ae2c3760 100644 --- a/lib/gitlab/conflict/file_collection.rb +++ b/lib/gitlab/conflict/file_collection.rb @@ -13,12 +13,13 @@ module Gitlab end def resolve(user, commit_message, files) + msg = commit_message || default_commit_message + resolution = Gitlab::Git::Conflict::Resolution.new(user, files, msg) args = { source_branch: merge_request.source_branch, - target_branch: merge_request.target_branch, - commit_message: commit_message || default_commit_message + target_branch: merge_request.target_branch } - resolver.resolve_conflicts(@source_repo, user, files, args) + resolver.resolve_conflicts(@source_repo, resolution, args) ensure @merge_request.clear_memoized_shas end diff --git a/lib/gitlab/cycle_analytics/base_query.rb b/lib/gitlab/cycle_analytics/base_query.rb index dcbdf9a64b0..8b3bc3e440d 100644 --- a/lib/gitlab/cycle_analytics/base_query.rb +++ b/lib/gitlab/cycle_analytics/base_query.rb @@ -15,7 +15,6 @@ module Gitlab query = mr_closing_issues_table.join(issue_table).on(issue_table[:id].eq(mr_closing_issues_table[:issue_id])) .join(issue_metrics_table).on(issue_table[:id].eq(issue_metrics_table[:issue_id])) .where(issue_table[:project_id].eq(@project.id)) # rubocop:disable Gitlab/ModuleWithInstanceVariables - .where(issue_table[:deleted_at].eq(nil)) .where(issue_table[:created_at].gteq(@options[:from])) # rubocop:disable Gitlab/ModuleWithInstanceVariables # Load merge_requests diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 3f65bc912de..592a1956ceb 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -385,10 +385,27 @@ module Gitlab # necessary since we copy over old values further down. change_column_default(table, new, old_col.default) if old_col.default - trigger_name = rename_trigger_name(table, old, new) + install_rename_triggers(table, old, new) + + update_column_in_batches(table, new, Arel::Table.new(table)[old]) + + change_column_null(table, new, false) unless old_col.null + + copy_indexes(table, old, new) + copy_foreign_keys(table, old, new) + end + + # Installs triggers in a table that keep a new column in sync with an old + # one. + # + # table - The name of the table to install the trigger in. + # old_column - The name of the old column. + # new_column - The name of the new column. + def install_rename_triggers(table, old_column, new_column) + trigger_name = rename_trigger_name(table, old_column, new_column) quoted_table = quote_table_name(table) - quoted_old = quote_column_name(old) - quoted_new = quote_column_name(new) + quoted_old = quote_column_name(old_column) + quoted_new = quote_column_name(new_column) if Database.postgresql? install_rename_triggers_for_postgresql(trigger_name, quoted_table, @@ -397,13 +414,6 @@ module Gitlab install_rename_triggers_for_mysql(trigger_name, quoted_table, quoted_old, quoted_new) end - - update_column_in_batches(table, new, Arel::Table.new(table)[old]) - - change_column_null(table, new, false) unless old_col.null - - copy_indexes(table, old, new) - copy_foreign_keys(table, old, new) end # Changes the type of a column concurrently. @@ -455,6 +465,98 @@ module Gitlab remove_column(table, old) end + # Changes the column type of a table using a background migration. + # + # Because this method uses a background migration it's more suitable for + # large tables. For small tables it's better to use + # `change_column_type_concurrently` since it can complete its work in a + # much shorter amount of time and doesn't rely on Sidekiq. + # + # Example usage: + # + # class Issue < ActiveRecord::Base + # self.table_name = 'issues' + # + # include EachBatch + # + # def self.to_migrate + # where('closed_at IS NOT NULL') + # end + # end + # + # change_column_type_using_background_migration( + # Issue.to_migrate, + # :closed_at, + # :datetime_with_timezone + # ) + # + # Reverting a migration like this is done exactly the same way, just with + # a different type to migrate to (e.g. `:datetime` in the above example). + # + # relation - An ActiveRecord relation to use for scheduling jobs and + # figuring out what table we're modifying. This relation _must_ + # have the EachBatch module included. + # + # column - The name of the column for which the type will be changed. + # + # new_type - The new type of the column. + # + # batch_size - The number of rows to schedule in a single background + # migration. + # + # interval - The time interval between every background migration. + def change_column_type_using_background_migration( + relation, + column, + new_type, + batch_size: 10_000, + interval: 10.minutes + ) + + unless relation.model < EachBatch + raise TypeError, 'The relation must include the EachBatch module' + end + + temp_column = "#{column}_for_type_change" + table = relation.table_name + max_index = 0 + + add_column(table, temp_column, new_type) + install_rename_triggers(table, column, temp_column) + + # Schedule the jobs that will copy the data from the old column to the + # new one. + relation.each_batch(of: batch_size) do |batch, index| + start_id, end_id = batch.pluck('MIN(id), MAX(id)').first + max_index = index + + BackgroundMigrationWorker.perform_in( + index * interval, + 'CopyColumn', + [table, column, temp_column, start_id, end_id] + ) + end + + # Schedule the renaming of the column to happen (initially) 1 hour after + # the last batch finished. + BackgroundMigrationWorker.perform_in( + (max_index * interval) + 1.hour, + 'CleanupConcurrentTypeChange', + [table, column, temp_column] + ) + + if perform_background_migration_inline? + # To ensure the schema is up to date immediately we perform the + # migration inline in dev / test environments. + Gitlab::BackgroundMigration.steal('CopyColumn') + Gitlab::BackgroundMigration.steal('CleanupConcurrentTypeChange') + end + end + + def perform_background_migration_inline? + Rails.env.test? || Rails.env.development? + end + # Performs a concurrent column rename when using PostgreSQL. def install_rename_triggers_for_postgresql(trigger, table, old, new) execute <<-EOF.strip_heredoc @@ -741,6 +843,12 @@ into similar problems in the future (e.g. when new tables are created). def queue_background_migration_jobs_by_range_at_intervals(model_class, job_class_name, delay_interval, batch_size: BACKGROUND_MIGRATION_BATCH_SIZE) raise "#{model_class} does not have an ID to use for batch ranges" unless model_class.column_names.include?('id') + # To not overload the worker too much we enforce a minimum interval both + # when scheduling and performing jobs. + if delay_interval < BackgroundMigrationWorker::MIN_INTERVAL + delay_interval = BackgroundMigrationWorker::MIN_INTERVAL + end + model_class.each_batch(of: batch_size) do |relation, index| start_id, end_id = relation.pluck('MIN(id), MAX(id)').first diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb index d32616862f0..979225dd216 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb @@ -26,6 +26,7 @@ module Gitlab move_repository(project, old_full_path, new_full_path) move_repository(project, "#{old_full_path}.wiki", "#{new_full_path}.wiki") end + move_uploads(old_full_path, new_full_path) unless project.hashed_storage?(:attachments) move_pages(old_full_path, new_full_path) end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index cd490aaa291..34b070dd375 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -116,8 +116,10 @@ module Gitlab new_content_sha || old_content_sha end + # Use #itself to check the value wrapped by a BatchLoader instance, rather + # than if the BatchLoader instance itself is falsey. def blob - new_blob || old_blob + new_blob&.itself || old_blob&.itself end attr_writer :highlighted_diff_lines @@ -173,7 +175,7 @@ module Gitlab end def binary? - has_binary_notice? || old_blob&.binary? || new_blob&.binary? + has_binary_notice? || try_blobs(:binary?) end def text? @@ -181,15 +183,15 @@ module Gitlab end def external_storage_error? - old_blob&.external_storage_error? || new_blob&.external_storage_error? + try_blobs(:external_storage_error?) end def stored_externally? - old_blob&.stored_externally? || new_blob&.stored_externally? + try_blobs(:stored_externally?) end def external_storage - old_blob&.external_storage || new_blob&.external_storage + try_blobs(:external_storage) end def content_changed? @@ -204,15 +206,15 @@ module Gitlab end def size - [old_blob&.size, new_blob&.size].compact.sum + valid_blobs.map(&:size).sum end def raw_size - [old_blob&.raw_size, new_blob&.raw_size].compact.sum + valid_blobs.map(&:raw_size).sum end def raw_binary? - old_blob&.raw_binary? || new_blob&.raw_binary? + try_blobs(:raw_binary?) end def raw_text? @@ -235,6 +237,19 @@ module Gitlab private + # The blob instances are instances of BatchLoader, which means calling + # &. directly on them won't work. Object#try also won't work, because Blob + # doesn't inherit from Object, but from BasicObject (via SimpleDelegator). + def try_blobs(meth) + old_blob&.itself&.public_send(meth) || new_blob&.itself&.public_send(meth) + end + + # We can't use #compact for the same reason we can't use &., but calling + # #nil? explicitly does work because it is proxied to the blob itself. + def valid_blobs + [old_blob, new_blob].reject(&:nil?) + end + def text_position_properties(line) { old_line: line.old_line, new_line: line.new_line } end diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index b669ee5b799..0f897e6316c 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -14,6 +14,7 @@ module Gitlab else @diff_lines = diff_lines end + @raw_lines = @diff_lines.map(&:text) end diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index 37face8e7d0..d3b49b1ec75 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -156,12 +156,14 @@ module Gitlab %W[git apply --3way #{patch_path}] ) do |output, status| puts output + unless status.zero? @failed_files = output.lines.reduce([]) do |memo, line| if line.start_with?('error: patch failed:') file = line.sub(/\Aerror: patch failed: /, '') memo << file unless file =~ IGNORED_FILES_REGEX end + memo end diff --git a/lib/gitlab/email/handler/create_merge_request_handler.rb b/lib/gitlab/email/handler/create_merge_request_handler.rb index e2f7c1d0257..3436306e122 100644 --- a/lib/gitlab/email/handler/create_merge_request_handler.rb +++ b/lib/gitlab/email/handler/create_merge_request_handler.rb @@ -10,6 +10,7 @@ module Gitlab def initialize(mail, mail_key) super(mail, mail_key) + if m = /\A([^\+]*)\+merge-request\+(.*)/.match(mail_key.to_s) @project_path, @incoming_email_token = m.captures end diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index 582028493e9..c0edcabc6fd 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -14,14 +14,7 @@ module Gitlab ENCODING_CONFIDENCE_THRESHOLD = 50 def encode!(message) - return nil unless message.respond_to?(:force_encoding) - return message if message.encoding == Encoding::UTF_8 && message.valid_encoding? - - if message.respond_to?(:frozen?) && message.frozen? - message = message.dup - end - - message.force_encoding("UTF-8") + message = force_encode_utf8(message) return message if message.valid_encoding? # return message if message type is binary @@ -35,6 +28,8 @@ module Gitlab # encode and clean the bad chars message.replace clean(message) + rescue ArgumentError + return nil rescue encoding = detect ? detect[:encoding] : "unknown" "--broken encoding: #{encoding}" @@ -54,8 +49,8 @@ module Gitlab end def encode_utf8(message) - return nil unless message.is_a?(String) - return message if message.encoding == Encoding::UTF_8 && message.valid_encoding? + message = force_encode_utf8(message) + return message if message.valid_encoding? detect = CharlockHolmes::EncodingDetector.detect(message) if detect && detect[:encoding] @@ -69,10 +64,31 @@ module Gitlab else clean(message) end + rescue ArgumentError + return nil + end + + def encode_binary(s) + return "" if s.nil? + + s.dup.force_encoding(Encoding::ASCII_8BIT) + end + + def binary_stringio(s) + StringIO.new(s || '').tap { |io| io.set_encoding(Encoding::ASCII_8BIT) } end private + def force_encode_utf8(message) + raise ArgumentError unless message.respond_to?(:force_encoding) + return message if message.encoding == Encoding::UTF_8 && message.valid_encoding? + + message = message.dup if message.respond_to?(:frozen?) && message.frozen? + + message.force_encoding("UTF-8") + end + def clean(message) message.encode("UTF-16BE", undef: :replace, invalid: :replace, replace: "") .encode("UTF-8") diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index 3f7b42456af..dbb8f317afe 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -71,5 +71,16 @@ module Gitlab redis.exists(@redis_shared_state_key) end end + + # Returns the TTL of the Redis key. + # + # This method will return `nil` if no TTL could be obtained. + def ttl + Gitlab::Redis::SharedState.with do |redis| + ttl = redis.ttl(@redis_shared_state_key) + + ttl if ttl.positive? + end + end end end diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb index 5e426b13ade..8953bc8c148 100644 --- a/lib/gitlab/fogbugz_import/importer.rb +++ b/lib/gitlab/fogbugz_import/importer.rb @@ -112,6 +112,7 @@ module Gitlab [bug['sCategory'], bug['sPriority']].each do |label| unless label.blank? labels << label + unless @known_labels.include?(label) create_label(label) @known_labels << label @@ -265,6 +266,7 @@ module Gitlab if content.blank? content = '*(No description has been entered for this issue)*' end + body << content body.join("\n\n") @@ -278,6 +280,7 @@ module Gitlab if content.blank? content = "*(No comment has been entered for this change)*" end + body << content if updates.any? diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 1f7c35cafaa..71647099f83 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -11,7 +11,7 @@ module Gitlab include Gitlab::EncodingHelper def ref_name(ref) - encode_utf8(ref).sub(/\Arefs\/(tags|heads|remotes)\//, '') + encode!(ref).sub(/\Arefs\/(tags|heads|remotes)\//, '') end def branch_name(ref) diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 228d97a87ab..031fccba92b 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -50,10 +50,19 @@ module Gitlab # to the caller to limit the number of blobs and blob_size_limit. # # Gitaly migration issue: https://gitlab.com/gitlab-org/gitaly/issues/798 - def batch(repository, blob_references, blob_size_limit: nil) - blob_size_limit ||= MAX_DATA_DISPLAY_SIZE - blob_references.map do |sha, path| - find_by_rugged(repository, sha, path, limit: blob_size_limit) + def batch(repository, blob_references, blob_size_limit: MAX_DATA_DISPLAY_SIZE) + Gitlab::GitalyClient.migrate(:list_blobs_by_sha_path) do |is_enabled| + if is_enabled + Gitlab::GitalyClient.allow_n_plus_1_calls do + blob_references.map do |sha, path| + find_by_gitaly(repository, sha, path, limit: blob_size_limit) + end + end + else + blob_references.map do |sha, path| + find_by_rugged(repository, sha, path, limit: blob_size_limit) + end + end end end @@ -122,13 +131,23 @@ module Gitlab ) end - def find_by_gitaly(repository, sha, path) + def find_by_gitaly(repository, sha, path, limit: MAX_DATA_DISPLAY_SIZE) path = path.sub(/\A\/*/, '') path = '/' if path.empty? name = File.basename(path) - entry = Gitlab::GitalyClient::CommitService.new(repository).tree_entry(sha, path, MAX_DATA_DISPLAY_SIZE) + + # Gitaly will think that setting the limit to 0 means unlimited, while + # the client might only need the metadata and thus set the limit to 0. + # In this method we'll then set the limit to 1, but clear the byte of data + # that we got back so for the outside world it looks like the limit was + # actually 0. + req_limit = limit == 0 ? 1 : limit + + entry = Gitlab::GitalyClient::CommitService.new(repository).tree_entry(sha, path, req_limit) return unless entry + entry.data = "" if limit == 0 + case entry.type when :COMMIT new( @@ -154,8 +173,8 @@ module Gitlab end def find_by_rugged(repository, sha, path, limit:) - commit = repository.lookup(sha) - root_tree = commit.tree + rugged_commit = repository.lookup(sha) + root_tree = rugged_commit.tree blob_entry = find_entry_by_path(repository, root_tree.oid, path) diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 145721dea76..016437b2419 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -15,8 +15,6 @@ module Gitlab attr_accessor *SERIALIZE_KEYS # rubocop:disable Lint/AmbiguousOperator - delegate :tree, to: :rugged_commit - def ==(other) return false unless other.is_a?(Gitlab::Git::Commit) @@ -452,6 +450,11 @@ module Gitlab ) end + # Is this the same as Blob.find_entry_by_path ? + def rugged_tree_entry(path) + rugged_commit.tree.path(path) + end + private def init_from_hash(hash) diff --git a/lib/gitlab/git/commit_stats.rb b/lib/gitlab/git/commit_stats.rb index 6bf49a0af18..8463b1eb794 100644 --- a/lib/gitlab/git/commit_stats.rb +++ b/lib/gitlab/git/commit_stats.rb @@ -34,13 +34,8 @@ module Gitlab def rugged_stats(commit) diff = commit.rugged_diff_from_parent - - diff.each_patch do |p| - # TODO: Use the new Rugged convenience methods when they're released - @additions += p.stat[0] - @deletions += p.stat[1] - @total += p.changes - end + _files_changed, @additions, @deletions = diff.stat + @total = @additions + @deletions end end end diff --git a/lib/gitlab/git/conflict/file.rb b/lib/gitlab/git/conflict/file.rb index b2a625e08fa..2a9cf10a068 100644 --- a/lib/gitlab/git/conflict/file.rb +++ b/lib/gitlab/git/conflict/file.rb @@ -2,7 +2,9 @@ module Gitlab module Git module Conflict class File - attr_reader :content, :their_path, :our_path, :our_mode, :repository, :commit_oid + attr_reader :their_path, :our_path, :our_mode, :repository, :commit_oid + + attr_accessor :content def initialize(repository, commit_oid, conflict, content) @repository = repository diff --git a/lib/gitlab/git/conflict/resolution.rb b/lib/gitlab/git/conflict/resolution.rb new file mode 100644 index 00000000000..ab9be683e15 --- /dev/null +++ b/lib/gitlab/git/conflict/resolution.rb @@ -0,0 +1,15 @@ +module Gitlab + module Git + module Conflict + class Resolution + attr_reader :user, :files, :commit_message + + def initialize(user, files, commit_message) + @user = user + @files = files + @commit_message = commit_message + end + end + end + end +end diff --git a/lib/gitlab/git/conflict/resolver.rb b/lib/gitlab/git/conflict/resolver.rb index 03e5c0fcd6f..74c9874d590 100644 --- a/lib/gitlab/git/conflict/resolver.rb +++ b/lib/gitlab/git/conflict/resolver.rb @@ -13,37 +13,27 @@ module Gitlab def conflicts @conflicts ||= begin - target_index = @target_repository.rugged.merge_commits(@our_commit_oid, @their_commit_oid) - - # We don't need to do `with_repo_branch_commit` here, because the target - # project always fetches source refs when creating merge request diffs. - conflict_files(@target_repository, target_index) + @target_repository.gitaly_migrate(:conflicts_list_conflict_files) do |is_enabled| + if is_enabled + gitaly_conflicts_client(@target_repository).list_conflict_files + else + rugged_list_conflict_files + end + end end + rescue GRPC::FailedPrecondition => e + raise Gitlab::Git::Conflict::Resolver::ConflictSideMissing.new(e.message) + rescue Rugged::OdbError, GRPC::BadStatus => e + raise Gitlab::Git::CommandError.new(e) end - def resolve_conflicts(source_repository, user, files, source_branch:, target_branch:, commit_message:) - source_repository.with_repo_branch_commit(@target_repository, target_branch) do - index = source_repository.rugged.merge_commits(@our_commit_oid, @their_commit_oid) - conflicts = conflict_files(source_repository, index) - - files.each do |file_params| - conflict_file = conflict_for_path(conflicts, file_params[:old_path], file_params[:new_path]) - - write_resolved_file_to_index(source_repository, index, conflict_file, file_params) - end - - unless index.conflicts.empty? - missing_files = index.conflicts.map { |file| file[:ours][:path] } - - raise ResolutionError, "Missing resolutions for the following files: #{missing_files.join(', ')}" + def resolve_conflicts(source_repository, resolution, source_branch:, target_branch:) + source_repository.gitaly_migrate(:conflicts_resolve_conflicts) do |is_enabled| + if is_enabled + gitaly_conflicts_client(source_repository).resolve_conflicts(@target_repository, resolution, source_branch, target_branch) + else + rugged_resolve_conflicts(source_repository, resolution, source_branch, target_branch) end - - commit_params = { - message: commit_message, - parents: [@our_commit_oid, @their_commit_oid] - } - - source_repository.commit_index(user, source_branch, index, commit_params) end end @@ -68,6 +58,10 @@ module Gitlab end end + def gitaly_conflicts_client(repository) + repository.gitaly_conflicts_client(@our_commit_oid, @their_commit_oid) + end + def write_resolved_file_to_index(repository, index, file, params) if params[:sections] resolved_lines = file.resolve_lines(params[:sections]) @@ -84,6 +78,40 @@ module Gitlab index.add(path: our_path, oid: oid, mode: file.our_mode) index.conflict_remove(our_path) end + + def rugged_list_conflict_files + target_index = @target_repository.rugged.merge_commits(@our_commit_oid, @their_commit_oid) + + # We don't need to do `with_repo_branch_commit` here, because the target + # project always fetches source refs when creating merge request diffs. + conflict_files(@target_repository, target_index) + end + + def rugged_resolve_conflicts(source_repository, resolution, source_branch, target_branch) + source_repository.with_repo_branch_commit(@target_repository, target_branch) do + index = source_repository.rugged.merge_commits(@our_commit_oid, @their_commit_oid) + conflicts = conflict_files(source_repository, index) + + resolution.files.each do |file_params| + conflict_file = conflict_for_path(conflicts, file_params[:old_path], file_params[:new_path]) + + write_resolved_file_to_index(source_repository, index, conflict_file, file_params) + end + + unless index.conflicts.empty? + missing_files = index.conflicts.map { |file| file[:ours][:path] } + + raise ResolutionError, "Missing resolutions for the following files: #{missing_files.join(', ')}" + end + + commit_params = { + message: resolution.commit_message, + parents: [@our_commit_oid, @their_commit_oid] + } + + source_repository.commit_index(resolution.user, source_branch, index, commit_params) + end + end end end end diff --git a/lib/gitlab/git/gitlab_projects.rb b/lib/gitlab/git/gitlab_projects.rb index d948d7895ed..976fa1ddfe6 100644 --- a/lib/gitlab/git/gitlab_projects.rb +++ b/lib/gitlab/git/gitlab_projects.rb @@ -2,6 +2,9 @@ module Gitlab module Git class GitlabProjects include Gitlab::Git::Popen + include Gitlab::Utils::StrongMemoize + + ShardNameNotFoundError = Class.new(StandardError) # Absolute path to directory where repositories are stored. # Example: /home/git/repositories @@ -38,36 +41,6 @@ module Gitlab io.read end - def rm_project - logger.info "Removing repository <#{repository_absolute_path}>." - FileUtils.rm_rf(repository_absolute_path) - end - - # Move repository from one directory to another - # - # Example: gitlab/gitlab-ci.git -> randx/six.git - # - # Won't work if target namespace directory does not exist - # - def mv_project(new_path) - new_absolute_path = File.join(shard_path, new_path) - - # verify that the source repo exists - unless File.exist?(repository_absolute_path) - logger.error "mv-project failed: source path <#{repository_absolute_path}> does not exist." - return false - end - - # ...and that the target repo does not exist - if File.exist?(new_absolute_path) - logger.error "mv-project failed: destination path <#{new_absolute_path}> already exists." - return false - end - - logger.info "Moving repository from <#{repository_absolute_path}> to <#{new_absolute_path}>." - FileUtils.mv(repository_absolute_path, new_absolute_path) - end - # Import project via git clone --bare # URL must be publicly cloneable def import_project(source, timeout) @@ -97,22 +70,13 @@ module Gitlab end def fork_repository(new_shard_path, new_repository_relative_path) - from_path = repository_absolute_path - to_path = File.join(new_shard_path, new_repository_relative_path) - - # The repository cannot already exist - if File.exist?(to_path) - logger.error "fork-repository failed: destination repository <#{to_path}> already exists." - return false + Gitlab::GitalyClient.migrate(:fork_repository) do |is_enabled| + if is_enabled + gitaly_fork_repository(new_shard_path, new_repository_relative_path) + else + git_fork_repository(new_shard_path, new_repository_relative_path) + end end - - # Ensure the namepsace / hashed storage directory exists - FileUtils.mkdir_p(File.dirname(to_path), mode: 0770) - - logger.info "Forking repository from <#{from_path}> to <#{to_path}>." - cmd = %W(git clone --bare --no-local -- #{from_path} #{to_path}) - - run(cmd, nil) && Gitlab::Git::Repository.create_hooks(to_path, global_hooks_path) end def fetch_remote(name, timeout, force:, tags:, ssh_key: nil, known_hosts: nil) @@ -253,6 +217,48 @@ module Gitlab known_hosts_file&.close! script&.close! end + + private + + def shard_name + strong_memoize(:shard_name) do + shard_name_from_shard_path(shard_path) + end + end + + def shard_name_from_shard_path(shard_path) + Gitlab.config.repositories.storages.find { |_, info| info['path'] == shard_path }&.first || + raise(ShardNameNotFoundError, "no shard found for path '#{shard_path}'") + end + + def git_fork_repository(new_shard_path, new_repository_relative_path) + from_path = repository_absolute_path + to_path = File.join(new_shard_path, new_repository_relative_path) + + # The repository cannot already exist + if File.exist?(to_path) + logger.error "fork-repository failed: destination repository <#{to_path}> already exists." + return false + end + + # Ensure the namepsace / hashed storage directory exists + FileUtils.mkdir_p(File.dirname(to_path), mode: 0770) + + logger.info "Forking repository from <#{from_path}> to <#{to_path}>." + cmd = %W(git clone --bare --no-local -- #{from_path} #{to_path}) + + run(cmd, nil) && Gitlab::Git::Repository.create_hooks(to_path, global_hooks_path) + end + + def gitaly_fork_repository(new_shard_path, new_repository_relative_path) + target_repository = Gitlab::Git::Repository.new(shard_name_from_shard_path(new_shard_path), new_repository_relative_path, nil) + raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil) + + Gitlab::GitalyClient::RepositoryService.new(target_repository).fork_repository(raw_repository) + rescue GRPC::BadStatus => e + logger.error "fork-repository failed: #{e.message}" + false + end end end end diff --git a/lib/gitlab/git/index.rb b/lib/gitlab/git/index.rb index db532600d1b..d94082a3e30 100644 --- a/lib/gitlab/git/index.rb +++ b/lib/gitlab/git/index.rb @@ -10,6 +10,7 @@ module Gitlab DEFAULT_MODE = 0o100644 ACTIONS = %w(create create_dir update move delete).freeze + ACTION_OPTIONS = %i(file_path previous_path content encoding).freeze attr_reader :repository, :raw_index @@ -20,6 +21,11 @@ module Gitlab delegate :read_tree, :get, to: :raw_index + def apply(action, options) + validate_action!(action) + public_send(action, options.slice(*ACTION_OPTIONS)) # rubocop:disable GitlabSecurity/PublicSend + end + def write_tree raw_index.write_tree(repository.rugged) end @@ -140,6 +146,12 @@ module Gitlab rescue Rugged::IndexError => e raise IndexError, e.message end + + def validate_action!(action) + unless ACTIONS.include?(action.to_s) + raise ArgumentError, "Unknown action '#{action}'" + end + end end end end diff --git a/lib/gitlab/git/operation_service.rb b/lib/gitlab/git/operation_service.rb index ef5bdbaf819..3fb0e2eed93 100644 --- a/lib/gitlab/git/operation_service.rb +++ b/lib/gitlab/git/operation_service.rb @@ -97,6 +97,11 @@ module Gitlab end end + def update_branch(branch_name, newrev, oldrev) + ref = Gitlab::Git::BRANCH_REF_PREFIX + branch_name + update_ref_in_hooks(ref, newrev, oldrev) + end + private # Returns [newrev, should_run_after_create, should_run_after_create_branch] diff --git a/lib/gitlab/git/remote_mirror.rb b/lib/gitlab/git/remote_mirror.rb new file mode 100644 index 00000000000..38e9d2a8554 --- /dev/null +++ b/lib/gitlab/git/remote_mirror.rb @@ -0,0 +1,75 @@ +module Gitlab + module Git + class RemoteMirror + def initialize(repository, ref_name) + @repository = repository + @ref_name = ref_name + end + + def update(only_branches_matching: [], only_tags_matching: []) + local_branches = refs_obj(@repository.local_branches, only_refs_matching: only_branches_matching) + remote_branches = refs_obj(@repository.remote_branches(@ref_name), only_refs_matching: only_branches_matching) + + updated_branches = changed_refs(local_branches, remote_branches) + push_branches(updated_branches.keys) if updated_branches.present? + + delete_refs(local_branches, remote_branches) + + local_tags = refs_obj(@repository.tags, only_refs_matching: only_tags_matching) + remote_tags = refs_obj(@repository.remote_tags(@ref_name), only_refs_matching: only_tags_matching) + + updated_tags = changed_refs(local_tags, remote_tags) + @repository.push_remote_branches(@ref_name, updated_tags.keys) if updated_tags.present? + + delete_refs(local_tags, remote_tags) + end + + private + + def refs_obj(refs, only_refs_matching: []) + refs.each_with_object({}) do |ref, refs| + next if only_refs_matching.present? && !only_refs_matching.include?(ref.name) + + refs[ref.name] = ref + end + end + + def changed_refs(local_refs, remote_refs) + local_refs.select do |ref_name, ref| + remote_ref = remote_refs[ref_name] + + remote_ref.nil? || ref.dereferenced_target != remote_ref.dereferenced_target + end + end + + def push_branches(branches) + default_branch, branches = branches.partition do |branch| + @repository.root_ref == branch + end + + # Push the default branch first so it works fine when remote mirror is empty. + branches.unshift(*default_branch) + + @repository.push_remote_branches(@ref_name, branches) + end + + def delete_refs(local_refs, remote_refs) + refs = refs_to_delete(local_refs, remote_refs) + + @repository.delete_remote_branches(@ref_name, refs.keys) if refs.present? + end + + def refs_to_delete(local_refs, remote_refs) + default_branch_id = @repository.commit.id + + remote_refs.select do |remote_ref_name, remote_ref| + next false if local_refs[remote_ref_name] # skip if branch or tag exist in local repo + + remote_ref_id = remote_ref.dereferenced_target.try(:id) + + remote_ref_id && @repository.rugged_is_ancestor?(remote_ref_id, default_branch_id) + end + end + end + end +end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 36dc6b820ce..d0467bca992 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -21,6 +21,7 @@ module Gitlab REBASE_WORKTREE_PREFIX = 'rebase'.freeze SQUASH_WORKTREE_PREFIX = 'squash'.freeze GITALY_INTERNAL_URL = 'ssh://gitaly/internal.git'.freeze + GITLAB_PROJECTS_TIMEOUT = Gitlab.config.gitlab_shell.git_timeout NoRepository = Class.new(StandardError) InvalidBlobName = Class.new(StandardError) @@ -83,7 +84,7 @@ module Gitlab # Rugged repo object attr_reader :rugged - attr_reader :storage, :gl_repository, :relative_path + attr_reader :gitlab_projects, :storage, :gl_repository, :relative_path # This initializer method is only used on the client side (gitlab-ce). # Gitaly-ruby uses a different initializer. @@ -93,6 +94,12 @@ module Gitlab @gl_repository = gl_repository storage_path = Gitlab.config.repositories.storages[@storage]['path'] + @gitlab_projects = Gitlab::Git::GitlabProjects.new( + storage_path, + relative_path, + global_hooks_path: Gitlab.config.gitlab_shell.hooks_path, + logger: Rails.logger + ) @path = File.join(storage_path, @relative_path) @name = @relative_path.split("/").last @attributes = Gitlab::Git::Attributes.new(path) @@ -126,7 +133,7 @@ module Gitlab end def exists? - Gitlab::GitalyClient.migrate(:repository_exists, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled| + Gitlab::GitalyClient.migrate(:repository_exists) do |enabled| if enabled gitaly_repository_client.exists? else @@ -188,7 +195,7 @@ module Gitlab end def local_branches(sort_by: nil) - gitaly_migrate(:local_branches, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| + gitaly_migrate(:local_branches) do |is_enabled| if is_enabled gitaly_ref_client.local_branches(sort_by: sort_by) else @@ -491,11 +498,13 @@ module Gitlab end def count_commits(options) + count_commits_options = process_count_commits_options(options) + gitaly_migrate(:count_commits) do |is_enabled| if is_enabled - count_commits_by_gitaly(options) + count_commits_by_gitaly(count_commits_options) else - count_commits_by_shelling_out(options) + count_commits_by_shelling_out(count_commits_options) end end end @@ -533,8 +542,8 @@ module Gitlab end # Counts the amount of commits between `from` and `to`. - def count_commits_between(from, to) - count_commits(ref: "#{from}..#{to}") + def count_commits_between(from, to, options = {}) + count_commits(from: from, to: to, **options) end # Returns the SHA of the most recent common ancestor of +from+ and +to+ @@ -562,7 +571,21 @@ module Gitlab end def merged_branch_names(branch_names = []) - Set.new(git_merged_branch_names(branch_names)) + return [] unless root_ref + + root_sha = find_branch(root_ref)&.target + + return [] unless root_sha + + branches = gitaly_migrate(:merged_branch_names) do |is_enabled| + if is_enabled + gitaly_merged_branch_names(branch_names, root_sha) + else + git_merged_branch_names(branch_names, root_sha) + end + end + + Set.new(branches) end # Return an array of Diff objects that represent the diff @@ -645,6 +668,7 @@ module Gitlab end end end + @refs_hash end @@ -919,7 +943,7 @@ module Gitlab # If `mirror_refmap` is present the remote is set as mirror with that mapping def add_remote(remote_name, url, mirror_refmap: nil) - gitaly_migrate(:operation_user_add_tag) do |is_enabled| + gitaly_migrate(:remote_add_remote) do |is_enabled| if is_enabled gitaly_remote_client.add_remote(remote_name, url, mirror_refmap) else @@ -929,7 +953,7 @@ module Gitlab end def remove_remote(remote_name) - gitaly_migrate(:operation_user_add_tag) do |is_enabled| + gitaly_migrate(:remote_remove_remote) do |is_enabled| if is_enabled gitaly_remote_client.remove_remote(remote_name) else @@ -1094,14 +1118,27 @@ module Gitlab end end - def write_ref(ref_path, ref) + def write_ref(ref_path, ref, old_ref: nil, shell: true) + if shell + shell_write_ref(ref_path, ref, old_ref) + else + rugged_write_ref(ref_path, ref) + end + end + + def shell_write_ref(ref_path, ref, old_ref) raise ArgumentError, "invalid ref_path #{ref_path.inspect}" if ref_path.include?(' ') raise ArgumentError, "invalid ref #{ref.inspect}" if ref.include?("\x00") + raise ArgumentError, "invalid old_ref #{old_ref.inspect}" if !old_ref.nil? && old_ref.include?("\x00") - input = "update #{ref_path}\x00#{ref}\x00\x00" + input = "update #{ref_path}\x00#{ref}\x00#{old_ref}\x00" run_git!(%w[update-ref --stdin -z]) { |stdin| stdin.write(input) } end + def rugged_write_ref(ref_path, ref) + rugged.references.create(ref_path, ref, force: true) + end + def fetch_ref(source_repository, source_ref:, target_ref:) Gitlab::Git.check_namespace!(source_repository) source_repository = RemoteRepository.new(source_repository) unless source_repository.is_a?(RemoteRepository) @@ -1154,23 +1191,13 @@ module Gitlab end def fetch_repository_as_mirror(repository) - remote_name = "tmp-#{SecureRandom.hex}" - - # Notice that this feature flag is not for `fetch_repository_as_mirror` - # as a whole but for the fetching mechanism (file path or gitaly-ssh). - url, env = gitaly_migrate(:fetch_internal) do |is_enabled| + gitaly_migrate(:remote_fetch_internal_remote) do |is_enabled| if is_enabled - repository = RemoteRepository.new(repository) unless repository.is_a?(RemoteRepository) - [GITALY_INTERNAL_URL, repository.fetch_env] + gitaly_remote_client.fetch_internal_remote(repository) else - [repository.path, nil] + rugged_fetch_repository_as_mirror(repository) end end - - add_remote(remote_name, url, mirror_refmap: :all_refs) - fetch_remote(remote_name, env: env) - ensure - remove_remote(remote_name) end def blob_at(sha, path) @@ -1178,7 +1205,7 @@ module Gitlab end # Items should be of format [[commit_id, path], [commit_id1, path1]] - def batch_blobs(items, blob_size_limit: nil) + def batch_blobs(items, blob_size_limit: Gitlab::Git::Blob::MAX_DATA_DISPLAY_SIZE) Gitlab::Git::Blob.batch(self, items, blob_size_limit: blob_size_limit) end @@ -1209,26 +1236,31 @@ module Gitlab end def rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) - rebase_path = worktree_path(REBASE_WORKTREE_PREFIX, rebase_id) - env = git_env_for_user(user) - - with_worktree(rebase_path, branch, env: env) do - run_git!( - %W(pull --rebase #{remote_repository.path} #{remote_branch}), - chdir: rebase_path, env: env - ) - - rebase_sha = run_git!(%w(rev-parse HEAD), chdir: rebase_path, env: env).strip - - Gitlab::Git::OperationService.new(user, self) - .update_branch(branch, rebase_sha, branch_sha) - - rebase_sha + gitaly_migrate(:rebase) do |is_enabled| + if is_enabled + gitaly_rebase(user, rebase_id, + branch: branch, + branch_sha: branch_sha, + remote_repository: remote_repository, + remote_branch: remote_branch) + else + git_rebase(user, rebase_id, + branch: branch, + branch_sha: branch_sha, + remote_repository: remote_repository, + remote_branch: remote_branch) + end end end def rebase_in_progress?(rebase_id) - fresh_worktree?(worktree_path(REBASE_WORKTREE_PREFIX, rebase_id)) + gitaly_migrate(:rebase_in_progress) do |is_enabled| + if is_enabled + gitaly_repository_client.rebase_in_progress?(rebase_id) + else + fresh_worktree?(worktree_path(REBASE_WORKTREE_PREFIX, rebase_id)) + end + end end def squash(user, squash_id, branch:, start_sha:, end_sha:, author:, message:) @@ -1266,6 +1298,60 @@ module Gitlab fresh_worktree?(worktree_path(SQUASH_WORKTREE_PREFIX, squash_id)) end + def push_remote_branches(remote_name, branch_names, forced: true) + success = @gitlab_projects.push_branches(remote_name, GITLAB_PROJECTS_TIMEOUT, forced, branch_names) + + success || gitlab_projects_error + end + + def delete_remote_branches(remote_name, branch_names) + success = @gitlab_projects.delete_remote_branches(remote_name, branch_names) + + success || gitlab_projects_error + end + + def delete_remote_branches(remote_name, branch_names) + success = @gitlab_projects.delete_remote_branches(remote_name, branch_names) + + success || gitlab_projects_error + end + + # rubocop:disable Metrics/ParameterLists + def multi_action( + user, branch_name:, message:, actions:, + author_email: nil, author_name: nil, + start_branch_name: nil, start_repository: self) + + OperationService.new(user, self).with_branch( + branch_name, + start_branch_name: start_branch_name, + start_repository: start_repository + ) do |start_commit| + index = Gitlab::Git::Index.new(self) + parents = [] + + if start_commit + index.read_tree(start_commit.rugged_commit.tree) + parents = [start_commit.sha] + end + + actions.each { |opts| index.apply(opts.delete(:action), opts) } + + committer = user_to_committer(user) + author = Gitlab::Git.committer_hash(email: author_email, name: author_name) || committer + options = { + tree: index.write_tree, + message: message, + parents: parents, + author: author, + committer: committer + } + + create_commit(options) + end + end + # rubocop:enable Metrics/ParameterLists + def gitaly_repository Gitlab::GitalyClient::Util.repository(@storage, @relative_path, @gl_repository) end @@ -1294,6 +1380,10 @@ module Gitlab @gitaly_remote_client ||= Gitlab::GitalyClient::RemoteService.new(self) end + def gitaly_conflicts_client(our_commit_oid, their_commit_oid) + Gitlab::GitalyClient::ConflictsService.new(self, our_commit_oid, their_commit_oid) + end + def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block) Gitlab::GitalyClient.migrate(method, status: status, &block) rescue GRPC::NotFound => e @@ -1411,14 +1501,7 @@ module Gitlab sort_branches(branches, sort_by) end - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/695 - def git_merged_branch_names(branch_names = []) - return [] unless root_ref - - root_sha = find_branch(root_ref)&.target - - return [] unless root_sha - + def git_merged_branch_names(branch_names, root_sha) git_arguments = %W[branch --merged #{root_sha} --format=%(refname:short)\ %(objectname)] + branch_names @@ -1432,6 +1515,34 @@ module Gitlab end end + def gitaly_merged_branch_names(branch_names, root_sha) + qualified_branch_names = branch_names.map { |b| "refs/heads/#{b}" } + + gitaly_ref_client.merged_branches(qualified_branch_names) + .reject { |b| b.target == root_sha } + .map(&:name) + end + + def process_count_commits_options(options) + if options[:from] || options[:to] + ref = + if options[:left_right] # Compare with merge-base for left-right + "#{options[:from]}...#{options[:to]}" + else + "#{options[:from]}..#{options[:to]}" + end + + options.merge(ref: ref) + + elsif options[:ref] && options[:left_right] + from, to = options[:ref].match(/\A([^\.]*)\.{2,3}([^\.]*)\z/)[1..2] + + options.merge(from: from, to: to) + else + options + end + end + def log_using_shell?(options) options[:path].present? || options[:disable_walk] || @@ -1654,19 +1765,59 @@ module Gitlab end def count_commits_by_gitaly(options) - gitaly_commit_client.commit_count(options[:ref], options) + if options[:left_right] + from = options[:from] + to = options[:to] + + right_count = gitaly_commit_client + .commit_count("#{from}..#{to}", options) + left_count = gitaly_commit_client + .commit_count("#{to}..#{from}", options) + + [left_count, right_count] + else + gitaly_commit_client.commit_count(options[:ref], options) + end end def count_commits_by_shelling_out(options) + cmd = count_commits_shelling_command(options) + + raw_output = IO.popen(cmd) { |io| io.read } + + process_count_commits_raw_output(raw_output, options) + end + + def count_commits_shelling_command(options) cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} rev-list] cmd << "--after=#{options[:after].iso8601}" if options[:after] cmd << "--before=#{options[:before].iso8601}" if options[:before] + cmd << "--max-count=#{options[:max_count]}" if options[:max_count] + cmd << "--left-right" if options[:left_right] cmd += %W[--count #{options[:ref]}] cmd += %W[-- #{options[:path]}] if options[:path].present? + cmd + end - raw_output = IO.popen(cmd) { |io| io.read } + def process_count_commits_raw_output(raw_output, options) + if options[:left_right] + result = raw_output.scan(/\d+/).map(&:to_i) - raw_output.to_i + if result.sum != options[:max_count] + result + else # Reaching max count, right is not accurate + right_option = + process_count_commits_options(options + .except(:left_right, :from, :to) + .merge(ref: options[:to])) + + right = count_commits_by_shelling_out(right_option) + + [result.first, right] # left should be accurate in the first call + end + else + raw_output.to_i + end end def gitaly_ls_files(ref) @@ -1886,6 +2037,40 @@ module Gitlab tree_id end + def gitaly_rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) + gitaly_operation_client.user_rebase(user, rebase_id, + branch: branch, + branch_sha: branch_sha, + remote_repository: remote_repository, + remote_branch: remote_branch) + end + + def git_rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) + rebase_path = worktree_path(REBASE_WORKTREE_PREFIX, rebase_id) + env = git_env_for_user(user) + + if remote_repository.is_a?(RemoteRepository) + env.merge!(remote_repository.fetch_env) + remote_repo_path = GITALY_INTERNAL_URL + else + remote_repo_path = remote_repository.path + end + + with_worktree(rebase_path, branch, env: env) do + run_git!( + %W(pull --rebase #{remote_repo_path} #{remote_branch}), + chdir: rebase_path, env: env + ) + + rebase_sha = run_git!(%w(rev-parse HEAD), chdir: rebase_path, env: env).strip + + Gitlab::Git::OperationService.new(user, self) + .update_branch(branch, rebase_sha, branch_sha) + + rebase_sha + end + end + def local_fetch_ref(source_path, source_ref:, target_ref:) args = %W(fetch --no-tags -f #{source_path} #{source_ref}:#{target_ref}) run_git(args) @@ -1936,9 +2121,23 @@ module Gitlab false end + def rugged_fetch_repository_as_mirror(repository) + remote_name = "tmp-#{SecureRandom.hex}" + repository = RemoteRepository.new(repository) unless repository.is_a?(RemoteRepository) + + add_remote(remote_name, GITALY_INTERNAL_URL, mirror_refmap: :all_refs) + fetch_remote(remote_name, env: repository.fetch_env) + ensure + remove_remote(remote_name) + end + def fetch_remote(remote_name = 'origin', env: nil) run_git(['fetch', remote_name], env: env).last.zero? end + + def gitlab_projects_error + raise CommandError, @gitlab_projects.output + end end end end diff --git a/lib/gitlab/git/storage/forked_storage_check.rb b/lib/gitlab/git/storage/forked_storage_check.rb index 1307f400700..0a4e557b59b 100644 --- a/lib/gitlab/git/storage/forked_storage_check.rb +++ b/lib/gitlab/git/storage/forked_storage_check.rb @@ -27,6 +27,7 @@ module Gitlab status = nil while status.nil? + if deadline > Time.now.utc sleep(wait_time) _pid, status = Process.wait2(filesystem_check_pid, Process::WNOHANG) diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index b753ac46291..4507ea923b4 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -330,22 +330,6 @@ module Gitlab Google::Protobuf::Timestamp.new(seconds: t.to_i) end - def self.encode(s) - return "" if s.nil? - - s.dup.force_encoding(Encoding::ASCII_8BIT) - end - - def self.binary_stringio(s) - io = StringIO.new(s || '') - io.set_encoding(Encoding::ASCII_8BIT) - io - end - - def self.encode_repeated(a) - Google::Protobuf::RepeatedField.new(:bytes, a.map { |s| self.encode(s) } ) - end - # The default timeout on all Gitaly calls def self.default_timeout return 0 if Sidekiq.server? diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index fb3e27770b4..fed05bb6c64 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -1,6 +1,8 @@ module Gitlab module GitalyClient class CommitService + include Gitlab::EncodingHelper + # The ID of empty tree. # See http://stackoverflow.com/a/40884093/1856239 and https://github.com/git/git/blob/3ad8b5bf26362ac67c9020bf8c30eee54a84f56d/cache.h#L1011-L1012 EMPTY_TREE_ID = '4b825dc642cb6eb9a060e54bf8d69288fbee4904'.freeze @@ -13,7 +15,7 @@ module Gitlab def ls_files(revision) request = Gitaly::ListFilesRequest.new( repository: @gitaly_repo, - revision: GitalyClient.encode(revision) + revision: encode_binary(revision) ) response = GitalyClient.call(@repository.storage, :commit_service, :list_files, request, timeout: GitalyClient.medium_timeout) @@ -73,7 +75,7 @@ module Gitlab request = Gitaly::TreeEntryRequest.new( repository: @gitaly_repo, revision: ref, - path: GitalyClient.encode(path), + path: encode_binary(path), limit: limit.to_i ) @@ -98,8 +100,8 @@ module Gitlab def tree_entries(repository, revision, path) request = Gitaly::GetTreeEntriesRequest.new( repository: @gitaly_repo, - revision: GitalyClient.encode(revision), - path: path.present? ? GitalyClient.encode(path) : '.' + revision: encode_binary(revision), + path: path.present? ? encode_binary(path) : '.' ) response = GitalyClient.call(@repository.storage, :commit_service, :get_tree_entries, request, timeout: GitalyClient.medium_timeout) @@ -112,8 +114,8 @@ module Gitlab type: gitaly_tree_entry.type.downcase, mode: gitaly_tree_entry.mode.to_s(8), name: File.basename(gitaly_tree_entry.path), - path: GitalyClient.encode(gitaly_tree_entry.path), - flat_path: GitalyClient.encode(gitaly_tree_entry.flat_path), + path: encode_binary(gitaly_tree_entry.path), + flat_path: encode_binary(gitaly_tree_entry.flat_path), commit_id: gitaly_tree_entry.commit_oid ) end @@ -128,6 +130,7 @@ module Gitlab request.after = Google::Protobuf::Timestamp.new(seconds: options[:after].to_i) if options[:after].present? request.before = Google::Protobuf::Timestamp.new(seconds: options[:before].to_i) if options[:before].present? request.path = options[:path] if options[:path].present? + request.max_count = options[:max_count] if options[:max_count].present? GitalyClient.call(@repository.storage, :commit_service, :count_commits, request, timeout: GitalyClient.medium_timeout).count end @@ -135,8 +138,8 @@ module Gitlab def last_commit_for_path(revision, path) request = Gitaly::LastCommitForPathRequest.new( repository: @gitaly_repo, - revision: GitalyClient.encode(revision), - path: GitalyClient.encode(path.to_s) + revision: encode_binary(revision), + path: encode_binary(path.to_s) ) gitaly_commit = GitalyClient.call(@repository.storage, :commit_service, :last_commit_for_path, request, timeout: GitalyClient.fast_timeout).commit @@ -202,8 +205,8 @@ module Gitlab def raw_blame(revision, path) request = Gitaly::RawBlameRequest.new( repository: @gitaly_repo, - revision: GitalyClient.encode(revision), - path: GitalyClient.encode(path) + revision: encode_binary(revision), + path: encode_binary(path) ) response = GitalyClient.call(@repository.storage, :commit_service, :raw_blame, request, timeout: GitalyClient.medium_timeout) @@ -213,7 +216,7 @@ module Gitlab def find_commit(revision) request = Gitaly::FindCommitRequest.new( repository: @gitaly_repo, - revision: GitalyClient.encode(revision) + revision: encode_binary(revision) ) response = GitalyClient.call(@repository.storage, :commit_service, :find_commit, request, timeout: GitalyClient.medium_timeout) @@ -224,7 +227,7 @@ module Gitlab def patch(revision) request = Gitaly::CommitPatchRequest.new( repository: @gitaly_repo, - revision: GitalyClient.encode(revision) + revision: encode_binary(revision) ) response = GitalyClient.call(@repository.storage, :diff_service, :commit_patch, request, timeout: GitalyClient.medium_timeout) @@ -234,7 +237,7 @@ module Gitlab def commit_stats(revision) request = Gitaly::CommitStatsRequest.new( repository: @gitaly_repo, - revision: GitalyClient.encode(revision) + revision: encode_binary(revision) ) GitalyClient.call(@repository.storage, :commit_service, :commit_stats, request, timeout: GitalyClient.medium_timeout) end @@ -250,9 +253,9 @@ module Gitlab ) request.after = GitalyClient.timestamp(options[:after]) if options[:after] request.before = GitalyClient.timestamp(options[:before]) if options[:before] - request.revision = GitalyClient.encode(options[:ref]) if options[:ref] + request.revision = encode_binary(options[:ref]) if options[:ref] - request.paths = GitalyClient.encode_repeated(Array(options[:path])) if options[:path].present? + request.paths = encode_repeated(Array(options[:path])) if options[:path].present? response = GitalyClient.call(@repository.storage, :commit_service, :find_commits, request, timeout: GitalyClient.medium_timeout) @@ -264,7 +267,7 @@ module Gitlab enum = Enumerator.new do |y| shas.each_slice(20) do |revs| - request.shas = GitalyClient.encode_repeated(revs) + request.shas = encode_repeated(revs) y.yield request @@ -303,7 +306,7 @@ module Gitlab repository: @gitaly_repo, left_commit_id: from_id, right_commit_id: to_id, - paths: options.fetch(:paths, []).compact.map { |path| GitalyClient.encode(path) } + paths: options.fetch(:paths, []).compact.map { |path| encode_binary(path) } } end @@ -314,6 +317,10 @@ module Gitlab end end end + + def encode_repeated(a) + Google::Protobuf::RepeatedField.new(:bytes, a.map { |s| encode_binary(s) } ) + end end end end diff --git a/lib/gitlab/gitaly_client/conflicts_service.rb b/lib/gitlab/gitaly_client/conflicts_service.rb new file mode 100644 index 00000000000..40f032cf873 --- /dev/null +++ b/lib/gitlab/gitaly_client/conflicts_service.rb @@ -0,0 +1,95 @@ +module Gitlab + module GitalyClient + class ConflictsService + include Gitlab::EncodingHelper + + MAX_MSG_SIZE = 128.kilobytes.freeze + + def initialize(repository, our_commit_oid, their_commit_oid) + @gitaly_repo = repository.gitaly_repository + @repository = repository + @our_commit_oid = our_commit_oid + @their_commit_oid = their_commit_oid + end + + def list_conflict_files + request = Gitaly::ListConflictFilesRequest.new( + repository: @gitaly_repo, + our_commit_oid: @our_commit_oid, + their_commit_oid: @their_commit_oid + ) + response = GitalyClient.call(@repository.storage, :conflicts_service, :list_conflict_files, request) + + files_from_response(response).to_a + end + + def resolve_conflicts(target_repository, resolution, source_branch, target_branch) + reader = binary_stringio(resolution.files.to_json) + + req_enum = Enumerator.new do |y| + header = resolve_conflicts_request_header(target_repository, resolution, source_branch, target_branch) + y.yield Gitaly::ResolveConflictsRequest.new(header: header) + + until reader.eof? + chunk = reader.read(MAX_MSG_SIZE) + + y.yield Gitaly::ResolveConflictsRequest.new(files_json: chunk) + end + end + + response = GitalyClient.call(@repository.storage, :conflicts_service, :resolve_conflicts, req_enum, remote_storage: target_repository.storage) + + if response.resolution_error.present? + raise Gitlab::Git::Conflict::Resolver::ResolutionError, response.resolution_error + end + end + + private + + def resolve_conflicts_request_header(target_repository, resolution, source_branch, target_branch) + Gitaly::ResolveConflictsRequestHeader.new( + repository: @gitaly_repo, + our_commit_oid: @our_commit_oid, + target_repository: target_repository.gitaly_repository, + their_commit_oid: @their_commit_oid, + source_branch: source_branch, + target_branch: target_branch, + commit_message: resolution.commit_message, + user: Gitlab::Git::User.from_gitlab(resolution.user).to_gitaly + ) + end + + def files_from_response(response) + files = [] + + response.each do |msg| + msg.files.each do |gitaly_file| + if gitaly_file.header + files << file_from_gitaly_header(gitaly_file.header) + else + files.last.content << gitaly_file.content + end + end + end + + files + end + + def file_from_gitaly_header(header) + Gitlab::Git::Conflict::File.new( + Gitlab::GitalyClient::Util.git_repository(header.repository), + header.commit_oid, + conflict_from_gitaly_file_header(header), + '' + ) + end + + def conflict_from_gitaly_file_header(header) + { + ours: { path: header.our_path, mode: header.our_mode }, + theirs: { path: header.their_path } + } + end + end + end +end diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 400a4af363b..7319de69d13 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -1,6 +1,8 @@ module Gitlab module GitalyClient class OperationService + include Gitlab::EncodingHelper + def initialize(repository) @gitaly_repo = repository.gitaly_repository @repository = repository @@ -9,7 +11,7 @@ module Gitlab def rm_tag(tag_name, user) request = Gitaly::UserDeleteTagRequest.new( repository: @gitaly_repo, - tag_name: GitalyClient.encode(tag_name), + tag_name: encode_binary(tag_name), user: Gitlab::Git::User.from_gitlab(user).to_gitaly ) @@ -24,9 +26,9 @@ module Gitlab request = Gitaly::UserCreateTagRequest.new( repository: @gitaly_repo, user: Gitlab::Git::User.from_gitlab(user).to_gitaly, - tag_name: GitalyClient.encode(tag_name), - target_revision: GitalyClient.encode(target), - message: GitalyClient.encode(message.to_s) + tag_name: encode_binary(tag_name), + target_revision: encode_binary(target), + message: encode_binary(message.to_s) ) response = GitalyClient.call(@repository.storage, :operation_service, :user_create_tag, request) @@ -44,12 +46,13 @@ module Gitlab def user_create_branch(branch_name, user, start_point) request = Gitaly::UserCreateBranchRequest.new( repository: @gitaly_repo, - branch_name: GitalyClient.encode(branch_name), + branch_name: encode_binary(branch_name), user: Gitlab::Git::User.from_gitlab(user).to_gitaly, - start_point: GitalyClient.encode(start_point) + start_point: encode_binary(start_point) ) response = GitalyClient.call(@repository.storage, :operation_service, :user_create_branch, request) + if response.pre_receive_error.present? raise Gitlab::Git::HooksService::PreReceiveError.new(response.pre_receive_error) end @@ -64,7 +67,7 @@ module Gitlab def user_delete_branch(branch_name, user) request = Gitaly::UserDeleteBranchRequest.new( repository: @gitaly_repo, - branch_name: GitalyClient.encode(branch_name), + branch_name: encode_binary(branch_name), user: Gitlab::Git::User.from_gitlab(user).to_gitaly ) @@ -89,8 +92,8 @@ module Gitlab repository: @gitaly_repo, user: Gitlab::Git::User.from_gitlab(user).to_gitaly, commit_id: source_sha, - branch: GitalyClient.encode(target_branch), - message: GitalyClient.encode(message) + branch: encode_binary(target_branch), + message: encode_binary(message) ) ) @@ -99,6 +102,7 @@ module Gitlab request_enum.push(Gitaly::UserMergeBranchRequest.new(apply: true)) branch_update = response_enum.next.branch_update + return if branch_update.nil? raise Gitlab::Git::CommitError.new('failed to apply merge to branch') unless branch_update.commit_id.present? Gitlab::Git::OperationService::BranchUpdate.from_gitaly(branch_update) @@ -111,7 +115,7 @@ module Gitlab repository: @gitaly_repo, user: Gitlab::Git::User.from_gitlab(user).to_gitaly, commit_id: source_sha, - branch: GitalyClient.encode(target_branch) + branch: encode_binary(target_branch) ) branch_update = GitalyClient.call( @@ -143,6 +147,34 @@ module Gitlab start_repository: start_repository) end + def user_rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) + request = Gitaly::UserRebaseRequest.new( + repository: @gitaly_repo, + user: Gitlab::Git::User.from_gitlab(user).to_gitaly, + rebase_id: rebase_id.to_s, + branch: encode_binary(branch), + branch_sha: branch_sha, + remote_repository: remote_repository.gitaly_repository, + remote_branch: encode_binary(remote_branch) + ) + + response = GitalyClient.call( + @repository.storage, + :operation_service, + :user_rebase, + request, + remote_storage: remote_repository.storage + ) + + if response.pre_receive_error.presence + raise Gitlab::Git::HooksService::PreReceiveError, response.pre_receive_error + elsif response.git_error.presence + raise Gitlab::Git::Repository::GitError, response.git_error + else + response.rebase_sha + end + end + private def call_cherry_pick_or_revert(rpc, user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) @@ -152,9 +184,9 @@ module Gitlab repository: @gitaly_repo, user: Gitlab::Git::User.from_gitlab(user).to_gitaly, commit: commit.to_gitaly_commit, - branch_name: GitalyClient.encode(branch_name), - message: GitalyClient.encode(message), - start_branch_name: GitalyClient.encode(start_branch_name.to_s), + branch_name: encode_binary(branch_name), + message: encode_binary(message), + start_branch_name: encode_binary(start_branch_name.to_s), start_repository: start_repository.gitaly_repository ) diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index 066e4e183c0..f8e2a27f3fe 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -14,12 +14,18 @@ module Gitlab request = Gitaly::FindAllBranchesRequest.new(repository: @gitaly_repo) response = GitalyClient.call(@storage, :ref_service, :find_all_branches, request) - response.flat_map do |message| - message.branches.map do |branch| - target_commit = Gitlab::Git::Commit.decorate(@repository, branch.target) - Gitlab::Git::Branch.new(@repository, branch.name, branch.target.id, target_commit) - end - end + consume_find_all_branches_response(response) + end + + def merged_branches(branch_names = []) + request = Gitaly::FindAllBranchesRequest.new( + repository: @gitaly_repo, + merged_only: true, + merged_branches: branch_names.map { |s| encode_binary(s) } + ) + response = GitalyClient.call(@storage, :ref_service, :find_all_branches, request) + + consume_find_all_branches_response(response) end def default_branch_name @@ -62,7 +68,7 @@ module Gitlab request = Gitaly::FindLocalBranchesRequest.new(repository: @gitaly_repo) request.sort_by = sort_by_param(sort_by) if sort_by response = GitalyClient.call(@storage, :ref_service, :find_local_branches, request) - consume_branches_response(response) + consume_find_local_branches_response(response) end def tags @@ -72,7 +78,7 @@ module Gitlab end def ref_exists?(ref_name) - request = Gitaly::RefExistsRequest.new(repository: @gitaly_repo, ref: GitalyClient.encode(ref_name)) + request = Gitaly::RefExistsRequest.new(repository: @gitaly_repo, ref: encode_binary(ref_name)) response = GitalyClient.call(@storage, :ref_service, :ref_exists, request) response.value rescue GRPC::InvalidArgument => e @@ -82,7 +88,7 @@ module Gitlab def find_branch(branch_name) request = Gitaly::FindBranchRequest.new( repository: @gitaly_repo, - name: GitalyClient.encode(branch_name) + name: encode_binary(branch_name) ) response = GitalyClient.call(@repository.storage, :ref_service, :find_branch, request) @@ -96,8 +102,8 @@ module Gitlab def create_branch(ref, start_point) request = Gitaly::CreateBranchRequest.new( repository: @gitaly_repo, - name: GitalyClient.encode(ref), - start_point: GitalyClient.encode(start_point) + name: encode_binary(ref), + start_point: encode_binary(start_point) ) response = GitalyClient.call(@repository.storage, :ref_service, :create_branch, request) @@ -121,7 +127,7 @@ module Gitlab def delete_branch(branch_name) request = Gitaly::DeleteBranchRequest.new( repository: @gitaly_repo, - name: GitalyClient.encode(branch_name) + name: encode_binary(branch_name) ) GitalyClient.call(@repository.storage, :ref_service, :delete_branch, request) @@ -151,7 +157,7 @@ module Gitlab enum_value end - def consume_branches_response(response) + def consume_find_local_branches_response(response) response.flat_map do |message| message.branches.map do |gitaly_branch| Gitlab::Git::Branch.new( @@ -164,6 +170,15 @@ module Gitlab end end + def consume_find_all_branches_response(response) + response.flat_map do |message| + message.branches.map do |branch| + target_commit = Gitlab::Git::Commit.decorate(@repository, branch.target) + Gitlab::Git::Branch.new(@repository, branch.name, branch.target.id, target_commit) + end + end + end + def consume_tags_response(response) response.flat_map do |message| message.tags.map { |gitaly_tag| Util.gitlab_tag_from_gitaly_tag(@repository, gitaly_tag) } diff --git a/lib/gitlab/gitaly_client/remote_service.rb b/lib/gitlab/gitaly_client/remote_service.rb index 9218f6cfd68..e58f641d69a 100644 --- a/lib/gitlab/gitaly_client/remote_service.rb +++ b/lib/gitlab/gitaly_client/remote_service.rb @@ -7,10 +7,12 @@ module Gitlab @storage = repository.storage end - def add_remote(name, url, mirror_refmap) + def add_remote(name, url, mirror_refmaps) request = Gitaly::AddRemoteRequest.new( - repository: @gitaly_repo, name: name, url: url, - mirror_refmap: mirror_refmap.to_s + repository: @gitaly_repo, + name: name, + url: url, + mirror_refmaps: Array.wrap(mirror_refmaps).map(&:to_s) ) GitalyClient.call(@storage, :remote_service, :add_remote, request) @@ -23,6 +25,19 @@ module Gitlab response.result end + + def fetch_internal_remote(repository) + request = Gitaly::FetchInternalRemoteRequest.new( + repository: @gitaly_repo, + remote_repository: repository.gitaly_repository + ) + + response = GitalyClient.call(@storage, :remote_service, + :fetch_internal_remote, request, + remote_storage: repository.storage) + + response.result + end end end end diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index c1f95396878..72ee92e78dc 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -1,6 +1,8 @@ module Gitlab module GitalyClient class RepositoryService + include Gitlab::EncodingHelper + def initialize(repository) @repository = repository @gitaly_repo = repository.gitaly_repository @@ -41,8 +43,11 @@ module Gitlab GitalyClient.call(@storage, :repository_service, :apply_gitattributes, request) end - def fetch_remote(remote, ssh_auth: nil, forced: false, no_tags: false) - request = Gitaly::FetchRemoteRequest.new(repository: @gitaly_repo, remote: remote, force: forced, no_tags: no_tags) + def fetch_remote(remote, ssh_auth:, forced:, no_tags:, timeout:) + request = Gitaly::FetchRemoteRequest.new( + repository: @gitaly_repo, remote: remote, force: forced, + no_tags: no_tags, timeout: timeout + ) if ssh_auth&.ssh_import? if ssh_auth.ssh_key_auth? && ssh_auth.ssh_private_key.present? @@ -72,13 +77,46 @@ module Gitlab def find_merge_base(*revisions) request = Gitaly::FindMergeBaseRequest.new( repository: @gitaly_repo, - revisions: revisions.map { |r| GitalyClient.encode(r) } + revisions: revisions.map { |r| encode_binary(r) } ) response = GitalyClient.call(@storage, :repository_service, :find_merge_base, request) response.base.presence end + def fork_repository(source_repository) + request = Gitaly::CreateForkRequest.new( + repository: @gitaly_repo, + source_repository: source_repository.gitaly_repository + ) + + GitalyClient.call( + @storage, + :repository_service, + :create_fork, + request, + remote_storage: source_repository.storage, + timeout: GitalyClient.default_timeout + ) + end + + def rebase_in_progress?(rebase_id) + request = Gitaly::IsRebaseInProgressRequest.new( + repository: @gitaly_repo, + rebase_id: rebase_id.to_s + ) + + response = GitalyClient.call( + @storage, + :repository_service, + :is_rebase_in_progress, + request, + timeout: GitalyClient.default_timeout + ) + + response.in_progress + end + def fetch_source_branch(source_repository, source_branch, local_ref) request = Gitaly::FetchSourceBranchRequest.new( repository: @gitaly_repo, diff --git a/lib/gitlab/gitaly_client/util.rb b/lib/gitlab/gitaly_client/util.rb index b1a033280b4..a8c6d478de8 100644 --- a/lib/gitlab/gitaly_client/util.rb +++ b/lib/gitlab/gitaly_client/util.rb @@ -12,12 +12,18 @@ module Gitlab Gitaly::Repository.new( storage_name: repository_storage, relative_path: relative_path, - gl_repository: gl_repository, + gl_repository: gl_repository.to_s, git_object_directory: git_object_directory.to_s, git_alternate_object_directories: git_alternate_object_directories ) end + def git_repository(gitaly_repository) + Gitlab::Git::Repository.new(gitaly_repository.storage_name, + gitaly_repository.relative_path, + gitaly_repository.gl_repository) + end + def gitlab_tag_from_gitaly_tag(repository, gitaly_tag) if gitaly_tag.target_commit.present? commit = Gitlab::Git::Commit.decorate(repository, gitaly_tag.target_commit) diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb index 337d225d081..5c5b170a3e0 100644 --- a/lib/gitlab/gitaly_client/wiki_service.rb +++ b/lib/gitlab/gitaly_client/wiki_service.rb @@ -3,6 +3,8 @@ require 'stringio' module Gitlab module GitalyClient class WikiService + include Gitlab::EncodingHelper + MAX_MSG_SIZE = 128.kilobytes.freeze def initialize(repository) @@ -13,12 +15,12 @@ module Gitlab def write_page(name, format, content, commit_details) request = Gitaly::WikiWritePageRequest.new( repository: @gitaly_repo, - name: GitalyClient.encode(name), + name: encode_binary(name), format: format.to_s, commit_details: gitaly_commit_details(commit_details) ) - strio = GitalyClient.binary_stringio(content) + strio = binary_stringio(content) enum = Enumerator.new do |y| until strio.eof? @@ -39,13 +41,13 @@ module Gitlab def update_page(page_path, title, format, content, commit_details) request = Gitaly::WikiUpdatePageRequest.new( repository: @gitaly_repo, - page_path: GitalyClient.encode(page_path), - title: GitalyClient.encode(title), + page_path: encode_binary(page_path), + title: encode_binary(title), format: format.to_s, commit_details: gitaly_commit_details(commit_details) ) - strio = GitalyClient.binary_stringio(content) + strio = binary_stringio(content) enum = Enumerator.new do |y| until strio.eof? @@ -63,7 +65,7 @@ module Gitlab def delete_page(page_path, commit_details) request = Gitaly::WikiDeletePageRequest.new( repository: @gitaly_repo, - page_path: GitalyClient.encode(page_path), + page_path: encode_binary(page_path), commit_details: gitaly_commit_details(commit_details) ) @@ -73,9 +75,9 @@ module Gitlab def find_page(title:, version: nil, dir: nil) request = Gitaly::WikiFindPageRequest.new( repository: @gitaly_repo, - title: GitalyClient.encode(title), - revision: GitalyClient.encode(version), - directory: GitalyClient.encode(dir) + title: encode_binary(title), + revision: encode_binary(version), + directory: encode_binary(dir) ) response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_find_page, request) @@ -102,8 +104,8 @@ module Gitlab def find_file(name, revision) request = Gitaly::WikiFindFileRequest.new( repository: @gitaly_repo, - name: GitalyClient.encode(name), - revision: GitalyClient.encode(revision) + name: encode_binary(name), + revision: encode_binary(revision) ) response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_find_file, request) @@ -158,9 +160,9 @@ module Gitlab def gitaly_commit_details(commit_details) Gitaly::WikiCommitDetails.new( - name: GitalyClient.encode(commit_details.name), - email: GitalyClient.encode(commit_details.email), - message: GitalyClient.encode(commit_details.message) + name: encode_binary(commit_details.name), + email: encode_binary(commit_details.email), + message: encode_binary(commit_details.message) ) end end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index dfcdfc307b6..9148d7571f2 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -21,6 +21,7 @@ module Gitlab gon.revision = Gitlab::REVISION gon.gitlab_logo = ActionController::Base.helpers.asset_path('gitlab_logo.png') gon.sprite_icons = IconsHelper.sprite_icon_path + gon.sprite_file_icons = IconsHelper.sprite_file_icons_path if current_user gon.current_user_id = current_user.id diff --git a/lib/gitlab/google_code_import/importer.rb b/lib/gitlab/google_code_import/importer.rb index ab38c0c3e34..46b49128140 100644 --- a/lib/gitlab/google_code_import/importer.rb +++ b/lib/gitlab/google_code_import/importer.rb @@ -302,6 +302,7 @@ module Gitlab else "#{project.namespace.full_path}/#{name}##{id}" end + text = "~~#{text}~~" if deleted text end @@ -329,6 +330,7 @@ module Gitlab if content.blank? content = "*(No comment has been entered for this change)*" end + body << content if updates.any? @@ -352,6 +354,7 @@ module Gitlab if content.blank? content = "*(No description has been entered for this issue)*" end + body << content if attachments.any? diff --git a/lib/gitlab/grape_logging/loggers/user_logger.rb b/lib/gitlab/grape_logging/loggers/user_logger.rb new file mode 100644 index 00000000000..fa172861967 --- /dev/null +++ b/lib/gitlab/grape_logging/loggers/user_logger.rb @@ -0,0 +1,18 @@ +# This grape_logging module (https://github.com/aserafin/grape_logging) makes it +# possible to log the user who performed the Grape API action by retrieving +# the user context from the request environment. +module Gitlab + module GrapeLogging + module Loggers + class UserLogger < ::GrapeLogging::Loggers::Base + def parameters(request, _) + params = request.env[::API::Helpers::API_USER_ENV] + + return {} unless params + + params.slice(:user_id, :username) + end + end + end + end +end diff --git a/lib/gitlab/hook_data/issue_builder.rb b/lib/gitlab/hook_data/issue_builder.rb index e29dd0d5b0e..f9b1a3caf5e 100644 --- a/lib/gitlab/hook_data/issue_builder.rb +++ b/lib/gitlab/hook_data/issue_builder.rb @@ -7,7 +7,6 @@ module Gitlab closed_at confidential created_at - deleted_at description due_date id diff --git a/lib/gitlab/hook_data/merge_request_builder.rb b/lib/gitlab/hook_data/merge_request_builder.rb index ae9b68eb648..aff786864f2 100644 --- a/lib/gitlab/hook_data/merge_request_builder.rb +++ b/lib/gitlab/hook_data/merge_request_builder.rb @@ -5,7 +5,6 @@ module Gitlab assignee_id author_id created_at - deleted_at description head_pipeline_id id diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index 2066005dddc..af203ff711d 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -3,7 +3,7 @@ module Gitlab extend self # For every version update, the version history in import_export.md has to be kept up to date. - VERSION = '0.2.1'.freeze + VERSION = '0.2.2'.freeze FILENAME_LIMIT = 50 def export_path(relative_path:) diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index 0135b3c6f22..dd5d35feab9 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -15,6 +15,11 @@ module Gitlab execute(%W(#{git_bin_path} --git-dir=#{repo_path} bundle create #{bundle_path} --all)) end + def git_clone_bundle(repo_path:, bundle_path:) + execute(%W(#{git_bin_path} clone --bare -- #{bundle_path} #{repo_path})) + Gitlab::Git::Repository.create_hooks(repo_path, File.expand_path(Gitlab.config.gitlab_shell.hooks_path)) + end + def mkdir_p(path) FileUtils.mkdir_p(path, mode: DEFAULT_MODE) FileUtils.chmod(DEFAULT_MODE, path) diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index f2b193c79cb..2daed10f678 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -49,8 +49,8 @@ project_tree: - :author - events: - :push_event_payload - - :stages - - :statuses + - stages: + - :statuses - :auto_devops - :triggers - :pipeline_schedules diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index c518943be59..4b5f9f3a926 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -148,6 +148,7 @@ module Gitlab else relation_hash = relation_item[sub_relation.to_s] end + [relation_hash, sub_relation] end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index d7d1b05e8b9..cb711a83433 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -62,6 +62,7 @@ module Gitlab when :notes then setup_note when :project_label, :project_labels then setup_label when :milestone, :milestones then setup_milestone + when 'Ci::Pipeline' then setup_pipeline else @relation_hash['project_id'] = @project.id end @@ -112,9 +113,7 @@ module Gitlab @relation_hash.delete('trace') # old export files have trace @relation_hash.delete('token') - imported_object do |object| - object.commit_id = nil - end + imported_object elsif @relation_name == :merge_requests MergeRequestParser.new(@project, @relation_hash.delete('diff_head_sha'), imported_object, @relation_hash).parse! else @@ -182,8 +181,9 @@ module Gitlab end def imported_object - yield(existing_or_new_object) if block_given? - existing_or_new_object.importing = true if existing_or_new_object.respond_to?(:importing) + if existing_or_new_object.respond_to?(:importing) + existing_or_new_object.importing = true + end existing_or_new_object rescue ActiveRecord::RecordNotUnique @@ -211,6 +211,14 @@ module Gitlab @relation_hash['diff'] = @relation_hash.delete('utf8_diff') end + def setup_pipeline + @relation_hash.fetch('stages').each do |stage| + stage.statuses.each do |status| + status.pipeline = imported_object + end + end + end + def existing_or_new_object # Only find existing records to avoid mapping tables such as milestones # Otherwise always create the record, skipping the extra SELECT clause. @@ -259,6 +267,7 @@ module Gitlab else %w[title group_id] end + finder_hash = parsed_relation_hash.slice(*finder_attributes) if label? diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index 32ca2809b2f..d0e5cfcfd3e 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -13,7 +13,7 @@ module Gitlab def restore return true unless File.exist?(@path_to_bundle) - gitlab_shell.import_repository(@project.repository_storage_path, @project.disk_path, @path_to_bundle) + git_clone_bundle(repo_path: @project.repository.path_to_repo, bundle_path: @path_to_bundle) rescue => e @shared.error(e) false diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index eeb03625479..60d5fa4d29a 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -7,6 +7,7 @@ module Gitlab module ImportSources ImportSource = Struct.new(:name, :title, :importer) + # We exclude `bare_repository` here as it has no import class associated ImportTable = [ ImportSource.new('github', 'GitHub', Gitlab::GithubImport::ParallelImporter), ImportSource.new('bitbucket', 'Bitbucket', Gitlab::BitbucketImport::Importer), diff --git a/lib/gitlab/insecure_key_fingerprint.rb b/lib/gitlab/insecure_key_fingerprint.rb new file mode 100644 index 00000000000..f85b6e9197f --- /dev/null +++ b/lib/gitlab/insecure_key_fingerprint.rb @@ -0,0 +1,23 @@ +module Gitlab + # + # Calculates the fingerprint of a given key without using + # openssh key validations. For this reason, only use + # for calculating the fingerprint to find the key with it. + # + # DO NOT use it for checking the validity of a ssh key. + # + class InsecureKeyFingerprint + attr_accessor :key + + # + # Gets the base64 encoded string representing a rsa or dsa key + # + def initialize(key_base64) + @key = key_base64 + end + + def fingerprint + OpenSSL::Digest::MD5.hexdigest(Base64.decode64(@key)).scan(/../).join(':') + end + end +end diff --git a/lib/gitlab/kubernetes/helm/pod.rb b/lib/gitlab/kubernetes/helm/pod.rb index 233f6bf6227..97ad3c97e95 100644 --- a/lib/gitlab/kubernetes/helm/pod.rb +++ b/lib/gitlab/kubernetes/helm/pod.rb @@ -14,6 +14,7 @@ module Gitlab generate_config_map spec['volumes'] = volumes_specification end + ::Kubeclient::Resource.new(metadata: metadata, spec: spec) end diff --git a/lib/gitlab/ldap/adapter.rb b/lib/gitlab/ldap/adapter.rb index 0afaa2306b5..76863e77dc3 100644 --- a/lib/gitlab/ldap/adapter.rb +++ b/lib/gitlab/ldap/adapter.rb @@ -74,7 +74,7 @@ module Gitlab def user_options(fields, value, limit) options = { - attributes: Gitlab::LDAP::Person.ldap_attributes(config).compact.uniq, + attributes: Gitlab::LDAP::Person.ldap_attributes(config), base: config.base } diff --git a/lib/gitlab/ldap/config.rb b/lib/gitlab/ldap/config.rb index c8f19cd52d5..cde60addcf7 100644 --- a/lib/gitlab/ldap/config.rb +++ b/lib/gitlab/ldap/config.rb @@ -42,6 +42,7 @@ module Gitlab else self.class.invalid_provider(provider) end + @options = config_for(@provider) # Use @provider, not provider end @@ -148,7 +149,7 @@ module Gitlab def default_attributes { - 'username' => %w(uid userid sAMAccountName), + 'username' => %w(uid sAMAccountName userid), 'email' => %w(mail email userPrincipalName), 'name' => 'cn', 'first_name' => 'givenName', diff --git a/lib/gitlab/ldap/person.rb b/lib/gitlab/ldap/person.rb index 38d7a9ba2f5..e81cec6ba1a 100644 --- a/lib/gitlab/ldap/person.rb +++ b/lib/gitlab/ldap/person.rb @@ -6,6 +6,8 @@ module Gitlab # Source: http://ctogonewild.com/2009/09/03/bitmask-searches-in-ldap/ AD_USER_DISABLED = Net::LDAP::Filter.ex("userAccountControl:1.2.840.113556.1.4.803", "2") + InvalidEntryError = Class.new(StandardError) + attr_accessor :entry, :provider def self.find_by_uid(uid, adapter) @@ -29,11 +31,12 @@ module Gitlab def self.ldap_attributes(config) [ - 'dn', # Used in `dn` - config.uid, # Used in `uid` - *config.attributes['name'], # Used in `name` - *config.attributes['email'] # Used in `email` - ] + 'dn', + config.uid, + *config.attributes['name'], + *config.attributes['email'], + *config.attributes['username'] + ].compact.uniq end def self.normalize_dn(dn) @@ -60,6 +63,8 @@ module Gitlab Rails.logger.debug { "Instantiating #{self.class.name} with LDIF:\n#{entry.to_ldif}" } @entry = entry @provider = provider + + validate_entry end def name @@ -71,7 +76,13 @@ module Gitlab end def username - uid + username = attribute_value(:username) + + # Depending on the attribute, multiple values may + # be returned. We need only one for username. + # Ex. `uid` returns only one value but `mail` may + # return an array of multiple email addresses. + [username].flatten.first end def email @@ -104,6 +115,19 @@ module Gitlab entry.public_send(selected_attr) # rubocop:disable GitlabSecurity/PublicSend end + + def validate_entry + allowed_attrs = self.class.ldap_attributes(config).map(&:downcase) + + # Net::LDAP::Entry transforms keys to symbols. Change to strings to compare. + entry_attrs = entry.attribute_names.map { |n| n.to_s.downcase } + invalid_attrs = entry_attrs - allowed_attrs + + if invalid_attrs.any? + raise InvalidEntryError, + "#{self.class.name} initialized with Net::LDAP::Entry containing invalid attributes(s): #{invalid_attrs}" + end + end end end end diff --git a/lib/gitlab/metrics/influx_db.rb b/lib/gitlab/metrics/influx_db.rb index 877cebf6786..ef44a13df51 100644 --- a/lib/gitlab/metrics/influx_db.rb +++ b/lib/gitlab/metrics/influx_db.rb @@ -169,6 +169,7 @@ module Gitlab end end end + @pool end end diff --git a/lib/gitlab/metrics/method_call.rb b/lib/gitlab/metrics/method_call.rb index 329b07af5db..c2f9db56824 100644 --- a/lib/gitlab/metrics/method_call.rb +++ b/lib/gitlab/metrics/method_call.rb @@ -5,7 +5,7 @@ module Gitlab # Class for tracking timing information about method calls class MethodCall @@measurement_enabled_cache = Concurrent::AtomicBoolean.new(false) - @@measurement_enabled_cache_expires_at = Concurrent::AtomicFixnum.new(Time.now.to_i) + @@measurement_enabled_cache_expires_at = Concurrent::AtomicReference.new(Time.now.to_i) MUTEX = Mutex.new BASE_LABELS = { module: nil, method: nil }.freeze attr_reader :real_time, :cpu_time, :call_count, :labels diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb index fee741b47be..cc1e92480be 100644 --- a/lib/gitlab/middleware/multipart.rb +++ b/lib/gitlab/middleware/multipart.rb @@ -47,6 +47,7 @@ module Gitlab else value = decorate_params_value(value, @request.params[key], tmp_path) end + @request.update_param(key, value) end @@ -60,6 +61,7 @@ module Gitlab unless path_hash.is_a?(Hash) && path_hash.count == 1 raise "invalid path: #{path_hash.inspect}" end + path_key, path_value = path_hash.first unless value_hash.is_a?(Hash) && value_hash[path_key] diff --git a/lib/gitlab/multi_collection_paginator.rb b/lib/gitlab/multi_collection_paginator.rb index c22d0a84860..43921a8c1c0 100644 --- a/lib/gitlab/multi_collection_paginator.rb +++ b/lib/gitlab/multi_collection_paginator.rb @@ -37,6 +37,7 @@ module Gitlab else per_page - first_collection_last_page_size end + hash[page] = second_collection.page(second_collection_page) .per(per_page - paginated_first_collection(page).size) .padding(offset) diff --git a/lib/gitlab/quick_actions/extractor.rb b/lib/gitlab/quick_actions/extractor.rb index 3ebfa3bd4b8..c0878a34fb1 100644 --- a/lib/gitlab/quick_actions/extractor.rb +++ b/lib/gitlab/quick_actions/extractor.rb @@ -126,6 +126,7 @@ module Gitlab command << match_data[1] unless match_data[1].empty? commands << command end + content = substitution.perform_substitution(self, content) end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index 8ad06480575..4178b436acf 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -24,6 +24,7 @@ module Gitlab # the pool will be used in a multi-threaded context size += Sidekiq.options[:concurrency] end + size end @@ -104,6 +105,7 @@ module Gitlab db_numbers = queries["db"] if queries.key?("db") config[:db] = db_numbers[0].to_i if db_numbers.any? end + config else redis_hash = ::Redis::Store::Factory.extract_host_options_from_uri(redis_url) diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 2c7b8af83f2..0002c7da8f1 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -37,7 +37,7 @@ module Gitlab end def environment_name_regex_chars - 'a-zA-Z0-9_/\\$\\{\\}\\. -' + 'a-zA-Z0-9_/\\$\\{\\}\\. \\-' end def environment_name_regex diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 7037e2e61cc..70b639501fd 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -82,7 +82,10 @@ module Gitlab end def issues - issues = IssuesFinder.new(current_user).execute.where(project_id: project_ids_relation) + issues = IssuesFinder.new(current_user).execute + unless default_project_filter + issues = issues.where(project_id: project_ids_relation) + end issues = if query =~ /#(\d+)\z/ @@ -112,6 +115,7 @@ module Gitlab else merge_requests.full_search(query) end + merge_requests.order('updated_at DESC') end diff --git a/lib/gitlab/seeder.rb b/lib/gitlab/seeder.rb index 30df7e4a831..94a481a0f2e 100644 --- a/lib/gitlab/seeder.rb +++ b/lib/gitlab/seeder.rb @@ -8,7 +8,7 @@ end module Gitlab class Seeder def self.quiet - mute_mailer unless Rails.env.test? + mute_mailer SeedFu.quiet = true diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb new file mode 100644 index 00000000000..d01213bb6e0 --- /dev/null +++ b/lib/gitlab/setup_helper.rb @@ -0,0 +1,61 @@ +module Gitlab + module SetupHelper + class << self + # We cannot create config.toml files for all possible Gitaly configuations. + # For instance, if Gitaly is running on another machine then it makes no + # sense to write a config.toml file on the current machine. This method will + # only generate a configuration for the most common and simplest case: when + # we have exactly one Gitaly process and we are sure it is running locally + # because it uses a Unix socket. + # For development and testing purposes, an extra storage is added to gitaly, + # which is not known to Rails, but must be explicitly stubbed. + def gitaly_configuration_toml(gitaly_dir, gitaly_ruby: true) + storages = [] + address = nil + + Gitlab.config.repositories.storages.each do |key, val| + if address + if address != val['gitaly_address'] + raise ArgumentError, "Your gitlab.yml contains more than one gitaly_address." + end + elsif URI(val['gitaly_address']).scheme != 'unix' + raise ArgumentError, "Automatic config.toml generation only supports 'unix:' addresses." + else + address = val['gitaly_address'] + end + + storages << { name: key, path: val['path'] } + end + + if Rails.env.test? + storages << { name: 'test_second_storage', path: Rails.root.join('tmp', 'tests', 'second_storage').to_s } + end + + config = { socket_path: address.sub(%r{\Aunix:}, ''), storage: storages } + config[:auth] = { token: 'secret' } if Rails.env.test? + config[:'gitaly-ruby'] = { dir: File.join(gitaly_dir, 'ruby') } if gitaly_ruby + config[:'gitlab-shell'] = { dir: Gitlab.config.gitlab_shell.path } + config[:bin_dir] = Gitlab.config.gitaly.client_path + + TOML.dump(config) + end + + # rubocop:disable Rails/Output + def create_gitaly_configuration(dir, force: false) + config_path = File.join(dir, 'config.toml') + FileUtils.rm_f(config_path) if force + + File.open(config_path, File::WRONLY | File::CREAT | File::EXCL) do |f| + f.puts gitaly_configuration_toml(dir) + end + rescue Errno::EEXIST + puts "Skipping config.toml generation:" + puts "A configuration file already exists." + rescue ArgumentError => e + puts "Skipping config.toml generation:" + puts e.message + end + # rubocop:enable Rails/Output + end + end +end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 9cdd3d22f18..f4a41dc3eda 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -71,7 +71,6 @@ module Gitlab # Ex. # add_repository("/path/to/storage", "gitlab/gitlab-ci") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def add_repository(storage, name) relative_path = name.dup relative_path << '.git' unless relative_path.end_with?('.git') @@ -100,8 +99,12 @@ module Gitlab # Ex. # import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://gitlab.com/gitlab-org/gitlab-test.git") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/874 def import_repository(storage, name, url) + if url.start_with?('.', '/') + raise Error.new("don't use disk paths with import_repository: #{url.inspect}") + end + # The timeout ensures the subprocess won't hang forever cmd = gitlab_projects(storage, "#{name}.git") success = cmd.import_project(url, git_timeout) @@ -122,11 +125,10 @@ module Gitlab # Ex. # fetch_remote(my_repo, "upstream") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def fetch_remote(repository, remote, ssh_auth: nil, forced: false, no_tags: false) gitaly_migrate(:fetch_remote) do |is_enabled| if is_enabled - repository.gitaly_repository_client.fetch_remote(remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags) + repository.gitaly_repository_client.fetch_remote(remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, timeout: git_timeout) else storage_path = Gitlab.config.repositories.storages[repository.storage]["path"] local_fetch_remote(storage_path, repository.relative_path, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags) @@ -134,7 +136,10 @@ module Gitlab end end - # Move repository + # Move repository reroutes to mv_directory which is an alias for + # mv_namespace. Given the underlying implementation is a move action, + # indescriminate of what the folders might be. + # # storage - project's storage path # path - project disk path # new_path - new project disk path @@ -142,9 +147,11 @@ module Gitlab # Ex. # mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/873 def mv_repository(storage, path, new_path) - gitlab_projects(storage, "#{path}.git").mv_project("#{new_path}.git") + return false if path.empty? || new_path.empty? + + !!mv_directory(storage, "#{path}.git", "#{new_path}.git") end # Fork repository to new path @@ -156,13 +163,15 @@ module Gitlab # Ex. # fork_repository("/path/to/forked_from/storage", "gitlab/gitlab-ci", "/path/to/forked_to/storage", "new-namespace/gitlab-ci") # - # Gitaly note: JV: not easy to migrate because this involves two Gitaly servers, not one. + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/817 def fork_repository(forked_from_storage, forked_from_disk_path, forked_to_storage, forked_to_disk_path) gitlab_projects(forked_from_storage, "#{forked_from_disk_path}.git") .fork_repository(forked_to_storage, "#{forked_to_disk_path}.git") end - # Remove repository from file system + # Removes a repository from file system, using rm_diretory which is an alias + # for rm_namespace. Given the underlying implementation removes the name + # passed as second argument on the passed storage. # # storage - project's storage path # name - project disk path @@ -170,9 +179,14 @@ module Gitlab # Ex. # remove_repository("/path/to/storage", "gitlab/gitlab-ci") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/873 def remove_repository(storage, name) - gitlab_projects(storage, "#{name}.git").rm_project + return false if name.empty? + + !!rm_directory(storage, "#{name}.git") + rescue ArgumentError => e + Rails.logger.warn("Repository does not exist: #{e} at: #{name}.git") + false end # Add new key to gitlab-shell @@ -181,6 +195,8 @@ module Gitlab # add_key("key-42", "sha-rsa ...") # def add_key(key_id, key_content) + return unless self.authorized_keys_enabled? + gitlab_shell_fast_execute([gitlab_shell_keys_path, 'add-key', key_id, self.class.strip_key(key_content)]) end @@ -190,6 +206,8 @@ module Gitlab # Ex. # batch_add_keys { |adder| adder.add_key("key-42", "sha-rsa ...") } def batch_add_keys(&block) + return unless self.authorized_keys_enabled? + IO.popen(%W(#{gitlab_shell_path}/bin/gitlab-keys batch-add-keys), 'w') do |io| yield(KeyAdder.new(io)) end @@ -200,10 +218,11 @@ module Gitlab # Ex. # remove_key("key-342", "sha-rsa ...") # - def remove_key(key_id, key_content) + def remove_key(key_id, key_content = nil) + return unless self.authorized_keys_enabled? + args = [gitlab_shell_keys_path, 'rm-key', key_id] args << key_content if key_content - gitlab_shell_fast_execute(args) end @@ -213,15 +232,67 @@ module Gitlab # remove_all_keys # def remove_all_keys + return unless self.authorized_keys_enabled? + gitlab_shell_fast_execute([gitlab_shell_keys_path, 'clear']) end + # Remove ssh keys from gitlab shell that are not in the DB + # + # Ex. + # remove_keys_not_found_in_db + # + def remove_keys_not_found_in_db + return unless self.authorized_keys_enabled? + + Rails.logger.info("Removing keys not found in DB") + + batch_read_key_ids do |ids_in_file| + ids_in_file.uniq! + keys_in_db = Key.where(id: ids_in_file) + + next unless ids_in_file.size > keys_in_db.count # optimization + + ids_to_remove = ids_in_file - keys_in_db.pluck(:id) + ids_to_remove.each do |id| + Rails.logger.info("Removing key-#{id} not found in DB") + remove_key("key-#{id}") + end + end + end + + # Iterate over all ssh key IDs from gitlab shell, in batches + # + # Ex. + # batch_read_key_ids { |batch| keys = Key.where(id: batch) } + # + def batch_read_key_ids(batch_size: 100, &block) + return unless self.authorized_keys_enabled? + + list_key_ids do |key_id_stream| + key_id_stream.lazy.each_slice(batch_size) do |lines| + key_ids = lines.map { |l| l.chomp.to_i } + yield(key_ids) + end + end + end + + # Stream all ssh key IDs from gitlab shell, separated by newlines + # + # Ex. + # list_key_ids + # + def list_key_ids(&block) + return unless self.authorized_keys_enabled? + + IO.popen(%W(#{gitlab_shell_path}/bin/gitlab-keys list-key-ids), &block) + end + # Add empty directory for storing repositories # # Ex. # add_namespace("/path/to/storage", "gitlab") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def add_namespace(storage, name) Gitlab::GitalyClient.migrate(:add_namespace) do |enabled| if enabled @@ -243,7 +314,6 @@ module Gitlab # Ex. # rm_namespace("/path/to/storage", "gitlab") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def rm_namespace(storage, name) Gitlab::GitalyClient.migrate(:remove_namespace) do |enabled| if enabled @@ -255,13 +325,13 @@ module Gitlab rescue GRPC::InvalidArgument => e raise ArgumentError, e.message end + alias_method :rm_directory, :rm_namespace # Move namespace directory inside repositories storage # # Ex. # mv_namespace("/path/to/storage", "gitlab", "gitlabhq") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def mv_namespace(storage, old_name, new_name) Gitlab::GitalyClient.migrate(:rename_namespace) do |enabled| if enabled @@ -275,6 +345,7 @@ module Gitlab rescue GRPC::InvalidArgument false end + alias_method :mv_directory, :mv_namespace def url_to_repo(path) Gitlab.config.gitlab_shell.ssh_path_prefix + "#{path}.git" @@ -306,47 +377,6 @@ module Gitlab end end - # Push branch to remote repository - # - # storage - project's storage path - # project_name - project's disk path - # remote_name - remote name - # branch_names - remote branch names to push - # forced - should we use --force flag - # - # Ex. - # push_remote_branches('/path/to/storage', 'gitlab-org/gitlab-test' 'upstream', ['feature']) - # - def push_remote_branches(storage, project_name, remote_name, branch_names, forced: true) - cmd = gitlab_projects(storage, "#{project_name}.git") - - success = cmd.push_branches(remote_name, git_timeout, forced, branch_names) - - raise Error, cmd.output unless success - - success - end - - # Delete branch from remote repository - # - # storage - project's storage path - # project_name - project's disk path - # remote_name - remote name - # branch_names - remote branch names - # - # Ex. - # delete_remote_branches('/path/to/storage', 'gitlab-org/gitlab-test', 'upstream', ['feature']) - # - def delete_remote_branches(storage, project_name, remote_name, branch_names) - cmd = gitlab_projects(storage, "#{project_name}.git") - - success = cmd.delete_remote_branches(remote_name, branch_names) - - raise Error, cmd.output unless success - - success - end - protected def gitlab_shell_path @@ -375,6 +405,14 @@ module Gitlab File.join(gitlab_shell_path, 'bin', 'gitlab-keys') end + def authorized_keys_enabled? + # Return true if nil to ensure the authorized_keys methods work while + # fixing the authorized_keys file during migration. + return true if Gitlab::CurrentSettings.current_application_settings.authorized_keys_enabled.nil? + + Gitlab::CurrentSettings.current_application_settings.authorized_keys_enabled + end + private def gitlab_projects(shard_path, disk_path) diff --git a/lib/gitlab/storage_check/cli.rb b/lib/gitlab/storage_check/cli.rb index 04bf1bf1d26..9b64c8e033a 100644 --- a/lib/gitlab/storage_check/cli.rb +++ b/lib/gitlab/storage_check/cli.rb @@ -59,9 +59,11 @@ module Gitlab if response.skipped_shards.any? warnings << "Skipped shards: #{response.skipped_shards.join(', ')}" end + if response.failing_shards.any? warnings << "Failing shards: #{response.failing_shards.join(', ')}" end + logger.warn(warnings.join(' - ')) if warnings.any? end end diff --git a/lib/gitlab/testing/request_blocker_middleware.rb b/lib/gitlab/testing/request_blocker_middleware.rb index 4a8e3c2eee0..53333b9b06b 100644 --- a/lib/gitlab/testing/request_blocker_middleware.rb +++ b/lib/gitlab/testing/request_blocker_middleware.rb @@ -37,12 +37,14 @@ module Gitlab def call(env) increment_active_requests + if block_requests? block_request(env) else sleep 0.2 if slow_requests? @app.call(env) end + ensure decrement_active_requests end diff --git a/lib/gitlab/timeless.rb b/lib/gitlab/timeless.rb index b290c716f97..76a1808c8ac 100644 --- a/lib/gitlab/timeless.rb +++ b/lib/gitlab/timeless.rb @@ -9,6 +9,7 @@ module Gitlab else block.call end + ensure model.record_timestamps = original_record_timestamps end diff --git a/lib/gitlab/upgrader.rb b/lib/gitlab/upgrader.rb index 961df0468a4..3b64cb32afa 100644 --- a/lib/gitlab/upgrader.rb +++ b/lib/gitlab/upgrader.rb @@ -12,6 +12,7 @@ module Gitlab puts "You are using the latest GitLab version" else puts "Newer GitLab version is available" + answer = if ARGV.first == "-y" "yes" else @@ -77,6 +78,7 @@ module Gitlab update_commands.each do |title, cmd| puts title puts " -> #{cmd.join(' ')}" + if system(env, *cmd) puts " -> OK" else diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index 11472ce6cce..6ced06a863d 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -57,11 +57,17 @@ module Gitlab } end - def highest_allowed_level + def allowed_levels restricted_levels = current_application_settings.restricted_visibility_levels - allowed_levels = self.values - restricted_levels - allowed_levels.max || PRIVATE + self.values - restricted_levels + end + + def closest_allowed_level(target_level) + highest_allowed_level = allowed_levels.select { |level| level <= target_level }.max + + # If all levels are restricted, fall back to PRIVATE + highest_allowed_level || PRIVATE end def allowed_for?(user, level) diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 5ab6cd5a4ef..0de183858aa 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -42,6 +42,7 @@ module Gitlab else raise "Unsupported action: #{action}" end + if feature_enabled params[:GitalyServer] = server end @@ -97,6 +98,9 @@ module Gitlab ) end + # If present DisableCache must be a Boolean. Otherwise workhorse ignores it. + params['DisableCache'] = true if git_archive_cache_disabled? + [ SEND_DATA_HEADER, "git-archive:#{encode(params)}" @@ -244,6 +248,10 @@ module Gitlab right_commit_id: diff_refs.head_sha } end + + def git_archive_cache_disabled? + ENV['WORKHORSE_ARCHIVE_CACHE_DISABLED'].present? || Feature.enabled?(:workhorse_archive_cache_disabled) + end end end end diff --git a/lib/google_api/cloud_platform/client.rb b/lib/google_api/cloud_platform/client.rb index b0563fb2d69..ff638c07755 100644 --- a/lib/google_api/cloud_platform/client.rb +++ b/lib/google_api/cloud_platform/client.rb @@ -1,4 +1,6 @@ require 'google/apis/container_v1' +require 'google/apis/cloudbilling_v1' +require 'google/apis/cloudresourcemanager_v1' module GoogleApi module CloudPlatform @@ -40,6 +42,22 @@ module GoogleApi true end + def projects_list + service = Google::Apis::CloudresourcemanagerV1::CloudResourceManagerService.new + service.authorization = access_token + + service.fetch_all(items: :projects) do |token| + service.list_projects(page_token: token, options: user_agent_header) + end + end + + def projects_get_billing_info(project_id) + service = Google::Apis::CloudbillingV1::CloudbillingService.new + service.authorization = access_token + + service.get_project_billing_info("projects/#{project_id}", options: user_agent_header) + end + def projects_zones_clusters_get(project_id, zone, cluster_id) service = Google::Apis::ContainerV1::ContainerService.new service.authorization = access_token diff --git a/lib/system_check/simple_executor.rb b/lib/system_check/simple_executor.rb index 8b145fb4511..d268f501b4a 100644 --- a/lib/system_check/simple_executor.rb +++ b/lib/system_check/simple_executor.rb @@ -66,6 +66,7 @@ module SystemCheck if check.can_repair? $stdout.print 'Trying to fix error automatically. ...' + if check.repair! $stdout.puts 'Success'.color(:green) return diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index 9dcf44fdc3e..2383bcf954b 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -46,6 +46,7 @@ namespace :gitlab do puts 'Removing all tables. Press `Ctrl-C` within 5 seconds to abort'.color(:yellow) sleep(5) end + # Drop all tables Load the schema to ensure we don't have any newer tables # hanging out from a failed upgrade $progress.puts 'Cleaning the database ... '.color(:blue) @@ -222,6 +223,7 @@ namespace :gitlab do task restore: :environment do $progress.puts "Restoring container registry images ... ".color(:blue) + if Gitlab.config.registry.enabled Backup::Registry.new.restore $progress.puts "done".color(:green) diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index dfade1f3885..31cd6bfe6e1 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -180,6 +180,7 @@ namespace :gitlab do puts "can't check, you have no projects".color(:magenta) return end + puts "" Project.find_each(batch_size: 100) do |project| @@ -210,6 +211,7 @@ namespace :gitlab do gitlab_shell_repo_base = gitlab_shell_path check_cmd = File.expand_path('bin/check', gitlab_shell_repo_base) puts "Running #{check_cmd}" + if system(check_cmd, chdir: gitlab_shell_repo_base) puts 'gitlab-shell self-check successful'.color(:green) else @@ -285,6 +287,7 @@ namespace :gitlab do return if process_count.zero? print 'Number of Sidekiq processes ... ' + if process_count == 1 puts '1'.color(:green) else @@ -387,14 +390,8 @@ namespace :gitlab do namespace :repo do desc "GitLab | Check the integrity of the repositories managed by GitLab" task check: :environment do - Gitlab.config.repositories.storages.each do |name, repository_storage| - namespace_dirs = Dir.glob(File.join(repository_storage['path'], '*')) - - namespace_dirs.each do |namespace_dir| - repo_dirs = Dir.glob(File.join(namespace_dir, '*')) - repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) } - end - end + puts "This task is deprecated. Please use gitlab:git:fsck instead".color(:red) + Rake::Task["gitlab:git:fsck"].execute end end @@ -461,35 +458,4 @@ namespace :gitlab do puts "FAIL. Please update gitlab-shell to #{required_version} from #{current_version}".color(:red) end end - - def check_repo_integrity(repo_dir) - puts "\nChecking repo at #{repo_dir.color(:yellow)}" - - git_fsck(repo_dir) - check_config_lock(repo_dir) - check_ref_locks(repo_dir) - end - - def git_fsck(repo_dir) - puts "Running `git fsck`".color(:yellow) - system(*%W(#{Gitlab.config.git.bin_path} fsck), chdir: repo_dir) - end - - def check_config_lock(repo_dir) - config_exists = File.exist?(File.join(repo_dir, 'config.lock')) - config_output = config_exists ? 'yes'.color(:red) : 'no'.color(:green) - puts "'config.lock' file exists?".color(:yellow) + " ... #{config_output}" - end - - def check_ref_locks(repo_dir) - lock_files = Dir.glob(File.join(repo_dir, 'refs/heads/*.lock')) - if lock_files.present? - puts "Ref lock files exist:".color(:red) - lock_files.each do |lock_file| - puts " #{lock_file}" - end - else - puts "No ref lock files exist".color(:green) - end - end end diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index eb0f757aea7..04d56509ac6 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -84,6 +84,7 @@ namespace :gitlab do next unless user.ldap_user? print "#{user.name} (#{user.ldap_identity.extern_uid}) ..." + if Gitlab::LDAP::Access.allowed?(user) puts " [OK]".color(:green) else diff --git a/lib/tasks/gitlab/dev.rake b/lib/tasks/gitlab/dev.rake index ba221e44e5d..77c28615856 100644 --- a/lib/tasks/gitlab/dev.rake +++ b/lib/tasks/gitlab/dev.rake @@ -14,6 +14,7 @@ namespace :gitlab do puts "Must specify a branch as an argument".color(:red) exit 1 end + args end diff --git a/lib/tasks/gitlab/git.rake b/lib/tasks/gitlab/git.rake index cf82134d97e..3f5dd2ae3b3 100644 --- a/lib/tasks/gitlab/git.rake +++ b/lib/tasks/gitlab/git.rake @@ -30,6 +30,20 @@ namespace :gitlab do end end + desc 'GitLab | Git | Check all repos integrity' + task fsck: :environment do + failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} fsck --name-objects --no-progress), "Checking integrity") do |repo| + check_config_lock(repo) + check_ref_locks(repo) + end + + if failures.empty? + puts "Done".color(:green) + else + output_failures(failures) + end + end + def perform_git_cmd(cmd, message) puts "Starting #{message} on all repositories" @@ -40,6 +54,8 @@ namespace :gitlab do else failures << repo end + + yield(repo) if block_given? end failures @@ -49,5 +65,24 @@ namespace :gitlab do puts "The following repositories reported errors:".color(:red) failures.each { |f| puts "- #{f}" } end + + def check_config_lock(repo_dir) + config_exists = File.exist?(File.join(repo_dir, 'config.lock')) + config_output = config_exists ? 'yes'.color(:red) : 'no'.color(:green) + + puts "'config.lock' file exists?".color(:yellow) + " ... #{config_output}" + end + + def check_ref_locks(repo_dir) + lock_files = Dir.glob(File.join(repo_dir, 'refs/heads/*.lock')) + + if lock_files.present? + puts "Ref lock files exist:".color(:red) + + lock_files.each { |lock_file| puts " #{lock_file}" } + else + puts "No ref lock files exist".color(:green) + end + end end end diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index 4d880c05f99..a2e68c0471b 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -5,9 +5,11 @@ namespace :gitlab do require 'toml' warn_user_is_not_gitlab + unless args.dir.present? abort %(Please specify the directory where you want to install gitaly:\n rake "gitlab:gitaly:install[/home/git/gitaly]") end + args.with_defaults(repo: 'https://gitlab.com/gitlab-org/gitaly.git') version = Gitlab::GitalyClient.expected_server_version @@ -21,8 +23,8 @@ namespace :gitlab do command << 'BUNDLE_FLAGS=--no-deployment' if Rails.env.test? + Gitlab::SetupHelper.create_gitaly_configuration(args.dir) Dir.chdir(args.dir) do - create_gitaly_configuration # In CI we run scripts/gitaly-test-build instead of this command unless ENV['CI'].present? Bundler.with_original_env { run_command!(command) } @@ -39,60 +41,7 @@ namespace :gitlab do # Exclude gitaly-ruby configuration because that depends on the gitaly # installation directory. - puts gitaly_configuration_toml(gitaly_ruby: false) - end - - private - - # We cannot create config.toml files for all possible Gitaly configuations. - # For instance, if Gitaly is running on another machine then it makes no - # sense to write a config.toml file on the current machine. This method will - # only generate a configuration for the most common and simplest case: when - # we have exactly one Gitaly process and we are sure it is running locally - # because it uses a Unix socket. - # For development and testing purposes, an extra storage is added to gitaly, - # which is not known to Rails, but must be explicitly stubbed. - def gitaly_configuration_toml(gitaly_ruby: true) - storages = [] - address = nil - - Gitlab.config.repositories.storages.each do |key, val| - if address - if address != val['gitaly_address'] - raise ArgumentError, "Your gitlab.yml contains more than one gitaly_address." - end - elsif URI(val['gitaly_address']).scheme != 'unix' - raise ArgumentError, "Automatic config.toml generation only supports 'unix:' addresses." - else - address = val['gitaly_address'] - end - - storages << { name: key, path: val['path'] } - end - - if Rails.env.test? - storages << { name: 'test_second_storage', path: Rails.root.join('tmp', 'tests', 'second_storage').to_s } - end - - config = { socket_path: address.sub(%r{\Aunix:}, ''), storage: storages } - config[:auth] = { token: 'secret' } if Rails.env.test? - config[:'gitaly-ruby'] = { dir: File.join(Dir.pwd, 'ruby') } if gitaly_ruby - config[:'gitlab-shell'] = { dir: Gitlab.config.gitlab_shell.path } - config[:bin_dir] = Gitlab.config.gitaly.client_path - - TOML.dump(config) - end - - def create_gitaly_configuration - File.open("config.toml", File::WRONLY | File::CREAT | File::EXCL) do |f| - f.puts gitaly_configuration_toml - end - rescue Errno::EEXIST - puts "Skipping config.toml generation:" - puts "A configuration file already exists." - rescue ArgumentError => e - puts "Skipping config.toml generation:" - puts e.message + puts Gitlab::SetupHelper.gitaly_configuration_toml('', gitaly_ruby: false) end end end diff --git a/lib/tasks/gitlab/list_repos.rake b/lib/tasks/gitlab/list_repos.rake index b732db9db6e..d7f28691098 100644 --- a/lib/tasks/gitlab/list_repos.rake +++ b/lib/tasks/gitlab/list_repos.rake @@ -8,6 +8,7 @@ namespace :gitlab do namespace_ids = Namespace.where(['updated_at > ?', date]).pluck(:id).sort scope = scope.where('id IN (?) OR namespace_id in (?)', project_ids, namespace_ids) end + scope.find_each do |project| base = File.join(project.repository_storage_path, project.disk_path) puts base + '.git' diff --git a/lib/tasks/gitlab/task_helpers.rb b/lib/tasks/gitlab/task_helpers.rb index 6723662703c..c1182af1014 100644 --- a/lib/tasks/gitlab/task_helpers.rb +++ b/lib/tasks/gitlab/task_helpers.rb @@ -130,7 +130,7 @@ module Gitlab def all_repos Gitlab.config.repositories.storages.each_value do |repository_storage| - IO.popen(%W(find #{repository_storage['path']} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find| + IO.popen(%W(find #{repository_storage['path']} -mindepth 2 -type d -name *.git)) do |find| find.each_line do |path| yield path.chomp end diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake index f44abc2b81b..a25f7ce59c7 100644 --- a/lib/tasks/gitlab/update_templates.rake +++ b/lib/tasks/gitlab/update_templates.rake @@ -10,6 +10,7 @@ namespace :gitlab do puts "This rake task is not meant fo production instances".red exit(1) end + admin = User.find_by(admin: true) unless admin diff --git a/lib/tasks/gitlab/uploads.rake b/lib/tasks/gitlab/uploads.rake new file mode 100644 index 00000000000..df31567ce64 --- /dev/null +++ b/lib/tasks/gitlab/uploads.rake @@ -0,0 +1,44 @@ +namespace :gitlab do + namespace :uploads do + desc 'GitLab | Uploads | Check integrity of uploaded files' + task check: :environment do + puts 'Checking integrity of uploaded files' + + uploads_batches do |batch| + batch.each do |upload| + puts "- Checking file (#{upload.id}): #{upload.absolute_path}".color(:green) + + if upload.exist? + check_checksum(upload) + else + puts " * File does not exist on the file system".color(:red) + end + end + end + + puts 'Done!' + end + + def batch_size + ENV.fetch('BATCH', 200).to_i + end + + def calculate_checksum(absolute_path) + Digest::SHA256.file(absolute_path).hexdigest + end + + def check_checksum(upload) + checksum = calculate_checksum(upload.absolute_path) + + if checksum != upload.checksum + puts " * File checksum (#{checksum}) does not match the one in the database (#{upload.checksum})".color(:red) + end + end + + def uploads_batches(&block) + Upload.all.in_batches(of: batch_size, start: ENV['ID_FROM'], finish: ENV['ID_TO']) do |relation| # rubocop: disable Cop/InBatches + yield relation + end + end + end +end diff --git a/lib/tasks/gitlab/workhorse.rake b/lib/tasks/gitlab/workhorse.rake index e7ac0b5859f..308ffb0e284 100644 --- a/lib/tasks/gitlab/workhorse.rake +++ b/lib/tasks/gitlab/workhorse.rake @@ -3,9 +3,11 @@ namespace :gitlab do desc "GitLab | Install or upgrade gitlab-workhorse" task :install, [:dir, :repo] => :environment do |t, args| warn_user_is_not_gitlab + unless args.dir.present? abort %(Please specify the directory where you want to install gitlab-workhorse:\n rake "gitlab:workhorse:install[/home/git/gitlab-workhorse]") end + args.with_defaults(repo: 'https://gitlab.com/gitlab-org/gitlab-workhorse.git') version = Gitlab::Workhorse.version diff --git a/lib/tasks/migrate/migrate_iids.rake b/lib/tasks/migrate/migrate_iids.rake index fc2cea8c016..aa2d01730d7 100644 --- a/lib/tasks/migrate/migrate_iids.rake +++ b/lib/tasks/migrate/migrate_iids.rake @@ -4,6 +4,7 @@ task migrate_iids: :environment do Issue.where(iid: nil).find_each(batch_size: 100) do |issue| begin issue.set_iid + if issue.update_attribute(:iid, issue.iid) print '.' else @@ -19,6 +20,7 @@ task migrate_iids: :environment do MergeRequest.where(iid: nil).find_each(batch_size: 100) do |mr| begin mr.set_iid + if mr.update_attribute(:iid, mr.iid) print '.' else @@ -34,6 +36,7 @@ task migrate_iids: :environment do Milestone.where(iid: nil).find_each(batch_size: 100) do |m| begin m.set_iid + if m.update_attribute(:iid, m.iid) print '.' else |