diff options
Diffstat (limited to 'lib')
80 files changed, 1849 insertions, 708 deletions
diff --git a/lib/after_commit_queue.rb b/lib/after_commit_queue.rb index db63c5038ae..a4d8507960e 100644 --- a/lib/after_commit_queue.rb +++ b/lib/after_commit_queue.rb @@ -14,7 +14,15 @@ module AfterCommitQueue def run_after_commit_or_now(&block) if AfterCommitQueue.inside_transaction? - run_after_commit(&block) + if ActiveRecord::Base.connection.current_transaction.records.include?(self) + run_after_commit(&block) + else + # If the current transaction does not include this record, we can run + # the block now, even if it queues a Sidekiq job. + Sidekiq::Worker.skipping_transaction_check do + instance_eval(&block) + end + end else instance_eval(&block) end diff --git a/lib/api/api.rb b/lib/api/api.rb index 8094597d238..e0d14281c96 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -119,6 +119,7 @@ module API mount ::API::Features mount ::API::Files mount ::API::Groups + mount ::API::GroupMilestones mount ::API::Internal mount ::API::Issues mount ::API::Jobs @@ -129,8 +130,6 @@ module API mount ::API::Members mount ::API::MergeRequestDiffs mount ::API::MergeRequests - mount ::API::ProjectMilestones - mount ::API::GroupMilestones mount ::API::Namespaces mount ::API::Notes mount ::API::NotificationSettings @@ -139,6 +138,7 @@ module API mount ::API::PipelineSchedules mount ::API::ProjectHooks mount ::API::Projects + mount ::API::ProjectMilestones mount ::API::ProjectSnippets mount ::API::ProtectedBranches mount ::API::Repositories diff --git a/lib/api/boards.rb b/lib/api/boards.rb index 366b0dc9a6f..6c706b2b4e1 100644 --- a/lib/api/boards.rb +++ b/lib/api/boards.rb @@ -1,45 +1,46 @@ module API class Boards < Grape::API + include BoardsResponses include PaginationParams before { authenticate! } + helpers do + def board_parent + user_project + end + end + params do requires :id, type: String, desc: 'The ID of a project' end resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do - desc 'Get all project boards' do - detail 'This feature was introduced in 8.13' - success Entities::Board - end - params do - use :pagination - end - get ':id/boards' do - authorize!(:read_board, user_project) - present paginate(user_project.boards), with: Entities::Board + segment ':id/boards' do + desc 'Get all project boards' do + detail 'This feature was introduced in 8.13' + success Entities::Board + end + params do + use :pagination + end + get '/' do + authorize!(:read_board, user_project) + present paginate(board_parent.boards), with: Entities::Board + end + + desc 'Find a project board' do + detail 'This feature was introduced in 10.4' + success Entities::Board + end + get '/:board_id' do + present board, with: Entities::Board + end end params do requires :board_id, type: Integer, desc: 'The ID of a board' end segment ':id/boards/:board_id' do - helpers do - def project_board - board = user_project.boards.first - - if params[:board_id] == board.id - board - else - not_found!('Board') - end - end - - def board_lists - project_board.lists.destroyable - end - end - desc 'Get the lists of a project board' do detail 'Does not include `done` list. This feature was introduced in 8.13' success Entities::List @@ -72,22 +73,13 @@ module API requires :label_id, type: Integer, desc: 'The ID of an existing label' end post '/lists' do - unless available_labels.exists?(params[:label_id]) + unless available_labels_for(user_project).exists?(params[:label_id]) render_api_error!({ error: 'Label not found!' }, 400) end authorize!(:admin_list, user_project) - service = ::Boards::Lists::CreateService.new(user_project, current_user, - { label_id: params[:label_id] }) - - list = service.execute(project_board) - - if list.valid? - present list, with: Entities::List - else - render_validation_error!(list) - end + create_list end desc 'Moves a board list to a new position' do @@ -99,18 +91,11 @@ module API requires :position, type: Integer, desc: 'The position of the list' end put '/lists/:list_id' do - list = project_board.lists.movable.find(params[:list_id]) + list = board_lists.find(params[:list_id]) authorize!(:admin_list, user_project) - service = ::Boards::Lists::MoveService.new(user_project, current_user, - { position: params[:position] }) - - if service.execute(list) - present list, with: Entities::List - else - render_api_error!({ error: "List could not be moved!" }, 400) - end + move_list(list) end desc 'Delete a board list' do @@ -124,12 +109,7 @@ module API authorize!(:admin_list, user_project) list = board_lists.find(params[:list_id]) - destroy_conditionally!(list) do |list| - service = ::Boards::Lists::DestroyService.new(user_project, current_user) - unless service.execute(list) - render_api_error!({ error: 'List could not be deleted!' }, 400) - end - end + destroy_list(list) end end end diff --git a/lib/api/boards_responses.rb b/lib/api/boards_responses.rb new file mode 100644 index 00000000000..ead0943a74d --- /dev/null +++ b/lib/api/boards_responses.rb @@ -0,0 +1,50 @@ +module API + module BoardsResponses + extend ActiveSupport::Concern + + included do + helpers do + def board + board_parent.boards.find(params[:board_id]) + end + + def board_lists + board.lists.destroyable + end + + def create_list + create_list_service = + ::Boards::Lists::CreateService.new(board_parent, current_user, { label_id: params[:label_id] }) + + list = create_list_service.execute(board) + + if list.valid? + present list, with: Entities::List + else + render_validation_error!(list) + end + end + + def move_list(list) + move_list_service = + ::Boards::Lists::MoveService.new(board_parent, current_user, { position: params[:position].to_i }) + + if move_list_service.execute(list) + present list, with: Entities::List + else + render_api_error!({ error: "List could not be moved!" }, 400) + end + end + + def destroy_list(list) + destroy_conditionally!(list) do |list| + service = ::Boards::Lists::DestroyService.new(board_parent, current_user) + unless service.execute(list) + render_api_error!({ error: 'List could not be deleted!' }, 400) + end + end + end + end + end + end +end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 928706dfda7..bd0c54a1b04 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -708,8 +708,9 @@ module API class ProjectService < Grape::Entity expose :id, :title, :created_at, :updated_at, :active - expose :push_events, :issues_events, :merge_requests_events - expose :tag_push_events, :note_events, :pipeline_events + expose :push_events, :issues_events, :confidential_issues_events + expose :merge_requests_events, :tag_push_events, :note_events + expose :pipeline_events, :wiki_page_events expose :job_events # Expose serialized properties expose :properties do |service, options| @@ -790,6 +791,8 @@ module API class Board < Grape::Entity expose :id + expose :project, using: Entities::BasicProjectDetails + expose :lists, using: Entities::List do |board| board.lists.destroyable end @@ -861,6 +864,8 @@ module API expose :active expose :is_shared expose :name + expose :online?, as: :online + expose :status end class RunnerDetails < Runner @@ -1132,6 +1137,7 @@ module API class PagesDomainBasic < Grape::Entity expose :domain expose :url + expose :project_id expose :certificate, as: :certificate_expiration, if: ->(pages_domain, _) { pages_domain.certificate? }, diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 9ba15893f55..bf388163ec8 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -69,13 +69,20 @@ module API end def wiki_page - page = user_project.wiki.find_page(params[:slug]) + page = ProjectWiki.new(user_project, current_user).find_page(params[:slug]) page || not_found!('Wiki Page') end - def available_labels - @available_labels ||= LabelsFinder.new(current_user, project_id: user_project.id).execute + def available_labels_for(label_parent) + search_params = + if label_parent.is_a?(Project) + { project_id: label_parent.id } + else + { group_id: label_parent.id, only_group_labels: true } + end + + LabelsFinder.new(current_user, search_params).execute end def find_user(id) @@ -141,7 +148,9 @@ module API end def find_project_label(id) - label = available_labels.find_by_id(id) || available_labels.find_by_title(id) + labels = available_labels_for(user_project) + label = labels.find_by_id(id) || labels.find_by_title(id) + label || not_found!('Label') end diff --git a/lib/api/internal.rb b/lib/api/internal.rb index ccaaeca10d4..79b302aae70 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -190,9 +190,12 @@ module API project = Gitlab::GlRepository.parse(params[:gl_repository]).first user = identify(params[:identifier]) - redirect_message = Gitlab::Checks::ProjectMoved.fetch_redirect_message(user.id, project.id) - if redirect_message - output[:redirected_message] = redirect_message + + # A user is not guaranteed to be returned; an orphaned write deploy + # key could be used + if user + redirect_message = Gitlab::Checks::ProjectMoved.fetch_redirect_message(user.id, project.id) + output[:redirected_message] = redirect_message if redirect_message end output diff --git a/lib/api/issues.rb b/lib/api/issues.rb index b29c5848aef..7aa10631d53 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -277,6 +277,19 @@ module API present paginate(merge_requests), with: Entities::MergeRequestBasic, current_user: current_user, project: user_project end + desc 'List participants for an issue' do + success Entities::UserBasic + end + params do + requires :issue_iid, type: Integer, desc: 'The internal ID of a project issue' + end + get ':id/issues/:issue_iid/participants' do + issue = find_project_issue(params[:issue_iid]) + participants = ::Kaminari.paginate_array(issue.participants) + + present paginate(participants), with: Entities::UserBasic, current_user: current_user, project: user_project + end + desc 'Get the user agent details for an issue' do success Entities::UserAgentDetail end diff --git a/lib/api/labels.rb b/lib/api/labels.rb index e41a1720ac1..81eaf56e48e 100644 --- a/lib/api/labels.rb +++ b/lib/api/labels.rb @@ -15,7 +15,7 @@ module API use :pagination end get ':id/labels' do - present paginate(available_labels), with: Entities::Label, current_user: current_user, project: user_project + present paginate(available_labels_for(user_project)), with: Entities::Label, current_user: current_user, project: user_project end desc 'Create a new label' do @@ -30,7 +30,7 @@ module API post ':id/labels' do authorize! :admin_label, user_project - label = available_labels.find_by(title: params[:name]) + label = available_labels_for(user_project).find_by(title: params[:name]) conflict!('Label already exists') if label priority = params.delete(:priority) diff --git a/lib/api/members.rb b/lib/api/members.rb index 22e4bdead41..5446f6b54b1 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -59,7 +59,9 @@ module API member = source.add_user(params[:user_id], params[:access_level], current_user: current_user, expires_at: params[:expires_at]) - if member.persisted? && member.valid? + if !member + not_allowed! # This currently can only be reached in EE + elsif member.persisted? && member.valid? present member.user, with: Entities::Member, member: member else render_validation_error!(member) diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 02f2b75ab9d..8f665b39fa8 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -185,6 +185,16 @@ module API present merge_request, with: Entities::MergeRequest, current_user: current_user, project: user_project end + desc 'Get the participants of a merge request' do + success Entities::UserBasic + end + get ':id/merge_requests/:merge_request_iid/participants' do + merge_request = find_merge_request_with_access(params[:merge_request_iid]) + participants = ::Kaminari.paginate_array(merge_request.participants) + + present paginate(participants), with: Entities::UserBasic + end + desc 'Get the commits of a merge request' do success Entities::Commit end diff --git a/lib/api/services.rb b/lib/api/services.rb index bbcc851d07a..a7f44e2869c 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -1,5 +1,143 @@ module API class Services < Grape::API + chat_notification_settings = [ + { + required: true, + name: :webhook, + type: String, + desc: 'The chat webhook' + }, + { + required: false, + name: :username, + type: String, + desc: 'The chat username' + }, + { + required: false, + name: :channel, + type: String, + desc: 'The default chat channel' + } + ] + + chat_notification_flags = [ + { + required: false, + name: :notify_only_broken_pipelines, + type: Boolean, + desc: 'Send notifications for broken pipelines' + }, + { + required: false, + name: :notify_only_default_branch, + type: Boolean, + desc: 'Send notifications only for the default branch' + } + ] + + chat_notification_channels = [ + { + required: false, + name: :push_channel, + type: String, + desc: 'The name of the channel to receive push_events notifications' + }, + { + required: false, + name: :issue_channel, + type: String, + desc: 'The name of the channel to receive issues_events notifications' + }, + { + required: false, + name: :confidential_issue_channel, + type: String, + desc: 'The name of the channel to receive confidential_issues_events notifications' + }, + { + required: false, + name: :merge_request_channel, + type: String, + desc: 'The name of the channel to receive merge_requests_events notifications' + }, + { + required: false, + name: :note_channel, + type: String, + desc: 'The name of the channel to receive note_events notifications' + }, + { + required: false, + name: :tag_push_channel, + type: String, + desc: 'The name of the channel to receive tag_push_events notifications' + }, + { + required: false, + name: :pipeline_channel, + type: String, + desc: 'The name of the channel to receive pipeline_events notifications' + }, + { + required: false, + name: :wiki_page_channel, + type: String, + desc: 'The name of the channel to receive wiki_page_events notifications' + } + ] + + chat_notification_events = [ + { + required: false, + name: :push_events, + type: Boolean, + desc: 'Enable notifications for push_events' + }, + { + required: false, + name: :issues_events, + type: Boolean, + desc: 'Enable notifications for issues_events' + }, + { + required: false, + name: :confidential_issues_events, + type: Boolean, + desc: 'Enable notifications for confidential_issues_events' + }, + { + required: false, + name: :merge_requests_events, + type: Boolean, + desc: 'Enable notifications for merge_requests_events' + }, + { + required: false, + name: :note_events, + type: Boolean, + desc: 'Enable notifications for note_events' + }, + { + required: false, + name: :tag_push_events, + type: Boolean, + desc: 'Enable notifications for tag_push_events' + }, + { + required: false, + name: :pipeline_events, + type: Boolean, + desc: 'Enable notifications for pipeline_events' + }, + { + required: false, + name: :wiki_page_events, + type: Boolean, + desc: 'Enable notifications for wiki_page_events' + } + ] + services = { 'asana' => [ { @@ -489,25 +627,11 @@ module API } ], 'slack' => [ - { - required: true, - name: :webhook, - type: String, - desc: 'The Slack webhook. e.g. https://hooks.slack.com/services/...' - }, - { - required: false, - name: :new_issue_url, - type: String, - desc: 'The user name' - }, - { - required: false, - name: :channel, - type: String, - desc: 'The channel name' - } - ], + chat_notification_settings, + chat_notification_flags, + chat_notification_channels, + chat_notification_events + ].flatten, 'microsoft-teams' => [ { required: true, @@ -517,19 +641,11 @@ module API } ], 'mattermost' => [ - { - required: true, - name: :webhook, - type: String, - desc: 'The Mattermost webhook. e.g. http://mattermost_host/hooks/...' - }, - { - required: false, - name: :username, - type: String, - desc: 'The username to use to post the message' - } - ], + chat_notification_settings, + chat_notification_flags, + chat_notification_channels, + chat_notification_events + ].flatten, 'teamcity' => [ { required: true, diff --git a/lib/api/v3/labels.rb b/lib/api/v3/labels.rb index bd5eb2175e8..4157462ec2a 100644 --- a/lib/api/v3/labels.rb +++ b/lib/api/v3/labels.rb @@ -11,7 +11,7 @@ module API success ::API::Entities::Label end get ':id/labels' do - present available_labels, with: ::API::Entities::Label, current_user: current_user, project: user_project + present available_labels_for(user_project), with: ::API::Entities::Label, current_user: current_user, project: user_project end desc 'Delete an existing label' do diff --git a/lib/backup/files.rb b/lib/backup/files.rb index 30a91647b77..287d591e88d 100644 --- a/lib/backup/files.rb +++ b/lib/backup/files.rb @@ -18,7 +18,7 @@ module Backup FileUtils.rm_f(backup_tarball) if ENV['STRATEGY'] == 'copy' - cmd = %W(cp -a #{app_files_dir} #{Gitlab.config.backup.path}) + cmd = %W(rsync -a --exclude=lost+found #{app_files_dir} #{Gitlab.config.backup.path}) output, status = Gitlab::Popen.popen(cmd) unless status.zero? @@ -26,10 +26,10 @@ module Backup abort 'Backup failed' end - run_pipeline!([%W(tar -C #{@backup_files_dir} -cf - .), %w(gzip -c -1)], out: [backup_tarball, 'w', 0600]) + run_pipeline!([%W(tar --exclude=lost+found -C #{@backup_files_dir} -cf - .), %w(gzip -c -1)], out: [backup_tarball, 'w', 0600]) FileUtils.rm_rf(@backup_files_dir) else - run_pipeline!([%W(tar -C #{app_files_dir} -cf - .), %w(gzip -c -1)], out: [backup_tarball, 'w', 0600]) + run_pipeline!([%W(tar --exclude=lost+found -C #{app_files_dir} -cf - .), %w(gzip -c -1)], out: [backup_tarball, 'w', 0600]) end end diff --git a/lib/banzai/filter/mermaid_filter.rb b/lib/banzai/filter/mermaid_filter.rb index b545b947a2c..65c131e08d9 100644 --- a/lib/banzai/filter/mermaid_filter.rb +++ b/lib/banzai/filter/mermaid_filter.rb @@ -2,16 +2,7 @@ module Banzai module Filter class MermaidFilter < HTML::Pipeline::Filter def call - doc.css('pre[lang="mermaid"]').add_class('mermaid') - doc.css('pre[lang="mermaid"]').add_class('js-render-mermaid') - - # The `<code></code>` blocks are added in the lib/banzai/filter/syntax_highlight_filter.rb - # We want to keep context and consistency, so we the blocks are added for all filters. - # Details: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/15107/diffs?diff_id=7962900#note_45495859 - doc.css('pre[lang="mermaid"]').each do |pre| - document = pre.at('code') - document.replace(document.content) - end + doc.css('pre[lang="mermaid"] > code').add_class('js-render-mermaid') doc end diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb index 758f15c8a67..5c197afd782 100644 --- a/lib/banzai/filter/relative_link_filter.rb +++ b/lib/banzai/filter/relative_link_filter.rb @@ -2,19 +2,21 @@ require 'uri' module Banzai module Filter - # HTML filter that "fixes" relative links to files in a repository. + # HTML filter that "fixes" relative links to uploads or files in a repository. # # Context options: # :commit + # :group # :project # :project_wiki # :ref # :requested_path class RelativeLinkFilter < HTML::Pipeline::Filter - def call - return doc unless linkable_files? + include Gitlab::Utils::StrongMemoize + def call @uri_types = {} + clear_memoization(:linkable_files) doc.search('a:not(.gfm)').each do |el| process_link_attr el.attribute('href') @@ -31,18 +33,40 @@ module Banzai protected def linkable_files? - context[:project_wiki].nil? && repository.try(:exists?) && !repository.empty? + strong_memoize(:linkable_files) do + context[:project_wiki].nil? && repository.try(:exists?) && !repository.empty? + end end def process_link_attr(html_attr) return if html_attr.blank? return if html_attr.value.start_with?('//') + if html_attr.value.start_with?('/uploads/') + process_link_to_upload_attr(html_attr) + elsif linkable_files? + process_link_to_repository_attr(html_attr) + end + end + + def process_link_to_upload_attr(html_attr) + uri_parts = [html_attr.value] + + if group + uri_parts.unshift(relative_url_root, 'groups', group.full_path, '-') + elsif project + uri_parts.unshift(relative_url_root, project.full_path) + end + + html_attr.value = File.join(*uri_parts) + end + + def process_link_to_repository_attr(html_attr) uri = URI(html_attr.value) if uri.relative? && uri.path.present? html_attr.value = rebuild_relative_uri(uri).to_s end - rescue URI::Error + rescue URI::Error, Addressable::URI::InvalidURIError # noop end @@ -51,7 +75,7 @@ module Banzai uri.path = [ relative_url_root, - context[:project].full_path, + project.full_path, uri_type(file_path), Addressable::URI.escape(ref), Addressable::URI.escape(file_path) @@ -123,11 +147,19 @@ module Banzai end def ref - context[:ref] || context[:project].default_branch + context[:ref] || project.default_branch + end + + def group + context[:group] + end + + def project + context[:project] end def repository - @repository ||= context[:project].try(:repository) + @repository ||= project&.repository end end end diff --git a/lib/banzai/filter/upload_link_filter.rb b/lib/banzai/filter/upload_link_filter.rb deleted file mode 100644 index d64f9ac4eb6..00000000000 --- a/lib/banzai/filter/upload_link_filter.rb +++ /dev/null @@ -1,60 +0,0 @@ -require 'uri' - -module Banzai - module Filter - # HTML filter that "fixes" relative upload links to files. - # Context options: - # :project (required) - Current project - # - class UploadLinkFilter < HTML::Pipeline::Filter - def call - return doc unless project || group - - doc.xpath('descendant-or-self::a[starts-with(@href, "/uploads/")]').each do |el| - process_link_attr el.attribute('href') - end - - doc.xpath('descendant-or-self::img[starts-with(@src, "/uploads/")]').each do |el| - process_link_attr el.attribute('src') - end - - doc - end - - protected - - def process_link_attr(html_attr) - html_attr.value = build_url(html_attr.value).to_s - end - - def build_url(uri) - base_path = Gitlab.config.gitlab.url - - if group - urls = Gitlab::Routing.url_helpers - # we need to get last 2 parts of the uri which are secret and filename - uri_parts = uri.split(File::SEPARATOR) - file_path = urls.show_group_uploads_path(group, uri_parts[-2], uri_parts[-1]) - File.join(base_path, file_path) - else - File.join(base_path, project.full_path, uri) - end - end - - def project - context[:project] - end - - def group - context[:group] - end - - # Ensure that a :project key exists in context - # - # Note that while the key might exist, its value could be nil! - def validate - needs :project - end - end - end -end diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index 55874ad50a3..c746f6f64e9 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -15,7 +15,6 @@ module Banzai Filter::MathFilter, Filter::MermaidFilter, - Filter::UploadLinkFilter, Filter::VideoLinkFilter, Filter::ImageLazyLoadFilter, Filter::ImageLinkFilter, diff --git a/lib/gitlab/background_migration/cleanup_concurrent_type_change.rb b/lib/gitlab/background_migration/cleanup_concurrent_type_change.rb new file mode 100644 index 00000000000..de622f657b2 --- /dev/null +++ b/lib/gitlab/background_migration/cleanup_concurrent_type_change.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Background migration for cleaning up a concurrent column rename. + class CleanupConcurrentTypeChange + include Database::MigrationHelpers + + RESCHEDULE_DELAY = 10.minutes + + # table - The name of the table the migration is performed for. + # old_column - The name of the old (to drop) column. + # new_column - The name of the new column. + def perform(table, old_column, new_column) + return unless column_exists?(:issues, new_column) + + rows_to_migrate = define_model_for(table) + .where(new_column => nil) + .where + .not(old_column => nil) + + if rows_to_migrate.any? + BackgroundMigrationWorker.perform_in( + RESCHEDULE_DELAY, + 'CleanupConcurrentTypeChange', + [table, old_column, new_column] + ) + else + cleanup_concurrent_column_type_change(table, old_column) + end + end + + # These methods are necessary so we can re-use the migration helpers in + # this class. + def connection + ActiveRecord::Base.connection + end + + def method_missing(name, *args, &block) + connection.__send__(name, *args, &block) # rubocop: disable GitlabSecurity/PublicSend + end + + def respond_to_missing?(*args) + connection.respond_to?(*args) || super + end + + def define_model_for(table) + Class.new(ActiveRecord::Base) do + self.table_name = table + end + end + end + end +end diff --git a/lib/gitlab/background_migration/copy_column.rb b/lib/gitlab/background_migration/copy_column.rb new file mode 100644 index 00000000000..a2cb215c230 --- /dev/null +++ b/lib/gitlab/background_migration/copy_column.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # CopyColumn is a simple (reusable) background migration that can be used to + # update the value of a column based on the value of another column in the + # same table. + # + # For this background migration to work the table that is migrated _has_ to + # have an `id` column as the primary key. + class CopyColumn + # table - The name of the table that contains the columns. + # copy_from - The column containing the data to copy. + # copy_to - The column to copy the data to. + # start_id - The start ID of the range of rows to update. + # end_id - The end ID of the range of rows to update. + def perform(table, copy_from, copy_to, start_id, end_id) + return unless connection.column_exists?(table, copy_to) + + quoted_table = connection.quote_table_name(table) + quoted_copy_from = connection.quote_column_name(copy_from) + quoted_copy_to = connection.quote_column_name(copy_to) + + # We're using raw SQL here since this job may be frequently executed. As + # a result dynamically defining models would lead to many unnecessary + # schema information queries. + connection.execute <<-SQL.strip_heredoc + UPDATE #{quoted_table} + SET #{quoted_copy_to} = #{quoted_copy_from} + WHERE id BETWEEN #{start_id} AND #{end_id} + SQL + end + + def connection + ActiveRecord::Base.connection + end + end + end +end diff --git a/lib/gitlab/background_migration/delete_conflicting_redirect_routes_range.rb b/lib/gitlab/background_migration/delete_conflicting_redirect_routes_range.rb index a1af045a71f..21b626dde56 100644 --- a/lib/gitlab/background_migration/delete_conflicting_redirect_routes_range.rb +++ b/lib/gitlab/background_migration/delete_conflicting_redirect_routes_range.rb @@ -1,44 +1,12 @@ # frozen_string_literal: true -# rubocop:disable Metrics/LineLength # rubocop:disable Style/Documentation module Gitlab module BackgroundMigration class DeleteConflictingRedirectRoutesRange - class Route < ActiveRecord::Base - self.table_name = 'routes' - end - - class RedirectRoute < ActiveRecord::Base - self.table_name = 'redirect_routes' - end - - # start_id - The start ID of the range of events to process - # end_id - The end ID of the range to process. def perform(start_id, end_id) - return unless migrate? - - conflicts = RedirectRoute.where(routes_match_redirects_clause(start_id, end_id)) - num_rows = conflicts.delete_all - - Rails.logger.info("Gitlab::BackgroundMigration::DeleteConflictingRedirectRoutesRange [#{start_id}, #{end_id}] - Deleted #{num_rows} redirect routes that were conflicting with routes.") - end - - def migrate? - Route.table_exists? && RedirectRoute.table_exists? - end - - def routes_match_redirects_clause(start_id, end_id) - <<~ROUTES_MATCH_REDIRECTS - EXISTS ( - SELECT 1 FROM routes - WHERE ( - LOWER(redirect_routes.path) = LOWER(routes.path) - OR LOWER(redirect_routes.path) LIKE LOWER(CONCAT(routes.path, '/%')) - ) - AND routes.id BETWEEN #{start_id} AND #{end_id} - ) - ROUTES_MATCH_REDIRECTS + # No-op. + # See https://gitlab.com/gitlab-com/infrastructure/issues/3460#note_53223252 end end end diff --git a/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb b/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb index 84ac00f1a5c..7088aa0860a 100644 --- a/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb +++ b/lib/gitlab/background_migration/migrate_events_to_push_event_payloads.rb @@ -128,8 +128,14 @@ module Gitlab end def process_event(event) - replicate_event(event) - create_push_event_payload(event) if event.push_event? + ActiveRecord::Base.transaction do + replicate_event(event) + create_push_event_payload(event) if event.push_event? + end + rescue ActiveRecord::InvalidForeignKey => e + # A foreign key error means the associated event was removed. In this + # case we'll just skip migrating the event. + Rails.logger.error("Unable to migrate event #{event.id}: #{e}") end def replicate_event(event) @@ -137,9 +143,6 @@ module Gitlab .with_indifferent_access.except(:title, :data) EventForMigration.create!(new_attributes) - rescue ActiveRecord::InvalidForeignKey - # A foreign key error means the associated event was removed. In this - # case we'll just skip migrating the event. end def create_push_event_payload(event) @@ -156,9 +159,6 @@ module Gitlab ref: event.trimmed_ref_name, commit_title: event.commit_title ) - rescue ActiveRecord::InvalidForeignKey - # A foreign key error means the associated event was removed. In this - # case we'll just skip migrating the event. end def find_events(start_id, end_id) diff --git a/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data.rb b/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data.rb new file mode 100644 index 00000000000..8a901a9bf39 --- /dev/null +++ b/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data.rb @@ -0,0 +1,135 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/LineLength +# rubocop:disable Metrics/MethodLength +# rubocop:disable Metrics/ClassLength +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class PopulateMergeRequestMetricsWithEventsData + def perform(min_merge_request_id, max_merge_request_id) + insert_metrics_for_range(min_merge_request_id, max_merge_request_id) + update_metrics_with_events_data(min_merge_request_id, max_merge_request_id) + end + + # Inserts merge_request_metrics records for merge_requests without it for + # a given merge request batch. + def insert_metrics_for_range(min, max) + metrics_not_exists_clause = + <<-SQL.strip_heredoc + NOT EXISTS (SELECT 1 FROM merge_request_metrics + WHERE merge_request_metrics.merge_request_id = merge_requests.id) + SQL + + MergeRequest.where(metrics_not_exists_clause).where(id: min..max).each_batch do |batch| + select_sql = batch.select(:id, :created_at, :updated_at).to_sql + + execute("INSERT INTO merge_request_metrics (merge_request_id, created_at, updated_at) #{select_sql}") + end + end + + def update_metrics_with_events_data(min, max) + if Gitlab::Database.postgresql? + # Uses WITH syntax in order to update merged and closed events with a single UPDATE. + # WITH is not supported by MySQL. + update_events_for_range(min, max) + else + update_merged_events_for_range(min, max) + update_closed_events_for_range(min, max) + end + end + + private + + # Updates merge_request_metrics latest_closed_at, latest_closed_by_id and merged_by_id + # based on the latest event records on events table for a given merge request batch. + def update_events_for_range(min, max) + sql = <<-SQL.strip_heredoc + WITH events_for_update AS ( + SELECT DISTINCT ON (target_id, action) target_id, action, author_id, updated_at + FROM events + WHERE target_id BETWEEN #{min} AND #{max} + AND target_type = 'MergeRequest' + AND action IN (#{Event::CLOSED},#{Event::MERGED}) + ORDER BY target_id, action, id DESC + ) + UPDATE merge_request_metrics met + SET latest_closed_at = latest_closed.updated_at, + latest_closed_by_id = latest_closed.author_id, + merged_by_id = latest_merged.author_id + FROM (SELECT * FROM events_for_update WHERE action = #{Event::CLOSED}) AS latest_closed + FULL OUTER JOIN + (SELECT * FROM events_for_update WHERE action = #{Event::MERGED}) AS latest_merged + USING (target_id) + WHERE target_id = merge_request_id; + SQL + + execute(sql) + end + + # Updates merge_request_metrics latest_closed_at, latest_closed_by_id based on the latest closed + # records on events table for a given merge request batch. + def update_closed_events_for_range(min, max) + sql = + <<-SQL.strip_heredoc + UPDATE merge_request_metrics metrics, + (#{select_events(min, max, Event::CLOSED)}) closed_events + SET metrics.latest_closed_by_id = closed_events.author_id, + metrics.latest_closed_at = closed_events.updated_at #{where_matches_closed_events}; + SQL + + execute(sql) + end + + # Updates merge_request_metrics merged_by_id based on the latest merged + # records on events table for a given merge request batch. + def update_merged_events_for_range(min, max) + sql = + <<-SQL.strip_heredoc + UPDATE merge_request_metrics metrics, + (#{select_events(min, max, Event::MERGED)}) merged_events + SET metrics.merged_by_id = merged_events.author_id #{where_matches_merged_events}; + SQL + + execute(sql) + end + + def execute(sql) + @connection ||= ActiveRecord::Base.connection + @connection.execute(sql) + end + + def select_events(min, max, action) + select_max_event_id = <<-SQL.strip_heredoc + SELECT max(id) + FROM events + WHERE action = #{action} + AND target_type = 'MergeRequest' + AND target_id BETWEEN #{min} AND #{max} + GROUP BY target_id + SQL + + <<-SQL.strip_heredoc + SELECT author_id, updated_at, target_id + FROM events + WHERE id IN(#{select_max_event_id}) + SQL + end + + def where_matches_closed_events + <<-SQL.strip_heredoc + WHERE metrics.merge_request_id = closed_events.target_id + AND metrics.latest_closed_at IS NULL + AND metrics.latest_closed_by_id IS NULL + SQL + end + + def where_matches_merged_events + <<-SQL.strip_heredoc + WHERE metrics.merge_request_id = merged_events.target_id + AND metrics.merged_by_id IS NULL + SQL + end + end + end +end diff --git a/lib/gitlab/bare_repository_import/importer.rb b/lib/gitlab/bare_repository_import/importer.rb index 298409d8b5a..709a901aa77 100644 --- a/lib/gitlab/bare_repository_import/importer.rb +++ b/lib/gitlab/bare_repository_import/importer.rb @@ -14,7 +14,7 @@ module Gitlab repos_to_import.each do |repo_path| bare_repo = Gitlab::BareRepositoryImport::Repository.new(import_path, repo_path) - if bare_repo.hashed? || bare_repo.wiki? + unless bare_repo.processable? log " * Skipping repo #{bare_repo.repo_path}".color(:yellow) next @@ -55,12 +55,15 @@ module Gitlab name: project_name, path: project_name, skip_disk_validation: true, - import_type: 'gitlab_project', + skip_wiki: bare_repo.wiki_exists?, + import_type: 'bare_repository', namespace_id: group&.id).execute if project.persisted? && mv_repo(project) log " * Created #{project.name} (#{project_full_path})".color(:green) + project.write_repository_config + ProjectCacheWorker.perform_async(project.id) else log " * Failed trying to create #{project.name} (#{project_full_path})".color(:red) diff --git a/lib/gitlab/bare_repository_import/repository.rb b/lib/gitlab/bare_repository_import/repository.rb index fa7891c8dcc..85b79362196 100644 --- a/lib/gitlab/bare_repository_import/repository.rb +++ b/lib/gitlab/bare_repository_import/repository.rb @@ -6,39 +6,56 @@ module Gitlab def initialize(root_path, repo_path) @root_path = root_path @repo_path = repo_path - @root_path << '/' unless root_path.ends_with?('/') + full_path = + if hashed? && !wiki? + repository.config.get('gitlab.fullpath') + else + repo_relative_path + end + # Split path into 'all/the/namespaces' and 'project_name' - @group_path, _, @project_name = repo_relative_path.rpartition('/') + @group_path, _, @project_name = full_path.to_s.rpartition('/') end def wiki_exists? File.exist?(wiki_path) end - def wiki? - @wiki ||= repo_path.end_with?('.wiki.git') - end - def wiki_path @wiki_path ||= repo_path.sub(/\.git$/, '.wiki.git') end - def hashed? - @hashed ||= group_path.start_with?('@hashed') - end - def project_full_path @project_full_path ||= "#{group_path}/#{project_name}" end + def processable? + return false if wiki? + return false if hashed? && (group_path.blank? || project_name.blank?) + + true + end + private + def wiki? + @wiki ||= repo_path.end_with?('.wiki.git') + end + + def hashed? + @hashed ||= repo_relative_path.include?('@hashed') + end + def repo_relative_path # Remove root path and `.git` at the end repo_path[@root_path.size...-4] end + + def repository + @repository ||= Rugged::Repository.new(repo_path) + end end end end diff --git a/lib/gitlab/checks/project_moved.rb b/lib/gitlab/checks/project_moved.rb index 3a1c0a3455e..dfb2f4d4054 100644 --- a/lib/gitlab/checks/project_moved.rb +++ b/lib/gitlab/checks/project_moved.rb @@ -21,6 +21,10 @@ module Gitlab end def add_redirect_message + # Don't bother with sending a redirect message for anonymous clones + # because they never see it via the `/internal/post_receive` endpoint + return unless user.present? && project.present? + Gitlab::Redis::SharedState.with do |redis| key = self.class.redirect_message_key(user.id, project.id) redis.setex(key, 5.minutes, redirect_message) diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb index 72b75791bbb..e25916528f4 100644 --- a/lib/gitlab/ci/ansi2html.rb +++ b/lib/gitlab/ci/ansi2html.rb @@ -234,7 +234,7 @@ module Gitlab # Most terminals show bold colored text in the light color variant # Let's mimic that here if @style_mask & STYLE_SWITCHES[:bold] != 0 - fg_color.sub!(/fg-(\w{2,}+)/, 'fg-l-\1') + fg_color.sub!(/fg-([a-z]{2,}+)/, 'fg-l-\1') end css_classes << fg_color end diff --git a/lib/gitlab/ci/config/entry/node.rb b/lib/gitlab/ci/config/entry/node.rb index 1fba0b2db0b..26505c91be3 100644 --- a/lib/gitlab/ci/config/entry/node.rb +++ b/lib/gitlab/ci/config/entry/node.rb @@ -93,9 +93,7 @@ module Gitlab private - def entries - @entries - end + attr_reader :entries end end end diff --git a/lib/gitlab/conflict/file_collection.rb b/lib/gitlab/conflict/file_collection.rb index 76aee5a3deb..0a3ae2c3760 100644 --- a/lib/gitlab/conflict/file_collection.rb +++ b/lib/gitlab/conflict/file_collection.rb @@ -13,12 +13,13 @@ module Gitlab end def resolve(user, commit_message, files) + msg = commit_message || default_commit_message + resolution = Gitlab::Git::Conflict::Resolution.new(user, files, msg) args = { source_branch: merge_request.source_branch, - target_branch: merge_request.target_branch, - commit_message: commit_message || default_commit_message + target_branch: merge_request.target_branch } - resolver.resolve_conflicts(@source_repo, user, files, args) + resolver.resolve_conflicts(@source_repo, resolution, args) ensure @merge_request.clear_memoized_shas end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 3f65bc912de..7b35c24d153 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -385,10 +385,27 @@ module Gitlab # necessary since we copy over old values further down. change_column_default(table, new, old_col.default) if old_col.default - trigger_name = rename_trigger_name(table, old, new) + install_rename_triggers(table, old, new) + + update_column_in_batches(table, new, Arel::Table.new(table)[old]) + + change_column_null(table, new, false) unless old_col.null + + copy_indexes(table, old, new) + copy_foreign_keys(table, old, new) + end + + # Installs triggers in a table that keep a new column in sync with an old + # one. + # + # table - The name of the table to install the trigger in. + # old_column - The name of the old column. + # new_column - The name of the new column. + def install_rename_triggers(table, old_column, new_column) + trigger_name = rename_trigger_name(table, old_column, new_column) quoted_table = quote_table_name(table) - quoted_old = quote_column_name(old) - quoted_new = quote_column_name(new) + quoted_old = quote_column_name(old_column) + quoted_new = quote_column_name(new_column) if Database.postgresql? install_rename_triggers_for_postgresql(trigger_name, quoted_table, @@ -397,13 +414,6 @@ module Gitlab install_rename_triggers_for_mysql(trigger_name, quoted_table, quoted_old, quoted_new) end - - update_column_in_batches(table, new, Arel::Table.new(table)[old]) - - change_column_null(table, new, false) unless old_col.null - - copy_indexes(table, old, new) - copy_foreign_keys(table, old, new) end # Changes the type of a column concurrently. @@ -455,6 +465,97 @@ module Gitlab remove_column(table, old) end + # Changes the column type of a table using a background migration. + # + # Because this method uses a background migration it's more suitable for + # large tables. For small tables it's better to use + # `change_column_type_concurrently` since it can complete its work in a + # much shorter amount of time and doesn't rely on Sidekiq. + # + # Example usage: + # + # class Issue < ActiveRecord::Base + # self.table_name = 'issues' + # + # include EachBatch + # + # def self.to_migrate + # where('closed_at IS NOT NULL') + # end + # end + # + # change_column_type_using_background_migration( + # Issue.to_migrate, + # :closed_at, + # :datetime_with_timezone + # ) + # + # Reverting a migration like this is done exactly the same way, just with + # a different type to migrate to (e.g. `:datetime` in the above example). + # + # relation - An ActiveRecord relation to use for scheduling jobs and + # figuring out what table we're modifying. This relation _must_ + # have the EachBatch module included. + # + # column - The name of the column for which the type will be changed. + # + # new_type - The new type of the column. + # + # batch_size - The number of rows to schedule in a single background + # migration. + # + # interval - The time interval between every background migration. + def change_column_type_using_background_migration( + relation, + column, + new_type, + batch_size: 10_000, + interval: 10.minutes + ) + unless relation.model < EachBatch + raise TypeError, 'The relation must include the EachBatch module' + end + + temp_column = "#{column}_for_type_change" + table = relation.table_name + max_index = 0 + + add_column(table, temp_column, new_type) + install_rename_triggers(table, column, temp_column) + + # Schedule the jobs that will copy the data from the old column to the + # new one. + relation.each_batch(of: batch_size) do |batch, index| + start_id, end_id = batch.pluck('MIN(id), MAX(id)').first + max_index = index + + BackgroundMigrationWorker.perform_in( + index * interval, + 'CopyColumn', + [table, column, temp_column, start_id, end_id] + ) + end + + # Schedule the renaming of the column to happen (initially) 1 hour after + # the last batch finished. + BackgroundMigrationWorker.perform_in( + (max_index * interval) + 1.hour, + 'CleanupConcurrentTypeChange', + [table, column, temp_column] + ) + + if perform_background_migration_inline? + # To ensure the schema is up to date immediately we perform the + # migration inline in dev / test environments. + Gitlab::BackgroundMigration.steal('CopyColumn') + Gitlab::BackgroundMigration.steal('CleanupConcurrentTypeChange') + end + end + + def perform_background_migration_inline? + Rails.env.test? || Rails.env.development? + end + # Performs a concurrent column rename when using PostgreSQL. def install_rename_triggers_for_postgresql(trigger, table, old, new) execute <<-EOF.strip_heredoc @@ -741,6 +842,12 @@ into similar problems in the future (e.g. when new tables are created). def queue_background_migration_jobs_by_range_at_intervals(model_class, job_class_name, delay_interval, batch_size: BACKGROUND_MIGRATION_BATCH_SIZE) raise "#{model_class} does not have an ID to use for batch ranges" unless model_class.column_names.include?('id') + # To not overload the worker too much we enforce a minimum interval both + # when scheduling and performing jobs. + if delay_interval < BackgroundMigrationWorker::MIN_INTERVAL + delay_interval = BackgroundMigrationWorker::MIN_INTERVAL + end + model_class.each_batch(of: batch_size) do |relation, index| start_id, end_id = relation.pluck('MIN(id), MAX(id)').first diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index d0cfe2386ca..34b070dd375 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -61,7 +61,9 @@ module Gitlab end def line_for_position(pos) - diff_lines.find { |line| position(line) == pos } + return nil unless pos.position_type == 'text' + + diff_lines.find { |line| line.old_line == pos.old_line && line.new_line == pos.new_line } end def position_for_line_code(code) @@ -114,8 +116,10 @@ module Gitlab new_content_sha || old_content_sha end + # Use #itself to check the value wrapped by a BatchLoader instance, rather + # than if the BatchLoader instance itself is falsey. def blob - new_blob || old_blob + new_blob&.itself || old_blob&.itself end attr_writer :highlighted_diff_lines @@ -171,7 +175,7 @@ module Gitlab end def binary? - has_binary_notice? || old_blob&.binary? || new_blob&.binary? + has_binary_notice? || try_blobs(:binary?) end def text? @@ -179,15 +183,15 @@ module Gitlab end def external_storage_error? - old_blob&.external_storage_error? || new_blob&.external_storage_error? + try_blobs(:external_storage_error?) end def stored_externally? - old_blob&.stored_externally? || new_blob&.stored_externally? + try_blobs(:stored_externally?) end def external_storage - old_blob&.external_storage || new_blob&.external_storage + try_blobs(:external_storage) end def content_changed? @@ -202,15 +206,15 @@ module Gitlab end def size - [old_blob&.size, new_blob&.size].compact.sum + valid_blobs.map(&:size).sum end def raw_size - [old_blob&.raw_size, new_blob&.raw_size].compact.sum + valid_blobs.map(&:raw_size).sum end def raw_binary? - old_blob&.raw_binary? || new_blob&.raw_binary? + try_blobs(:raw_binary?) end def raw_text? @@ -233,6 +237,19 @@ module Gitlab private + # The blob instances are instances of BatchLoader, which means calling + # &. directly on them won't work. Object#try also won't work, because Blob + # doesn't inherit from Object, but from BasicObject (via SimpleDelegator). + def try_blobs(meth) + old_blob&.itself&.public_send(meth) || new_blob&.itself&.public_send(meth) + end + + # We can't use #compact for the same reason we can't use &., but calling + # #nil? explicitly does work because it is proxied to the blob itself. + def valid_blobs + [old_blob, new_blob].reject(&:nil?) + end + def text_position_properties(line) { old_line: line.old_line, new_line: line.new_line } end diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index 582028493e9..c0edcabc6fd 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -14,14 +14,7 @@ module Gitlab ENCODING_CONFIDENCE_THRESHOLD = 50 def encode!(message) - return nil unless message.respond_to?(:force_encoding) - return message if message.encoding == Encoding::UTF_8 && message.valid_encoding? - - if message.respond_to?(:frozen?) && message.frozen? - message = message.dup - end - - message.force_encoding("UTF-8") + message = force_encode_utf8(message) return message if message.valid_encoding? # return message if message type is binary @@ -35,6 +28,8 @@ module Gitlab # encode and clean the bad chars message.replace clean(message) + rescue ArgumentError + return nil rescue encoding = detect ? detect[:encoding] : "unknown" "--broken encoding: #{encoding}" @@ -54,8 +49,8 @@ module Gitlab end def encode_utf8(message) - return nil unless message.is_a?(String) - return message if message.encoding == Encoding::UTF_8 && message.valid_encoding? + message = force_encode_utf8(message) + return message if message.valid_encoding? detect = CharlockHolmes::EncodingDetector.detect(message) if detect && detect[:encoding] @@ -69,10 +64,31 @@ module Gitlab else clean(message) end + rescue ArgumentError + return nil + end + + def encode_binary(s) + return "" if s.nil? + + s.dup.force_encoding(Encoding::ASCII_8BIT) + end + + def binary_stringio(s) + StringIO.new(s || '').tap { |io| io.set_encoding(Encoding::ASCII_8BIT) } end private + def force_encode_utf8(message) + raise ArgumentError unless message.respond_to?(:force_encoding) + return message if message.encoding == Encoding::UTF_8 && message.valid_encoding? + + message = message.dup if message.respond_to?(:frozen?) && message.frozen? + + message.force_encoding("UTF-8") + end + def clean(message) message.encode("UTF-16BE", undef: :replace, invalid: :replace, replace: "") .encode("UTF-8") diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index 3f7b42456af..dbb8f317afe 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -71,5 +71,16 @@ module Gitlab redis.exists(@redis_shared_state_key) end end + + # Returns the TTL of the Redis key. + # + # This method will return `nil` if no TTL could be obtained. + def ttl + Gitlab::Redis::SharedState.with do |redis| + ttl = redis.ttl(@redis_shared_state_key) + + ttl if ttl.positive? + end + end end end diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 1f7c35cafaa..71647099f83 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -11,7 +11,7 @@ module Gitlab include Gitlab::EncodingHelper def ref_name(ref) - encode_utf8(ref).sub(/\Arefs\/(tags|heads|remotes)\//, '') + encode!(ref).sub(/\Arefs\/(tags|heads|remotes)\//, '') end def branch_name(ref) diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 228d97a87ab..031fccba92b 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -50,10 +50,19 @@ module Gitlab # to the caller to limit the number of blobs and blob_size_limit. # # Gitaly migration issue: https://gitlab.com/gitlab-org/gitaly/issues/798 - def batch(repository, blob_references, blob_size_limit: nil) - blob_size_limit ||= MAX_DATA_DISPLAY_SIZE - blob_references.map do |sha, path| - find_by_rugged(repository, sha, path, limit: blob_size_limit) + def batch(repository, blob_references, blob_size_limit: MAX_DATA_DISPLAY_SIZE) + Gitlab::GitalyClient.migrate(:list_blobs_by_sha_path) do |is_enabled| + if is_enabled + Gitlab::GitalyClient.allow_n_plus_1_calls do + blob_references.map do |sha, path| + find_by_gitaly(repository, sha, path, limit: blob_size_limit) + end + end + else + blob_references.map do |sha, path| + find_by_rugged(repository, sha, path, limit: blob_size_limit) + end + end end end @@ -122,13 +131,23 @@ module Gitlab ) end - def find_by_gitaly(repository, sha, path) + def find_by_gitaly(repository, sha, path, limit: MAX_DATA_DISPLAY_SIZE) path = path.sub(/\A\/*/, '') path = '/' if path.empty? name = File.basename(path) - entry = Gitlab::GitalyClient::CommitService.new(repository).tree_entry(sha, path, MAX_DATA_DISPLAY_SIZE) + + # Gitaly will think that setting the limit to 0 means unlimited, while + # the client might only need the metadata and thus set the limit to 0. + # In this method we'll then set the limit to 1, but clear the byte of data + # that we got back so for the outside world it looks like the limit was + # actually 0. + req_limit = limit == 0 ? 1 : limit + + entry = Gitlab::GitalyClient::CommitService.new(repository).tree_entry(sha, path, req_limit) return unless entry + entry.data = "" if limit == 0 + case entry.type when :COMMIT new( @@ -154,8 +173,8 @@ module Gitlab end def find_by_rugged(repository, sha, path, limit:) - commit = repository.lookup(sha) - root_tree = commit.tree + rugged_commit = repository.lookup(sha) + root_tree = rugged_commit.tree blob_entry = find_entry_by_path(repository, root_tree.oid, path) diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 145721dea76..016437b2419 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -15,8 +15,6 @@ module Gitlab attr_accessor *SERIALIZE_KEYS # rubocop:disable Lint/AmbiguousOperator - delegate :tree, to: :rugged_commit - def ==(other) return false unless other.is_a?(Gitlab::Git::Commit) @@ -452,6 +450,11 @@ module Gitlab ) end + # Is this the same as Blob.find_entry_by_path ? + def rugged_tree_entry(path) + rugged_commit.tree.path(path) + end + private def init_from_hash(hash) diff --git a/lib/gitlab/git/commit_stats.rb b/lib/gitlab/git/commit_stats.rb index 6bf49a0af18..8463b1eb794 100644 --- a/lib/gitlab/git/commit_stats.rb +++ b/lib/gitlab/git/commit_stats.rb @@ -34,13 +34,8 @@ module Gitlab def rugged_stats(commit) diff = commit.rugged_diff_from_parent - - diff.each_patch do |p| - # TODO: Use the new Rugged convenience methods when they're released - @additions += p.stat[0] - @deletions += p.stat[1] - @total += p.changes - end + _files_changed, @additions, @deletions = diff.stat + @total = @additions + @deletions end end end diff --git a/lib/gitlab/git/conflict/file.rb b/lib/gitlab/git/conflict/file.rb index b2a625e08fa..2a9cf10a068 100644 --- a/lib/gitlab/git/conflict/file.rb +++ b/lib/gitlab/git/conflict/file.rb @@ -2,7 +2,9 @@ module Gitlab module Git module Conflict class File - attr_reader :content, :their_path, :our_path, :our_mode, :repository, :commit_oid + attr_reader :their_path, :our_path, :our_mode, :repository, :commit_oid + + attr_accessor :content def initialize(repository, commit_oid, conflict, content) @repository = repository diff --git a/lib/gitlab/git/conflict/resolution.rb b/lib/gitlab/git/conflict/resolution.rb new file mode 100644 index 00000000000..ab9be683e15 --- /dev/null +++ b/lib/gitlab/git/conflict/resolution.rb @@ -0,0 +1,15 @@ +module Gitlab + module Git + module Conflict + class Resolution + attr_reader :user, :files, :commit_message + + def initialize(user, files, commit_message) + @user = user + @files = files + @commit_message = commit_message + end + end + end + end +end diff --git a/lib/gitlab/git/conflict/resolver.rb b/lib/gitlab/git/conflict/resolver.rb index 03e5c0fcd6f..74c9874d590 100644 --- a/lib/gitlab/git/conflict/resolver.rb +++ b/lib/gitlab/git/conflict/resolver.rb @@ -13,37 +13,27 @@ module Gitlab def conflicts @conflicts ||= begin - target_index = @target_repository.rugged.merge_commits(@our_commit_oid, @their_commit_oid) - - # We don't need to do `with_repo_branch_commit` here, because the target - # project always fetches source refs when creating merge request diffs. - conflict_files(@target_repository, target_index) + @target_repository.gitaly_migrate(:conflicts_list_conflict_files) do |is_enabled| + if is_enabled + gitaly_conflicts_client(@target_repository).list_conflict_files + else + rugged_list_conflict_files + end + end end + rescue GRPC::FailedPrecondition => e + raise Gitlab::Git::Conflict::Resolver::ConflictSideMissing.new(e.message) + rescue Rugged::OdbError, GRPC::BadStatus => e + raise Gitlab::Git::CommandError.new(e) end - def resolve_conflicts(source_repository, user, files, source_branch:, target_branch:, commit_message:) - source_repository.with_repo_branch_commit(@target_repository, target_branch) do - index = source_repository.rugged.merge_commits(@our_commit_oid, @their_commit_oid) - conflicts = conflict_files(source_repository, index) - - files.each do |file_params| - conflict_file = conflict_for_path(conflicts, file_params[:old_path], file_params[:new_path]) - - write_resolved_file_to_index(source_repository, index, conflict_file, file_params) - end - - unless index.conflicts.empty? - missing_files = index.conflicts.map { |file| file[:ours][:path] } - - raise ResolutionError, "Missing resolutions for the following files: #{missing_files.join(', ')}" + def resolve_conflicts(source_repository, resolution, source_branch:, target_branch:) + source_repository.gitaly_migrate(:conflicts_resolve_conflicts) do |is_enabled| + if is_enabled + gitaly_conflicts_client(source_repository).resolve_conflicts(@target_repository, resolution, source_branch, target_branch) + else + rugged_resolve_conflicts(source_repository, resolution, source_branch, target_branch) end - - commit_params = { - message: commit_message, - parents: [@our_commit_oid, @their_commit_oid] - } - - source_repository.commit_index(user, source_branch, index, commit_params) end end @@ -68,6 +58,10 @@ module Gitlab end end + def gitaly_conflicts_client(repository) + repository.gitaly_conflicts_client(@our_commit_oid, @their_commit_oid) + end + def write_resolved_file_to_index(repository, index, file, params) if params[:sections] resolved_lines = file.resolve_lines(params[:sections]) @@ -84,6 +78,40 @@ module Gitlab index.add(path: our_path, oid: oid, mode: file.our_mode) index.conflict_remove(our_path) end + + def rugged_list_conflict_files + target_index = @target_repository.rugged.merge_commits(@our_commit_oid, @their_commit_oid) + + # We don't need to do `with_repo_branch_commit` here, because the target + # project always fetches source refs when creating merge request diffs. + conflict_files(@target_repository, target_index) + end + + def rugged_resolve_conflicts(source_repository, resolution, source_branch, target_branch) + source_repository.with_repo_branch_commit(@target_repository, target_branch) do + index = source_repository.rugged.merge_commits(@our_commit_oid, @their_commit_oid) + conflicts = conflict_files(source_repository, index) + + resolution.files.each do |file_params| + conflict_file = conflict_for_path(conflicts, file_params[:old_path], file_params[:new_path]) + + write_resolved_file_to_index(source_repository, index, conflict_file, file_params) + end + + unless index.conflicts.empty? + missing_files = index.conflicts.map { |file| file[:ours][:path] } + + raise ResolutionError, "Missing resolutions for the following files: #{missing_files.join(', ')}" + end + + commit_params = { + message: resolution.commit_message, + parents: [@our_commit_oid, @their_commit_oid] + } + + source_repository.commit_index(resolution.user, source_branch, index, commit_params) + end + end end end end diff --git a/lib/gitlab/git/gitlab_projects.rb b/lib/gitlab/git/gitlab_projects.rb index d948d7895ed..cba638c06db 100644 --- a/lib/gitlab/git/gitlab_projects.rb +++ b/lib/gitlab/git/gitlab_projects.rb @@ -2,6 +2,9 @@ module Gitlab module Git class GitlabProjects include Gitlab::Git::Popen + include Gitlab::Utils::StrongMemoize + + ShardNameNotFoundError = Class.new(StandardError) # Absolute path to directory where repositories are stored. # Example: /home/git/repositories @@ -97,22 +100,13 @@ module Gitlab end def fork_repository(new_shard_path, new_repository_relative_path) - from_path = repository_absolute_path - to_path = File.join(new_shard_path, new_repository_relative_path) - - # The repository cannot already exist - if File.exist?(to_path) - logger.error "fork-repository failed: destination repository <#{to_path}> already exists." - return false + Gitlab::GitalyClient.migrate(:fork_repository) do |is_enabled| + if is_enabled + gitaly_fork_repository(new_shard_path, new_repository_relative_path) + else + git_fork_repository(new_shard_path, new_repository_relative_path) + end end - - # Ensure the namepsace / hashed storage directory exists - FileUtils.mkdir_p(File.dirname(to_path), mode: 0770) - - logger.info "Forking repository from <#{from_path}> to <#{to_path}>." - cmd = %W(git clone --bare --no-local -- #{from_path} #{to_path}) - - run(cmd, nil) && Gitlab::Git::Repository.create_hooks(to_path, global_hooks_path) end def fetch_remote(name, timeout, force:, tags:, ssh_key: nil, known_hosts: nil) @@ -253,6 +247,48 @@ module Gitlab known_hosts_file&.close! script&.close! end + + private + + def shard_name + strong_memoize(:shard_name) do + shard_name_from_shard_path(shard_path) + end + end + + def shard_name_from_shard_path(shard_path) + Gitlab.config.repositories.storages.find { |_, info| info['path'] == shard_path }&.first || + raise(ShardNameNotFoundError, "no shard found for path '#{shard_path}'") + end + + def git_fork_repository(new_shard_path, new_repository_relative_path) + from_path = repository_absolute_path + to_path = File.join(new_shard_path, new_repository_relative_path) + + # The repository cannot already exist + if File.exist?(to_path) + logger.error "fork-repository failed: destination repository <#{to_path}> already exists." + return false + end + + # Ensure the namepsace / hashed storage directory exists + FileUtils.mkdir_p(File.dirname(to_path), mode: 0770) + + logger.info "Forking repository from <#{from_path}> to <#{to_path}>." + cmd = %W(git clone --bare --no-local -- #{from_path} #{to_path}) + + run(cmd, nil) && Gitlab::Git::Repository.create_hooks(to_path, global_hooks_path) + end + + def gitaly_fork_repository(new_shard_path, new_repository_relative_path) + target_repository = Gitlab::Git::Repository.new(shard_name_from_shard_path(new_shard_path), new_repository_relative_path, nil) + raw_repository = Gitlab::Git::Repository.new(shard_name, repository_relative_path, nil) + + Gitlab::GitalyClient::RepositoryService.new(target_repository).fork_repository(raw_repository) + rescue GRPC::BadStatus => e + logger.error "fork-repository failed: #{e.message}" + false + end end end end diff --git a/lib/gitlab/git/index.rb b/lib/gitlab/git/index.rb index db532600d1b..d94082a3e30 100644 --- a/lib/gitlab/git/index.rb +++ b/lib/gitlab/git/index.rb @@ -10,6 +10,7 @@ module Gitlab DEFAULT_MODE = 0o100644 ACTIONS = %w(create create_dir update move delete).freeze + ACTION_OPTIONS = %i(file_path previous_path content encoding).freeze attr_reader :repository, :raw_index @@ -20,6 +21,11 @@ module Gitlab delegate :read_tree, :get, to: :raw_index + def apply(action, options) + validate_action!(action) + public_send(action, options.slice(*ACTION_OPTIONS)) # rubocop:disable GitlabSecurity/PublicSend + end + def write_tree raw_index.write_tree(repository.rugged) end @@ -140,6 +146,12 @@ module Gitlab rescue Rugged::IndexError => e raise IndexError, e.message end + + def validate_action!(action) + unless ACTIONS.include?(action.to_s) + raise ArgumentError, "Unknown action '#{action}'" + end + end end end end diff --git a/lib/gitlab/git/operation_service.rb b/lib/gitlab/git/operation_service.rb index ef5bdbaf819..3fb0e2eed93 100644 --- a/lib/gitlab/git/operation_service.rb +++ b/lib/gitlab/git/operation_service.rb @@ -97,6 +97,11 @@ module Gitlab end end + def update_branch(branch_name, newrev, oldrev) + ref = Gitlab::Git::BRANCH_REF_PREFIX + branch_name + update_ref_in_hooks(ref, newrev, oldrev) + end + private # Returns [newrev, should_run_after_create, should_run_after_create_branch] diff --git a/lib/gitlab/git/remote_mirror.rb b/lib/gitlab/git/remote_mirror.rb new file mode 100644 index 00000000000..38e9d2a8554 --- /dev/null +++ b/lib/gitlab/git/remote_mirror.rb @@ -0,0 +1,75 @@ +module Gitlab + module Git + class RemoteMirror + def initialize(repository, ref_name) + @repository = repository + @ref_name = ref_name + end + + def update(only_branches_matching: [], only_tags_matching: []) + local_branches = refs_obj(@repository.local_branches, only_refs_matching: only_branches_matching) + remote_branches = refs_obj(@repository.remote_branches(@ref_name), only_refs_matching: only_branches_matching) + + updated_branches = changed_refs(local_branches, remote_branches) + push_branches(updated_branches.keys) if updated_branches.present? + + delete_refs(local_branches, remote_branches) + + local_tags = refs_obj(@repository.tags, only_refs_matching: only_tags_matching) + remote_tags = refs_obj(@repository.remote_tags(@ref_name), only_refs_matching: only_tags_matching) + + updated_tags = changed_refs(local_tags, remote_tags) + @repository.push_remote_branches(@ref_name, updated_tags.keys) if updated_tags.present? + + delete_refs(local_tags, remote_tags) + end + + private + + def refs_obj(refs, only_refs_matching: []) + refs.each_with_object({}) do |ref, refs| + next if only_refs_matching.present? && !only_refs_matching.include?(ref.name) + + refs[ref.name] = ref + end + end + + def changed_refs(local_refs, remote_refs) + local_refs.select do |ref_name, ref| + remote_ref = remote_refs[ref_name] + + remote_ref.nil? || ref.dereferenced_target != remote_ref.dereferenced_target + end + end + + def push_branches(branches) + default_branch, branches = branches.partition do |branch| + @repository.root_ref == branch + end + + # Push the default branch first so it works fine when remote mirror is empty. + branches.unshift(*default_branch) + + @repository.push_remote_branches(@ref_name, branches) + end + + def delete_refs(local_refs, remote_refs) + refs = refs_to_delete(local_refs, remote_refs) + + @repository.delete_remote_branches(@ref_name, refs.keys) if refs.present? + end + + def refs_to_delete(local_refs, remote_refs) + default_branch_id = @repository.commit.id + + remote_refs.select do |remote_ref_name, remote_ref| + next false if local_refs[remote_ref_name] # skip if branch or tag exist in local repo + + remote_ref_id = remote_ref.dereferenced_target.try(:id) + + remote_ref_id && @repository.rugged_is_ancestor?(remote_ref_id, default_branch_id) + end + end + end + end +end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 044c60caa05..283134e043e 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -21,6 +21,7 @@ module Gitlab REBASE_WORKTREE_PREFIX = 'rebase'.freeze SQUASH_WORKTREE_PREFIX = 'squash'.freeze GITALY_INTERNAL_URL = 'ssh://gitaly/internal.git'.freeze + GITLAB_PROJECTS_TIMEOUT = Gitlab.config.gitlab_shell.git_timeout NoRepository = Class.new(StandardError) InvalidBlobName = Class.new(StandardError) @@ -56,11 +57,12 @@ module Gitlab # Do nothing if hooks already exist unless real_local_hooks_path == File.realpath(global_hooks_path) - # Move the existing hooks somewhere safe - FileUtils.mv( - local_hooks_path, - "#{local_hooks_path}.old.#{Time.now.to_i}" - ) if File.exist?(local_hooks_path) + if File.exist?(local_hooks_path) + # Move the existing hooks somewhere safe + FileUtils.mv( + local_hooks_path, + "#{local_hooks_path}.old.#{Time.now.to_i}") + end # Create the hooks symlink FileUtils.ln_sf(global_hooks_path, local_hooks_path) @@ -82,7 +84,7 @@ module Gitlab # Rugged repo object attr_reader :rugged - attr_reader :storage, :gl_repository, :relative_path + attr_reader :gitlab_projects, :storage, :gl_repository, :relative_path # This initializer method is only used on the client side (gitlab-ce). # Gitaly-ruby uses a different initializer. @@ -92,6 +94,12 @@ module Gitlab @gl_repository = gl_repository storage_path = Gitlab.config.repositories.storages[@storage]['path'] + @gitlab_projects = Gitlab::Git::GitlabProjects.new( + storage_path, + relative_path, + global_hooks_path: Gitlab.config.gitlab_shell.hooks_path, + logger: Rails.logger + ) @path = File.join(storage_path, @relative_path) @name = @relative_path.split("/").last @attributes = Gitlab::Git::Attributes.new(path) @@ -125,7 +133,7 @@ module Gitlab end def exists? - Gitlab::GitalyClient.migrate(:repository_exists, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |enabled| + Gitlab::GitalyClient.migrate(:repository_exists) do |enabled| if enabled gitaly_repository_client.exists? else @@ -187,7 +195,7 @@ module Gitlab end def local_branches(sort_by: nil) - gitaly_migrate(:local_branches, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| + gitaly_migrate(:local_branches) do |is_enabled| if is_enabled gitaly_ref_client.local_branches(sort_by: sort_by) else @@ -490,11 +498,13 @@ module Gitlab end def count_commits(options) + count_commits_options = process_count_commits_options(options) + gitaly_migrate(:count_commits) do |is_enabled| if is_enabled - count_commits_by_gitaly(options) + count_commits_by_gitaly(count_commits_options) else - count_commits_by_shelling_out(options) + count_commits_by_shelling_out(count_commits_options) end end end @@ -532,8 +542,8 @@ module Gitlab end # Counts the amount of commits between `from` and `to`. - def count_commits_between(from, to) - count_commits(ref: "#{from}..#{to}") + def count_commits_between(from, to, options = {}) + count_commits(from: from, to: to, **options) end # Returns the SHA of the most recent common ancestor of +from+ and +to+ @@ -918,31 +928,23 @@ module Gitlab # If `mirror_refmap` is present the remote is set as mirror with that mapping def add_remote(remote_name, url, mirror_refmap: nil) - rugged.remotes.create(remote_name, url) - - set_remote_as_mirror(remote_name, refmap: mirror_refmap) if mirror_refmap - rescue Rugged::ConfigError - remote_update(remote_name, url: url) + gitaly_migrate(:remote_add_remote) do |is_enabled| + if is_enabled + gitaly_remote_client.add_remote(remote_name, url, mirror_refmap) + else + rugged_add_remote(remote_name, url, mirror_refmap) + end + end end def remove_remote(remote_name) - # When a remote is deleted all its remote refs are deleted too, but in - # the case of mirrors we map its refs (that would usualy go under - # [remote_name]/) to the top level namespace. We clean the mapping so - # those don't get deleted. - if rugged.config["remote.#{remote_name}.mirror"] - rugged.config.delete("remote.#{remote_name}.fetch") + gitaly_migrate(:remote_remove_remote) do |is_enabled| + if is_enabled + gitaly_remote_client.remove_remote(remote_name) + else + rugged_remove_remote(remote_name) + end end - - rugged.remotes.delete(remote_name) - true - rescue Rugged::ConfigError - false - end - - # Returns true if a remote exists. - def remote_exists?(name) - rugged.remotes[name].present? end # Update the specified remote using the values in the +options+ hash @@ -1161,23 +1163,13 @@ module Gitlab end def fetch_repository_as_mirror(repository) - remote_name = "tmp-#{SecureRandom.hex}" - - # Notice that this feature flag is not for `fetch_repository_as_mirror` - # as a whole but for the fetching mechanism (file path or gitaly-ssh). - url, env = gitaly_migrate(:fetch_internal) do |is_enabled| + gitaly_migrate(:remote_fetch_internal_remote) do |is_enabled| if is_enabled - repository = RemoteRepository.new(repository) unless repository.is_a?(RemoteRepository) - [GITALY_INTERNAL_URL, repository.fetch_env] + gitaly_remote_client.fetch_internal_remote(repository) else - [repository.path, nil] + rugged_fetch_repository_as_mirror(repository) end end - - add_remote(remote_name, url, mirror_refmap: :all_refs) - fetch_remote(remote_name, env: env) - ensure - remove_remote(remote_name) end def blob_at(sha, path) @@ -1185,7 +1177,7 @@ module Gitlab end # Items should be of format [[commit_id, path], [commit_id1, path1]] - def batch_blobs(items, blob_size_limit: nil) + def batch_blobs(items, blob_size_limit: Gitlab::Git::Blob::MAX_DATA_DISPLAY_SIZE) Gitlab::Git::Blob.batch(self, items, blob_size_limit: blob_size_limit) end @@ -1219,9 +1211,16 @@ module Gitlab rebase_path = worktree_path(REBASE_WORKTREE_PREFIX, rebase_id) env = git_env_for_user(user) + if remote_repository.is_a?(RemoteRepository) + env.merge!(remote_repository.fetch_env) + remote_repo_path = GITALY_INTERNAL_URL + else + remote_repo_path = remote_repository.path + end + with_worktree(rebase_path, branch, env: env) do run_git!( - %W(pull --rebase #{remote_repository.path} #{remote_branch}), + %W(pull --rebase #{remote_repo_path} #{remote_branch}), chdir: rebase_path, env: env ) @@ -1273,6 +1272,60 @@ module Gitlab fresh_worktree?(worktree_path(SQUASH_WORKTREE_PREFIX, squash_id)) end + def push_remote_branches(remote_name, branch_names, forced: true) + success = @gitlab_projects.push_branches(remote_name, GITLAB_PROJECTS_TIMEOUT, forced, branch_names) + + success || gitlab_projects_error + end + + def delete_remote_branches(remote_name, branch_names) + success = @gitlab_projects.delete_remote_branches(remote_name, branch_names) + + success || gitlab_projects_error + end + + def delete_remote_branches(remote_name, branch_names) + success = @gitlab_projects.delete_remote_branches(remote_name, branch_names) + + success || gitlab_projects_error + end + + # rubocop:disable Metrics/ParameterLists + def multi_action( + user, branch_name:, message:, actions:, + author_email: nil, author_name: nil, + start_branch_name: nil, start_repository: self) + + OperationService.new(user, self).with_branch( + branch_name, + start_branch_name: start_branch_name, + start_repository: start_repository + ) do |start_commit| + index = Gitlab::Git::Index.new(self) + parents = [] + + if start_commit + index.read_tree(start_commit.rugged_commit.tree) + parents = [start_commit.sha] + end + + actions.each { |opts| index.apply(opts.delete(:action), opts) } + + committer = user_to_committer(user) + author = Gitlab::Git.committer_hash(email: author_email, name: author_name) || committer + options = { + tree: index.write_tree, + message: message, + parents: parents, + author: author, + committer: committer + } + + create_commit(options) + end + end + # rubocop:enable Metrics/ParameterLists + def gitaly_repository Gitlab::GitalyClient::Util.repository(@storage, @relative_path, @gl_repository) end @@ -1297,6 +1350,14 @@ module Gitlab @gitaly_operation_client ||= Gitlab::GitalyClient::OperationService.new(self) end + def gitaly_remote_client + @gitaly_remote_client ||= Gitlab::GitalyClient::RemoteService.new(self) + end + + def gitaly_conflicts_client(our_commit_oid, their_commit_oid) + Gitlab::GitalyClient::ConflictsService.new(self, our_commit_oid, their_commit_oid) + end + def gitaly_migrate(method, status: Gitlab::GitalyClient::MigrationStatus::OPT_IN, &block) Gitlab::GitalyClient.migrate(method, status: status, &block) rescue GRPC::NotFound => e @@ -1435,6 +1496,26 @@ module Gitlab end end + def process_count_commits_options(options) + if options[:from] || options[:to] + ref = + if options[:left_right] # Compare with merge-base for left-right + "#{options[:from]}...#{options[:to]}" + else + "#{options[:from]}..#{options[:to]}" + end + + options.merge(ref: ref) + + elsif options[:ref] && options[:left_right] + from, to = options[:ref].match(/\A([^\.]*)\.{2,3}([^\.]*)\z/)[1..2] + + options.merge(from: from, to: to) + else + options + end + end + def log_using_shell?(options) options[:path].present? || options[:disable_walk] || @@ -1657,19 +1738,59 @@ module Gitlab end def count_commits_by_gitaly(options) - gitaly_commit_client.commit_count(options[:ref], options) + if options[:left_right] + from = options[:from] + to = options[:to] + + right_count = gitaly_commit_client + .commit_count("#{from}..#{to}", options) + left_count = gitaly_commit_client + .commit_count("#{to}..#{from}", options) + + [left_count, right_count] + else + gitaly_commit_client.commit_count(options[:ref], options) + end end def count_commits_by_shelling_out(options) + cmd = count_commits_shelling_command(options) + + raw_output = IO.popen(cmd) { |io| io.read } + + process_count_commits_raw_output(raw_output, options) + end + + def count_commits_shelling_command(options) cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} rev-list] cmd << "--after=#{options[:after].iso8601}" if options[:after] cmd << "--before=#{options[:before].iso8601}" if options[:before] + cmd << "--max-count=#{options[:max_count]}" if options[:max_count] + cmd << "--left-right" if options[:left_right] cmd += %W[--count #{options[:ref]}] cmd += %W[-- #{options[:path]}] if options[:path].present? + cmd + end - raw_output = IO.popen(cmd) { |io| io.read } + def process_count_commits_raw_output(raw_output, options) + if options[:left_right] + result = raw_output.scan(/\d+/).map(&:to_i) + + if result.sum != options[:max_count] + result + else # Reaching max count, right is not accurate + right_option = + process_count_commits_options(options + .except(:left_right, :from, :to) + .merge(ref: options[:to])) - raw_output.to_i + right = count_commits_by_shelling_out(right_option) + + [result.first, right] # left should be accurate in the first call + end + else + raw_output.to_i + end end def gitaly_ls_files(ref) @@ -1916,9 +2037,46 @@ module Gitlab raise ArgumentError, 'Invalid merge source' end + def rugged_add_remote(remote_name, url, mirror_refmap) + rugged.remotes.create(remote_name, url) + + set_remote_as_mirror(remote_name, refmap: mirror_refmap) if mirror_refmap + rescue Rugged::ConfigError + remote_update(remote_name, url: url) + end + + def rugged_remove_remote(remote_name) + # When a remote is deleted all its remote refs are deleted too, but in + # the case of mirrors we map its refs (that would usualy go under + # [remote_name]/) to the top level namespace. We clean the mapping so + # those don't get deleted. + if rugged.config["remote.#{remote_name}.mirror"] + rugged.config.delete("remote.#{remote_name}.fetch") + end + + rugged.remotes.delete(remote_name) + true + rescue Rugged::ConfigError + false + end + + def rugged_fetch_repository_as_mirror(repository) + remote_name = "tmp-#{SecureRandom.hex}" + repository = RemoteRepository.new(repository) unless repository.is_a?(RemoteRepository) + + add_remote(remote_name, GITALY_INTERNAL_URL, mirror_refmap: :all_refs) + fetch_remote(remote_name, env: repository.fetch_env) + ensure + remove_remote(remote_name) + end + def fetch_remote(remote_name = 'origin', env: nil) run_git(['fetch', remote_name], env: env).last.zero? end + + def gitlab_projects_error + raise CommandError, @gitlab_projects.output + end end end end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index b753ac46291..4507ea923b4 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -330,22 +330,6 @@ module Gitlab Google::Protobuf::Timestamp.new(seconds: t.to_i) end - def self.encode(s) - return "" if s.nil? - - s.dup.force_encoding(Encoding::ASCII_8BIT) - end - - def self.binary_stringio(s) - io = StringIO.new(s || '') - io.set_encoding(Encoding::ASCII_8BIT) - io - end - - def self.encode_repeated(a) - Google::Protobuf::RepeatedField.new(:bytes, a.map { |s| self.encode(s) } ) - end - # The default timeout on all Gitaly calls def self.default_timeout return 0 if Sidekiq.server? diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index fb3e27770b4..fed05bb6c64 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -1,6 +1,8 @@ module Gitlab module GitalyClient class CommitService + include Gitlab::EncodingHelper + # The ID of empty tree. # See http://stackoverflow.com/a/40884093/1856239 and https://github.com/git/git/blob/3ad8b5bf26362ac67c9020bf8c30eee54a84f56d/cache.h#L1011-L1012 EMPTY_TREE_ID = '4b825dc642cb6eb9a060e54bf8d69288fbee4904'.freeze @@ -13,7 +15,7 @@ module Gitlab def ls_files(revision) request = Gitaly::ListFilesRequest.new( repository: @gitaly_repo, - revision: GitalyClient.encode(revision) + revision: encode_binary(revision) ) response = GitalyClient.call(@repository.storage, :commit_service, :list_files, request, timeout: GitalyClient.medium_timeout) @@ -73,7 +75,7 @@ module Gitlab request = Gitaly::TreeEntryRequest.new( repository: @gitaly_repo, revision: ref, - path: GitalyClient.encode(path), + path: encode_binary(path), limit: limit.to_i ) @@ -98,8 +100,8 @@ module Gitlab def tree_entries(repository, revision, path) request = Gitaly::GetTreeEntriesRequest.new( repository: @gitaly_repo, - revision: GitalyClient.encode(revision), - path: path.present? ? GitalyClient.encode(path) : '.' + revision: encode_binary(revision), + path: path.present? ? encode_binary(path) : '.' ) response = GitalyClient.call(@repository.storage, :commit_service, :get_tree_entries, request, timeout: GitalyClient.medium_timeout) @@ -112,8 +114,8 @@ module Gitlab type: gitaly_tree_entry.type.downcase, mode: gitaly_tree_entry.mode.to_s(8), name: File.basename(gitaly_tree_entry.path), - path: GitalyClient.encode(gitaly_tree_entry.path), - flat_path: GitalyClient.encode(gitaly_tree_entry.flat_path), + path: encode_binary(gitaly_tree_entry.path), + flat_path: encode_binary(gitaly_tree_entry.flat_path), commit_id: gitaly_tree_entry.commit_oid ) end @@ -128,6 +130,7 @@ module Gitlab request.after = Google::Protobuf::Timestamp.new(seconds: options[:after].to_i) if options[:after].present? request.before = Google::Protobuf::Timestamp.new(seconds: options[:before].to_i) if options[:before].present? request.path = options[:path] if options[:path].present? + request.max_count = options[:max_count] if options[:max_count].present? GitalyClient.call(@repository.storage, :commit_service, :count_commits, request, timeout: GitalyClient.medium_timeout).count end @@ -135,8 +138,8 @@ module Gitlab def last_commit_for_path(revision, path) request = Gitaly::LastCommitForPathRequest.new( repository: @gitaly_repo, - revision: GitalyClient.encode(revision), - path: GitalyClient.encode(path.to_s) + revision: encode_binary(revision), + path: encode_binary(path.to_s) ) gitaly_commit = GitalyClient.call(@repository.storage, :commit_service, :last_commit_for_path, request, timeout: GitalyClient.fast_timeout).commit @@ -202,8 +205,8 @@ module Gitlab def raw_blame(revision, path) request = Gitaly::RawBlameRequest.new( repository: @gitaly_repo, - revision: GitalyClient.encode(revision), - path: GitalyClient.encode(path) + revision: encode_binary(revision), + path: encode_binary(path) ) response = GitalyClient.call(@repository.storage, :commit_service, :raw_blame, request, timeout: GitalyClient.medium_timeout) @@ -213,7 +216,7 @@ module Gitlab def find_commit(revision) request = Gitaly::FindCommitRequest.new( repository: @gitaly_repo, - revision: GitalyClient.encode(revision) + revision: encode_binary(revision) ) response = GitalyClient.call(@repository.storage, :commit_service, :find_commit, request, timeout: GitalyClient.medium_timeout) @@ -224,7 +227,7 @@ module Gitlab def patch(revision) request = Gitaly::CommitPatchRequest.new( repository: @gitaly_repo, - revision: GitalyClient.encode(revision) + revision: encode_binary(revision) ) response = GitalyClient.call(@repository.storage, :diff_service, :commit_patch, request, timeout: GitalyClient.medium_timeout) @@ -234,7 +237,7 @@ module Gitlab def commit_stats(revision) request = Gitaly::CommitStatsRequest.new( repository: @gitaly_repo, - revision: GitalyClient.encode(revision) + revision: encode_binary(revision) ) GitalyClient.call(@repository.storage, :commit_service, :commit_stats, request, timeout: GitalyClient.medium_timeout) end @@ -250,9 +253,9 @@ module Gitlab ) request.after = GitalyClient.timestamp(options[:after]) if options[:after] request.before = GitalyClient.timestamp(options[:before]) if options[:before] - request.revision = GitalyClient.encode(options[:ref]) if options[:ref] + request.revision = encode_binary(options[:ref]) if options[:ref] - request.paths = GitalyClient.encode_repeated(Array(options[:path])) if options[:path].present? + request.paths = encode_repeated(Array(options[:path])) if options[:path].present? response = GitalyClient.call(@repository.storage, :commit_service, :find_commits, request, timeout: GitalyClient.medium_timeout) @@ -264,7 +267,7 @@ module Gitlab enum = Enumerator.new do |y| shas.each_slice(20) do |revs| - request.shas = GitalyClient.encode_repeated(revs) + request.shas = encode_repeated(revs) y.yield request @@ -303,7 +306,7 @@ module Gitlab repository: @gitaly_repo, left_commit_id: from_id, right_commit_id: to_id, - paths: options.fetch(:paths, []).compact.map { |path| GitalyClient.encode(path) } + paths: options.fetch(:paths, []).compact.map { |path| encode_binary(path) } } end @@ -314,6 +317,10 @@ module Gitlab end end end + + def encode_repeated(a) + Google::Protobuf::RepeatedField.new(:bytes, a.map { |s| encode_binary(s) } ) + end end end end diff --git a/lib/gitlab/gitaly_client/conflicts_service.rb b/lib/gitlab/gitaly_client/conflicts_service.rb new file mode 100644 index 00000000000..40f032cf873 --- /dev/null +++ b/lib/gitlab/gitaly_client/conflicts_service.rb @@ -0,0 +1,95 @@ +module Gitlab + module GitalyClient + class ConflictsService + include Gitlab::EncodingHelper + + MAX_MSG_SIZE = 128.kilobytes.freeze + + def initialize(repository, our_commit_oid, their_commit_oid) + @gitaly_repo = repository.gitaly_repository + @repository = repository + @our_commit_oid = our_commit_oid + @their_commit_oid = their_commit_oid + end + + def list_conflict_files + request = Gitaly::ListConflictFilesRequest.new( + repository: @gitaly_repo, + our_commit_oid: @our_commit_oid, + their_commit_oid: @their_commit_oid + ) + response = GitalyClient.call(@repository.storage, :conflicts_service, :list_conflict_files, request) + + files_from_response(response).to_a + end + + def resolve_conflicts(target_repository, resolution, source_branch, target_branch) + reader = binary_stringio(resolution.files.to_json) + + req_enum = Enumerator.new do |y| + header = resolve_conflicts_request_header(target_repository, resolution, source_branch, target_branch) + y.yield Gitaly::ResolveConflictsRequest.new(header: header) + + until reader.eof? + chunk = reader.read(MAX_MSG_SIZE) + + y.yield Gitaly::ResolveConflictsRequest.new(files_json: chunk) + end + end + + response = GitalyClient.call(@repository.storage, :conflicts_service, :resolve_conflicts, req_enum, remote_storage: target_repository.storage) + + if response.resolution_error.present? + raise Gitlab::Git::Conflict::Resolver::ResolutionError, response.resolution_error + end + end + + private + + def resolve_conflicts_request_header(target_repository, resolution, source_branch, target_branch) + Gitaly::ResolveConflictsRequestHeader.new( + repository: @gitaly_repo, + our_commit_oid: @our_commit_oid, + target_repository: target_repository.gitaly_repository, + their_commit_oid: @their_commit_oid, + source_branch: source_branch, + target_branch: target_branch, + commit_message: resolution.commit_message, + user: Gitlab::Git::User.from_gitlab(resolution.user).to_gitaly + ) + end + + def files_from_response(response) + files = [] + + response.each do |msg| + msg.files.each do |gitaly_file| + if gitaly_file.header + files << file_from_gitaly_header(gitaly_file.header) + else + files.last.content << gitaly_file.content + end + end + end + + files + end + + def file_from_gitaly_header(header) + Gitlab::Git::Conflict::File.new( + Gitlab::GitalyClient::Util.git_repository(header.repository), + header.commit_oid, + conflict_from_gitaly_file_header(header), + '' + ) + end + + def conflict_from_gitaly_file_header(header) + { + ours: { path: header.our_path, mode: header.our_mode }, + theirs: { path: header.their_path } + } + end + end + end +end diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 400a4af363b..ae1753ff0ae 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -1,6 +1,8 @@ module Gitlab module GitalyClient class OperationService + include Gitlab::EncodingHelper + def initialize(repository) @gitaly_repo = repository.gitaly_repository @repository = repository @@ -9,7 +11,7 @@ module Gitlab def rm_tag(tag_name, user) request = Gitaly::UserDeleteTagRequest.new( repository: @gitaly_repo, - tag_name: GitalyClient.encode(tag_name), + tag_name: encode_binary(tag_name), user: Gitlab::Git::User.from_gitlab(user).to_gitaly ) @@ -24,9 +26,9 @@ module Gitlab request = Gitaly::UserCreateTagRequest.new( repository: @gitaly_repo, user: Gitlab::Git::User.from_gitlab(user).to_gitaly, - tag_name: GitalyClient.encode(tag_name), - target_revision: GitalyClient.encode(target), - message: GitalyClient.encode(message.to_s) + tag_name: encode_binary(tag_name), + target_revision: encode_binary(target), + message: encode_binary(message.to_s) ) response = GitalyClient.call(@repository.storage, :operation_service, :user_create_tag, request) @@ -44,9 +46,9 @@ module Gitlab def user_create_branch(branch_name, user, start_point) request = Gitaly::UserCreateBranchRequest.new( repository: @gitaly_repo, - branch_name: GitalyClient.encode(branch_name), + branch_name: encode_binary(branch_name), user: Gitlab::Git::User.from_gitlab(user).to_gitaly, - start_point: GitalyClient.encode(start_point) + start_point: encode_binary(start_point) ) response = GitalyClient.call(@repository.storage, :operation_service, :user_create_branch, request) @@ -64,7 +66,7 @@ module Gitlab def user_delete_branch(branch_name, user) request = Gitaly::UserDeleteBranchRequest.new( repository: @gitaly_repo, - branch_name: GitalyClient.encode(branch_name), + branch_name: encode_binary(branch_name), user: Gitlab::Git::User.from_gitlab(user).to_gitaly ) @@ -89,8 +91,8 @@ module Gitlab repository: @gitaly_repo, user: Gitlab::Git::User.from_gitlab(user).to_gitaly, commit_id: source_sha, - branch: GitalyClient.encode(target_branch), - message: GitalyClient.encode(message) + branch: encode_binary(target_branch), + message: encode_binary(message) ) ) @@ -99,6 +101,7 @@ module Gitlab request_enum.push(Gitaly::UserMergeBranchRequest.new(apply: true)) branch_update = response_enum.next.branch_update + return if branch_update.nil? raise Gitlab::Git::CommitError.new('failed to apply merge to branch') unless branch_update.commit_id.present? Gitlab::Git::OperationService::BranchUpdate.from_gitaly(branch_update) @@ -111,7 +114,7 @@ module Gitlab repository: @gitaly_repo, user: Gitlab::Git::User.from_gitlab(user).to_gitaly, commit_id: source_sha, - branch: GitalyClient.encode(target_branch) + branch: encode_binary(target_branch) ) branch_update = GitalyClient.call( @@ -152,9 +155,9 @@ module Gitlab repository: @gitaly_repo, user: Gitlab::Git::User.from_gitlab(user).to_gitaly, commit: commit.to_gitaly_commit, - branch_name: GitalyClient.encode(branch_name), - message: GitalyClient.encode(message), - start_branch_name: GitalyClient.encode(start_branch_name.to_s), + branch_name: encode_binary(branch_name), + message: encode_binary(message), + start_branch_name: encode_binary(start_branch_name.to_s), start_repository: start_repository.gitaly_repository ) diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index 066e4e183c0..5bce1009878 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -72,7 +72,7 @@ module Gitlab end def ref_exists?(ref_name) - request = Gitaly::RefExistsRequest.new(repository: @gitaly_repo, ref: GitalyClient.encode(ref_name)) + request = Gitaly::RefExistsRequest.new(repository: @gitaly_repo, ref: encode_binary(ref_name)) response = GitalyClient.call(@storage, :ref_service, :ref_exists, request) response.value rescue GRPC::InvalidArgument => e @@ -82,7 +82,7 @@ module Gitlab def find_branch(branch_name) request = Gitaly::FindBranchRequest.new( repository: @gitaly_repo, - name: GitalyClient.encode(branch_name) + name: encode_binary(branch_name) ) response = GitalyClient.call(@repository.storage, :ref_service, :find_branch, request) @@ -96,8 +96,8 @@ module Gitlab def create_branch(ref, start_point) request = Gitaly::CreateBranchRequest.new( repository: @gitaly_repo, - name: GitalyClient.encode(ref), - start_point: GitalyClient.encode(start_point) + name: encode_binary(ref), + start_point: encode_binary(start_point) ) response = GitalyClient.call(@repository.storage, :ref_service, :create_branch, request) @@ -121,7 +121,7 @@ module Gitlab def delete_branch(branch_name) request = Gitaly::DeleteBranchRequest.new( repository: @gitaly_repo, - name: GitalyClient.encode(branch_name) + name: encode_binary(branch_name) ) GitalyClient.call(@repository.storage, :ref_service, :delete_branch, request) diff --git a/lib/gitlab/gitaly_client/remote_service.rb b/lib/gitlab/gitaly_client/remote_service.rb new file mode 100644 index 00000000000..559a901b9a3 --- /dev/null +++ b/lib/gitlab/gitaly_client/remote_service.rb @@ -0,0 +1,41 @@ +module Gitlab + module GitalyClient + class RemoteService + def initialize(repository) + @repository = repository + @gitaly_repo = repository.gitaly_repository + @storage = repository.storage + end + + def add_remote(name, url, mirror_refmap) + request = Gitaly::AddRemoteRequest.new( + repository: @gitaly_repo, name: name, url: url, + mirror_refmap: mirror_refmap.to_s + ) + + GitalyClient.call(@storage, :remote_service, :add_remote, request) + end + + def remove_remote(name) + request = Gitaly::RemoveRemoteRequest.new(repository: @gitaly_repo, name: name) + + response = GitalyClient.call(@storage, :remote_service, :remove_remote, request) + + response.result + end + + def fetch_internal_remote(repository) + request = Gitaly::FetchInternalRemoteRequest.new( + repository: @gitaly_repo, + remote_repository: repository.gitaly_repository + ) + + response = GitalyClient.call(@storage, :remote_service, + :fetch_internal_remote, request, + remote_storage: repository.storage) + + response.result + end + end + end +end diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index c1f95396878..d43d80da960 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -1,6 +1,8 @@ module Gitlab module GitalyClient class RepositoryService + include Gitlab::EncodingHelper + def initialize(repository) @repository = repository @gitaly_repo = repository.gitaly_repository @@ -72,13 +74,29 @@ module Gitlab def find_merge_base(*revisions) request = Gitaly::FindMergeBaseRequest.new( repository: @gitaly_repo, - revisions: revisions.map { |r| GitalyClient.encode(r) } + revisions: revisions.map { |r| encode_binary(r) } ) response = GitalyClient.call(@storage, :repository_service, :find_merge_base, request) response.base.presence end + def fork_repository(source_repository) + request = Gitaly::CreateForkRequest.new( + repository: @gitaly_repo, + source_repository: source_repository.gitaly_repository + ) + + GitalyClient.call( + @storage, + :repository_service, + :create_fork, + request, + remote_storage: source_repository.storage, + timeout: GitalyClient.default_timeout + ) + end + def fetch_source_branch(source_repository, source_branch, local_ref) request = Gitaly::FetchSourceBranchRequest.new( repository: @gitaly_repo, diff --git a/lib/gitlab/gitaly_client/util.rb b/lib/gitlab/gitaly_client/util.rb index b1a033280b4..a8c6d478de8 100644 --- a/lib/gitlab/gitaly_client/util.rb +++ b/lib/gitlab/gitaly_client/util.rb @@ -12,12 +12,18 @@ module Gitlab Gitaly::Repository.new( storage_name: repository_storage, relative_path: relative_path, - gl_repository: gl_repository, + gl_repository: gl_repository.to_s, git_object_directory: git_object_directory.to_s, git_alternate_object_directories: git_alternate_object_directories ) end + def git_repository(gitaly_repository) + Gitlab::Git::Repository.new(gitaly_repository.storage_name, + gitaly_repository.relative_path, + gitaly_repository.gl_repository) + end + def gitlab_tag_from_gitaly_tag(repository, gitaly_tag) if gitaly_tag.target_commit.present? commit = Gitlab::Git::Commit.decorate(repository, gitaly_tag.target_commit) diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb index 337d225d081..5c5b170a3e0 100644 --- a/lib/gitlab/gitaly_client/wiki_service.rb +++ b/lib/gitlab/gitaly_client/wiki_service.rb @@ -3,6 +3,8 @@ require 'stringio' module Gitlab module GitalyClient class WikiService + include Gitlab::EncodingHelper + MAX_MSG_SIZE = 128.kilobytes.freeze def initialize(repository) @@ -13,12 +15,12 @@ module Gitlab def write_page(name, format, content, commit_details) request = Gitaly::WikiWritePageRequest.new( repository: @gitaly_repo, - name: GitalyClient.encode(name), + name: encode_binary(name), format: format.to_s, commit_details: gitaly_commit_details(commit_details) ) - strio = GitalyClient.binary_stringio(content) + strio = binary_stringio(content) enum = Enumerator.new do |y| until strio.eof? @@ -39,13 +41,13 @@ module Gitlab def update_page(page_path, title, format, content, commit_details) request = Gitaly::WikiUpdatePageRequest.new( repository: @gitaly_repo, - page_path: GitalyClient.encode(page_path), - title: GitalyClient.encode(title), + page_path: encode_binary(page_path), + title: encode_binary(title), format: format.to_s, commit_details: gitaly_commit_details(commit_details) ) - strio = GitalyClient.binary_stringio(content) + strio = binary_stringio(content) enum = Enumerator.new do |y| until strio.eof? @@ -63,7 +65,7 @@ module Gitlab def delete_page(page_path, commit_details) request = Gitaly::WikiDeletePageRequest.new( repository: @gitaly_repo, - page_path: GitalyClient.encode(page_path), + page_path: encode_binary(page_path), commit_details: gitaly_commit_details(commit_details) ) @@ -73,9 +75,9 @@ module Gitlab def find_page(title:, version: nil, dir: nil) request = Gitaly::WikiFindPageRequest.new( repository: @gitaly_repo, - title: GitalyClient.encode(title), - revision: GitalyClient.encode(version), - directory: GitalyClient.encode(dir) + title: encode_binary(title), + revision: encode_binary(version), + directory: encode_binary(dir) ) response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_find_page, request) @@ -102,8 +104,8 @@ module Gitlab def find_file(name, revision) request = Gitaly::WikiFindFileRequest.new( repository: @gitaly_repo, - name: GitalyClient.encode(name), - revision: GitalyClient.encode(revision) + name: encode_binary(name), + revision: encode_binary(revision) ) response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_find_file, request) @@ -158,9 +160,9 @@ module Gitlab def gitaly_commit_details(commit_details) Gitaly::WikiCommitDetails.new( - name: GitalyClient.encode(commit_details.name), - email: GitalyClient.encode(commit_details.email), - message: GitalyClient.encode(commit_details.message) + name: encode_binary(commit_details.name), + email: encode_binary(commit_details.email), + message: encode_binary(commit_details.message) ) end end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index dfcdfc307b6..9148d7571f2 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -21,6 +21,7 @@ module Gitlab gon.revision = Gitlab::REVISION gon.gitlab_logo = ActionController::Base.helpers.asset_path('gitlab_logo.png') gon.sprite_icons = IconsHelper.sprite_icon_path + gon.sprite_file_icons = IconsHelper.sprite_file_icons_path if current_user gon.current_user_id = current_user.id diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index 2066005dddc..af203ff711d 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -3,7 +3,7 @@ module Gitlab extend self # For every version update, the version history in import_export.md has to be kept up to date. - VERSION = '0.2.1'.freeze + VERSION = '0.2.2'.freeze FILENAME_LIMIT = 50 def export_path(relative_path:) diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index 0135b3c6f22..dd5d35feab9 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -15,6 +15,11 @@ module Gitlab execute(%W(#{git_bin_path} --git-dir=#{repo_path} bundle create #{bundle_path} --all)) end + def git_clone_bundle(repo_path:, bundle_path:) + execute(%W(#{git_bin_path} clone --bare -- #{bundle_path} #{repo_path})) + Gitlab::Git::Repository.create_hooks(repo_path, File.expand_path(Gitlab.config.gitlab_shell.hooks_path)) + end + def mkdir_p(path) FileUtils.mkdir_p(path, mode: DEFAULT_MODE) FileUtils.chmod(DEFAULT_MODE, path) diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index f2b193c79cb..2daed10f678 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -49,8 +49,8 @@ project_tree: - :author - events: - :push_event_payload - - :stages - - :statuses + - stages: + - :statuses - :auto_devops - :triggers - :pipeline_schedules diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index d7d1b05e8b9..05dbaf6322c 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -62,6 +62,7 @@ module Gitlab when :notes then setup_note when :project_label, :project_labels then setup_label when :milestone, :milestones then setup_milestone + when 'Ci::Pipeline' then setup_pipeline else @relation_hash['project_id'] = @project.id end @@ -112,9 +113,7 @@ module Gitlab @relation_hash.delete('trace') # old export files have trace @relation_hash.delete('token') - imported_object do |object| - object.commit_id = nil - end + imported_object elsif @relation_name == :merge_requests MergeRequestParser.new(@project, @relation_hash.delete('diff_head_sha'), imported_object, @relation_hash).parse! else @@ -182,8 +181,9 @@ module Gitlab end def imported_object - yield(existing_or_new_object) if block_given? - existing_or_new_object.importing = true if existing_or_new_object.respond_to?(:importing) + if existing_or_new_object.respond_to?(:importing) + existing_or_new_object.importing = true + end existing_or_new_object rescue ActiveRecord::RecordNotUnique @@ -211,6 +211,14 @@ module Gitlab @relation_hash['diff'] = @relation_hash.delete('utf8_diff') end + def setup_pipeline + @relation_hash.fetch('stages').each do |stage| + stage.statuses.each do |status| + status.pipeline = imported_object + end + end + end + def existing_or_new_object # Only find existing records to avoid mapping tables such as milestones # Otherwise always create the record, skipping the extra SELECT clause. diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index 32ca2809b2f..d0e5cfcfd3e 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -13,7 +13,7 @@ module Gitlab def restore return true unless File.exist?(@path_to_bundle) - gitlab_shell.import_repository(@project.repository_storage_path, @project.disk_path, @path_to_bundle) + git_clone_bundle(repo_path: @project.repository.path_to_repo, bundle_path: @path_to_bundle) rescue => e @shared.error(e) false diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index eeb03625479..60d5fa4d29a 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -7,6 +7,7 @@ module Gitlab module ImportSources ImportSource = Struct.new(:name, :title, :importer) + # We exclude `bare_repository` here as it has no import class associated ImportTable = [ ImportSource.new('github', 'GitHub', Gitlab::GithubImport::ParallelImporter), ImportSource.new('bitbucket', 'Bitbucket', Gitlab::BitbucketImport::Importer), diff --git a/lib/gitlab/kubernetes/helm.rb b/lib/gitlab/kubernetes/helm.rb index 407cdefc04d..0f0588b8b23 100644 --- a/lib/gitlab/kubernetes/helm.rb +++ b/lib/gitlab/kubernetes/helm.rb @@ -1,96 +1,8 @@ module Gitlab module Kubernetes - class Helm + module Helm HELM_VERSION = '2.7.0'.freeze NAMESPACE = 'gitlab-managed-apps'.freeze - INSTALL_DEPS = <<-EOS.freeze - set -eo pipefail - apk add -U ca-certificates openssl >/dev/null - wget -q -O - https://kubernetes-helm.storage.googleapis.com/helm-v${HELM_VERSION}-linux-amd64.tar.gz | tar zxC /tmp >/dev/null - mv /tmp/linux-amd64/helm /usr/bin/ - EOS - - InstallCommand = Struct.new(:name, :install_helm, :chart) do - def pod_name - "install-#{name}" - end - end - - def initialize(kubeclient) - @kubeclient = kubeclient - @namespace = Gitlab::Kubernetes::Namespace.new(NAMESPACE, kubeclient) - end - - def install(command) - @namespace.ensure_exists! - @kubeclient.create_pod(pod_resource(command)) - end - - ## - # Returns Pod phase - # - # https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase - # - # values: "Pending", "Running", "Succeeded", "Failed", "Unknown" - # - def installation_status(pod_name) - @kubeclient.get_pod(pod_name, @namespace.name).status.phase - end - - def installation_log(pod_name) - @kubeclient.get_pod_log(pod_name, @namespace.name).body - end - - def delete_installation_pod!(pod_name) - @kubeclient.delete_pod(pod_name, @namespace.name) - end - - private - - def pod_resource(command) - labels = { 'gitlab.org/action': 'install', 'gitlab.org/application': command.name } - metadata = { name: command.pod_name, namespace: @namespace.name, labels: labels } - container = { - name: 'helm', - image: 'alpine:3.6', - env: generate_pod_env(command), - command: %w(/bin/sh), - args: %w(-c $(COMMAND_SCRIPT)) - } - spec = { containers: [container], restartPolicy: 'Never' } - - ::Kubeclient::Resource.new(metadata: metadata, spec: spec) - end - - def generate_pod_env(command) - { - HELM_VERSION: HELM_VERSION, - TILLER_NAMESPACE: @namespace.name, - COMMAND_SCRIPT: generate_script(command) - }.map { |key, value| { name: key, value: value } } - end - - def generate_script(command) - [ - INSTALL_DEPS, - helm_init_command(command), - helm_install_command(command) - ].join("\n") - end - - def helm_init_command(command) - if command.install_helm - 'helm init >/dev/null' - else - 'helm init --client-only >/dev/null' - end - end - - def helm_install_command(command) - return if command.chart.nil? - - "helm install #{command.chart} --name #{command.name} --namespace #{@namespace.name} >/dev/null" - end end end end diff --git a/lib/gitlab/kubernetes/helm/api.rb b/lib/gitlab/kubernetes/helm/api.rb new file mode 100644 index 00000000000..737081ddc5b --- /dev/null +++ b/lib/gitlab/kubernetes/helm/api.rb @@ -0,0 +1,42 @@ +module Gitlab + module Kubernetes + module Helm + class Api + def initialize(kubeclient) + @kubeclient = kubeclient + @namespace = Gitlab::Kubernetes::Namespace.new(Gitlab::Kubernetes::Helm::NAMESPACE, kubeclient) + end + + def install(command) + @namespace.ensure_exists! + @kubeclient.create_pod(pod_resource(command)) + end + + ## + # Returns Pod phase + # + # https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase + # + # values: "Pending", "Running", "Succeeded", "Failed", "Unknown" + # + def installation_status(pod_name) + @kubeclient.get_pod(pod_name, @namespace.name).status.phase + end + + def installation_log(pod_name) + @kubeclient.get_pod_log(pod_name, @namespace.name).body + end + + def delete_installation_pod!(pod_name) + @kubeclient.delete_pod(pod_name, @namespace.name) + end + + private + + def pod_resource(command) + Gitlab::Kubernetes::Helm::Pod.new(command, @namespace.name, @kubeclient).generate + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/helm/install_command.rb b/lib/gitlab/kubernetes/helm/install_command.rb new file mode 100644 index 00000000000..8d8c441a4b1 --- /dev/null +++ b/lib/gitlab/kubernetes/helm/install_command.rb @@ -0,0 +1,53 @@ +module Gitlab + module Kubernetes + module Helm + class InstallCommand + attr_reader :name, :install_helm, :chart, :chart_values_file + + def initialize(name, install_helm: false, chart: false, chart_values_file: false) + @name = name + @install_helm = install_helm + @chart = chart + @chart_values_file = chart_values_file + end + + def pod_name + "install-#{name}" + end + + def generate_script(namespace_name) + [ + install_dps_command, + init_command, + complete_command(namespace_name) + ].join("\n") + end + + private + + def init_command + if install_helm + 'helm init >/dev/null' + else + 'helm init --client-only >/dev/null' + end + end + + def complete_command(namespace_name) + return unless chart + + "helm install #{chart} --name #{name} --namespace #{namespace_name} >/dev/null" + end + + def install_dps_command + <<~HEREDOC + set -eo pipefail + apk add -U ca-certificates openssl >/dev/null + wget -q -O - https://kubernetes-helm.storage.googleapis.com/helm-v#{Gitlab::Kubernetes::Helm::HELM_VERSION}-linux-amd64.tar.gz | tar zxC /tmp >/dev/null + mv /tmp/linux-amd64/helm /usr/bin/ + HEREDOC + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/helm/pod.rb b/lib/gitlab/kubernetes/helm/pod.rb new file mode 100644 index 00000000000..233f6bf6227 --- /dev/null +++ b/lib/gitlab/kubernetes/helm/pod.rb @@ -0,0 +1,69 @@ +module Gitlab + module Kubernetes + module Helm + class Pod + def initialize(command, namespace_name, kubeclient) + @command = command + @namespace_name = namespace_name + @kubeclient = kubeclient + end + + def generate + spec = { containers: [container_specification], restartPolicy: 'Never' } + if command.chart_values_file + generate_config_map + spec['volumes'] = volumes_specification + end + ::Kubeclient::Resource.new(metadata: metadata, spec: spec) + end + + private + + attr_reader :command, :namespace_name, :kubeclient + + def container_specification + container = { + name: 'helm', + image: 'alpine:3.6', + env: generate_pod_env(command), + command: %w(/bin/sh), + args: %w(-c $(COMMAND_SCRIPT)) + } + container[:volumeMounts] = volume_mounts_specification if command.chart_values_file + container + end + + def labels + { 'gitlab.org/action': 'install', 'gitlab.org/application': command.name } + end + + def metadata + { name: command.pod_name, namespace: namespace_name, labels: labels } + end + + def volume_mounts_specification + [{ name: 'config-volume', mountPath: '/etc/config' }] + end + + def volumes_specification + [{ name: 'config-volume', configMap: { name: 'values-config' } }] + end + + def generate_pod_env(command) + { + HELM_VERSION: Gitlab::Kubernetes::Helm::HELM_VERSION, + TILLER_NAMESPACE: namespace_name, + COMMAND_SCRIPT: command.generate_script(namespace_name) + }.map { |key, value| { name: key, value: value } } + end + + def generate_config_map + resource = ::Kubeclient::Resource.new + resource.metadata = { name: 'values-config', namespace: namespace_name } + resource.data = YAML.load_file(command.chart_values_file) + kubeclient.create_config_map(resource) + end + end + end + end +end diff --git a/lib/gitlab/ldap/adapter.rb b/lib/gitlab/ldap/adapter.rb index 0afaa2306b5..76863e77dc3 100644 --- a/lib/gitlab/ldap/adapter.rb +++ b/lib/gitlab/ldap/adapter.rb @@ -74,7 +74,7 @@ module Gitlab def user_options(fields, value, limit) options = { - attributes: Gitlab::LDAP::Person.ldap_attributes(config).compact.uniq, + attributes: Gitlab::LDAP::Person.ldap_attributes(config), base: config.base } diff --git a/lib/gitlab/ldap/config.rb b/lib/gitlab/ldap/config.rb index c8f19cd52d5..0d9a554fc18 100644 --- a/lib/gitlab/ldap/config.rb +++ b/lib/gitlab/ldap/config.rb @@ -148,7 +148,7 @@ module Gitlab def default_attributes { - 'username' => %w(uid userid sAMAccountName), + 'username' => %w(uid sAMAccountName userid), 'email' => %w(mail email userPrincipalName), 'name' => 'cn', 'first_name' => 'givenName', diff --git a/lib/gitlab/ldap/person.rb b/lib/gitlab/ldap/person.rb index 38d7a9ba2f5..e81cec6ba1a 100644 --- a/lib/gitlab/ldap/person.rb +++ b/lib/gitlab/ldap/person.rb @@ -6,6 +6,8 @@ module Gitlab # Source: http://ctogonewild.com/2009/09/03/bitmask-searches-in-ldap/ AD_USER_DISABLED = Net::LDAP::Filter.ex("userAccountControl:1.2.840.113556.1.4.803", "2") + InvalidEntryError = Class.new(StandardError) + attr_accessor :entry, :provider def self.find_by_uid(uid, adapter) @@ -29,11 +31,12 @@ module Gitlab def self.ldap_attributes(config) [ - 'dn', # Used in `dn` - config.uid, # Used in `uid` - *config.attributes['name'], # Used in `name` - *config.attributes['email'] # Used in `email` - ] + 'dn', + config.uid, + *config.attributes['name'], + *config.attributes['email'], + *config.attributes['username'] + ].compact.uniq end def self.normalize_dn(dn) @@ -60,6 +63,8 @@ module Gitlab Rails.logger.debug { "Instantiating #{self.class.name} with LDIF:\n#{entry.to_ldif}" } @entry = entry @provider = provider + + validate_entry end def name @@ -71,7 +76,13 @@ module Gitlab end def username - uid + username = attribute_value(:username) + + # Depending on the attribute, multiple values may + # be returned. We need only one for username. + # Ex. `uid` returns only one value but `mail` may + # return an array of multiple email addresses. + [username].flatten.first end def email @@ -104,6 +115,19 @@ module Gitlab entry.public_send(selected_attr) # rubocop:disable GitlabSecurity/PublicSend end + + def validate_entry + allowed_attrs = self.class.ldap_attributes(config).map(&:downcase) + + # Net::LDAP::Entry transforms keys to symbols. Change to strings to compare. + entry_attrs = entry.attribute_names.map { |n| n.to_s.downcase } + invalid_attrs = entry_attrs - allowed_attrs + + if invalid_attrs.any? + raise InvalidEntryError, + "#{self.class.name} initialized with Net::LDAP::Entry containing invalid attributes(s): #{invalid_attrs}" + end + end end end end diff --git a/lib/gitlab/metrics/method_call.rb b/lib/gitlab/metrics/method_call.rb index 329b07af5db..c2f9db56824 100644 --- a/lib/gitlab/metrics/method_call.rb +++ b/lib/gitlab/metrics/method_call.rb @@ -5,7 +5,7 @@ module Gitlab # Class for tracking timing information about method calls class MethodCall @@measurement_enabled_cache = Concurrent::AtomicBoolean.new(false) - @@measurement_enabled_cache_expires_at = Concurrent::AtomicFixnum.new(Time.now.to_i) + @@measurement_enabled_cache_expires_at = Concurrent::AtomicReference.new(Time.now.to_i) MUTEX = Mutex.new BASE_LABELS = { module: nil, method: nil }.freeze attr_reader :real_time, :cpu_time, :call_count, :labels diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 7037e2e61cc..ca48c6df602 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -82,7 +82,10 @@ module Gitlab end def issues - issues = IssuesFinder.new(current_user).execute.where(project_id: project_ids_relation) + issues = IssuesFinder.new(current_user).execute + unless default_project_filter + issues = issues.where(project_id: project_ids_relation) + end issues = if query =~ /#(\d+)\z/ diff --git a/lib/gitlab/seeder.rb b/lib/gitlab/seeder.rb index 30df7e4a831..94a481a0f2e 100644 --- a/lib/gitlab/seeder.rb +++ b/lib/gitlab/seeder.rb @@ -8,7 +8,7 @@ end module Gitlab class Seeder def self.quiet - mute_mailer unless Rails.env.test? + mute_mailer SeedFu.quiet = true diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb new file mode 100644 index 00000000000..d01213bb6e0 --- /dev/null +++ b/lib/gitlab/setup_helper.rb @@ -0,0 +1,61 @@ +module Gitlab + module SetupHelper + class << self + # We cannot create config.toml files for all possible Gitaly configuations. + # For instance, if Gitaly is running on another machine then it makes no + # sense to write a config.toml file on the current machine. This method will + # only generate a configuration for the most common and simplest case: when + # we have exactly one Gitaly process and we are sure it is running locally + # because it uses a Unix socket. + # For development and testing purposes, an extra storage is added to gitaly, + # which is not known to Rails, but must be explicitly stubbed. + def gitaly_configuration_toml(gitaly_dir, gitaly_ruby: true) + storages = [] + address = nil + + Gitlab.config.repositories.storages.each do |key, val| + if address + if address != val['gitaly_address'] + raise ArgumentError, "Your gitlab.yml contains more than one gitaly_address." + end + elsif URI(val['gitaly_address']).scheme != 'unix' + raise ArgumentError, "Automatic config.toml generation only supports 'unix:' addresses." + else + address = val['gitaly_address'] + end + + storages << { name: key, path: val['path'] } + end + + if Rails.env.test? + storages << { name: 'test_second_storage', path: Rails.root.join('tmp', 'tests', 'second_storage').to_s } + end + + config = { socket_path: address.sub(%r{\Aunix:}, ''), storage: storages } + config[:auth] = { token: 'secret' } if Rails.env.test? + config[:'gitaly-ruby'] = { dir: File.join(gitaly_dir, 'ruby') } if gitaly_ruby + config[:'gitlab-shell'] = { dir: Gitlab.config.gitlab_shell.path } + config[:bin_dir] = Gitlab.config.gitaly.client_path + + TOML.dump(config) + end + + # rubocop:disable Rails/Output + def create_gitaly_configuration(dir, force: false) + config_path = File.join(dir, 'config.toml') + FileUtils.rm_f(config_path) if force + + File.open(config_path, File::WRONLY | File::CREAT | File::EXCL) do |f| + f.puts gitaly_configuration_toml(dir) + end + rescue Errno::EEXIST + puts "Skipping config.toml generation:" + puts "A configuration file already exists." + rescue ArgumentError => e + puts "Skipping config.toml generation:" + puts e.message + end + # rubocop:enable Rails/Output + end + end +end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 9cdd3d22f18..564047bbd34 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -71,7 +71,6 @@ module Gitlab # Ex. # add_repository("/path/to/storage", "gitlab/gitlab-ci") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def add_repository(storage, name) relative_path = name.dup relative_path << '.git' unless relative_path.end_with?('.git') @@ -100,8 +99,12 @@ module Gitlab # Ex. # import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://gitlab.com/gitlab-org/gitlab-test.git") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/874 def import_repository(storage, name, url) + if url.start_with?('.', '/') + raise Error.new("don't use disk paths with import_repository: #{url.inspect}") + end + # The timeout ensures the subprocess won't hang forever cmd = gitlab_projects(storage, "#{name}.git") success = cmd.import_project(url, git_timeout) @@ -122,7 +125,6 @@ module Gitlab # Ex. # fetch_remote(my_repo, "upstream") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def fetch_remote(repository, remote, ssh_auth: nil, forced: false, no_tags: false) gitaly_migrate(:fetch_remote) do |is_enabled| if is_enabled @@ -142,7 +144,7 @@ module Gitlab # Ex. # mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/873 def mv_repository(storage, path, new_path) gitlab_projects(storage, "#{path}.git").mv_project("#{new_path}.git") end @@ -156,7 +158,7 @@ module Gitlab # Ex. # fork_repository("/path/to/forked_from/storage", "gitlab/gitlab-ci", "/path/to/forked_to/storage", "new-namespace/gitlab-ci") # - # Gitaly note: JV: not easy to migrate because this involves two Gitaly servers, not one. + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/817 def fork_repository(forked_from_storage, forked_from_disk_path, forked_to_storage, forked_to_disk_path) gitlab_projects(forked_from_storage, "#{forked_from_disk_path}.git") .fork_repository(forked_to_storage, "#{forked_to_disk_path}.git") @@ -170,7 +172,7 @@ module Gitlab # Ex. # remove_repository("/path/to/storage", "gitlab/gitlab-ci") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/873 def remove_repository(storage, name) gitlab_projects(storage, "#{name}.git").rm_project end @@ -221,7 +223,6 @@ module Gitlab # Ex. # add_namespace("/path/to/storage", "gitlab") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def add_namespace(storage, name) Gitlab::GitalyClient.migrate(:add_namespace) do |enabled| if enabled @@ -243,7 +244,6 @@ module Gitlab # Ex. # rm_namespace("/path/to/storage", "gitlab") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def rm_namespace(storage, name) Gitlab::GitalyClient.migrate(:remove_namespace) do |enabled| if enabled @@ -261,7 +261,6 @@ module Gitlab # Ex. # mv_namespace("/path/to/storage", "gitlab", "gitlabhq") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def mv_namespace(storage, old_name, new_name) Gitlab::GitalyClient.migrate(:rename_namespace) do |enabled| if enabled @@ -306,47 +305,6 @@ module Gitlab end end - # Push branch to remote repository - # - # storage - project's storage path - # project_name - project's disk path - # remote_name - remote name - # branch_names - remote branch names to push - # forced - should we use --force flag - # - # Ex. - # push_remote_branches('/path/to/storage', 'gitlab-org/gitlab-test' 'upstream', ['feature']) - # - def push_remote_branches(storage, project_name, remote_name, branch_names, forced: true) - cmd = gitlab_projects(storage, "#{project_name}.git") - - success = cmd.push_branches(remote_name, git_timeout, forced, branch_names) - - raise Error, cmd.output unless success - - success - end - - # Delete branch from remote repository - # - # storage - project's storage path - # project_name - project's disk path - # remote_name - remote name - # branch_names - remote branch names - # - # Ex. - # delete_remote_branches('/path/to/storage', 'gitlab-org/gitlab-test', 'upstream', ['feature']) - # - def delete_remote_branches(storage, project_name, remote_name, branch_names) - cmd = gitlab_projects(storage, "#{project_name}.git") - - success = cmd.delete_remote_branches(remote_name, branch_names) - - raise Error, cmd.output unless success - - success - end - protected def gitlab_shell_path diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index 11472ce6cce..6ced06a863d 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -57,11 +57,17 @@ module Gitlab } end - def highest_allowed_level + def allowed_levels restricted_levels = current_application_settings.restricted_visibility_levels - allowed_levels = self.values - restricted_levels - allowed_levels.max || PRIVATE + self.values - restricted_levels + end + + def closest_allowed_level(target_level) + highest_allowed_level = allowed_levels.select { |level| level <= target_level }.max + + # If all levels are restricted, fall back to PRIVATE + highest_allowed_level || PRIVATE end def allowed_for?(user, level) diff --git a/lib/google_api/cloud_platform/client.rb b/lib/google_api/cloud_platform/client.rb index b0563fb2d69..f05d001fd02 100644 --- a/lib/google_api/cloud_platform/client.rb +++ b/lib/google_api/cloud_platform/client.rb @@ -1,4 +1,6 @@ require 'google/apis/container_v1' +require 'google/apis/cloudbilling_v1' +require 'google/apis/cloudresourcemanager_v1' module GoogleApi module CloudPlatform @@ -40,6 +42,22 @@ module GoogleApi true end + def projects_list + service = Google::Apis::CloudresourcemanagerV1::CloudResourceManagerService.new + service.authorization = access_token + + service.fetch_all(items: :projects) do |token| + service.list_projects(page_token: token) + end + end + + def projects_get_billing_info(project_name) + service = Google::Apis::CloudbillingV1::CloudbillingService.new + service.authorization = access_token + + service.get_project_billing_info("projects/#{project_name}") + end + def projects_zones_clusters_get(project_id, zone, cluster_id) service = Google::Apis::ContainerV1::ContainerService.new service.authorization = access_token diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index dfade1f3885..903e84359cd 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -387,14 +387,8 @@ namespace :gitlab do namespace :repo do desc "GitLab | Check the integrity of the repositories managed by GitLab" task check: :environment do - Gitlab.config.repositories.storages.each do |name, repository_storage| - namespace_dirs = Dir.glob(File.join(repository_storage['path'], '*')) - - namespace_dirs.each do |namespace_dir| - repo_dirs = Dir.glob(File.join(namespace_dir, '*')) - repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) } - end - end + puts "This task is deprecated. Please use gitlab:git:fsck instead".color(:red) + Rake::Task["gitlab:git:fsck"].execute end end @@ -461,35 +455,4 @@ namespace :gitlab do puts "FAIL. Please update gitlab-shell to #{required_version} from #{current_version}".color(:red) end end - - def check_repo_integrity(repo_dir) - puts "\nChecking repo at #{repo_dir.color(:yellow)}" - - git_fsck(repo_dir) - check_config_lock(repo_dir) - check_ref_locks(repo_dir) - end - - def git_fsck(repo_dir) - puts "Running `git fsck`".color(:yellow) - system(*%W(#{Gitlab.config.git.bin_path} fsck), chdir: repo_dir) - end - - def check_config_lock(repo_dir) - config_exists = File.exist?(File.join(repo_dir, 'config.lock')) - config_output = config_exists ? 'yes'.color(:red) : 'no'.color(:green) - puts "'config.lock' file exists?".color(:yellow) + " ... #{config_output}" - end - - def check_ref_locks(repo_dir) - lock_files = Dir.glob(File.join(repo_dir, 'refs/heads/*.lock')) - if lock_files.present? - puts "Ref lock files exist:".color(:red) - lock_files.each do |lock_file| - puts " #{lock_file}" - end - else - puts "No ref lock files exist".color(:green) - end - end end diff --git a/lib/tasks/gitlab/git.rake b/lib/tasks/gitlab/git.rake index cf82134d97e..3f5dd2ae3b3 100644 --- a/lib/tasks/gitlab/git.rake +++ b/lib/tasks/gitlab/git.rake @@ -30,6 +30,20 @@ namespace :gitlab do end end + desc 'GitLab | Git | Check all repos integrity' + task fsck: :environment do + failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} fsck --name-objects --no-progress), "Checking integrity") do |repo| + check_config_lock(repo) + check_ref_locks(repo) + end + + if failures.empty? + puts "Done".color(:green) + else + output_failures(failures) + end + end + def perform_git_cmd(cmd, message) puts "Starting #{message} on all repositories" @@ -40,6 +54,8 @@ namespace :gitlab do else failures << repo end + + yield(repo) if block_given? end failures @@ -49,5 +65,24 @@ namespace :gitlab do puts "The following repositories reported errors:".color(:red) failures.each { |f| puts "- #{f}" } end + + def check_config_lock(repo_dir) + config_exists = File.exist?(File.join(repo_dir, 'config.lock')) + config_output = config_exists ? 'yes'.color(:red) : 'no'.color(:green) + + puts "'config.lock' file exists?".color(:yellow) + " ... #{config_output}" + end + + def check_ref_locks(repo_dir) + lock_files = Dir.glob(File.join(repo_dir, 'refs/heads/*.lock')) + + if lock_files.present? + puts "Ref lock files exist:".color(:red) + + lock_files.each { |lock_file| puts " #{lock_file}" } + else + puts "No ref lock files exist".color(:green) + end + end end end diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index 4d880c05f99..4507b841964 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -21,8 +21,8 @@ namespace :gitlab do command << 'BUNDLE_FLAGS=--no-deployment' if Rails.env.test? + Gitlab::SetupHelper.create_gitaly_configuration(args.dir) Dir.chdir(args.dir) do - create_gitaly_configuration # In CI we run scripts/gitaly-test-build instead of this command unless ENV['CI'].present? Bundler.with_original_env { run_command!(command) } @@ -39,60 +39,7 @@ namespace :gitlab do # Exclude gitaly-ruby configuration because that depends on the gitaly # installation directory. - puts gitaly_configuration_toml(gitaly_ruby: false) - end - - private - - # We cannot create config.toml files for all possible Gitaly configuations. - # For instance, if Gitaly is running on another machine then it makes no - # sense to write a config.toml file on the current machine. This method will - # only generate a configuration for the most common and simplest case: when - # we have exactly one Gitaly process and we are sure it is running locally - # because it uses a Unix socket. - # For development and testing purposes, an extra storage is added to gitaly, - # which is not known to Rails, but must be explicitly stubbed. - def gitaly_configuration_toml(gitaly_ruby: true) - storages = [] - address = nil - - Gitlab.config.repositories.storages.each do |key, val| - if address - if address != val['gitaly_address'] - raise ArgumentError, "Your gitlab.yml contains more than one gitaly_address." - end - elsif URI(val['gitaly_address']).scheme != 'unix' - raise ArgumentError, "Automatic config.toml generation only supports 'unix:' addresses." - else - address = val['gitaly_address'] - end - - storages << { name: key, path: val['path'] } - end - - if Rails.env.test? - storages << { name: 'test_second_storage', path: Rails.root.join('tmp', 'tests', 'second_storage').to_s } - end - - config = { socket_path: address.sub(%r{\Aunix:}, ''), storage: storages } - config[:auth] = { token: 'secret' } if Rails.env.test? - config[:'gitaly-ruby'] = { dir: File.join(Dir.pwd, 'ruby') } if gitaly_ruby - config[:'gitlab-shell'] = { dir: Gitlab.config.gitlab_shell.path } - config[:bin_dir] = Gitlab.config.gitaly.client_path - - TOML.dump(config) - end - - def create_gitaly_configuration - File.open("config.toml", File::WRONLY | File::CREAT | File::EXCL) do |f| - f.puts gitaly_configuration_toml - end - rescue Errno::EEXIST - puts "Skipping config.toml generation:" - puts "A configuration file already exists." - rescue ArgumentError => e - puts "Skipping config.toml generation:" - puts e.message + puts Gitlab::SetupHelper.gitaly_configuration_toml('', gitaly_ruby: false) end end end diff --git a/lib/tasks/gitlab/task_helpers.rb b/lib/tasks/gitlab/task_helpers.rb index 6723662703c..c1182af1014 100644 --- a/lib/tasks/gitlab/task_helpers.rb +++ b/lib/tasks/gitlab/task_helpers.rb @@ -130,7 +130,7 @@ module Gitlab def all_repos Gitlab.config.repositories.storages.each_value do |repository_storage| - IO.popen(%W(find #{repository_storage['path']} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find| + IO.popen(%W(find #{repository_storage['path']} -mindepth 2 -type d -name *.git)) do |find| find.each_line do |path| yield path.chomp end diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake index 9cc986535e1..c9e3eed82f2 100644 --- a/lib/tasks/migrate/setup_postgresql.rake +++ b/lib/tasks/migrate/setup_postgresql.rake @@ -6,6 +6,7 @@ require Rails.root.join('db/migrate/20161212142807_add_lower_path_index_to_route require Rails.root.join('db/migrate/20170317203554_index_routes_path_for_like') require Rails.root.join('db/migrate/20170724214302_add_lower_path_index_to_redirect_routes') require Rails.root.join('db/migrate/20170503185032_index_redirect_routes_path_for_like') +require Rails.root.join('db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb') desc 'GitLab | Sets up PostgreSQL' task setup_postgresql: :environment do @@ -15,4 +16,5 @@ task setup_postgresql: :environment do IndexRoutesPathForLike.new.up AddLowerPathIndexToRedirectRoutes.new.up IndexRedirectRoutesPathForLike.new.up + AddIndexOnNamespacesLowerName.new.up end |