diff options
Diffstat (limited to 'lib')
93 files changed, 1470 insertions, 1919 deletions
diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 13cfba728fa..4b223a391ae 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -45,6 +45,7 @@ module API present( paginate(::Kaminari.paginate_array(branches)), with: Entities::Branch, + current_user: current_user, project: user_project, merged_branch_names: merged_branch_names ) @@ -63,7 +64,7 @@ module API get do branch = find_branch!(params[:branch]) - present branch, with: Entities::Branch, project: user_project + present branch, with: Entities::Branch, current_user: current_user, project: user_project end end @@ -101,7 +102,7 @@ module API end if protected_branch.valid? - present branch, with: Entities::Branch, project: user_project + present branch, with: Entities::Branch, current_user: current_user, project: user_project else render_api_error!(protected_branch.errors.full_messages, 422) end @@ -121,7 +122,7 @@ module API protected_branch = user_project.protected_branches.find_by(name: branch.name) protected_branch&.destroy - present branch, with: Entities::Branch, project: user_project + present branch, with: Entities::Branch, current_user: current_user, project: user_project end desc 'Create branch' do @@ -140,6 +141,7 @@ module API if result[:status] == :success present result[:branch], with: Entities::Branch, + current_user: current_user, project: user_project else render_api_error!(result[:message], 400) diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 1cc8fcb8408..bb48a86fe9e 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -349,6 +349,10 @@ module API expose :developers_can_merge do |repo_branch, options| options[:project].protected_branches.developers_can?(:merge, repo_branch.name) end + + expose :can_push do |repo_branch, options| + Gitlab::UserAccess.new(options[:current_user], project: options[:project]).can_push_to_branch?(repo_branch.name) + end end class TreeObject < Grape::Entity diff --git a/lib/api/files.rb b/lib/api/files.rb index 1598d3c00b8..29d7489bd7c 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -5,6 +5,8 @@ module API # Prevents returning plain/text responses for files with .txt extension after_validation { content_type "application/json" } + helpers ::API::Helpers::HeadersHelpers + helpers do def commit_params(attrs) { @@ -40,6 +42,20 @@ module API } end + def blob_data + { + file_name: @blob.name, + file_path: @blob.path, + size: @blob.size, + encoding: "base64", + content_sha256: Digest::SHA256.hexdigest(@blob.data), + ref: params[:ref], + blob_id: @blob.id, + commit_id: @commit.id, + last_commit_id: @repo.last_commit_id_for_path(@commit.sha, params[:file_path]) + } + end + params :simple_file_params do requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' requires :branch, type: String, desc: 'Name of the branch to commit into. To create a new branch, also provide `start_branch`.' @@ -61,6 +77,17 @@ module API requires :id, type: String, desc: 'The project ID' end resource :projects, requirements: FILE_ENDPOINT_REQUIREMENTS do + desc 'Get raw file metadata from repository' + params do + requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' + requires :ref, type: String, desc: 'The name of branch, tag or commit' + end + head ":id/repository/files/:file_path/raw", requirements: FILE_ENDPOINT_REQUIREMENTS do + assign_file_vars! + + set_http_headers(blob_data) + end + desc 'Get raw file contents from the repository' params do requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' @@ -69,9 +96,22 @@ module API get ":id/repository/files/:file_path/raw", requirements: FILE_ENDPOINT_REQUIREMENTS do assign_file_vars! + set_http_headers(blob_data) + send_git_blob @repo, @blob end + desc 'Get file metadata from repository' + params do + requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' + requires :ref, type: String, desc: 'The name of branch, tag or commit' + end + head ":id/repository/files/:file_path", requirements: FILE_ENDPOINT_REQUIREMENTS do + assign_file_vars! + + set_http_headers(blob_data) + end + desc 'Get a file from the repository' params do requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' @@ -80,17 +120,11 @@ module API get ":id/repository/files/:file_path", requirements: FILE_ENDPOINT_REQUIREMENTS do assign_file_vars! - { - file_name: @blob.name, - file_path: @blob.path, - size: @blob.size, - encoding: "base64", - content: Base64.strict_encode64(@blob.data), - ref: params[:ref], - blob_id: @blob.id, - commit_id: @commit.id, - last_commit_id: @repo.last_commit_id_for_path(@commit.sha, params[:file_path]) - } + data = blob_data + + set_http_headers(data) + + data.merge(content: Base64.strict_encode64(@blob.data)) end desc 'Create new file in repository' diff --git a/lib/api/groups.rb b/lib/api/groups.rb index 03b6b30a0d8..f633dd88d06 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -32,7 +32,7 @@ module API optional :all_available, type: Boolean, desc: 'Show all group that you have access to' optional :search, type: String, desc: 'Search for a specific group' optional :owned, type: Boolean, default: false, desc: 'Limit by owned by authenticated user' - optional :order_by, type: String, values: %w[name path], default: 'name', desc: 'Order by name or path' + optional :order_by, type: String, values: %w[name path id], default: 'name', desc: 'Order by name, path or id' optional :sort, type: String, values: %w[asc desc], default: 'asc', desc: 'Sort by asc (ascending) or desc (descending)' use :pagination end @@ -46,7 +46,9 @@ module API groups = GroupsFinder.new(current_user, find_params).execute groups = groups.search(params[:search]) if params[:search].present? groups = groups.where.not(id: params[:skip_groups]) if params[:skip_groups].present? - groups = groups.reorder(params[:order_by] => params[:sort]) + order_options = { params[:order_by] => params[:sort] } + order_options["id"] ||= "asc" + groups = groups.reorder(order_options) groups end @@ -54,6 +56,8 @@ module API def find_group_projects(params) group = find_group!(params[:id]) projects = GroupProjectsFinder.new(group: group, current_user: current_user, params: project_finder_params).execute + projects = projects.with_issues_available_for_user(current_user) if params[:with_issues_enabled] + projects = projects.with_merge_requests_enabled if params[:with_merge_requests_enabled] projects = reorder_projects(projects) paginate(projects) end @@ -189,6 +193,8 @@ module API desc: 'Return only the ID, URL, name, and path of each project' optional :owned, type: Boolean, default: false, desc: 'Limit by owned by authenticated user' optional :starred, type: Boolean, default: false, desc: 'Limit by starred status' + optional :with_issues_enabled, type: Boolean, default: false, desc: 'Limit by enabled issues feature' + optional :with_merge_requests_enabled, type: Boolean, default: false, desc: 'Limit by enabled merge requests feature' use :pagination use :with_custom_attributes diff --git a/lib/api/helpers/headers_helpers.rb b/lib/api/helpers/headers_helpers.rb new file mode 100644 index 00000000000..cde51fccc62 --- /dev/null +++ b/lib/api/helpers/headers_helpers.rb @@ -0,0 +1,11 @@ +module API + module Helpers + module HeadersHelpers + def set_http_headers(header_data) + header_data.each do |key, value| + header "X-Gitlab-#{key.to_s.split('_').collect(&:capitalize).join('-')}", value + end + end + end + end +end diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index af7d2471b34..0f46bc4c98e 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -72,8 +72,8 @@ module API end params :merge_requests_params do - optional :state, type: String, values: %w[opened closed merged all], default: 'all', - desc: 'Return opened, closed, merged, or all merge requests' + optional :state, type: String, values: %w[opened closed locked merged all], default: 'all', + desc: 'Return opened, closed, locked, merged, or all merge requests' optional :order_by, type: String, values: %w[created_at updated_at], default: 'created_at', desc: 'Return merge requests ordered by `created_at` or `updated_at` fields.' optional :sort, type: String, values: %w[asc desc], default: 'desc', diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb index 8374a57edfa..5d33a13d035 100644 --- a/lib/api/pipelines.rb +++ b/lib/api/pipelines.rb @@ -31,7 +31,7 @@ module API get ':id/pipelines' do authorize! :read_pipeline, user_project - pipelines = PipelinesFinder.new(user_project, params).execute + pipelines = PipelinesFinder.new(user_project, current_user, params).execute present paginate(pipelines), with: Entities::PipelineBasic end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 3ef3680c5d9..b83da00502d 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -459,6 +459,23 @@ module API conflict!(error.message) end end + + desc 'Transfer a project to a new namespace' + params do + requires :namespace, type: String, desc: 'The ID or path of the new namespace' + end + put ":id/transfer" do + authorize! :change_namespace, user_project + + namespace = find_namespace!(params[:namespace]) + result = ::Projects::TransferService.new(user_project, current_user).execute(namespace) + + if result + present user_project, with: Entities::Project + else + render_api_error!("Failed to transfer project #{user_project.errors.messages}", 400) + end + end end end end diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index bb3fa99af38..33a9646ac3b 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -100,9 +100,10 @@ module API params do requires :from, type: String, desc: 'The commit, branch name, or tag name to start comparison' requires :to, type: String, desc: 'The commit, branch name, or tag name to stop comparison' + optional :straight, type: Boolean, desc: 'Comparison method, `true` for direct comparison between `from` and `to` (`from`..`to`), `false` to compare using merge base (`from`...`to`)', default: false end get ':id/repository/compare' do - compare = Gitlab::Git::Compare.new(user_project.repository.raw_repository, params[:from], params[:to]) + compare = Gitlab::Git::Compare.new(user_project.repository.raw_repository, params[:from], params[:to], straight: params[:straight]) present compare, with: Entities::Compare end diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 0119c5d6851..af762db517c 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -4,7 +4,6 @@ require_relative 'helper' module Backup class Repository include Backup::Helper - # rubocop:disable Metrics/AbcSize attr_reader :progress @@ -18,61 +17,26 @@ module Backup Project.find_each(batch_size: 1000) do |project| progress.print " * #{display_repo_path(project)} ... " - path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do - path_to_repo(project) - end - path_to_project_bundle = path_to_bundle(project) - - # Create namespace dir or hashed path if missing if project.hashed_storage?(:repository) FileUtils.mkdir_p(File.dirname(File.join(backup_repos_path, project.disk_path))) else FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.full_path)) if project.namespace end - if empty_repo?(project) - progress.puts "[SKIPPED]".color(:cyan) + if !empty_repo?(project) + backup_project(project) + progress.puts "[DONE]".color(:green) else - in_path(path_to_project_repo) do |dir| - FileUtils.mkdir_p(path_to_tars(project)) - cmd = %W(tar -cf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir}) - output, status = Gitlab::Popen.popen(cmd) - - unless status.zero? - progress_warn(project, cmd.join(' '), output) - end - end - - cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_project_repo} bundle create #{path_to_project_bundle} --all) - output, status = Gitlab::Popen.popen(cmd) - - if status.zero? - progress.puts "[DONE]".color(:green) - else - progress_warn(project, cmd.join(' '), output) - end + progress.puts "[SKIPPED]".color(:cyan) end wiki = ProjectWiki.new(project) - path_to_wiki_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do - path_to_repo(wiki) - end - path_to_wiki_bundle = path_to_bundle(wiki) - if File.exist?(path_to_wiki_repo) - progress.print " * #{display_repo_path(wiki)} ... " - - if empty_repo?(wiki) - progress.puts " [SKIPPED]".color(:cyan) - else - cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_wiki_repo} bundle create #{path_to_wiki_bundle} --all) - output, status = Gitlab::Popen.popen(cmd) - if status.zero? - progress.puts " [DONE]".color(:green) - else - progress_warn(wiki, cmd.join(' '), output) - end - end + if !empty_repo?(wiki) + backup_project(wiki) + progress.puts "[DONE] Wiki".color(:green) + else + progress.puts "[SKIPPED] Wiki".color(:cyan) end end end @@ -83,8 +47,40 @@ module Backup end end + def backup_project(project) + gitaly_migrate(:repository_backup, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| + if is_enabled + backup_project_gitaly(project) + else + backup_project_local(project) + end + end + + backup_custom_hooks(project) + rescue => e + progress_warn(project, e, 'Failed to backup repo') + end + + def backup_project_gitaly(project) + path_to_project_bundle = path_to_bundle(project) + Gitlab::GitalyClient::RepositoryService.new(project.repository) + .create_bundle(path_to_project_bundle) + end + + def backup_project_local(project) + path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do + path_to_repo(project) + end + + path_to_project_bundle = path_to_bundle(project) + + cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_project_repo} bundle create #{path_to_project_bundle} --all) + output, status = Gitlab::Popen.popen(cmd) + progress_warn(project, cmd.join(' '), output) unless status.zero? + end + def delete_all_repositories(name, repository_storage) - gitaly_migrate(:delete_all_repositories) do |is_enabled| + gitaly_migrate(:delete_all_repositories, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| if is_enabled Gitlab::GitalyClient::StorageService.new(name).delete_all_repositories else @@ -97,8 +93,6 @@ module Backup path = repository_storage.legacy_disk_path return unless File.exist?(path) - # Move all files in the existing repos directory except . and .. to - # repositories.old.<timestamp> directory bk_repos_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}-repositories.old." + Time.now.to_i.to_s) FileUtils.mkdir_p(bk_repos_path, mode: 0700) files = Dir.glob(File.join(path, "*"), File::FNM_DOTMATCH) - [File.join(path, "."), File.join(path, "..")] @@ -129,13 +123,47 @@ module Backup .restore_custom_hooks(custom_hooks_path) end + def local_backup_custom_hooks(project) + in_path(path_to_tars(project)) do |dir| + path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do + path_to_repo(project) + end + break unless File.exist?(File.join(path_to_project_repo, dir)) + + FileUtils.mkdir_p(path_to_tars(project)) + cmd = %W(tar -cf #{path_to_tars(project, dir)} -c #{path_to_project_repo} #{dir}) + output, status = Gitlab::Popen.popen(cmd) + + unless status.zero? + progress_warn(project, cmd.join(' '), output) + end + end + end + + def gitaly_backup_custom_hooks(project) + FileUtils.mkdir_p(path_to_tars(project)) + custom_hooks_path = path_to_tars(project, 'custom_hooks') + Gitlab::GitalyClient::RepositoryService.new(project.repository) + .backup_custom_hooks(custom_hooks_path) + end + + def backup_custom_hooks(project) + gitaly_migrate(:backup_custom_hooks, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| + if is_enabled + gitaly_backup_custom_hooks(project) + else + local_backup_custom_hooks(project) + end + end + end + def restore_custom_hooks(project) in_path(path_to_tars(project)) do |dir| - gitaly_migrate(:restore_custom_hooks) do |is_enabled| + gitaly_migrate(:restore_custom_hooks, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| if is_enabled - local_restore_custom_hooks(project, dir) - else gitaly_restore_custom_hooks(project, dir) + else + local_restore_custom_hooks(project, dir) end end end @@ -178,6 +206,8 @@ module Backup progress.print " * #{wiki.full_path} ... " begin wiki.repository.create_from_bundle(path_to_wiki_bundle) + restore_custom_hooks(wiki) + progress.puts "[DONE]".color(:green) rescue => e progress.puts "[Failed] restoring #{wiki.full_path} wiki".color(:red) @@ -186,7 +216,6 @@ module Backup end end end - # rubocop:enable Metrics/AbcSize protected @@ -224,9 +253,7 @@ module Backup def prepare FileUtils.rm_rf(backup_repos_path) - # Ensure the parent dir of backup_repos_path exists FileUtils.mkdir_p(Gitlab.config.backup.path) - # Fail if somebody raced to create backup_repos_path before us FileUtils.mkdir(backup_repos_path, mode: 0700) end @@ -242,7 +269,6 @@ module Backup end def empty_repo?(project_or_wiki) - # Protect against stale caches project_or_wiki.repository.expire_emptiness_caches project_or_wiki.repository.empty? end diff --git a/lib/banzai/filter/blockquote_fence_filter.rb b/lib/banzai/filter/blockquote_fence_filter.rb index d2c4b1e4d76..fbfcd72c916 100644 --- a/lib/banzai/filter/blockquote_fence_filter.rb +++ b/lib/banzai/filter/blockquote_fence_filter.rb @@ -10,7 +10,7 @@ module Banzai ^``` .+? - \n```$ + \n```\ *$ ) | (?<html> @@ -19,9 +19,9 @@ module Banzai # Anything, including `>>>` blocks which are ignored by this filter # </tag> - ^<[^>]+?>\n + ^<[^>]+?>\ *\n .+? - \n<\/[^>]+?>$ + \n<\/[^>]+?>\ *$ ) | (?: @@ -30,14 +30,14 @@ module Banzai # Anything, including code and HTML blocks # >>> - ^>>>\n + ^>>>\ *\n (?<quote> (?: # Any character that doesn't introduce a code or HTML block (?! ^``` | - ^<[^>]+?>\n + ^<[^>]+?>\ *\n ) . | @@ -48,7 +48,7 @@ module Banzai \g<html> )+? ) - \n>>>$ + \n>>>\ *$ ) }mx.freeze diff --git a/lib/banzai/filter/emoji_filter.rb b/lib/banzai/filter/emoji_filter.rb index e1261e7bbbe..4eccd9d5ed5 100644 --- a/lib/banzai/filter/emoji_filter.rb +++ b/lib/banzai/filter/emoji_filter.rb @@ -3,10 +3,6 @@ module Banzai # HTML filter that replaces :emoji: and unicode with images. # # Based on HTML::Pipeline::EmojiFilter - # - # Context options: - # :asset_root - # :asset_host class EmojiFilter < HTML::Pipeline::Filter IGNORED_ANCESTOR_TAGS = %w(pre code tt).to_set diff --git a/lib/banzai/filter/gollum_tags_filter.rb b/lib/banzai/filter/gollum_tags_filter.rb index 4bc82ecb4d6..bb9f488cd87 100644 --- a/lib/banzai/filter/gollum_tags_filter.rb +++ b/lib/banzai/filter/gollum_tags_filter.rb @@ -56,10 +56,12 @@ module Banzai # Pattern to match allowed image extensions ALLOWED_IMAGE_EXTENSIONS = /.+(jpg|png|gif|svg|bmp)\z/i.freeze + # Do not perform linking inside these tags. + IGNORED_ANCESTOR_TAGS = %w(pre code tt).to_set + def call doc.search(".//text()").each do |node| - # Do not perform linking inside <code> blocks - next unless node.ancestors('code').empty? + next if has_ancestor?(node, IGNORED_ANCESTOR_TAGS) # A Gollum ToC tag is `[[_TOC_]]`, but due to MarkdownFilter running # before this one, it will be converted into `[[<em>TOC</em>]]`, so it diff --git a/lib/banzai/filter/merge_request_reference_filter.rb b/lib/banzai/filter/merge_request_reference_filter.rb index 5cbdb01c130..10c40568006 100644 --- a/lib/banzai/filter/merge_request_reference_filter.rb +++ b/lib/banzai/filter/merge_request_reference_filter.rb @@ -25,7 +25,10 @@ module Banzai extras = super if commit_ref = object_link_commit_ref(object, matches) - return extras.unshift(commit_ref) + klass = reference_class(:commit, tooltip: false) + commit_ref_tag = %(<span class="#{klass}">#{commit_ref}</span>) + + return extras.unshift(commit_ref_tag) end path = matches[:path] if matches.names.include?("path") diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index 2f023f4f242..2411dd2cdfc 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -65,8 +65,12 @@ module Banzai context[:skip_project_check] end - def reference_class(type) - "gfm gfm-#{type} has-tooltip" + def reference_class(type, tooltip: true) + gfm_klass = "gfm gfm-#{type}" + + return gfm_klass unless tooltip + + "#{gfm_klass} has-tooltip" end # Ensure that a :project key exists in context diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index 6786b9d07b6..8275bb9e149 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -4,31 +4,25 @@ module Banzai # # Extends HTML::Pipeline::SanitizationFilter with a custom whitelist. class SanitizationFilter < HTML::Pipeline::SanitizationFilter + include Gitlab::Utils::StrongMemoize + UNSAFE_PROTOCOLS = %w(data javascript vbscript).freeze TABLE_ALIGNMENT_PATTERN = /text-align: (?<alignment>center|left|right)/ def whitelist - whitelist = super - - customize_whitelist(whitelist) - - whitelist + strong_memoize(:whitelist) do + customize_whitelist(super.dup) + end end private - def customized?(transformers) - transformers.last.source_location[0] == __FILE__ - end - def customize_whitelist(whitelist) - # Only push these customizations once - return if customized?(whitelist[:transformers]) - - # Allow table alignment; we whitelist specific style properties in a + # Allow table alignment; we whitelist specific text-align values in a # transformer below whitelist[:attributes]['th'] = %w(style) whitelist[:attributes]['td'] = %w(style) + whitelist[:css] = { properties: ['text-align'] } # Allow span elements whitelist[:elements].push('span') diff --git a/lib/banzai/filter/table_of_contents_filter.rb b/lib/banzai/filter/table_of_contents_filter.rb index 97244159985..b32660a8341 100644 --- a/lib/banzai/filter/table_of_contents_filter.rb +++ b/lib/banzai/filter/table_of_contents_filter.rb @@ -92,7 +92,7 @@ module Banzai def text return '' unless node - @text ||= node.text + @text ||= EscapeUtils.escape_html(node.text) end private diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index a1f24e8b093..0d9b874ef85 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -44,11 +44,7 @@ module Banzai def self.transform_context(context) context[:only_path] = true unless context.key?(:only_path) - context.merge( - # EmojiFilter - asset_host: Gitlab::Application.config.asset_host, - asset_root: Gitlab.config.gitlab.base_url - ) + context end end end diff --git a/lib/gitaly/server.rb b/lib/gitaly/server.rb index 605e93022e7..2760211fee8 100644 --- a/lib/gitaly/server.rb +++ b/lib/gitaly/server.rb @@ -22,6 +22,18 @@ module Gitaly server_version == Gitlab::GitalyClient.expected_server_version end + def read_writeable? + readable? && writeable? + end + + def readable? + storage_status&.readable + end + + def writeable? + storage_status&.writeable + end + def address Gitlab::GitalyClient.address(@storage) rescue RuntimeError => e @@ -30,13 +42,17 @@ module Gitaly private + def storage_status + @storage_status ||= info.storage_statuses.find { |s| s.storage_name == storage } + end + def info @info ||= begin Gitlab::GitalyClient::ServerService.new(@storage).info rescue GRPC::Unavailable, GRPC::GRPC::DeadlineExceeded # This will show the server as being out of date - Gitaly::ServerInfoResponse.new(git_version: '', server_version: '') + Gitaly::ServerInfoResponse.new(git_version: '', server_version: '', storage_statuses: []) end end end diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index 6c5d0788a0a..e7283b2f9e8 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -74,6 +74,10 @@ module Gitlab gl_user end + def bypass_two_factor? + false + end + protected def should_save? diff --git a/lib/gitlab/auth/saml/auth_hash.rb b/lib/gitlab/auth/saml/auth_hash.rb index c345a7e3f6c..3bc5e2864df 100644 --- a/lib/gitlab/auth/saml/auth_hash.rb +++ b/lib/gitlab/auth/saml/auth_hash.rb @@ -6,6 +6,17 @@ module Gitlab Array.wrap(get_raw(Gitlab::Auth::Saml::Config.groups)) end + def authn_context + response_object = auth_hash.extra[:response_object] + return nil if response_object.blank? + + document = response_object.decrypted_document + document ||= response_object.document + return nil if document.blank? + + extract_authn_context(document) + end + private def get_raw(key) @@ -13,6 +24,10 @@ module Gitlab # otherwise just the first value is returned auth_hash.extra[:raw_info].all[key] end + + def extract_authn_context(document) + REXML::XPath.first(document, "//saml:AuthnStatement/saml:AuthnContext/saml:AuthnContextClassRef/text()").to_s + end end end end diff --git a/lib/gitlab/auth/saml/config.rb b/lib/gitlab/auth/saml/config.rb index 5fa9581f837..625dab7c6f4 100644 --- a/lib/gitlab/auth/saml/config.rb +++ b/lib/gitlab/auth/saml/config.rb @@ -7,6 +7,10 @@ module Gitlab Gitlab::Auth::OAuth::Provider.config_for('saml') end + def upstream_two_factor_authn_contexts + options.args[:upstream_two_factor_authn_contexts] + end + def groups options[:groups_attribute] end diff --git a/lib/gitlab/auth/saml/user.rb b/lib/gitlab/auth/saml/user.rb index b8c84c37cd5..6c3b75f3eb0 100644 --- a/lib/gitlab/auth/saml/user.rb +++ b/lib/gitlab/auth/saml/user.rb @@ -34,6 +34,10 @@ module Gitlab gl_user.changed? || gl_user.identities.any?(&:changed?) end + def bypass_two_factor? + saml_config.upstream_two_factor_authn_contexts&.include?(auth_hash.authn_context) + end + protected def saml_config diff --git a/lib/gitlab/background_migration/cleanup_concurrent_rename.rb b/lib/gitlab/background_migration/cleanup_concurrent_rename.rb new file mode 100644 index 00000000000..d3f366f3480 --- /dev/null +++ b/lib/gitlab/background_migration/cleanup_concurrent_rename.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Background migration for cleaning up a concurrent column rename. + class CleanupConcurrentRename < CleanupConcurrentSchemaChange + RESCHEDULE_DELAY = 10.minutes + + def cleanup_concurrent_schema_change(table, old_column, new_column) + cleanup_concurrent_column_rename(table, old_column, new_column) + end + end + end +end diff --git a/lib/gitlab/background_migration/cleanup_concurrent_schema_change.rb b/lib/gitlab/background_migration/cleanup_concurrent_schema_change.rb new file mode 100644 index 00000000000..54f77f184d5 --- /dev/null +++ b/lib/gitlab/background_migration/cleanup_concurrent_schema_change.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Base class for cleaning up concurrent schema changes. + class CleanupConcurrentSchemaChange + include Database::MigrationHelpers + + # table - The name of the table the migration is performed for. + # old_column - The name of the old (to drop) column. + # new_column - The name of the new column. + def perform(table, old_column, new_column) + return unless column_exists?(table, new_column) + + rows_to_migrate = define_model_for(table) + .where(new_column => nil) + .where + .not(old_column => nil) + + if rows_to_migrate.any? + BackgroundMigrationWorker.perform_in( + RESCHEDULE_DELAY, + self.class.name, + [table, old_column, new_column] + ) + else + cleanup_concurrent_schema_change(table, old_column, new_column) + end + end + + # These methods are necessary so we can re-use the migration helpers in + # this class. + def connection + ActiveRecord::Base.connection + end + + def method_missing(name, *args, &block) + connection.__send__(name, *args, &block) # rubocop: disable GitlabSecurity/PublicSend + end + + def respond_to_missing?(*args) + connection.respond_to?(*args) || super + end + + def define_model_for(table) + Class.new(ActiveRecord::Base) do + self.table_name = table + end + end + end + end +end diff --git a/lib/gitlab/background_migration/cleanup_concurrent_type_change.rb b/lib/gitlab/background_migration/cleanup_concurrent_type_change.rb index de622f657b2..48411095dbb 100644 --- a/lib/gitlab/background_migration/cleanup_concurrent_type_change.rb +++ b/lib/gitlab/background_migration/cleanup_concurrent_type_change.rb @@ -2,52 +2,12 @@ module Gitlab module BackgroundMigration - # Background migration for cleaning up a concurrent column rename. - class CleanupConcurrentTypeChange - include Database::MigrationHelpers - + # Background migration for cleaning up a concurrent column type changeb. + class CleanupConcurrentTypeChange < CleanupConcurrentSchemaChange RESCHEDULE_DELAY = 10.minutes - # table - The name of the table the migration is performed for. - # old_column - The name of the old (to drop) column. - # new_column - The name of the new column. - def perform(table, old_column, new_column) - return unless column_exists?(:issues, new_column) - - rows_to_migrate = define_model_for(table) - .where(new_column => nil) - .where - .not(old_column => nil) - - if rows_to_migrate.any? - BackgroundMigrationWorker.perform_in( - RESCHEDULE_DELAY, - 'CleanupConcurrentTypeChange', - [table, old_column, new_column] - ) - else - cleanup_concurrent_column_type_change(table, old_column) - end - end - - # These methods are necessary so we can re-use the migration helpers in - # this class. - def connection - ActiveRecord::Base.connection - end - - def method_missing(name, *args, &block) - connection.__send__(name, *args, &block) # rubocop: disable GitlabSecurity/PublicSend - end - - def respond_to_missing?(*args) - connection.respond_to?(*args) || super - end - - def define_model_for(table) - Class.new(ActiveRecord::Base) do - self.table_name = table - end + def cleanup_concurrent_schema_change(table, old_column, new_column) + cleanup_concurrent_column_type_change(table, old_column) end end end diff --git a/lib/gitlab/background_migration/delete_diff_files.rb b/lib/gitlab/background_migration/delete_diff_files.rb new file mode 100644 index 00000000000..0b785e1b056 --- /dev/null +++ b/lib/gitlab/background_migration/delete_diff_files.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true +# rubocop:disable Metrics/AbcSize +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + class DeleteDiffFiles + def perform(merge_request_diff_id) + merge_request_diff = MergeRequestDiff.find_by(id: merge_request_diff_id) + + return unless merge_request_diff + return unless should_delete_diff_files?(merge_request_diff) + + MergeRequestDiff.transaction do + merge_request_diff.update_column(:state, 'without_files') + + # explain (analyze, buffers) when deleting 453 diff files: + # + # Delete on merge_request_diff_files (cost=0.57..8487.35 rows=4846 width=6) (actual time=43.265..43.265 rows=0 loops=1) + # Buffers: shared hit=2043 read=259 dirtied=254 + # -> Index Scan using index_merge_request_diff_files_on_mr_diff_id_and_order on merge_request_diff_files (cost=0.57..8487.35 rows=4846 width=6) (actu + # al time=0.466..26.317 rows=453 loops=1) + # Index Cond: (merge_request_diff_id = 463448) + # Buffers: shared hit=17 read=84 + # Planning time: 0.107 ms + # Execution time: 43.287 ms + # + MergeRequestDiffFile.where(merge_request_diff_id: merge_request_diff.id).delete_all + end + end + + private + + def should_delete_diff_files?(merge_request_diff) + return false if merge_request_diff.state == 'without_files' + + merge_request = merge_request_diff.merge_request + + return false unless merge_request.state == 'merged' + return false if merge_request_diff.id == merge_request.latest_merge_request_diff_id + + true + end + end + end +end diff --git a/lib/gitlab/cache/request_cache.rb b/lib/gitlab/cache/request_cache.rb index ecc85f847d4..671b8e7e1b1 100644 --- a/lib/gitlab/cache/request_cache.rb +++ b/lib/gitlab/cache/request_cache.rb @@ -1,41 +1,6 @@ module Gitlab module Cache - # This module provides a simple way to cache values in RequestStore, - # and the cache key would be based on the class name, method name, - # optionally customized instance level values, optionally customized - # method level values, and optional method arguments. - # - # A simple example: - # - # class UserAccess - # extend Gitlab::Cache::RequestCache - # - # request_cache_key do - # [user&.id, project&.id] - # end - # - # request_cache def can_push_to_branch?(ref) - # # ... - # end - # end - # - # This way, the result of `can_push_to_branch?` would be cached in - # `RequestStore.store` based on the cache key. If RequestStore is not - # currently active, then it would be stored in a hash saved in an - # instance variable, so the cache logic would be the same. - # Here's another example using customized method level values: - # - # class Commit - # extend Gitlab::Cache::RequestCache - # - # def author - # User.find_by_any_email(author_email.downcase) - # end - # request_cache(:author) { author_email.downcase } - # end - # - # So that we could have different strategies for different methods - # + # See https://docs.gitlab.com/ee/development/utilities.html#requestcache module RequestCache def self.extended(klass) return if klass < self diff --git a/lib/gitlab/checks/commit_check.rb b/lib/gitlab/checks/commit_check.rb index 43a52b493bb..22310e313ac 100644 --- a/lib/gitlab/checks/commit_check.rb +++ b/lib/gitlab/checks/commit_check.rb @@ -37,7 +37,7 @@ module Gitlab def validate_lfs_file_locks? strong_memoize(:validate_lfs_file_locks) do - project.lfs_enabled? && project.lfs_file_locks.any? && newrev && oldrev + project.lfs_enabled? && newrev && oldrev && project.any_lfs_file_locks? end end diff --git a/lib/gitlab/checks/force_push.rb b/lib/gitlab/checks/force_push.rb index c9c3050cfc2..87af4a90572 100644 --- a/lib/gitlab/checks/force_push.rb +++ b/lib/gitlab/checks/force_push.rb @@ -7,18 +7,10 @@ module Gitlab # Created or deleted branch return false if Gitlab::Git.blank_ref?(oldrev) || Gitlab::Git.blank_ref?(newrev) - GitalyClient.migrate(:force_push) do |is_enabled| - if is_enabled - !project - .repository - .gitaly_commit_client - .ancestor?(oldrev, newrev) - else - Gitlab::Git::RevList.new( - project.repository.raw, oldrev: oldrev, newrev: newrev - ).missed_ref.present? - end - end + !project + .repository + .gitaly_commit_client + .ancestor?(oldrev, newrev) end end end diff --git a/lib/gitlab/ci/variables/collection/item.rb b/lib/gitlab/ci/variables/collection/item.rb index d00e5b07f95..222aa06b800 100644 --- a/lib/gitlab/ci/variables/collection/item.rb +++ b/lib/gitlab/ci/variables/collection/item.rb @@ -4,6 +4,9 @@ module Gitlab class Collection class Item def initialize(key:, value:, public: true, file: false) + raise ArgumentError, "`value` must be of type String, while it was: #{value.class}" unless + value.is_a?(String) || value.nil? + @variable = { key: key, value: value, public: public, file: file } diff --git a/lib/gitlab/database/median.rb b/lib/gitlab/database/median.rb index 3cac007a42c..f64e3d53138 100644 --- a/lib/gitlab/database/median.rb +++ b/lib/gitlab/database/median.rb @@ -33,7 +33,13 @@ module Gitlab end def mysql_median_datetime_sql(arel_table, query_so_far, column_sym) - query = arel_table + arel_from = if Gitlab.rails5? + arel_table.from + else + arel_table + end + + query = arel_from .from(arel_table.project(Arel.sql('*')).order(arel_table[column_sym]).as(arel_table.table_name)) .project(average([arel_table[column_sym]], 'median')) .where( diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index c21bae5e16b..4fe5b4cc835 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -596,6 +596,97 @@ module Gitlab end end + # Renames a column using a background migration. + # + # Because this method uses a background migration it's more suitable for + # large tables. For small tables it's better to use + # `rename_column_concurrently` since it can complete its work in a much + # shorter amount of time and doesn't rely on Sidekiq. + # + # Example usage: + # + # rename_column_using_background_migration( + # :users, + # :feed_token, + # :rss_token + # ) + # + # table - The name of the database table containing the column. + # + # old - The old column name. + # + # new - The new column name. + # + # type - The type of the new column. If no type is given the old column's + # type is used. + # + # batch_size - The number of rows to schedule in a single background + # migration. + # + # interval - The time interval between every background migration. + def rename_column_using_background_migration( + table, + old_column, + new_column, + type: nil, + batch_size: 10_000, + interval: 10.minutes + ) + + check_trigger_permissions!(table) + + old_col = column_for(table, old_column) + new_type = type || old_col.type + max_index = 0 + + add_column(table, new_column, new_type, + limit: old_col.limit, + precision: old_col.precision, + scale: old_col.scale) + + # We set the default value _after_ adding the column so we don't end up + # updating any existing data with the default value. This isn't + # necessary since we copy over old values further down. + change_column_default(table, new_column, old_col.default) if old_col.default + + install_rename_triggers(table, old_column, new_column) + + model = Class.new(ActiveRecord::Base) do + self.table_name = table + + include ::EachBatch + end + + # Schedule the jobs that will copy the data from the old column to the + # new one. Rows with NULL values in our source column are skipped since + # the target column is already NULL at this point. + model.where.not(old_column => nil).each_batch(of: batch_size) do |batch, index| + start_id, end_id = batch.pluck('MIN(id), MAX(id)').first + max_index = index + + BackgroundMigrationWorker.perform_in( + index * interval, + 'CopyColumn', + [table, old_column, new_column, start_id, end_id] + ) + end + + # Schedule the renaming of the column to happen (initially) 1 hour after + # the last batch finished. + BackgroundMigrationWorker.perform_in( + (max_index * interval) + 1.hour, + 'CleanupConcurrentRename', + [table, old_column, new_column] + ) + + if perform_background_migration_inline? + # To ensure the schema is up to date immediately we perform the + # migration inline in dev / test environments. + Gitlab::BackgroundMigration.steal('CopyColumn') + Gitlab::BackgroundMigration.steal('CleanupConcurrentRename') + end + end + def perform_background_migration_inline? Rails.env.test? || Rails.env.development? end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb index 62d4d0a92a6..26ae6966746 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb @@ -37,6 +37,7 @@ module Gitlab class Namespace < ActiveRecord::Base include MigrationClasses::Routable self.table_name = 'namespaces' + self.inheritance_column = :_type_disabled belongs_to :parent, class_name: "#{MigrationClasses.name}::Namespace" has_one :route, as: :source diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 2820293ad5c..a9e209d5b71 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -130,11 +130,13 @@ module Gitlab # Array of Gitlab::Diff::Line objects def diff_lines - @diff_lines ||= Gitlab::Diff::Parser.new.parse(raw_diff.each_line).to_a + @diff_lines ||= + Gitlab::Diff::Parser.new.parse(raw_diff.each_line, diff_file: self).to_a end def highlighted_diff_lines - @highlighted_diff_lines ||= Gitlab::Diff::Highlight.new(self, repository: self.repository).highlight + @highlighted_diff_lines ||= + Gitlab::Diff::Highlight.new(self, repository: self.repository).highlight end # Array[<Hash>] with right/left keys that contains Gitlab::Diff::Line objects which text is hightlighted @@ -239,8 +241,33 @@ module Gitlab simple_viewer.is_a?(DiffViewer::Text) && (ignore_errors || simple_viewer.render_error.nil?) end + # This adds the bottom match line to the array if needed. It contains + # the data to load more context lines. + def diff_lines_for_serializer + lines = highlighted_diff_lines + + return if lines.empty? + + last_line = lines.last + + if last_line.new_pos < total_blob_lines(blob) && !deleted_file? + match_line = Gitlab::Diff::Line.new("", 'match', nil, last_line.old_pos, last_line.new_pos) + lines.push(match_line) + end + + lines + end + private + def total_blob_lines(blob) + @total_lines ||= begin + line_count = blob.lines.size + line_count -= 1 if line_count > 0 && blob.lines.last.blank? + line_count + end + end + # We can't use Object#try because Blob doesn't inherit from Object, but # from BasicObject (via SimpleDelegator). def try_blobs(meth) diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index a1e904cfef4..2b3ebfbb9ff 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -1,22 +1,26 @@ module Gitlab module Diff class Line - attr_reader :type, :index, :old_pos, :new_pos + attr_reader :line_code, :type, :index, :old_pos, :new_pos attr_writer :rich_text attr_accessor :text - def initialize(text, type, index, old_pos, new_pos, parent_file: nil) + def initialize(text, type, index, old_pos, new_pos, parent_file: nil, line_code: nil) @text, @type, @index = text, type, index @old_pos, @new_pos = old_pos, new_pos @parent_file = parent_file + + # When line code is not provided from cache store we build it + # using the parent_file(Diff::File or Conflict::File). + @line_code = line_code || calculate_line_code end def self.init_from_hash(hash) - new(hash[:text], hash[:type], hash[:index], hash[:old_pos], hash[:new_pos]) + new(hash[:text], hash[:type], hash[:index], hash[:old_pos], hash[:new_pos], line_code: hash[:line_code]) end def serialize_keys - @serialize_keys ||= %i(text type index old_pos new_pos) + @serialize_keys ||= %i(line_code text type index old_pos new_pos) end def to_hash @@ -62,20 +66,37 @@ module Gitlab end def rich_text - @parent_file.highlight_lines! if @parent_file && !@rich_text + @parent_file.try(:highlight_lines!) if @parent_file && !@rich_text @rich_text end + def meta_positions + return unless meta? + + { + old_pos: old_pos, + new_pos: new_pos + } + end + def as_json(opts = nil) { + line_code: line_code, type: type, old_line: old_line, new_line: new_line, text: text, - rich_text: rich_text || text + rich_text: rich_text || text, + meta_data: meta_positions } end + + private + + def calculate_line_code + @parent_file&.line_code(self) + end end end end diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb index 8302f30a0a2..7ae7ed286ed 100644 --- a/lib/gitlab/diff/parser.rb +++ b/lib/gitlab/diff/parser.rb @@ -3,7 +3,7 @@ module Gitlab class Parser include Enumerable - def parse(lines) + def parse(lines, diff_file: nil) return [] if lines.blank? @lines = lines @@ -31,17 +31,17 @@ module Gitlab next if line_old <= 1 && line_new <= 1 # top of file - yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new) + yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new, parent_file: diff_file) line_obj_index += 1 next elsif line[0] == '\\' type = "#{context}-nonewline" - yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new) + yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new, parent_file: diff_file) line_obj_index += 1 else type = identification_type(line) - yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new) + yielder << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new, parent_file: diff_file) line_obj_index += 1 end diff --git a/lib/gitlab/favicon.rb b/lib/gitlab/favicon.rb index faf7016d73a..4850a6c0430 100644 --- a/lib/gitlab/favicon.rb +++ b/lib/gitlab/favicon.rb @@ -2,10 +2,10 @@ module Gitlab class Favicon class << self def main - return appearance_favicon.url if appearance_favicon.exists? - image_name = - if Gitlab::Utils.to_boolean(ENV['CANARY']) + if appearance_favicon.exists? + appearance_favicon.url + elsif Gitlab::Utils.to_boolean(ENV['CANARY']) 'favicon-yellow.png' elsif Rails.env.development? 'favicon-blue.png' @@ -13,7 +13,7 @@ module Gitlab 'favicon.png' end - ActionController::Base.helpers.image_path(image_name) + ActionController::Base.helpers.image_path(image_name, host: host) end def status_overlay(status_name) @@ -22,7 +22,7 @@ module Gitlab "#{status_name}.png" ) - ActionController::Base.helpers.image_path(path) + ActionController::Base.helpers.image_path(path, host: host) end def available_status_names @@ -35,6 +35,17 @@ module Gitlab private + # we only want to create full urls when there's a different asset_host + # configured. + def host + asset_host = ActionController::Base.asset_host + if asset_host.nil? || asset_host == Gitlab.config.gitlab.base_url + nil + else + Gitlab.config.gitlab.base_url + end + end + def appearance RequestStore.store[:appearance] ||= (Appearance.current || Appearance.new) end diff --git a/lib/gitlab/file_finder.rb b/lib/gitlab/file_finder.rb index f42088f980e..af8270c8db8 100644 --- a/lib/gitlab/file_finder.rb +++ b/lib/gitlab/file_finder.rb @@ -14,14 +14,21 @@ module Gitlab end def find(query) - by_content = find_by_content(query) + query = Gitlab::Search::Query.new(query) do + filter :filename, matcher: ->(filter, blob) { blob.filename =~ /#{filter[:regex_value]}$/i } + filter :path, matcher: ->(filter, blob) { blob.filename =~ /#{filter[:regex_value]}/i } + filter :extension, matcher: ->(filter, blob) { blob.filename =~ /\.#{filter[:regex_value]}$/i } + end + + by_content = find_by_content(query.term) already_found = Set.new(by_content.map(&:filename)) - by_filename = find_by_filename(query, except: already_found) + by_filename = find_by_filename(query.term, except: already_found) + + files = (by_content + by_filename) + .sort_by(&:filename) - (by_content + by_filename) - .sort_by(&:filename) - .map { |blob| [blob.filename, blob] } + query.filter_results(files).map { |blob| [blob.filename, blob] } end private diff --git a/lib/gitlab/gfm/uploads_rewriter.rb b/lib/gitlab/gfm/uploads_rewriter.rb index b6eeb5d9a2b..f7e66697da3 100644 --- a/lib/gitlab/gfm/uploads_rewriter.rb +++ b/lib/gitlab/gfm/uploads_rewriter.rb @@ -23,11 +23,8 @@ module Gitlab file = find_file(@source_project, $~[:secret], $~[:file]) break markdown unless file.try(:exists?) - new_uploader = FileUploader.new(target_project) - with_link_in_tmp_dir(file.file) do |open_tmp_file| - new_uploader.store!(open_tmp_file) - end - new_uploader.markdown_link + moved = FileUploader.copy_to(file, target_project) + moved.markdown_link end end @@ -48,20 +45,7 @@ module Gitlab def find_file(project, secret, file) uploader = FileUploader.new(project, secret: secret) uploader.retrieve_from_store!(file) - uploader.file - end - - # Because the uploaders use 'move_to_store' we must have a temporary - # file that is allowed to be (re)moved. - def with_link_in_tmp_dir(file) - dir = Dir.mktmpdir('UploadsRewriter', File.dirname(file)) - # The filename matters to Carrierwave so we make sure to preserve it - tmp_file = File.join(dir, File.basename(file)) - File.link(file, tmp_file) - # Open the file to placate Carrierwave - File.open(tmp_file) { |open_file| yield open_file } - ensure - FileUtils.rm_rf(dir) + uploader end end end diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 156d077a69c..604bb11e712 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -21,13 +21,31 @@ module Gitlab attr_accessor :name, :path, :size, :data, :mode, :id, :commit_id, :loaded_size, :binary class << self - def find(repository, sha, path) - Gitlab::GitalyClient.migrate(:project_raw_show) do |is_enabled| - if is_enabled - find_by_gitaly(repository, sha, path) - else - find_by_rugged(repository, sha, path, limit: MAX_DATA_DISPLAY_SIZE) - end + def find(repository, sha, path, limit: MAX_DATA_DISPLAY_SIZE) + return unless path + + path = path.sub(%r{\A/*}, '') + path = '/' if path.empty? + name = File.basename(path) + + # Gitaly will think that setting the limit to 0 means unlimited, while + # the client might only need the metadata and thus set the limit to 0. + # In this method we'll then set the limit to 1, but clear the byte of data + # that we got back so for the outside world it looks like the limit was + # actually 0. + req_limit = limit == 0 ? 1 : limit + + entry = Gitlab::GitalyClient::CommitService.new(repository).tree_entry(sha, path, req_limit) + return unless entry + + entry.data = "" if limit == 0 + + case entry.type + when :COMMIT + new(id: entry.oid, name: name, size: 0, data: '', path: path, commit_id: sha) + when :BLOB + new(id: entry.oid, name: name, size: entry.size, data: entry.data.dup, mode: entry.mode.to_s(8), + path: path, commit_id: sha, binary: binary?(entry.data)) end end @@ -56,7 +74,7 @@ module Gitlab repository.gitaly_blob_client.get_blobs(blob_references, blob_size_limit).to_a else blob_references.map do |sha, path| - find_by_rugged(repository, sha, path, limit: blob_size_limit) + find(repository, sha, path, limit: blob_size_limit) end end end @@ -136,85 +154,6 @@ module Gitlab ) end - def find_by_gitaly(repository, sha, path, limit: MAX_DATA_DISPLAY_SIZE) - return unless path - - path = path.sub(%r{\A/*}, '') - path = '/' if path.empty? - name = File.basename(path) - - # Gitaly will think that setting the limit to 0 means unlimited, while - # the client might only need the metadata and thus set the limit to 0. - # In this method we'll then set the limit to 1, but clear the byte of data - # that we got back so for the outside world it looks like the limit was - # actually 0. - req_limit = limit == 0 ? 1 : limit - - entry = Gitlab::GitalyClient::CommitService.new(repository).tree_entry(sha, path, req_limit) - return unless entry - - entry.data = "" if limit == 0 - - case entry.type - when :COMMIT - new( - id: entry.oid, - name: name, - size: 0, - data: '', - path: path, - commit_id: sha - ) - when :BLOB - new( - id: entry.oid, - name: name, - size: entry.size, - data: entry.data.dup, - mode: entry.mode.to_s(8), - path: path, - commit_id: sha, - binary: binary?(entry.data) - ) - end - end - - def find_by_rugged(repository, sha, path, limit:) - return unless path - - # Strip any leading / characters from the path - path = path.sub(%r{\A/*}, '') - - rugged_commit = repository.lookup(sha) - root_tree = rugged_commit.tree - - blob_entry = find_entry_by_path(repository, root_tree.oid, *path.split('/')) - - return nil unless blob_entry - - if blob_entry[:type] == :commit - submodule_blob(blob_entry, path, sha) - else - blob = repository.lookup(blob_entry[:oid]) - - if blob - new( - id: blob.oid, - name: blob_entry[:name], - size: blob.size, - # Rugged::Blob#content is expensive; don't call it if we don't have to. - data: limit.zero? ? '' : blob.content(limit), - mode: blob_entry[:filemode].to_s(8), - path: path, - commit_id: sha, - binary: blob.binary? - ) - end - end - rescue Rugged::ReferenceError - nil - end - def rugged_raw(repository, sha, limit:) blob = repository.lookup(sha) diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index c9806cdb85f..36d56e411d8 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -63,12 +63,8 @@ module Gitlab # This saves us an RPC round trip. return nil if commit_id.include?(':') - commit = repo.gitaly_migrate(:find_commit) do |is_enabled| - if is_enabled - repo.gitaly_commit_client.find_commit(commit_id) - else - rugged_find(repo, commit_id) - end + commit = repo.wrapped_gitaly_errors do + repo.gitaly_commit_client.find_commit(commit_id) end decorate(repo, commit) if commit @@ -78,12 +74,6 @@ module Gitlab nil end - def rugged_find(repo, commit_id) - obj = repo.rev_parse_target(commit_id) - - obj.is_a?(Rugged::Commit) ? obj : nil - end - # Get last commit for HEAD # # Ex. @@ -116,15 +106,9 @@ module Gitlab # Commit.between(repo, '29eda46b', 'master') # def between(repo, base, head) - Gitlab::GitalyClient.migrate(:commits_between) do |is_enabled| - if is_enabled - repo.gitaly_commit_client.between(base, head) - else - repo.rugged_commits_between(base, head).map { |c| decorate(repo, c) } - end + repo.wrapped_gitaly_errors do + repo.gitaly_commit_client.between(base, head) end - rescue Rugged::ReferenceError - [] end # Returns commits collection @@ -149,56 +133,9 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/326 def find_all(repo, options = {}) - Gitlab::GitalyClient.migrate(:find_all_commits) do |is_enabled| - if is_enabled - find_all_by_gitaly(repo, options) - else - find_all_by_rugged(repo, options) - end - end - end - - def find_all_by_rugged(repo, options = {}) - actual_options = options.dup - - allowed_options = [:ref, :max_count, :skip, :order] - - actual_options.keep_if do |key| - allowed_options.include?(key) - end - - default_options = { skip: 0 } - actual_options = default_options.merge(actual_options) - - rugged = repo.rugged - walker = Rugged::Walker.new(rugged) - - if actual_options[:ref] - walker.push(rugged.rev_parse_oid(actual_options[:ref])) - else - rugged.references.each("refs/heads/*") do |ref| - walker.push(ref.target_id) - end + repo.wrapped_gitaly_errors do + Gitlab::GitalyClient::CommitService.new(repo).find_all_commits(options) end - - walker.sorting(rugged_sort_type(actual_options[:order])) - - commits = [] - offset = actual_options[:skip] - limit = actual_options[:max_count] - walker.each(offset: offset, limit: limit) do |commit| - commits.push(decorate(repo, commit)) - end - - walker.reset - - commits - rescue Rugged::OdbError - [] - end - - def find_all_by_gitaly(repo, options = {}) - Gitlab::GitalyClient::CommitService.new(repo).find_all_commits(options) end def decorate(repository, commit, ref = nil) @@ -220,19 +157,7 @@ module Gitlab end def shas_with_signatures(repository, shas) - GitalyClient.migrate(:filter_shas_with_signatures) do |is_enabled| - if is_enabled - Gitlab::GitalyClient::CommitService.new(repository).filter_shas_with_signatures(shas) - else - shas.select do |sha| - begin - Rugged::Commit.extract_signature(repository.rugged, sha) - rescue Rugged::OdbError - false - end - end - end - end + Gitlab::GitalyClient::CommitService.new(repository).filter_shas_with_signatures(shas) end # Only to be used when the object ids will not necessarily have a @@ -250,13 +175,7 @@ module Gitlab end def extract_signature(repository, commit_id) - repository.gitaly_migrate(:extract_commit_signature) do |is_enabled| - if is_enabled - repository.gitaly_commit_client.extract_signature(commit_id) - else - rugged_extract_signature(repository, commit_id) - end - end + repository.gitaly_commit_client.extract_signature(commit_id) end def extract_signature_lazily(repository, commit_id) @@ -276,36 +195,9 @@ module Gitlab end def batch_signature_extraction(repository, commit_ids) - repository.gitaly_migrate(:extract_commit_signature_in_batch) do |is_enabled| - if is_enabled - gitaly_batch_signature_extraction(repository, commit_ids) - else - rugged_batch_signature_extraction(repository, commit_ids) - end - end - end - - def gitaly_batch_signature_extraction(repository, commit_ids) repository.gitaly_commit_client.get_commit_signatures(commit_ids) end - def rugged_batch_signature_extraction(repository, commit_ids) - commit_ids.each_with_object({}) do |commit_id, signatures| - signature_data = rugged_extract_signature(repository, commit_id) - next unless signature_data - - signatures[commit_id] = signature_data - end - end - - def rugged_extract_signature(repository, commit_id) - begin - Rugged::Commit.extract_signature(repository.rugged, commit_id) - rescue Rugged::OdbError - nil - end - end - def get_message(repository, commit_id) BatchLoader.for({ repository: repository, commit_id: commit_id }).batch do |items, loader| items_by_repo = items.group_by { |i| i[:repository] } @@ -323,13 +215,7 @@ module Gitlab end def get_messages(repository, commit_ids) - repository.gitaly_migrate(:commit_messages) do |is_enabled| - if is_enabled - repository.gitaly_commit_client.get_commit_messages(commit_ids) - else - commit_ids.map { |id| [id, rugged_find(repository, id).message] }.to_h - end - end + repository.gitaly_commit_client.get_commit_messages(commit_ids) end end @@ -381,15 +267,11 @@ module Gitlab # empty repo. See Repository#diff for keys allowed in the +options+ # hash. def diff_from_parent(options = {}) - Gitlab::GitalyClient.migrate(:commit_raw_diffs) do |is_enabled| - if is_enabled - @repository.gitaly_commit_client.diff_from_parent(self, options) - else - rugged_diff_from_parent(options) - end - end + @repository.gitaly_commit_client.diff_from_parent(self, options) end + # Not to be called directly, but right now its used for tests and in old + # migrations def rugged_diff_from_parent(options = {}) options ||= {} break_rewrites = options[:break_rewrites] @@ -497,13 +379,18 @@ module Gitlab def tree_entry(path) return unless path.present? - @repository.gitaly_migrate(:commit_tree_entry) do |is_migrated| - if is_migrated - gitaly_tree_entry(path) - else - rugged_tree_entry(path) - end - end + # We're only interested in metadata, so limit actual data to 1 byte + # since Gitaly doesn't support "send no data" option. + entry = @repository.gitaly_commit_client.tree_entry(id, path, 1) + return unless entry + + # To be compatible with the rugged format + entry = entry.to_h + entry.delete(:data) + entry[:name] = File.basename(path) + entry[:type] = entry[:type].downcase + + entry end def to_gitaly_commit @@ -566,28 +453,6 @@ module Gitlab SERIALIZE_KEYS end - def gitaly_tree_entry(path) - # We're only interested in metadata, so limit actual data to 1 byte - # since Gitaly doesn't support "send no data" option. - entry = @repository.gitaly_commit_client.tree_entry(id, path, 1) - return unless entry - - # To be compatible with the rugged format - entry = entry.to_h - entry.delete(:data) - entry[:name] = File.basename(path) - entry[:type] = entry[:type].downcase - - entry - end - - # Is this the same as Blob.find_entry_by_path ? - def rugged_tree_entry(path) - rugged_commit.tree.path(path) - rescue Rugged::TreeError - nil - end - def gitaly_commit_author_from_rugged(author_or_committer) Gitaly::CommitAuthor.new( name: author_or_committer[:name].b, diff --git a/lib/gitlab/git/conflict/resolver.rb b/lib/gitlab/git/conflict/resolver.rb index c3cb0264112..0e4a973301f 100644 --- a/lib/gitlab/git/conflict/resolver.rb +++ b/lib/gitlab/git/conflict/resolver.rb @@ -12,14 +12,8 @@ module Gitlab end def conflicts - @conflicts ||= begin - @target_repository.gitaly_migrate(:conflicts_list_conflict_files) do |is_enabled| - if is_enabled - gitaly_conflicts_client(@target_repository).list_conflict_files.to_a - else - rugged_list_conflict_files - end - end + @conflicts ||= @target_repository.wrapped_gitaly_errors do + gitaly_conflicts_client(@target_repository).list_conflict_files.to_a end rescue GRPC::FailedPrecondition => e raise Gitlab::Git::Conflict::Resolver::ConflictSideMissing.new(e.message) @@ -28,12 +22,8 @@ module Gitlab end def resolve_conflicts(source_repository, resolution, source_branch:, target_branch:) - source_repository.gitaly_migrate(:conflicts_resolve_conflicts) do |is_enabled| - if is_enabled - gitaly_conflicts_client(source_repository).resolve_conflicts(@target_repository, resolution, source_branch, target_branch) - else - rugged_resolve_conflicts(source_repository, resolution, source_branch, target_branch) - end + source_repository.wrapped_gitaly_errors do + gitaly_conflicts_client(source_repository).resolve_conflicts(@target_repository, resolution, source_branch, target_branch) end end @@ -61,57 +51,6 @@ module Gitlab def gitaly_conflicts_client(repository) repository.gitaly_conflicts_client(@our_commit_oid, @their_commit_oid) end - - def write_resolved_file_to_index(repository, index, file, params) - if params[:sections] - resolved_lines = file.resolve_lines(params[:sections]) - new_file = resolved_lines.map { |line| line[:full_line] }.join("\n") - - new_file << "\n" if file.our_blob.data.end_with?("\n") - elsif params[:content] - new_file = file.resolve_content(params[:content]) - end - - our_path = file.our_path - - oid = repository.rugged.write(new_file, :blob) - index.add(path: our_path, oid: oid, mode: file.our_mode) - index.conflict_remove(our_path) - end - - def rugged_list_conflict_files - target_index = @target_repository.rugged.merge_commits(@our_commit_oid, @their_commit_oid) - - # We don't need to do `with_repo_branch_commit` here, because the target - # project always fetches source refs when creating merge request diffs. - conflict_files(@target_repository, target_index) - end - - def rugged_resolve_conflicts(source_repository, resolution, source_branch, target_branch) - source_repository.with_repo_branch_commit(@target_repository, target_branch) do - index = source_repository.rugged.merge_commits(@our_commit_oid, @their_commit_oid) - conflicts = conflict_files(source_repository, index) - - resolution.files.each do |file_params| - conflict_file = conflict_for_path(conflicts, file_params[:old_path], file_params[:new_path]) - - write_resolved_file_to_index(source_repository, index, conflict_file, file_params) - end - - unless index.conflicts.empty? - missing_files = index.conflicts.map { |file| file[:ours][:path] } - - raise ResolutionError, "Missing resolutions for the following files: #{missing_files.join(', ')}" - end - - commit_params = { - message: resolution.commit_message, - parents: [@our_commit_oid, @their_commit_oid] - } - - source_repository.commit_index(resolution.user, source_branch, index, commit_params) - end - end end end end diff --git a/lib/gitlab/git/gitlab_projects.rb b/lib/gitlab/git/gitlab_projects.rb index 8475645971e..5ff15a787f0 100644 --- a/lib/gitlab/git/gitlab_projects.rb +++ b/lib/gitlab/git/gitlab_projects.rb @@ -61,22 +61,15 @@ module Gitlab end def fetch_remote(name, timeout, force:, tags:, ssh_key: nil, known_hosts: nil, prune: true) - tags_option = tags ? '--tags' : '--no-tags' - logger.info "Fetching remote #{name} for repository #{repository_absolute_path}." - cmd = %W(#{Gitlab.config.git.bin_path} fetch #{name} --quiet) - cmd << '--prune' if prune - cmd << '--force' if force - cmd << tags_option + cmd = fetch_remote_command(name, tags, prune, force) setup_ssh_auth(ssh_key, known_hosts) do |env| - success = run_with_timeout(cmd, timeout, repository_absolute_path, env) - - unless success - logger.error "Fetching remote #{name} for repository #{repository_absolute_path} failed." + run_with_timeout(cmd, timeout, repository_absolute_path, env).tap do |success| + unless success + logger.error "Fetching remote #{name} for repository #{repository_absolute_path} failed." + end end - - success end end @@ -202,6 +195,14 @@ module Gitlab private + def fetch_remote_command(name, tags, prune, force) + %W(#{Gitlab.config.git.bin_path} fetch #{name} --quiet).tap do |cmd| + cmd << '--prune' if prune + cmd << '--force' if force + cmd << (tags ? '--tags' : '--no-tags') + end + end + def git_import_repository(source, timeout) # Skip import if repo already exists return false if File.exist?(repository_absolute_path) diff --git a/lib/gitlab/git/lfs_changes.rb b/lib/gitlab/git/lfs_changes.rb index f3cc388ea41..f0fab1e76a3 100644 --- a/lib/gitlab/git/lfs_changes.rb +++ b/lib/gitlab/git/lfs_changes.rb @@ -7,67 +7,11 @@ module Gitlab end def new_pointers(object_limit: nil, not_in: nil) - @repository.gitaly_migrate(:blob_get_new_lfs_pointers) do |is_enabled| - if is_enabled - @repository.gitaly_blob_client.get_new_lfs_pointers(@newrev, object_limit, not_in) - else - git_new_pointers(object_limit, not_in) - end - end + @repository.gitaly_blob_client.get_new_lfs_pointers(@newrev, object_limit, not_in) end def all_pointers - @repository.gitaly_migrate(:blob_get_all_lfs_pointers) do |is_enabled| - if is_enabled - @repository.gitaly_blob_client.get_all_lfs_pointers(@newrev) - else - git_all_pointers - end - end - end - - private - - def git_new_pointers(object_limit, not_in) - @new_pointers ||= begin - rev_list.new_objects(rev_list_params(not_in: not_in)) do |object_ids| - object_ids = object_ids.take(object_limit) if object_limit - - Gitlab::Git::Blob.batch_lfs_pointers(@repository, object_ids) - end - end - end - - def git_all_pointers - params = {} - if rev_list_supports_new_options? - params[:options] = ["--filter=blob:limit=#{Gitlab::Git::Blob::LFS_POINTER_MAX_SIZE}"] - end - - rev_list.all_objects(rev_list_params(params)) do |object_ids| - Gitlab::Git::Blob.batch_lfs_pointers(@repository, object_ids) - end - end - - def rev_list - Gitlab::Git::RevList.new(@repository, newrev: @newrev) - end - - # We're passing the `--in-commit-order` arg to ensure we don't wait - # for git to traverse all commits before returning pointers. - # This is required in order to improve the performance of LFS integrity check - def rev_list_params(params = {}) - params[:options] ||= [] - params[:options] << "--in-commit-order" if rev_list_supports_new_options? - params[:require_path] = true - - params - end - - def rev_list_supports_new_options? - return @option_supported if defined?(@option_supported) - - @option_supported = Gitlab::Git.version >= Gitlab::VersionInfo.parse('2.16.0') + @repository.gitaly_blob_client.get_all_lfs_pointers(@newrev) end end end diff --git a/lib/gitlab/git/remote_mirror.rb b/lib/gitlab/git/remote_mirror.rb index ebe46722890..e4743b4db0a 100644 --- a/lib/gitlab/git/remote_mirror.rb +++ b/lib/gitlab/git/remote_mirror.rb @@ -7,81 +7,8 @@ module Gitlab end def update(only_branches_matching: []) - @repository.gitaly_migrate(:remote_update_remote_mirror) do |is_enabled| - if is_enabled - gitaly_update(only_branches_matching) - else - rugged_update(only_branches_matching) - end - end - end - - private - - def gitaly_update(only_branches_matching) - @repository.gitaly_remote_client.update_remote_mirror(@ref_name, only_branches_matching) - end - - def rugged_update(only_branches_matching) - local_branches = refs_obj(@repository.local_branches, only_refs_matching: only_branches_matching) - remote_branches = refs_obj(@repository.remote_branches(@ref_name), only_refs_matching: only_branches_matching) - - updated_branches = changed_refs(local_branches, remote_branches) - push_branches(updated_branches.keys) if updated_branches.present? - - delete_refs(local_branches, remote_branches) - - local_tags = refs_obj(@repository.tags) - remote_tags = refs_obj(@repository.remote_tags(@ref_name)) - - updated_tags = changed_refs(local_tags, remote_tags) - @repository.push_remote_branches(@ref_name, updated_tags.keys) if updated_tags.present? - - delete_refs(local_tags, remote_tags) - end - - def refs_obj(refs, only_refs_matching: []) - refs.each_with_object({}) do |ref, refs| - next if only_refs_matching.present? && !only_refs_matching.include?(ref.name) - - refs[ref.name] = ref - end - end - - def changed_refs(local_refs, remote_refs) - local_refs.select do |ref_name, ref| - remote_ref = remote_refs[ref_name] - - remote_ref.nil? || ref.dereferenced_target != remote_ref.dereferenced_target - end - end - - def push_branches(branches) - default_branch, branches = branches.partition do |branch| - @repository.root_ref == branch - end - - # Push the default branch first so it works fine when remote mirror is empty. - branches.unshift(*default_branch) - - @repository.push_remote_branches(@ref_name, branches) - end - - def delete_refs(local_refs, remote_refs) - refs = refs_to_delete(local_refs, remote_refs) - - @repository.delete_remote_branches(@ref_name, refs.keys) if refs.present? - end - - def refs_to_delete(local_refs, remote_refs) - default_branch_id = @repository.commit.id - - remote_refs.select do |remote_ref_name, remote_ref| - next false if local_refs[remote_ref_name] # skip if branch or tag exist in local repo - - remote_ref_id = remote_ref.dereferenced_target.try(:id) - - remote_ref_id && @repository.rugged_is_ancestor?(remote_ref_id, default_branch_id) + @repository.wrapped_gitaly_errors do + @repository.gitaly_remote_client.update_remote_mirror(@ref_name, only_branches_matching) end end end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index eb5d6318dcb..12fd6898694 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -403,13 +403,7 @@ module Gitlab # Return repo size in megabytes def size - size = gitaly_migrate(:repository_size) do |is_enabled| - if is_enabled - size_by_gitaly - else - size_by_shelling_out - end - end + size = gitaly_repository_client.repository_size (size.to_f / 1024).round(2) end @@ -472,13 +466,21 @@ module Gitlab end def count_commits(options) - count_commits_options = process_count_commits_options(options) + options = process_count_commits_options(options.dup) - gitaly_migrate(:count_commits, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - count_commits_by_gitaly(count_commits_options) + wrapped_gitaly_errors do + if options[:left_right] + from = options[:from] + to = options[:to] + + right_count = gitaly_commit_client + .commit_count("#{from}..#{to}", options) + left_count = gitaly_commit_client + .commit_count("#{to}..#{from}", options) + + [left_count, right_count] else - count_commits_by_shelling_out(count_commits_options) + gitaly_commit_client.commit_count(options[:ref], options) end end end @@ -490,27 +492,6 @@ module Gitlab Ref.dereference_object(obj) end - # Return a collection of Rugged::Commits between the two revspec arguments. - # See http://git-scm.com/docs/git-rev-parse.html#_specifying_revisions for - # a detailed list of valid arguments. - # - # Gitaly note: JV: to be deprecated in favor of Commit.between - def rugged_commits_between(from, to) - walker = Rugged::Walker.new(rugged) - walker.sorting(Rugged::SORT_NONE | Rugged::SORT_REVERSE) - - sha_from = sha_from_ref(from) - sha_to = sha_from_ref(to) - - walker.push(sha_to) - walker.hide(sha_from) - - commits = walker.to_a - walker.reset - - commits - end - # Counts the amount of commits between `from` and `to`. def count_commits_between(from, to, options = {}) count_commits(from: from, to: to, **options) @@ -521,32 +502,17 @@ module Gitlab def raw_changes_between(old_rev, new_rev) @raw_changes_between ||= {} - @raw_changes_between[[old_rev, new_rev]] ||= begin - return [] if new_rev.blank? || new_rev == Gitlab::Git::BLANK_SHA + @raw_changes_between[[old_rev, new_rev]] ||= + begin + return [] if new_rev.blank? || new_rev == Gitlab::Git::BLANK_SHA - gitaly_migrate(:raw_changes_between) do |is_enabled| - if is_enabled + wrapped_gitaly_errors do gitaly_repository_client.raw_changes_between(old_rev, new_rev) .each_with_object([]) do |msg, arr| msg.raw_changes.each { |change| arr << ::Gitlab::Git::RawDiffChange.new(change) } end - else - result = [] - - circuit_breaker.perform do - Open3.pipeline_r(git_diff_cmd(old_rev, new_rev), format_git_cat_file_script, git_cat_file_cmd) do |last_stdout, wait_threads| - last_stdout.each_line { |line| result << ::Gitlab::Git::RawDiffChange.new(line.chomp!) } - - if wait_threads.any? { |waiter| !waiter.value&.success? } - raise ::Gitlab::Git::Repository::GitError, "Unabled to obtain changes between #{old_rev} and #{new_rev}" - end - end - end - - result end end - end rescue ArgumentError => e raise Gitlab::Git::Repository::GitError.new(e) end @@ -562,24 +528,9 @@ module Gitlab end end - # Gitaly note: JV: check gitlab-ee before removing this method. - def rugged_is_ancestor?(ancestor_id, descendant_id) - return false if ancestor_id.nil? || descendant_id.nil? - - rugged_merge_base(ancestor_id, descendant_id) == ancestor_id - rescue Rugged::OdbError - false - end - # Returns true is +from+ is direct ancestor to +to+, otherwise false def ancestor?(from, to) - Gitlab::GitalyClient.migrate(:is_ancestor) do |is_enabled| - if is_enabled - gitaly_commit_client.ancestor?(from, to) - else - rugged_is_ancestor?(from, to) - end - end + gitaly_commit_client.ancestor?(from, to) end def merged_branch_names(branch_names = []) @@ -605,12 +556,8 @@ module Gitlab # diff options. The +options+ hash can also include :break_rewrites to # split larger rewrites into delete/add pairs. def diff(from, to, options = {}, *paths) - iterator = gitaly_migrate(:diff_between) do |is_enabled| - if is_enabled - gitaly_commit_client.diff(from, to, options.merge(paths: paths)) - else - diff_patches(from, to, options, *paths) - end + iterator = wrapped_gitaly_errors do + gitaly_commit_client.diff(from, to, options.merge(paths: paths)) end Gitlab::Git::DiffCollection.new(iterator, options) @@ -620,17 +567,7 @@ module Gitlab def ref_name_for_sha(ref_path, sha) raise ArgumentError, "sha can't be empty" unless sha.present? - gitaly_migrate(:find_ref_name) do |is_enabled| - if is_enabled - gitaly_ref_client.find_ref_name(sha, ref_path) - else - args = %W(for-each-ref --count=1 #{ref_path} --contains #{sha}) - - # Not found -> ["", 0] - # Found -> ["b8d95eb4969eefacb0a58f6a28f6803f8070e7b9 commit\trefs/environments/production/77\n", 0] - run_git(args).first.split.last - end - end + gitaly_ref_client.find_ref_name(sha, ref_path) end # Get refs hash which key is is the commit id @@ -676,15 +613,9 @@ module Gitlab end # Return total commits count accessible from passed ref - # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/330 def commit_count(ref) - gitaly_migrate(:commit_count, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_commit_client.commit_count(ref) - else - rugged_commit_count(ref) - end + wrapped_gitaly_errors do + gitaly_commit_client.commit_count(ref) end end @@ -713,38 +644,30 @@ module Gitlab end def add_branch(branch_name, user:, target:) - gitaly_operation_client.user_create_branch(branch_name, user, target) - rescue GRPC::FailedPrecondition => ex - raise InvalidRef, ex + wrapped_gitaly_errors do + gitaly_operation_client.user_create_branch(branch_name, user, target) + end end def add_tag(tag_name, user:, target:, message: nil) - gitaly_migrate(:operation_user_add_tag, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_add_tag(tag_name, user: user, target: target, message: message) - else - rugged_add_tag(tag_name, user: user, target: target, message: message) - end + wrapped_gitaly_errors do + gitaly_operation_client.add_tag(tag_name, user, target, message) end end + def update_branch(branch_name, user:, newrev:, oldrev:) + OperationService.new(user, self).update_branch(branch_name, newrev, oldrev) + end + def rm_branch(branch_name, user:) - gitaly_migrate(:operation_user_delete_branch, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_operations_client.user_delete_branch(branch_name, user) - else - OperationService.new(user, self).rm_branch(find_branch(branch_name)) - end + wrapped_gitaly_errors do + gitaly_operation_client.user_delete_branch(branch_name, user) end end def rm_tag(tag_name, user:) - gitaly_migrate(:operation_user_delete_tag) do |is_enabled| - if is_enabled - gitaly_operations_client.rm_tag(tag_name, user) - else - Gitlab::Git::OperationService.new(user, self).rm_tag(find_tag(tag_name)) - end + wrapped_gitaly_errors do + gitaly_operation_client.rm_tag(tag_name, user) end end @@ -753,72 +676,29 @@ module Gitlab end def merge(user, source_sha, target_branch, message, &block) - gitaly_migrate(:operation_user_merge_branch) do |is_enabled| - if is_enabled - gitaly_operation_client.user_merge_branch(user, source_sha, target_branch, message, &block) - else - rugged_merge(user, source_sha, target_branch, message, &block) - end - end - end - - def rugged_merge(user, source_sha, target_branch, message) - committer = Gitlab::Git.committer_hash(email: user.email, name: user.name) - - OperationService.new(user, self).with_branch(target_branch) do |start_commit| - our_commit = start_commit.sha - their_commit = source_sha - - raise 'Invalid merge target' unless our_commit - raise 'Invalid merge source' unless their_commit - - merge_index = rugged.merge_commits(our_commit, their_commit) - break if merge_index.conflicts? - - options = { - parents: [our_commit, their_commit], - tree: merge_index.write_tree(rugged), - message: message, - author: committer, - committer: committer - } - - commit_id = create_commit(options) - - yield commit_id - - commit_id + wrapped_gitaly_errors do + gitaly_operation_client.user_merge_branch(user, source_sha, target_branch, message, &block) end - rescue Gitlab::Git::CommitError # when merge_index.conflicts? - nil end def ff_merge(user, source_sha, target_branch) - gitaly_migrate(:operation_user_ff_branch) do |is_enabled| - if is_enabled - gitaly_ff_merge(user, source_sha, target_branch) - else - rugged_ff_merge(user, source_sha, target_branch) - end + wrapped_gitaly_errors do + gitaly_operation_client.user_ff_branch(user, source_sha, target_branch) end end def revert(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) - gitaly_migrate(:revert, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - args = { - user: user, - commit: commit, - branch_name: branch_name, - message: message, - start_branch_name: start_branch_name, - start_repository: start_repository - } + args = { + user: user, + commit: commit, + branch_name: branch_name, + message: message, + start_branch_name: start_branch_name, + start_repository: start_repository + } - if is_enabled - gitaly_operations_client.user_revert(args) - else - rugged_revert(args) - end + wrapped_gitaly_errors do + gitaly_operation_client.user_revert(args) end end @@ -836,21 +716,17 @@ module Gitlab end def cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) - gitaly_migrate(:cherry_pick) do |is_enabled| - args = { - user: user, - commit: commit, - branch_name: branch_name, - message: message, - start_branch_name: start_branch_name, - start_repository: start_repository - } + args = { + user: user, + commit: commit, + branch_name: branch_name, + message: message, + start_branch_name: start_branch_name, + start_repository: start_repository + } - if is_enabled - gitaly_operations_client.user_cherry_pick(args) - else - rugged_cherry_pick(args) - end + wrapped_gitaly_errors do + gitaly_operation_client.user_cherry_pick(args) end end @@ -959,13 +835,7 @@ module Gitlab # # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/327 def ls_files(ref) - gitaly_migrate(:ls_files) do |is_enabled| - if is_enabled - gitaly_ls_files(ref) - else - git_ls_files(ref) - end - end + gitaly_commit_client.ls_files(ref) end # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/328 @@ -984,21 +854,7 @@ module Gitlab def info_attributes return @info_attributes if @info_attributes - content = - gitaly_migrate(:get_info_attributes, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_repository_client.info_attributes - else - attributes_path = File.join(File.expand_path(path), 'info', 'attributes') - - if File.exist?(attributes_path) - File.read(attributes_path) - else - "" - end - end - end - + content = gitaly_repository_client.info_attributes @info_attributes = AttributesParser.new(content) end @@ -1027,45 +883,14 @@ module Gitlab end def languages(ref = nil) - gitaly_migrate(:commit_languages, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_commit_client.languages(ref) - else - ref ||= rugged.head.target_id - languages = Linguist::Repository.new(rugged, ref).languages - total = languages.map(&:last).sum - - languages = languages.map do |language| - name, share = language - color = Linguist::Language[name].color || "##{Digest::SHA256.hexdigest(name)[0...6]}" - { - value: (share.to_f * 100 / total).round(2), - label: name, - color: color, - highlight: color - } - end - - languages.sort do |x, y| - y[:value] <=> x[:value] - end - end + wrapped_gitaly_errors do + gitaly_commit_client.languages(ref) end end def license_short_name - gitaly_migrate(:license_short_name, - status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_repository_client.license_short_name - else - begin - # The licensee gem creates a Rugged object from the path: - # https://github.com/benbalter/licensee/blob/v8.7.0/lib/licensee/projects/git_project.rb - Licensee.license(path).try(:key) - rescue Rugged::Error - end - end + wrapped_gitaly_errors do + gitaly_repository_client.license_short_name end end @@ -1217,16 +1042,7 @@ module Gitlab end def create_from_bundle(bundle_path) - gitaly_migrate(:create_repo_from_bundle) do |is_enabled| - if is_enabled - gitaly_repository_client.create_from_bundle(bundle_path) - else - run_git!(%W(clone --bare -- #{bundle_path} #{path}), chdir: nil) - self.class.create_hooks(path, File.expand_path(Gitlab.config.gitlab_shell.hooks_path)) - end - end - - true + gitaly_repository_client.create_from_bundle(bundle_path) end def create_from_snapshot(url, auth) @@ -1234,51 +1050,31 @@ module Gitlab end def rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) - gitaly_migrate(:rebase) do |is_enabled| - if is_enabled - gitaly_rebase(user, rebase_id, - branch: branch, - branch_sha: branch_sha, - remote_repository: remote_repository, - remote_branch: remote_branch) - else - git_rebase(user, rebase_id, - branch: branch, - branch_sha: branch_sha, - remote_repository: remote_repository, - remote_branch: remote_branch) - end + wrapped_gitaly_errors do + gitaly_operation_client.user_rebase(user, rebase_id, + branch: branch, + branch_sha: branch_sha, + remote_repository: remote_repository, + remote_branch: remote_branch) end end def rebase_in_progress?(rebase_id) - gitaly_migrate(:rebase_in_progress) do |is_enabled| - if is_enabled - gitaly_repository_client.rebase_in_progress?(rebase_id) - else - fresh_worktree?(worktree_path(REBASE_WORKTREE_PREFIX, rebase_id)) - end + wrapped_gitaly_errors do + gitaly_repository_client.rebase_in_progress?(rebase_id) end end def squash(user, squash_id, branch:, start_sha:, end_sha:, author:, message:) - gitaly_migrate(:squash) do |is_enabled| - if is_enabled - gitaly_operation_client.user_squash(user, squash_id, branch, + wrapped_gitaly_errors do + gitaly_operation_client.user_squash(user, squash_id, branch, start_sha, end_sha, author, message) - else - git_squash(user, squash_id, branch, start_sha, end_sha, author, message) - end end end def squash_in_progress?(squash_id) - gitaly_migrate(:squash_in_progress, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_repository_client.squash_in_progress?(squash_id) - else - fresh_worktree?(worktree_path(SQUASH_WORKTREE_PREFIX, squash_id)) - end + wrapped_gitaly_errors do + gitaly_repository_client.squash_in_progress?(squash_id) end end @@ -1318,15 +1114,10 @@ module Gitlab author_email: nil, author_name: nil, start_branch_name: nil, start_repository: self) - gitaly_migrate(:operation_user_commit_files) do |is_enabled| - if is_enabled - gitaly_operation_client.user_commit_files(user, branch_name, + wrapped_gitaly_errors do + gitaly_operation_client.user_commit_files(user, branch_name, message, actions, author_email, author_name, start_branch_name, start_repository) - else - rugged_multi_action(user, branch_name, message, actions, - author_email, author_name, start_branch_name, start_repository) - end end end # rubocop:enable Metrics/ParameterLists @@ -1335,16 +1126,10 @@ module Gitlab return unless full_path.present? # This guard avoids Gitaly log/error spam - unless exists? - raise NoRepository, 'repository does not exist' - end + raise NoRepository, 'repository does not exist' unless exists? - gitaly_migrate(:write_config) do |is_enabled| - if is_enabled - gitaly_repository_client.write_config(full_path: full_path) - else - rugged_write_config(full_path: full_path) - end + wrapped_gitaly_errors do + gitaly_repository_client.write_config(full_path: full_path) end end @@ -1352,10 +1137,6 @@ module Gitlab Gitlab::GitalyClient::Util.repository(@storage, @relative_path, @gl_repository) end - def gitaly_operations_client - @gitaly_operations_client ||= Gitlab::GitalyClient::OperationService.new(self) - end - def gitaly_ref_client @gitaly_ref_client ||= Gitlab::GitalyClient::RefService.new(self) end @@ -1430,25 +1211,14 @@ module Gitlab safe_query = Regexp.escape(query) ref ||= root_ref - gitaly_migrate(:search_files_by_content) do |is_enabled| - if is_enabled - gitaly_repository_client.search_files_by_content(ref, safe_query) - else - offset = 2 - args = %W(grep -i -I -n -z --before-context #{offset} --after-context #{offset} -E -e #{safe_query} #{ref}) - - run_git(args).first.scrub.split(/^--\n/) - end - end + gitaly_repository_client.search_files_by_content(ref, safe_query) end def can_be_merged?(source_sha, target_branch) - gitaly_migrate(:can_be_merged) do |is_enabled| - if is_enabled - gitaly_can_be_merged?(source_sha, find_branch(target_branch, true).target) - else - rugged_can_be_merged?(source_sha, target_branch) - end + if target_sha = find_branch(target_branch, true)&.target + !gitaly_conflicts_client(source_sha, target_sha).conflicts? + else + false end end @@ -1458,15 +1228,7 @@ module Gitlab return [] if empty? || safe_query.blank? - gitaly_migrate(:search_files_by_name) do |is_enabled| - if is_enabled - gitaly_repository_client.search_files_by_name(ref, safe_query) - else - args = %W(ls-tree -r --name-status --full-tree #{ref} -- #{safe_query}) - - run_git(args).first.lines.map(&:strip) - end - end + gitaly_repository_client.search_files_by_name(ref, safe_query) end def find_commits_by_message(query, ref, path, limit, offset) @@ -1514,10 +1276,6 @@ module Gitlab run_git!(args, lazy_block: block) end - def missed_ref(oldrev, newrev) - run_git!(['rev-list', '--max-count=1', oldrev, "^#{newrev}"]) - end - def with_worktree(worktree_path, branch, sparse_checkout_files: nil, env:) base_args = %w(worktree add --detach) @@ -1621,21 +1379,6 @@ module Gitlab end end - # This function is duplicated in Gitaly-Go, don't change it! - # https://gitlab.com/gitlab-org/gitaly/merge_requests/698 - def fresh_worktree?(path) - File.exist?(path) && !clean_stuck_worktree(path) - end - - # This function is duplicated in Gitaly-Go, don't change it! - # https://gitlab.com/gitlab-org/gitaly/merge_requests/698 - def clean_stuck_worktree(path) - return false unless File.mtime(path) < 15.minutes.ago - - FileUtils.rm_rf(path) - true - end - # Adding a worktree means checking out the repository. For large repos, # this can be very expensive, so set up sparse checkout for the worktree # to only check out the files we're interested in. @@ -1861,17 +1604,6 @@ module Gitlab tmp_entry end - # Return the Rugged patches for the diff between +from+ and +to+. - def diff_patches(from, to, options = {}, *paths) - options ||= {} - break_rewrites = options[:break_rewrites] - actual_options = Gitlab::Git::Diff.filter_diff_options(options.merge(paths: paths)) - - diff = rugged.diff(from, to, actual_options) - diff.find_similar!(break_rewrites: break_rewrites) - diff.each_patch - end - def sort_branches(branches, sort_by) case sort_by when 'name' @@ -1894,106 +1626,6 @@ module Gitlab commit(sha) end - def size_by_shelling_out - popen(%w(du -sk), path).first.strip.to_i - end - - def size_by_gitaly - gitaly_repository_client.repository_size - end - - def count_commits_by_gitaly(options) - if options[:left_right] - from = options[:from] - to = options[:to] - - right_count = gitaly_commit_client - .commit_count("#{from}..#{to}", options) - left_count = gitaly_commit_client - .commit_count("#{to}..#{from}", options) - - [left_count, right_count] - else - gitaly_commit_client.commit_count(options[:ref], options) - end - end - - def count_commits_by_shelling_out(options) - cmd = count_commits_shelling_command(options) - - raw_output, _status = run_git(cmd) - - process_count_commits_raw_output(raw_output, options) - end - - def count_commits_shelling_command(options) - cmd = %w[rev-list] - cmd << "--after=#{options[:after].iso8601}" if options[:after] - cmd << "--before=#{options[:before].iso8601}" if options[:before] - cmd << "--max-count=#{options[:max_count]}" if options[:max_count] - cmd << "--left-right" if options[:left_right] - cmd << '--count' - - cmd << if options[:all] - '--all' - elsif options[:ref] - options[:ref] - else - raise ArgumentError, "Please specify a valid ref or set the 'all' attribute to true" - end - - cmd += %W[-- #{options[:path]}] if options[:path].present? - cmd - end - - def process_count_commits_raw_output(raw_output, options) - if options[:left_right] - result = raw_output.scan(/\d+/).map(&:to_i) - - if result.sum != options[:max_count] - result - else # Reaching max count, right is not accurate - right_option = - process_count_commits_options(options - .except(:left_right, :from, :to) - .merge(ref: options[:to])) - - right = count_commits_by_shelling_out(right_option) - - [result.first, right] # left should be accurate in the first call - end - else - raw_output.to_i - end - end - - def gitaly_ls_files(ref) - gitaly_commit_client.ls_files(ref) - end - - def git_ls_files(ref) - actual_ref = ref || root_ref - - begin - sha_from_ref(actual_ref) - rescue Rugged::OdbError, Rugged::InvalidError, Rugged::ReferenceError - # Return an empty array if the ref wasn't found - return [] - end - - cmd = %W(ls-tree -r --full-tree --full-name -- #{actual_ref}) - raw_output, _status = run_git(cmd) - - lines = raw_output.split("\n").map do |f| - stuff, path = f.split("\t") - _mode, type, _sha = stuff.split(" ") - path if type == "blob" - # Contain only blob type - end - - lines.compact - end - # Returns true if the given ref name exists # # Ref names must start with `refs/`. @@ -2033,33 +1665,6 @@ module Gitlab false end - def gitaly_add_tag(tag_name, user:, target:, message: nil) - gitaly_operations_client.add_tag(tag_name, user, target, message) - end - - def rugged_add_tag(tag_name, user:, target:, message: nil) - target_object = Ref.dereference_object(lookup(target)) - raise InvalidRef.new("target not found: #{target}") unless target_object - - user = Gitlab::Git::User.from_gitlab(user) unless user.respond_to?(:gl_id) - - options = nil # Use nil, not the empty hash. Rugged cares about this. - if message - options = { - message: message, - tagger: Gitlab::Git.committer_hash(email: user.email, name: user.name) - } - end - - Gitlab::Git::OperationService.new(user, self).add_tag(tag_name, target_object.oid, options) - - find_tag(tag_name) - rescue Rugged::ReferenceError => ex - raise InvalidRef, ex - rescue Rugged::TagError - raise TagExistsError - end - def rugged_create_branch(ref, start_point) rugged_ref = rugged.branches.create(ref, start_point) target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) @@ -2104,28 +1709,6 @@ module Gitlab end end - def rugged_revert(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) - OperationService.new(user, self).with_branch( - branch_name, - start_branch_name: start_branch_name, - start_repository: start_repository - ) do |start_commit| - - Gitlab::Git.check_namespace!(commit, start_repository) - - revert_tree_id = check_revert_content(commit, start_commit.sha) - raise CreateTreeError unless revert_tree_id - - committer = user_to_committer(user) - - create_commit(message: message, - author: committer, - committer: committer, - tree: revert_tree_id, - parents: [start_commit.sha]) - end - end - def rugged_cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) OperationService.new(user, self).with_branch( branch_name, @@ -2165,72 +1748,6 @@ module Gitlab tree_id end - def gitaly_rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) - gitaly_operation_client.user_rebase(user, rebase_id, - branch: branch, - branch_sha: branch_sha, - remote_repository: remote_repository, - remote_branch: remote_branch) - end - - def git_rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:) - rebase_path = worktree_path(REBASE_WORKTREE_PREFIX, rebase_id) - env = git_env_for_user(user) - - if remote_repository.is_a?(RemoteRepository) - env.merge!(remote_repository.fetch_env) - remote_repo_path = GITALY_INTERNAL_URL - else - remote_repo_path = remote_repository.path - end - - with_worktree(rebase_path, branch, env: env) do - run_git!( - %W(pull --rebase #{remote_repo_path} #{remote_branch}), - chdir: rebase_path, env: env - ) - - rebase_sha = run_git!(%w(rev-parse HEAD), chdir: rebase_path, env: env).strip - - Gitlab::Git::OperationService.new(user, self) - .update_branch(branch, rebase_sha, branch_sha) - - rebase_sha - end - end - - def git_squash(user, squash_id, branch, start_sha, end_sha, author, message) - squash_path = worktree_path(SQUASH_WORKTREE_PREFIX, squash_id) - env = git_env_for_user(user).merge( - 'GIT_AUTHOR_NAME' => author.name, - 'GIT_AUTHOR_EMAIL' => author.email - ) - diff_range = "#{start_sha}...#{end_sha}" - diff_files = run_git!( - %W(diff --name-only --diff-filter=ar --binary #{diff_range}) - ).chomp - - with_worktree(squash_path, branch, sparse_checkout_files: diff_files, env: env) do - # Apply diff of the `diff_range` to the worktree - diff = run_git!(%W(diff --binary #{diff_range})) - run_git!(%w(apply --index --whitespace=nowarn), chdir: squash_path, env: env) do |stdin| - stdin.binmode - stdin.write(diff) - end - - # Commit the `diff_range` diff - run_git!(%W(commit --no-verify --message #{message}), chdir: squash_path, env: env) - - # Return the squash sha. May print a warning for ambiguous refs, but - # we can ignore that with `--quiet` and just take the SHA, if present. - # HEAD here always refers to the current HEAD commit, even if there is - # another ref called HEAD. - run_git!( - %w(rev-parse --quiet --verify HEAD), chdir: squash_path, env: env - ).chomp - end - end - def local_fetch_ref(source_path, source_ref:, target_ref:) args = %W(fetch --no-tags -f #{source_path} #{source_ref}:#{target_ref}) run_git(args) @@ -2242,22 +1759,6 @@ module Gitlab run_git(args, env: source_repository.fetch_env) end - def gitaly_ff_merge(user, source_sha, target_branch) - gitaly_operations_client.user_ff_branch(user, source_sha, target_branch) - rescue GRPC::FailedPrecondition => e - raise CommitError, e - end - - def rugged_ff_merge(user, source_sha, target_branch) - OperationService.new(user, self).with_branch(target_branch) do |our_commit| - raise ArgumentError, 'Invalid merge target' unless our_commit - - source_sha - end - rescue Rugged::ReferenceError, InvalidRef - raise ArgumentError, 'Invalid merge source' - end - def rugged_add_remote(remote_name, url, mirror_refmap) rugged.remotes.create(remote_name, url) @@ -2309,51 +1810,10 @@ module Gitlab remove_remote(remote_name) end - def rugged_multi_action( - user, branch_name, message, actions, author_email, author_name, - start_branch_name, start_repository) - - OperationService.new(user, self).with_branch( - branch_name, - start_branch_name: start_branch_name, - start_repository: start_repository - ) do |start_commit| - index = Gitlab::Git::Index.new(self) - parents = [] - - if start_commit - index.read_tree(start_commit.rugged_commit.tree) - parents = [start_commit.sha] - end - - actions.each { |opts| index.apply(opts.delete(:action), opts) } - - committer = user_to_committer(user) - author = Gitlab::Git.committer_hash(email: author_email, name: author_name) || committer - options = { - tree: index.write_tree, - message: message, - parents: parents, - author: author, - committer: committer - } - - create_commit(options) - end - end - def fetch_remote(remote_name = 'origin', env: nil) run_git(['fetch', remote_name], env: env).last.zero? end - def gitaly_can_be_merged?(their_commit, our_commit) - !gitaly_conflicts_client(our_commit, their_commit).conflicts? - end - - def rugged_can_be_merged?(their_commit, our_commit) - !rugged.merge_commits(our_commit, their_commit).conflicts? - end - def gitlab_projects_error raise CommandError, @gitlab_projects.output end @@ -2393,16 +1853,6 @@ module Gitlab nil end - def rugged_commit_count(ref) - walker = Rugged::Walker.new(rugged) - walker.sorting(Rugged::SORT_TOPO | Rugged::SORT_REVERSE) - oid = rugged.rev_parse_oid(ref) - walker.push(oid) - walker.count - rescue Rugged::ReferenceError - 0 - end - def rev_list_param(spec) spec == :all ? ['--all'] : spec end diff --git a/lib/gitlab/git/rev_list.rb b/lib/gitlab/git/rev_list.rb index 4e661eceffb..5fdad077eea 100644 --- a/lib/gitlab/git/rev_list.rb +++ b/lib/gitlab/git/rev_list.rb @@ -1,5 +1,3 @@ -# Gitaly note: JV: will probably be migrated indirectly by migrating the call sites. - module Gitlab module Git class RevList @@ -45,13 +43,6 @@ module Gitlab &lazy_block) end - # This methods returns an array of missed references - # - # Should become obsolete after https://gitlab.com/gitlab-org/gitaly/issues/348. - def missed_ref - repository.missed_ref(oldrev, newrev).split("\n") - end - private def execute(args) diff --git a/lib/gitlab/git/tag.rb b/lib/gitlab/git/tag.rb index e44284572fd..bbf2ecdb1fa 100644 --- a/lib/gitlab/git/tag.rb +++ b/lib/gitlab/git/tag.rb @@ -28,18 +28,7 @@ module Gitlab end def get_messages(repository, tag_ids) - repository.gitaly_migrate(:tag_messages) do |is_enabled| - if is_enabled - repository.gitaly_ref_client.get_tag_messages(tag_ids) - else - tag_ids.map do |id| - tag = repository.rugged.lookup(id) - message = tag.is_a?(Rugged::Commit) ? "" : tag.message - - [id, message] - end.to_h - end - end + repository.gitaly_ref_client.get_tag_messages(tag_ids) end end diff --git a/lib/gitlab/git/version.rb b/lib/gitlab/git/version.rb index 11184ca3457..1e14e8b652a 100644 --- a/lib/gitlab/git/version.rb +++ b/lib/gitlab/git/version.rb @@ -4,7 +4,7 @@ module Gitlab extend Gitlab::Git::Popen def self.git_version - Gitlab::VersionInfo.parse(popen(%W(#{Gitlab.config.git.bin_path} --version), nil).first) + Gitlab::VersionInfo.parse(Gitaly::Server.all.first.git_binary_version) end end end diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index 1ab8c4e0229..8ee46b59830 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -27,63 +27,38 @@ module Gitlab end def write_page(name, format, content, commit_details) - @repository.gitaly_migrate(:wiki_write_page) do |is_enabled| - if is_enabled - gitaly_write_page(name, format, content, commit_details) - else - gollum_write_page(name, format, content, commit_details) - end + @repository.wrapped_gitaly_errors do + gitaly_write_page(name, format, content, commit_details) end end def delete_page(page_path, commit_details) - @repository.gitaly_migrate(:wiki_delete_page) do |is_enabled| - if is_enabled - gitaly_delete_page(page_path, commit_details) - else - gollum_delete_page(page_path, commit_details) - end + @repository.wrapped_gitaly_errors do + gitaly_delete_page(page_path, commit_details) end end def update_page(page_path, title, format, content, commit_details) - @repository.gitaly_migrate(:wiki_update_page) do |is_enabled| - if is_enabled - gitaly_update_page(page_path, title, format, content, commit_details) - else - gollum_update_page(page_path, title, format, content, commit_details) - end + @repository.wrapped_gitaly_errors do + gitaly_update_page(page_path, title, format, content, commit_details) end end def pages(limit: nil) - @repository.gitaly_migrate(:wiki_get_all_pages) do |is_enabled| - if is_enabled - gitaly_get_all_pages - else - gollum_get_all_pages(limit: limit) - end + @repository.wrapped_gitaly_errors do + gitaly_get_all_pages end end def page(title:, version: nil, dir: nil) - @repository.gitaly_migrate(:wiki_find_page, - status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_find_page(title: title, version: version, dir: dir) - else - gollum_find_page(title: title, version: version, dir: dir) - end + @repository.wrapped_gitaly_errors do + gitaly_find_page(title: title, version: version, dir: dir) end end def file(name, version) - @repository.gitaly_migrate(:wiki_find_file) do |is_enabled| - if is_enabled - gitaly_find_file(name, version) - else - gollum_find_file(name, version) - end + @repository.wrapped_gitaly_errors do + gitaly_find_file(name, version) end end @@ -92,24 +67,15 @@ module Gitlab # :per_page - The number of items per page. # :limit - Total number of items to return. def page_versions(page_path, options = {}) - @repository.gitaly_migrate(:wiki_page_versions) do |is_enabled| - if is_enabled - versions = gitaly_wiki_client.page_versions(page_path, options) - - # Gitaly uses gollum-lib to get the versions. Gollum defaults to 20 - # per page, but also fetches 20 if `limit` or `per_page` < 20. - # Slicing returns an array with the expected number of items. - slice_bound = options[:limit] || options[:per_page] || Gollum::Page.per_page - versions[0..slice_bound] - else - current_page = gollum_page_by_path(page_path) - - commits_from_page(current_page, options).map do |gitlab_git_commit| - gollum_page = gollum_wiki.page(current_page.title, gitlab_git_commit.id) - Gitlab::Git::WikiPageVersion.new(gitlab_git_commit, gollum_page&.format) - end - end + versions = @repository.wrapped_gitaly_errors do + gitaly_wiki_client.page_versions(page_path, options) end + + # Gitaly uses gollum-lib to get the versions. Gollum defaults to 20 + # per page, but also fetches 20 if `limit` or `per_page` < 20. + # Slicing returns an array with the expected number of items. + slice_bound = options[:limit] || options[:per_page] || Gollum::Page.per_page + versions[0..slice_bound] end def count_page_versions(page_path) @@ -131,46 +97,13 @@ module Gitlab def page_formatted_data(title:, dir: nil, version: nil) version = version&.id - @repository.gitaly_migrate(:wiki_page_formatted_data, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - gitaly_wiki_client.get_formatted_data(title: title, dir: dir, version: version) - else - # We don't use #page because if wiki_find_page feature is enabled, we would - # get a page without formatted_data. - gollum_find_page(title: title, dir: dir, version: version)&.formatted_data - end + @repository.wrapped_gitaly_errors do + gitaly_wiki_client.get_formatted_data(title: title, dir: dir, version: version) end end - def gollum_wiki - @gollum_wiki ||= Gollum::Wiki.new(@repository.path) - end - private - # options: - # :page - The Integer page number. - # :per_page - The number of items per page. - # :limit - Total number of items to return. - def commits_from_page(gollum_page, options = {}) - unless options[:limit] - options[:offset] = ([1, options.delete(:page).to_i].max - 1) * Gollum::Page.per_page - options[:limit] = (options.delete(:per_page) || Gollum::Page.per_page).to_i - end - - @repository.log(ref: gollum_page.last_version.id, - path: gollum_page.path, - limit: options[:limit], - offset: options[:offset]) - end - - def gollum_page_by_path(page_path) - page_name = Gollum::Page.canonicalize_filename(page_path) - page_dir = File.split(page_path).first - - gollum_wiki.paged(page_name, page_dir) - end - def new_page(gollum_page) Gitlab::Git::WikiPage.new(gollum_page, new_version(gollum_page, gollum_page.version.id)) end @@ -199,65 +132,6 @@ module Gitlab @gitaly_wiki_client ||= Gitlab::GitalyClient::WikiService.new(@repository) end - def gollum_write_page(name, format, content, commit_details) - assert_type!(format, Symbol) - assert_type!(commit_details, CommitDetails) - - with_committer_with_hooks(commit_details) do |committer| - filename = File.basename(name) - dir = (tmp_dir = File.dirname(name)) == '.' ? '' : tmp_dir - - gollum_wiki.write_page(filename, format, content, { committer: committer }, dir) - end - rescue Gollum::DuplicatePageError => e - raise Gitlab::Git::Wiki::DuplicatePageError, e.message - end - - def gollum_delete_page(page_path, commit_details) - assert_type!(commit_details, CommitDetails) - - with_committer_with_hooks(commit_details) do |committer| - gollum_wiki.delete_page(gollum_page_by_path(page_path), committer: committer) - end - end - - def gollum_update_page(page_path, title, format, content, commit_details) - assert_type!(format, Symbol) - assert_type!(commit_details, CommitDetails) - - with_committer_with_hooks(commit_details) do |committer| - page = gollum_page_by_path(page_path) - # Instead of performing two renames if the title has changed, - # the update_page will only update the format and content and - # the rename_page will do anything related to moving/renaming - gollum_wiki.update_page(page, page.name, format, content, committer: committer) - gollum_wiki.rename_page(page, title, committer: committer) - end - end - - def gollum_find_page(title:, version: nil, dir: nil) - if version - version = Gitlab::Git::Commit.find(@repository, version).id - end - - gollum_page = gollum_wiki.page(title, version, dir) - return unless gollum_page - - new_page(gollum_page) - end - - def gollum_find_file(name, version) - version ||= self.class.default_ref - gollum_file = gollum_wiki.file(name, version) - return unless gollum_file - - Gitlab::Git::WikiFile.new(gollum_file) - end - - def gollum_get_all_pages(limit: nil) - gollum_wiki.pages(limit: limit).map { |gollum_page| new_page(gollum_page) } - end - def gitaly_write_page(name, format, content, commit_details) gitaly_wiki_client.write_page(name, format, content, commit_details) end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 7f2e6441f16..d979ba0eb14 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -76,6 +76,13 @@ module Gitlab end def tree_entry(ref, path, limit = nil) + if Pathname.new(path).cleanpath.to_s.start_with?('../') + # The TreeEntry RPC should return an empty reponse in this case but in + # Gitaly 0.107.0 and earlier we get an exception instead. This early return + # saves us a Gitaly roundtrip while also avoiding the exception. + return + end + request = Gitaly::TreeEntryRequest.new( repository: @gitaly_repo, revision: encode_binary(ref), @@ -317,6 +324,8 @@ module Gitlab return if signature.blank? && signed_text.blank? [signature, signed_text] + rescue GRPC::InvalidArgument => ex + raise ArgumentError, ex end def get_commit_signatures(commit_ids) @@ -334,6 +343,8 @@ module Gitlab end signatures + rescue GRPC::InvalidArgument => ex + raise ArgumentError, ex end def get_commit_messages(commit_ids) @@ -357,7 +368,7 @@ module Gitlab def call_commit_diff(request_params, options = {}) request_params[:ignore_whitespace_change] = options.fetch(:ignore_whitespace_change, false) request_params[:enforce_limits] = options.fetch(:limits, true) - request_params[:collapse_diffs] = request_params[:enforce_limits] || !options.fetch(:expanded, true) + request_params[:collapse_diffs] = !options.fetch(:expanded, true) request_params.merge!(Gitlab::Git::DiffCollection.collection_limits(options).to_h) request = Gitaly::CommitDiffRequest.new(request_params) diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index e9d4bb4c4b6..c04183a348f 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -64,6 +64,8 @@ module Gitlab target_commit = Gitlab::Git::Commit.decorate(@repository, branch.target_commit) Gitlab::Git::Branch.new(@repository, branch.name, target_commit.id, target_commit) + rescue GRPC::FailedPrecondition => ex + raise Gitlab::Git::Repository::InvalidRef, ex end def user_delete_branch(branch_name, user) @@ -133,6 +135,8 @@ module Gitlab request ).branch_update Gitlab::Git::OperationService::BranchUpdate.from_gitaly(branch_update) + rescue GRPC::FailedPrecondition => e + raise Gitlab::Git::CommitError, e end def user_cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 4340f779e53..ca986434221 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -196,20 +196,21 @@ module Gitlab end def create_bundle(save_path) - request = Gitaly::CreateBundleRequest.new(repository: @gitaly_repo) - response = GitalyClient.call( - @storage, - :repository_service, + gitaly_fetch_stream_to_file( + save_path, :create_bundle, - request, - timeout: GitalyClient.default_timeout + Gitaly::CreateBundleRequest, + GitalyClient.default_timeout ) + end - File.open(save_path, 'wb') do |f| - response.each do |message| - f.write(message.data) - end - end + def backup_custom_hooks(save_path) + gitaly_fetch_stream_to_file( + save_path, + :backup_custom_hooks, + Gitaly::BackupCustomHooksRequest, + GitalyClient.default_timeout + ) end def create_from_bundle(bundle_path) @@ -309,6 +310,25 @@ module Gitlab private + def gitaly_fetch_stream_to_file(save_path, rpc_name, request_class, timeout) + request = request_class.new(repository: @gitaly_repo) + response = GitalyClient.call( + @storage, + :repository_service, + rpc_name, + request, + timeout: timeout + ) + + File.open(save_path, 'wb') do |f| + response.each do |message| + f.write(message.data) + end + end + # If the file is empty means that we recieved an empty stream, we delete the file + FileUtils.rm(save_path) if File.zero?(save_path) + end + def gitaly_repo_stream_request(file_path, rpc_name, request_class, timeout) request = request_class.new(repository: @gitaly_repo) enum = Enumerator.new do |y| diff --git a/lib/gitlab/graphql/connections.rb b/lib/gitlab/graphql/connections.rb new file mode 100644 index 00000000000..2582ffeb2a8 --- /dev/null +++ b/lib/gitlab/graphql/connections.rb @@ -0,0 +1,12 @@ +module Gitlab + module Graphql + module Connections + def self.use(_schema) + GraphQL::Relay::BaseConnection.register_connection_implementation( + ActiveRecord::Relation, + Gitlab::Graphql::Connections::KeysetConnection + ) + end + end + end +end diff --git a/lib/gitlab/graphql/connections/keyset_connection.rb b/lib/gitlab/graphql/connections/keyset_connection.rb new file mode 100644 index 00000000000..abee2afe144 --- /dev/null +++ b/lib/gitlab/graphql/connections/keyset_connection.rb @@ -0,0 +1,73 @@ +module Gitlab + module Graphql + module Connections + class KeysetConnection < GraphQL::Relay::BaseConnection + def cursor_from_node(node) + encode(node[order_field].to_s) + end + + def sliced_nodes + @sliced_nodes ||= + begin + sliced = nodes + + sliced = sliced.where(before_slice) if before.present? + sliced = sliced.where(after_slice) if after.present? + + sliced + end + end + + def paged_nodes + if first && last + raise Gitlab::Graphql::Errors::ArgumentError.new("Can only provide either `first` or `last`, not both") + end + + if last + sliced_nodes.last(limit_value) + else + sliced_nodes.limit(limit_value) + end + end + + private + + def before_slice + if sort_direction == :asc + table[order_field].lt(decode(before)) + else + table[order_field].gt(decode(before)) + end + end + + def after_slice + if sort_direction == :asc + table[order_field].gt(decode(after)) + else + table[order_field].lt(decode(after)) + end + end + + def limit_value + @limit_value ||= [first, last, max_page_size].compact.min + end + + def table + nodes.arel_table + end + + def order_info + @order_info ||= nodes.order_values.first + end + + def order_field + @order_field ||= order_info&.expr&.name || nodes.primary_key + end + + def sort_direction + @order_direction ||= order_info&.direction || :desc + end + end + end + end +end diff --git a/lib/gitlab/graphql/errors.rb b/lib/gitlab/graphql/errors.rb new file mode 100644 index 00000000000..1d8e8307ab9 --- /dev/null +++ b/lib/gitlab/graphql/errors.rb @@ -0,0 +1,8 @@ +module Gitlab + module Graphql + module Errors + BaseError = Class.new(GraphQL::ExecutionError) + ArgumentError = Class.new(BaseError) + end + end +end diff --git a/lib/gitlab/graphql/expose_permissions.rb b/lib/gitlab/graphql/expose_permissions.rb new file mode 100644 index 00000000000..e3779995406 --- /dev/null +++ b/lib/gitlab/graphql/expose_permissions.rb @@ -0,0 +1,15 @@ +module Gitlab + module Graphql + module ExposePermissions + extend ActiveSupport::Concern + prepended do + def self.expose_permissions(permission_type, description: 'Permissions for the current user on the resource') + field :user_permissions, permission_type, + description: description, + null: false, + resolve: -> (obj, _, _) { obj } + end + end + end + end +end diff --git a/lib/gitlab/graphql/present/instrumentation.rb b/lib/gitlab/graphql/present/instrumentation.rb index 1688262974b..f87fd147b15 100644 --- a/lib/gitlab/graphql/present/instrumentation.rb +++ b/lib/gitlab/graphql/present/instrumentation.rb @@ -3,6 +3,8 @@ module Gitlab module Present class Instrumentation def instrument(type, field) + return field unless field.metadata[:type_class] + presented_in = field.metadata[:type_class].owner return field unless presented_in.respond_to?(:presenter_class) return field unless presented_in.presenter_class @@ -10,9 +12,18 @@ module Gitlab old_resolver = field.resolve_proc resolve_with_presenter = -> (presented_type, args, context) do + # We need to wrap the original presentation type into a type that + # uses the presenter as an object. object = presented_type.object + + if object.is_a?(presented_in.presenter_class) + next old_resolver.call(presented_type, args, context) + end + presenter = presented_in.presenter_class.new(object, **context.to_h) - old_resolver.call(presenter, args, context) + wrapped = presented_type.class.new(presenter, context) + + old_resolver.call(wrapped, args, context) end field.redefine do diff --git a/lib/gitlab/health_checks/db_check.rb b/lib/gitlab/health_checks/db_check.rb index e27e16ddaf6..08495c0a59e 100644 --- a/lib/gitlab/health_checks/db_check.rb +++ b/lib/gitlab/health_checks/db_check.rb @@ -17,7 +17,7 @@ module Gitlab def check catch_timeout 10.seconds do if Gitlab::Database.postgresql? - ActiveRecord::Base.connection.execute('SELECT 1 as ping')&.first&.[]('ping') + ActiveRecord::Base.connection.execute('SELECT 1 as ping')&.first&.[]('ping')&.to_s else ActiveRecord::Base.connection.execute('SELECT 1 as ping')&.first&.first&.to_s end diff --git a/lib/gitlab/health_checks/fs_shards_check.rb b/lib/gitlab/health_checks/fs_shards_check.rb deleted file mode 100644 index fcbf266b80b..00000000000 --- a/lib/gitlab/health_checks/fs_shards_check.rb +++ /dev/null @@ -1,168 +0,0 @@ -module Gitlab - module HealthChecks - class FsShardsCheck - extend BaseAbstractCheck - RANDOM_STRING = SecureRandom.hex(1000).freeze - COMMAND_TIMEOUT = '1'.freeze - TIMEOUT_EXECUTABLE = 'timeout'.freeze - - class << self - def readiness - repository_storages.map do |storage_name| - begin - if !storage_circuitbreaker_test(storage_name) - HealthChecks::Result.new(false, 'circuitbreaker tripped', shard: storage_name) - elsif !storage_stat_test(storage_name) - HealthChecks::Result.new(false, 'cannot stat storage', shard: storage_name) - else - with_temp_file(storage_name) do |tmp_file_path| - if !storage_write_test(tmp_file_path) - HealthChecks::Result.new(false, 'cannot write to storage', shard: storage_name) - elsif !storage_read_test(tmp_file_path) - HealthChecks::Result.new(false, 'cannot read from storage', shard: storage_name) - else - HealthChecks::Result.new(true, nil, shard: storage_name) - end - end - end - rescue RuntimeError => ex - message = "unexpected error #{ex} when checking storage #{storage_name}" - Rails.logger.error(message) - HealthChecks::Result.new(false, message, shard: storage_name) - end - end - end - - def metrics - repository_storages.flat_map do |storage_name| - [ - storage_stat_metrics(storage_name), - storage_write_metrics(storage_name), - storage_read_metrics(storage_name), - storage_circuitbreaker_metrics(storage_name) - ].flatten - end - end - - private - - def operation_metrics(ok_metric, latency_metric, **labels) - result, elapsed = yield - [ - metric(latency_metric, elapsed, **labels), - metric(ok_metric, result ? 1 : 0, **labels) - ] - rescue RuntimeError => ex - Rails.logger.error("unexpected error #{ex} when checking #{ok_metric}") - [metric(ok_metric, 0, **labels)] - end - - def repository_storages - storages_paths.keys - end - - def storages_paths - Gitlab.config.repositories.storages - end - - def exec_with_timeout(cmd_args, *args, &block) - Gitlab::Popen.popen([TIMEOUT_EXECUTABLE, COMMAND_TIMEOUT].concat(cmd_args), *args, &block) - end - - def with_temp_file(storage_name) - temp_file_path = Dir::Tmpname.create(%w(fs_shards_check +deleted), storage_path(storage_name)) { |path| path } - yield temp_file_path - ensure - delete_test_file(temp_file_path) - end - - def storage_path(storage_name) - Gitlab::GitalyClient::StorageSettings.allow_disk_access do - storages_paths[storage_name]&.legacy_disk_path - end - end - - # All below test methods use shell commands to perform actions on storage volumes. - # In case a storage volume have connectivity problems causing pure Ruby IO operation to wait indefinitely, - # we can rely on shell commands to be terminated once `timeout` kills them. - # - # However we also fallback to pure Ruby file operations in case a specific shell command is missing - # so we are still able to perform healthchecks and gather metrics from such system. - - def delete_test_file(tmp_path) - _, status = exec_with_timeout(%W{ rm -f #{tmp_path} }) - status.zero? - rescue Errno::ENOENT - File.delete(tmp_path) rescue Errno::ENOENT - end - - def storage_stat_test(storage_name) - stat_path = File.join(storage_path(storage_name), '.') - begin - _, status = exec_with_timeout(%W{ stat #{stat_path} }) - status.zero? - rescue Errno::ENOENT - File.exist?(stat_path) && File::Stat.new(stat_path).readable? - end - end - - def storage_write_test(tmp_path) - _, status = exec_with_timeout(%W{ tee #{tmp_path} }) do |stdin| - stdin.write(RANDOM_STRING) - end - status.zero? - rescue Errno::ENOENT - written_bytes = File.write(tmp_path, RANDOM_STRING) rescue Errno::ENOENT - written_bytes == RANDOM_STRING.length - end - - def storage_read_test(tmp_path) - _, status = exec_with_timeout(%W{ diff #{tmp_path} - }) do |stdin| - stdin.write(RANDOM_STRING) - end - status.zero? - rescue Errno::ENOENT - file_contents = File.read(tmp_path) rescue Errno::ENOENT - file_contents == RANDOM_STRING - end - - def storage_circuitbreaker_test(storage_name) - Gitlab::Git::Storage::CircuitBreaker.build(storage_name).perform { "OK" } - rescue Gitlab::Git::Storage::Inaccessible - nil - end - - def storage_stat_metrics(storage_name) - operation_metrics(:filesystem_accessible, :filesystem_access_latency_seconds, shard: storage_name) do - with_timing { storage_stat_test(storage_name) } - end - end - - def storage_write_metrics(storage_name) - operation_metrics(:filesystem_writable, :filesystem_write_latency_seconds, shard: storage_name) do - with_temp_file(storage_name) do |tmp_file_path| - with_timing { storage_write_test(tmp_file_path) } - end - end - end - - def storage_read_metrics(storage_name) - operation_metrics(:filesystem_readable, :filesystem_read_latency_seconds, shard: storage_name) do - with_temp_file(storage_name) do |tmp_file_path| - storage_write_test(tmp_file_path) # writes data used by read test - with_timing { storage_read_test(tmp_file_path) } - end - end - end - - def storage_circuitbreaker_metrics(storage_name) - operation_metrics(:filesystem_circuitbreaker, - :filesystem_circuitbreaker_latency_seconds, - shard: storage_name) do - with_timing { storage_circuitbreaker_test(storage_name) } - end - end - end - end - end -end diff --git a/lib/gitlab/health_checks/gitaly_check.rb b/lib/gitlab/health_checks/gitaly_check.rb index 11416c002e3..1f623e0b6ec 100644 --- a/lib/gitlab/health_checks/gitaly_check.rb +++ b/lib/gitlab/health_checks/gitaly_check.rb @@ -13,14 +13,14 @@ module Gitlab end def metrics - repository_storages.flat_map do |storage_name| - result, elapsed = with_timing { check(storage_name) } - labels = { shard: storage_name } + Gitaly::Server.all.flat_map do |server| + result, elapsed = with_timing { server.read_writeable? } + labels = { shard: server.storage } [ - metric("#{metric_prefix}_success", successful?(result) ? 1 : 0, **labels), + metric("#{metric_prefix}_success", result ? 1 : 0, **labels), metric("#{metric_prefix}_latency_seconds", elapsed, **labels) - ].flatten + ] end end @@ -36,10 +36,6 @@ module Gitlab METRIC_PREFIX end - def successful?(result) - result[:success] - end - def repository_storages storages.keys end diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 3772ef11c7f..343487bc361 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -21,7 +21,8 @@ module Gitlab 'nl_NL' => 'Nederlands', 'tr_TR' => 'Türkçe', 'id_ID' => 'Bahasa Indonesia', - 'fil_PH' => 'Filipino' + 'fil_PH' => 'Filipino', + 'pl_PL' => 'Polski' }.freeze def available_locales diff --git a/lib/gitlab/i18n/metadata_entry.rb b/lib/gitlab/i18n/metadata_entry.rb index 35d57459a3d..36fc1bcdcb7 100644 --- a/lib/gitlab/i18n/metadata_entry.rb +++ b/lib/gitlab/i18n/metadata_entry.rb @@ -3,16 +3,25 @@ module Gitlab class MetadataEntry attr_reader :entry_data + # Avoid testing too many plurals if `nplurals` was incorrectly set. + # Based on info on https://www.gnu.org/software/gettext/manual/html_node/Plural-forms.html + # which mentions special cases for numbers ending in 2 digits + MAX_FORMS_TO_TEST = 101 + def initialize(entry_data) @entry_data = entry_data end - def expected_plurals + def expected_forms return nil unless plural_information plural_information['nplurals'].to_i end + def forms_to_test + @forms_to_test ||= [expected_forms, MAX_FORMS_TO_TEST].compact.min + end + private def plural_information diff --git a/lib/gitlab/i18n/po_linter.rb b/lib/gitlab/i18n/po_linter.rb index 7d3ff8c7f58..d8e7269a2c2 100644 --- a/lib/gitlab/i18n/po_linter.rb +++ b/lib/gitlab/i18n/po_linter.rb @@ -1,6 +1,8 @@ module Gitlab module I18n class PoLinter + include Gitlab::Utils::StrongMemoize + attr_reader :po_path, :translation_entries, :metadata_entry, :locale VARIABLE_REGEX = /%{\w*}|%[a-z]/.freeze @@ -34,7 +36,7 @@ module Gitlab end @translation_entries = entries.map do |entry_data| - Gitlab::I18n::TranslationEntry.new(entry_data, metadata_entry.expected_plurals) + Gitlab::I18n::TranslationEntry.new(entry_data, metadata_entry.expected_forms) end nil @@ -48,7 +50,7 @@ module Gitlab translation_entries.each do |entry| errors_for_entry = validate_entry(entry) - errors[join_message(entry.msgid)] = errors_for_entry if errors_for_entry.any? + errors[entry.msgid] = errors_for_entry if errors_for_entry.any? end errors @@ -62,6 +64,7 @@ module Gitlab validate_newlines(errors, entry) validate_number_of_plurals(errors, entry) validate_unescaped_chars(errors, entry) + validate_translation(errors, entry) errors end @@ -81,35 +84,39 @@ module Gitlab end def validate_number_of_plurals(errors, entry) - return unless metadata_entry&.expected_plurals + return unless metadata_entry&.expected_forms return unless entry.translated? - if entry.has_plural? && entry.all_translations.size != metadata_entry.expected_plurals - errors << "should have #{metadata_entry.expected_plurals} "\ - "#{'translations'.pluralize(metadata_entry.expected_plurals)}" + if entry.has_plural? && entry.all_translations.size != metadata_entry.expected_forms + errors << "should have #{metadata_entry.expected_forms} "\ + "#{'translations'.pluralize(metadata_entry.expected_forms)}" end end def validate_newlines(errors, entry) - if entry.msgid_contains_newlines? + if entry.msgid_has_multiple_lines? errors << 'is defined over multiple lines, this breaks some tooling.' end - if entry.plural_id_contains_newlines? + if entry.plural_id_has_multiple_lines? errors << 'plural is defined over multiple lines, this breaks some tooling.' end - if entry.translations_contain_newlines? + if entry.translations_have_multiple_lines? errors << 'has translations defined over multiple lines, this breaks some tooling.' end end def validate_variables(errors, entry) if entry.has_singular_translation? + validate_variables_in_message(errors, entry.msgid, entry.msgid) + validate_variables_in_message(errors, entry.msgid, entry.singular_translation) end if entry.has_plural? + validate_variables_in_message(errors, entry.plural_id, entry.plural_id) + entry.plural_translations.each do |translation| validate_variables_in_message(errors, entry.plural_id, translation) end @@ -117,41 +124,98 @@ module Gitlab end def validate_variables_in_message(errors, message_id, message_translation) - message_id = join_message(message_id) required_variables = message_id.scan(VARIABLE_REGEX) validate_unnamed_variables(errors, required_variables) - validate_translation(errors, message_id, required_variables) validate_variable_usage(errors, message_translation, required_variables) end - def validate_translation(errors, message_id, used_variables) + def validate_translation(errors, entry) + Gitlab::I18n.with_locale(locale) do + if entry.has_plural? + translate_plural(entry) + else + translate_singular(entry) + end + end + + # `sprintf` could raise an `ArgumentError` when invalid passing something + # other than a Hash when using named variables + # + # `sprintf` could raise `TypeError` when passing a wrong type when using + # unnamed variables + # + # FastGettext::Translation could raise `RuntimeError` (raised as a string), + # or as subclassess `NoTextDomainConfigured` & `InvalidFormat` + # + # `FastGettext::Translation` could raise `ArgumentError` as subclassess + # `InvalidEncoding`, `IllegalSequence` & `InvalidCharacter` + rescue ArgumentError, TypeError, RuntimeError => e + errors << "Failure translating to #{locale}: #{e.message}" + end + + def translate_singular(entry) + used_variables = entry.msgid.scan(VARIABLE_REGEX) variables = fill_in_variables(used_variables) - begin - Gitlab::I18n.with_locale(locale) do - translated = if message_id.include?('|') - FastGettext::Translation.s_(message_id) - else - FastGettext::Translation._(message_id) - end + translation = if entry.msgid.include?('|') + FastGettext::Translation.s_(entry.msgid) + else + FastGettext::Translation._(entry.msgid) + end - translated % variables + translation % variables if used_variables.any? + end + + def translate_plural(entry) + used_variables = entry.plural_id.scan(VARIABLE_REGEX) + variables = fill_in_variables(used_variables) + + numbers_covering_all_plurals.map do |number| + translation = FastGettext::Translation.n_(entry.msgid, entry.plural_id, number) + + translation % variables if used_variables.any? + end + end + + def numbers_covering_all_plurals + @numbers_covering_all_plurals ||= calculate_numbers_covering_all_plurals + end + + def calculate_numbers_covering_all_plurals + required_numbers = [] + discovered_indexes = [] + counter = 0 + + while discovered_indexes.size < metadata_entry.forms_to_test && counter < Gitlab::I18n::MetadataEntry::MAX_FORMS_TO_TEST + index_for_count = index_for_pluralization(counter) + + unless discovered_indexes.include?(index_for_count) + discovered_indexes << index_for_count + required_numbers << counter end - # `sprintf` could raise an `ArgumentError` when invalid passing something - # other than a Hash when using named variables - # - # `sprintf` could raise `TypeError` when passing a wrong type when using - # unnamed variables - # - # FastGettext::Translation could raise `RuntimeError` (raised as a string), - # or as subclassess `NoTextDomainConfigured` & `InvalidFormat` - # - # `FastGettext::Translation` could raise `ArgumentError` as subclassess - # `InvalidEncoding`, `IllegalSequence` & `InvalidCharacter` - rescue ArgumentError, TypeError, RuntimeError => e - errors << "Failure translating to #{locale} with #{variables}: #{e.message}" + counter += 1 + end + + required_numbers + end + + def index_for_pluralization(counter) + # This calls the C function that defines the pluralization rule, it can + # return a boolean (`false` represents 0, `true` represents 1) or an integer + # that specifies the plural form to be used for the given number + pluralization_result = Gitlab::I18n.with_locale(locale) do + FastGettext.pluralisation_rule.call(counter) + end + + case pluralization_result + when false + 0 + when true + 1 + else + pluralization_result end end @@ -172,14 +236,18 @@ module Gitlab end def validate_unnamed_variables(errors, variables) - if variables.size > 1 && variables.any? { |variable_name| unnamed_variable?(variable_name) } + unnamed_variables, named_variables = variables.partition { |name| unnamed_variable?(name) } + + if unnamed_variables.any? && named_variables.any? + errors << 'is combining named variables with unnamed variables' + end + + if unnamed_variables.size > 1 errors << 'is combining multiple unnamed variables' end end def validate_variable_usage(errors, translation, required_variables) - translation = join_message(translation) - # We don't need to validate when the message is empty. # In this case we fall back to the default, which has all the the # required variables. @@ -205,10 +273,6 @@ module Gitlab def validate_flags(errors, entry) errors << "is marked #{entry.flag}" if entry.flag end - - def join_message(message) - Array(message).join - end end end end diff --git a/lib/gitlab/i18n/translation_entry.rb b/lib/gitlab/i18n/translation_entry.rb index e6c95afca7e..54adb98f42d 100644 --- a/lib/gitlab/i18n/translation_entry.rb +++ b/lib/gitlab/i18n/translation_entry.rb @@ -11,11 +11,11 @@ module Gitlab end def msgid - entry_data[:msgid] + @msgid ||= Array(entry_data[:msgid]).join end def plural_id - entry_data[:msgid_plural] + @plural_id ||= Array(entry_data[:msgid_plural]).join end def has_plural? @@ -23,12 +23,11 @@ module Gitlab end def singular_translation - all_translations.first if has_singular_translation? + all_translations.first.to_s if has_singular_translation? end def all_translations - @all_translations ||= entry_data.fetch_values(*translation_keys) - .reject(&:empty?) + @all_translations ||= translation_entries.map { |translation| Array(translation).join } end def translated? @@ -54,16 +53,16 @@ module Gitlab nplurals > 1 || !has_plural? end - def msgid_contains_newlines? - msgid.is_a?(Array) + def msgid_has_multiple_lines? + entry_data[:msgid].is_a?(Array) end - def plural_id_contains_newlines? - plural_id.is_a?(Array) + def plural_id_has_multiple_lines? + entry_data[:msgid_plural].is_a?(Array) end - def translations_contain_newlines? - all_translations.any? { |translation| translation.is_a?(Array) } + def translations_have_multiple_lines? + translation_entries.any? { |translation| translation.is_a?(Array) } end def msgid_contains_unescaped_chars? @@ -84,6 +83,11 @@ module Gitlab private + def translation_entries + @translation_entries ||= entry_data.fetch_values(*translation_keys) + .reject(&:empty?) + end + def translation_keys @translation_keys ||= entry_data.keys.select { |key| key.to_s =~ /\Amsgstr(\[\d+\])?\z/ } end diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index b713fa7e1cd..53fe2f8e436 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -3,7 +3,7 @@ module Gitlab extend self # For every version update, the version history in import_export.md has to be kept up to date. - VERSION = '0.2.3'.freeze + VERSION = '0.2.4'.freeze FILENAME_LIMIT = 50 def export_path(relative_path:) diff --git a/lib/gitlab/import_export/group_project_object_builder.rb b/lib/gitlab/import_export/group_project_object_builder.rb new file mode 100644 index 00000000000..6c2af770119 --- /dev/null +++ b/lib/gitlab/import_export/group_project_object_builder.rb @@ -0,0 +1,90 @@ +module Gitlab + module ImportExport + # Given a class, it finds or creates a new object + # (initializes in the case of Label) at group or project level. + # If it does not exist in the group, it creates it at project level. + # + # Example: + # `GroupProjectObjectBuilder.build(Label, label_attributes)` + # finds or initializes a label with the given attributes. + # + # It also adds some logic around Group Labels/Milestones for edge cases. + class GroupProjectObjectBuilder + def self.build(*args) + Project.transaction do + new(*args).find + end + end + + def initialize(klass, attributes) + @klass = klass < Label ? Label : klass + @attributes = attributes + @group = @attributes['group'] + @project = @attributes['project'] + end + + def find + find_object || @klass.create(project_attributes) + end + + private + + def find_object + @klass.where(where_clause).first + end + + def where_clause + @attributes.slice('title').map do |key, value| + scope_clause = table[:project_id].eq(@project.id) + scope_clause = scope_clause.or(table[:group_id].eq(@group.id)) if @group + + table[key].eq(value).and(scope_clause) + end.reduce(:or) + end + + def table + @table ||= @klass.arel_table + end + + def project_attributes + @attributes.except('group').tap do |atts| + if label? + atts['type'] = 'ProjectLabel' # Always create project labels + elsif milestone? + if atts['group_id'] # Transform new group milestones into project ones + atts['iid'] = nil + atts.delete('group_id') + else + claim_iid + end + end + end + end + + def label? + @klass == Label + end + + def milestone? + @klass == Milestone + end + + # If an existing group milestone used the IID + # claim the IID back and set the group milestone to use one available + # This is necessary to fix situations like the following: + # - Importing into a user namespace project with exported group milestones + # where the IID of the Group milestone could conflict with a project one. + def claim_iid + # The milestone has to be a group milestone, as it's the only case where + # we set the IID as the maximum. The rest of them are fixed. + milestone = @project.milestones.find_by(iid: @attributes['iid']) + + return unless milestone + + milestone.iid = nil + milestone.ensure_project_iid! + milestone.save! + end + end + end +end diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 4eb67fbe11e..76b99b1de16 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -1,8 +1,8 @@ module Gitlab module ImportExport class ProjectTreeRestorer - # Relations which cannot have both group_id and project_id at the same time - RESTRICT_PROJECT_AND_GROUP = %i(milestone milestones).freeze + # Relations which cannot be saved at project level (and have a group assigned) + GROUP_MODELS = [GroupLabel, Milestone].freeze def initialize(user:, shared:, project:) @path = File.join(shared.export_path, 'project.json') @@ -70,12 +70,23 @@ module Gitlab def save_relation_hash(relation_hash_batch, relation_key) relation_hash = create_relation(relation_key, relation_hash_batch) + remove_group_models(relation_hash) if relation_hash.is_a?(Array) + @saved = false unless restored_project.append_or_update_attribute(relation_key, relation_hash) # Restore the project again, extra query that skips holding the AR objects in memory @restored_project = Project.find(@project_id) end + # Remove project models that became group models as we found them at group level. + # This no longer required saving them at the root project level. + # For example, in the case of an existing group label that matched the title. + def remove_group_models(relation_hash) + relation_hash.reject! do |value| + GROUP_MODELS.include?(value.class) && value.group_id + end + end + def default_relation_list reader.tree.reject do |model| model.is_a?(Hash) && model[:project_members] @@ -170,7 +181,7 @@ module Gitlab def create_relation(relation, relation_hash_list) relation_array = [relation_hash_list].flatten.map do |relation_hash| Gitlab::ImportExport::RelationFactory.create(relation_sym: relation.to_sym, - relation_hash: parsed_relation_hash(relation_hash, relation.to_sym), + relation_hash: relation_hash, members_mapper: members_mapper, user: @user, project: @restored_project, @@ -180,18 +191,6 @@ module Gitlab relation_hash_list.is_a?(Array) ? relation_array : relation_array.first end - def parsed_relation_hash(relation_hash, relation_type) - if RESTRICT_PROJECT_AND_GROUP.include?(relation_type) - params = {} - params['group_id'] = restored_project.group.try(:id) if relation_hash['group_id'] - params['project_id'] = restored_project.id if relation_hash['project_id'] - else - params = { 'group_id' => restored_project.group.try(:id), 'project_id' => restored_project.id } - end - - relation_hash.merge(params) - end - def reader @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared) end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index c5cf290f191..091e81028bb 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -54,6 +54,8 @@ module Gitlab @project = project @imported_object_retries = 0 + @relation_hash['project_id'] = @project.id + # Remove excluded keys from relation_hash # We don't do this in the parsed_relation_hash because of the 'transformed attributes' # For example, MergeRequestDiffFiles exports its diff attribute as utf8_diff. Then, @@ -80,15 +82,12 @@ module Gitlab case @relation_name when :merge_request_diff_files then setup_diff when :notes then setup_note - when :project_label, :project_labels then setup_label - when :milestone, :milestones then setup_milestone when 'Ci::Pipeline' then setup_pipeline - else - @relation_hash['project_id'] = @project.id end update_user_references update_project_references + update_group_references remove_duplicate_assignees reset_tokens! @@ -151,39 +150,23 @@ module Gitlab end def update_project_references - project_id = @relation_hash.delete('project_id') - # If source and target are the same, populate them with the new project ID. if @relation_hash['source_project_id'] - @relation_hash['source_project_id'] = same_source_and_target? ? project_id : MergeRequestParser::FORKED_PROJECT_ID + @relation_hash['source_project_id'] = same_source_and_target? ? @relation_hash['project_id'] : MergeRequestParser::FORKED_PROJECT_ID end - # project_id may not be part of the export, but we always need to populate it if required. - @relation_hash['project_id'] = project_id - @relation_hash['target_project_id'] = project_id if @relation_hash['target_project_id'] + @relation_hash['target_project_id'] = @relation_hash['project_id'] if @relation_hash['target_project_id'] end def same_source_and_target? @relation_hash['target_project_id'] && @relation_hash['target_project_id'] == @relation_hash['source_project_id'] end - def setup_label - # If there's no group, move the label to a project label - if @relation_hash['type'] == 'GroupLabel' && @relation_hash['group_id'] - @relation_hash['project_id'] = nil - @relation_name = :group_label - else - @relation_hash['group_id'] = nil - @relation_hash['type'] = 'ProjectLabel' - end - end + def update_group_references + return unless EXISTING_OBJECT_CHECK.include?(@relation_name) + return unless @relation_hash['group_id'] - def setup_milestone - if @relation_hash['group_id'] - @relation_hash['group_id'] = @project.group.id - else - @relation_hash['project_id'] = @project.id - end + @relation_hash['group_id'] = @project.group&.id end def reset_tokens! @@ -271,15 +254,7 @@ module Gitlab end def existing_object - @existing_object ||= - begin - existing_object = find_or_create_object! - - # Done in two steps, as MySQL behaves differently than PostgreSQL using - # the +find_or_create_by+ method and does not return the ID the second time. - existing_object.update!(parsed_relation_hash) - existing_object - end + @existing_object ||= find_or_create_object! end def unknown_service? @@ -288,29 +263,16 @@ module Gitlab end def find_or_create_object! - finder_attributes = if @relation_name == :group_label - %w[title group_id] - elsif parsed_relation_hash['project_id'] - %w[title project_id] - else - %w[title group_id] - end - - finder_hash = parsed_relation_hash.slice(*finder_attributes) - - if label? - label = relation_class.find_or_initialize_by(finder_hash) - parsed_relation_hash.delete('priorities') if label.persisted? - - label.save! - label - else - relation_class.find_or_create_by(finder_hash) + return relation_class.find_or_create_by(project_id: @project.id) if @relation_name == :project_feature + + # Can't use IDs as validation exists calling `group` or `project` attributes + finder_hash = parsed_relation_hash.tap do |hash| + hash['group'] = @project.group if relation_class.attribute_method?('group_id') + hash['project'] = @project + hash.delete('project_id') end - end - def label? - @relation_name.to_s.include?('label') + GroupProjectObjectBuilder.build(relation_class, finder_hash) end end end diff --git a/lib/gitlab/kubernetes/helm/install_command.rb b/lib/gitlab/kubernetes/helm/install_command.rb index 30af3e97b4a..d2133a6d65b 100644 --- a/lib/gitlab/kubernetes/helm/install_command.rb +++ b/lib/gitlab/kubernetes/helm/install_command.rb @@ -2,11 +2,12 @@ module Gitlab module Kubernetes module Helm class InstallCommand < BaseCommand - attr_reader :name, :chart, :repository, :values + attr_reader :name, :chart, :version, :repository, :values - def initialize(name, chart:, values:, repository: nil) + def initialize(name, chart:, values:, version: nil, repository: nil) @name = name @chart = chart + @version = version @values = values @repository = repository end @@ -39,9 +40,13 @@ module Gitlab def script_command <<~HEREDOC - helm install #{chart} --name #{name} --namespace #{Gitlab::Kubernetes::Helm::NAMESPACE} -f /data/helm/#{name}/config/values.yaml >/dev/null + helm install #{chart} --name #{name}#{optional_version_flag} --namespace #{Gitlab::Kubernetes::Helm::NAMESPACE} -f /data/helm/#{name}/config/values.yaml >/dev/null HEREDOC end + + def optional_version_flag + " --version #{version}" if version + end end end end diff --git a/lib/gitlab/metrics/samplers/influx_sampler.rb b/lib/gitlab/metrics/samplers/influx_sampler.rb index 5a0f7f28fc8..ad97632e4eb 100644 --- a/lib/gitlab/metrics/samplers/influx_sampler.rb +++ b/lib/gitlab/metrics/samplers/influx_sampler.rb @@ -16,12 +16,6 @@ module Gitlab @last_minor_gc = Delta.new(GC.stat[:minor_gc_count]) @last_major_gc = Delta.new(GC.stat[:major_gc_count]) - - if Gitlab::Metrics.mri? - require 'allocations' - - Allocations.start - end end def sample diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index 4e1ea62351f..7b2b3bedf04 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -20,39 +20,29 @@ module Gitlab {} end - def initialize(interval) - super(interval) - - if Metrics.mri? - require 'allocations' - - Allocations.start - end - end - def init_metrics metrics = {} - metrics[:sampler_duration] = Metrics.histogram(with_prefix(:sampler_duration, :seconds), 'Sampler time', { worker: nil }) - metrics[:total_time] = Metrics.gauge(with_prefix(:gc, :time_total), 'Total GC time', labels, :livesum) + metrics[:sampler_duration] = Metrics.counter(with_prefix(:sampler, :duration_seconds_total), 'Sampler time', labels) + metrics[:total_time] = Metrics.counter(with_prefix(:gc, :duration_seconds_total), 'Total GC time', labels) GC.stat.keys.each do |key| - metrics[key] = Metrics.gauge(with_prefix(:gc, key), to_doc_string(key), labels, :livesum) + metrics[key] = Metrics.gauge(with_prefix(:gc_stat, key), to_doc_string(key), labels, :livesum) end - metrics[:objects_total] = Metrics.gauge(with_prefix(:objects, :total), 'Objects total', labels.merge(class: nil), :livesum) - metrics[:memory_usage] = Metrics.gauge(with_prefix(:memory, :usage_total), 'Memory used total', labels, :livesum) - metrics[:file_descriptors] = Metrics.gauge(with_prefix(:file, :descriptors_total), 'File descriptors total', labels, :livesum) + metrics[:memory_usage] = Metrics.gauge(with_prefix(:memory, :bytes), 'Memory used', labels, :livesum) + metrics[:file_descriptors] = Metrics.gauge(with_prefix(:file, :descriptors), 'File descriptors used', labels, :livesum) metrics end def sample start_time = System.monotonic_time - sample_gc - metrics[:memory_usage].set(labels, System.memory_usage) - metrics[:file_descriptors].set(labels, System.file_descriptor_count) + metrics[:memory_usage].set(labels.merge(worker_label), System.memory_usage) + metrics[:file_descriptors].set(labels.merge(worker_label), System.file_descriptor_count) + + sample_gc - metrics[:sampler_duration].observe(labels.merge(worker_label), System.monotonic_time - start_time) + metrics[:sampler_duration].increment(labels, System.monotonic_time - start_time) ensure GC::Profiler.clear end @@ -60,11 +50,13 @@ module Gitlab private def sample_gc - metrics[:total_time].set(labels, GC::Profiler.total_time * 1000) - + # Collect generic GC stats. GC.stat.each do |key, value| metrics[key].set(labels, value) end + + # Collect the GC time since last sample in float seconds. + metrics[:total_time].increment(labels, GC::Profiler.total_time) end def worker_label diff --git a/lib/gitlab/metrics/web_transaction.rb b/lib/gitlab/metrics/web_transaction.rb index 3799aaebf1c..723ca576aab 100644 --- a/lib/gitlab/metrics/web_transaction.rb +++ b/lib/gitlab/metrics/web_transaction.rb @@ -3,6 +3,7 @@ module Gitlab class WebTransaction < Transaction CONTROLLER_KEY = 'action_controller.instance'.freeze ENDPOINT_KEY = 'api.endpoint'.freeze + ALLOWED_SUFFIXES = Set.new(%w[json js atom rss xml zip]) def initialize(env) super() @@ -32,9 +33,13 @@ module Gitlab # Devise exposes a method called "request_format" that does the below. # However, this method is not available to all controllers (e.g. certain # Doorkeeper controllers). As such we use the underlying code directly. - suffix = controller.request.format.try(:ref) + suffix = controller.request.format.try(:ref).to_s - if suffix && suffix != :html + # Sometimes the request format is set to silly data such as + # "application/xrds+xml" or actual URLs. To prevent such values from + # increasing the cardinality of our metrics, we limit the number of + # possible suffixes. + if suffix && ALLOWED_SUFFIXES.include?(suffix) action += ".#{suffix}" end diff --git a/lib/gitlab/middleware/read_only/controller.rb b/lib/gitlab/middleware/read_only/controller.rb index 45b644e6510..4a99b7cca5c 100644 --- a/lib/gitlab/middleware/read_only/controller.rb +++ b/lib/gitlab/middleware/read_only/controller.rb @@ -4,8 +4,18 @@ module Gitlab class Controller DISALLOWED_METHODS = %w(POST PATCH PUT DELETE).freeze APPLICATION_JSON = 'application/json'.freeze + APPLICATION_JSON_TYPES = %W{#{APPLICATION_JSON} application/vnd.git-lfs+json}.freeze ERROR_MESSAGE = 'You cannot perform write operations on a read-only instance'.freeze + WHITELISTED_GIT_ROUTES = { + 'projects/git_http' => %w{git_upload_pack git_receive_pack} + }.freeze + + WHITELISTED_GIT_LFS_ROUTES = { + 'projects/lfs_api' => %w{batch}, + 'projects/lfs_locks_api' => %w{verify create unlock} + }.freeze + def initialize(app, env) @app = app @env = env @@ -36,7 +46,7 @@ module Gitlab end def json_request? - request.media_type == APPLICATION_JSON + APPLICATION_JSON_TYPES.include?(request.media_type) end def rack_flash @@ -63,22 +73,27 @@ module Gitlab grack_route || ReadOnly.internal_routes.any? { |path| request.path.include?(path) } || lfs_route || sidekiq_route end - def sidekiq_route - request.path.start_with?('/admin/sidekiq') - end - def grack_route # Calling route_hash may be expensive. Only do it if we think there's a possible match - return false unless request.path.end_with?('.git/git-upload-pack') + return false unless + request.path.end_with?('.git/git-upload-pack', '.git/git-receive-pack') - route_hash[:controller] == 'projects/git_http' && route_hash[:action] == 'git_upload_pack' + WHITELISTED_GIT_ROUTES[route_hash[:controller]]&.include?(route_hash[:action]) end def lfs_route # Calling route_hash may be expensive. Only do it if we think there's a possible match - return false unless request.path.end_with?('/info/lfs/objects/batch') + unless request.path.end_with?('/info/lfs/objects/batch', + '/info/lfs/locks', '/info/lfs/locks/verify') || + %r{/info/lfs/locks/\d+/unlock\z}.match?(request.path) + return false + end + + WHITELISTED_GIT_LFS_ROUTES[route_hash[:controller]]&.include?(route_hash[:action]) + end - route_hash[:controller] == 'projects/lfs_api' && route_hash[:action] == 'batch' + def sidekiq_route + request.path.start_with?('/admin/sidekiq') end end end diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index e5191f5c7f9..61653044433 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -30,6 +30,7 @@ module Gitlab dashboard deploy.html explore + favicon.ico favicon.png files groups diff --git a/lib/gitlab/profiler.rb b/lib/gitlab/profiler.rb index 18540e64d4c..ecff6ab5d5e 100644 --- a/lib/gitlab/profiler.rb +++ b/lib/gitlab/profiler.rb @@ -11,6 +11,7 @@ module Gitlab lib/gitlab/etag_caching/ lib/gitlab/metrics/ lib/gitlab/middleware/ + ee/lib/gitlab/middleware/ lib/gitlab/performance_bar/ lib/gitlab/request_profiler/ lib/gitlab/profiler.rb @@ -98,11 +99,7 @@ module Gitlab super - backtrace = Rails.backtrace_cleaner.clean(caller) - - backtrace.each do |caller_line| - next if caller_line.match(Regexp.union(IGNORE_BACKTRACES)) - + Gitlab::Profiler.clean_backtrace(caller).each do |caller_line| stripped_caller_line = caller_line.sub("#{Rails.root}/", '') super(" ↳ #{stripped_caller_line}") @@ -112,6 +109,12 @@ module Gitlab end end + def self.clean_backtrace(backtrace) + Array(Rails.backtrace_cleaner.clean(backtrace)).reject do |line| + line.match(Regexp.union(IGNORE_BACKTRACES)) + end + end + def self.with_custom_logger(logger) original_colorize_logging = ActiveSupport::LogSubscriber.colorize_logging original_activerecord_logger = ActiveRecord::Base.logger diff --git a/lib/gitlab/repository_cache_adapter.rb b/lib/gitlab/repository_cache_adapter.rb index 7f64a8c9e46..2ec871f0754 100644 --- a/lib/gitlab/repository_cache_adapter.rb +++ b/lib/gitlab/repository_cache_adapter.rb @@ -25,6 +25,11 @@ module Gitlab raise NotImplementedError end + # List of cached methods. Should be overridden by the including class + def cached_methods + raise NotImplementedError + end + # Caches the supplied block both in a cache and in an instance variable. # # The cache key and instance variable are named the same way as the value of @@ -67,6 +72,11 @@ module Gitlab # Expires the caches of a specific set of methods def expire_method_caches(methods) methods.each do |key| + unless cached_methods.include?(key.to_sym) + Rails.logger.error "Requested to expire non-existent method '#{key}' for Repository" + next + end + cache.expire(key) ivar = cache_instance_variable_name(key) diff --git a/lib/gitlab/request_forgery_protection.rb b/lib/gitlab/request_forgery_protection.rb index ccfe0d6bed3..a502ad8a541 100644 --- a/lib/gitlab/request_forgery_protection.rb +++ b/lib/gitlab/request_forgery_protection.rb @@ -5,7 +5,7 @@ module Gitlab module RequestForgeryProtection class Controller < ActionController::Base - protect_from_forgery with: :exception + protect_from_forgery with: :exception, prepend: true rescue_from ActionController::InvalidAuthenticityToken do |e| logger.warn "This CSRF token verification failure is handled internally by `GitLab::RequestForgeryProtection`" diff --git a/lib/gitlab/search/parsed_query.rb b/lib/gitlab/search/parsed_query.rb new file mode 100644 index 00000000000..23595f23f01 --- /dev/null +++ b/lib/gitlab/search/parsed_query.rb @@ -0,0 +1,23 @@ +module Gitlab + module Search + class ParsedQuery + attr_reader :term, :filters + + def initialize(term, filters) + @term = term + @filters = filters + end + + def filter_results(results) + filters = @filters.reject { |filter| filter[:matcher].nil? } + return unless filters + + results.select do |result| + filters.all? do |filter| + filter[:matcher].call(filter, result) + end + end + end + end + end +end diff --git a/lib/gitlab/search/query.rb b/lib/gitlab/search/query.rb new file mode 100644 index 00000000000..8583bce7792 --- /dev/null +++ b/lib/gitlab/search/query.rb @@ -0,0 +1,55 @@ +module Gitlab + module Search + class Query < SimpleDelegator + def initialize(query, filter_opts = {}, &block) + @raw_query = query.dup + @filters = [] + @filter_options = { default_parser: :downcase.to_proc }.merge(filter_opts) + + self.instance_eval(&block) if block_given? + + @query = Gitlab::Search::ParsedQuery.new(*extract_filters) + # set the ParsedQuery as our default delegator thanks to SimpleDelegator + super(@query) + end + + private + + def filter(name, **attributes) + filter = { parser: @filter_options[:default_parser], name: name }.merge(attributes) + + @filters << filter + end + + def filter_options(**options) + @filter_options.merge!(options) + end + + def extract_filters + fragments = [] + + filters = @filters.each_with_object([]) do |filter, parsed_filters| + match = @raw_query.split.find { |part| part =~ /\A#{filter[:name]}:/ } + next unless match + + input = match.split(':')[1..-1].join + next if input.empty? + + filter[:value] = parse_filter(filter, input) + filter[:regex_value] = Regexp.escape(filter[:value]).gsub('\*', '.*?') + fragments << match + + parsed_filters << filter + end + + query = (@raw_query.split - fragments).join(' ') + + [query, filters] + end + + def parse_filter(filter, input) + filter[:parser].call(input) + end + end + end +end diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb index 4a87f43597e..b2d75aac1d0 100644 --- a/lib/gitlab/setup_helper.rb +++ b/lib/gitlab/setup_helper.rb @@ -24,6 +24,7 @@ module Gitlab address = val['gitaly_address'] end + # https://gitlab.com/gitlab-org/gitaly/issues/1238 Gitlab::GitalyClient::StorageSettings.allow_disk_access do storages << { name: key, path: val.legacy_disk_path } end diff --git a/lib/gitlab/shard_health_cache.rb b/lib/gitlab/shard_health_cache.rb new file mode 100644 index 00000000000..3f03f46d4b1 --- /dev/null +++ b/lib/gitlab/shard_health_cache.rb @@ -0,0 +1,41 @@ +module Gitlab + class ShardHealthCache + HEALTHY_SHARDS_KEY = 'gitlab-healthy-shards'.freeze + HEALTHY_SHARDS_TIMEOUT = 300 + + # Clears the Redis set storing the list of healthy shards + def self.clear + Gitlab::Redis::Cache.with { |redis| redis.del(HEALTHY_SHARDS_KEY) } + end + + # Updates the list of healthy shards using a Redis set + # + # shards - An array of shard names to store + def self.update(shards) + Gitlab::Redis::Cache.with do |redis| + redis.multi do |m| + m.del(HEALTHY_SHARDS_KEY) + shards.each { |shard_name| m.sadd(HEALTHY_SHARDS_KEY, shard_name) } + m.expire(HEALTHY_SHARDS_KEY, HEALTHY_SHARDS_TIMEOUT) + end + end + end + + # Returns an array of strings of healthy shards + def self.cached_healthy_shards + Gitlab::Redis::Cache.with { |redis| redis.smembers(HEALTHY_SHARDS_KEY) } + end + + # Checks whether the given shard name is in the list of healthy shards. + # + # shard_name - The string to check + def self.healthy_shard?(shard_name) + Gitlab::Redis::Cache.with { |redis| redis.sismember(HEALTHY_SHARDS_KEY, shard_name) } + end + + # Returns the number of healthy shards in the Redis set + def self.healthy_shard_count + Gitlab::Redis::Cache.with { |redis| redis.scard(HEALTHY_SHARDS_KEY) } + end + end +end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 4b8aae4f5a2..5cedd9e84c2 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -1,5 +1,4 @@ -# Gitaly note: JV: two sets of straightforward RPC's. 1 Hard RPC: fork_repository. -# SSH key operations are not part of Gitaly so will never be migrated. +# Gitaly note: SSH key operations are not part of Gitaly so will never be migrated. require 'securerandom' @@ -153,8 +152,6 @@ module Gitlab # # Ex. # mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new") - # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/873 def mv_repository(storage, path, new_path) return false if path.empty? || new_path.empty? @@ -169,19 +166,11 @@ module Gitlab # # Ex. # fork_repository("nfs-file06", "gitlab/gitlab-ci", "nfs-file07", "new-namespace/gitlab-ci") - # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/817 def fork_repository(forked_from_storage, forked_from_disk_path, forked_to_storage, forked_to_disk_path) forked_from_relative_path = "#{forked_from_disk_path}.git" fork_args = [forked_to_storage, "#{forked_to_disk_path}.git"] - gitaly_migrate(:fork_repository, status: Gitlab::GitalyClient::MigrationStatus::OPT_OUT) do |is_enabled| - if is_enabled - GitalyGitlabProjects.new(forked_from_storage, forked_from_relative_path).fork_repository(*fork_args) - else - gitlab_projects(forked_from_storage, forked_from_relative_path).fork_repository(*fork_args) - end - end + GitalyGitlabProjects.new(forked_from_storage, forked_from_relative_path).fork_repository(*fork_args) end # Removes a repository from file system, using rm_diretory which is an alias @@ -193,8 +182,6 @@ module Gitlab # # Ex. # remove_repository("/path/to/storage", "gitlab/gitlab-ci") - # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/873 def remove_repository(storage, name) return false if name.empty? diff --git a/lib/gitlab/slash_commands/presenters/access.rb b/lib/gitlab/slash_commands/presenters/access.rb index 1a817eb735b..81f7cd3ffe8 100644 --- a/lib/gitlab/slash_commands/presenters/access.rb +++ b/lib/gitlab/slash_commands/presenters/access.rb @@ -15,7 +15,7 @@ module Gitlab if @resource ":wave: Hi there! Before I do anything for you, please [connect your GitLab account](#{@resource})." else - ":sweat_smile: Couldn't identify you, nor can I autorize you!" + ":sweat_smile: Couldn't identify you, nor can I authorize you!" end ephemeral_response(text: message) diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 59a222b086c..dff0c97eeb4 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -24,7 +24,6 @@ module Gitlab installation_type: Gitlab::INSTALLATION_TYPE, active_user_count: User.active.count, recorded_at: Time.now, - mattermost_enabled: Gitlab.config.mattermost.enabled, edition: 'CE' } @@ -91,13 +90,14 @@ module Gitlab def features_usage_data_ce { - signup: Gitlab::CurrentSettings.allow_signup?, - ldap: Gitlab.config.ldap.enabled, - gravatar: Gitlab::CurrentSettings.gravatar_enabled?, - omniauth: Gitlab.config.omniauth.enabled, - reply_by_email: Gitlab::IncomingEmail.enabled?, - container_registry: Gitlab.config.registry.enabled, - gitlab_shared_runners: Gitlab.config.gitlab_ci.shared_runners_enabled + container_registry_enabled: Gitlab.config.registry.enabled, + gitlab_shared_runners_enabled: Gitlab.config.gitlab_ci.shared_runners_enabled, + gravatar_enabled: Gitlab::CurrentSettings.gravatar_enabled?, + ldap_enabled: Gitlab.config.ldap.enabled, + mattermost_enabled: Gitlab.config.mattermost.enabled, + omniauth_enabled: Gitlab.config.omniauth.enabled, + reply_by_email_enabled: Gitlab::IncomingEmail.enabled?, + signup_enabled: Gitlab::CurrentSettings.allow_signup? } end diff --git a/lib/gitlab/verify/uploads.rb b/lib/gitlab/verify/uploads.rb index 01f09ab8df7..73fc43cb590 100644 --- a/lib/gitlab/verify/uploads.rb +++ b/lib/gitlab/verify/uploads.rb @@ -12,7 +12,7 @@ module Gitlab private def all_relation - Upload.all + Upload.all.preload(:model) end def local?(upload) diff --git a/lib/mysql_zero_date.rb b/lib/mysql_zero_date.rb new file mode 100644 index 00000000000..64634f789da --- /dev/null +++ b/lib/mysql_zero_date.rb @@ -0,0 +1,18 @@ +# Disable NO_ZERO_DATE mode for mysql in rails 5. +# We use zero date as a default value +# (config/initializers/active_record_mysql_timestamp.rb), in +# Rails 5 using zero date fails by default (https://gitlab.com/gitlab-org/gitlab-ce/-/jobs/75450216) +# and NO_ZERO_DATE has to be explicitly disabled. Disabling strict mode +# is not sufficient. + +require 'active_record/connection_adapters/abstract_mysql_adapter' + +module MysqlZeroDate + def configure_connection + super + + @connection.query "SET @@SESSION.sql_mode = REPLACE(@@SESSION.sql_mode, 'NO_ZERO_DATE', '');" # rubocop:disable Gitlab/ModuleWithInstanceVariables + end +end + +ActiveRecord::ConnectionAdapters::AbstractMysqlAdapter.prepend(MysqlZeroDate) if Gitlab.rails5? diff --git a/lib/tasks/flay.rake b/lib/tasks/flay.rake index 4b4881cecb8..4bec013a141 100644 --- a/lib/tasks/flay.rake +++ b/lib/tasks/flay.rake @@ -1,6 +1,6 @@ desc 'Code duplication analyze via flay' task :flay do - output = `bundle exec flay --mass 35 app/ lib/gitlab/ 2> #{File::NULL}` + output = `bundle exec flay --mass 35 app/ lib/gitlab/ ee/ 2> #{File::NULL}` if output.include?("Similar code found") || output.include?("IDENTICAL code found") puts output diff --git a/lib/tasks/gettext.rake b/lib/tasks/gettext.rake index 247d7be7d78..21998dd2f5b 100644 --- a/lib/tasks/gettext.rake +++ b/lib/tasks/gettext.rake @@ -4,7 +4,7 @@ namespace :gettext do # Customize list of translatable files # See: https://github.com/grosser/gettext_i18n_rails#customizing-list-of-translatable-files def files_to_translate - folders = %W(app lib config #{locale_path}).join(',') + folders = %W(ee app lib config #{locale_path}).join(',') exts = %w(rb erb haml slim rhtml js jsx vue handlebars hbs mustache).join(',') Dir.glob( @@ -16,7 +16,6 @@ namespace :gettext do # See: https://gitlab.com/gitlab-org/gitlab-ce/issues/33014#note_31218998 FileUtils.touch(File.join(Rails.root, 'locale/gitlab.pot')) - Rake::Task['gettext:pack'].invoke Rake::Task['gettext:po_to_json'].invoke end @@ -50,6 +49,41 @@ namespace :gettext do end end + task :updated_check do + # Removing all pre-translated files speeds up `gettext:find` as the + # files don't need to be merged. + # Having `LC_MESSAGES/gitlab.mo files present also confuses the output. + FileUtils.rm Dir['locale/**/gitlab.*'] + + # Make sure we start out with a clean pot.file + `git checkout -- locale/gitlab.pot` + + # `gettext:find` writes touches to temp files to `stderr` which would cause + # `static-analysis` to report failures. We can ignore these. + silence_stream($stderr) do + Rake::Task['gettext:find'].invoke + end + + pot_diff = `git diff -- locale/gitlab.pot`.strip + + # reset the locale folder for potential next tasks + `git checkout -- locale` + + if pot_diff.present? + raise <<~MSG + Newly translated strings found, please add them to `gitlab.pot` by running: + + rm locale/**/gitlab.*; bin/rake gettext:find; git checkout -- locale/*/gitlab.po + + Then commit and push the resulting changes to `locale/gitlab.pot`. + + The diff was: + + #{pot_diff} + MSG + end + end + def report_errors_for_file(file, errors_for_file) puts "Errors in `#{file}`:" diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 139ab70e125..69166851816 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -46,7 +46,9 @@ namespace :gitlab do desc 'Configures the database by running migrate, or by loading the schema and seeding if needed' task configure: :environment do - if ActiveRecord::Base.connection.tables.any? + # Check if we have existing db tables + # The schema_migrations table will still exist if drop_tables was called + if ActiveRecord::Base.connection.tables.count > 1 Rake::Task['db:migrate'].invoke else Rake::Task['db:schema:load'].invoke diff --git a/lib/tasks/gitlab/import_export.rake b/lib/tasks/gitlab/import_export.rake index 44074397c05..900dbf7be24 100644 --- a/lib/tasks/gitlab/import_export.rake +++ b/lib/tasks/gitlab/import_export.rake @@ -10,15 +10,22 @@ namespace :gitlab do puts YAML.load_file(Gitlab::ImportExport.config_file)['project_tree'].to_yaml(SortKeys: true) end - desc 'GitLab | Bumps the Import/Export version for test_project_export.tar.gz' - task bump_test_version: :environment do - Dir.mktmpdir do |tmp_dir| - system("tar -zxf spec/features/projects/import_export/test_project_export.tar.gz -C #{tmp_dir} > /dev/null") - File.write(File.join(tmp_dir, 'VERSION'), Gitlab::ImportExport.version, mode: 'w') - system("tar -zcvf spec/features/projects/import_export/test_project_export.tar.gz -C #{tmp_dir} . > /dev/null") + desc 'GitLab | Bumps the Import/Export version in fixtures and project templates' + task bump_version: :environment do + archives = Dir['vendor/project_templates/*.tar.gz'] + archives.push('spec/features/projects/import_export/test_project_export.tar.gz') + + archives.each do |archive| + raise ArgumentError unless File.exist?(archive) + + Dir.mktmpdir do |tmp_dir| + system("tar -zxf #{archive} -C #{tmp_dir} > /dev/null") + File.write(File.join(tmp_dir, 'VERSION'), Gitlab::ImportExport.version, mode: 'w') + system("tar -zcvf #{archive} -C #{tmp_dir} . > /dev/null") + end end - puts "Updated to #{Gitlab::ImportExport.version}" + puts "Updated #{archives} to #{Gitlab::ImportExport.version}." end end end diff --git a/lib/tasks/lint.rake b/lib/tasks/lint.rake index 8b86a5c72a5..006fcdd31a4 100644 --- a/lib/tasks/lint.rake +++ b/lib/tasks/lint.rake @@ -17,16 +17,26 @@ unless Rails.env.production? Rake::Task['eslint'].invoke end + desc "GitLab | lint | Lint HAML files" + task :haml do + begin + Rake::Task['haml_lint'].invoke + rescue RuntimeError # The haml_lint tasks raise a RuntimeError + exit(1) + end + end + desc "GitLab | lint | Run several lint checks" task :all do status = 0 %w[ config_lint - haml_lint + lint:haml scss_lint flay gettext:lint + gettext:updated_check lint:static_verification ].each do |task| pid = Process.fork do @@ -38,13 +48,12 @@ unless Rails.env.production? $stderr.reopen(wr_err) begin - begin - Rake::Task[task].invoke - rescue RuntimeError # The haml_lint tasks raise a RuntimeError - exit(1) - end + Rake::Task[task].invoke rescue SystemExit => ex - msg = "*** Rake task #{task} failed with the following error(s):" + msg = "*** Rake task #{task} exited:" + raise ex + rescue => ex + msg = "*** Rake task #{task} raised #{ex.class}:" raise ex ensure $stdout.reopen(stdout) |