diff options
Diffstat (limited to 'lib')
69 files changed, 642 insertions, 1493 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index 843f75d3096..e89d9337853 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -118,6 +118,7 @@ module API mount ::API::Namespaces mount ::API::Notes mount ::API::Discussions + mount ::API::ResourceLabelEvents mount ::API::NotificationSettings mount ::API::PagesDomains mount ::API::Pipelines diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 3e445e6b1fa..ac33c2924ca 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -9,14 +9,6 @@ module API before { authorize! :download_code, user_project } helpers do - def find_branch!(branch_name) - begin - user_project.repository.find_branch(branch_name) || not_found!('Branch') - rescue Gitlab::Git::CommandError - render_api_error!('The branch refname is invalid', 400) - end - end - params :filter_params do optional :search, type: String, desc: 'Return list of branches matching the search criteria' optional :sort, type: String, desc: 'Return list of branches sorted by the given field' @@ -77,7 +69,7 @@ module API success Entities::Branch end params do - requires :branch, type: String, desc: 'The name of the branch' + requires :branch, type: String, desc: 'The name of the branch', allow_blank: false optional :developers_can_push, type: Boolean, desc: 'Flag if developers can push to that branch' optional :developers_can_merge, type: Boolean, desc: 'Flag if developers can merge to that branch' end @@ -114,7 +106,7 @@ module API success Entities::Branch end params do - requires :branch, type: String, desc: 'The name of the branch' + requires :branch, type: String, desc: 'The name of the branch', allow_blank: false end put ':id/repository/branches/:branch/unprotect', requirements: BRANCH_ENDPOINT_REQUIREMENTS do authorize_admin_project @@ -130,8 +122,8 @@ module API success Entities::Branch end params do - requires :branch, type: String, desc: 'The name of the branch' - requires :ref, type: String, desc: 'Create branch from commit sha or existing branch' + requires :branch, type: String, desc: 'The name of the branch', allow_blank: false + requires :ref, type: String, desc: 'Create branch from commit sha or existing branch', allow_blank: false end post ':id/repository/branches' do authorize_push_project @@ -151,7 +143,7 @@ module API desc 'Delete a branch' params do - requires :branch, type: String, desc: 'The name of the branch' + requires :branch, type: String, desc: 'The name of the branch', allow_blank: false end delete ':id/repository/branches/:branch', requirements: BRANCH_ENDPOINT_REQUIREMENTS do authorize_push_project diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 92329465b2c..f86ac60b54b 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -71,7 +71,7 @@ module API detail 'This feature was introduced in GitLab 8.13' end params do - requires :branch, type: String, desc: 'Name of the branch to commit into. To create a new branch, also provide `start_branch`.' + requires :branch, type: String, desc: 'Name of the branch to commit into. To create a new branch, also provide `start_branch`.', allow_blank: false requires :commit_message, type: String, desc: 'Commit message' requires :actions, type: Array[Hash], desc: 'Actions to perform in commit' optional :start_branch, type: String, desc: 'Name of the branch to start the new commit from' @@ -151,7 +151,7 @@ module API end params do requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag to be cherry picked' - requires :branch, type: String, desc: 'The name of the branch' + requires :branch, type: String, desc: 'The name of the branch', allow_blank: false end post ':id/repository/commits/:sha/cherry_pick', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS do authorize_push_to_branch!(params[:branch]) @@ -159,8 +159,7 @@ module API commit = user_project.commit(params[:sha]) not_found!('Commit') unless commit - branch = user_project.repository.find_branch(params[:branch]) - not_found!('Branch') unless branch + find_branch!(params[:branch]) commit_params = { commit: commit, @@ -171,7 +170,7 @@ module API result = ::Commits::CherryPickService.new(user_project, current_user, commit_params).execute if result[:status] == :success - branch = user_project.repository.find_branch(params[:branch]) + branch = find_branch!(params[:branch]) present user_project.repository.commit(branch.dereferenced_target), with: Entities::Commit else render_api_error!(result[:message], 400) diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 90abee94f6a..f0eafbaeb94 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -1437,5 +1437,19 @@ module API badge.type == 'ProjectBadge' ? 'project' : 'group' end end + + class ResourceLabelEvent < Grape::Entity + expose :id + expose :user, using: Entities::UserBasic + expose :created_at + expose :resource_type do |event, options| + event.issuable.class.name + end + expose :resource_id do |event, options| + event.issuable.id + end + expose :label, using: Entities::LabelBasic + expose :action + end end end diff --git a/lib/api/files.rb b/lib/api/files.rb index ff4f75c12df..ac02488d30c 100644 --- a/lib/api/files.rb +++ b/lib/api/files.rb @@ -58,7 +58,7 @@ module API params :simple_file_params do requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' - requires :branch, type: String, desc: 'Name of the branch to commit into. To create a new branch, also provide `start_branch`.' + requires :branch, type: String, desc: 'Name of the branch to commit into. To create a new branch, also provide `start_branch`.', allow_blank: false requires :commit_message, type: String, allow_blank: false, desc: 'Commit message' optional :start_branch, type: String, desc: 'Name of the branch to start the new commit from' optional :author_email, type: String, desc: 'The email of the author' @@ -80,7 +80,7 @@ module API desc 'Get raw file metadata from repository' params do requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' - requires :ref, type: String, desc: 'The name of branch, tag or commit' + requires :ref, type: String, desc: 'The name of branch, tag or commit', allow_blank: false end head ":id/repository/files/:file_path/raw", requirements: FILE_ENDPOINT_REQUIREMENTS do assign_file_vars! @@ -91,7 +91,7 @@ module API desc 'Get raw file contents from the repository' params do requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' - requires :ref, type: String, desc: 'The name of branch, tag commit' + requires :ref, type: String, desc: 'The name of branch, tag commit', allow_blank: false end get ":id/repository/files/:file_path/raw", requirements: FILE_ENDPOINT_REQUIREMENTS do assign_file_vars! @@ -104,7 +104,7 @@ module API desc 'Get file metadata from repository' params do requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' - requires :ref, type: String, desc: 'The name of branch, tag or commit' + requires :ref, type: String, desc: 'The name of branch, tag or commit', allow_blank: false end head ":id/repository/files/:file_path", requirements: FILE_ENDPOINT_REQUIREMENTS do assign_file_vars! @@ -115,7 +115,7 @@ module API desc 'Get a file from the repository' params do requires :file_path, type: String, desc: 'The url encoded path to the file. Ex. lib%2Fclass%2Erb' - requires :ref, type: String, desc: 'The name of branch, tag or commit' + requires :ref, type: String, desc: 'The name of branch, tag or commit', allow_blank: false end get ":id/repository/files/:file_path", requirements: FILE_ENDPOINT_REQUIREMENTS do assign_file_vars! diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index be17653dbb2..c7ecddeccf0 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -103,10 +103,12 @@ module API end def find_project(id) + projects = Project.without_deleted + if id.is_a?(Integer) || id =~ /^\d+$/ - Project.find_by(id: id) + projects.find_by(id: id) elsif id.include?("/") - Project.find_by_full_path(id) + projects.find_by_full_path(id) end end @@ -156,6 +158,12 @@ module API end end + def find_branch!(branch_name) + user_project.repository.find_branch(branch_name) || not_found!('Branch') + rescue Gitlab::Git::CommandError + render_api_error!('The branch refname is invalid', 400) + end + def find_project_label(id) labels = available_labels_for(user_project) label = labels.find_by_id(id) || labels.find_by_title(id) @@ -380,7 +388,7 @@ module API end def project_finder_params - finder_params = {} + finder_params = { without_deleted: true } finder_params[:owned] = true if params[:owned].present? finder_params[:non_public] = true if params[:membership].present? finder_params[:starred] = true if params[:starred].present? diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb index 381d5e8968c..98672f2f765 100644 --- a/lib/api/helpers/projects_helpers.rb +++ b/lib/api/helpers/projects_helpers.rb @@ -26,6 +26,7 @@ module API optional :avatar, type: File, desc: 'Avatar image for project' optional :printing_merge_request_link_enabled, type: Boolean, desc: 'Show link to create/view merge request when pushing from the command line' optional :merge_method, type: String, values: %w(ff rebase_merge merge), desc: 'The merge method used when merging merge requests' + optional :initialize_with_readme, type: Boolean, desc: "Initialize a project with a README.md" end params :optional_project_params do diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 0990e2a1fba..e3e8cb71c09 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -74,6 +74,7 @@ module API gl_repository: gl_repository, gl_id: Gitlab::GlId.gl_id(user), gl_username: user&.username, + git_config_options: [], # This repository_path is a bogus value but gitlab-shell still requires # its presence. https://gitlab.com/gitlab-org/gitlab-shell/issues/135 @@ -81,6 +82,13 @@ module API gitaly: gitaly_payload(params[:action]) } + + # Custom option for git-receive-pack command + receive_max_input_size = Gitlab::CurrentSettings.receive_max_input_size.to_i + if receive_max_input_size > 0 + payload[:git_config_options] << "receive.maxInputSize=#{receive_max_input_size.megabytes}" + end + response_with_status(**payload) when ::Gitlab::GitAccessResult::CustomAction response_with_status(code: 300, message: check_result.message, payload: check_result.payload) diff --git a/lib/api/pipeline_schedules.rb b/lib/api/pipeline_schedules.rb index 37f32411296..ae4a7654ec1 100644 --- a/lib/api/pipeline_schedules.rb +++ b/lib/api/pipeline_schedules.rb @@ -39,7 +39,7 @@ module API end params do requires :description, type: String, desc: 'The description of pipeline schedule' - requires :ref, type: String, desc: 'The branch/tag name will be triggered' + requires :ref, type: String, desc: 'The branch/tag name will be triggered', allow_blank: false requires :cron, type: String, desc: 'The cron' optional :cron_timezone, type: String, default: 'UTC', desc: 'The timezone' optional :active, type: Boolean, default: true, desc: 'The activation of pipeline schedule' diff --git a/lib/api/project_export.rb b/lib/api/project_export.rb index 15c57a2fc02..8562ae6d737 100644 --- a/lib/api/project_export.rb +++ b/lib/api/project_export.rb @@ -21,12 +21,8 @@ module API detail 'This feature was introduced in GitLab 10.6.' end get ':id/export/download' do - path = user_project.export_project_path - - if path - present_disk_file!(path, File.basename(path), 'application/gzip') - elsif user_project.export_project_object_exists? - present_carrierwave_file!(user_project.import_export_upload.export_file) + if user_project.export_file_exists? + present_carrierwave_file!(user_project.export_file) else render_api_error!('404 Not found or has expired', 404) end diff --git a/lib/api/resource_label_events.rb b/lib/api/resource_label_events.rb new file mode 100644 index 00000000000..5ac3adeb990 --- /dev/null +++ b/lib/api/resource_label_events.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module API + class ResourceLabelEvents < Grape::API + include PaginationParams + helpers ::API::Helpers::NotesHelpers + + before { authenticate! } + + EVENTABLE_TYPES = [Issue, MergeRequest].freeze + + EVENTABLE_TYPES.each do |eventable_type| + parent_type = eventable_type.parent_class.to_s.underscore + eventables_str = eventable_type.to_s.underscore.pluralize + + params do + requires :id, type: String, desc: "The ID of a #{parent_type}" + end + resource parent_type.pluralize.to_sym, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + desc "Get a list of #{eventable_type.to_s.downcase} resource label events" do + success Entities::ResourceLabelEvent + detail 'This feature was introduced in 11.3' + end + params do + requires :eventable_id, types: [Integer, String], desc: 'The ID of the eventable' + use :pagination + end + get ":id/#{eventables_str}/:eventable_id/resource_label_events" do + eventable = find_noteable(parent_type, eventables_str, params[:eventable_id]) + events = eventable.resource_label_events.includes(:label, :user) + + present paginate(events), with: Entities::ResourceLabelEvent + end + + desc "Get a single #{eventable_type.to_s.downcase} resource label event" do + success Entities::ResourceLabelEvent + detail 'This feature was introduced in 11.3' + end + params do + requires :event_id, type: String, desc: 'The ID of a resource label event' + requires :eventable_id, types: [Integer, String], desc: 'The ID of the eventable' + end + get ":id/#{eventables_str}/:eventable_id/resource_label_events/:event_id" do + eventable = find_noteable(parent_type, eventables_str, params[:eventable_id]) + event = eventable.resource_label_events.find(params[:event_id]) + + present event, with: Entities::ResourceLabelEvent + end + end + end + end +end diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb index b29e660c6e0..be95ef9e928 100644 --- a/lib/api/triggers.rb +++ b/lib/api/triggers.rb @@ -10,7 +10,7 @@ module API success Entities::Pipeline end params do - requires :ref, type: String, desc: 'The commit sha or name of a branch or tag' + requires :ref, type: String, desc: 'The commit sha or name of a branch or tag', allow_blank: false requires :token, type: String, desc: 'The unique token of trigger' optional :variables, type: Hash, desc: 'The list of variables to be injected into build' end diff --git a/lib/api/users.rb b/lib/api/users.rb index b0811bb4aad..a4ae597e252 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -361,6 +361,7 @@ module API params do requires :id, type: Integer, desc: 'The ID of the user' requires :email, type: String, desc: 'The email of the user' + optional :skip_confirmation, type: Boolean, desc: 'Skip confirmation of email and assume it is verified' end post ":id/emails" do authenticated_as_admin! diff --git a/lib/banzai/filter/spaced_link_filter.rb b/lib/banzai/filter/spaced_link_filter.rb index 574a8a6c7a5..a27f1d46863 100644 --- a/lib/banzai/filter/spaced_link_filter.rb +++ b/lib/banzai/filter/spaced_link_filter.rb @@ -8,22 +8,31 @@ module Banzai # # Based on Banzai::Filter::AutolinkFilter # - # CommonMark does not allow spaces in the url portion of a link. - # For example, `[example](page slug)` is not valid. However, + # CommonMark does not allow spaces in the url portion of a link/url. + # For example, `[example](page slug)` is not valid. + # Neither is ``. However, particularly # in our wikis, we support (via RedCarpet) this type of link, allowing # wiki pages to be easily linked by their title. This filter adds that functionality. - # The intent is for this to only be used in Wikis - in general, we want - # to adhere to CommonMark's spec. + # + # This is a small extension to the CommonMark spec. If they start allowing + # spaces in urls, we could then remove this filter. # class SpacedLinkFilter < HTML::Pipeline::Filter include ActionView::Helpers::TagHelper # Pattern to match a standard markdown link # - # Rubular: http://rubular.com/r/z9EAHxYmKI - LINK_PATTERN = /\[([^\]]+)\]\(([^)"]+)(?: \"([^\"]+)\")?\)/ - - # Text matching LINK_PATTERN inside these elements will not be linked + # Rubular: http://rubular.com/r/2EXEQ49rg5 + LINK_OR_IMAGE_PATTERN = %r{ + (?<preview_operator>!)? + \[(?<text>.+?)\] + \( + (?<new_link>.+?) + (?<title>\ ".+?")? + \) + }x + + # Text matching LINK_OR_IMAGE_PATTERN inside these elements will not be linked IGNORE_PARENTS = %w(a code kbd pre script style).to_set # The XPath query to use for finding text nodes to parse. @@ -38,7 +47,7 @@ module Banzai doc.xpath(TEXT_QUERY).each do |node| content = node.to_html - next unless content.match(LINK_PATTERN) + next unless content.match(LINK_OR_IMAGE_PATTERN) html = spaced_link_filter(content) @@ -53,25 +62,37 @@ module Banzai private def spaced_link_match(link) - match = LINK_PATTERN.match(link) - return link unless match && match[1] && match[2] + match = LINK_OR_IMAGE_PATTERN.match(link) + return link unless match # escape the spaces in the url so that it's a valid markdown link, # then run it through the markdown processor again, let it do its magic - text = match[1] - new_link = match[2].gsub(' ', '%20') - title = match[3] ? " \"#{match[3]}\"" : '' - html = Banzai::Filter::MarkdownFilter.call("[#{text}](#{new_link}#{title})", context) + html = Banzai::Filter::MarkdownFilter.call(transform_markdown(match), context) # link is wrapped in a <p>, so strip that off html.sub('<p>', '').chomp('</p>') end def spaced_link_filter(text) - Gitlab::StringRegexMarker.new(CGI.unescapeHTML(text), text.html_safe).mark(LINK_PATTERN) do |link, left:, right:| + Gitlab::StringRegexMarker.new(CGI.unescapeHTML(text), text.html_safe).mark(LINK_OR_IMAGE_PATTERN) do |link, left:, right:| spaced_link_match(link) end end + + def transform_markdown(match) + preview_operator, text, new_link, title = process_match(match) + + "#{preview_operator}[#{text}](#{new_link}#{title})" + end + + def process_match(match) + [ + match[:preview_operator], + match[:text], + match[:new_link].gsub(' ', '%20'), + match[:title] + ] + end end end end diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index e9be05e174e..bd34614f149 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -16,6 +16,7 @@ module Banzai Filter::MathFilter, Filter::ColorFilter, Filter::MermaidFilter, + Filter::SpacedLinkFilter, Filter::VideoLinkFilter, Filter::ImageLazyLoadFilter, Filter::ImageLinkFilter, diff --git a/lib/banzai/pipeline/label_pipeline.rb b/lib/banzai/pipeline/label_pipeline.rb new file mode 100644 index 00000000000..725cccc4b2b --- /dev/null +++ b/lib/banzai/pipeline/label_pipeline.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Banzai + module Pipeline + class LabelPipeline < BasePipeline + def self.filters + @filters ||= FilterArray[ + Filter::SanitizationFilter, + Filter::LabelReferenceFilter + ] + end + end + end +end diff --git a/lib/banzai/pipeline/wiki_pipeline.rb b/lib/banzai/pipeline/wiki_pipeline.rb index 737ff0cc818..c37b8e71cb0 100644 --- a/lib/banzai/pipeline/wiki_pipeline.rb +++ b/lib/banzai/pipeline/wiki_pipeline.rb @@ -5,7 +5,6 @@ module Banzai @filters ||= begin super.insert_after(Filter::TableOfContentsFilter, Filter::GollumTagsFilter) .insert_before(Filter::TaskListFilter, Filter::WikiLinkFilter) - .insert_before(Filter::WikiLinkFilter, Filter::SpacedLinkFilter) end end end diff --git a/lib/banzai/renderer/common_mark/html.rb b/lib/banzai/renderer/common_mark/html.rb index 46b609c36b0..0b27316da1b 100644 --- a/lib/banzai/renderer/common_mark/html.rb +++ b/lib/banzai/renderer/common_mark/html.rb @@ -4,15 +4,11 @@ module Banzai class HTML < CommonMarker::HtmlRenderer def code_block(node) block do - code = node.string_content - lang = node.fence_info - lang_attr = lang.present? ? %Q{ lang="#{lang}"} : '' - result = - "<pre>" \ - "<code#{lang_attr}>#{ERB::Util.html_escape(code)}</code>" \ - "</pre>" - - out(result) + out("<pre#{sourcepos(node)}><code") + out(' lang="', node.fence_info, '"') if node.fence_info.present? + out('>') + out(escape_html(node.string_content)) + out('</code></pre>') end end end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 46dad59eb8c..fe98d25af29 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -1,6 +1,6 @@ module Gitlab module Ci - ## + # # Base GitLab CI Configuration facade # class Config @@ -15,6 +15,8 @@ module Gitlab @global.compose! rescue Loader::FormatError, Extendable::ExtensionError => e raise Config::ConfigError, e.message + rescue ::Gitlab::Ci::External::Processor::FileError => e + raise ::Gitlab::Ci::YamlProcessor::ValidationError, e.message end def valid? @@ -64,9 +66,22 @@ module Gitlab @global.jobs_value end - # 'opts' argument is used in EE see /ee/lib/ee/gitlab/ci/config.rb + private + def build_config(config, opts = {}) - Loader.new(config).load! + initial_config = Loader.new(config).load! + project = opts.fetch(:project, nil) + + if project + process_external_files(initial_config, project, opts) + else + initial_config + end + end + + def process_external_files(config, project, opts) + sha = opts.fetch(:sha) { project.repository.root_ref_sha } + ::Gitlab::Ci::External::Processor.new(config, project, sha).perform end end end diff --git a/lib/gitlab/ci/external/file/base.rb b/lib/gitlab/ci/external/file/base.rb new file mode 100644 index 00000000000..f4da07b0b02 --- /dev/null +++ b/lib/gitlab/ci/external/file/base.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module External + module File + class Base + YAML_WHITELIST_EXTENSION = /(yml|yaml)$/i.freeze + + def initialize(location, opts = {}) + @location = location + end + + def valid? + location.match(YAML_WHITELIST_EXTENSION) && content + end + + def content + raise NotImplementedError, 'content must be implemented and return a string or nil' + end + + def error_message + raise NotImplementedError, 'error_message must be implemented and return a string' + end + end + end + end + end +end diff --git a/lib/gitlab/ci/external/file/local.rb b/lib/gitlab/ci/external/file/local.rb new file mode 100644 index 00000000000..1aa7f687507 --- /dev/null +++ b/lib/gitlab/ci/external/file/local.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module External + module File + class Local < Base + attr_reader :location, :project, :sha + + def initialize(location, opts = {}) + super + + @project = opts.fetch(:project) + @sha = opts.fetch(:sha) + end + + def content + @content ||= fetch_local_content + end + + def error_message + "Local file '#{location}' is not valid." + end + + private + + def fetch_local_content + project.repository.blob_data_at(sha, location) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/external/file/remote.rb b/lib/gitlab/ci/external/file/remote.rb new file mode 100644 index 00000000000..59bb3e8999e --- /dev/null +++ b/lib/gitlab/ci/external/file/remote.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module External + module File + class Remote < Base + include Gitlab::Utils::StrongMemoize + attr_reader :location + + def content + return @content if defined?(@content) + + @content = strong_memoize(:content) do + begin + Gitlab::HTTP.get(location) + rescue Gitlab::HTTP::Error, Timeout::Error, SocketError, Gitlab::HTTP::BlockedUrlError + nil + end + end + end + + def error_message + "Remote file '#{location}' is not valid." + end + end + end + end + end +end diff --git a/lib/gitlab/ci/external/mapper.rb b/lib/gitlab/ci/external/mapper.rb new file mode 100644 index 00000000000..58bd6a19acf --- /dev/null +++ b/lib/gitlab/ci/external/mapper.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module External + class Mapper + def initialize(values, project, sha) + @locations = Array(values.fetch(:include, [])) + @project = project + @sha = sha + end + + def process + locations.map { |location| build_external_file(location) } + end + + private + + attr_reader :locations, :project, :sha + + def build_external_file(location) + if ::Gitlab::UrlSanitizer.valid?(location) + Gitlab::Ci::External::File::Remote.new(location) + else + options = { project: project, sha: sha } + Gitlab::Ci::External::File::Local.new(location, options) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/external/processor.rb b/lib/gitlab/ci/external/processor.rb new file mode 100644 index 00000000000..76cf3ce89f9 --- /dev/null +++ b/lib/gitlab/ci/external/processor.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module External + class Processor + FileError = Class.new(StandardError) + + def initialize(values, project, sha) + @values = values + @external_files = Gitlab::Ci::External::Mapper.new(values, project, sha).process + @content = {} + end + + def perform + return values if external_files.empty? + + external_files.each do |external_file| + validate_external_file(external_file) + @content.deep_merge!(content_of(external_file)) + end + + append_inline_content + remove_include_keyword + end + + private + + attr_reader :values, :external_files, :content + + def validate_external_file(external_file) + unless external_file.valid? + raise FileError, external_file.error_message + end + end + + def content_of(external_file) + Gitlab::Ci::Config::Loader.new(external_file.content).load! + end + + def append_inline_content + @content.deep_merge!(@values) + end + + def remove_include_keyword + content.delete(:include) + content + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb index 508b4814631..2fa9a0d4541 100644 --- a/lib/gitlab/ci/status/build/failed.rb +++ b/lib/gitlab/ci/status/build/failed.rb @@ -13,6 +13,8 @@ module Gitlab runner_unsupported: 'unsupported runner' }.freeze + private_constant :REASONS + def status_tooltip base_message end @@ -25,6 +27,10 @@ module Gitlab build.failed? end + def self.reasons + REASONS + end + private def base_message @@ -36,7 +42,7 @@ module Gitlab end def failure_reason_message - REASONS.fetch(subject.failure_reason.to_sym) + self.class.reasons.fetch(subject.failure_reason.to_sym) end end end diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index 4c28489f45a..58ca077e636 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -7,7 +7,11 @@ module Gitlab def initialize(contributor, current_user = nil) @contributor = contributor @current_user = current_user - @projects = ContributedProjectsFinder.new(contributor).execute(current_user) + @projects = if @contributor.include_private_contributions? + ContributedProjectsFinder.new(@contributor).execute(@contributor) + else + ContributedProjectsFinder.new(contributor).execute(current_user) + end end def activity_dates @@ -36,13 +40,9 @@ module Gitlab def events_by_date(date) return Event.none unless can_read_cross_project? - events = Event.contributions.where(author_id: contributor.id) + Event.contributions.where(author_id: contributor.id) .where(created_at: date.beginning_of_day..date.end_of_day) .where(project_id: projects) - - # Use visible_to_user? instead of the complicated logic in activity_dates - # because we're only viewing the events for a single day. - events.select { |event| event.visible_to_user?(current_user) } end def starting_year diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 9147ef401da..c9dbd2d2e5f 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -13,6 +13,10 @@ module Gitlab Gitlab::FakeApplicationSettings.new(::ApplicationSetting.defaults.merge(attributes || {})) end + def clear_in_memory_application_settings! + @in_memory_application_settings = nil + end + def method_missing(name, *args, &block) current_application_settings.send(name, *args, &block) # rubocop:disable GitlabSecurity/PublicSend end diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb index c79d8d3cb21..2acb0e43b69 100644 --- a/lib/gitlab/diff/file_collection/base.rb +++ b/lib/gitlab/diff/file_collection/base.rb @@ -2,7 +2,7 @@ module Gitlab module Diff module FileCollection class Base - attr_reader :project, :diff_options, :diff_refs, :fallback_diff_refs + attr_reader :project, :diff_options, :diff_refs, :fallback_diff_refs, :diffable delegate :count, :size, :real_size, to: :diff_files @@ -33,6 +33,14 @@ module Gitlab diff_files.find { |diff_file| diff_file.new_path == new_path } end + def clear_cache + # No-op + end + + def write_cache + # No-op + end + private def decorate_diff!(diff) diff --git a/lib/gitlab/diff/file_collection/merge_request_diff.rb b/lib/gitlab/diff/file_collection/merge_request_diff.rb index be25e1bab21..0dd073a3a8e 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff.rb @@ -2,6 +2,8 @@ module Gitlab module Diff module FileCollection class MergeRequestDiff < Base + extend ::Gitlab::Utils::Override + def initialize(merge_request_diff, diff_options:) @merge_request_diff = merge_request_diff @@ -13,70 +15,35 @@ module Gitlab end def diff_files - # Make sure to _not_ send any method call to Gitlab::Diff::File - # _before_ all of them were collected (`super`). Premature method calls will - # trigger N+1 RPCs to Gitaly through BatchLoader records (Blob.lazy). - # diff_files = super - diff_files.each { |diff_file| cache_highlight!(diff_file) if cacheable?(diff_file) } - store_highlight_cache + diff_files.each { |diff_file| cache.decorate(diff_file) } diff_files end - def real_size - @merge_request_diff.real_size + override :write_cache + def write_cache + cache.write_if_empty end - def clear_cache! - Rails.cache.delete(cache_key) + override :clear_cache + def clear_cache + cache.clear end def cache_key - [@merge_request_diff, 'highlighted-diff-files', Gitlab::Diff::Line::SERIALIZE_KEYS, diff_options] - end - - private - - def highlight_diff_file_from_cache!(diff_file, cache_diff_lines) - diff_file.highlighted_diff_lines = cache_diff_lines.map do |line| - Gitlab::Diff::Line.init_from_hash(line) - end + cache.key end - # - # If we find the highlighted diff files lines on the cache we replace existing diff_files lines (no highlighted) - # for the highlighted ones, so we just skip their execution. - # If the highlighted diff files lines are not cached we calculate and cache them. - # - # The content of the cache is a Hash where the key identifies the file and the values are Arrays of - # hashes that represent serialized diff lines. - # - def cache_highlight!(diff_file) - item_key = diff_file.file_identifier - - if highlight_cache[item_key] - highlight_diff_file_from_cache!(diff_file, highlight_cache[item_key]) - else - highlight_cache[item_key] = diff_file.highlighted_diff_lines.map(&:to_hash) - end - end - - def highlight_cache - return @highlight_cache if defined?(@highlight_cache) - - @highlight_cache = Rails.cache.read(cache_key) || {} - @highlight_cache_was_empty = @highlight_cache.empty? - @highlight_cache + def real_size + @merge_request_diff.real_size end - def store_highlight_cache - Rails.cache.write(cache_key, highlight_cache, expires_in: 1.week) if @highlight_cache_was_empty - end + private - def cacheable?(diff_file) - @merge_request_diff.present? && diff_file.text? && diff_file.diffable? + def cache + @cache ||= Gitlab::Diff::HighlightCache.new(self) end end end diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb new file mode 100644 index 00000000000..e4390771db2 --- /dev/null +++ b/lib/gitlab/diff/highlight_cache.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true +# +module Gitlab + module Diff + class HighlightCache + delegate :diffable, to: :@diff_collection + delegate :diff_options, to: :@diff_collection + + def initialize(diff_collection, backend: Rails.cache) + @backend = backend + @diff_collection = diff_collection + end + + # - Reads from cache + # - Assigns DiffFile#highlighted_diff_lines for cached files + def decorate(diff_file) + if content = read_file(diff_file) + diff_file.highlighted_diff_lines = content.map do |line| + Gitlab::Diff::Line.init_from_hash(line) + end + end + end + + # It populates a Hash in order to submit a single write to the memory + # cache. This avoids excessive IO generated by N+1's (1 writing for + # each highlighted line or file). + def write_if_empty + return if cached_content.present? + + @diff_collection.diff_files.each do |diff_file| + next unless cacheable?(diff_file) + + diff_file_id = diff_file.file_identifier + + cached_content[diff_file_id] = diff_file.highlighted_diff_lines.map(&:to_hash) + end + + cache.write(key, cached_content, expires_in: 1.week) + end + + def clear + cache.delete(key) + end + + def key + [diffable, 'highlighted-diff-files', Gitlab::Diff::Line::SERIALIZE_KEYS, diff_options] + end + + private + + def read_file(diff_file) + cached_content[diff_file.file_identifier] + end + + def cache + @backend + end + + def cached_content + @cached_content ||= cache.read(key) || {} + end + + def cacheable?(diff_file) + diffable.present? && diff_file.text? && diff_file.diffable? + end + end + end +end diff --git a/lib/gitlab/fake_application_settings.rb b/lib/gitlab/fake_application_settings.rb index bb14a8cd9e7..2c827265d8c 100644 --- a/lib/gitlab/fake_application_settings.rb +++ b/lib/gitlab/fake_application_settings.rb @@ -5,12 +5,6 @@ # column type without parsing db/schema.rb. module Gitlab class FakeApplicationSettings < OpenStruct - def initialize(options = {}) - super - - FakeApplicationSettings.define_predicate_methods(options) - end - # Mimic ActiveRecord predicate methods for boolean values def self.define_predicate_methods(options) options.each do |key, value| @@ -23,5 +17,23 @@ module Gitlab end end end + + def initialize(options = {}) + super + + FakeApplicationSettings.define_predicate_methods(options) + end + + def key_restriction_for(type) + 0 + end + + def allowed_key_types + ApplicationSetting::SUPPORTED_KEY_TYPES + end + + def pick_repository_storage + repository_storages.sample + end end end diff --git a/lib/gitlab/git/committer_with_hooks.rb b/lib/gitlab/git/committer_with_hooks.rb deleted file mode 100644 index 4198be7c9c9..00000000000 --- a/lib/gitlab/git/committer_with_hooks.rb +++ /dev/null @@ -1,47 +0,0 @@ -module Gitlab - module Git - class CommitterWithHooks < Gollum::Committer - attr_reader :gl_wiki - - def initialize(gl_wiki, options = {}) - @gl_wiki = gl_wiki - super(gl_wiki.gollum_wiki, options) - end - - def commit - # TODO: Remove after 10.8 - return super unless allowed_to_run_hooks? - - result = Gitlab::Git::OperationService.new(git_user, gl_wiki.repository).with_branch( - @wiki.ref, - start_branch_name: @wiki.ref - ) do |start_commit| - super(false) - end - - result[:newrev] - rescue Gitlab::Git::PreReceiveError => e - message = "Custom Hook failed: #{e.message}" - raise Gitlab::Git::Wiki::OperationError, message - end - - private - - # TODO: Remove after 10.8 - def allowed_to_run_hooks? - @options[:user_id] != 0 && @options[:username].present? - end - - def git_user - @git_user ||= Gitlab::Git::User.new(@options[:username], - @options[:name], - @options[:email], - gitlab_id) - end - - def gitlab_id - Gitlab::GlId.gl_id_from_id_value(@options[:user_id]) - end - end - end -end diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index 61ce10ca131..f6b51dc3982 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -1,6 +1,3 @@ -# Gitaly note: JV: needs RPC for Gitlab::Git::Diff.between. - -# Gitlab::Git::Diff is a wrapper around native Rugged::Diff object module Gitlab module Git class Diff @@ -52,20 +49,31 @@ module Gitlab repo.diff(common_commit, head, actual_options, *paths) end - # Return a copy of the +options+ hash containing only keys that can be - # passed to Rugged. Allowed options are: + # Return a copy of the +options+ hash containing only recognized keys. + # Allowed options are: # # :ignore_whitespace_change :: # If true, changes in amount of whitespace will be ignored. # - # :disable_pathspec_match :: - # If true, the given +*paths+ will be applied as exact matches, - # instead of as fnmatch patterns. + # :max_files :: + # Limit how many files will patches be allowed for before collapsing + # + # :max_lines :: + # Limit how many patch lines (across all files) will be allowed for + # before collapsing # + # :limits :: + # A hash with additional limits to check before collapsing patches. + # Allowed keys are: `max_bytes`, `safe_max_files`, `safe_max_lines` + # and `safe_max_bytes` + # + # :expanded :: + # If true, patch raw data will not be included in the diff after + # `max_files`, `max_lines` or any of the limits in `limits` are + # exceeded def filter_diff_options(options, default_options = {}) - allowed_options = [:ignore_whitespace_change, - :disable_pathspec_match, :paths, - :max_files, :max_lines, :limits, :expanded] + allowed_options = [:ignore_whitespace_change, :max_files, :max_lines, + :limits, :expanded] if default_options actual_defaults = default_options.dup @@ -93,7 +101,7 @@ module Gitlab # # "Binary files a/file/path and b/file/path differ\n" # This is used when we detect that a diff is binary - # using CharlockHolmes when Rugged treats it as text. + # using CharlockHolmes. def binary_message(old_path, new_path) "Binary files #{old_path} and #{new_path} differ\n" end @@ -106,8 +114,6 @@ module Gitlab when Hash init_from_hash(raw_diff) prune_diff_if_eligible - when Rugged::Patch, Rugged::Diff::Delta - init_from_rugged(raw_diff) when Gitlab::GitalyClient::Diff init_from_gitaly(raw_diff) prune_diff_if_eligible @@ -184,31 +190,6 @@ module Gitlab private - def init_from_rugged(rugged) - if rugged.is_a?(Rugged::Patch) - init_from_rugged_patch(rugged) - d = rugged.delta - else - d = rugged - end - - @new_path = encode!(d.new_file[:path]) - @old_path = encode!(d.old_file[:path]) - @a_mode = d.old_file[:mode].to_s(8) - @b_mode = d.new_file[:mode].to_s(8) - @new_file = d.added? - @renamed_file = d.renamed? - @deleted_file = d.deleted? - end - - def init_from_rugged_patch(patch) - # Don't bother initializing diffs that are too large. If a diff is - # binary we're not going to display anything so we skip the size check. - return if !patch.delta.binary? && prune_large_patch(patch) - - @diff = encode!(strip_diff_headers(patch.to_s)) - end - def init_from_hash(hash) raw_diff = hash.symbolize_keys @@ -262,23 +243,6 @@ module Gitlab false end - - # Strip out the information at the beginning of the patch's text to match - # Grit's output - def strip_diff_headers(diff_text) - # Delete everything up to the first line that starts with '---' or - # 'Binary' - diff_text.sub!(/\A.*?^(---|Binary)/m, '\1') - - if diff_text.start_with?('---', 'Binary') - diff_text - else - # If the diff_text did not contain a line starting with '---' or - # 'Binary', return the empty string. No idea why; we are just - # preserving behavior from before the refactor. - '' - end - end end end end diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index 219c69893ad..20dce8d0e06 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -11,7 +11,7 @@ module Gitlab delegate :max_files, :max_lines, :max_bytes, :safe_max_files, :safe_max_lines, :safe_max_bytes, to: :limits - def self.collection_limits(options = {}) + def self.limits(options = {}) limits = {} limits[:max_files] = options.fetch(:max_files, DEFAULT_LIMITS[:max_files]) limits[:max_lines] = options.fetch(:max_lines, DEFAULT_LIMITS[:max_lines]) @@ -19,13 +19,14 @@ module Gitlab limits[:safe_max_files] = [limits[:max_files], DEFAULT_LIMITS[:max_files]].min limits[:safe_max_lines] = [limits[:max_lines], DEFAULT_LIMITS[:max_lines]].min limits[:safe_max_bytes] = limits[:safe_max_files] * 5.kilobytes # Average 5 KB per file + limits[:max_patch_bytes] = Gitlab::Git::Diff::SIZE_LIMIT OpenStruct.new(limits) end def initialize(iterator, options = {}) @iterator = iterator - @limits = self.class.collection_limits(options) + @limits = self.class.limits(options) @enforce_limits = !!options.fetch(:limits, true) @expanded = !!options.fetch(:expanded, true) diff --git a/lib/gitlab/git/gitlab_projects.rb b/lib/gitlab/git/gitlab_projects.rb deleted file mode 100644 index 5ff15a787f0..00000000000 --- a/lib/gitlab/git/gitlab_projects.rb +++ /dev/null @@ -1,253 +0,0 @@ -module Gitlab - module Git - class GitlabProjects - include Gitlab::Git::Popen - include Gitlab::Utils::StrongMemoize - - # Name of shard where repositories are stored. - # Example: nfs-file06 - attr_reader :shard_name - - # Relative path is a directory name for repository with .git at the end. - # Example: gitlab-org/gitlab-test.git - attr_reader :repository_relative_path - - # This is the path at which the gitlab-shell hooks directory can be found. - # It's essential for integration between git and GitLab proper. All new - # repositories should have their hooks directory symlinked here. - attr_reader :global_hooks_path - - attr_reader :logger - - def initialize(shard_name, repository_relative_path, global_hooks_path:, logger:) - @shard_name = shard_name - @repository_relative_path = repository_relative_path - - @logger = logger - @global_hooks_path = global_hooks_path - @output = StringIO.new - end - - def output - io = @output.dup - io.rewind - io.read - end - - # Absolute path to the repository. - # Example: /home/git/repositorities/gitlab-org/gitlab-test.git - # Probably will be removed when we fully migrate to Gitaly, part of - # https://gitlab.com/gitlab-org/gitaly/issues/1124. - def repository_absolute_path - strong_memoize(:repository_absolute_path) do - File.join(shard_path, repository_relative_path) - end - end - - def shard_path - strong_memoize(:shard_path) do - Gitlab.config.repositories.storages.fetch(shard_name).legacy_disk_path - end - end - - # Import project via git clone --bare - # URL must be publicly cloneable - def import_project(source, timeout) - git_import_repository(source, timeout) - end - - def fork_repository(new_shard_name, new_repository_relative_path) - git_fork_repository(new_shard_name, new_repository_relative_path) - end - - def fetch_remote(name, timeout, force:, tags:, ssh_key: nil, known_hosts: nil, prune: true) - logger.info "Fetching remote #{name} for repository #{repository_absolute_path}." - cmd = fetch_remote_command(name, tags, prune, force) - - setup_ssh_auth(ssh_key, known_hosts) do |env| - run_with_timeout(cmd, timeout, repository_absolute_path, env).tap do |success| - unless success - logger.error "Fetching remote #{name} for repository #{repository_absolute_path} failed." - end - end - end - end - - def push_branches(remote_name, timeout, force, branch_names) - logger.info "Pushing branches from #{repository_absolute_path} to remote #{remote_name}: #{branch_names}" - cmd = %W(#{Gitlab.config.git.bin_path} push) - cmd << '--force' if force - cmd += %W(-- #{remote_name}).concat(branch_names) - - success = run_with_timeout(cmd, timeout, repository_absolute_path) - - unless success - logger.error("Pushing branches to remote #{remote_name} failed.") - end - - success - end - - def delete_remote_branches(remote_name, branch_names) - branches = branch_names.map { |branch_name| ":#{branch_name}" } - - logger.info "Pushing deleted branches from #{repository_absolute_path} to remote #{remote_name}: #{branch_names}" - cmd = %W(#{Gitlab.config.git.bin_path} push -- #{remote_name}).concat(branches) - - success = run(cmd, repository_absolute_path) - - unless success - logger.error("Pushing deleted branches to remote #{remote_name} failed.") - end - - success - end - - protected - - def run(*args) - output, exitstatus = popen(*args) - @output << output - - exitstatus&.zero? - end - - def run_with_timeout(*args) - output, exitstatus = popen_with_timeout(*args) - @output << output - - exitstatus&.zero? - rescue Timeout::Error - @output.puts('Timed out') - - false - end - - def mask_password_in_url(url) - result = URI(url) - result.password = "*****" unless result.password.nil? - result.user = "*****" unless result.user.nil? # it's needed for oauth access_token - result - rescue - url - end - - def remove_origin_in_repo - cmd = %W(#{Gitlab.config.git.bin_path} remote rm origin) - run(cmd, repository_absolute_path) - end - - # Builds a small shell script that can be used to execute SSH with a set of - # custom options. - # - # Options are expanded as `'-oKey="Value"'`, so SSH will correctly interpret - # paths with spaces in them. We trust the user not to embed single or double - # quotes in the key or value. - def custom_ssh_script(options = {}) - args = options.map { |k, v| %Q{'-o#{k}="#{v}"'} }.join(' ') - - [ - "#!/bin/sh", - "exec ssh #{args} \"$@\"" - ].join("\n") - end - - # Known hosts data and private keys can be passed to gitlab-shell in the - # environment. If present, this method puts them into temporary files, writes - # a script that can substitute as `ssh`, setting the options to respect those - # files, and yields: { "GIT_SSH" => "/tmp/myScript" } - def setup_ssh_auth(key, known_hosts) - options = {} - - if key - key_file = Tempfile.new('gitlab-shell-key-file') - key_file.chmod(0o400) - key_file.write(key) - key_file.close - - options['IdentityFile'] = key_file.path - options['IdentitiesOnly'] = 'yes' - end - - if known_hosts - known_hosts_file = Tempfile.new('gitlab-shell-known-hosts') - known_hosts_file.chmod(0o400) - known_hosts_file.write(known_hosts) - known_hosts_file.close - - options['StrictHostKeyChecking'] = 'yes' - options['UserKnownHostsFile'] = known_hosts_file.path - end - - return yield({}) if options.empty? - - script = Tempfile.new('gitlab-shell-ssh-wrapper') - script.chmod(0o755) - script.write(custom_ssh_script(options)) - script.close - - yield('GIT_SSH' => script.path) - ensure - key_file&.close! - known_hosts_file&.close! - script&.close! - end - - private - - def fetch_remote_command(name, tags, prune, force) - %W(#{Gitlab.config.git.bin_path} fetch #{name} --quiet).tap do |cmd| - cmd << '--prune' if prune - cmd << '--force' if force - cmd << (tags ? '--tags' : '--no-tags') - end - end - - def git_import_repository(source, timeout) - # Skip import if repo already exists - return false if File.exist?(repository_absolute_path) - - masked_source = mask_password_in_url(source) - - logger.info "Importing project from <#{masked_source}> to <#{repository_absolute_path}>." - cmd = %W(#{Gitlab.config.git.bin_path} clone --bare -- #{source} #{repository_absolute_path}) - - success = run_with_timeout(cmd, timeout, nil) - - unless success - logger.error("Importing project from <#{masked_source}> to <#{repository_absolute_path}> failed.") - FileUtils.rm_rf(repository_absolute_path) - return false - end - - Gitlab::Git::Repository.create_hooks(repository_absolute_path, global_hooks_path) - - # The project was imported successfully. - # Remove the origin URL since it may contain password. - remove_origin_in_repo - - true - end - - def git_fork_repository(new_shard_name, new_repository_relative_path) - from_path = repository_absolute_path - new_shard_path = Gitlab.config.repositories.storages.fetch(new_shard_name).legacy_disk_path - to_path = File.join(new_shard_path, new_repository_relative_path) - - # The repository cannot already exist - if File.exist?(to_path) - logger.error "fork-repository failed: destination repository <#{to_path}> already exists." - return false - end - - # Ensure the namepsace / hashed storage directory exists - FileUtils.mkdir_p(File.dirname(to_path), mode: 0770) - - logger.info "Forking repository from <#{from_path}> to <#{to_path}>." - cmd = %W(#{Gitlab.config.git.bin_path} clone --bare --no-local -- #{from_path} #{to_path}) - - run(cmd, nil) && Gitlab::Git::Repository.create_hooks(to_path, global_hooks_path) - end - end - end -end diff --git a/lib/gitlab/git/hook.rb b/lib/gitlab/git/hook.rb deleted file mode 100644 index 94ff5b4980a..00000000000 --- a/lib/gitlab/git/hook.rb +++ /dev/null @@ -1,108 +0,0 @@ -# Gitaly note: JV: looks like this is only used by Gitlab::Git::HooksService in -# app/services. We shouldn't bother migrating this until we know how -# Gitlab::Git::HooksService will be migrated. - -module Gitlab - module Git - class Hook - GL_PROTOCOL = 'web'.freeze - attr_reader :name, :path, :repository - - def initialize(name, repository) - @name = name - @repository = repository - @path = File.join(repo_path, 'hooks', name) - end - - def repo_path - repository.path - end - - def exists? - File.exist?(path) - end - - def trigger(gl_id, gl_username, oldrev, newrev, ref) - return [true, nil] unless exists? - - Bundler.with_clean_env do - case name - when "pre-receive", "post-receive" - call_receive_hook(gl_id, gl_username, oldrev, newrev, ref) - when "update" - call_update_hook(gl_id, gl_username, oldrev, newrev, ref) - end - end - end - - private - - def call_receive_hook(gl_id, gl_username, oldrev, newrev, ref) - changes = [oldrev, newrev, ref].join(" ") - - exit_status = false - exit_message = nil - - vars = { - 'GL_ID' => gl_id, - 'GL_USERNAME' => gl_username, - 'PWD' => repo_path, - 'GL_PROTOCOL' => GL_PROTOCOL, - 'GL_REPOSITORY' => repository.gl_repository - } - - options = { - chdir: repo_path - } - - Open3.popen3(vars, path, options) do |stdin, stdout, stderr, wait_thr| - exit_status = true - stdin.sync = true - - # in git, pre- and post- receive hooks may just exit without - # reading stdin. We catch the exception to avoid a broken pipe - # warning - begin - # inject all the changes as stdin to the hook - changes.lines do |line| - stdin.puts line - end - rescue Errno::EPIPE - end - - stdin.close - - unless wait_thr.value == 0 - exit_status = false - exit_message = retrieve_error_message(stderr, stdout) - end - end - - [exit_status, exit_message] - end - - def call_update_hook(gl_id, gl_username, oldrev, newrev, ref) - env = { - 'GL_ID' => gl_id, - 'GL_USERNAME' => gl_username, - 'PWD' => repo_path - } - - options = { - chdir: repo_path - } - - args = [ref, oldrev, newrev] - - stdout, stderr, status = Open3.capture3(env, path, *args, options) - [status.success?, stderr.presence || stdout] - end - - def retrieve_error_message(stderr, stdout) - err_message = stderr.read - err_message = err_message.blank? ? stdout.read : err_message - err_message - end - end - end -end diff --git a/lib/gitlab/git/hooks_service.rb b/lib/gitlab/git/hooks_service.rb deleted file mode 100644 index e67cacdb95a..00000000000 --- a/lib/gitlab/git/hooks_service.rb +++ /dev/null @@ -1,35 +0,0 @@ -module Gitlab - module Git - class HooksService - attr_accessor :oldrev, :newrev, :ref - - def execute(pusher, repository, oldrev, newrev, ref) - @repository = repository - @gl_id = pusher.gl_id - @gl_username = pusher.username - @oldrev = oldrev - @newrev = newrev - @ref = ref - - %w(pre-receive update).each do |hook_name| - status, message = run_hook(hook_name) - - unless status - raise PreReceiveError, message - end - end - - yield(self).tap do - run_hook('post-receive') - end - end - - private - - def run_hook(name) - hook = Gitlab::Git::Hook.new(name, @repository) - hook.trigger(@gl_id, @gl_username, oldrev, newrev, ref) - end - end - end -end diff --git a/lib/gitlab/git/index.rb b/lib/gitlab/git/index.rb index d94082a3e30..c2e4274e3ee 100644 --- a/lib/gitlab/git/index.rb +++ b/lib/gitlab/git/index.rb @@ -1,157 +1,7 @@ -# Gitaly note: JV: When the time comes I think we will want to copy this -# class into Gitaly. None of its methods look like they should be RPC's. -# The RPC's will be at a higher level. - module Gitlab module Git class Index IndexError = Class.new(StandardError) - - DEFAULT_MODE = 0o100644 - - ACTIONS = %w(create create_dir update move delete).freeze - ACTION_OPTIONS = %i(file_path previous_path content encoding).freeze - - attr_reader :repository, :raw_index - - def initialize(repository) - @repository = repository - @raw_index = repository.rugged.index - end - - delegate :read_tree, :get, to: :raw_index - - def apply(action, options) - validate_action!(action) - public_send(action, options.slice(*ACTION_OPTIONS)) # rubocop:disable GitlabSecurity/PublicSend - end - - def write_tree - raw_index.write_tree(repository.rugged) - end - - def dir_exists?(path) - raw_index.find { |entry| entry[:path].start_with?("#{path}/") } - end - - def create(options) - options = normalize_options(options) - - if get(options[:file_path]) - raise IndexError, "A file with this name already exists" - end - - add_blob(options) - end - - def create_dir(options) - options = normalize_options(options) - - if get(options[:file_path]) - raise IndexError, "A file with this name already exists" - end - - if dir_exists?(options[:file_path]) - raise IndexError, "A directory with this name already exists" - end - - options = options.dup - options[:file_path] += '/.gitkeep' - options[:content] = '' - - add_blob(options) - end - - def update(options) - options = normalize_options(options) - - file_entry = get(options[:file_path]) - unless file_entry - raise IndexError, "A file with this name doesn't exist" - end - - add_blob(options, mode: file_entry[:mode]) - end - - def move(options) - options = normalize_options(options) - - file_entry = get(options[:previous_path]) - unless file_entry - raise IndexError, "A file with this name doesn't exist" - end - - if get(options[:file_path]) - raise IndexError, "A file with this name already exists" - end - - raw_index.remove(options[:previous_path]) - - add_blob(options, mode: file_entry[:mode]) - end - - def delete(options) - options = normalize_options(options) - - unless get(options[:file_path]) - raise IndexError, "A file with this name doesn't exist" - end - - raw_index.remove(options[:file_path]) - end - - private - - def normalize_options(options) - options = options.dup - options[:file_path] = normalize_path(options[:file_path]) if options[:file_path] - options[:previous_path] = normalize_path(options[:previous_path]) if options[:previous_path] - options - end - - def normalize_path(path) - unless path - raise IndexError, "You must provide a file path" - end - - pathname = Gitlab::Git::PathHelper.normalize_path(path.dup) - - pathname.each_filename do |segment| - if segment == '..' - raise IndexError, 'Path cannot include directory traversal' - end - end - - pathname.to_s - end - - def add_blob(options, mode: nil) - content = options[:content] - unless content - raise IndexError, "You must provide content" - end - - content = Base64.decode64(content) if options[:encoding] == 'base64' - - detect = CharlockHolmes::EncodingDetector.new.detect(content) - unless detect && detect[:type] == :binary - # When writing to the repo directly as we are doing here, - # the `core.autocrlf` config isn't taken into account. - content.gsub!("\r\n", "\n") if repository.autocrlf - end - - oid = repository.rugged.write(content, :blob) - - raw_index.add(path: options[:file_path], oid: oid, mode: mode || DEFAULT_MODE) - rescue Rugged::IndexError => e - raise IndexError, e.message - end - - def validate_action!(action) - unless ACTIONS.include?(action.to_s) - raise ArgumentError, "Unknown action '#{action}'" - end - end end end end diff --git a/lib/gitlab/git/operation_service.rb b/lib/gitlab/git/operation_service.rb index 57d748343be..0584629ac84 100644 --- a/lib/gitlab/git/operation_service.rb +++ b/lib/gitlab/git/operation_service.rb @@ -1,8 +1,6 @@ module Gitlab module Git class OperationService - include Gitlab::Git::Popen - BranchUpdate = Struct.new(:newrev, :repo_created, :branch_created) do alias_method :repo_created?, :repo_created alias_method :branch_created?, :branch_created @@ -17,177 +15,6 @@ module Gitlab ) end end - - attr_reader :user, :repository - - def initialize(user, new_repository) - if user - user = Gitlab::Git::User.from_gitlab(user) unless user.respond_to?(:gl_id) - @user = user - end - - # Refactoring aid - Gitlab::Git.check_namespace!(new_repository) - - @repository = new_repository - end - - def add_branch(branch_name, newrev) - ref = Gitlab::Git::BRANCH_REF_PREFIX + branch_name - oldrev = Gitlab::Git::BLANK_SHA - - update_ref_in_hooks(ref, newrev, oldrev) - end - - def rm_branch(branch) - ref = Gitlab::Git::BRANCH_REF_PREFIX + branch.name - oldrev = branch.target - newrev = Gitlab::Git::BLANK_SHA - - update_ref_in_hooks(ref, newrev, oldrev) - end - - def add_tag(tag_name, newrev, options = {}) - ref = Gitlab::Git::TAG_REF_PREFIX + tag_name - oldrev = Gitlab::Git::BLANK_SHA - - with_hooks(ref, newrev, oldrev) do |service| - # We want to pass the OID of the tag object to the hooks. For an - # annotated tag we don't know that OID until after the tag object - # (raw_tag) is created in the repository. That is why we have to - # update the value after creating the tag object. Only the - # "post-receive" hook will receive the correct value in this case. - raw_tag = repository.rugged.tags.create(tag_name, newrev, options) - service.newrev = raw_tag.target_id - end - end - - def rm_tag(tag) - ref = Gitlab::Git::TAG_REF_PREFIX + tag.name - oldrev = tag.target - newrev = Gitlab::Git::BLANK_SHA - - update_ref_in_hooks(ref, newrev, oldrev) do - repository.rugged.tags.delete(tag_name) - end - end - - # Whenever `start_branch_name` is passed, if `branch_name` doesn't exist, - # it would be created from `start_branch_name`. - # If `start_repository` is passed, and the branch doesn't exist, - # it would try to find the commits from it instead of current repository. - def with_branch( - branch_name, - start_branch_name: nil, - start_repository: repository, - &block) - - Gitlab::Git.check_namespace!(start_repository) - start_repository = RemoteRepository.new(start_repository) unless start_repository.is_a?(RemoteRepository) - - start_branch_name = nil if start_repository.empty? - - if start_branch_name && !start_repository.branch_exists?(start_branch_name) - raise ArgumentError, "Cannot find branch #{start_branch_name} in #{start_repository.relative_path}" - end - - update_branch_with_hooks(branch_name) do - repository.with_repo_branch_commit( - start_repository, - start_branch_name || branch_name, - &block) - end - end - - def update_branch(branch_name, newrev, oldrev) - ref = Gitlab::Git::BRANCH_REF_PREFIX + branch_name - update_ref_in_hooks(ref, newrev, oldrev) - end - - private - - # Returns [newrev, should_run_after_create, should_run_after_create_branch] - def update_branch_with_hooks(branch_name) - update_autocrlf_option - - was_empty = repository.empty? - - # Make commit - newrev = yield - - unless newrev - raise Gitlab::Git::CommitError.new('Failed to create commit') - end - - branch = repository.find_branch(branch_name) - oldrev = find_oldrev_from_branch(newrev, branch) - - ref = Gitlab::Git::BRANCH_REF_PREFIX + branch_name - update_ref_in_hooks(ref, newrev, oldrev) - - BranchUpdate.new(newrev, was_empty, was_empty || Gitlab::Git.blank_ref?(oldrev)) - end - - def find_oldrev_from_branch(newrev, branch) - return Gitlab::Git::BLANK_SHA unless branch - - oldrev = branch.target - - merge_base = repository.merge_base(newrev, branch.target) - raise Gitlab::Git::Repository::InvalidRef unless merge_base - - if oldrev == merge_base - oldrev - else - raise Gitlab::Git::CommitError.new('Branch diverged') - end - end - - def update_ref_in_hooks(ref, newrev, oldrev) - with_hooks(ref, newrev, oldrev) do - update_ref(ref, newrev, oldrev) - end - end - - def with_hooks(ref, newrev, oldrev) - Gitlab::Git::HooksService.new.execute( - user, - repository, - oldrev, - newrev, - ref) do |service| - - yield(service) - end - end - - # Gitaly note: JV: wait with migrating #update_ref until we know how to migrate its call sites. - def update_ref(ref, newrev, oldrev) - # We use 'git update-ref' because libgit2/rugged currently does not - # offer 'compare and swap' ref updates. Without compare-and-swap we can - # (and have!) accidentally reset the ref to an earlier state, clobbering - # commits. See also https://github.com/libgit2/libgit2/issues/1534. - command = %W[#{Gitlab.config.git.bin_path} update-ref --stdin -z] - - output, status = popen( - command, - repository.path) do |stdin| - stdin.write("update #{ref}\x00#{newrev}\x00#{oldrev}\x00") - end - - unless status.zero? - Gitlab::GitLogger.error("'git update-ref' in #{repository.path}: #{output}") - raise Gitlab::Git::CommitError.new( - "Could not update branch #{Gitlab::Git.branch_name(ref)}." \ - " Please refresh and try again.") - end - end - - def update_autocrlf_option - if repository.autocrlf != :input - repository.autocrlf = :input - end - end end end end diff --git a/lib/gitlab/git/popen.rb b/lib/gitlab/git/popen.rb deleted file mode 100644 index 7426688fc55..00000000000 --- a/lib/gitlab/git/popen.rb +++ /dev/null @@ -1,112 +0,0 @@ -# Gitaly note: JV: no RPC's here. - -require 'open3' - -module Gitlab - module Git - module Popen - FAST_GIT_PROCESS_TIMEOUT = 15.seconds - - def popen(cmd, path, vars = {}, lazy_block: nil) - unless cmd.is_a?(Array) - raise "System commands must be given as an array of strings" - end - - path ||= Dir.pwd - vars['PWD'] = path - options = { chdir: path } - - cmd_output = "" - cmd_status = 0 - Open3.popen3(vars, *cmd, options) do |stdin, stdout, stderr, wait_thr| - stdout.set_encoding(Encoding::ASCII_8BIT) - - # stderr and stdout pipes can block if stderr/stdout aren't drained: https://bugs.ruby-lang.org/issues/9082 - # Mimic what Ruby does with capture3: https://github.com/ruby/ruby/blob/1ec544695fa02d714180ef9c34e755027b6a2103/lib/open3.rb#L257-L273 - err_reader = Thread.new { stderr.read } - - yield(stdin) if block_given? - stdin.close - - if lazy_block - cmd_output = lazy_block.call(stdout.lazy) - cmd_status = 0 - break - else - cmd_output << stdout.read - end - - cmd_output << err_reader.value - cmd_status = wait_thr.value.exitstatus - end - - [cmd_output, cmd_status] - end - - def popen_with_timeout(cmd, timeout, path, vars = {}) - unless cmd.is_a?(Array) - raise "System commands must be given as an array of strings" - end - - path ||= Dir.pwd - vars['PWD'] = path - - unless File.directory?(path) - FileUtils.mkdir_p(path) - end - - rout, wout = IO.pipe - rerr, werr = IO.pipe - - pid = Process.spawn(vars, *cmd, out: wout, err: werr, chdir: path, pgroup: true) - # stderr and stdout pipes can block if stderr/stdout aren't drained: https://bugs.ruby-lang.org/issues/9082 - # Mimic what Ruby does with capture3: https://github.com/ruby/ruby/blob/1ec544695fa02d714180ef9c34e755027b6a2103/lib/open3.rb#L257-L273 - out_reader = Thread.new { rout.read } - err_reader = Thread.new { rerr.read } - - begin - # close write ends so we could read them - wout.close - werr.close - - status = process_wait_with_timeout(pid, timeout) - - cmd_output = out_reader.value - cmd_output << err_reader.value # Copying the behaviour of `popen` which merges stderr into output - - [cmd_output, status.exitstatus] - rescue Timeout::Error => e - kill_process_group_for_pid(pid) - - raise e - ensure - wout.close unless wout.closed? - werr.close unless werr.closed? - - rout.close - rerr.close - end - end - - def process_wait_with_timeout(pid, timeout) - deadline = timeout.seconds.from_now - wait_time = 0.01 - - while deadline > Time.now - sleep(wait_time) - _, status = Process.wait2(pid, Process::WNOHANG) - - return status unless status.nil? - end - - raise Timeout::Error, "Timeout waiting for process ##{pid}" - end - - def kill_process_group_for_pid(pid) - Process.kill("KILL", -pid) - Process.wait(pid) - rescue Errno::ESRCH - end - end - end -end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 9521a2d63a0..1b8d320ff3b 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -6,7 +6,6 @@ module Gitlab module Git class Repository include Gitlab::Git::RepositoryMirroring - include Gitlab::Git::Popen include Gitlab::EncodingHelper include Gitlab::Utils::StrongMemoize @@ -73,7 +72,7 @@ module Gitlab # Relative path of repo attr_reader :relative_path - attr_reader :gitlab_projects, :storage, :gl_repository, :relative_path + attr_reader :storage, :gl_repository, :relative_path # This initializer method is only used on the client side (gitlab-ce). # Gitaly-ruby uses a different initializer. @@ -82,13 +81,6 @@ module Gitlab @relative_path = relative_path @gl_repository = gl_repository - @gitlab_projects = Gitlab::Git::GitlabProjects.new( - storage, - relative_path, - global_hooks_path: Gitlab.config.gitlab_shell.hooks_path, - logger: Rails.logger - ) - @name = @relative_path.split("/").last end @@ -121,10 +113,6 @@ module Gitlab raise NoRepository.new('no repository for such path') end - def cleanup - @rugged&.close - end - def circuit_breaker @circuit_breaker ||= Gitlab::Git::Storage::CircuitBreaker.for_storage(storage) end @@ -148,10 +136,6 @@ module Gitlab end end - def reload_rugged - @rugged = nil - end - # Directly find a branch with a simple name (e.g. master) # def find_branch(name) @@ -250,15 +234,6 @@ module Gitlab end end - # Returns an Array of all ref names, except when it's matching pattern - # - # regexp - The pattern for ref names we don't want - def all_ref_names_except(prefixes) - rugged.references.reject do |ref| - prefixes.any? { |p| ref.name.start_with?(p) } - end.map(&:name) - end - def archive_metadata(ref, storage_path, project_path, format = "tar.gz", append_sha:) ref ||= root_ref commit = Gitlab::Git::Commit.find(self, ref) @@ -331,7 +306,7 @@ module Gitlab (size.to_f / 1024).round(2) end - # Use the Rugged Walker API to build an array of commits. + # Build an array of commits. # # Usage. # repo.log( @@ -591,19 +566,6 @@ module Gitlab end end - def check_revert_content(target_commit, source_sha) - args = [target_commit.sha, source_sha] - args << { mainline: 1 } if target_commit.merge_commit? - - revert_index = rugged.revert_commit(*args) - return false if revert_index.conflicts? - - tree_id = revert_index.write_tree(rugged) - return false unless diff_exists?(source_sha, tree_id) - - tree_id - end - def cherry_pick(user:, commit:, branch_name:, message:, start_branch_name:, start_repository:) args = { user: user, @@ -619,10 +581,6 @@ module Gitlab end end - def diff_exists?(sha1, sha2) - rugged.diff(sha1, sha2).size > 0 - end - def user_to_committer(user) Gitlab::Git.committer_hash(email: user.email, name: user.name) end @@ -666,6 +624,14 @@ module Gitlab end end + def find_remote_root_ref(remote_name) + return unless remote_name.present? + + wrapped_gitaly_errors do + gitaly_remote_client.find_remote_root_ref(remote_name) + end + end + AUTOCRLF_VALUES = { "true" => true, "false" => false, @@ -738,48 +704,6 @@ module Gitlab end end - def with_repo_branch_commit(start_repository, start_branch_name) - Gitlab::Git.check_namespace!(start_repository) - start_repository = RemoteRepository.new(start_repository) unless start_repository.is_a?(RemoteRepository) - - return yield nil if start_repository.empty? - - if start_repository.same_repository?(self) - yield commit(start_branch_name) - else - start_commit_id = start_repository.commit_id(start_branch_name) - - return yield nil unless start_commit_id - - if branch_commit = commit(start_commit_id) - yield branch_commit - else - with_repo_tmp_commit( - start_repository, start_branch_name, start_commit_id) do |tmp_commit| - yield tmp_commit - end - end - end - end - - def with_repo_tmp_commit(start_repository, start_branch_name, sha) - source_ref = start_branch_name - - unless Gitlab::Git.branch_ref?(source_ref) - source_ref = "#{Gitlab::Git::BRANCH_REF_PREFIX}#{source_ref}" - end - - tmp_ref = fetch_ref( - start_repository, - source_ref: source_ref, - target_ref: "refs/tmp/#{SecureRandom.hex}" - ) - - yield commit(sha) - ensure - delete_refs(tmp_ref) if tmp_ref - end - def fetch_source_branch!(source_repository, source_branch, local_ref) wrapped_gitaly_errors do gitaly_repository_client.fetch_source_branch(source_repository, source_branch, local_ref) @@ -809,21 +733,6 @@ module Gitlab end end - # This method, fetch_ref, is used from within - # Gitlab::Git::OperationService. OperationService will eventually only - # exist in gitaly-ruby. When we delete OperationService from gitlab-ce - # we can also remove fetch_ref. - def fetch_ref(source_repository, source_ref:, target_ref:) - Gitlab::Git.check_namespace!(source_repository) - source_repository = RemoteRepository.new(source_repository) unless source_repository.is_a?(RemoteRepository) - - args = %W(fetch --no-tags -f #{GITALY_INTERNAL_URL} #{source_ref}:#{target_ref}) - message, status = run_git(args, env: source_repository.fetch_env) - raise Gitlab::Git::CommandError, message if status != 0 - - target_ref - end - # Refactoring aid; allows us to copy code from app/models/repository.rb def commit(ref = 'HEAD') Gitlab::Git::Commit.find(self, ref) @@ -891,24 +800,6 @@ module Gitlab end end - def push_remote_branches(remote_name, branch_names, forced: true) - success = @gitlab_projects.push_branches(remote_name, GITLAB_PROJECTS_TIMEOUT, forced, branch_names) - - success || gitlab_projects_error - end - - def delete_remote_branches(remote_name, branch_names) - success = @gitlab_projects.delete_remote_branches(remote_name, branch_names) - - success || gitlab_projects_error - end - - def delete_remote_branches(remote_name, branch_names) - success = @gitlab_projects.delete_remote_branches(remote_name, branch_names) - - success || gitlab_projects_error - end - def bundle_to_disk(save_path) wrapped_gitaly_errors do gitaly_repository_client.create_bundle(save_path) @@ -1056,37 +947,12 @@ module Gitlab end end - def shell_blame(sha, path) - output, _status = run_git(%W(blame -p #{sha} -- #{path})) - output - end - def last_commit_for_path(sha, path) wrapped_gitaly_errors do gitaly_commit_client.last_commit_for_path(sha, path) end end - def rev_list(including: [], excluding: [], options: [], objects: false, &block) - args = ['rev-list'] - - args.push(*rev_list_param(including)) - - exclude_param = *rev_list_param(excluding) - if exclude_param.any? - args.push('--not') - args.push(*exclude_param) - end - - args.push('--objects') if objects - - if options.any? - args.push(*options) - end - - run_git!(args, lazy_block: block) - end - def checksum # The exists? RPC is much cheaper, so we perform this request first raise NoRepository, "Repository does not exists" unless exists? @@ -1104,44 +970,6 @@ module Gitlab end end - def run_git(args, chdir: path, env: {}, nice: false, lazy_block: nil, &block) - cmd = [Gitlab.config.git.bin_path, *args] - cmd.unshift("nice") if nice - - object_directories = alternate_object_directories - if object_directories.any? - env['GIT_ALTERNATE_OBJECT_DIRECTORIES'] = object_directories.join(File::PATH_SEPARATOR) - end - - circuit_breaker.perform do - popen(cmd, chdir, env, lazy_block: lazy_block, &block) - end - end - - def run_git!(args, chdir: path, env: {}, nice: false, lazy_block: nil, &block) - output, status = run_git(args, chdir: chdir, env: env, nice: nice, lazy_block: lazy_block, &block) - - raise GitError, output unless status.zero? - - output - end - - def run_git_with_timeout(args, timeout, env: {}) - circuit_breaker.perform do - popen_with_timeout([Gitlab.config.git.bin_path, *args], timeout, path, env) - end - end - - def git_env_for_user(user) - { - 'GIT_COMMITTER_NAME' => user.name, - 'GIT_COMMITTER_EMAIL' => user.email, - 'GL_ID' => Gitlab::GlId.gl_id(user), - 'GL_PROTOCOL' => Gitlab::Git::Hook::GL_PROTOCOL, - 'GL_REPOSITORY' => gl_repository - } - end - def gitaly_merged_branch_names(branch_names, root_sha) qualified_branch_names = branch_names.map { |b| "refs/heads/#{b}" } @@ -1192,23 +1020,6 @@ module Gitlab Gitlab::Git::HookEnv.all(gl_repository).values_at(*ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES).flatten.compact end - def sort_branches(branches, sort_by) - case sort_by - when 'name' - branches.sort_by(&:name) - when 'updated_desc' - branches.sort do |a, b| - b.dereferenced_target.committed_date <=> a.dereferenced_target.committed_date - end - when 'updated_asc' - branches.sort do |a, b| - a.dereferenced_target.committed_date <=> b.dereferenced_target.committed_date - end - else - branches - end - end - # Returns true if the given ref name exists # # Ref names must start with `refs/`. @@ -1223,14 +1034,6 @@ module Gitlab def gitaly_delete_refs(*ref_names) gitaly_ref_client.delete_refs(refs: ref_names) if ref_names.any? end - - def gitlab_projects_error - raise CommandError, @gitlab_projects.output - end - - def rev_list_param(spec) - spec == :all ? ['--all'] : spec - end end end end diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index cb851b76a23..e0867aeb5a7 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -50,51 +50,6 @@ module Gitlab entry[:oid] end end - - def tree_entries_from_rugged(repository, sha, path, recursive) - current_path_entries = get_tree_entries_from_rugged(repository, sha, path) - ordered_entries = [] - - current_path_entries.each do |entry| - ordered_entries << entry - - if recursive && entry.dir? - ordered_entries.concat(tree_entries_from_rugged(repository, sha, entry.path, true)) - end - end - - ordered_entries - end - - def get_tree_entries_from_rugged(repository, sha, path) - commit = repository.lookup(sha) - root_tree = commit.tree - - tree = if path - id = find_id_by_path(repository, root_tree.oid, path) - if id - repository.lookup(id) - else - [] - end - else - root_tree - end - - tree.map do |entry| - new( - id: entry[:oid], - root_id: root_tree.oid, - name: entry[:name], - type: entry[:type], - mode: entry[:filemode].to_s(8), - path: path ? File.join(path, entry[:name]) : entry[:name], - commit_id: sha - ) - end - rescue Rugged::ReferenceError - [] - end end def initialize(options) diff --git a/lib/gitlab/git/version.rb b/lib/gitlab/git/version.rb index 1e14e8b652a..4bd91898457 100644 --- a/lib/gitlab/git/version.rb +++ b/lib/gitlab/git/version.rb @@ -1,8 +1,6 @@ module Gitlab module Git module Version - extend Gitlab::Git::Popen - def self.git_version Gitlab::VersionInfo.parse(Gitaly::Server.all.first.git_binary_version) end diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index 9d992be66eb..ae92a624e05 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -163,20 +163,6 @@ module Gitlab Gitlab::Git::WikiPage.new(wiki_page, version) end end - - def committer_with_hooks(commit_details) - Gitlab::Git::CommitterWithHooks.new(self, commit_details.to_h) - end - - def with_committer_with_hooks(commit_details, &block) - committer = committer_with_hooks(commit_details) - - yield committer - - committer.commit - - nil - end end end end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 93720500711..30cd09a0ca7 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -241,8 +241,6 @@ module Gitlab end elsif user # User access is verified in check_change_access! - elsif authed_via_jwt? - # Authenticated via JWT else raise UnauthorizedError, ERROR_MESSAGES[:upload] end @@ -331,10 +329,6 @@ module Gitlab !Gitlab.config.gitlab_shell.receive_pack end - def authed_via_jwt? - false - end - protected def changes_list diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 6a97cd8ed17..aa5b4f94090 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -369,7 +369,7 @@ module Gitlab request_params[:ignore_whitespace_change] = options.fetch(:ignore_whitespace_change, false) request_params[:enforce_limits] = options.fetch(:limits, true) request_params[:collapse_diffs] = !options.fetch(:expanded, true) - request_params.merge!(Gitlab::Git::DiffCollection.collection_limits(options).to_h) + request_params.merge!(Gitlab::Git::DiffCollection.limits(options).to_h) request = Gitaly::CommitDiffRequest.new(request_params) response = GitalyClient.call(@repository.storage, :diff_service, :commit_diff, request, timeout: GitalyClient.medium_timeout) diff --git a/lib/gitlab/gitaly_client/remote_service.rb b/lib/gitlab/gitaly_client/remote_service.rb index 1381e033d4b..4661448621b 100644 --- a/lib/gitlab/gitaly_client/remote_service.rb +++ b/lib/gitlab/gitaly_client/remote_service.rb @@ -1,6 +1,8 @@ module Gitlab module GitalyClient class RemoteService + include Gitlab::EncodingHelper + MAX_MSG_SIZE = 128.kilobytes.freeze def self.exists?(remote_url) @@ -52,6 +54,18 @@ module Gitlab response.result end + def find_remote_root_ref(remote_name) + request = Gitaly::FindRemoteRootRefRequest.new( + repository: @gitaly_repo, + remote: remote_name + ) + + response = GitalyClient.call(@storage, :remote_service, + :find_remote_root_ref, request) + + encode_utf8(response.ref) + end + def update_remote_mirror(ref_name, only_branches_matching) req_enum = Enumerator.new do |y| y.yield Gitaly::UpdateRemoteMirrorRequest.new( diff --git a/lib/gitlab/gitaly_client/storage_service.rb b/lib/gitlab/gitaly_client/storage_service.rb index eb0e910665b..3a26dd58ff4 100644 --- a/lib/gitlab/gitaly_client/storage_service.rb +++ b/lib/gitlab/gitaly_client/storage_service.rb @@ -5,6 +5,14 @@ module Gitlab @storage = storage end + # Returns all directories in the git storage directory, lexically ordered + def list_directories(depth: 1) + request = Gitaly::ListDirectoriesRequest.new(storage_name: @storage, depth: depth) + + GitalyClient.call(@storage, :storage_service, :list_directories, request) + .flat_map(&:paths) + end + # Delete all repositories in the storage. This is a slow and VERY DESTRUCTIVE operation. def delete_all_repositories request = Gitaly::DeleteAllRepositoriesRequest.new(storage_name: @storage) diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index be3710c5b7f..53fe2f8e436 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -40,10 +40,6 @@ module Gitlab "#{basename[0..FILENAME_LIMIT]}_export.tar.gz" end - def object_storage? - Feature.enabled?(:import_export_object_storage) - end - def version VERSION end diff --git a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb index 83134bb0769..7cbf653dd97 100644 --- a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb +++ b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb @@ -53,7 +53,7 @@ module Gitlab end def self.lock_file_path(project) - return unless project.export_path || object_storage? + return unless project.export_path || export_file_exists? lock_path = project.import_export_shared.archive_path @@ -83,8 +83,8 @@ module Gitlab errors.full_messages.each { |msg| project.import_export_shared.add_error_message(msg) } end - def object_storage? - project.export_project_object_exists? + def export_file_exists? + project.export_file_exists? end end end diff --git a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb index dce8f89c0ab..4f29bdcea2c 100644 --- a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb +++ b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb @@ -23,7 +23,7 @@ module Gitlab def strategy_execute handle_response_error(send_file) - project.remove_exported_project_file + project.remove_exports end def handle_response_error(response) @@ -40,15 +40,11 @@ module Gitlab def send_file Gitlab::HTTP.public_send(http_method.downcase, url, send_file_options) # rubocop:disable GitlabSecurity/PublicSend ensure - export_file.close if export_file && !object_storage? + export_file.close if export_file end def export_file - if object_storage? - project.import_export_upload.export_file.file.open - else - File.open(project.export_project_path) - end + project.export_file.open end def send_file_options @@ -63,11 +59,7 @@ module Gitlab end def export_size - if object_storage? - project.import_export_upload.export_file.file.size - else - File.size(project.export_project_path) - end + project.export_file.file.size end end end diff --git a/lib/gitlab/import_export/avatar_restorer.rb b/lib/gitlab/import_export/avatar_restorer.rb index cfa595629f4..17796430811 100644 --- a/lib/gitlab/import_export/avatar_restorer.rb +++ b/lib/gitlab/import_export/avatar_restorer.rb @@ -19,7 +19,7 @@ module Gitlab private def avatar_export_file - @avatar_export_file ||= Dir["#{avatar_export_path}/*"].first + @avatar_export_file ||= Dir["#{avatar_export_path}/**/*"].find { |f| File.file?(f) } end def avatar_export_path diff --git a/lib/gitlab/import_export/avatar_saver.rb b/lib/gitlab/import_export/avatar_saver.rb index 31ef0490cb3..6ffebf83dd2 100644 --- a/lib/gitlab/import_export/avatar_saver.rb +++ b/lib/gitlab/import_export/avatar_saver.rb @@ -1,8 +1,6 @@ module Gitlab module ImportExport class AvatarSaver - include Gitlab::ImportExport::CommandLineUtil - def initialize(project:, shared:) @project = project @shared = shared @@ -14,19 +12,12 @@ module Gitlab Gitlab::ImportExport::UploadsManager.new( project: @project, shared: @shared, - relative_export_path: 'avatar', - from: avatar_path + relative_export_path: 'avatar' ).save rescue => e @shared.error(e) false end - - private - - def avatar_path - @project.avatar.path - end end end end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index f69f98a78a3..a19b3c88627 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -19,6 +19,9 @@ project_tree: - milestone: - events: - :push_event_payload + - resource_label_events: + - label: + :priorities - :issue_assignees - snippets: - :award_emoji @@ -45,6 +48,9 @@ project_tree: - milestone: - events: - :push_event_payload + - resource_label_events: + - label: + :priorities - pipelines: - notes: - :author @@ -64,6 +70,7 @@ project_tree: - :create_access_levels - :project_feature - :custom_attributes + - :prometheus_metrics - :project_badges - :ci_cd_settings @@ -108,6 +115,9 @@ excluded_attributes: - :remote_mirror_available_overridden - :description_html - :repository_languages + prometheus_metrics: + - :common + - :identifier snippets: - :expired_at merge_request_diff: @@ -133,6 +143,10 @@ excluded_attributes: - :event_id project_badges: - :group_id + resource_label_events: + - :reference + - :reference_html + - :epic_id methods: labels: diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index 4e179f63d8c..72d5b9b830c 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -92,8 +92,6 @@ module Gitlab end def remove_import_file - return unless Gitlab::ImportExport.object_storage? - upload = @project.import_export_upload return unless upload&.import_file&.file diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index f4106e03a57..00ea4b833e2 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -199,7 +199,7 @@ module Gitlab end def excluded_keys_for_relation(relation) - @reader.attributes_finder.find_excluded_keys(relation) + reader.attributes_finder.find_excluded_keys(relation) end end end diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb index 3cd153a4fd2..59a74083395 100644 --- a/lib/gitlab/import_export/saver.rb +++ b/lib/gitlab/import_export/saver.rb @@ -18,7 +18,7 @@ module Gitlab Rails.logger.info("Saved project export #{archive_file}") - save_on_object_storage if use_object_storage? + save_upload else @shared.error(Gitlab::ImportExport::Error.new(error_message)) false @@ -27,10 +27,8 @@ module Gitlab @shared.error(e) false ensure - if use_object_storage? - remove_archive - remove_export_path - end + remove_archive + remove_export_path end private @@ -51,7 +49,7 @@ module Gitlab @archive_file ||= File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(project: @project)) end - def save_on_object_storage + def save_upload upload = ImportExportUpload.find_or_initialize_by(project: @project) File.open(archive_file) { |file| upload.export_file = file } @@ -59,12 +57,8 @@ module Gitlab upload.save! end - def use_object_storage? - Gitlab::ImportExport.object_storage? - end - def error_message - "Unable to save #{archive_file} into #{@shared.export_path}. Object storage enabled: #{use_object_storage?}" + "Unable to save #{archive_file} into #{@shared.export_path}." end end end diff --git a/lib/gitlab/import_export/uploads_manager.rb b/lib/gitlab/import_export/uploads_manager.rb index e0d4235e65b..8511319cb1c 100644 --- a/lib/gitlab/import_export/uploads_manager.rb +++ b/lib/gitlab/import_export/uploads_manager.rb @@ -5,18 +5,13 @@ module Gitlab UPLOADS_BATCH_SIZE = 100 - def initialize(project:, shared:, relative_export_path: 'uploads', from: nil) + def initialize(project:, shared:, relative_export_path: 'uploads') @project = project @shared = shared @relative_export_path = relative_export_path - @from = from || default_uploads_path end def save - if File.file?(@from) && @relative_export_path == 'avatar' - copy_files(@from, File.join(uploads_export_path, @project.avatar.filename)) - end - copy_project_uploads true @@ -55,17 +50,11 @@ module Gitlab copy_files(uploader.absolute_path, File.join(uploads_export_path, uploader.upload.path)) else - next unless Gitlab::ImportExport.object_storage? - download_and_copy(uploader) end end end - def default_uploads_path - FileUploader.absolute_base_dir(@project) - end - def uploads_export_path @uploads_export_path ||= File.join(@shared.export_path, @relative_export_path) end diff --git a/lib/gitlab/import_export/uploads_restorer.rb b/lib/gitlab/import_export/uploads_restorer.rb index 25f85936227..b4313ff4cb4 100644 --- a/lib/gitlab/import_export/uploads_restorer.rb +++ b/lib/gitlab/import_export/uploads_restorer.rb @@ -2,30 +2,14 @@ module Gitlab module ImportExport class UploadsRestorer < UploadsSaver def restore - if Gitlab::ImportExport.object_storage? - Gitlab::ImportExport::UploadsManager.new( - project: @project, - shared: @shared - ).restore - elsif File.directory?(uploads_export_path) - copy_files(uploads_export_path, uploads_path) - - true - else - true # Proceed without uploads - end + Gitlab::ImportExport::UploadsManager.new( + project: @project, + shared: @shared + ).restore rescue => e @shared.error(e) false end - - def uploads_path - FileUploader.absolute_base_dir(@project) - end - - def uploads_export_path - @uploads_export_path ||= File.join(@shared.export_path, 'uploads') - end end end end diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb index b3f17af5661..0275f686c5e 100644 --- a/lib/gitlab/import_export/uploads_saver.rb +++ b/lib/gitlab/import_export/uploads_saver.rb @@ -1,8 +1,6 @@ module Gitlab module ImportExport class UploadsSaver - include Gitlab::ImportExport::CommandLineUtil - def initialize(project:, shared:) @project = project @shared = shared diff --git a/lib/gitlab/project_service_logger.rb b/lib/gitlab/project_service_logger.rb new file mode 100644 index 00000000000..e84dca97962 --- /dev/null +++ b/lib/gitlab/project_service_logger.rb @@ -0,0 +1,7 @@ +module Gitlab + class ProjectServiceLogger < Gitlab::JsonLogger + def self.file_name_noext + 'integrations_json' + end + end +end diff --git a/lib/gitlab/prometheus/additional_metrics_parser.rb b/lib/gitlab/prometheus/additional_metrics_parser.rb index bb1172f82a1..a240d090074 100644 --- a/lib/gitlab/prometheus/additional_metrics_parser.rb +++ b/lib/gitlab/prometheus/additional_metrics_parser.rb @@ -5,7 +5,7 @@ module Gitlab MUTEX = Mutex.new extend self - def load_groups_from_yaml(file_name = 'additional_metrics.yml') + def load_groups_from_yaml(file_name) yaml_metrics_raw(file_name).map(&method(:group_from_entry)) end diff --git a/lib/gitlab/prometheus/metric_group.rb b/lib/gitlab/prometheus/metric_group.rb index e91c6fb2e27..d696a8fc00c 100644 --- a/lib/gitlab/prometheus/metric_group.rb +++ b/lib/gitlab/prometheus/metric_group.rb @@ -4,10 +4,13 @@ module Gitlab include ActiveModel::Model attr_accessor :name, :priority, :metrics + validates :name, :priority, :metrics, presence: true def self.common_metrics - AdditionalMetricsParser.load_groups_from_yaml + ::PrometheusMetric.common.group_by(&:group_title).map do |name, metrics| + MetricGroup.new(name: name, priority: 0, metrics: metrics.map(&:to_query_metric)) + end end # EE only diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index a17cd27e82d..c8fa50757f6 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -368,15 +368,6 @@ module Gitlab private - def gitlab_projects(shard_name, disk_path) - Gitlab::Git::GitlabProjects.new( - shard_name, - disk_path, - global_hooks_path: Gitlab.config.gitlab_shell.hooks_path, - logger: Rails.logger - ) - end - def gitlab_shell_fast_execute(cmd) output, status = gitlab_shell_fast_execute_helper(cmd) diff --git a/lib/gitlab/template_helper.rb b/lib/gitlab/template_helper.rb index f24a01e6cf5..fc498dde723 100644 --- a/lib/gitlab/template_helper.rb +++ b/lib/gitlab/template_helper.rb @@ -1,24 +1,9 @@ module Gitlab module TemplateHelper - include Gitlab::Utils::StrongMemoize - def prepare_template_environment(file) return unless file - if Gitlab::ImportExport.object_storage? - params[:import_export_upload] = ImportExportUpload.new(import_file: file) - else - FileUtils.mkdir_p(File.dirname(import_upload_path)) - FileUtils.copy_entry(file.path, import_upload_path) - - params[:import_source] = import_upload_path - end - end - - def import_upload_path - strong_memoize(:import_upload_path) do - Gitlab::ImportExport.import_upload_path(filename: tmp_filename) - end + params[:import_export_upload] = ImportExportUpload.new(import_file: file) end def tmp_filename diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index a9629a92a50..30a8c3578d8 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -22,18 +22,27 @@ module Gitlab project = repository.project - { + attrs = { GL_ID: Gitlab::GlId.gl_id(user), GL_REPOSITORY: Gitlab::GlRepository.gl_repository(project, is_wiki), GL_USERNAME: user&.username, ShowAllRefs: show_all_refs, Repository: repository.gitaly_repository.to_h, RepoPath: 'ignored but not allowed to be empty in gitlab-workhorse', + GitConfigOptions: [], GitalyServer: { address: Gitlab::GitalyClient.address(project.repository_storage), token: Gitlab::GitalyClient.token(project.repository_storage) } } + + # Custom option for git-receive-pack command + receive_max_input_size = Gitlab::CurrentSettings.receive_max_input_size.to_i + if receive_max_input_size > 0 + attrs[:GitConfigOptions] << "receive.maxInputSize=#{receive_max_input_size.megabytes}" + end + + attrs end def send_git_blob(repository, blob) diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index c8a8863443e..e8ae5dfa540 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -1,40 +1,29 @@ -# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/954 -# +# frozen_string_literal: true +require 'set' + namespace :gitlab do namespace :cleanup do - HASHED_REPOSITORY_NAME = '@hashed'.freeze - desc "GitLab | Cleanup | Clean namespaces" task dirs: :gitlab_environment do - warn_user_is_not_gitlab + namespaces = Set.new(Namespace.pluck(:path)) + namespaces << Storage::HashedProject::ROOT_PATH_PREFIX - namespaces = Namespace.pluck(:path) - namespaces << HASHED_REPOSITORY_NAME # add so that it will be ignored - Gitlab.config.repositories.storages.each do |name, repository_storage| - git_base_path = Gitlab::GitalyClient::StorageSettings.allow_disk_access { repository_storage.legacy_disk_path } - all_dirs = Dir.glob(git_base_path + '/*') + Gitaly::Server.all.each do |server| + all_dirs = Gitlab::GitalyClient::StorageService + .new(server.storage) + .list_directories(depth: 0) + .reject { |dir| dir.ends_with?('.git') || namespaces.include?(File.basename(dir)) } - puts git_base_path.color(:yellow) puts "Looking for directories to remove... " - - all_dirs.reject! do |dir| - # skip if git repo - dir =~ /.git$/ - end - - all_dirs.reject! do |dir| - dir_name = File.basename dir - - # skip if namespace present - namespaces.include?(dir_name) - end - all_dirs.each do |dir_path| if remove? - if FileUtils.rm_rf dir_path - puts "Removed...#{dir_path}".color(:red) - else - puts "Cannot remove #{dir_path}".color(:red) + begin + Gitlab::GitalyClient::NamespaceService.new(server.storage) + .remove(dir_path) + + puts "Removed...#{dir_path}" + rescue StandardError => e + puts "Cannot remove #{dir_path}: #{e.message}".color(:red) end else puts "Can be removed: #{dir_path}".color(:red) @@ -49,29 +38,29 @@ namespace :gitlab do desc "GitLab | Cleanup | Clean repositories" task repos: :gitlab_environment do - warn_user_is_not_gitlab - move_suffix = "+orphaned+#{Time.now.to_i}" - Gitlab.config.repositories.storages.each do |name, repository_storage| - repo_root = Gitlab::GitalyClient::StorageSettings.allow_disk_access { repository_storage.legacy_disk_path } - - # Look for global repos (legacy, depth 1) and normal repos (depth 2) - IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find| - find.each_line do |path| - path.chomp! - repo_with_namespace = path - .sub(repo_root, '') - .sub(%r{^/*}, '') - .chomp('.git') - .chomp('.wiki') - - # TODO ignoring hashed repositories for now. But revisit to fully support - # possible orphaned hashed repos - next if repo_with_namespace.start_with?("#{HASHED_REPOSITORY_NAME}/") || Project.find_by_full_path(repo_with_namespace) - - new_path = path + move_suffix - puts path.inspect + ' -> ' + new_path.inspect - File.rename(path, new_path) + + Gitaly::Server.all.each do |server| + Gitlab::GitalyClient::StorageService + .new(server.storage) + .list_directories + .each do |path| + repo_with_namespace = path.chomp('.git').chomp('.wiki') + + # TODO ignoring hashed repositories for now. But revisit to fully support + # possible orphaned hashed repos + next if repo_with_namespace.start_with?(Storage::HashedProject::ROOT_PATH_PREFIX) + next if Project.find_by_full_path(repo_with_namespace) + + new_path = path + move_suffix + puts path.inspect + ' -> ' + new_path.inspect + + begin + Gitlab::GitalyClient::NamespaceService + .new(server.storage) + .rename(path, new_path) + rescue StandardError => e + puts "Error occured while moving the repository: #{e.message}".color(:red) end end end diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake index a25f7ce59c7..ef6a32d6730 100644 --- a/lib/tasks/gitlab/update_templates.rake +++ b/lib/tasks/gitlab/update_templates.rake @@ -6,6 +6,8 @@ namespace :gitlab do desc "GitLab | Update project templates" task :update_project_templates do + include Gitlab::ImportExport::CommandLineUtil + if Rails.env.production? puts "This rake task is not meant fo production instances".red exit(1) @@ -52,7 +54,7 @@ namespace :gitlab do end Projects::ImportExport::ExportService.new(project, admin).execute - FileUtils.cp(project.export_project_path, template.archive_path) + download_or_copy_upload(project.export_file, template.archive_path) Projects::DestroyService.new(admin, project).execute puts "Exported #{template.name}".green end diff --git a/lib/tasks/migrate/add_limits_mysql.rake b/lib/tasks/migrate/add_limits_mysql.rake index 9b05876034c..c77fa49d586 100644 --- a/lib/tasks/migrate/add_limits_mysql.rake +++ b/lib/tasks/migrate/add_limits_mysql.rake @@ -3,6 +3,7 @@ require Rails.root.join('db/migrate/markdown_cache_limits_to_mysql') require Rails.root.join('db/migrate/merge_request_diff_file_limits_to_mysql') require Rails.root.join('db/migrate/limits_ci_build_trace_chunks_raw_data_for_mysql') require Rails.root.join('db/migrate/gpg_keys_limits_to_mysql') +require Rails.root.join('db/migrate/prometheus_metrics_limits_to_mysql') desc "GitLab | Add limits to strings in mysql database" task add_limits_mysql: :environment do @@ -12,4 +13,5 @@ task add_limits_mysql: :environment do MergeRequestDiffFileLimitsToMysql.new.up LimitsCiBuildTraceChunksRawDataForMysql.new.up IncreaseMysqlTextLimitForGpgKeys.new.up + PrometheusMetricsLimitsToMysql.new.up end |