diff options
| author | Matija Čupić <matteeyah@gmail.com> | 2018-02-04 23:38:59 +0100 |
|---|---|---|
| committer | Matija Čupić <matteeyah@gmail.com> | 2018-02-04 23:38:59 +0100 |
| commit | 3366f377c1d4cbb02ecc5a2e47b059ed375c5e09 (patch) | |
| tree | 6a19813b4820ad6d2813bf86b30d8e5be2bd10c6 /lib | |
| parent | 0abce36cd20cdd3579138bee835d28519a5593f2 (diff) | |
| parent | cf887a8b3108edb715ee5618377f4ffab1824d85 (diff) | |
| download | gitlab-ce-3366f377c1d4cbb02ecc5a2e47b059ed375c5e09.tar.gz | |
Merge branch 'master' into 38265-stuckcijobsworker-wrongly-detects-cancels-stuck-builds-when-per-job-timeout-is-more-than-an-hour
Diffstat (limited to 'lib')
113 files changed, 1290 insertions, 731 deletions
diff --git a/lib/api/access_requests.rb b/lib/api/access_requests.rb index 374b611f55e..60ae5e6b9a2 100644 --- a/lib/api/access_requests.rb +++ b/lib/api/access_requests.rb @@ -24,7 +24,7 @@ module API access_requesters = AccessRequestsFinder.new(source).execute!(current_user) access_requesters = paginate(access_requesters.includes(:user)) - present access_requesters.map(&:user), with: Entities::AccessRequester, source: source + present access_requesters, with: Entities::AccessRequester end desc "Requests access for the authenticated user to a #{source_type}." do @@ -36,7 +36,7 @@ module API access_requester = source.request_access(current_user) if access_requester.persisted? - present access_requester.user, with: Entities::AccessRequester, access_requester: access_requester + present access_requester, with: Entities::AccessRequester else render_validation_error!(access_requester) end @@ -56,7 +56,7 @@ module API member = ::Members::ApproveAccessRequestService.new(source, current_user, declared_params).execute status :created - present member.user, with: Entities::Member, member: member + present member, with: Entities::Member end desc 'Denies an access request for the given user.' do diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 7b9a80a234b..e13463ec66b 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -205,22 +205,15 @@ module API expose :build_artifacts_size, as: :job_artifacts_size end - class Member < UserBasic - expose :access_level do |user, options| - member = options[:member] || options[:source].members.find_by(user_id: user.id) - member.access_level - end - expose :expires_at do |user, options| - member = options[:member] || options[:source].members.find_by(user_id: user.id) - member.expires_at - end + class Member < Grape::Entity + expose :user, merge: true, using: UserBasic + expose :access_level + expose :expires_at end - class AccessRequester < UserBasic - expose :requested_at do |user, options| - access_requester = options[:access_requester] || options[:source].requesters.find_by(user_id: user.id) - access_requester.requested_at - end + class AccessRequester < Grape::Entity + expose :user, merge: true, using: UserBasic + expose :requested_at end class Group < Grape::Entity @@ -507,7 +500,15 @@ module API expose :work_in_progress?, as: :work_in_progress expose :milestone, using: Entities::Milestone expose :merge_when_pipeline_succeeds - expose :merge_status + + # Ideally we should deprecate `MergeRequest#merge_status` exposure and + # use `MergeRequest#mergeable?` instead (boolean). + # See https://gitlab.com/gitlab-org/gitlab-ce/issues/42344 for more + # information. + expose :merge_status do |merge_request| + merge_request.check_if_can_be_merged + merge_request.merge_status + end expose :diff_head_sha, as: :sha expose :merge_commit_sha expose :user_notes_count diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb index 87ba40e26e1..fbe30192a16 100644 --- a/lib/api/helpers/runner.rb +++ b/lib/api/helpers/runner.rb @@ -1,14 +1,12 @@ module API module Helpers module Runner - include Gitlab::CurrentSettings - JOB_TOKEN_HEADER = 'HTTP_JOB_TOKEN'.freeze JOB_TOKEN_PARAM = :token def runner_registration_token_valid? ActiveSupport::SecurityUtils.variable_size_secure_compare(params[:token], - current_application_settings.runners_registration_token) + Gitlab::CurrentSettings.runners_registration_token) end def get_runner_version_from_params @@ -53,7 +51,7 @@ module API end def max_artifacts_size - current_application_settings.max_artifacts_size.megabytes.to_i + Gitlab::CurrentSettings.max_artifacts_size.megabytes.to_i end end end diff --git a/lib/api/members.rb b/lib/api/members.rb index 130c6d6da71..bc1de37284a 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -21,10 +21,11 @@ module API get ":id/members" do source = find_source(source_type, params[:id]) - users = source.users - users = users.merge(User.search(params[:query])) if params[:query].present? + members = source.members.where.not(user_id: nil).includes(:user) + members = members.joins(:user).merge(User.search(params[:query])) if params[:query].present? + members = paginate(members) - present paginate(users), with: Entities::Member, source: source + present members, with: Entities::Member end desc 'Gets a member of a group or project.' do @@ -39,7 +40,7 @@ module API members = source.members member = members.find_by!(user_id: params[:user_id]) - present member.user, with: Entities::Member, member: member + present member, with: Entities::Member end desc 'Adds a member to a group or project.' do @@ -62,7 +63,7 @@ module API if !member not_allowed! # This currently can only be reached in EE elsif member.persisted? && member.valid? - present member.user, with: Entities::Member, member: member + present member, with: Entities::Member else render_validation_error!(member) end @@ -83,7 +84,7 @@ module API member = source.members.find_by!(user_id: params.delete(:user_id)) if member.update_attributes(declared_params(include_missing: false)) - present member.user, with: Entities::Member, member: member + present member, with: Entities::Member else render_validation_error!(member) end diff --git a/lib/api/runner.rb b/lib/api/runner.rb index 50cb1df92ad..5469cba69a6 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -214,9 +214,9 @@ module API job = authenticate_job! forbidden!('Job is not running!') unless job.running? - artifacts_upload_path = JobArtifactUploader.artifacts_upload_path - artifacts = uploaded_file(:file, artifacts_upload_path) - metadata = uploaded_file(:metadata, artifacts_upload_path) + workhorse_upload_path = JobArtifactUploader.workhorse_upload_path + artifacts = uploaded_file(:file, workhorse_upload_path) + metadata = uploaded_file(:metadata, workhorse_upload_path) bad_request!('Missing artifacts file!') unless artifacts file_to_large! unless artifacts.size < max_artifacts_size diff --git a/lib/api/templates.rb b/lib/api/templates.rb index 6550b331fb8..41862768a3f 100644 --- a/lib/api/templates.rb +++ b/lib/api/templates.rb @@ -17,15 +17,15 @@ module API } }.freeze PROJECT_TEMPLATE_REGEX = - /[\<\{\[] + %r{[\<\{\[] (project|description| one\sline\s.+\swhat\sit\sdoes\.) # matching the start and end is enough here - [\>\}\]]/xi.freeze + [\>\}\]]}xi.freeze YEAR_TEMPLATE_REGEX = /[<{\[](year|yyyy)[>}\]]/i.freeze FULLNAME_TEMPLATE_REGEX = - /[\<\{\[] + %r{[\<\{\[] (fullname|name\sof\s(author|copyright\sowner)) - [\>\}\]]/xi.freeze + [\>\}\]]}xi.freeze helpers do def parsed_license_template diff --git a/lib/api/v3/members.rb b/lib/api/v3/members.rb index 46145cac7a5..d7bde8ceb89 100644 --- a/lib/api/v3/members.rb +++ b/lib/api/v3/members.rb @@ -22,10 +22,11 @@ module API get ":id/members" do source = find_source(source_type, params[:id]) - users = source.users - users = users.merge(User.search(params[:query])) if params[:query].present? + members = source.members.where.not(user_id: nil).includes(:user) + members = members.joins(:user).merge(User.search(params[:query])) if params[:query].present? + members = paginate(members) - present paginate(users), with: ::API::Entities::Member, source: source + present members, with: ::API::Entities::Member end desc 'Gets a member of a group or project.' do @@ -40,7 +41,7 @@ module API members = source.members member = members.find_by!(user_id: params[:user_id]) - present member.user, with: ::API::Entities::Member, member: member + present member, with: ::API::Entities::Member end desc 'Adds a member to a group or project.' do @@ -69,7 +70,7 @@ module API end if member.persisted? && member.valid? - present member.user, with: ::API::Entities::Member, member: member + present member, with: ::API::Entities::Member else # This is to ensure back-compatibility but 400 behavior should be used # for all validation errors in 9.0! @@ -93,7 +94,7 @@ module API member = source.members.find_by!(user_id: params.delete(:user_id)) if member.update_attributes(declared_params(include_missing: false)) - present member.user, with: ::API::Entities::Member, member: member + present member, with: ::API::Entities::Member else # This is to ensure back-compatibility but 400 behavior should be used # for all validation errors in 9.0! @@ -125,7 +126,7 @@ module API else ::Members::DestroyService.new(source, current_user, declared_params).execute - present member.user, with: ::API::Entities::Member, member: member + present member, with: ::API::Entities::Member end end end diff --git a/lib/api/v3/projects.rb b/lib/api/v3/projects.rb index a7f0813bf74..c856ba99f09 100644 --- a/lib/api/v3/projects.rb +++ b/lib/api/v3/projects.rb @@ -173,7 +173,7 @@ module API use :sort_params use :pagination end - get "/search/:query", requirements: { query: /[^\/]+/ } do + get "/search/:query", requirements: { query: %r{[^/]+} } do search_service = Search::GlobalService.new(current_user, search: params[:query]).execute projects = search_service.objects('projects', params[:page], false) projects = projects.reorder(params[:order_by] => params[:sort]) diff --git a/lib/api/v3/templates.rb b/lib/api/v3/templates.rb index 7298203df10..b82b02b5f49 100644 --- a/lib/api/v3/templates.rb +++ b/lib/api/v3/templates.rb @@ -16,15 +16,15 @@ module API } }.freeze PROJECT_TEMPLATE_REGEX = - /[\<\{\[] + %r{[\<\{\[] (project|description| one\sline\s.+\swhat\sit\sdoes\.) # matching the start and end is enough here - [\>\}\]]/xi.freeze + [\>\}\]]}xi.freeze YEAR_TEMPLATE_REGEX = /[<{\[](year|yyyy)[>}\]]/i.freeze FULLNAME_TEMPLATE_REGEX = - /[\<\{\[] + %r{[\<\{\[] (fullname|name\sof\s(author|copyright\sowner)) - [\>\}\]]/xi.freeze + [\>\}\]]}xi.freeze DEPRECATION_MESSAGE = ' This endpoint is deprecated and has been removed in V4.'.freeze helpers do diff --git a/lib/backup/artifacts.rb b/lib/backup/artifacts.rb index 7a582a20056..4383124d150 100644 --- a/lib/backup/artifacts.rb +++ b/lib/backup/artifacts.rb @@ -3,7 +3,7 @@ require 'backup/files' module Backup class Artifacts < Files def initialize - super('artifacts', LegacyArtifactUploader.local_store_path) + super('artifacts', JobArtifactUploader.root) end def create_files_dir diff --git a/lib/banzai/color_parser.rb b/lib/banzai/color_parser.rb new file mode 100644 index 00000000000..355c364b07b --- /dev/null +++ b/lib/banzai/color_parser.rb @@ -0,0 +1,44 @@ +module Banzai + module ColorParser + ALPHA = /0(?:\.\d+)?|\.\d+|1(?:\.0+)?/ # 0.0..1.0 + PERCENTS = /(?:\d{1,2}|100)%/ # 00%..100% + ALPHA_CHANNEL = /(?:,\s*(?:#{ALPHA}|#{PERCENTS}))?/ + BITS = /\d{1,2}|1\d\d|2(?:[0-4]\d|5[0-5])/ # 00..255 + DEGS = /-?\d+(?:deg)?/i # [-]digits[deg] + RADS = /-?(?:\d+(?:\.\d+)?|\.\d+)rad/i # [-](digits[.digits] OR .digits)rad + HEX_FORMAT = /\#(?:\h{3}|\h{4}|\h{6}|\h{8})/ + RGB_FORMAT = %r{ + (?:rgba? + \( + (?: + (?:(?:#{BITS},\s*){2}#{BITS}) + | + (?:(?:#{PERCENTS},\s*){2}#{PERCENTS}) + ) + #{ALPHA_CHANNEL} + \) + ) + }xi + HSL_FORMAT = %r{ + (?:hsla? + \( + (?:#{DEGS}|#{RADS}),\s*#{PERCENTS},\s*#{PERCENTS} + #{ALPHA_CHANNEL} + \) + ) + }xi + + FORMATS = [HEX_FORMAT, RGB_FORMAT, HSL_FORMAT].freeze + + COLOR_FORMAT = /\A(#{Regexp.union(FORMATS)})\z/ix + + # Public: Analyzes whether the String is a color code. + # + # text - The String to be parsed. + # + # Returns the recognized color String or nil if none was found. + def self.parse(text) + text if COLOR_FORMAT =~ text + end + end +end diff --git a/lib/banzai/filter/color_filter.rb b/lib/banzai/filter/color_filter.rb new file mode 100644 index 00000000000..6ab29ac281f --- /dev/null +++ b/lib/banzai/filter/color_filter.rb @@ -0,0 +1,31 @@ +module Banzai + module Filter + # HTML filter that renders `color` followed by a color "chip". + # + class ColorFilter < HTML::Pipeline::Filter + COLOR_CHIP_CLASS = 'gfm-color_chip'.freeze + + def call + doc.css('code').each do |node| + color = ColorParser.parse(node.content) + node << color_chip(color) if color + end + + doc + end + + private + + def color_chip(color) + checkerboard = doc.document.create_element('span', class: COLOR_CHIP_CLASS) + chip = doc.document.create_element('span', style: inline_styles(color: color)) + + checkerboard << chip + end + + def inline_styles(color:) + "background-color: #{color};" + end + end + end +end diff --git a/lib/banzai/filter/emoji_filter.rb b/lib/banzai/filter/emoji_filter.rb index 6255a611dbe..b82c6ca6393 100644 --- a/lib/banzai/filter/emoji_filter.rb +++ b/lib/banzai/filter/emoji_filter.rb @@ -54,9 +54,9 @@ module Banzai # Build a regexp that matches all valid :emoji: names. def self.emoji_pattern @emoji_pattern ||= - /(?<=[^[:alnum:]:]|\n|^) + %r{(?<=[^[:alnum:]:]|\n|^) :(#{Gitlab::Emoji.emojis_names.map { |name| Regexp.escape(name) }.join('|')}): - (?=[^[:alnum:]:]|$)/x + (?=[^[:alnum:]:]|$)}x end # Build a regexp that matches all valid unicode emojis names. diff --git a/lib/banzai/filter/gollum_tags_filter.rb b/lib/banzai/filter/gollum_tags_filter.rb index 2e259904673..c2b42673376 100644 --- a/lib/banzai/filter/gollum_tags_filter.rb +++ b/lib/banzai/filter/gollum_tags_filter.rb @@ -51,10 +51,10 @@ module Banzai # See https://github.com/gollum/gollum/wiki # # Rubular: http://rubular.com/r/7dQnE5CUCH - TAGS_PATTERN = %r{\[\[(.+?)\]\]}.freeze + TAGS_PATTERN = /\[\[(.+?)\]\]/.freeze # Pattern to match allowed image extensions - ALLOWED_IMAGE_EXTENSIONS = %r{.+(jpg|png|gif|svg|bmp)\z}i.freeze + ALLOWED_IMAGE_EXTENSIONS = /.+(jpg|png|gif|svg|bmp)\z/i.freeze def call search_text_nodes(doc).each do |node| diff --git a/lib/banzai/pipeline/broadcast_message_pipeline.rb b/lib/banzai/pipeline/broadcast_message_pipeline.rb index adc09c8afbd..5dd572de3a1 100644 --- a/lib/banzai/pipeline/broadcast_message_pipeline.rb +++ b/lib/banzai/pipeline/broadcast_message_pipeline.rb @@ -7,6 +7,7 @@ module Banzai Filter::SanitizationFilter, Filter::EmojiFilter, + Filter::ColorFilter, Filter::AutolinkFilter, Filter::ExternalLinkFilter ] diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index c746f6f64e9..4001b8a85e3 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -14,6 +14,7 @@ module Banzai Filter::SyntaxHighlightFilter, Filter::MathFilter, + Filter::ColorFilter, Filter::MermaidFilter, Filter::VideoLinkFilter, Filter::ImageLazyLoadFilter, diff --git a/lib/container_registry/registry.rb b/lib/container_registry/registry.rb index 63bce655f57..f90d711474a 100644 --- a/lib/container_registry/registry.rb +++ b/lib/container_registry/registry.rb @@ -11,7 +11,7 @@ module ContainerRegistry private def default_path - @uri.sub(/^https?:\/\//, '') + @uri.sub(%r{^https?://}, '') end end end diff --git a/lib/email_template_interceptor.rb b/lib/email_template_interceptor.rb index f2bf3d0fb2b..3978a6d9fe4 100644 --- a/lib/email_template_interceptor.rb +++ b/lib/email_template_interceptor.rb @@ -1,10 +1,8 @@ # Read about interceptors in http://guides.rubyonrails.org/action_mailer_basics.html#intercepting-emails class EmailTemplateInterceptor - extend Gitlab::CurrentSettings - def self.delivering_email(message) # Remove HTML part if HTML emails are disabled. - unless current_application_settings.html_emails_enabled + unless Gitlab::CurrentSettings.html_emails_enabled message.parts.delete_if do |part| part.content_type.start_with?('text/html') end diff --git a/lib/extracts_path.rb b/lib/extracts_path.rb index d8aca3304c5..a9b04c183ad 100644 --- a/lib/extracts_path.rb +++ b/lib/extracts_path.rb @@ -56,7 +56,7 @@ module ExtractsPath if valid_refs.length == 0 # No exact ref match, so just try our best - pair = id.match(/([^\/]+)(.*)/).captures + pair = id.match(%r{([^/]+)(.*)}).captures else # There is a distinct possibility that multiple refs prefix the ID. # Use the longest match to maximize the chance that we have the @@ -68,7 +68,7 @@ module ExtractsPath end # Remove ending slashes from path - pair[1].gsub!(/^\/|\/$/, '') + pair[1].gsub!(%r{^/|/$}, '') pair end diff --git a/lib/gitaly/server.rb b/lib/gitaly/server.rb new file mode 100644 index 00000000000..605e93022e7 --- /dev/null +++ b/lib/gitaly/server.rb @@ -0,0 +1,43 @@ +module Gitaly + class Server + def self.all + Gitlab.config.repositories.storages.keys.map { |s| Gitaly::Server.new(s) } + end + + attr_reader :storage + + def initialize(storage) + @storage = storage + end + + def server_version + info.server_version + end + + def git_binary_version + info.git_version + end + + def up_to_date? + server_version == Gitlab::GitalyClient.expected_server_version + end + + def address + Gitlab::GitalyClient.address(@storage) + rescue RuntimeError => e + "Error getting the address: #{e.message}" + end + + private + + def info + @info ||= + begin + Gitlab::GitalyClient::ServerService.new(@storage).info + rescue GRPC::Unavailable, GRPC::GRPC::DeadlineExceeded + # This will show the server as being out of date + Gitaly::ServerInfoResponse.new(git_version: '', server_version: '') + end + end + end +end diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb index cead1c7eacd..ee7f4be6b9f 100644 --- a/lib/gitlab/asciidoc.rb +++ b/lib/gitlab/asciidoc.rb @@ -6,8 +6,6 @@ module Gitlab # Parser/renderer for the AsciiDoc format that uses Asciidoctor and filters # the resulting HTML through HTML pipeline filters. module Asciidoc - extend Gitlab::CurrentSettings - DEFAULT_ADOC_ATTRS = [ 'showtitle', 'idprefix=user-content-', 'idseparator=-', 'env=gitlab', 'env-gitlab', 'source-highlighter=html-pipeline', 'icons=font' @@ -33,9 +31,9 @@ module Gitlab def self.plantuml_setup Asciidoctor::PlantUml.configure do |conf| - conf.url = current_application_settings.plantuml_url - conf.svg_enable = current_application_settings.plantuml_enabled - conf.png_enable = current_application_settings.plantuml_enabled + conf.url = Gitlab::CurrentSettings.plantuml_url + conf.svg_enable = Gitlab::CurrentSettings.plantuml_enabled + conf.png_enable = Gitlab::CurrentSettings.plantuml_enabled conf.txt_enable = false end end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 65d7fd3ec70..05932378173 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -14,8 +14,6 @@ module Gitlab DEFAULT_SCOPES = [:api].freeze class << self - include Gitlab::CurrentSettings - def find_for_git_client(login, password, project:, ip:) raise "Must provide an IP for rate limiting" if ip.nil? @@ -57,7 +55,7 @@ module Gitlab if user.nil? || user.ldap_user? # Second chance - try LDAP authentication Gitlab::LDAP::Authentication.login(login, password) - elsif current_application_settings.password_authentication_enabled_for_git? + elsif Gitlab::CurrentSettings.password_authentication_enabled_for_git? user if user.active? && user.valid_password?(password) end end @@ -87,7 +85,7 @@ module Gitlab private def authenticate_using_internal_or_ldap_password? - current_application_settings.password_authentication_enabled_for_git? || Gitlab::LDAP::Config.enabled? + Gitlab::CurrentSettings.password_authentication_enabled_for_git? || Gitlab::LDAP::Config.enabled? end def service_request_check(login, password, project) diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index 81e95e5832d..8a8e770940e 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -12,7 +12,7 @@ module Gitlab # Ends with /:random_hex/:filename FILE_UPLOADER_PATH = %r{/\h+/[^/]+\z} - FULL_PATH_CAPTURE = %r{\A(.+)#{FILE_UPLOADER_PATH}} + FULL_PATH_CAPTURE = /\A(.+)#{FILE_UPLOADER_PATH}/ # These regex patterns are tested against a relative path, relative to # the upload directory. @@ -143,7 +143,7 @@ module Gitlab end def absolute_path - File.join(CarrierWave.root, path) + File.join(Gitlab.config.uploads.storage_path, path) end end diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 4e0121ca34d..a7a1bbe1752 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -11,9 +11,12 @@ module Gitlab FIND_BATCH_SIZE = 500 RELATIVE_UPLOAD_DIR = "uploads".freeze - ABSOLUTE_UPLOAD_DIR = "#{CarrierWave.root}/#{RELATIVE_UPLOAD_DIR}".freeze + ABSOLUTE_UPLOAD_DIR = File.join( + Gitlab.config.uploads.storage_path, + RELATIVE_UPLOAD_DIR + ) FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads'.freeze - START_WITH_CARRIERWAVE_ROOT_REGEX = %r{\A#{CarrierWave.root}/} + START_WITH_ROOT_REGEX = %r{\A#{Gitlab.config.uploads.storage_path}/} EXCLUDED_HASHED_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/@hashed/*".freeze EXCLUDED_TMP_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/tmp/*".freeze @@ -81,7 +84,7 @@ module Gitlab paths = [] stdout.each_line("\0") do |line| - paths << line.chomp("\0").sub(START_WITH_CARRIERWAVE_ROOT_REGEX, '') + paths << line.chomp("\0").sub(START_WITH_ROOT_REGEX, '') if paths.size >= batch_size yield(paths) diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb index 5b2f09e03ea..428c0505808 100644 --- a/lib/gitlab/ci/build/artifacts/metadata/entry.rb +++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb @@ -97,7 +97,7 @@ module Gitlab end def total_size - descendant_pattern = %r{^#{Regexp.escape(@path.to_s)}} + descendant_pattern = /^#{Regexp.escape(@path.to_s)}/ entries.sum do |path, entry| (entry[:size] if path =~ descendant_pattern).to_i end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 91fd9cc7631..b7c596a973d 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -1,73 +1,79 @@ module Gitlab module CurrentSettings - extend self + class << self + def current_application_settings + if RequestStore.active? + RequestStore.fetch(:current_application_settings) { ensure_application_settings! } + else + ensure_application_settings! + end + end - def current_application_settings - if RequestStore.active? - RequestStore.fetch(:current_application_settings) { ensure_application_settings! } - else - ensure_application_settings! + def fake_application_settings(defaults = ::ApplicationSetting.defaults) + Gitlab::FakeApplicationSettings.new(defaults) end - end - delegate :sidekiq_throttling_enabled?, to: :current_application_settings + def method_missing(name, *args, &block) + current_application_settings.send(name, *args, &block) # rubocop:disable GitlabSecurity/PublicSend + end - def fake_application_settings(defaults = ::ApplicationSetting.defaults) - FakeApplicationSettings.new(defaults) - end + def respond_to_missing?(name, include_private = false) + current_application_settings.respond_to?(name, include_private) || super + end - private + private - def ensure_application_settings! - return in_memory_application_settings if ENV['IN_MEMORY_APPLICATION_SETTINGS'] == 'true' + def ensure_application_settings! + return in_memory_application_settings if ENV['IN_MEMORY_APPLICATION_SETTINGS'] == 'true' - cached_application_settings || uncached_application_settings - end + cached_application_settings || uncached_application_settings + end - def cached_application_settings - begin - ::ApplicationSetting.cached - rescue ::Redis::BaseError, ::Errno::ENOENT, ::Errno::EADDRNOTAVAIL - # In case Redis isn't running or the Redis UNIX socket file is not available + def cached_application_settings + begin + ::ApplicationSetting.cached + rescue ::Redis::BaseError, ::Errno::ENOENT, ::Errno::EADDRNOTAVAIL + # In case Redis isn't running or the Redis UNIX socket file is not available + end end - end - def uncached_application_settings - return fake_application_settings unless connect_to_db? + def uncached_application_settings + return fake_application_settings unless connect_to_db? - db_settings = ::ApplicationSetting.current + db_settings = ::ApplicationSetting.current - # If there are pending migrations, it's possible there are columns that - # need to be added to the application settings. To prevent Rake tasks - # and other callers from failing, use any loaded settings and return - # defaults for missing columns. - if ActiveRecord::Migrator.needs_migration? - defaults = ::ApplicationSetting.defaults - defaults.merge!(db_settings.attributes.symbolize_keys) if db_settings.present? - return fake_application_settings(defaults) - end + # If there are pending migrations, it's possible there are columns that + # need to be added to the application settings. To prevent Rake tasks + # and other callers from failing, use any loaded settings and return + # defaults for missing columns. + if ActiveRecord::Migrator.needs_migration? + defaults = ::ApplicationSetting.defaults + defaults.merge!(db_settings.attributes.symbolize_keys) if db_settings.present? + return fake_application_settings(defaults) + end - return db_settings if db_settings.present? + return db_settings if db_settings.present? - ::ApplicationSetting.create_from_defaults || in_memory_application_settings - end + ::ApplicationSetting.create_from_defaults || in_memory_application_settings + end - def in_memory_application_settings - @in_memory_application_settings ||= ::ApplicationSetting.new(::ApplicationSetting.defaults) # rubocop:disable Gitlab/ModuleWithInstanceVariables - rescue ActiveRecord::StatementInvalid, ActiveRecord::UnknownAttributeError - # In case migrations the application_settings table is not created yet, - # we fallback to a simple OpenStruct - fake_application_settings - end + def in_memory_application_settings + @in_memory_application_settings ||= ::ApplicationSetting.new(::ApplicationSetting.defaults) # rubocop:disable Gitlab/ModuleWithInstanceVariables + rescue ActiveRecord::StatementInvalid, ActiveRecord::UnknownAttributeError + # In case migrations the application_settings table is not created yet, + # we fallback to a simple OpenStruct + fake_application_settings + end - def connect_to_db? - # When the DBMS is not available, an exception (e.g. PG::ConnectionBad) is raised - active_db_connection = ActiveRecord::Base.connection.active? rescue false + def connect_to_db? + # When the DBMS is not available, an exception (e.g. PG::ConnectionBad) is raised + active_db_connection = ActiveRecord::Base.connection.active? rescue false - active_db_connection && - ActiveRecord::Base.connection.table_exists?('application_settings') - rescue ActiveRecord::NoDatabaseError - false + active_db_connection && + ActiveRecord::Base.connection.table_exists?('application_settings') + rescue ActiveRecord::NoDatabaseError + false + end end end end diff --git a/lib/gitlab/dependency_linker/composer_json_linker.rb b/lib/gitlab/dependency_linker/composer_json_linker.rb index 0245bf4077a..cfd4ec15125 100644 --- a/lib/gitlab/dependency_linker/composer_json_linker.rb +++ b/lib/gitlab/dependency_linker/composer_json_linker.rb @@ -11,7 +11,7 @@ module Gitlab end def package_url(name) - "https://packagist.org/packages/#{name}" if name =~ %r{\A#{REPO_REGEX}\z} + "https://packagist.org/packages/#{name}" if name =~ /\A#{REPO_REGEX}\z/ end end end diff --git a/lib/gitlab/dependency_linker/gemfile_linker.rb b/lib/gitlab/dependency_linker/gemfile_linker.rb index d034ea67387..bfea836bcb2 100644 --- a/lib/gitlab/dependency_linker/gemfile_linker.rb +++ b/lib/gitlab/dependency_linker/gemfile_linker.rb @@ -15,7 +15,7 @@ module Gitlab link_regex(/(github:|:github\s*=>)\s*['"](?<name>[^'"]+)['"]/, &method(:github_url)) # Link `git: "https://gitlab.example.com/user/repo"` to https://gitlab.example.com/user/repo - link_regex(%r{(git:|:git\s*=>)\s*['"](?<name>#{URL_REGEX})['"]}, &:itself) + link_regex(/(git:|:git\s*=>)\s*['"](?<name>#{URL_REGEX})['"]/, &:itself) # Link `source "https://rubygems.org"` to https://rubygems.org link_method_call('source', URL_REGEX, &:itself) diff --git a/lib/gitlab/dependency_linker/podspec_linker.rb b/lib/gitlab/dependency_linker/podspec_linker.rb index a52c7a02439..924e55e9820 100644 --- a/lib/gitlab/dependency_linker/podspec_linker.rb +++ b/lib/gitlab/dependency_linker/podspec_linker.rb @@ -12,7 +12,7 @@ module Gitlab def link_dependencies link_method_call('homepage', URL_REGEX, &:itself) - link_regex(%r{(git:|:git\s*=>)\s*['"](?<name>#{URL_REGEX})['"]}, &:itself) + link_regex(/(git:|:git\s*=>)\s*['"](?<name>#{URL_REGEX})['"]/, &:itself) link_method_call('license', &method(:license_url)) link_regex(/license\s*=\s*\{\s*(type:|:type\s*=>)\s*#{STRING_REGEX}/, &method(:license_url)) diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index d3b49b1ec75..0fb71976883 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -5,7 +5,7 @@ module Gitlab DEFAULT_CE_PROJECT_URL = 'https://gitlab.com/gitlab-org/gitlab-ce'.freeze EE_REPO_URL = 'https://gitlab.com/gitlab-org/gitlab-ee.git'.freeze CHECK_DIR = Rails.root.join('ee_compat_check') - IGNORED_FILES_REGEX = /(VERSION|CHANGELOG\.md:\d+)/.freeze + IGNORED_FILES_REGEX = %r{VERSION|CHANGELOG\.md|db/schema\.rb}i.freeze PLEASE_READ_THIS_BANNER = %Q{ ============================================================ ===================== PLEASE READ THIS ===================== diff --git a/lib/gitlab/email/reply_parser.rb b/lib/gitlab/email/reply_parser.rb index 558df87f36d..01c28d051ee 100644 --- a/lib/gitlab/email/reply_parser.rb +++ b/lib/gitlab/email/reply_parser.rb @@ -43,7 +43,7 @@ module Gitlab return "" unless decoded # Certain trigger phrases that means we didn't parse correctly - if decoded =~ /(Content\-Type\:|multipart\/alternative|text\/plain)/ + if decoded =~ %r{(Content\-Type\:|multipart/alternative|text/plain)} return "" end diff --git a/lib/gitlab/file_detector.rb b/lib/gitlab/file_detector.rb index 0e9ef4f897c..cc2638172ec 100644 --- a/lib/gitlab/file_detector.rb +++ b/lib/gitlab/file_detector.rb @@ -6,14 +6,14 @@ module Gitlab module FileDetector PATTERNS = { # Project files - readme: /\Areadme[^\/]*\z/i, - changelog: /\A(changelog|history|changes|news)[^\/]*\z/i, - license: /\A(licen[sc]e|copying)(\.[^\/]+)?\z/i, - contributing: /\Acontributing[^\/]*\z/i, + readme: %r{\Areadme[^/]*\z}i, + changelog: %r{\A(changelog|history|changes|news)[^/]*\z}i, + license: %r{\A(licen[sc]e|copying)(\.[^/]+)?\z}i, + contributing: %r{\Acontributing[^/]*\z}i, version: 'version', avatar: /\Alogo\.(png|jpg|gif)\z/, - issue_template: /\A\.gitlab\/issue_templates\/[^\/]+\.md\z/, - merge_request_template: /\A\.gitlab\/merge_request_templates\/[^\/]+\.md\z/, + issue_template: %r{\A\.gitlab/issue_templates/[^/]+\.md\z}, + merge_request_template: %r{\A\.gitlab/merge_request_templates/[^/]+\.md\z}, # Configuration files gitignore: '.gitignore', @@ -22,17 +22,17 @@ module Gitlab route_map: '.gitlab/route-map.yml', # Dependency files - cartfile: /\ACartfile[^\/]*\z/, + cartfile: %r{\ACartfile[^/]*\z}, composer_json: 'composer.json', gemfile: /\A(Gemfile|gems\.rb)\z/, gemfile_lock: 'Gemfile.lock', - gemspec: /\A[^\/]*\.gemspec\z/, + gemspec: %r{\A[^/]*\.gemspec\z}, godeps_json: 'Godeps.json', package_json: 'package.json', podfile: 'Podfile', - podspec_json: /\A[^\/]*\.podspec\.json\z/, - podspec: /\A[^\/]*\.podspec\z/, - requirements_txt: /\A[^\/]*requirements\.txt\z/, + podspec_json: %r{\A[^/]*\.podspec\.json\z}, + podspec: %r{\A[^/]*\.podspec\z}, + requirements_txt: %r{\A[^/]*requirements\.txt\z}, yarn_lock: 'yarn.lock' }.freeze diff --git a/lib/gitlab/gfm/uploads_rewriter.rb b/lib/gitlab/gfm/uploads_rewriter.rb index 8fab5489616..3fdc3c27f73 100644 --- a/lib/gitlab/gfm/uploads_rewriter.rb +++ b/lib/gitlab/gfm/uploads_rewriter.rb @@ -27,7 +27,7 @@ module Gitlab with_link_in_tmp_dir(file.file) do |open_tmp_file| new_uploader.store!(open_tmp_file) end - new_uploader.to_markdown + new_uploader.markdown_link end end diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 71647099f83..d4e893b881c 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -6,12 +6,13 @@ module Gitlab CommandError = Class.new(StandardError) CommitError = Class.new(StandardError) + OSError = Class.new(StandardError) class << self include Gitlab::EncodingHelper def ref_name(ref) - encode!(ref).sub(/\Arefs\/(tags|heads|remotes)\//, '') + encode!(ref).sub(%r{\Arefs/(tags|heads|remotes)/}, '') end def branch_name(ref) diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb index 31effdba292..6d6ed065f79 100644 --- a/lib/gitlab/git/blame.rb +++ b/lib/gitlab/git/blame.rb @@ -42,9 +42,7 @@ module Gitlab end def load_blame_by_shelling_out - cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{@repo.path} blame -p #{@sha} -- #{@path}) - # Read in binary mode to ensure ASCII-8BIT - IO.popen(cmd, 'rb') {|io| io.read } + @repo.shell_blame(@sha, @path) end def process_raw_blame(output) diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 13120120223..4828301dbb9 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -107,7 +107,7 @@ module Gitlab def find_entry_by_path(repository, root_id, path) root_tree = repository.lookup(root_id) # Strip leading slashes - path[/^\/*/] = '' + path[%r{^/*}] = '' path_arr = path.split('/') entry = root_tree.find do |entry| @@ -140,7 +140,7 @@ module Gitlab def find_by_gitaly(repository, sha, path, limit: MAX_DATA_DISPLAY_SIZE) return unless path - path = path.sub(/\A\/*/, '') + path = path.sub(%r{\A/*}, '') path = '/' if path.empty? name = File.basename(path) diff --git a/lib/gitlab/git/branch.rb b/lib/gitlab/git/branch.rb index 3487e099381..ae7e88f0503 100644 --- a/lib/gitlab/git/branch.rb +++ b/lib/gitlab/git/branch.rb @@ -1,5 +1,3 @@ -# Gitaly note: JV: no RPC's here. - module Gitlab module Git class Branch < Ref diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index ca94b4baa59..a203587aec1 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -44,7 +44,7 @@ module Gitlab # branch1...branch2) From the git documentation: # "git diff A...B" is equivalent to "git diff # $(git-merge-base A B) B" - repo.merge_base_commit(head, base) + repo.merge_base(head, base) end options ||= {} diff --git a/lib/gitlab/git/operation_service.rb b/lib/gitlab/git/operation_service.rb index 3fb0e2eed93..280def182d5 100644 --- a/lib/gitlab/git/operation_service.rb +++ b/lib/gitlab/git/operation_service.rb @@ -131,7 +131,10 @@ module Gitlab oldrev = branch.target - if oldrev == repository.merge_base(newrev, branch.target) + merge_base = repository.merge_base(newrev, branch.target) + raise Gitlab::Git::Repository::InvalidRef unless merge_base + + if oldrev == merge_base oldrev else raise Gitlab::Git::CommitError.new('Branch diverged') diff --git a/lib/gitlab/git/path_helper.rb b/lib/gitlab/git/path_helper.rb index 42c80aabd0a..155cf52f050 100644 --- a/lib/gitlab/git/path_helper.rb +++ b/lib/gitlab/git/path_helper.rb @@ -6,7 +6,7 @@ module Gitlab class << self def normalize_path(filename) # Strip all leading slashes so that //foo -> foo - filename[/^\/*/] = '' + filename[%r{^/*}] = '' # Expand relative paths (e.g. foo/../bar) filename = Pathname.new(filename) diff --git a/lib/gitlab/git/popen.rb b/lib/gitlab/git/popen.rb index 1ccca13ce2f..e0bd2bbe47b 100644 --- a/lib/gitlab/git/popen.rb +++ b/lib/gitlab/git/popen.rb @@ -19,6 +19,8 @@ module Gitlab cmd_output = "" cmd_status = 0 Open3.popen3(vars, *cmd, options) do |stdin, stdout, stderr, wait_thr| + stdout.set_encoding(Encoding::ASCII_8BIT) + yield(stdin) if block_given? stdin.close diff --git a/lib/gitlab/git/ref.rb b/lib/gitlab/git/ref.rb index a3ba9475ad0..fa71a4e7ea7 100644 --- a/lib/gitlab/git/ref.rb +++ b/lib/gitlab/git/ref.rb @@ -23,7 +23,7 @@ module Gitlab # Ex. # Ref.extract_branch_name('refs/heads/master') #=> 'master' def self.extract_branch_name(str) - str.gsub(/\Arefs\/heads\//, '') + str.gsub(%r{\Arefs/heads/}, '') end # Gitaly: this method will probably be migrated indirectly via its call sites. diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 638d335b523..35eb4a097e9 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -462,15 +462,18 @@ module Gitlab path: nil, follow: false, skip_merges: false, - disable_walk: false, after: nil, before: nil } options = default_options.merge(options) - options[:limit] ||= 0 options[:offset] ||= 0 + limit = options[:limit] + if limit == 0 || !limit.is_a?(Integer) + raise ArgumentError.new("invalid Repository#log limit: #{limit.inspect}") + end + gitaly_migrate(:find_commits) do |is_enabled| if is_enabled gitaly_commit_client.find_commits(options) @@ -490,11 +493,7 @@ module Gitlab return [] end - if log_using_shell?(options) - log_by_shell(sha, options) - else - log_by_walk(sha, options) - end + log_by_shell(sha, options) end def count_commits(options) @@ -547,29 +546,34 @@ module Gitlab end # Returns the SHA of the most recent common ancestor of +from+ and +to+ - def merge_base_commit(from, to) + def merge_base(from, to) gitaly_migrate(:merge_base) do |is_enabled| if is_enabled gitaly_repository_client.find_merge_base(from, to) else - rugged.merge_base(from, to) + rugged_merge_base(from, to) end end end - alias_method :merge_base, :merge_base_commit # Gitaly note: JV: check gitlab-ee before removing this method. def rugged_is_ancestor?(ancestor_id, descendant_id) return false if ancestor_id.nil? || descendant_id.nil? - merge_base_commit(ancestor_id, descendant_id) == ancestor_id + rugged_merge_base(ancestor_id, descendant_id) == ancestor_id rescue Rugged::OdbError false end # Returns true is +from+ is direct ancestor to +to+, otherwise false def ancestor?(from, to) - gitaly_commit_client.ancestor?(from, to) + Gitlab::GitalyClient.migrate(:is_ancestor) do |is_enabled| + if is_enabled + gitaly_commit_client.ancestor?(from, to) + else + rugged_is_ancestor?(from, to) + end + end end def merged_branch_names(branch_names = []) @@ -614,11 +618,11 @@ module Gitlab if is_enabled gitaly_ref_client.find_ref_name(sha, ref_path) else - args = %W(#{Gitlab.config.git.bin_path} for-each-ref --count=1 #{ref_path} --contains #{sha}) + args = %W(for-each-ref --count=1 #{ref_path} --contains #{sha}) # Not found -> ["", 0] # Found -> ["b8d95eb4969eefacb0a58f6a28f6803f8070e7b9 commit\trefs/environments/production/77\n", 0] - popen(args, @path).first.split.last + run_git(args).first.split.last end end end @@ -676,11 +680,7 @@ module Gitlab if is_enabled gitaly_commit_client.commit_count(ref) else - walker = Rugged::Walker.new(rugged) - walker.sorting(Rugged::SORT_TOPO | Rugged::SORT_REVERSE) - oid = rugged.rev_parse_oid(ref) - walker.push(oid) - walker.count + rugged_commit_count(ref) end end end @@ -883,17 +883,12 @@ module Gitlab end def delete_refs(*ref_names) - instructions = ref_names.map do |ref| - "delete #{ref}\x00\x00" - end - - command = %W[#{Gitlab.config.git.bin_path} update-ref --stdin -z] - message, status = popen(command, path) do |stdin| - stdin.write(instructions.join) - end - - unless status.zero? - raise GitError.new("Could not delete refs #{ref_names}: #{message}") + gitaly_migrate(:delete_refs) do |is_enabled| + if is_enabled + gitaly_delete_refs(*ref_names) + else + git_delete_refs(*ref_names) + end end end @@ -1102,10 +1097,14 @@ module Gitlab end def write_ref(ref_path, ref, old_ref: nil, shell: true) - if shell - shell_write_ref(ref_path, ref, old_ref) - else - rugged_write_ref(ref_path, ref) + ref_path = "#{Gitlab::Git::BRANCH_REF_PREFIX}#{ref_path}" unless ref_path.start_with?("refs/") || ref_path == "HEAD" + + gitaly_migrate(:write_ref) do |is_enabled| + if is_enabled + gitaly_repository_client.write_ref(ref_path, ref, old_ref, shell) + else + local_write_ref(ref_path, ref, old_ref: old_ref, shell: shell) + end end end @@ -1128,13 +1127,6 @@ module Gitlab end # Refactoring aid; allows us to copy code from app/models/repository.rb - def run_git_with_timeout(args, timeout, env: {}) - circuit_breaker.perform do - popen_with_timeout([Gitlab.config.git.bin_path, *args], timeout, path, env) - end - end - - # Refactoring aid; allows us to copy code from app/models/repository.rb def commit(ref = 'HEAD') Gitlab::Git::Commit.find(self, ref) end @@ -1314,7 +1306,15 @@ module Gitlab # rubocop:enable Metrics/ParameterLists def write_config(full_path:) - rugged.config['gitlab.fullpath'] = full_path if full_path.present? + return unless full_path.present? + + gitaly_migrate(:write_config) do |is_enabled| + if is_enabled + gitaly_repository_client.write_config(full_path: full_path) + else + rugged_write_config(full_path: full_path) + end + end end def gitaly_repository @@ -1363,20 +1363,23 @@ module Gitlab raise CommandError.new(e) end - def refs_contains_sha(ref_type, sha) - args = %W(#{ref_type} --contains #{sha}) - names = run_git(args).first - - if names.respond_to?(:split) - names = names.split("\n").map(&:strip) - - names.each do |name| - name.slice! '* ' + def branch_names_contains_sha(sha) + gitaly_migrate(:branch_names_contains_sha) do |is_enabled| + if is_enabled + gitaly_ref_client.branch_names_contains_sha(sha) + else + refs_contains_sha(:branch, sha) end + end + end - names - else - [] + def tag_names_contains_sha(sha) + gitaly_migrate(:tag_names_contains_sha) do |is_enabled| + if is_enabled + gitaly_ref_client.tag_names_contains_sha(sha) + else + refs_contains_sha(:tag, sha) + end end end @@ -1389,8 +1392,18 @@ module Gitlab run_git(args).first.scrub.split(/^--$/) end + def can_be_merged?(source_sha, target_branch) + gitaly_migrate(:can_be_merged) do |is_enabled| + if is_enabled + gitaly_can_be_merged?(source_sha, find_branch(target_branch, true).target) + else + rugged_can_be_merged?(source_sha, target_branch) + end + end + end + def search_files_by_name(query, ref) - safe_query = Regexp.escape(query.sub(/^\/*/, "")) + safe_query = Regexp.escape(query.sub(%r{^/*}, "")) return [] if empty? || safe_query.blank? @@ -1409,8 +1422,60 @@ module Gitlab end end + def shell_blame(sha, path) + output, _status = run_git(%W(blame -p #{sha} -- #{path})) + output + end + + def can_be_merged?(source_sha, target_branch) + gitaly_migrate(:can_be_merged) do |is_enabled| + if is_enabled + gitaly_can_be_merged?(source_sha, find_branch(target_branch).target) + else + rugged_can_be_merged?(source_sha, target_branch) + end + end + end + + def last_commit_id_for_path(sha, path) + gitaly_migrate(:last_commit_for_path) do |is_enabled| + if is_enabled + last_commit_for_path_by_gitaly(sha, path).id + else + last_commit_id_for_path_by_shelling_out(sha, path) + end + end + end + private + def local_write_ref(ref_path, ref, old_ref: nil, shell: true) + if shell + shell_write_ref(ref_path, ref, old_ref) + else + rugged_write_ref(ref_path, ref) + end + end + + def refs_contains_sha(ref_type, sha) + args = %W(#{ref_type} --contains #{sha}) + names = run_git(args).first + + return [] unless names.respond_to?(:split) + + names = names.split("\n").map(&:strip) + + names.each do |name| + name.slice! '* ' + end + + names + end + + def rugged_write_config(full_path:) + rugged.config['gitlab.fullpath'] = full_path + end + def shell_write_ref(ref_path, ref, old_ref) raise ArgumentError, "invalid ref_path #{ref_path.inspect}" if ref_path.include?(' ') raise ArgumentError, "invalid ref #{ref.inspect}" if ref.include?("\x00") @@ -1433,6 +1498,12 @@ module Gitlab def run_git(args, chdir: path, env: {}, nice: false, &block) cmd = [Gitlab.config.git.bin_path, *args] cmd.unshift("nice") if nice + + object_directories = alternate_object_directories + if object_directories.any? + env['GIT_ALTERNATE_OBJECT_DIRECTORIES'] = object_directories.join(File::PATH_SEPARATOR) + end + circuit_breaker.perform do popen(cmd, chdir, env, &block) end @@ -1446,6 +1517,12 @@ module Gitlab output end + def run_git_with_timeout(args, timeout, env: {}) + circuit_breaker.perform do + popen_with_timeout([Gitlab.config.git.bin_path, *args], timeout, path, env) + end + end + def fresh_worktree?(path) File.exist?(path) && !clean_stuck_worktree(path) end @@ -1460,7 +1537,7 @@ module Gitlab if sparse_checkout_files # Create worktree without checking out run_git!(base_args + ['--no-checkout', worktree_path], env: env) - worktree_git_path = run_git!(%w(rev-parse --git-dir), chdir: worktree_path) + worktree_git_path = run_git!(%w(rev-parse --git-dir), chdir: worktree_path).chomp configure_sparse_checkout(worktree_git_path, sparse_checkout_files) @@ -1593,24 +1670,6 @@ module Gitlab end end - def log_using_shell?(options) - options[:path].present? || - options[:disable_walk] || - options[:skip_merges] || - options[:after] || - options[:before] - end - - def log_by_walk(sha, options) - walk_options = { - show: sha, - sort: Rugged::SORT_NONE, - limit: options[:limit], - offset: options[:offset] - } - Rugged::Walker.walk(rugged, walk_options).to_a - end - # Gitaly note: JV: although #log_by_shell shells out to Git I think the # complexity is such that we should migrate it as Ruby before trying to # do it in Go. @@ -1624,7 +1683,7 @@ module Gitlab offset_in_ruby = use_follow_flag && options[:offset].present? limit += offset if offset_in_ruby - cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} log] + cmd = %w[log] cmd << "--max-count=#{limit}" cmd << '--format=%H' cmd << "--skip=#{offset}" unless offset_in_ruby @@ -1640,7 +1699,7 @@ module Gitlab cmd += Array(options[:path]) end - raw_output = IO.popen(cmd) { |io| io.read } + raw_output, _status = run_git(cmd) lines = offset_in_ruby ? raw_output.lines.drop(offset) : raw_output.lines lines.map! { |c| Rugged::Commit.new(rugged, c.strip) } @@ -1678,18 +1737,23 @@ module Gitlab end def alternate_object_directories - relative_paths = Gitlab::Git::Env.all.values_at(*ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES).flatten.compact + relative_paths = relative_object_directories if relative_paths.any? relative_paths.map { |d| File.join(path, d) } else - Gitlab::Git::Env.all.values_at(*ALLOWED_OBJECT_DIRECTORIES_VARIABLES) - .flatten - .compact - .flat_map { |d| d.split(File::PATH_SEPARATOR) } + absolute_object_directories.flat_map { |d| d.split(File::PATH_SEPARATOR) } end end + def relative_object_directories + Gitlab::Git::Env.all.values_at(*ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES).flatten.compact + end + + def absolute_object_directories + Gitlab::Git::Env.all.values_at(*ALLOWED_OBJECT_DIRECTORIES_VARIABLES).flatten.compact + end + # Get the content of a blob for a given commit. If the blob is a commit # (for submodules) then return the blob's OID. def blob_content(commit, blob_name) @@ -1833,13 +1897,13 @@ module Gitlab def count_commits_by_shelling_out(options) cmd = count_commits_shelling_command(options) - raw_output = IO.popen(cmd) { |io| io.read } + raw_output, _status = run_git(cmd) process_count_commits_raw_output(raw_output, options) end def count_commits_shelling_command(options) - cmd = %W[#{Gitlab.config.git.bin_path} --git-dir=#{path} rev-list] + cmd = %w[rev-list] cmd << "--after=#{options[:after].iso8601}" if options[:after] cmd << "--before=#{options[:before].iso8601}" if options[:before] cmd << "--max-count=#{options[:max_count]}" if options[:max_count] @@ -1884,20 +1948,17 @@ module Gitlab return [] end - cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path} ls-tree) - cmd += %w(-r) - cmd += %w(--full-tree) - cmd += %w(--full-name) - cmd += %W(-- #{actual_ref}) + cmd = %W(ls-tree -r --full-tree --full-name -- #{actual_ref}) + raw_output, _status = run_git(cmd) - raw_output = IO.popen(cmd, &:read).split("\n").map do |f| + lines = raw_output.split("\n").map do |f| stuff, path = f.split("\t") _mode, type, _sha = stuff.split(" ") path if type == "blob" # Contain only blob type end - raw_output.compact + lines.compact end # Returns true if the given ref name exists @@ -1971,7 +2032,7 @@ module Gitlab target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) rescue Rugged::ReferenceError => e - raise InvalidRef.new("Branch #{ref} already exists") if e.to_s =~ /'refs\/heads\/#{ref}'/ + raise InvalidRef.new("Branch #{ref} already exists") if e.to_s =~ %r{'refs/heads/#{ref}'} raise InvalidRef.new("Invalid reference #{start_point}") end @@ -2144,7 +2205,7 @@ module Gitlab source_sha end - rescue Rugged::ReferenceError + rescue Rugged::ReferenceError, InvalidRef raise ArgumentError, 'Invalid merge source' end @@ -2156,6 +2217,24 @@ module Gitlab remote_update(remote_name, url: url) end + def git_delete_refs(*ref_names) + instructions = ref_names.map do |ref| + "delete #{ref}\x00\x00" + end + + message, status = run_git(%w[update-ref --stdin -z]) do |stdin| + stdin.write(instructions.join) + end + + unless status.zero? + raise GitError.new("Could not delete refs #{ref_names}: #{message}") + end + end + + def gitaly_delete_refs(*ref_names) + gitaly_ref_client.delete_refs(refs: ref_names) + end + def rugged_remove_remote(remote_name) # When a remote is deleted all its remote refs are deleted too, but in # the case of mirrors we map its refs (that would usualy go under @@ -2218,6 +2297,14 @@ module Gitlab run_git(['fetch', remote_name], env: env).last.zero? end + def gitaly_can_be_merged?(their_commit, our_commit) + !gitaly_conflicts_client(our_commit, their_commit).conflicts? + end + + def rugged_can_be_merged?(their_commit, our_commit) + !rugged.merge_commits(our_commit, their_commit).conflicts? + end + def gitlab_projects_error raise CommandError, @gitlab_projects.output end @@ -2241,6 +2328,39 @@ module Gitlab .commits_by_message(query, revision: ref, path: path, limit: limit, offset: offset) .map { |c| commit(c) } end + + def gitaly_can_be_merged?(their_commit, our_commit) + !gitaly_conflicts_client(our_commit, their_commit).conflicts? + end + + def rugged_can_be_merged?(their_commit, our_commit) + !rugged.merge_commits(our_commit, their_commit).conflicts? + end + + def last_commit_for_path_by_gitaly(sha, path) + gitaly_commit_client.last_commit_for_path(sha, path) + end + + def last_commit_id_for_path_by_shelling_out(sha, path) + args = %W(rev-list --max-count=1 #{sha} -- #{path}) + run_git_with_timeout(args, Gitlab::Git::Popen::FAST_GIT_PROCESS_TIMEOUT).first.strip + end + + def rugged_merge_base(from, to) + rugged.merge_base(from, to) + rescue Rugged::ReferenceError + nil + end + + def rugged_commit_count(ref) + walker = Rugged::Walker.new(rugged) + walker.sorting(Rugged::SORT_TOPO | Rugged::SORT_REVERSE) + oid = rugged.rev_parse_oid(ref) + walker.push(oid) + walker.count + rescue Rugged::ReferenceError + 0 + end end end end diff --git a/lib/gitlab/git/repository_mirroring.rb b/lib/gitlab/git/repository_mirroring.rb index effb1f0ca19..dc424a433fb 100644 --- a/lib/gitlab/git/repository_mirroring.rb +++ b/lib/gitlab/git/repository_mirroring.rb @@ -43,7 +43,7 @@ module Gitlab branches = [] rugged.references.each("refs/remotes/#{remote_name}/*").map do |ref| - name = ref.name.sub(/\Arefs\/remotes\/#{remote_name}\//, '') + name = ref.name.sub(%r{\Arefs/remotes/#{remote_name}/}, '') begin target_commit = Gitlab::Git::Commit.find(self, ref.target) diff --git a/lib/gitlab/git/tag.rb b/lib/gitlab/git/tag.rb index bc4e160dce9..8a8f7b051ed 100644 --- a/lib/gitlab/git/tag.rb +++ b/lib/gitlab/git/tag.rb @@ -1,5 +1,3 @@ -# Gitaly note: JV: no RPC's here. -# module Gitlab module Git class Tag < Ref diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index 5cf336af3c6..ba6058fd3c9 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -83,6 +83,8 @@ module Gitlab commit_id: sha ) end + rescue Rugged::ReferenceError + [] end end diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index d4a53d32c28..ccdb8975342 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -117,6 +117,20 @@ module Gitlab page.url_path end + def page_formatted_data(title:, dir: nil, version: nil) + version = version&.id + + @repository.gitaly_migrate(:wiki_page_formatted_data) do |is_enabled| + if is_enabled + gitaly_wiki_client.get_formatted_data(title: title, dir: dir, version: version) + else + # We don't use #page because if wiki_find_page feature is enabled, we would + # get a page without formatted_data. + gollum_find_page(title: title, dir: dir, version: version)&.formatted_data + end + end + end + private # options: diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 6bd256f57c7..c5d3e944f7d 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -6,6 +6,7 @@ require 'grpc/health/v1/health_services_pb' module Gitlab module GitalyClient + include Gitlab::Metrics::Methods module MigrationStatus DISABLED = 1 OPT_IN = 2 @@ -33,8 +34,6 @@ module Gitlab CLIENT_NAME = (Sidekiq.server? ? 'gitlab-sidekiq' : 'gitlab-web').freeze MUTEX = Mutex.new - METRICS_MUTEX = Mutex.new - private_constant :MUTEX, :METRICS_MUTEX class << self attr_accessor :query_time @@ -42,28 +41,14 @@ module Gitlab self.query_time = 0 - def self.migrate_histogram - @migrate_histogram ||= - METRICS_MUTEX.synchronize do - # If a thread was blocked on the mutex, the value was set already - return @migrate_histogram if @migrate_histogram - - Gitlab::Metrics.histogram(:gitaly_migrate_call_duration_seconds, - "Gitaly migration call execution timings", - gitaly_enabled: nil, feature: nil) - end + define_histogram :gitaly_migrate_call_duration_seconds do + docstring "Gitaly migration call execution timings" + base_labels gitaly_enabled: nil, feature: nil end - def self.gitaly_call_histogram - @gitaly_call_histogram ||= - METRICS_MUTEX.synchronize do - # If a thread was blocked on the mutex, the value was set already - return @gitaly_call_histogram if @gitaly_call_histogram - - Gitlab::Metrics.histogram(:gitaly_controller_action_duration_seconds, - "Gitaly endpoint histogram by controller and action combination", - Gitlab::Metrics::Transaction::BASE_LABELS.merge(gitaly_service: nil, rpc: nil)) - end + define_histogram :gitaly_controller_action_duration_seconds do + docstring "Gitaly endpoint histogram by controller and action combination" + base_labels Gitlab::Metrics::Transaction::BASE_LABELS.merge(gitaly_service: nil, rpc: nil) end def self.stub(name, storage) @@ -145,7 +130,7 @@ module Gitlab # Keep track, seperately, for the performance bar self.query_time += duration - gitaly_call_histogram.observe( + gitaly_controller_action_duration_seconds.observe( current_transaction_labels.merge(gitaly_service: service.to_s, rpc: rpc.to_s), duration) end @@ -247,7 +232,7 @@ module Gitlab yield is_enabled ensure total_time = Gitlab::Metrics::System.monotonic_time - start - migrate_histogram.observe({ gitaly_enabled: is_enabled, feature: feature }, total_time) + gitaly_migrate_call_duration_seconds.observe({ gitaly_enabled: is_enabled, feature: feature }, total_time) feature_stack.shift Thread.current[:gitaly_feature_stack] = nil if feature_stack.empty? end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index cadc7149301..5767f06b0ce 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -257,7 +257,7 @@ module Gitlab offset: options[:offset], follow: options[:follow], skip_merges: options[:skip_merges], - disable_walk: options[:disable_walk] + disable_walk: true # This option is deprecated. The 'walk' implementation is being removed. ) request.after = GitalyClient.timestamp(options[:after]) if options[:after] request.before = GitalyClient.timestamp(options[:before]) if options[:before] diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index c2b4155e6a5..cd2734b5a07 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -103,7 +103,13 @@ module Gitlab request_enum.push(Gitaly::UserMergeBranchRequest.new(apply: true)) - branch_update = response_enum.next.branch_update + second_response = response_enum.next + + if second_response.pre_receive_error.present? + raise Gitlab::Git::HooksService::PreReceiveError, second_response.pre_receive_error + end + + branch_update = second_response.branch_update return if branch_update.nil? raise Gitlab::Git::CommitError.new('failed to apply merge to branch') unless branch_update.commit_id.present? diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index f8e2a27f3fe..ba6b577fd17 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -133,13 +133,42 @@ module Gitlab GitalyClient.call(@repository.storage, :ref_service, :delete_branch, request) end - def delete_refs(except_with_prefixes:) + def delete_refs(refs: [], except_with_prefixes: []) request = Gitaly::DeleteRefsRequest.new( repository: @gitaly_repo, - except_with_prefix: except_with_prefixes + refs: refs.map { |r| encode_binary(r) }, + except_with_prefix: except_with_prefixes.map { |r| encode_binary(r) } ) - GitalyClient.call(@repository.storage, :ref_service, :delete_refs, request) + response = GitalyClient.call(@repository.storage, :ref_service, :delete_refs, request) + + raise Gitlab::Git::Repository::GitError, response.git_error if response.git_error.present? + end + + # Limit: 0 implies no limit, thus all tag names will be returned + def tag_names_contains_sha(sha, limit: 0) + request = Gitaly::ListTagNamesContainingCommitRequest.new( + repository: @gitaly_repo, + commit_id: sha, + limit: limit + ) + + stream = GitalyClient.call(@repository.storage, :ref_service, :list_tag_names_containing_commit, request) + + consume_ref_contains_sha_response(stream, :tag_names) + end + + # Limit: 0 implies no limit, thus all tag names will be returned + def branch_names_contains_sha(sha, limit: 0) + request = Gitaly::ListBranchNamesContainingCommitRequest.new( + repository: @gitaly_repo, + commit_id: sha, + limit: limit + ) + + stream = GitalyClient.call(@repository.storage, :ref_service, :list_branch_names_containing_commit, request) + + consume_ref_contains_sha_response(stream, :branch_names) end private @@ -212,6 +241,13 @@ module Gitlab Gitlab::Git::Commit.decorate(@repository, hash) end + def consume_ref_contains_sha_response(stream, collection_name) + stream.each_with_object([]) do |response, array| + encoded_names = response.send(collection_name).map { |b| Gitlab::Git.ref_name(b) } # rubocop:disable GitlabSecurity/PublicSend + array.concat(encoded_names) + end + end + def invalid_ref!(message) raise Gitlab::Git::Repository::InvalidRef.new(message) end diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index b0dbaf11598..60706b4f0d8 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -203,6 +203,35 @@ module Gitlab timeout: GitalyClient.default_timeout ) end + + def write_ref(ref_path, ref, old_ref, shell) + request = Gitaly::WriteRefRequest.new( + repository: @gitaly_repo, + ref: ref_path.b, + revision: ref.b, + shell: shell + ) + request.old_revision = old_ref.b unless old_ref.nil? + + response = GitalyClient.call(@storage, :repository_service, :write_ref, request) + + raise Gitlab::Git::CommandError, encode!(response.error) if response.error.present? + + true + end + + def write_config(full_path:) + request = Gitaly::WriteConfigRequest.new(repository: @gitaly_repo, full_path: full_path) + response = GitalyClient.call( + @storage, + :repository_service, + :write_config, + request, + timeout: GitalyClient.fast_timeout + ) + + raise Gitlab::Git::OSError.new(response.error) unless response.error.empty? + end end end end diff --git a/lib/gitlab/gitaly_client/server_service.rb b/lib/gitlab/gitaly_client/server_service.rb new file mode 100644 index 00000000000..2e1076d1f66 --- /dev/null +++ b/lib/gitlab/gitaly_client/server_service.rb @@ -0,0 +1,16 @@ +module Gitlab + module GitalyClient + # Meant for extraction of server data, and later maybe to perform misc task + # + # Not meant for connection logic, look in Gitlab::GitalyClient + class ServerService + def initialize(storage) + @storage = storage + end + + def info + GitalyClient.call(@storage, :server_service, :server_info, Gitaly::ServerInfoRequest.new) + end + end + end +end diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb index 5c5b170a3e0..8e87a8cc36f 100644 --- a/lib/gitlab/gitaly_client/wiki_service.rb +++ b/lib/gitlab/gitaly_client/wiki_service.rb @@ -127,6 +127,18 @@ module Gitlab wiki_file end + def get_formatted_data(title:, dir: nil, version: nil) + request = Gitaly::WikiGetFormattedDataRequest.new( + repository: @gitaly_repo, + title: encode_binary(title), + revision: encode_binary(version), + directory: encode_binary(dir) + ) + + response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_get_formatted_data, request) + response.reduce("") { |memo, msg| memo << msg.data } + end + private # If a block is given and the yielded value is true, iteration will be diff --git a/lib/gitlab/github_import/representation/diff_note.rb b/lib/gitlab/github_import/representation/diff_note.rb index bb7439a0641..be1334ca98a 100644 --- a/lib/gitlab/github_import/representation/diff_note.rb +++ b/lib/gitlab/github_import/representation/diff_note.rb @@ -13,7 +13,7 @@ module Gitlab :diff_hunk, :author, :note, :created_at, :updated_at, :github_id - NOTEABLE_ID_REGEX = /\/pull\/(?<iid>\d+)/i + NOTEABLE_ID_REGEX = %r{/pull/(?<iid>\d+)}i # Builds a diff note from a GitHub API response. # diff --git a/lib/gitlab/github_import/representation/note.rb b/lib/gitlab/github_import/representation/note.rb index a68bc4c002f..070e3b2db8d 100644 --- a/lib/gitlab/github_import/representation/note.rb +++ b/lib/gitlab/github_import/representation/note.rb @@ -12,7 +12,7 @@ module Gitlab expose_attribute :noteable_id, :noteable_type, :author, :note, :created_at, :updated_at, :github_id - NOTEABLE_TYPE_REGEX = /\/(?<type>(pull|issues))\/(?<iid>\d+)/i + NOTEABLE_TYPE_REGEX = %r{/(?<type>(pull|issues))/(?<iid>\d+)}i # Builds a note from a GitHub API response. # diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 9148d7571f2..86a90d57d9c 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -3,12 +3,11 @@ module Gitlab module GonHelper include WebpackHelper - include Gitlab::CurrentSettings def add_gon_variables gon.api_version = 'v4' gon.default_avatar_url = URI.join(Gitlab.config.gitlab.url, ActionController::Base.helpers.image_path('no_avatar.png')).to_s - gon.max_file_size = current_application_settings.max_attachment_size + gon.max_file_size = Gitlab::CurrentSettings.max_attachment_size gon.asset_host = ActionController::Base.asset_host gon.webpack_public_path = webpack_public_path gon.relative_url_root = Gitlab.config.gitlab.relative_url_root @@ -16,7 +15,7 @@ module Gitlab gon.user_color_scheme = Gitlab::ColorSchemes.for_user(current_user).css_class gon.katex_css_url = ActionController::Base.helpers.asset_path('katex.css') gon.katex_js_url = ActionController::Base.helpers.asset_path('katex.js') - gon.sentry_dsn = current_application_settings.clientside_sentry_dsn if current_application_settings.clientside_sentry_enabled + gon.sentry_dsn = Gitlab::CurrentSettings.clientside_sentry_dsn if Gitlab::CurrentSettings.clientside_sentry_enabled gon.gitlab_url = Gitlab.config.gitlab.url gon.revision = Gitlab::REVISION gon.gitlab_logo = ActionController::Base.helpers.asset_path('gitlab_logo.png') diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index 5c971564a73..0f4c3498036 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -59,7 +59,7 @@ module Gitlab end def extracted_files - Dir.glob("#{@shared.export_path}/**/*", File::FNM_DOTMATCH).reject { |f| f =~ /.*\/\.{1,2}$/ } + Dir.glob("#{@shared.export_path}/**/*", File::FNM_DOTMATCH).reject { |f| f =~ %r{.*/\.{1,2}$} } end end end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index cb711a83433..759833a5ee5 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -139,13 +139,12 @@ module Gitlab end def setup_label - return unless @relation_hash['type'] == 'GroupLabel' - # If there's no group, move the label to a project label - if @relation_hash['group_id'] + if @relation_hash['type'] == 'GroupLabel' && @relation_hash['group_id'] @relation_hash['project_id'] = nil @relation_name = :group_label else + @relation_hash['group_id'] = nil @relation_hash['type'] = 'ProjectLabel' end end diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb index 627a487d577..2f08dda55fd 100644 --- a/lib/gitlab/import_export/uploads_saver.rb +++ b/lib/gitlab/import_export/uploads_saver.rb @@ -17,15 +17,13 @@ module Gitlab false end - private + def uploads_path + FileUploader.absolute_base_dir(@project) + end def uploads_export_path File.join(@shared.export_path, 'uploads') end - - def uploads_path - FileUploader.dynamic_path_segment(@project) - end end end end diff --git a/lib/gitlab/legacy_github_import/project_creator.rb b/lib/gitlab/legacy_github_import/project_creator.rb index 41e7eac4d08..cbabe5454ca 100644 --- a/lib/gitlab/legacy_github_import/project_creator.rb +++ b/lib/gitlab/legacy_github_import/project_creator.rb @@ -1,8 +1,6 @@ module Gitlab module LegacyGithubImport class ProjectCreator - include Gitlab::CurrentSettings - attr_reader :repo, :name, :namespace, :current_user, :session_data, :type def initialize(repo, name, namespace, current_user, session_data, type: 'github') @@ -36,7 +34,7 @@ module Gitlab end def visibility_level - repo.private ? Gitlab::VisibilityLevel::PRIVATE : current_application_settings.default_project_visibility + repo.private ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::CurrentSettings.default_project_visibility end # diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index 4779755bb22..7d63ca5627d 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -1,7 +1,7 @@ module Gitlab module Metrics - extend Gitlab::Metrics::InfluxDb - extend Gitlab::Metrics::Prometheus + include Gitlab::Metrics::InfluxDb + include Gitlab::Metrics::Prometheus def self.enabled? influx_metrics_enabled? || prometheus_metrics_enabled? diff --git a/lib/gitlab/metrics/influx_db.rb b/lib/gitlab/metrics/influx_db.rb index ef44a13df51..66f30e3b397 100644 --- a/lib/gitlab/metrics/influx_db.rb +++ b/lib/gitlab/metrics/influx_db.rb @@ -1,179 +1,187 @@ module Gitlab module Metrics module InfluxDb - include Gitlab::CurrentSettings - extend self + extend ActiveSupport::Concern + include Gitlab::Metrics::Methods + + EXECUTION_MEASUREMENT_BUCKETS = [0.001, 0.01, 0.1, 1].freeze MUTEX = Mutex.new private_constant :MUTEX - def influx_metrics_enabled? - settings[:enabled] || false - end + class_methods do + def influx_metrics_enabled? + settings[:enabled] || false + end - # Prometheus histogram buckets used for arbitrary code measurements - EXECUTION_MEASUREMENT_BUCKETS = [0.001, 0.002, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1].freeze - RAILS_ROOT = Rails.root.to_s - METRICS_ROOT = Rails.root.join('lib', 'gitlab', 'metrics').to_s - PATH_REGEX = /^#{RAILS_ROOT}\/?/ - - def settings - @settings ||= { - enabled: current_application_settings[:metrics_enabled], - pool_size: current_application_settings[:metrics_pool_size], - timeout: current_application_settings[:metrics_timeout], - method_call_threshold: current_application_settings[:metrics_method_call_threshold], - host: current_application_settings[:metrics_host], - port: current_application_settings[:metrics_port], - sample_interval: current_application_settings[:metrics_sample_interval] || 15, - packet_size: current_application_settings[:metrics_packet_size] || 1 - } - end + # Prometheus histogram buckets used for arbitrary code measurements + + def settings + @settings ||= begin + current_settings = Gitlab::CurrentSettings.current_application_settings + + { + enabled: current_settings[:metrics_enabled], + pool_size: current_settings[:metrics_pool_size], + timeout: current_settings[:metrics_timeout], + method_call_threshold: current_settings[:metrics_method_call_threshold], + host: current_settings[:metrics_host], + port: current_settings[:metrics_port], + sample_interval: current_settings[:metrics_sample_interval] || 15, + packet_size: current_settings[:metrics_packet_size] || 1 + } + end + end - def mri? - RUBY_ENGINE == 'ruby' - end + def mri? + RUBY_ENGINE == 'ruby' + end - def method_call_threshold - # This is memoized since this method is called for every instrumented - # method. Loading data from an external cache on every method call slows - # things down too much. - # in milliseconds - @method_call_threshold ||= settings[:method_call_threshold] - end + def method_call_threshold + # This is memoized since this method is called for every instrumented + # method. Loading data from an external cache on every method call slows + # things down too much. + # in milliseconds + @method_call_threshold ||= settings[:method_call_threshold] + end - def submit_metrics(metrics) - prepared = prepare_metrics(metrics) + def submit_metrics(metrics) + prepared = prepare_metrics(metrics) - pool&.with do |connection| - prepared.each_slice(settings[:packet_size]) do |slice| - begin - connection.write_points(slice) - rescue StandardError + pool&.with do |connection| + prepared.each_slice(settings[:packet_size]) do |slice| + begin + connection.write_points(slice) + rescue StandardError + end end end + rescue Errno::EADDRNOTAVAIL, SocketError => ex + Gitlab::EnvironmentLogger.error('Cannot resolve InfluxDB address. GitLab Performance Monitoring will not work.') + Gitlab::EnvironmentLogger.error(ex) end - rescue Errno::EADDRNOTAVAIL, SocketError => ex - Gitlab::EnvironmentLogger.error('Cannot resolve InfluxDB address. GitLab Performance Monitoring will not work.') - Gitlab::EnvironmentLogger.error(ex) - end - def prepare_metrics(metrics) - metrics.map do |hash| - new_hash = hash.symbolize_keys + def prepare_metrics(metrics) + metrics.map do |hash| + new_hash = hash.symbolize_keys - new_hash[:tags].each do |key, value| - if value.blank? - new_hash[:tags].delete(key) - else - new_hash[:tags][key] = escape_value(value) + new_hash[:tags].each do |key, value| + if value.blank? + new_hash[:tags].delete(key) + else + new_hash[:tags][key] = escape_value(value) + end end + + new_hash end + end - new_hash + def escape_value(value) + value.to_s.gsub('=', '\\=') end - end - def escape_value(value) - value.to_s.gsub('=', '\\=') - end + # Measures the execution time of a block. + # + # Example: + # + # Gitlab::Metrics.measure(:find_by_username_duration) do + # User.find_by_username(some_username) + # end + # + # name - The name of the field to store the execution time in. + # + # Returns the value yielded by the supplied block. + def measure(name) + trans = current_transaction + + return yield unless trans + + real_start = Time.now.to_f + cpu_start = System.cpu_time + + retval = yield + + cpu_stop = System.cpu_time + real_stop = Time.now.to_f + + real_time = (real_stop - real_start) + cpu_time = cpu_stop - cpu_start + + real_duration_seconds = fetch_histogram("gitlab_#{name}_real_duration_seconds".to_sym) do + docstring "Measure #{name}" + base_labels Transaction::BASE_LABELS + buckets EXECUTION_MEASUREMENT_BUCKETS + end - # Measures the execution time of a block. - # - # Example: - # - # Gitlab::Metrics.measure(:find_by_username_duration) do - # User.find_by_username(some_username) - # end - # - # name - The name of the field to store the execution time in. - # - # Returns the value yielded by the supplied block. - def measure(name) - trans = current_transaction - - return yield unless trans - - real_start = Time.now.to_f - cpu_start = System.cpu_time - - retval = yield - - cpu_stop = System.cpu_time - real_stop = Time.now.to_f - - real_time = (real_stop - real_start) - cpu_time = cpu_stop - cpu_start - - Gitlab::Metrics.histogram("gitlab_#{name}_real_duration_seconds".to_sym, - "Measure #{name}", - Transaction::BASE_LABELS, - EXECUTION_MEASUREMENT_BUCKETS) - .observe(trans.labels, real_time) - - Gitlab::Metrics.histogram("gitlab_#{name}_cpu_duration_seconds".to_sym, - "Measure #{name}", - Transaction::BASE_LABELS, - EXECUTION_MEASUREMENT_BUCKETS) - .observe(trans.labels, cpu_time / 1000.0) - - # InfluxDB stores the _real_time time values as milliseconds - trans.increment("#{name}_real_time", real_time * 1000, false) - trans.increment("#{name}_cpu_time", cpu_time, false) - trans.increment("#{name}_call_count", 1, false) - - retval - end + real_duration_seconds.observe(trans.labels, real_time) - # Sets the action of the current transaction (if any) - # - # action - The name of the action. - def action=(action) - trans = current_transaction + cpu_duration_seconds = fetch_histogram("gitlab_#{name}_cpu_duration_seconds".to_sym) do + docstring "Measure #{name}" + base_labels Transaction::BASE_LABELS + buckets EXECUTION_MEASUREMENT_BUCKETS + with_feature "prometheus_metrics_measure_#{name}_cpu_duration" + end + cpu_duration_seconds.observe(trans.labels, cpu_time) - trans&.action = action - end + # InfluxDB stores the _real_time and _cpu_time time values as milliseconds + trans.increment("#{name}_real_time", real_time.in_milliseconds, false) + trans.increment("#{name}_cpu_time", cpu_time.in_milliseconds, false) + trans.increment("#{name}_call_count", 1, false) - # Tracks an event. - # - # See `Gitlab::Metrics::Transaction#add_event` for more details. - def add_event(*args) - trans = current_transaction + retval + end - trans&.add_event(*args) - end + # Sets the action of the current transaction (if any) + # + # action - The name of the action. + def action=(action) + trans = current_transaction - # Returns the prefix to use for the name of a series. - def series_prefix - @series_prefix ||= Sidekiq.server? ? 'sidekiq_' : 'rails_' - end + trans&.action = action + end - # Allow access from other metrics related middlewares - def current_transaction - Transaction.current - end + # Tracks an event. + # + # See `Gitlab::Metrics::Transaction#add_event` for more details. + def add_event(*args) + trans = current_transaction - # When enabled this should be set before being used as the usual pattern - # "@foo ||= bar" is _not_ thread-safe. - # rubocop:disable Gitlab/ModuleWithInstanceVariables - def pool - if influx_metrics_enabled? - if @pool.nil? - MUTEX.synchronize do - @pool ||= ConnectionPool.new(size: settings[:pool_size], timeout: settings[:timeout]) do - host = settings[:host] - port = settings[:port] - - InfluxDB::Client - .new(udp: { host: host, port: port }) + trans&.add_event(*args) + end + + # Returns the prefix to use for the name of a series. + def series_prefix + @series_prefix ||= Sidekiq.server? ? 'sidekiq_' : 'rails_' + end + + # Allow access from other metrics related middlewares + def current_transaction + Transaction.current + end + + # When enabled this should be set before being used as the usual pattern + # "@foo ||= bar" is _not_ thread-safe. + # rubocop:disable Gitlab/ModuleWithInstanceVariables + def pool + if influx_metrics_enabled? + if @pool.nil? + MUTEX.synchronize do + @pool ||= ConnectionPool.new(size: settings[:pool_size], timeout: settings[:timeout]) do + host = settings[:host] + port = settings[:port] + + InfluxDB::Client + .new(udp: { host: host, port: port }) + end end end - end - @pool + @pool + end end + # rubocop:enable Gitlab/ModuleWithInstanceVariables end - # rubocop:enable Gitlab/ModuleWithInstanceVariables end end end diff --git a/lib/gitlab/metrics/method_call.rb b/lib/gitlab/metrics/method_call.rb index c2f9db56824..b11520a79bb 100644 --- a/lib/gitlab/metrics/method_call.rb +++ b/lib/gitlab/metrics/method_call.rb @@ -4,26 +4,15 @@ module Gitlab module Metrics # Class for tracking timing information about method calls class MethodCall - @@measurement_enabled_cache = Concurrent::AtomicBoolean.new(false) - @@measurement_enabled_cache_expires_at = Concurrent::AtomicReference.new(Time.now.to_i) - MUTEX = Mutex.new + include Gitlab::Metrics::Methods BASE_LABELS = { module: nil, method: nil }.freeze attr_reader :real_time, :cpu_time, :call_count, :labels - def self.call_duration_histogram - return @call_duration_histogram if @call_duration_histogram - - MUTEX.synchronize do - @call_duration_histogram ||= Gitlab::Metrics.histogram( - :gitlab_method_call_duration_seconds, - 'Method calls real duration', - Transaction::BASE_LABELS.merge(BASE_LABELS), - [0.01, 0.05, 0.1, 0.5, 1]) - end - end - - def self.measurement_enabled_cache_expires_at - @@measurement_enabled_cache_expires_at + define_histogram :gitlab_method_call_duration_seconds do + docstring 'Method calls real duration' + base_labels Transaction::BASE_LABELS.merge(BASE_LABELS) + buckets [0.01, 0.05, 0.1, 0.5, 1] + with_feature :prometheus_metrics_method_instrumentation end # name - The full name of the method (including namespace) such as @@ -53,8 +42,8 @@ module Gitlab @cpu_time += cpu_time @call_count += 1 - if call_measurement_enabled? && above_threshold? - self.class.call_duration_histogram.observe(@transaction.labels.merge(labels), real_time) + if above_threshold? + self.class.gitlab_method_call_duration_seconds.observe(@transaction.labels.merge(labels), real_time) end retval @@ -78,17 +67,6 @@ module Gitlab def above_threshold? real_time.in_milliseconds >= Metrics.method_call_threshold end - - def call_measurement_enabled? - expires_at = @@measurement_enabled_cache_expires_at.value - if expires_at < Time.now.to_i - if @@measurement_enabled_cache_expires_at.compare_and_set(expires_at, 1.minute.from_now.to_i) - @@measurement_enabled_cache.value = Feature.get(:prometheus_metrics_method_instrumentation).enabled? - end - end - - @@measurement_enabled_cache.value - end end end end diff --git a/lib/gitlab/metrics/methods.rb b/lib/gitlab/metrics/methods.rb new file mode 100644 index 00000000000..cd7c1e507f7 --- /dev/null +++ b/lib/gitlab/metrics/methods.rb @@ -0,0 +1,129 @@ +# rubocop:disable Style/ClassVars + +module Gitlab + module Metrics + module Methods + extend ActiveSupport::Concern + + included do + @@_metric_provider_mutex ||= Mutex.new + @@_metrics_provider_cache = {} + end + + class_methods do + def reload_metric!(name) + @@_metrics_provider_cache.delete(name) + end + + private + + def define_metric(type, name, opts = {}, &block) + if respond_to?(name) + raise ArgumentError, "method #{name} already exists" + end + + define_singleton_method(name) do + # inlining fetch_metric method to avoid method call overhead when instrumenting hot spots + @@_metrics_provider_cache[name] || init_metric(type, name, opts, &block) + end + end + + def fetch_metric(type, name, opts = {}, &block) + @@_metrics_provider_cache[name] || init_metric(type, name, opts, &block) + end + + def init_metric(type, name, opts = {}, &block) + options = MetricOptions.new(opts) + options.evaluate(&block) + + if disabled_by_feature(options) + synchronized_cache_fill(name) { NullMetric.instance } + else + synchronized_cache_fill(name) { build_metric!(type, name, options) } + end + end + + def synchronized_cache_fill(key) + @@_metric_provider_mutex.synchronize do + @@_metrics_provider_cache[key] ||= yield + end + end + + def disabled_by_feature(options) + options.with_feature && !Feature.get(options.with_feature).enabled? + end + + def build_metric!(type, name, options) + case type + when :gauge + Gitlab::Metrics.gauge(name, options.docstring, options.base_labels, options.multiprocess_mode) + when :counter + Gitlab::Metrics.counter(name, options.docstring, options.base_labels) + when :histogram + Gitlab::Metrics.histogram(name, options.docstring, options.base_labels, options.buckets) + when :summary + raise NotImplementedError, "summary metrics are not currently supported" + else + raise ArgumentError, "uknown metric type #{type}" + end + end + + # Fetch and/or initialize counter metric + # @param [Symbol] name + # @param [Hash] opts + def fetch_counter(name, opts = {}, &block) + fetch_metric(:counter, name, opts, &block) + end + + # Fetch and/or initialize gauge metric + # @param [Symbol] name + # @param [Hash] opts + def fetch_gauge(name, opts = {}, &block) + fetch_metric(:gauge, name, opts, &block) + end + + # Fetch and/or initialize histogram metric + # @param [Symbol] name + # @param [Hash] opts + def fetch_histogram(name, opts = {}, &block) + fetch_metric(:histogram, name, opts, &block) + end + + # Fetch and/or initialize summary metric + # @param [Symbol] name + # @param [Hash] opts + def fetch_summary(name, opts = {}, &block) + fetch_metric(:summary, name, opts, &block) + end + + # Define metric accessor method for a Counter + # @param [Symbol] name + # @param [Hash] opts + def define_counter(name, opts = {}, &block) + define_metric(:counter, name, opts, &block) + end + + # Define metric accessor method for a Gauge + # @param [Symbol] name + # @param [Hash] opts + def define_gauge(name, opts = {}, &block) + define_metric(:gauge, name, opts, &block) + end + + # Define metric accessor method for a Histogram + # @param [Symbol] name + # @param [Hash] opts + def define_histogram(name, opts = {}, &block) + define_metric(:histogram, name, opts, &block) + end + + # Define metric accessor method for a Summary + # @param [Symbol] name + # @param [Hash] opts + def define_summary(name, opts = {}, &block) + define_metric(:summary, name, opts, &block) + end + end + end + end +end diff --git a/lib/gitlab/metrics/methods/metric_options.rb b/lib/gitlab/metrics/methods/metric_options.rb new file mode 100644 index 00000000000..70e122d4e15 --- /dev/null +++ b/lib/gitlab/metrics/methods/metric_options.rb @@ -0,0 +1,61 @@ +module Gitlab + module Metrics + module Methods + class MetricOptions + SMALL_NETWORK_BUCKETS = [0.005, 0.01, 0.1, 1, 10].freeze + + def initialize(options = {}) + @multiprocess_mode = options[:multiprocess_mode] || :all + @buckets = options[:buckets] || SMALL_NETWORK_BUCKETS + @base_labels = options[:base_labels] || {} + @docstring = options[:docstring] + @with_feature = options[:with_feature] + end + + # Documentation describing metric in metrics endpoint '/-/metrics' + def docstring(docstring = nil) + @docstring = docstring unless docstring.nil? + + @docstring + end + + # Gauge aggregation mode for multiprocess metrics + # - :all (default) returns each gauge for every process + # - :livesum all process'es gauges summed up + # - :max maximum value of per process gauges + # - :min minimum value of per process gauges + def multiprocess_mode(mode = nil) + @multiprocess_mode = mode unless mode.nil? + + @multiprocess_mode + end + + # Measurement buckets for histograms + def buckets(buckets = nil) + @buckets = buckets unless buckets.nil? + + @buckets + end + + # Base labels are merged with per metric labels + def base_labels(base_labels = nil) + @base_labels = base_labels unless base_labels.nil? + + @base_labels + end + + # Use feature toggle to control whether certain metric is enabled/disabled + def with_feature(name = nil) + @with_feature = name unless name.nil? + + @with_feature + end + + def evaluate(&block) + instance_eval(&block) if block_given? + self + end + end + end + end +end diff --git a/lib/gitlab/metrics/null_metric.rb b/lib/gitlab/metrics/null_metric.rb index 3b5a2907195..aabada5c21a 100644 --- a/lib/gitlab/metrics/null_metric.rb +++ b/lib/gitlab/metrics/null_metric.rb @@ -2,6 +2,8 @@ module Gitlab module Metrics # Mocks ::Prometheus::Client::Metric and all derived metrics class NullMetric + include Singleton + def method_missing(name, *args, &block) nil end diff --git a/lib/gitlab/metrics/prometheus.rb b/lib/gitlab/metrics/prometheus.rb index b0b8e8436db..d12ba0ec176 100644 --- a/lib/gitlab/metrics/prometheus.rb +++ b/lib/gitlab/metrics/prometheus.rb @@ -3,73 +3,76 @@ require 'prometheus/client' module Gitlab module Metrics module Prometheus - include Gitlab::CurrentSettings - include Gitlab::Utils::StrongMemoize + extend ActiveSupport::Concern REGISTRY_MUTEX = Mutex.new PROVIDER_MUTEX = Mutex.new - def metrics_folder_present? - multiprocess_files_dir = ::Prometheus::Client.configuration.multiprocess_files_dir + class_methods do + include Gitlab::Utils::StrongMemoize - multiprocess_files_dir && - ::Dir.exist?(multiprocess_files_dir) && - ::File.writable?(multiprocess_files_dir) - end + def metrics_folder_present? + multiprocess_files_dir = ::Prometheus::Client.configuration.multiprocess_files_dir - def prometheus_metrics_enabled? - strong_memoize(:prometheus_metrics_enabled) do - prometheus_metrics_enabled_unmemoized + multiprocess_files_dir && + ::Dir.exist?(multiprocess_files_dir) && + ::File.writable?(multiprocess_files_dir) + end + + def prometheus_metrics_enabled? + strong_memoize(:prometheus_metrics_enabled) do + prometheus_metrics_enabled_unmemoized + end end - end - def registry - strong_memoize(:registry) do - REGISTRY_MUTEX.synchronize do - strong_memoize(:registry) do - ::Prometheus::Client.registry + def registry + strong_memoize(:registry) do + REGISTRY_MUTEX.synchronize do + strong_memoize(:registry) do + ::Prometheus::Client.registry + end end end end - end - def counter(name, docstring, base_labels = {}) - safe_provide_metric(:counter, name, docstring, base_labels) - end + def counter(name, docstring, base_labels = {}) + safe_provide_metric(:counter, name, docstring, base_labels) + end - def summary(name, docstring, base_labels = {}) - safe_provide_metric(:summary, name, docstring, base_labels) - end + def summary(name, docstring, base_labels = {}) + safe_provide_metric(:summary, name, docstring, base_labels) + end - def gauge(name, docstring, base_labels = {}, multiprocess_mode = :all) - safe_provide_metric(:gauge, name, docstring, base_labels, multiprocess_mode) - end + def gauge(name, docstring, base_labels = {}, multiprocess_mode = :all) + safe_provide_metric(:gauge, name, docstring, base_labels, multiprocess_mode) + end - def histogram(name, docstring, base_labels = {}, buckets = ::Prometheus::Client::Histogram::DEFAULT_BUCKETS) - safe_provide_metric(:histogram, name, docstring, base_labels, buckets) - end + def histogram(name, docstring, base_labels = {}, buckets = ::Prometheus::Client::Histogram::DEFAULT_BUCKETS) + safe_provide_metric(:histogram, name, docstring, base_labels, buckets) + end - private + private - def safe_provide_metric(method, name, *args) - metric = provide_metric(name) - return metric if metric + def safe_provide_metric(method, name, *args) + metric = provide_metric(name) + return metric if metric - PROVIDER_MUTEX.synchronize do - provide_metric(name) || registry.method(method).call(name, *args) + PROVIDER_MUTEX.synchronize do + provide_metric(name) || registry.method(method).call(name, *args) + end end - end - def provide_metric(name) - if prometheus_metrics_enabled? - registry.get(name) - else - NullMetric.new + def provide_metric(name) + if prometheus_metrics_enabled? + registry.get(name) + else + NullMetric.instance + end end - end - def prometheus_metrics_enabled_unmemoized - metrics_folder_present? && current_application_settings[:prometheus_metrics_enabled] || false + def prometheus_metrics_enabled_unmemoized + metrics_folder_present? && Gitlab::CurrentSettings.prometheus_metrics_enabled || false + end end end end diff --git a/lib/gitlab/metrics/subscribers/action_view.rb b/lib/gitlab/metrics/subscribers/action_view.rb index 3da474fc1ec..b600e8a2a50 100644 --- a/lib/gitlab/metrics/subscribers/action_view.rb +++ b/lib/gitlab/metrics/subscribers/action_view.rb @@ -3,6 +3,14 @@ module Gitlab module Subscribers # Class for tracking the rendering timings of views. class ActionView < ActiveSupport::Subscriber + include Gitlab::Metrics::Methods + define_histogram :gitlab_view_rendering_duration_seconds do + docstring 'View rendering time' + base_labels Transaction::BASE_LABELS.merge({ path: nil }) + buckets [0.001, 0.01, 0.1, 1, 10.0] + with_feature :prometheus_metrics_view_instrumentation + end + attach_to :action_view SERIES = 'views'.freeze @@ -15,30 +23,18 @@ module Gitlab private - def metric_view_rendering_duration_seconds - @metric_view_rendering_duration_seconds ||= Gitlab::Metrics.histogram( - :gitlab_view_rendering_duration_seconds, - 'View rendering time', - Transaction::BASE_LABELS.merge({ path: nil }), - [0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.500, 2.0, 10.0] - ) - end - def track(event) values = values_for(event) tags = tags_for(event) - metric_view_rendering_duration_seconds.observe( - current_transaction.labels.merge(tags), - event.duration - ) + self.class.gitlab_view_rendering_duration_seconds.observe(current_transaction.labels.merge(tags), event.duration) current_transaction.increment(:view_duration, event.duration) current_transaction.add_metric(SERIES, values, tags) end def relative_path(path) - path.gsub(/^#{Rails.root.to_s}\/?/, '') + path.gsub(%r{^#{Rails.root.to_s}/?}, '') end def values_for(event) diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index ead1acb8d44..4b3e8d0a6a0 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -3,12 +3,13 @@ module Gitlab module Subscribers # Class for tracking the total query duration of a transaction. class ActiveRecord < ActiveSupport::Subscriber + include Gitlab::Metrics::Methods attach_to :active_record def sql(event) return unless current_transaction - metric_sql_duration_seconds.observe(current_transaction.labels, event.duration / 1000.0) + self.class.gitlab_sql_duration_seconds.observe(current_transaction.labels, event.duration / 1000.0) current_transaction.increment(:sql_duration, event.duration, false) current_transaction.increment(:sql_count, 1, false) @@ -16,17 +17,14 @@ module Gitlab private - def current_transaction - Transaction.current + define_histogram :gitlab_sql_duration_seconds do + docstring 'SQL time' + base_labels Transaction::BASE_LABELS + buckets [0.001, 0.01, 0.1, 1.0, 10.0] end - def metric_sql_duration_seconds - @metric_sql_duration_seconds ||= Gitlab::Metrics.histogram( - :gitlab_sql_duration_seconds, - 'SQL time', - Transaction::BASE_LABELS, - [0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.500, 2.0, 10.0] - ) + def current_transaction + Transaction.current end end end diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index e7975c023a9..45b9e14ba55 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -2,11 +2,12 @@ module Gitlab module Metrics # Class for storing metrics information of a single transaction. class Transaction + include Gitlab::Metrics::Methods + # base labels shared among all transactions BASE_LABELS = { controller: nil, action: nil }.freeze THREAD_KEY = :_gitlab_metrics_transaction - METRICS_MUTEX = Mutex.new # The series to store events (e.g. Git pushes) in. EVENT_SERIES = 'events'.freeze @@ -54,8 +55,8 @@ module Gitlab @memory_after = System.memory_usage @finished_at = System.monotonic_time - self.class.metric_transaction_duration_seconds.observe(labels, duration) - self.class.metric_transaction_allocated_memory_bytes.observe(labels, allocated_memory * 1024.0) + self.class.gitlab_transaction_duration_seconds.observe(labels, duration) + self.class.gitlab_transaction_allocated_memory_bytes.observe(labels, allocated_memory * 1024.0) Thread.current[THREAD_KEY] = nil end @@ -72,7 +73,7 @@ module Gitlab # event_name - The name of the event (e.g. "git_push"). # tags - A set of tags to attach to the event. def add_event(event_name, tags = {}) - self.class.metric_event_counter(event_name, tags).increment(tags.merge(labels)) + self.class.transaction_metric(event_name, :counter, prefix: 'event_', tags: tags).increment(tags.merge(labels)) @metrics << Metric.new(EVENT_SERIES, { count: 1 }, tags.merge(event: event_name), :event) end @@ -86,12 +87,12 @@ module Gitlab end def increment(name, value, use_prometheus = true) - self.class.metric_transaction_counter(name).increment(labels, value) if use_prometheus + self.class.transaction_metric(name, :counter).increment(labels, value) if use_prometheus @values[name] += value end def set(name, value, use_prometheus = true) - self.class.metric_transaction_gauge(name).set(labels, value) if use_prometheus + self.class.transaction_metric(name, :gauge).set(labels, value) if use_prometheus @values[name] = value end @@ -136,64 +137,28 @@ module Gitlab "#{labels[:controller]}##{labels[:action]}" if labels && !labels.empty? end - def self.metric_transaction_duration_seconds - return @metric_transaction_duration_seconds if @metric_transaction_duration_seconds - - METRICS_MUTEX.synchronize do - @metric_transaction_duration_seconds ||= Gitlab::Metrics.histogram( - :gitlab_transaction_duration_seconds, - 'Transaction duration', - BASE_LABELS, - [0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.500, 2.0, 10.0] - ) - end - end - - def self.metric_transaction_allocated_memory_bytes - return @metric_transaction_allocated_memory_bytes if @metric_transaction_allocated_memory_bytes - - METRICS_MUTEX.synchronize do - @metric_transaction_allocated_memory_bytes ||= Gitlab::Metrics.histogram( - :gitlab_transaction_allocated_memory_bytes, - 'Transaction allocated memory bytes', - BASE_LABELS, - [1000, 10000, 20000, 500000, 1000000, 2000000, 5000000, 10000000, 20000000, 100000000] - ) - end + define_histogram :gitlab_transaction_duration_seconds do + docstring 'Transaction duration' + base_labels BASE_LABELS + buckets [0.001, 0.01, 0.1, 1.0, 10.0] end - def self.metric_event_counter(event_name, tags) - return @metric_event_counters[event_name] if @metric_event_counters&.has_key?(event_name) - - METRICS_MUTEX.synchronize do - @metric_event_counters ||= {} - @metric_event_counters[event_name] ||= Gitlab::Metrics.counter( - "gitlab_transaction_event_#{event_name}_total".to_sym, - "Transaction event #{event_name} counter", - tags.merge(BASE_LABELS) - ) - end - end - - def self.metric_transaction_counter(name) - return @metric_transaction_counters[name] if @metric_transaction_counters&.has_key?(name) - - METRICS_MUTEX.synchronize do - @metric_transaction_counters ||= {} - @metric_transaction_counters[name] ||= Gitlab::Metrics.counter( - "gitlab_transaction_#{name}_total".to_sym, "Transaction #{name} counter", BASE_LABELS - ) - end + define_histogram :gitlab_transaction_allocated_memory_bytes do + docstring 'Transaction allocated memory bytes' + base_labels BASE_LABELS + buckets [100, 1000, 10000, 100000, 1000000, 10000000] + with_feature :prometheus_metrics_transaction_allocated_memory end - def self.metric_transaction_gauge(name) - return @metric_transaction_gauges[name] if @metric_transaction_gauges&.has_key?(name) + def self.transaction_metric(name, type, prefix: nil, tags: {}) + metric_name = "gitlab_transaction_#{prefix}#{name}_total".to_sym + fetch_metric(type, metric_name) do + docstring "Transaction #{prefix}#{name} #{type}" + base_labels tags.merge(BASE_LABELS) - METRICS_MUTEX.synchronize do - @metric_transaction_gauges ||= {} - @metric_transaction_gauges[name] ||= Gitlab::Metrics.gauge( - "gitlab_transaction_#{name}".to_sym, "Transaction gauge #{name}", BASE_LABELS, :livesum - ) + if type == :gauge + multiprocess_mode :livesum + end end end end diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb index c6a56277922..1a570f480c6 100644 --- a/lib/gitlab/middleware/go.rb +++ b/lib/gitlab/middleware/go.rb @@ -4,7 +4,6 @@ module Gitlab module Middleware class Go include ActionView::Helpers::TagHelper - include Gitlab::CurrentSettings PROJECT_PATH_REGEX = %r{\A(#{Gitlab::PathRegex.full_namespace_route_regex}/#{Gitlab::PathRegex.project_route_regex})/}.freeze @@ -42,7 +41,7 @@ module Gitlab project_url = URI.join(config.gitlab.url, path) import_prefix = strip_url(project_url.to_s) - repository_url = if current_application_settings.enabled_git_access_protocol == 'ssh' + repository_url = if Gitlab::CurrentSettings.enabled_git_access_protocol == 'ssh' shell = config.gitlab_shell port = ":#{shell.ssh_port}" unless shell.ssh_port == 22 "ssh://#{shell.ssh_user}@#{shell.ssh_host}#{port}/#{path}.git" @@ -56,12 +55,12 @@ module Gitlab end def strip_url(url) - url.gsub(/\Ahttps?:\/\//, '') + url.gsub(%r{\Ahttps?://}, '') end def project_path(request) path_info = request.env["PATH_INFO"] - path_info.sub!(/^\//, '') + path_info.sub!(%r{^/}, '') project_path_match = "#{path_info}/".match(PROJECT_PATH_REGEX) return unless project_path_match diff --git a/lib/gitlab/middleware/static.rb b/lib/gitlab/middleware/static.rb index 85ffa8aca68..aa1e9dc0fdb 100644 --- a/lib/gitlab/middleware/static.rb +++ b/lib/gitlab/middleware/static.rb @@ -1,7 +1,7 @@ module Gitlab module Middleware class Static < ActionDispatch::Static - UPLOADS_REGEX = /\A\/uploads(\/|\z)/.freeze + UPLOADS_REGEX = %r{\A/uploads(/|\z)}.freeze def call(env) return @app.call(env) if env['PATH_INFO'] =~ UPLOADS_REGEX diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index fff9360ea27..e40a001d20c 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -55,7 +55,7 @@ module Gitlab user ||= find_or_build_ldap_user if auto_link_ldap_user? user ||= build_new_user if signup_enabled? - user.external = true if external_provider? && user + user.external = true if external_provider? && user&.new_record? user end diff --git a/lib/gitlab/performance_bar.rb b/lib/gitlab/performance_bar.rb index e29e168fc5a..6c2b2036074 100644 --- a/lib/gitlab/performance_bar.rb +++ b/lib/gitlab/performance_bar.rb @@ -1,7 +1,5 @@ module Gitlab module PerformanceBar - extend Gitlab::CurrentSettings - ALLOWED_USER_IDS_KEY = 'performance_bar_allowed_user_ids:v2'.freeze EXPIRY_TIME = 5.minutes @@ -13,7 +11,7 @@ module Gitlab end def self.allowed_group_id - current_application_settings.performance_bar_allowed_group_id + Gitlab::CurrentSettings.performance_bar_allowed_group_id end def self.allowed_user_ids diff --git a/lib/gitlab/polling_interval.rb b/lib/gitlab/polling_interval.rb index 4780675a492..fe4bdfe3831 100644 --- a/lib/gitlab/polling_interval.rb +++ b/lib/gitlab/polling_interval.rb @@ -1,12 +1,10 @@ module Gitlab class PollingInterval - extend Gitlab::CurrentSettings - HEADER_NAME = 'Poll-Interval'.freeze def self.set_header(response, interval:) if polling_enabled? - multiplier = current_application_settings.polling_interval_multiplier + multiplier = Gitlab::CurrentSettings.polling_interval_multiplier value = (interval * multiplier).to_i else value = -1 @@ -16,7 +14,7 @@ module Gitlab end def self.polling_enabled? - !current_application_settings.polling_interval_multiplier.zero? + !Gitlab::CurrentSettings.polling_interval_multiplier.zero? end end end diff --git a/lib/gitlab/popen.rb b/lib/gitlab/popen.rb index 4bc5cda8cb5..b9832a724c4 100644 --- a/lib/gitlab/popen.rb +++ b/lib/gitlab/popen.rb @@ -5,7 +5,17 @@ module Gitlab module Popen extend self - def popen(cmd, path = nil, vars = {}) + Result = Struct.new(:cmd, :stdout, :stderr, :status, :duration) + + # Returns [stdout + stderr, status] + def popen(cmd, path = nil, vars = {}, &block) + result = popen_with_detail(cmd, path, vars, &block) + + [result.stdout << result.stderr, result.status&.exitstatus] + end + + # Returns Result + def popen_with_detail(cmd, path = nil, vars = {}) unless cmd.is_a?(Array) raise "System commands must be given as an array of strings" end @@ -18,18 +28,21 @@ module Gitlab FileUtils.mkdir_p(path) end - cmd_output = "" - cmd_status = 0 + cmd_stdout = '' + cmd_stderr = '' + cmd_status = nil + start = Time.now + Open3.popen3(vars, *cmd, options) do |stdin, stdout, stderr, wait_thr| yield(stdin) if block_given? stdin.close - cmd_output << stdout.read - cmd_output << stderr.read - cmd_status = wait_thr.value.exitstatus + cmd_stdout = stdout.read + cmd_stderr = stderr.read + cmd_status = wait_thr.value end - [cmd_output, cmd_status] + Result.new(cmd, cmd_stdout, cmd_stderr, cmd_status, Time.now - start) end end end diff --git a/lib/gitlab/popen/runner.rb b/lib/gitlab/popen/runner.rb new file mode 100644 index 00000000000..f44035a48bb --- /dev/null +++ b/lib/gitlab/popen/runner.rb @@ -0,0 +1,46 @@ +module Gitlab + module Popen + class Runner + attr_reader :results + + def initialize + @results = [] + end + + def run(commands, &block) + commands.each do |cmd| + # yield doesn't support blocks, so we need to use a block variable + block.call(cmd) do # rubocop:disable Performance/RedundantBlockCall + cmd_result = Gitlab::Popen.popen_with_detail(cmd) + + results << cmd_result + + cmd_result + end + end + end + + def all_success_and_clean? + all_success? && all_stderr_empty? + end + + def all_success? + results.all? { |result| result.status.success? } + end + + def all_stderr_empty? + results.all? { |result| result.stderr.empty? } + end + + def failed_results + results.reject { |result| result.status.success? } + end + + def warned_results + results.select do |result| + result.status.success? && !result.stderr.empty? + end + end + end + end +end diff --git a/lib/gitlab/protocol_access.rb b/lib/gitlab/protocol_access.rb index 09fa14764e6..2819c7d062c 100644 --- a/lib/gitlab/protocol_access.rb +++ b/lib/gitlab/protocol_access.rb @@ -1,14 +1,12 @@ module Gitlab module ProtocolAccess - extend Gitlab::CurrentSettings - def self.allowed?(protocol) if protocol == 'web' true - elsif current_application_settings.enabled_git_access_protocol.blank? + elsif Gitlab::CurrentSettings.enabled_git_access_protocol.blank? true else - protocol == current_application_settings.enabled_git_access_protocol + protocol == Gitlab::CurrentSettings.enabled_git_access_protocol end end end diff --git a/lib/gitlab/quick_actions/spend_time_and_date_separator.rb b/lib/gitlab/quick_actions/spend_time_and_date_separator.rb index 3f52402b31f..7328c517a30 100644 --- a/lib/gitlab/quick_actions/spend_time_and_date_separator.rb +++ b/lib/gitlab/quick_actions/spend_time_and_date_separator.rb @@ -9,7 +9,7 @@ module Gitlab # if date doesn't present return time with current date # in other cases return nil class SpendTimeAndDateSeparator - DATE_REGEX = /(\d{2,4}[\/\-.]\d{1,2}[\/\-.]\d{1,2})/ + DATE_REGEX = %r{(\d{2,4}[/\-.]\d{1,2}[/\-.]\d{1,2})} def initialize(spend_command_arg) @spend_arg = spend_command_arg diff --git a/lib/gitlab/recaptcha.rb b/lib/gitlab/recaptcha.rb index c463dd487a0..c9efa28d7e7 100644 --- a/lib/gitlab/recaptcha.rb +++ b/lib/gitlab/recaptcha.rb @@ -1,12 +1,10 @@ module Gitlab module Recaptcha - extend Gitlab::CurrentSettings - def self.load_configurations! - if current_application_settings.recaptcha_enabled + if Gitlab::CurrentSettings.recaptcha_enabled ::Recaptcha.configure do |config| - config.public_key = current_application_settings.recaptcha_site_key - config.private_key = current_application_settings.recaptcha_private_key + config.public_key = Gitlab::CurrentSettings.recaptcha_site_key + config.private_key = Gitlab::CurrentSettings.recaptcha_private_key end true @@ -14,7 +12,7 @@ module Gitlab end def self.enabled? - current_application_settings.recaptcha_enabled + Gitlab::CurrentSettings.recaptcha_enabled end end end diff --git a/lib/gitlab/redis/cache.rb b/lib/gitlab/redis/cache.rb index 9bf019b72e6..a991933e910 100644 --- a/lib/gitlab/redis/cache.rb +++ b/lib/gitlab/redis/cache.rb @@ -1,5 +1,5 @@ # please require all dependencies below: -require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper) +require_relative 'wrapper' unless defined?(::Rails) && ::Rails.root.present? module Gitlab module Redis diff --git a/lib/gitlab/repo_path.rb b/lib/gitlab/repo_path.rb index 3591fa9145e..79265cf952d 100644 --- a/lib/gitlab/repo_path.rb +++ b/lib/gitlab/repo_path.rb @@ -30,7 +30,7 @@ module Gitlab raise NotFoundError.new("No known storage path matches #{repo_path.inspect}") end - result.sub(/\A\/*/, '') + result.sub(%r{\A/*}, '') end def self.find_project(project_path) diff --git a/lib/gitlab/seeder.rb b/lib/gitlab/seeder.rb index 94a481a0f2e..98f005cb61b 100644 --- a/lib/gitlab/seeder.rb +++ b/lib/gitlab/seeder.rb @@ -5,9 +5,15 @@ module DeliverNever end end +module MuteNotifications + def new_note(note) + end +end + module Gitlab class Seeder def self.quiet + mute_notifications mute_mailer SeedFu.quiet = true @@ -18,6 +24,10 @@ module Gitlab puts "\nOK".color(:green) end + def self.mute_notifications + NotificationService.prepend(MuteNotifications) + end + def self.mute_mailer ActionMailer::MessageDelivery.prepend(DeliverNever) end diff --git a/lib/gitlab/sentry.rb b/lib/gitlab/sentry.rb index 159d0e7952e..4a22fc80f75 100644 --- a/lib/gitlab/sentry.rb +++ b/lib/gitlab/sentry.rb @@ -1,9 +1,7 @@ module Gitlab module Sentry - extend Gitlab::CurrentSettings - def self.enabled? - Rails.env.production? && current_application_settings.sentry_enabled? + Rails.env.production? && Gitlab::CurrentSettings.sentry_enabled? end def self.context(current_user = nil) diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb index d01213bb6e0..e90a90508a2 100644 --- a/lib/gitlab/setup_helper.rb +++ b/lib/gitlab/setup_helper.rb @@ -31,7 +31,7 @@ module Gitlab storages << { name: 'test_second_storage', path: Rails.root.join('tmp', 'tests', 'second_storage').to_s } end - config = { socket_path: address.sub(%r{\Aunix:}, ''), storage: storages } + config = { socket_path: address.sub(/\Aunix:/, ''), storage: storages } config[:auth] = { token: 'secret' } if Rails.env.test? config[:'gitaly-ruby'] = { dir: File.join(gitaly_dir, 'ruby') } if gitaly_ruby config[:'gitlab-shell'] = { dir: Gitlab.config.gitlab_shell.path } diff --git a/lib/gitlab/sherlock/file_sample.rb b/lib/gitlab/sherlock/file_sample.rb index 8a3e1a5e5bf..89072b01f2e 100644 --- a/lib/gitlab/sherlock/file_sample.rb +++ b/lib/gitlab/sherlock/file_sample.rb @@ -16,7 +16,7 @@ module Gitlab end def relative_path - @relative_path ||= @file.gsub(/^#{Rails.root.to_s}\/?/, '') + @relative_path ||= @file.gsub(%r{^#{Rails.root.to_s}/?}, '') end def to_param diff --git a/lib/gitlab/sherlock/middleware.rb b/lib/gitlab/sherlock/middleware.rb index 687332fc5fc..4c88e33699a 100644 --- a/lib/gitlab/sherlock/middleware.rb +++ b/lib/gitlab/sherlock/middleware.rb @@ -2,7 +2,7 @@ module Gitlab module Sherlock # Rack middleware used for tracking request metrics. class Middleware - CONTENT_TYPES = /text\/html|application\/json/i + CONTENT_TYPES = %r{text/html|application/json}i IGNORE_PATHS = %r{^/sherlock} diff --git a/lib/gitlab/sherlock/query.rb b/lib/gitlab/sherlock/query.rb index 948bf5e6528..02ddc3f47eb 100644 --- a/lib/gitlab/sherlock/query.rb +++ b/lib/gitlab/sherlock/query.rb @@ -4,7 +4,7 @@ module Gitlab attr_reader :id, :query, :started_at, :finished_at, :backtrace # SQL identifiers that should be prefixed with newlines. - PREFIX_NEWLINE = / + PREFIX_NEWLINE = %r{ \s+(FROM |(LEFT|RIGHT)?INNER\s+JOIN |(LEFT|RIGHT)?OUTER\s+JOIN @@ -13,7 +13,7 @@ module Gitlab |GROUP\s+BY |ORDER\s+BY |LIMIT - |OFFSET)\s+/ix # Vim indent breaks when this is on a newline :< + |OFFSET)\s+}ix # Vim indent breaks when this is on a newline :< # Creates a new Query using a String and a separate Array of bindings. # diff --git a/lib/gitlab/ssh_public_key.rb b/lib/gitlab/ssh_public_key.rb index 89ca1298120..545e7c74f7e 100644 --- a/lib/gitlab/ssh_public_key.rb +++ b/lib/gitlab/ssh_public_key.rb @@ -21,6 +21,22 @@ module Gitlab technology(name)&.supported_sizes end + def self.sanitize(key_content) + ssh_type, *parts = key_content.strip.split + + return key_content if parts.empty? + + parts.each_with_object("#{ssh_type} ").with_index do |(part, content), index| + content << part + + if Gitlab::SSHPublicKey.new(content).valid? + break [content, parts[index + 1]].compact.join(' ') # Add the comment part if present + elsif parts.size == index + 1 # return original content if we've reached the last element + break key_content + end + end + end + attr_reader :key_text, :key # Unqualified MD5 fingerprint for compatibility @@ -37,23 +53,23 @@ module Gitlab end def valid? - key.present? + key.present? && bits && technology.supported_sizes.include?(bits) end def type - technology.name if valid? + technology.name if key.present? end def bits - return unless valid? + return if key.blank? case type when :rsa - key.n.num_bits + key.n&.num_bits when :dsa - key.p.num_bits + key.p&.num_bits when :ecdsa - key.group.order.num_bits + key.group.order&.num_bits when :ed25519 256 else diff --git a/lib/tasks/gitlab/task_helpers.rb b/lib/gitlab/task_helpers.rb index c1182af1014..34bee6fecbe 100644 --- a/lib/tasks/gitlab/task_helpers.rb +++ b/lib/gitlab/task_helpers.rb @@ -1,6 +1,7 @@ require 'rainbow/ext/string' require 'gitlab/utils/strong_memoize' +# rubocop:disable Rails/Output module Gitlab TaskFailedError = Class.new(StandardError) TaskAbortedByUserError = Class.new(StandardError) @@ -96,11 +97,9 @@ module Gitlab end def gid_for(group_name) - begin - Etc.getgrnam(group_name).gid - rescue ArgumentError # no group - "group #{group_name} doesn't exist" - end + Etc.getgrnam(group_name).gid + rescue ArgumentError # no group + "group #{group_name} doesn't exist" end def gitlab_user diff --git a/lib/gitlab/upgrader.rb b/lib/gitlab/upgrader.rb index 3b64cb32afa..024be6aca44 100644 --- a/lib/gitlab/upgrader.rb +++ b/lib/gitlab/upgrader.rb @@ -1,6 +1,3 @@ -require_relative "popen" -require_relative "version_info" - module Gitlab class Upgrader def execute @@ -52,7 +49,7 @@ module Gitlab def fetch_git_tags remote_tags, _ = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} ls-remote --tags https://gitlab.com/gitlab-org/gitlab-ce.git)) - remote_tags.split("\n").grep(/tags\/v#{current_version.major}/) + remote_tags.split("\n").grep(%r{tags/v#{current_version.major}}) end def update_commands diff --git a/lib/gitlab/uploads_transfer.rb b/lib/gitlab/uploads_transfer.rb index b5f41240529..7d7400bdabf 100644 --- a/lib/gitlab/uploads_transfer.rb +++ b/lib/gitlab/uploads_transfer.rb @@ -1,7 +1,7 @@ module Gitlab class UploadsTransfer < ProjectTransfer def root_dir - File.join(CarrierWave.root, FileUploader.base_dir) + FileUploader.root end end end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 2adcc9809b3..9d13d1d781f 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -1,8 +1,6 @@ module Gitlab class UsageData class << self - include Gitlab::CurrentSettings - def data(force_refresh: false) Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) { uncached_data } end @@ -19,7 +17,7 @@ module Gitlab def license_usage_data usage_data = { - uuid: current_application_settings.uuid, + uuid: Gitlab::CurrentSettings.uuid, hostname: Gitlab.config.gitlab.host, version: Gitlab::VERSION, active_user_count: User.active.count, @@ -79,9 +77,9 @@ module Gitlab def features_usage_data_ce { - signup: current_application_settings.allow_signup?, + signup: Gitlab::CurrentSettings.allow_signup?, ldap: Gitlab.config.ldap.enabled, - gravatar: current_application_settings.gravatar_enabled?, + gravatar: Gitlab::CurrentSettings.gravatar_enabled?, omniauth: Gitlab.config.omniauth.enabled, reply_by_email: Gitlab::IncomingEmail.enabled?, container_registry: Gitlab.config.registry.enabled, diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index 6ced06a863d..2612208a927 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -5,7 +5,6 @@ # module Gitlab module VisibilityLevel - extend CurrentSettings extend ActiveSupport::Concern included do @@ -58,9 +57,9 @@ module Gitlab end def allowed_levels - restricted_levels = current_application_settings.restricted_visibility_levels + restricted_levels = Gitlab::CurrentSettings.restricted_visibility_levels - self.values - restricted_levels + self.values - Array(restricted_levels) end def closest_allowed_level(target_level) @@ -81,7 +80,7 @@ module Gitlab end def non_restricted_level?(level) - restricted_levels = current_application_settings.restricted_visibility_levels + restricted_levels = Gitlab::CurrentSettings.restricted_visibility_levels if restricted_levels.nil? true diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 633da44b22d..b3f8b0d174d 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -55,14 +55,14 @@ module Gitlab def lfs_upload_ok(oid, size) { - StoreLFSPath: "#{Gitlab.config.lfs.storage_path}/tmp/upload", + StoreLFSPath: LfsObjectUploader.workhorse_upload_path, LfsOid: oid, LfsSize: size } end def artifact_upload_ok - { TempPath: JobArtifactUploader.artifacts_upload_path } + { TempPath: JobArtifactUploader.workhorse_upload_path } end def send_git_blob(repository, blob) @@ -147,8 +147,11 @@ module Gitlab end def send_artifacts_entry(build, entry) + file = build.artifacts_file + archive = file.file_storage? ? file.path : file.url + params = { - 'Archive' => build.artifacts_file.path, + 'Archive' => archive, 'Entry' => Base64.encode64(entry.to_s) } diff --git a/lib/support/nginx/gitlab b/lib/support/nginx/gitlab index 54f51d9d633..0e27a28ea6e 100644 --- a/lib/support/nginx/gitlab +++ b/lib/support/nginx/gitlab @@ -17,6 +17,8 @@ ## See installation.md#using-https for additional HTTPS configuration details. upstream gitlab-workhorse { + # Gitlab socket file, + # for Omnibus this would be: unix:/var/opt/gitlab/gitlab-workhorse/socket server unix:/home/git/gitlab/tmp/sockets/gitlab-workhorse.socket fail_timeout=0; } @@ -110,6 +112,8 @@ server { error_page 502 /502.html; error_page 503 /503.html; location ~ ^/(404|422|500|502|503)\.html$ { + # Location to the Gitlab's public directory, + # for Omnibus this would be: /opt/gitlab/embedded/service/gitlab-rails/public. root /home/git/gitlab/public; internal; } diff --git a/lib/support/nginx/gitlab-ssl b/lib/support/nginx/gitlab-ssl index ed8131ef24f..8218d68f9ba 100644 --- a/lib/support/nginx/gitlab-ssl +++ b/lib/support/nginx/gitlab-ssl @@ -21,6 +21,8 @@ ## See installation.md#using-https for additional HTTPS configuration details. upstream gitlab-workhorse { + # Gitlab socket file, + # for Omnibus this would be: unix:/var/opt/gitlab/gitlab-workhorse/socket server unix:/home/git/gitlab/tmp/sockets/gitlab-workhorse.socket fail_timeout=0; } @@ -160,6 +162,8 @@ server { error_page 502 /502.html; error_page 503 /503.html; location ~ ^/(404|422|500|502|503)\.html$ { + # Location to the Gitlab's public directory, + # for Omnibus this would be: /opt/gitlab/embedded/service/gitlab-rails/public root /home/git/gitlab/public; internal; } diff --git a/lib/system_check/app/git_version_check.rb b/lib/system_check/app/git_version_check.rb index 6ee8c8874ec..44ec888c197 100644 --- a/lib/system_check/app/git_version_check.rb +++ b/lib/system_check/app/git_version_check.rb @@ -5,7 +5,7 @@ module SystemCheck set_check_pass -> { "yes (#{self.current_version})" } def self.required_version - @required_version ||= Gitlab::VersionInfo.new(2, 7, 3) + @required_version ||= Gitlab::VersionInfo.new(2, 9, 5) end def self.current_version diff --git a/lib/system_check/helpers.rb b/lib/system_check/helpers.rb index c42ae4fe4c4..914ed794601 100644 --- a/lib/system_check/helpers.rb +++ b/lib/system_check/helpers.rb @@ -1,5 +1,3 @@ -require 'tasks/gitlab/task_helpers' - module SystemCheck module Helpers include ::Gitlab::TaskHelpers diff --git a/lib/tasks/flay.rake b/lib/tasks/flay.rake index 7ad2b2e4d39..b1e012e70c5 100644 --- a/lib/tasks/flay.rake +++ b/lib/tasks/flay.rake @@ -1,6 +1,6 @@ desc 'Code duplication analyze via flay' task :flay do - output = `bundle exec flay --mass 35 app/ lib/gitlab/` + output = `bundle exec flay --mass 35 app/ lib/gitlab/ 2> #{File::NULL}` if output.include? "Similar code found" puts output diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index 2383bcf954b..24e37f6c6cc 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -4,7 +4,7 @@ namespace :gitlab do namespace :backup do # Create backup of GitLab system desc "GitLab | Create a backup of the GitLab system" - task create: :environment do + task create: :gitlab_environment do warn_user_is_not_gitlab configure_cron_mode @@ -25,7 +25,7 @@ namespace :gitlab do # Restore backup of GitLab system desc 'GitLab | Restore a previously created backup' - task restore: :environment do + task restore: :gitlab_environment do warn_user_is_not_gitlab configure_cron_mode @@ -73,7 +73,7 @@ namespace :gitlab do end namespace :repo do - task create: :environment do + task create: :gitlab_environment do $progress.puts "Dumping repositories ...".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("repositories") @@ -84,7 +84,7 @@ namespace :gitlab do end end - task restore: :environment do + task restore: :gitlab_environment do $progress.puts "Restoring repositories ...".color(:blue) Backup::Repository.new.restore $progress.puts "done".color(:green) @@ -92,7 +92,7 @@ namespace :gitlab do end namespace :db do - task create: :environment do + task create: :gitlab_environment do $progress.puts "Dumping database ... ".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("db") @@ -103,7 +103,7 @@ namespace :gitlab do end end - task restore: :environment do + task restore: :gitlab_environment do $progress.puts "Restoring database ... ".color(:blue) Backup::Database.new.restore $progress.puts "done".color(:green) @@ -111,7 +111,7 @@ namespace :gitlab do end namespace :builds do - task create: :environment do + task create: :gitlab_environment do $progress.puts "Dumping builds ... ".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("builds") @@ -122,7 +122,7 @@ namespace :gitlab do end end - task restore: :environment do + task restore: :gitlab_environment do $progress.puts "Restoring builds ... ".color(:blue) Backup::Builds.new.restore $progress.puts "done".color(:green) @@ -130,7 +130,7 @@ namespace :gitlab do end namespace :uploads do - task create: :environment do + task create: :gitlab_environment do $progress.puts "Dumping uploads ... ".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("uploads") @@ -141,7 +141,7 @@ namespace :gitlab do end end - task restore: :environment do + task restore: :gitlab_environment do $progress.puts "Restoring uploads ... ".color(:blue) Backup::Uploads.new.restore $progress.puts "done".color(:green) @@ -149,7 +149,7 @@ namespace :gitlab do end namespace :artifacts do - task create: :environment do + task create: :gitlab_environment do $progress.puts "Dumping artifacts ... ".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("artifacts") @@ -160,7 +160,7 @@ namespace :gitlab do end end - task restore: :environment do + task restore: :gitlab_environment do $progress.puts "Restoring artifacts ... ".color(:blue) Backup::Artifacts.new.restore $progress.puts "done".color(:green) @@ -168,7 +168,7 @@ namespace :gitlab do end namespace :pages do - task create: :environment do + task create: :gitlab_environment do $progress.puts "Dumping pages ... ".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("pages") @@ -179,7 +179,7 @@ namespace :gitlab do end end - task restore: :environment do + task restore: :gitlab_environment do $progress.puts "Restoring pages ... ".color(:blue) Backup::Pages.new.restore $progress.puts "done".color(:green) @@ -187,7 +187,7 @@ namespace :gitlab do end namespace :lfs do - task create: :environment do + task create: :gitlab_environment do $progress.puts "Dumping lfs objects ... ".color(:blue) if ENV["SKIP"] && ENV["SKIP"].include?("lfs") @@ -198,7 +198,7 @@ namespace :gitlab do end end - task restore: :environment do + task restore: :gitlab_environment do $progress.puts "Restoring lfs objects ... ".color(:blue) Backup::Lfs.new.restore $progress.puts "done".color(:green) @@ -206,7 +206,7 @@ namespace :gitlab do end namespace :registry do - task create: :environment do + task create: :gitlab_environment do $progress.puts "Dumping container registry images ... ".color(:blue) if Gitlab.config.registry.enabled @@ -221,7 +221,7 @@ namespace :gitlab do end end - task restore: :environment do + task restore: :gitlab_environment do $progress.puts "Restoring container registry images ... ".color(:blue) if Gitlab.config.registry.enabled diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index a584eb97cf5..e05a3aad824 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -1,7 +1,3 @@ -# Temporary hack, until we migrate all checks to SystemCheck format -require 'system_check' -require 'system_check/helpers' - namespace :gitlab do desc 'GitLab | Check the configuration of GitLab and its environment' task check: %w{gitlab:gitlab_shell:check @@ -12,7 +8,7 @@ namespace :gitlab do namespace :app do desc 'GitLab | Check the configuration of the GitLab Rails app' - task check: :environment do + task check: :gitlab_environment do warn_user_is_not_gitlab checks = [ @@ -43,7 +39,7 @@ namespace :gitlab do namespace :gitlab_shell do desc "GitLab | Check the configuration of GitLab Shell" - task check: :environment do + task check: :gitlab_environment do warn_user_is_not_gitlab start_checking "GitLab Shell" @@ -251,7 +247,7 @@ namespace :gitlab do namespace :sidekiq do desc "GitLab | Check the configuration of Sidekiq" - task check: :environment do + task check: :gitlab_environment do warn_user_is_not_gitlab start_checking "Sidekiq" @@ -310,7 +306,7 @@ namespace :gitlab do namespace :incoming_email do desc "GitLab | Check the configuration of Reply by email" - task check: :environment do + task check: :gitlab_environment do warn_user_is_not_gitlab if Gitlab.config.incoming_email.enabled @@ -333,7 +329,7 @@ namespace :gitlab do end namespace :ldap do - task :check, [:limit] => :environment do |_, args| + task :check, [:limit] => :gitlab_environment do |_, args| # Only show up to 100 results because LDAP directories can be very big. # This setting only affects the `rake gitlab:check` script. args.with_defaults(limit: 100) @@ -389,7 +385,7 @@ namespace :gitlab do namespace :repo do desc "GitLab | Check the integrity of the repositories managed by GitLab" - task check: :environment do + task check: :gitlab_environment do puts "This task is deprecated. Please use gitlab:git:fsck instead".color(:red) Rake::Task["gitlab:git:fsck"].execute end @@ -397,7 +393,7 @@ namespace :gitlab do namespace :orphans do desc 'Gitlab | Check for orphaned namespaces and repositories' - task check: :environment do + task check: :gitlab_environment do warn_user_is_not_gitlab checks = [ SystemCheck::Orphans::NamespaceCheck, @@ -408,7 +404,7 @@ namespace :gitlab do end desc 'GitLab | Check for orphaned namespaces in the repositories path' - task check_namespaces: :environment do + task check_namespaces: :gitlab_environment do warn_user_is_not_gitlab checks = [SystemCheck::Orphans::NamespaceCheck] @@ -416,7 +412,7 @@ namespace :gitlab do end desc 'GitLab | Check for orphaned repositories in the repositories path' - task check_repositories: :environment do + task check_repositories: :gitlab_environment do warn_user_is_not_gitlab checks = [SystemCheck::Orphans::RepositoryCheck] @@ -426,7 +422,7 @@ namespace :gitlab do namespace :user do desc "GitLab | Check the integrity of a specific user's repositories" - task :check_repos, [:username] => :environment do |t, args| + task :check_repos, [:username] => :gitlab_environment do |t, args| username = args[:username] || prompt("Check repository integrity for username? ".color(:blue)) user = User.find_by(username: username) if user diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index ab601b0d66b..5a53eac0897 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -5,7 +5,7 @@ namespace :gitlab do HASHED_REPOSITORY_NAME = '@hashed'.freeze desc "GitLab | Cleanup | Clean namespaces" - task dirs: :environment do + task dirs: :gitlab_environment do warn_user_is_not_gitlab remove_flag = ENV['REMOVE'] @@ -49,7 +49,7 @@ namespace :gitlab do end desc "GitLab | Cleanup | Clean repositories" - task repos: :environment do + task repos: :gitlab_environment do warn_user_is_not_gitlab move_suffix = "+orphaned+#{Time.now.to_i}" @@ -78,7 +78,7 @@ namespace :gitlab do end desc "GitLab | Cleanup | Block users that have been removed in LDAP" - task block_removed_ldap_users: :environment do + task block_removed_ldap_users: :gitlab_environment do warn_user_is_not_gitlab block_flag = ENV['BLOCK'] @@ -109,7 +109,7 @@ namespace :gitlab do # released. So likely this should only be run once on gitlab.com # Faulty refs are moved so they are kept around, else some features break. desc 'GitLab | Cleanup | Remove faulty deployment refs' - task move_faulty_deployment_refs: :environment do + task move_faulty_deployment_refs: :gitlab_environment do projects = Project.where(id: Deployment.select(:project_id).distinct) projects.find_each do |project| diff --git a/lib/tasks/gitlab/git.rake b/lib/tasks/gitlab/git.rake index 3f5dd2ae3b3..cb4f7e5c8a8 100644 --- a/lib/tasks/gitlab/git.rake +++ b/lib/tasks/gitlab/git.rake @@ -1,7 +1,7 @@ namespace :gitlab do namespace :git do desc "GitLab | Git | Repack" - task repack: :environment do + task repack: :gitlab_environment do failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} repack -a --quiet), "Repacking repo") if failures.empty? puts "Done".color(:green) @@ -11,7 +11,7 @@ namespace :gitlab do end desc "GitLab | Git | Run garbage collection on all repos" - task gc: :environment do + task gc: :gitlab_environment do failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} gc --auto --quiet), "Garbage Collecting") if failures.empty? puts "Done".color(:green) @@ -21,7 +21,7 @@ namespace :gitlab do end desc "GitLab | Git | Prune all repos" - task prune: :environment do + task prune: :gitlab_environment do failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} prune), "Git Prune") if failures.empty? puts "Done".color(:green) @@ -31,7 +31,7 @@ namespace :gitlab do end desc 'GitLab | Git | Check all repos integrity' - task fsck: :environment do + task fsck: :gitlab_environment do failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} fsck --name-objects --no-progress), "Checking integrity") do |repo| check_config_lock(repo) check_ref_locks(repo) diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index a2e68c0471b..107ff1d8aeb 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -1,7 +1,7 @@ namespace :gitlab do namespace :gitaly do desc "GitLab | Install or upgrade gitaly" - task :install, [:dir, :repo] => :environment do |t, args| + task :install, [:dir, :repo] => :gitlab_environment do |t, args| require 'toml' warn_user_is_not_gitlab @@ -21,7 +21,11 @@ namespace :gitlab do _, status = Gitlab::Popen.popen(%w[which gmake]) command << (status.zero? ? 'gmake' : 'make') - command << 'BUNDLE_FLAGS=--no-deployment' if Rails.env.test? + if Rails.env.test? + command.push( + 'BUNDLE_FLAGS=--no-deployment', + "BUNDLE_PATH=#{Bundler.bundle_path}") + end Gitlab::SetupHelper.create_gitaly_configuration(args.dir) Dir.chdir(args.dir) do diff --git a/lib/tasks/gitlab/helpers.rake b/lib/tasks/gitlab/helpers.rake index b0a24790c4a..14d1125a03d 100644 --- a/lib/tasks/gitlab/helpers.rake +++ b/lib/tasks/gitlab/helpers.rake @@ -1,8 +1,6 @@ -require 'tasks/gitlab/task_helpers' - # Prevent StateMachine warnings from outputting during a cron task StateMachines::Machine.ignore_method_conflicts = true if ENV['CRON'] -namespace :gitlab do +task gitlab_environment: :environment do extend SystemCheck::Helpers end diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake index e9fb6a008b0..45e9a1a1c72 100644 --- a/lib/tasks/gitlab/info.rake +++ b/lib/tasks/gitlab/info.rake @@ -1,7 +1,7 @@ namespace :gitlab do namespace :env do desc "GitLab | Show information about GitLab and its environment" - task info: :environment do + task info: :gitlab_environment do # check if there is an RVM environment rvm_version = run_and_match(%w(rvm --version), /[\d\.]+/).try(:to_s) # check Ruby version diff --git a/lib/tasks/gitlab/setup.rake b/lib/tasks/gitlab/setup.rake index 05fcb8e3da5..1d903c81358 100644 --- a/lib/tasks/gitlab/setup.rake +++ b/lib/tasks/gitlab/setup.rake @@ -1,6 +1,6 @@ namespace :gitlab do desc "GitLab | Setup production application" - task setup: :environment do + task setup: :gitlab_environment do setup_db end diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index 12ae4199b69..844664b12d4 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -1,7 +1,7 @@ namespace :gitlab do namespace :shell do desc "GitLab | Install or upgrade gitlab-shell" - task :install, [:repo] => :environment do |t, args| + task :install, [:repo] => :gitlab_environment do |t, args| warn_user_is_not_gitlab default_version = Gitlab::Shell.version_required @@ -58,12 +58,12 @@ namespace :gitlab do end desc "GitLab | Setup gitlab-shell" - task setup: :environment do + task setup: :gitlab_environment do setup end desc "GitLab | Build missing projects" - task build_missing_projects: :environment do + task build_missing_projects: :gitlab_environment do Project.find_each(batch_size: 1000) do |project| path_to_repo = project.repository.path_to_repo if File.exist?(path_to_repo) @@ -80,7 +80,7 @@ namespace :gitlab do end desc 'Create or repair repository hooks symlink' - task create_hooks: :environment do + task create_hooks: :gitlab_environment do warn_user_is_not_gitlab puts 'Creating/Repairing hooks symlinks for all repositories' diff --git a/lib/tasks/gitlab/workhorse.rake b/lib/tasks/gitlab/workhorse.rake index 308ffb0e284..b917a293095 100644 --- a/lib/tasks/gitlab/workhorse.rake +++ b/lib/tasks/gitlab/workhorse.rake @@ -1,7 +1,7 @@ namespace :gitlab do namespace :workhorse do desc "GitLab | Install or upgrade gitlab-workhorse" - task :install, [:dir, :repo] => :environment do |t, args| + task :install, [:dir, :repo] => :gitlab_environment do |t, args| warn_user_is_not_gitlab unless args.dir.present? diff --git a/lib/tasks/haml-lint.rake b/lib/tasks/haml-lint.rake index ad2d034b0b4..5c0cc4990fc 100644 --- a/lib/tasks/haml-lint.rake +++ b/lib/tasks/haml-lint.rake @@ -2,5 +2,14 @@ unless Rails.env.production? require 'haml_lint/rake_task' require 'haml_lint/inline_javascript' + # Workaround for warnings from parser/current + # TODO: Remove this after we update parser gem + task :haml_lint do + require 'parser' + def Parser.warn(*args) + puts(*args) # static-analysis ignores stdout if status is 0 + end + end + HamlLint::RakeTask.new end diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake index c996537cfbe..31cbd651edb 100644 --- a/lib/tasks/migrate/setup_postgresql.rake +++ b/lib/tasks/migrate/setup_postgresql.rake @@ -1,16 +1,14 @@ -require Rails.root.join('lib/gitlab/database') -require Rails.root.join('lib/gitlab/database/migration_helpers') -require Rails.root.join('db/migrate/20151007120511_namespaces_projects_path_lower_indexes') -require Rails.root.join('db/migrate/20151008110232_add_users_lower_username_email_indexes') -require Rails.root.join('db/migrate/20161212142807_add_lower_path_index_to_routes') -require Rails.root.join('db/migrate/20170317203554_index_routes_path_for_like') -require Rails.root.join('db/migrate/20170724214302_add_lower_path_index_to_redirect_routes') -require Rails.root.join('db/migrate/20170503185032_index_redirect_routes_path_for_like') -require Rails.root.join('db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb') -require Rails.root.join('db/migrate/20180113220114_rework_redirect_routes_indexes.rb') - desc 'GitLab | Sets up PostgreSQL' task setup_postgresql: :environment do + require Rails.root.join('db/migrate/20151007120511_namespaces_projects_path_lower_indexes') + require Rails.root.join('db/migrate/20151008110232_add_users_lower_username_email_indexes') + require Rails.root.join('db/migrate/20161212142807_add_lower_path_index_to_routes') + require Rails.root.join('db/migrate/20170317203554_index_routes_path_for_like') + require Rails.root.join('db/migrate/20170724214302_add_lower_path_index_to_redirect_routes') + require Rails.root.join('db/migrate/20170503185032_index_redirect_routes_path_for_like') + require Rails.root.join('db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb') + require Rails.root.join('db/migrate/20180113220114_rework_redirect_routes_indexes.rb') + NamespacesProjectsPathLowerIndexes.new.up AddUsersLowerUsernameEmailIndexes.new.up AddLowerPathIndexToRoutes.new.up |