diff options
| author | Mike Greiling <mike@pixelcog.com> | 2018-02-06 13:28:13 -0600 |
|---|---|---|
| committer | Mike Greiling <mike@pixelcog.com> | 2018-02-06 13:28:13 -0600 |
| commit | 47f2754a14549ccd18e4be8b3b6604b378450d6d (patch) | |
| tree | f2235166a76bcf9e7761e1b613e63110961869da /lib | |
| parent | e71a27f082c49a8e132be632cb8fe97f810af987 (diff) | |
| parent | e776096e84d01ab3d1d07a028b65e7430e195114 (diff) | |
| download | gitlab-ce-47f2754a14549ccd18e4be8b3b6604b378450d6d.tar.gz | |
Merge branch 'master' into pawel/connect_to_prometheus_through_proxy-30480
* master: (242 commits)
Validate user namespace before saving so that errors persist on model
Reset Project's column information in spec/lib/gitlab/background_migration/populate_merge_request_metrics_with_events_data_spec.rb
Explicitly set cwd in Sidekiq memory killer instead of depending on getcwd
Downgrade google-protobuf
Close low level rugged repository in project cache worker
File upload UI obeys LFS filters
Resolve "Add a link to documentation on how to get external ip in the Kubernetes cluster details page"
Upgrade GitLab Workhorse to v3.6.0
Add sorting options for /users API (admin only)
improvements from feedback
[ci-skip] add changelog
remove file after `Upload#destroy`
Fix a hardcoded pipeline ID in a spinach step
Override group sidebar links
Replace "cluster" with "Kubernetes cluster"
Reorder async/sync tasks in BuildFinishedWorker to read traces efficiently
Fix tests for Drop filename enforcement
Revert using expand_fixture_path in factory
Revert "Add FixtureHelpers for FactoryGirl"
Refactor :trace to :trace_live in spec
...
Diffstat (limited to 'lib')
102 files changed, 871 insertions, 339 deletions
diff --git a/lib/api/api_guard.rb b/lib/api/api_guard.rb index 9aeebc34525..c2113551207 100644 --- a/lib/api/api_guard.rb +++ b/lib/api/api_guard.rb @@ -42,7 +42,7 @@ module API include Gitlab::Auth::UserAuthFinders def find_current_user! - user = find_user_from_access_token || find_user_from_warden + user = find_user_from_sources return unless user forbidden!('User is blocked') unless Gitlab::UserAccess.new(user).allowed? && user.can?(:access_api) @@ -50,6 +50,10 @@ module API user end + def find_user_from_sources + find_user_from_access_token || find_user_from_warden + end + private # An array of scopes that were registered (using `allow_access_with_scope`) diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 0791a110c39..1794207e29b 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -29,6 +29,8 @@ module API use :pagination end get ':id/repository/branches' do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42329') + repository = user_project.repository branches = ::Kaminari.paginate_array(repository.branches.sort_by(&:name)) merged_branch_names = repository.merged_branch_names(branches.map(&:name)) diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb index 2cae53dba53..3d0d1287407 100644 --- a/lib/api/helpers/runner.rb +++ b/lib/api/helpers/runner.rb @@ -1,15 +1,13 @@ module API module Helpers module Runner - include Gitlab::CurrentSettings - JOB_TOKEN_HEADER = 'HTTP_JOB_TOKEN'.freeze JOB_TOKEN_PARAM = :token UPDATE_RUNNER_EVERY = 10 * 60 def runner_registration_token_valid? ActiveSupport::SecurityUtils.variable_size_secure_compare(params[:token], - current_application_settings.runners_registration_token) + Gitlab::CurrentSettings.runners_registration_token) end def get_runner_version_from_params @@ -70,7 +68,7 @@ module API end def max_artifacts_size - current_application_settings.max_artifacts_size.megabytes.to_i + Gitlab::CurrentSettings.max_artifacts_size.megabytes.to_i end end end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index c99fe3ab5b3..b6c278c89d0 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -161,6 +161,8 @@ module API use :issue_params end post ':id/issues' do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42320') + authorize! :create_issue, user_project # Setting created_at time only allowed for admins and project owners @@ -201,6 +203,8 @@ module API :labels, :created_at, :due_date, :confidential, :state_event end put ':id/issues/:issue_iid' do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42322') + issue = user_project.issues.find_by!(iid: params.delete(:issue_iid)) authorize! :update_issue, issue @@ -234,6 +238,8 @@ module API requires :to_project_id, type: Integer, desc: 'The ID of the new project' end post ':id/issues/:issue_iid/move' do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42323') + issue = user_project.issues.find_by(iid: params[:issue_iid]) not_found!('Issue') unless issue diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 420aaf1c964..719afa09295 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -152,6 +152,8 @@ module API use :optional_params end post ":id/merge_requests" do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42316') + authorize! :create_merge_request, user_project mr_params = declared_params(include_missing: false) @@ -256,6 +258,8 @@ module API at_least_one_of(*at_least_one_of_ce) end put ':id/merge_requests/:merge_request_iid' do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42318') + merge_request = find_merge_request_with_access(params.delete(:merge_request_iid), :update_merge_request) mr_params = declared_params(include_missing: false) @@ -283,6 +287,8 @@ module API optional :sha, type: String, desc: 'When present, must have the HEAD SHA of the source branch' end put ':id/merge_requests/:merge_request_iid/merge' do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42317') + merge_request = find_project_merge_request(params[:merge_request_iid]) merge_when_pipeline_succeeds = to_boolean(params[:merge_when_pipeline_succeeds]) diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb index 675c963bae2..d2b8b832e4e 100644 --- a/lib/api/pipelines.rb +++ b/lib/api/pipelines.rb @@ -42,6 +42,8 @@ module API requires :ref, type: String, desc: 'Reference' end post ':id/pipeline' do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42124') + authorize! :create_pipeline, user_project new_pipeline = Ci::CreatePipelineService.new(user_project, diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 8b5e4f8edcc..5b481121a10 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -210,6 +210,8 @@ module API optional :namespace, type: String, desc: 'The ID or name of the namespace that the project will be forked into' end post ':id/fork' do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42284') + fork_params = declared_params(include_missing: false) namespace_id = fork_params[:namespace] diff --git a/lib/api/runner.rb b/lib/api/runner.rb index 80feb629d54..1f80646a2ea 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -215,9 +215,9 @@ module API job = authenticate_job! forbidden!('Job is not running!') unless job.running? - artifacts_upload_path = JobArtifactUploader.artifacts_upload_path - artifacts = uploaded_file(:file, artifacts_upload_path) - metadata = uploaded_file(:metadata, artifacts_upload_path) + workhorse_upload_path = JobArtifactUploader.workhorse_upload_path + artifacts = uploaded_file(:file, workhorse_upload_path) + metadata = uploaded_file(:metadata, workhorse_upload_path) bad_request!('Missing artifacts file!') unless artifacts file_to_large! unless artifacts.size < max_artifacts_size diff --git a/lib/api/templates.rb b/lib/api/templates.rb index 6550b331fb8..41862768a3f 100644 --- a/lib/api/templates.rb +++ b/lib/api/templates.rb @@ -17,15 +17,15 @@ module API } }.freeze PROJECT_TEMPLATE_REGEX = - /[\<\{\[] + %r{[\<\{\[] (project|description| one\sline\s.+\swhat\sit\sdoes\.) # matching the start and end is enough here - [\>\}\]]/xi.freeze + [\>\}\]]}xi.freeze YEAR_TEMPLATE_REGEX = /[<{\[](year|yyyy)[>}\]]/i.freeze FULLNAME_TEMPLATE_REGEX = - /[\<\{\[] + %r{[\<\{\[] (fullname|name\sof\s(author|copyright\sowner)) - [\>\}\]]/xi.freeze + [\>\}\]]}xi.freeze helpers do def parsed_license_template diff --git a/lib/api/triggers.rb b/lib/api/triggers.rb index dd6801664b1..b3709455bc3 100644 --- a/lib/api/triggers.rb +++ b/lib/api/triggers.rb @@ -15,6 +15,8 @@ module API optional :variables, type: Hash, desc: 'The list of variables to be injected into build' end post ":id/(ref/:ref/)trigger/pipeline", requirements: { ref: /.+/ } do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42283') + # validate variables params[:variables] = params[:variables].to_h unless params[:variables].all? { |key, value| key.is_a?(String) && value.is_a?(String) } diff --git a/lib/api/users.rb b/lib/api/users.rb index e5de31ad51b..3cc12724b8a 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -18,6 +18,14 @@ module API User.find_by(id: id) || not_found!('User') end + def reorder_users(users) + if params[:order_by] && params[:sort] + users.reorder(params[:order_by] => params[:sort]) + else + users + end + end + params :optional_attributes do optional :skype, type: String, desc: 'The Skype username' optional :linkedin, type: String, desc: 'The LinkedIn username' @@ -35,6 +43,13 @@ module API optional :avatar, type: File, desc: 'Avatar image for user' all_or_none_of :extern_uid, :provider end + + params :sort_params do + optional :order_by, type: String, values: %w[id name username created_at updated_at], + default: 'id', desc: 'Return users ordered by a field' + optional :sort, type: String, values: %w[asc desc], default: 'desc', + desc: 'Return users sorted in ascending and descending order' + end end desc 'Get the list of users' do @@ -53,16 +68,18 @@ module API optional :created_before, type: DateTime, desc: 'Return users created before the specified time' all_or_none_of :extern_uid, :provider + use :sort_params use :pagination end get do authenticated_as_admin! if params[:external].present? || (params[:extern_uid].present? && params[:provider].present?) unless current_user&.admin? - params.except!(:created_after, :created_before) + params.except!(:created_after, :created_before, :order_by, :sort) end users = UsersFinder.new(current_user, params).execute + users = reorder_users(users) authorized = can?(current_user, :read_users_list) @@ -383,6 +400,8 @@ module API optional :hard_delete, type: Boolean, desc: "Whether to remove a user's contributions" end delete ":id" do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42279') + authenticated_as_admin! user = User.find_by(id: params[:id]) diff --git a/lib/api/v3/branches.rb b/lib/api/v3/branches.rb index b201bf77667..25176c5b38e 100644 --- a/lib/api/v3/branches.rb +++ b/lib/api/v3/branches.rb @@ -14,6 +14,8 @@ module API success ::API::Entities::Branch end get ":id/repository/branches" do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42276') + repository = user_project.repository branches = repository.branches.sort_by(&:name) merged_branch_names = repository.merged_branch_names(branches.map(&:name)) diff --git a/lib/api/v3/issues.rb b/lib/api/v3/issues.rb index cb371fdbab8..b59947d81d9 100644 --- a/lib/api/v3/issues.rb +++ b/lib/api/v3/issues.rb @@ -134,6 +134,8 @@ module API use :issue_params end post ':id/issues' do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42131') + # Setting created_at time only allowed for admins and project owners unless current_user.admin? || user_project.owner == current_user params.delete(:created_at) @@ -169,6 +171,8 @@ module API :labels, :created_at, :due_date, :confidential, :state_event end put ':id/issues/:issue_id' do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42132') + issue = user_project.issues.find(params.delete(:issue_id)) authorize! :update_issue, issue @@ -201,6 +205,8 @@ module API requires :to_project_id, type: Integer, desc: 'The ID of the new project' end post ':id/issues/:issue_id/move' do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42133') + issue = user_project.issues.find_by(id: params[:issue_id]) not_found!('Issue') unless issue diff --git a/lib/api/v3/merge_requests.rb b/lib/api/v3/merge_requests.rb index 0a24fea52a3..ce216497996 100644 --- a/lib/api/v3/merge_requests.rb +++ b/lib/api/v3/merge_requests.rb @@ -91,6 +91,8 @@ module API use :optional_params end post ":id/merge_requests" do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42126') + authorize! :create_merge_request, user_project mr_params = declared_params(include_missing: false) @@ -167,6 +169,8 @@ module API :remove_source_branch end put path do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42127') + merge_request = find_merge_request_with_access(params.delete(:merge_request_id), :update_merge_request) mr_params = declared_params(include_missing: false) diff --git a/lib/api/v3/pipelines.rb b/lib/api/v3/pipelines.rb index c48cbd2b765..6d31c12f572 100644 --- a/lib/api/v3/pipelines.rb +++ b/lib/api/v3/pipelines.rb @@ -19,6 +19,8 @@ module API desc: 'Either running, branches, or tags' end get ':id/pipelines' do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42123') + authorize! :read_pipeline, user_project pipelines = PipelinesFinder.new(user_project, scope: params[:scope]).execute diff --git a/lib/api/v3/projects.rb b/lib/api/v3/projects.rb index a7f0813bf74..c856ba99f09 100644 --- a/lib/api/v3/projects.rb +++ b/lib/api/v3/projects.rb @@ -173,7 +173,7 @@ module API use :sort_params use :pagination end - get "/search/:query", requirements: { query: /[^\/]+/ } do + get "/search/:query", requirements: { query: %r{[^/]+} } do search_service = Search::GlobalService.new(current_user, search: params[:query]).execute projects = search_service.objects('projects', params[:page], false) projects = projects.reorder(params[:order_by] => params[:sort]) diff --git a/lib/api/v3/templates.rb b/lib/api/v3/templates.rb index 7298203df10..b82b02b5f49 100644 --- a/lib/api/v3/templates.rb +++ b/lib/api/v3/templates.rb @@ -16,15 +16,15 @@ module API } }.freeze PROJECT_TEMPLATE_REGEX = - /[\<\{\[] + %r{[\<\{\[] (project|description| one\sline\s.+\swhat\sit\sdoes\.) # matching the start and end is enough here - [\>\}\]]/xi.freeze + [\>\}\]]}xi.freeze YEAR_TEMPLATE_REGEX = /[<{\[](year|yyyy)[>}\]]/i.freeze FULLNAME_TEMPLATE_REGEX = - /[\<\{\[] + %r{[\<\{\[] (fullname|name\sof\s(author|copyright\sowner)) - [\>\}\]]/xi.freeze + [\>\}\]]}xi.freeze DEPRECATION_MESSAGE = ' This endpoint is deprecated and has been removed in V4.'.freeze helpers do diff --git a/lib/api/v3/triggers.rb b/lib/api/v3/triggers.rb index 534911fde5c..34f07dfb486 100644 --- a/lib/api/v3/triggers.rb +++ b/lib/api/v3/triggers.rb @@ -16,6 +16,8 @@ module API optional :variables, type: Hash, desc: 'The list of variables to be injected into build' end post ":id/(ref/:ref/)trigger/builds", requirements: { ref: /.+/ } do + Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42121') + # validate variables params[:variables] = params[:variables].to_h unless params[:variables].all? { |key, value| key.is_a?(String) && value.is_a?(String) } diff --git a/lib/backup/artifacts.rb b/lib/backup/artifacts.rb index 7a582a20056..4383124d150 100644 --- a/lib/backup/artifacts.rb +++ b/lib/backup/artifacts.rb @@ -3,7 +3,7 @@ require 'backup/files' module Backup class Artifacts < Files def initialize - super('artifacts', LegacyArtifactUploader.local_store_path) + super('artifacts', JobArtifactUploader.root) end def create_files_dir diff --git a/lib/banzai/color_parser.rb b/lib/banzai/color_parser.rb new file mode 100644 index 00000000000..355c364b07b --- /dev/null +++ b/lib/banzai/color_parser.rb @@ -0,0 +1,44 @@ +module Banzai + module ColorParser + ALPHA = /0(?:\.\d+)?|\.\d+|1(?:\.0+)?/ # 0.0..1.0 + PERCENTS = /(?:\d{1,2}|100)%/ # 00%..100% + ALPHA_CHANNEL = /(?:,\s*(?:#{ALPHA}|#{PERCENTS}))?/ + BITS = /\d{1,2}|1\d\d|2(?:[0-4]\d|5[0-5])/ # 00..255 + DEGS = /-?\d+(?:deg)?/i # [-]digits[deg] + RADS = /-?(?:\d+(?:\.\d+)?|\.\d+)rad/i # [-](digits[.digits] OR .digits)rad + HEX_FORMAT = /\#(?:\h{3}|\h{4}|\h{6}|\h{8})/ + RGB_FORMAT = %r{ + (?:rgba? + \( + (?: + (?:(?:#{BITS},\s*){2}#{BITS}) + | + (?:(?:#{PERCENTS},\s*){2}#{PERCENTS}) + ) + #{ALPHA_CHANNEL} + \) + ) + }xi + HSL_FORMAT = %r{ + (?:hsla? + \( + (?:#{DEGS}|#{RADS}),\s*#{PERCENTS},\s*#{PERCENTS} + #{ALPHA_CHANNEL} + \) + ) + }xi + + FORMATS = [HEX_FORMAT, RGB_FORMAT, HSL_FORMAT].freeze + + COLOR_FORMAT = /\A(#{Regexp.union(FORMATS)})\z/ix + + # Public: Analyzes whether the String is a color code. + # + # text - The String to be parsed. + # + # Returns the recognized color String or nil if none was found. + def self.parse(text) + text if COLOR_FORMAT =~ text + end + end +end diff --git a/lib/banzai/filter/color_filter.rb b/lib/banzai/filter/color_filter.rb new file mode 100644 index 00000000000..6ab29ac281f --- /dev/null +++ b/lib/banzai/filter/color_filter.rb @@ -0,0 +1,31 @@ +module Banzai + module Filter + # HTML filter that renders `color` followed by a color "chip". + # + class ColorFilter < HTML::Pipeline::Filter + COLOR_CHIP_CLASS = 'gfm-color_chip'.freeze + + def call + doc.css('code').each do |node| + color = ColorParser.parse(node.content) + node << color_chip(color) if color + end + + doc + end + + private + + def color_chip(color) + checkerboard = doc.document.create_element('span', class: COLOR_CHIP_CLASS) + chip = doc.document.create_element('span', style: inline_styles(color: color)) + + checkerboard << chip + end + + def inline_styles(color:) + "background-color: #{color};" + end + end + end +end diff --git a/lib/banzai/filter/emoji_filter.rb b/lib/banzai/filter/emoji_filter.rb index 6255a611dbe..b82c6ca6393 100644 --- a/lib/banzai/filter/emoji_filter.rb +++ b/lib/banzai/filter/emoji_filter.rb @@ -54,9 +54,9 @@ module Banzai # Build a regexp that matches all valid :emoji: names. def self.emoji_pattern @emoji_pattern ||= - /(?<=[^[:alnum:]:]|\n|^) + %r{(?<=[^[:alnum:]:]|\n|^) :(#{Gitlab::Emoji.emojis_names.map { |name| Regexp.escape(name) }.join('|')}): - (?=[^[:alnum:]:]|$)/x + (?=[^[:alnum:]:]|$)}x end # Build a regexp that matches all valid unicode emojis names. diff --git a/lib/banzai/filter/gollum_tags_filter.rb b/lib/banzai/filter/gollum_tags_filter.rb index 2e259904673..c2b42673376 100644 --- a/lib/banzai/filter/gollum_tags_filter.rb +++ b/lib/banzai/filter/gollum_tags_filter.rb @@ -51,10 +51,10 @@ module Banzai # See https://github.com/gollum/gollum/wiki # # Rubular: http://rubular.com/r/7dQnE5CUCH - TAGS_PATTERN = %r{\[\[(.+?)\]\]}.freeze + TAGS_PATTERN = /\[\[(.+?)\]\]/.freeze # Pattern to match allowed image extensions - ALLOWED_IMAGE_EXTENSIONS = %r{.+(jpg|png|gif|svg|bmp)\z}i.freeze + ALLOWED_IMAGE_EXTENSIONS = /.+(jpg|png|gif|svg|bmp)\z/i.freeze def call search_text_nodes(doc).each do |node| diff --git a/lib/banzai/pipeline/broadcast_message_pipeline.rb b/lib/banzai/pipeline/broadcast_message_pipeline.rb index adc09c8afbd..5dd572de3a1 100644 --- a/lib/banzai/pipeline/broadcast_message_pipeline.rb +++ b/lib/banzai/pipeline/broadcast_message_pipeline.rb @@ -7,6 +7,7 @@ module Banzai Filter::SanitizationFilter, Filter::EmojiFilter, + Filter::ColorFilter, Filter::AutolinkFilter, Filter::ExternalLinkFilter ] diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index c746f6f64e9..4001b8a85e3 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -14,6 +14,7 @@ module Banzai Filter::SyntaxHighlightFilter, Filter::MathFilter, + Filter::ColorFilter, Filter::MermaidFilter, Filter::VideoLinkFilter, Filter::ImageLazyLoadFilter, diff --git a/lib/carrier_wave_string_file.rb b/lib/carrier_wave_string_file.rb new file mode 100644 index 00000000000..6c848902e4a --- /dev/null +++ b/lib/carrier_wave_string_file.rb @@ -0,0 +1,5 @@ +class CarrierWaveStringFile < StringIO + def original_filename + "" + end +end diff --git a/lib/container_registry/registry.rb b/lib/container_registry/registry.rb index 63bce655f57..f90d711474a 100644 --- a/lib/container_registry/registry.rb +++ b/lib/container_registry/registry.rb @@ -11,7 +11,7 @@ module ContainerRegistry private def default_path - @uri.sub(/^https?:\/\//, '') + @uri.sub(%r{^https?://}, '') end end end diff --git a/lib/email_template_interceptor.rb b/lib/email_template_interceptor.rb index f2bf3d0fb2b..3978a6d9fe4 100644 --- a/lib/email_template_interceptor.rb +++ b/lib/email_template_interceptor.rb @@ -1,10 +1,8 @@ # Read about interceptors in http://guides.rubyonrails.org/action_mailer_basics.html#intercepting-emails class EmailTemplateInterceptor - extend Gitlab::CurrentSettings - def self.delivering_email(message) # Remove HTML part if HTML emails are disabled. - unless current_application_settings.html_emails_enabled + unless Gitlab::CurrentSettings.html_emails_enabled message.parts.delete_if do |part| part.content_type.start_with?('text/html') end diff --git a/lib/extracts_path.rb b/lib/extracts_path.rb index d8aca3304c5..a9b04c183ad 100644 --- a/lib/extracts_path.rb +++ b/lib/extracts_path.rb @@ -56,7 +56,7 @@ module ExtractsPath if valid_refs.length == 0 # No exact ref match, so just try our best - pair = id.match(/([^\/]+)(.*)/).captures + pair = id.match(%r{([^/]+)(.*)}).captures else # There is a distinct possibility that multiple refs prefix the ID. # Use the longest match to maximize the chance that we have the @@ -68,7 +68,7 @@ module ExtractsPath end # Remove ending slashes from path - pair[1].gsub!(/^\/|\/$/, '') + pair[1].gsub!(%r{^/|/$}, '') pair end diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb index cead1c7eacd..ee7f4be6b9f 100644 --- a/lib/gitlab/asciidoc.rb +++ b/lib/gitlab/asciidoc.rb @@ -6,8 +6,6 @@ module Gitlab # Parser/renderer for the AsciiDoc format that uses Asciidoctor and filters # the resulting HTML through HTML pipeline filters. module Asciidoc - extend Gitlab::CurrentSettings - DEFAULT_ADOC_ATTRS = [ 'showtitle', 'idprefix=user-content-', 'idseparator=-', 'env=gitlab', 'env-gitlab', 'source-highlighter=html-pipeline', 'icons=font' @@ -33,9 +31,9 @@ module Gitlab def self.plantuml_setup Asciidoctor::PlantUml.configure do |conf| - conf.url = current_application_settings.plantuml_url - conf.svg_enable = current_application_settings.plantuml_enabled - conf.png_enable = current_application_settings.plantuml_enabled + conf.url = Gitlab::CurrentSettings.plantuml_url + conf.svg_enable = Gitlab::CurrentSettings.plantuml_enabled + conf.png_enable = Gitlab::CurrentSettings.plantuml_enabled conf.txt_enable = false end end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 65d7fd3ec70..05932378173 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -14,8 +14,6 @@ module Gitlab DEFAULT_SCOPES = [:api].freeze class << self - include Gitlab::CurrentSettings - def find_for_git_client(login, password, project:, ip:) raise "Must provide an IP for rate limiting" if ip.nil? @@ -57,7 +55,7 @@ module Gitlab if user.nil? || user.ldap_user? # Second chance - try LDAP authentication Gitlab::LDAP::Authentication.login(login, password) - elsif current_application_settings.password_authentication_enabled_for_git? + elsif Gitlab::CurrentSettings.password_authentication_enabled_for_git? user if user.active? && user.valid_password?(password) end end @@ -87,7 +85,7 @@ module Gitlab private def authenticate_using_internal_or_ldap_password? - current_application_settings.password_authentication_enabled_for_git? || Gitlab::LDAP::Config.enabled? + Gitlab::CurrentSettings.password_authentication_enabled_for_git? || Gitlab::LDAP::Config.enabled? end def service_request_check(login, password, project) diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index 81e95e5832d..8a8e770940e 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -12,7 +12,7 @@ module Gitlab # Ends with /:random_hex/:filename FILE_UPLOADER_PATH = %r{/\h+/[^/]+\z} - FULL_PATH_CAPTURE = %r{\A(.+)#{FILE_UPLOADER_PATH}} + FULL_PATH_CAPTURE = /\A(.+)#{FILE_UPLOADER_PATH}/ # These regex patterns are tested against a relative path, relative to # the upload directory. @@ -143,7 +143,7 @@ module Gitlab end def absolute_path - File.join(CarrierWave.root, path) + File.join(Gitlab.config.uploads.storage_path, path) end end diff --git a/lib/gitlab/background_migration/prepare_untracked_uploads.rb b/lib/gitlab/background_migration/prepare_untracked_uploads.rb index 4e0121ca34d..a7a1bbe1752 100644 --- a/lib/gitlab/background_migration/prepare_untracked_uploads.rb +++ b/lib/gitlab/background_migration/prepare_untracked_uploads.rb @@ -11,9 +11,12 @@ module Gitlab FIND_BATCH_SIZE = 500 RELATIVE_UPLOAD_DIR = "uploads".freeze - ABSOLUTE_UPLOAD_DIR = "#{CarrierWave.root}/#{RELATIVE_UPLOAD_DIR}".freeze + ABSOLUTE_UPLOAD_DIR = File.join( + Gitlab.config.uploads.storage_path, + RELATIVE_UPLOAD_DIR + ) FOLLOW_UP_MIGRATION = 'PopulateUntrackedUploads'.freeze - START_WITH_CARRIERWAVE_ROOT_REGEX = %r{\A#{CarrierWave.root}/} + START_WITH_ROOT_REGEX = %r{\A#{Gitlab.config.uploads.storage_path}/} EXCLUDED_HASHED_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/@hashed/*".freeze EXCLUDED_TMP_UPLOADS_PATH = "#{ABSOLUTE_UPLOAD_DIR}/tmp/*".freeze @@ -81,7 +84,7 @@ module Gitlab paths = [] stdout.each_line("\0") do |line| - paths << line.chomp("\0").sub(START_WITH_CARRIERWAVE_ROOT_REGEX, '') + paths << line.chomp("\0").sub(START_WITH_ROOT_REGEX, '') if paths.size >= batch_size yield(paths) diff --git a/lib/gitlab/badge/coverage/report.rb b/lib/gitlab/badge/coverage/report.rb index 9a0482306b7..778d78185ff 100644 --- a/lib/gitlab/badge/coverage/report.rb +++ b/lib/gitlab/badge/coverage/report.rb @@ -23,7 +23,7 @@ module Gitlab @coverage ||= raw_coverage return unless @coverage - @coverage.to_i + @coverage.to_f.round(2) end def metadata diff --git a/lib/gitlab/badge/coverage/template.rb b/lib/gitlab/badge/coverage/template.rb index fcecb1d9665..afbf9dd17e3 100644 --- a/lib/gitlab/badge/coverage/template.rb +++ b/lib/gitlab/badge/coverage/template.rb @@ -25,7 +25,7 @@ module Gitlab end def value_text - @status ? "#{@status}%" : 'unknown' + @status ? ("%.2f%%" % @status) : 'unknown' end def key_width @@ -33,7 +33,7 @@ module Gitlab end def value_width - @status ? 36 : 58 + @status ? 54 : 58 end def value_color diff --git a/lib/gitlab/checks/force_push.rb b/lib/gitlab/checks/force_push.rb index dc5d285ea65..c9c3050cfc2 100644 --- a/lib/gitlab/checks/force_push.rb +++ b/lib/gitlab/checks/force_push.rb @@ -15,8 +15,8 @@ module Gitlab .ancestor?(oldrev, newrev) else Gitlab::Git::RevList.new( - path_to_repo: project.repository.path_to_repo, - oldrev: oldrev, newrev: newrev).missed_ref.present? + project.repository.raw, oldrev: oldrev, newrev: newrev + ).missed_ref.present? end end end diff --git a/lib/gitlab/ci/build/artifacts/metadata/entry.rb b/lib/gitlab/ci/build/artifacts/metadata/entry.rb index 5b2f09e03ea..428c0505808 100644 --- a/lib/gitlab/ci/build/artifacts/metadata/entry.rb +++ b/lib/gitlab/ci/build/artifacts/metadata/entry.rb @@ -97,7 +97,7 @@ module Gitlab end def total_size - descendant_pattern = %r{^#{Regexp.escape(@path.to_s)}} + descendant_pattern = /^#{Regexp.escape(@path.to_s)}/ entries.sum do |path, entry| (entry[:size] if path =~ descendant_pattern).to_i end diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index baf55b1fa07..f2e5124c8a8 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -52,12 +52,14 @@ module Gitlab end def exist? - current_path.present? || old_trace.present? + trace_artifact&.exists? || current_path.present? || old_trace.present? end def read stream = Gitlab::Ci::Trace::Stream.new do - if current_path + if trace_artifact + trace_artifact.open + elsif current_path File.open(current_path, "rb") elsif old_trace StringIO.new(old_trace) @@ -82,6 +84,8 @@ module Gitlab end def erase! + trace_artifact&.destroy + paths.each do |trace_path| FileUtils.rm(trace_path, force: true) end @@ -137,6 +141,10 @@ module Gitlab "#{job.id}.log" ) if job.project&.ci_id end + + def trace_artifact + job.job_artifacts_trace + end end end end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 91fd9cc7631..b7c596a973d 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -1,73 +1,79 @@ module Gitlab module CurrentSettings - extend self + class << self + def current_application_settings + if RequestStore.active? + RequestStore.fetch(:current_application_settings) { ensure_application_settings! } + else + ensure_application_settings! + end + end - def current_application_settings - if RequestStore.active? - RequestStore.fetch(:current_application_settings) { ensure_application_settings! } - else - ensure_application_settings! + def fake_application_settings(defaults = ::ApplicationSetting.defaults) + Gitlab::FakeApplicationSettings.new(defaults) end - end - delegate :sidekiq_throttling_enabled?, to: :current_application_settings + def method_missing(name, *args, &block) + current_application_settings.send(name, *args, &block) # rubocop:disable GitlabSecurity/PublicSend + end - def fake_application_settings(defaults = ::ApplicationSetting.defaults) - FakeApplicationSettings.new(defaults) - end + def respond_to_missing?(name, include_private = false) + current_application_settings.respond_to?(name, include_private) || super + end - private + private - def ensure_application_settings! - return in_memory_application_settings if ENV['IN_MEMORY_APPLICATION_SETTINGS'] == 'true' + def ensure_application_settings! + return in_memory_application_settings if ENV['IN_MEMORY_APPLICATION_SETTINGS'] == 'true' - cached_application_settings || uncached_application_settings - end + cached_application_settings || uncached_application_settings + end - def cached_application_settings - begin - ::ApplicationSetting.cached - rescue ::Redis::BaseError, ::Errno::ENOENT, ::Errno::EADDRNOTAVAIL - # In case Redis isn't running or the Redis UNIX socket file is not available + def cached_application_settings + begin + ::ApplicationSetting.cached + rescue ::Redis::BaseError, ::Errno::ENOENT, ::Errno::EADDRNOTAVAIL + # In case Redis isn't running or the Redis UNIX socket file is not available + end end - end - def uncached_application_settings - return fake_application_settings unless connect_to_db? + def uncached_application_settings + return fake_application_settings unless connect_to_db? - db_settings = ::ApplicationSetting.current + db_settings = ::ApplicationSetting.current - # If there are pending migrations, it's possible there are columns that - # need to be added to the application settings. To prevent Rake tasks - # and other callers from failing, use any loaded settings and return - # defaults for missing columns. - if ActiveRecord::Migrator.needs_migration? - defaults = ::ApplicationSetting.defaults - defaults.merge!(db_settings.attributes.symbolize_keys) if db_settings.present? - return fake_application_settings(defaults) - end + # If there are pending migrations, it's possible there are columns that + # need to be added to the application settings. To prevent Rake tasks + # and other callers from failing, use any loaded settings and return + # defaults for missing columns. + if ActiveRecord::Migrator.needs_migration? + defaults = ::ApplicationSetting.defaults + defaults.merge!(db_settings.attributes.symbolize_keys) if db_settings.present? + return fake_application_settings(defaults) + end - return db_settings if db_settings.present? + return db_settings if db_settings.present? - ::ApplicationSetting.create_from_defaults || in_memory_application_settings - end + ::ApplicationSetting.create_from_defaults || in_memory_application_settings + end - def in_memory_application_settings - @in_memory_application_settings ||= ::ApplicationSetting.new(::ApplicationSetting.defaults) # rubocop:disable Gitlab/ModuleWithInstanceVariables - rescue ActiveRecord::StatementInvalid, ActiveRecord::UnknownAttributeError - # In case migrations the application_settings table is not created yet, - # we fallback to a simple OpenStruct - fake_application_settings - end + def in_memory_application_settings + @in_memory_application_settings ||= ::ApplicationSetting.new(::ApplicationSetting.defaults) # rubocop:disable Gitlab/ModuleWithInstanceVariables + rescue ActiveRecord::StatementInvalid, ActiveRecord::UnknownAttributeError + # In case migrations the application_settings table is not created yet, + # we fallback to a simple OpenStruct + fake_application_settings + end - def connect_to_db? - # When the DBMS is not available, an exception (e.g. PG::ConnectionBad) is raised - active_db_connection = ActiveRecord::Base.connection.active? rescue false + def connect_to_db? + # When the DBMS is not available, an exception (e.g. PG::ConnectionBad) is raised + active_db_connection = ActiveRecord::Base.connection.active? rescue false - active_db_connection && - ActiveRecord::Base.connection.table_exists?('application_settings') - rescue ActiveRecord::NoDatabaseError - false + active_db_connection && + ActiveRecord::Base.connection.table_exists?('application_settings') + rescue ActiveRecord::NoDatabaseError + false + end end end end diff --git a/lib/gitlab/dependency_linker/composer_json_linker.rb b/lib/gitlab/dependency_linker/composer_json_linker.rb index 0245bf4077a..cfd4ec15125 100644 --- a/lib/gitlab/dependency_linker/composer_json_linker.rb +++ b/lib/gitlab/dependency_linker/composer_json_linker.rb @@ -11,7 +11,7 @@ module Gitlab end def package_url(name) - "https://packagist.org/packages/#{name}" if name =~ %r{\A#{REPO_REGEX}\z} + "https://packagist.org/packages/#{name}" if name =~ /\A#{REPO_REGEX}\z/ end end end diff --git a/lib/gitlab/dependency_linker/gemfile_linker.rb b/lib/gitlab/dependency_linker/gemfile_linker.rb index d034ea67387..bfea836bcb2 100644 --- a/lib/gitlab/dependency_linker/gemfile_linker.rb +++ b/lib/gitlab/dependency_linker/gemfile_linker.rb @@ -15,7 +15,7 @@ module Gitlab link_regex(/(github:|:github\s*=>)\s*['"](?<name>[^'"]+)['"]/, &method(:github_url)) # Link `git: "https://gitlab.example.com/user/repo"` to https://gitlab.example.com/user/repo - link_regex(%r{(git:|:git\s*=>)\s*['"](?<name>#{URL_REGEX})['"]}, &:itself) + link_regex(/(git:|:git\s*=>)\s*['"](?<name>#{URL_REGEX})['"]/, &:itself) # Link `source "https://rubygems.org"` to https://rubygems.org link_method_call('source', URL_REGEX, &:itself) diff --git a/lib/gitlab/dependency_linker/podspec_linker.rb b/lib/gitlab/dependency_linker/podspec_linker.rb index a52c7a02439..924e55e9820 100644 --- a/lib/gitlab/dependency_linker/podspec_linker.rb +++ b/lib/gitlab/dependency_linker/podspec_linker.rb @@ -12,7 +12,7 @@ module Gitlab def link_dependencies link_method_call('homepage', URL_REGEX, &:itself) - link_regex(%r{(git:|:git\s*=>)\s*['"](?<name>#{URL_REGEX})['"]}, &:itself) + link_regex(/(git:|:git\s*=>)\s*['"](?<name>#{URL_REGEX})['"]/, &:itself) link_method_call('license', &method(:license_url)) link_regex(/license\s*=\s*\{\s*(type:|:type\s*=>)\s*#{STRING_REGEX}/, &method(:license_url)) diff --git a/lib/gitlab/email/reply_parser.rb b/lib/gitlab/email/reply_parser.rb index 558df87f36d..01c28d051ee 100644 --- a/lib/gitlab/email/reply_parser.rb +++ b/lib/gitlab/email/reply_parser.rb @@ -43,7 +43,7 @@ module Gitlab return "" unless decoded # Certain trigger phrases that means we didn't parse correctly - if decoded =~ /(Content\-Type\:|multipart\/alternative|text\/plain)/ + if decoded =~ %r{(Content\-Type\:|multipart/alternative|text/plain)} return "" end diff --git a/lib/gitlab/file_detector.rb b/lib/gitlab/file_detector.rb index 0e9ef4f897c..cc2638172ec 100644 --- a/lib/gitlab/file_detector.rb +++ b/lib/gitlab/file_detector.rb @@ -6,14 +6,14 @@ module Gitlab module FileDetector PATTERNS = { # Project files - readme: /\Areadme[^\/]*\z/i, - changelog: /\A(changelog|history|changes|news)[^\/]*\z/i, - license: /\A(licen[sc]e|copying)(\.[^\/]+)?\z/i, - contributing: /\Acontributing[^\/]*\z/i, + readme: %r{\Areadme[^/]*\z}i, + changelog: %r{\A(changelog|history|changes|news)[^/]*\z}i, + license: %r{\A(licen[sc]e|copying)(\.[^/]+)?\z}i, + contributing: %r{\Acontributing[^/]*\z}i, version: 'version', avatar: /\Alogo\.(png|jpg|gif)\z/, - issue_template: /\A\.gitlab\/issue_templates\/[^\/]+\.md\z/, - merge_request_template: /\A\.gitlab\/merge_request_templates\/[^\/]+\.md\z/, + issue_template: %r{\A\.gitlab/issue_templates/[^/]+\.md\z}, + merge_request_template: %r{\A\.gitlab/merge_request_templates/[^/]+\.md\z}, # Configuration files gitignore: '.gitignore', @@ -22,17 +22,17 @@ module Gitlab route_map: '.gitlab/route-map.yml', # Dependency files - cartfile: /\ACartfile[^\/]*\z/, + cartfile: %r{\ACartfile[^/]*\z}, composer_json: 'composer.json', gemfile: /\A(Gemfile|gems\.rb)\z/, gemfile_lock: 'Gemfile.lock', - gemspec: /\A[^\/]*\.gemspec\z/, + gemspec: %r{\A[^/]*\.gemspec\z}, godeps_json: 'Godeps.json', package_json: 'package.json', podfile: 'Podfile', - podspec_json: /\A[^\/]*\.podspec\.json\z/, - podspec: /\A[^\/]*\.podspec\z/, - requirements_txt: /\A[^\/]*requirements\.txt\z/, + podspec_json: %r{\A[^/]*\.podspec\.json\z}, + podspec: %r{\A[^/]*\.podspec\z}, + requirements_txt: %r{\A[^/]*requirements\.txt\z}, yarn_lock: 'yarn.lock' }.freeze diff --git a/lib/gitlab/gfm/uploads_rewriter.rb b/lib/gitlab/gfm/uploads_rewriter.rb index 8fab5489616..1b74f735679 100644 --- a/lib/gitlab/gfm/uploads_rewriter.rb +++ b/lib/gitlab/gfm/uploads_rewriter.rb @@ -27,7 +27,7 @@ module Gitlab with_link_in_tmp_dir(file.file) do |open_tmp_file| new_uploader.store!(open_tmp_file) end - new_uploader.to_markdown + new_uploader.markdown_link end end @@ -46,7 +46,7 @@ module Gitlab private def find_file(project, secret, file) - uploader = FileUploader.new(project, secret) + uploader = FileUploader.new(project, secret: secret) uploader.retrieve_from_store!(file) uploader.file end diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 71647099f83..d4e893b881c 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -6,12 +6,13 @@ module Gitlab CommandError = Class.new(StandardError) CommitError = Class.new(StandardError) + OSError = Class.new(StandardError) class << self include Gitlab::EncodingHelper def ref_name(ref) - encode!(ref).sub(/\Arefs\/(tags|heads|remotes)\//, '') + encode!(ref).sub(%r{\Arefs/(tags|heads|remotes)/}, '') end def branch_name(ref) diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 13120120223..4828301dbb9 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -107,7 +107,7 @@ module Gitlab def find_entry_by_path(repository, root_id, path) root_tree = repository.lookup(root_id) # Strip leading slashes - path[/^\/*/] = '' + path[%r{^/*}] = '' path_arr = path.split('/') entry = root_tree.find do |entry| @@ -140,7 +140,7 @@ module Gitlab def find_by_gitaly(repository, sha, path, limit: MAX_DATA_DISPLAY_SIZE) return unless path - path = path.sub(/\A\/*/, '') + path = path.sub(%r{\A/*}, '') path = '/' if path.empty? name = File.basename(path) diff --git a/lib/gitlab/git/branch.rb b/lib/gitlab/git/branch.rb index 3487e099381..ae7e88f0503 100644 --- a/lib/gitlab/git/branch.rb +++ b/lib/gitlab/git/branch.rb @@ -1,5 +1,3 @@ -# Gitaly note: JV: no RPC's here. - module Gitlab module Git class Branch < Ref diff --git a/lib/gitlab/git/lfs_changes.rb b/lib/gitlab/git/lfs_changes.rb index 732dd5d998a..48434047fce 100644 --- a/lib/gitlab/git/lfs_changes.rb +++ b/lib/gitlab/git/lfs_changes.rb @@ -25,8 +25,7 @@ module Gitlab private def rev_list - ::Gitlab::Git::RevList.new(path_to_repo: @repository.path_to_repo, - newrev: @newrev) + Gitlab::Git::RevList.new(@repository, newrev: @newrev) end end end diff --git a/lib/gitlab/git/lfs_pointer_file.rb b/lib/gitlab/git/lfs_pointer_file.rb new file mode 100644 index 00000000000..da12ed7d125 --- /dev/null +++ b/lib/gitlab/git/lfs_pointer_file.rb @@ -0,0 +1,25 @@ +module Gitlab + module Git + class LfsPointerFile + def initialize(data) + @data = data + end + + def pointer + @pointer ||= <<~FILE + version https://git-lfs.github.com/spec/v1 + oid sha256:#{sha256} + size #{size} + FILE + end + + def size + @size ||= @data.bytesize + end + + def sha256 + @sha256 ||= Digest::SHA256.hexdigest(@data) + end + end + end +end diff --git a/lib/gitlab/git/path_helper.rb b/lib/gitlab/git/path_helper.rb index 42c80aabd0a..155cf52f050 100644 --- a/lib/gitlab/git/path_helper.rb +++ b/lib/gitlab/git/path_helper.rb @@ -6,7 +6,7 @@ module Gitlab class << self def normalize_path(filename) # Strip all leading slashes so that //foo -> foo - filename[/^\/*/] = '' + filename[%r{^/*}] = '' # Expand relative paths (e.g. foo/../bar) filename = Pathname.new(filename) diff --git a/lib/gitlab/git/popen.rb b/lib/gitlab/git/popen.rb index e0bd2bbe47b..c1767046ff0 100644 --- a/lib/gitlab/git/popen.rb +++ b/lib/gitlab/git/popen.rb @@ -25,7 +25,7 @@ module Gitlab stdin.close if lazy_block - return lazy_block.call(stdout.lazy) + return [lazy_block.call(stdout.lazy), 0] else cmd_output << stdout.read end diff --git a/lib/gitlab/git/ref.rb b/lib/gitlab/git/ref.rb index a3ba9475ad0..fa71a4e7ea7 100644 --- a/lib/gitlab/git/ref.rb +++ b/lib/gitlab/git/ref.rb @@ -23,7 +23,7 @@ module Gitlab # Ex. # Ref.extract_branch_name('refs/heads/master') #=> 'master' def self.extract_branch_name(str) - str.gsub(/\Arefs\/heads\//, '') + str.gsub(%r{\Arefs/heads/}, '') end # Gitaly: this method will probably be migrated indirectly via its call sites. diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 8137c582c0f..d7510061def 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -128,6 +128,10 @@ module Gitlab raise NoRepository.new('no repository for such path') end + def cleanup + @rugged&.close + end + def circuit_breaker @circuit_breaker ||= Gitlab::Git::Storage::CircuitBreaker.for_storage(storage) end @@ -462,7 +466,6 @@ module Gitlab path: nil, follow: false, skip_merges: false, - disable_walk: false, after: nil, before: nil } @@ -494,11 +497,7 @@ module Gitlab return [] end - if log_using_shell?(options) - log_by_shell(sha, options) - else - log_by_walk(sha, options) - end + log_by_shell(sha, options) end def count_commits(options) @@ -1227,33 +1226,13 @@ module Gitlab end def squash(user, squash_id, branch:, start_sha:, end_sha:, author:, message:) - squash_path = worktree_path(SQUASH_WORKTREE_PREFIX, squash_id) - env = git_env_for_user(user).merge( - 'GIT_AUTHOR_NAME' => author.name, - 'GIT_AUTHOR_EMAIL' => author.email - ) - diff_range = "#{start_sha}...#{end_sha}" - diff_files = run_git!( - %W(diff --name-only --diff-filter=a --binary #{diff_range}) - ).chomp - - with_worktree(squash_path, branch, sparse_checkout_files: diff_files, env: env) do - # Apply diff of the `diff_range` to the worktree - diff = run_git!(%W(diff --binary #{diff_range})) - run_git!(%w(apply --index), chdir: squash_path, env: env) do |stdin| - stdin.write(diff) + gitaly_migrate(:squash) do |is_enabled| + if is_enabled + gitaly_operation_client.user_squash(user, squash_id, branch, + start_sha, end_sha, author, message) + else + git_squash(user, squash_id, branch, start_sha, end_sha, author, message) end - - # Commit the `diff_range` diff - run_git!(%W(commit --no-verify --message #{message}), chdir: squash_path, env: env) - - # Return the squash sha. May print a warning for ambiguous refs, but - # we can ignore that with `--quiet` and just take the SHA, if present. - # HEAD here always refers to the current HEAD commit, even if there is - # another ref called HEAD. - run_git!( - %w(rev-parse --quiet --verify HEAD), chdir: squash_path, env: env - ).chomp end end @@ -1311,7 +1290,15 @@ module Gitlab # rubocop:enable Metrics/ParameterLists def write_config(full_path:) - rugged.config['gitlab.fullpath'] = full_path if full_path.present? + return unless full_path.present? + + gitaly_migrate(:write_config) do |is_enabled| + if is_enabled + gitaly_repository_client.write_config(full_path: full_path) + else + rugged_write_config(full_path: full_path) + end + end end def gitaly_repository @@ -1360,20 +1347,23 @@ module Gitlab raise CommandError.new(e) end - def refs_contains_sha(ref_type, sha) - args = %W(#{ref_type} --contains #{sha}) - names = run_git(args).first - - if names.respond_to?(:split) - names = names.split("\n").map(&:strip) - - names.each do |name| - name.slice! '* ' + def branch_names_contains_sha(sha) + gitaly_migrate(:branch_names_contains_sha) do |is_enabled| + if is_enabled + gitaly_ref_client.branch_names_contains_sha(sha) + else + refs_contains_sha(:branch, sha) end + end + end - names - else - [] + def tag_names_contains_sha(sha) + gitaly_migrate(:tag_names_contains_sha) do |is_enabled| + if is_enabled + gitaly_ref_client.tag_names_contains_sha(sha) + else + refs_contains_sha(:tag, sha) + end end end @@ -1397,7 +1387,7 @@ module Gitlab end def search_files_by_name(query, ref) - safe_query = Regexp.escape(query.sub(/^\/*/, "")) + safe_query = Regexp.escape(query.sub(%r{^/*}, "")) return [] if empty? || safe_query.blank? @@ -1441,6 +1431,26 @@ module Gitlab end end + def rev_list(including: [], excluding: [], objects: false, &block) + args = ['rev-list'] + + args.push(*rev_list_param(including)) + + exclude_param = *rev_list_param(excluding) + if exclude_param.any? + args.push('--not') + args.push(*exclude_param) + end + + args.push('--objects') if objects + + run_git!(args, lazy_block: block) + end + + def missed_ref(oldrev, newrev) + run_git!(['rev-list', '--max-count=1', oldrev, "^#{newrev}"]) + end + private def local_write_ref(ref_path, ref, old_ref: nil, shell: true) @@ -1451,6 +1461,25 @@ module Gitlab end end + def refs_contains_sha(ref_type, sha) + args = %W(#{ref_type} --contains #{sha}) + names = run_git(args).first + + return [] unless names.respond_to?(:split) + + names = names.split("\n").map(&:strip) + + names.each do |name| + name.slice! '* ' + end + + names + end + + def rugged_write_config(full_path:) + rugged.config['gitlab.fullpath'] = full_path + end + def shell_write_ref(ref_path, ref, old_ref) raise ArgumentError, "invalid ref_path #{ref_path.inspect}" if ref_path.include?(' ') raise ArgumentError, "invalid ref #{ref.inspect}" if ref.include?("\x00") @@ -1470,7 +1499,7 @@ module Gitlab Rails.logger.error "Unable to create #{ref_path} reference for repository #{path}: #{ex}" end - def run_git(args, chdir: path, env: {}, nice: false, &block) + def run_git(args, chdir: path, env: {}, nice: false, lazy_block: nil, &block) cmd = [Gitlab.config.git.bin_path, *args] cmd.unshift("nice") if nice @@ -1480,12 +1509,12 @@ module Gitlab end circuit_breaker.perform do - popen(cmd, chdir, env, &block) + popen(cmd, chdir, env, lazy_block: lazy_block, &block) end end - def run_git!(args, chdir: path, env: {}, nice: false, &block) - output, status = run_git(args, chdir: chdir, env: env, nice: nice, &block) + def run_git!(args, chdir: path, env: {}, nice: false, lazy_block: nil, &block) + output, status = run_git(args, chdir: chdir, env: env, nice: nice, lazy_block: lazy_block, &block) raise GitError, output unless status.zero? @@ -1512,7 +1541,7 @@ module Gitlab if sparse_checkout_files # Create worktree without checking out run_git!(base_args + ['--no-checkout', worktree_path], env: env) - worktree_git_path = run_git!(%w(rev-parse --git-dir), chdir: worktree_path) + worktree_git_path = run_git!(%w(rev-parse --git-dir), chdir: worktree_path).chomp configure_sparse_checkout(worktree_git_path, sparse_checkout_files) @@ -1645,24 +1674,6 @@ module Gitlab end end - def log_using_shell?(options) - options[:path].present? || - options[:disable_walk] || - options[:skip_merges] || - options[:after] || - options[:before] - end - - def log_by_walk(sha, options) - walk_options = { - show: sha, - sort: Rugged::SORT_NONE, - limit: options[:limit], - offset: options[:offset] - } - Rugged::Walker.walk(rugged, walk_options).to_a - end - # Gitaly note: JV: although #log_by_shell shells out to Git I think the # complexity is such that we should migrate it as Ruby before trying to # do it in Go. @@ -2025,7 +2036,7 @@ module Gitlab target_commit = Gitlab::Git::Commit.find(self, rugged_ref.target) Gitlab::Git::Branch.new(self, rugged_ref.name, rugged_ref.target, target_commit) rescue Rugged::ReferenceError => e - raise InvalidRef.new("Branch #{ref} already exists") if e.to_s =~ /'refs\/heads\/#{ref}'/ + raise InvalidRef.new("Branch #{ref} already exists") if e.to_s =~ %r{'refs/heads/#{ref}'} raise InvalidRef.new("Invalid reference #{start_point}") end @@ -2175,6 +2186,37 @@ module Gitlab end end + def git_squash(user, squash_id, branch, start_sha, end_sha, author, message) + squash_path = worktree_path(SQUASH_WORKTREE_PREFIX, squash_id) + env = git_env_for_user(user).merge( + 'GIT_AUTHOR_NAME' => author.name, + 'GIT_AUTHOR_EMAIL' => author.email + ) + diff_range = "#{start_sha}...#{end_sha}" + diff_files = run_git!( + %W(diff --name-only --diff-filter=a --binary #{diff_range}) + ).chomp + + with_worktree(squash_path, branch, sparse_checkout_files: diff_files, env: env) do + # Apply diff of the `diff_range` to the worktree + diff = run_git!(%W(diff --binary #{diff_range})) + run_git!(%w(apply --index), chdir: squash_path, env: env) do |stdin| + stdin.write(diff) + end + + # Commit the `diff_range` diff + run_git!(%W(commit --no-verify --message #{message}), chdir: squash_path, env: env) + + # Return the squash sha. May print a warning for ambiguous refs, but + # we can ignore that with `--quiet` and just take the SHA, if present. + # HEAD here always refers to the current HEAD commit, even if there is + # another ref called HEAD. + run_git!( + %w(rev-parse --quiet --verify HEAD), chdir: squash_path, env: env + ).chomp + end + end + def local_fetch_ref(source_path, source_ref:, target_ref:) args = %W(fetch --no-tags -f #{source_path} #{source_ref}:#{target_ref}) run_git(args) @@ -2354,6 +2396,10 @@ module Gitlab rescue Rugged::ReferenceError 0 end + + def rev_list_param(spec) + spec == :all ? ['--all'] : spec + end end end end diff --git a/lib/gitlab/git/repository_mirroring.rb b/lib/gitlab/git/repository_mirroring.rb index effb1f0ca19..dc424a433fb 100644 --- a/lib/gitlab/git/repository_mirroring.rb +++ b/lib/gitlab/git/repository_mirroring.rb @@ -43,7 +43,7 @@ module Gitlab branches = [] rugged.references.each("refs/remotes/#{remote_name}/*").map do |ref| - name = ref.name.sub(/\Arefs\/remotes\/#{remote_name}\//, '') + name = ref.name.sub(%r{\Arefs/remotes/#{remote_name}/}, '') begin target_commit = Gitlab::Git::Commit.find(self, ref.target) diff --git a/lib/gitlab/git/rev_list.rb b/lib/gitlab/git/rev_list.rb index f8b2e7e0e21..38c3a55f96f 100644 --- a/lib/gitlab/git/rev_list.rb +++ b/lib/gitlab/git/rev_list.rb @@ -5,17 +5,17 @@ module Gitlab class RevList include Gitlab::Git::Popen - attr_reader :oldrev, :newrev, :path_to_repo + attr_reader :oldrev, :newrev, :repository - def initialize(path_to_repo:, newrev:, oldrev: nil) + def initialize(repository, newrev:, oldrev: nil) @oldrev = oldrev @newrev = newrev - @path_to_repo = path_to_repo + @repository = repository end # This method returns an array of new commit references def new_refs - execute([*base_args, newrev, '--not', '--all']) + repository.rev_list(including: newrev, excluding: :all).split("\n") end # Finds newly added objects @@ -28,66 +28,39 @@ module Gitlab # When given a block it will yield objects as a lazy enumerator so # the caller can limit work done instead of processing megabytes of data def new_objects(require_path: nil, not_in: nil, &lazy_block) - args = [*base_args, newrev, *not_in_refs(not_in), '--objects'] + opts = { + including: newrev, + excluding: not_in.nil? ? :all : not_in, + require_path: require_path + } - get_objects(args, require_path: require_path, &lazy_block) + get_objects(opts, &lazy_block) end def all_objects(require_path: nil, &lazy_block) - args = [*base_args, '--all', '--objects'] - - get_objects(args, require_path: require_path, &lazy_block) + get_objects(including: :all, require_path: require_path, &lazy_block) end # This methods returns an array of missed references # # Should become obsolete after https://gitlab.com/gitlab-org/gitaly/issues/348. def missed_ref - execute([*base_args, '--max-count=1', oldrev, "^#{newrev}"]) + repository.missed_ref(oldrev, newrev).split("\n") end private - def not_in_refs(references) - return ['--not', '--all'] unless references - return [] if references.empty? - - references.prepend('--not') - end - def execute(args) - output, status = popen(args, nil, Gitlab::Git::Env.to_env_hash) - - unless status.zero? - raise "Got a non-zero exit code while calling out `#{args.join(' ')}`: #{output}" - end - - output.split("\n") - end - - def lazy_execute(args, &lazy_block) - popen(args, nil, Gitlab::Git::Env.to_env_hash, lazy_block: lazy_block) - end - - def base_args - [ - Gitlab.config.git.bin_path, - "--git-dir=#{path_to_repo}", - 'rev-list' - ] + repository.rev_list(args).split("\n") end - def get_objects(args, require_path: nil) - if block_given? - lazy_execute(args) do |lazy_output| - objects = objects_from_output(lazy_output, require_path: require_path) + def get_objects(including: [], excluding: [], require_path: nil) + opts = { including: including, excluding: excluding, objects: true } - yield(objects) - end - else - object_output = execute(args) + repository.rev_list(opts) do |lazy_output| + objects = objects_from_output(lazy_output, require_path: require_path) - objects_from_output(object_output, require_path: require_path) + yield(objects) end end diff --git a/lib/gitlab/git/tag.rb b/lib/gitlab/git/tag.rb index bc4e160dce9..8a8f7b051ed 100644 --- a/lib/gitlab/git/tag.rb +++ b/lib/gitlab/git/tag.rb @@ -1,5 +1,3 @@ -# Gitaly note: JV: no RPC's here. -# module Gitlab module Git class Tag < Ref diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index ccdb8975342..39040d56971 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -25,8 +25,9 @@ module Gitlab @repository.exists? end + # Disabled because of https://gitlab.com/gitlab-org/gitaly/merge_requests/539 def write_page(name, format, content, commit_details) - @repository.gitaly_migrate(:wiki_write_page) do |is_enabled| + @repository.gitaly_migrate(:wiki_write_page, status: Gitlab::GitalyClient::MigrationStatus::DISABLED) do |is_enabled| if is_enabled gitaly_write_page(name, format, content, commit_details) gollum_wiki.clear_cache @@ -47,8 +48,9 @@ module Gitlab end end + # Disable because of https://gitlab.com/gitlab-org/gitlab-ce/issues/42094 def update_page(page_path, title, format, content, commit_details) - @repository.gitaly_migrate(:wiki_update_page) do |is_enabled| + @repository.gitaly_migrate(:wiki_update_page, status: Gitlab::GitalyClient::MigrationStatus::DISABLED) do |is_enabled| if is_enabled gitaly_update_page(page_path, title, format, content, commit_details) gollum_wiki.clear_cache @@ -68,8 +70,9 @@ module Gitlab end end + # Disable because of https://gitlab.com/gitlab-org/gitlab-ce/issues/42039 def page(title:, version: nil, dir: nil) - @repository.gitaly_migrate(:wiki_find_page) do |is_enabled| + @repository.gitaly_migrate(:wiki_find_page, status: Gitlab::GitalyClient::MigrationStatus::DISABLED) do |is_enabled| if is_enabled gitaly_find_page(title: title, version: version, dir: dir) else @@ -93,11 +96,23 @@ module Gitlab # :per_page - The number of items per page. # :limit - Total number of items to return. def page_versions(page_path, options = {}) - current_page = gollum_page_by_path(page_path) + @repository.gitaly_migrate(:wiki_page_versions) do |is_enabled| + if is_enabled + versions = gitaly_wiki_client.page_versions(page_path, options) + + # Gitaly uses gollum-lib to get the versions. Gollum defaults to 20 + # per page, but also fetches 20 if `limit` or `per_page` < 20. + # Slicing returns an array with the expected number of items. + slice_bound = options[:limit] || options[:per_page] || Gollum::Page.per_page + versions[0..slice_bound] + else + current_page = gollum_page_by_path(page_path) - commits_from_page(current_page, options).map do |gitlab_git_commit| - gollum_page = gollum_wiki.page(current_page.title, gitlab_git_commit.id) - Gitlab::Git::WikiPageVersion.new(gitlab_git_commit, gollum_page&.format) + commits_from_page(current_page, options).map do |gitlab_git_commit| + gollum_page = gollum_wiki.page(current_page.title, gitlab_git_commit.id) + Gitlab::Git::WikiPageVersion.new(gitlab_git_commit, gollum_page&.format) + end + end end end @@ -192,7 +207,10 @@ module Gitlab assert_type!(format, Symbol) assert_type!(commit_details, CommitDetails) - gollum_wiki.write_page(name, format, content, commit_details.to_h) + filename = File.basename(name) + dir = (tmp_dir = File.dirname(name)) == '.' ? '' : tmp_dir + + gollum_wiki.write_page(filename, format, content, commit_details.to_h, dir) nil rescue Gollum::DuplicatePageError => e @@ -210,7 +228,15 @@ module Gitlab assert_type!(format, Symbol) assert_type!(commit_details, CommitDetails) - gollum_wiki.update_page(gollum_page_by_path(page_path), title, format, content, commit_details.to_h) + page = gollum_page_by_path(page_path) + committer = Gollum::Committer.new(page.wiki, commit_details.to_h) + + # Instead of performing two renames if the title has changed, + # the update_page will only update the format and content and + # the rename_page will do anything related to moving/renaming + gollum_wiki.update_page(page, page.name, format, content, committer: committer) + gollum_wiki.rename_page(page, title, committer: committer) + committer.commit nil end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index cadc7149301..5767f06b0ce 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -257,7 +257,7 @@ module Gitlab offset: options[:offset], follow: options[:follow], skip_merges: options[:skip_merges], - disable_walk: options[:disable_walk] + disable_walk: true # This option is deprecated. The 'walk' implementation is being removed. ) request.after = GitalyClient.timestamp(options[:after]) if options[:after] request.before = GitalyClient.timestamp(options[:before]) if options[:before] diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index cd2734b5a07..831cfd1e014 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -183,6 +183,32 @@ module Gitlab end end + def user_squash(user, squash_id, branch, start_sha, end_sha, author, message) + request = Gitaly::UserSquashRequest.new( + repository: @gitaly_repo, + user: Gitlab::Git::User.from_gitlab(user).to_gitaly, + squash_id: squash_id.to_s, + branch: encode_binary(branch), + start_sha: start_sha, + end_sha: end_sha, + author: Gitlab::Git::User.from_gitlab(author).to_gitaly, + commit_message: encode_binary(message) + ) + + response = GitalyClient.call( + @repository.storage, + :operation_service, + :user_squash, + request + ) + + if response.git_error.presence + raise Gitlab::Git::Repository::GitError, response.git_error + end + + response.squash_sha + end + def user_commit_files( user, branch_name, commit_message, actions, author_email, author_name, start_branch_name, start_repository) diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index 8b9a224b700..ba6b577fd17 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -145,6 +145,32 @@ module Gitlab raise Gitlab::Git::Repository::GitError, response.git_error if response.git_error.present? end + # Limit: 0 implies no limit, thus all tag names will be returned + def tag_names_contains_sha(sha, limit: 0) + request = Gitaly::ListTagNamesContainingCommitRequest.new( + repository: @gitaly_repo, + commit_id: sha, + limit: limit + ) + + stream = GitalyClient.call(@repository.storage, :ref_service, :list_tag_names_containing_commit, request) + + consume_ref_contains_sha_response(stream, :tag_names) + end + + # Limit: 0 implies no limit, thus all tag names will be returned + def branch_names_contains_sha(sha, limit: 0) + request = Gitaly::ListBranchNamesContainingCommitRequest.new( + repository: @gitaly_repo, + commit_id: sha, + limit: limit + ) + + stream = GitalyClient.call(@repository.storage, :ref_service, :list_branch_names_containing_commit, request) + + consume_ref_contains_sha_response(stream, :branch_names) + end + private def consume_refs_response(response) @@ -215,6 +241,13 @@ module Gitlab Gitlab::Git::Commit.decorate(@repository, hash) end + def consume_ref_contains_sha_response(stream, collection_name) + stream.each_with_object([]) do |response, array| + encoded_names = response.send(collection_name).map { |b| Gitlab::Git.ref_name(b) } # rubocop:disable GitlabSecurity/PublicSend + array.concat(encoded_names) + end + end + def invalid_ref!(message) raise Gitlab::Git::Repository::InvalidRef.new(message) end diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 7adf32af209..60706b4f0d8 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -219,6 +219,19 @@ module Gitlab true end + + def write_config(full_path:) + request = Gitaly::WriteConfigRequest.new(repository: @gitaly_repo, full_path: full_path) + response = GitalyClient.call( + @storage, + :repository_service, + :write_config, + request, + timeout: GitalyClient.fast_timeout + ) + + raise Gitlab::Git::OSError.new(response.error) unless response.error.empty? + end end end end diff --git a/lib/gitlab/gitaly_client/wiki_page.rb b/lib/gitlab/gitaly_client/wiki_page.rb index 7339468e911..a02d15db5dd 100644 --- a/lib/gitlab/gitaly_client/wiki_page.rb +++ b/lib/gitlab/gitaly_client/wiki_page.rb @@ -4,6 +4,7 @@ module Gitlab ATTRS = %i(title format url_path path name historical raw_data).freeze include AttributesBag + include Gitlab::EncodingHelper def initialize(params) super @@ -11,6 +12,10 @@ module Gitlab # All gRPC strings in a response are frozen, so we get an unfrozen # version here so appending to `raw_data` doesn't blow up. @raw_data = @raw_data.dup + + @title = encode_utf8(@title) + @path = encode_utf8(@path) + @name = encode_utf8(@name) end def historical? diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb index 8e87a8cc36f..0d8dd5cb8f4 100644 --- a/lib/gitlab/gitaly_client/wiki_service.rb +++ b/lib/gitlab/gitaly_client/wiki_service.rb @@ -101,6 +101,30 @@ module Gitlab pages end + # options: + # :page - The Integer page number. + # :per_page - The number of items per page. + # :limit - Total number of items to return. + def page_versions(page_path, options) + request = Gitaly::WikiGetPageVersionsRequest.new( + repository: @gitaly_repo, + page_path: encode_binary(page_path), + page: options[:page] || 1, + per_page: options[:per_page] || Gollum::Page.per_page + ) + + stream = GitalyClient.call(@repository.storage, :wiki_service, :wiki_get_page_versions, request) + + versions = [] + stream.each do |message| + message.versions.each do |version| + versions << new_wiki_page_version(version) + end + end + + versions + end + def find_file(name, revision) request = Gitaly::WikiFindFileRequest.new( repository: @gitaly_repo, @@ -141,7 +165,7 @@ module Gitlab private - # If a block is given and the yielded value is true, iteration will be + # If a block is given and the yielded value is truthy, iteration will be # stopped early at that point; else the iterator is consumed entirely. # The iterator is traversed with `next` to allow resuming the iteration. def wiki_page_from_iterator(iterator) @@ -158,10 +182,7 @@ module Gitlab else wiki_page = GitalyClient::WikiPage.new(page.to_h) - version = Gitlab::Git::WikiPageVersion.new( - Gitlab::Git::Commit.decorate(@repository, page.version.commit), - page.version.format - ) + version = new_wiki_page_version(page.version) end end @@ -170,6 +191,13 @@ module Gitlab [wiki_page, version] end + def new_wiki_page_version(version) + Gitlab::Git::WikiPageVersion.new( + Gitlab::Git::Commit.decorate(@repository, version.commit), + version.format + ) + end + def gitaly_commit_details(commit_details) Gitaly::WikiCommitDetails.new( name: encode_binary(commit_details.name), diff --git a/lib/gitlab/github_import/representation/diff_note.rb b/lib/gitlab/github_import/representation/diff_note.rb index bb7439a0641..be1334ca98a 100644 --- a/lib/gitlab/github_import/representation/diff_note.rb +++ b/lib/gitlab/github_import/representation/diff_note.rb @@ -13,7 +13,7 @@ module Gitlab :diff_hunk, :author, :note, :created_at, :updated_at, :github_id - NOTEABLE_ID_REGEX = /\/pull\/(?<iid>\d+)/i + NOTEABLE_ID_REGEX = %r{/pull/(?<iid>\d+)}i # Builds a diff note from a GitHub API response. # diff --git a/lib/gitlab/github_import/representation/note.rb b/lib/gitlab/github_import/representation/note.rb index a68bc4c002f..070e3b2db8d 100644 --- a/lib/gitlab/github_import/representation/note.rb +++ b/lib/gitlab/github_import/representation/note.rb @@ -12,7 +12,7 @@ module Gitlab expose_attribute :noteable_id, :noteable_type, :author, :note, :created_at, :updated_at, :github_id - NOTEABLE_TYPE_REGEX = /\/(?<type>(pull|issues))\/(?<iid>\d+)/i + NOTEABLE_TYPE_REGEX = %r{/(?<type>(pull|issues))/(?<iid>\d+)}i # Builds a note from a GitHub API response. # diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 9148d7571f2..86a90d57d9c 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -3,12 +3,11 @@ module Gitlab module GonHelper include WebpackHelper - include Gitlab::CurrentSettings def add_gon_variables gon.api_version = 'v4' gon.default_avatar_url = URI.join(Gitlab.config.gitlab.url, ActionController::Base.helpers.image_path('no_avatar.png')).to_s - gon.max_file_size = current_application_settings.max_attachment_size + gon.max_file_size = Gitlab::CurrentSettings.max_attachment_size gon.asset_host = ActionController::Base.asset_host gon.webpack_public_path = webpack_public_path gon.relative_url_root = Gitlab.config.gitlab.relative_url_root @@ -16,7 +15,7 @@ module Gitlab gon.user_color_scheme = Gitlab::ColorSchemes.for_user(current_user).css_class gon.katex_css_url = ActionController::Base.helpers.asset_path('katex.css') gon.katex_js_url = ActionController::Base.helpers.asset_path('katex.js') - gon.sentry_dsn = current_application_settings.clientside_sentry_dsn if current_application_settings.clientside_sentry_enabled + gon.sentry_dsn = Gitlab::CurrentSettings.clientside_sentry_dsn if Gitlab::CurrentSettings.clientside_sentry_enabled gon.gitlab_url = Gitlab.config.gitlab.url gon.revision = Gitlab::REVISION gon.gitlab_logo = ActionController::Base.helpers.asset_path('gitlab_logo.png') diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index 5c971564a73..0f4c3498036 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -59,7 +59,7 @@ module Gitlab end def extracted_files - Dir.glob("#{@shared.export_path}/**/*", File::FNM_DOTMATCH).reject { |f| f =~ /.*\/\.{1,2}$/ } + Dir.glob("#{@shared.export_path}/**/*", File::FNM_DOTMATCH).reject { |f| f =~ %r{.*/\.{1,2}$} } end end end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index cb711a83433..759833a5ee5 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -139,13 +139,12 @@ module Gitlab end def setup_label - return unless @relation_hash['type'] == 'GroupLabel' - # If there's no group, move the label to a project label - if @relation_hash['group_id'] + if @relation_hash['type'] == 'GroupLabel' && @relation_hash['group_id'] @relation_hash['project_id'] = nil @relation_name = :group_label else + @relation_hash['group_id'] = nil @relation_hash['type'] = 'ProjectLabel' end end diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb index 627a487d577..2f08dda55fd 100644 --- a/lib/gitlab/import_export/uploads_saver.rb +++ b/lib/gitlab/import_export/uploads_saver.rb @@ -17,15 +17,13 @@ module Gitlab false end - private + def uploads_path + FileUploader.absolute_base_dir(@project) + end def uploads_export_path File.join(@shared.export_path, 'uploads') end - - def uploads_path - FileUploader.dynamic_path_segment(@project) - end end end end diff --git a/lib/gitlab/kubernetes/helm/pod.rb b/lib/gitlab/kubernetes/helm/pod.rb index a3216759cae..ca5e06009fa 100644 --- a/lib/gitlab/kubernetes/helm/pod.rb +++ b/lib/gitlab/kubernetes/helm/pod.rb @@ -64,7 +64,7 @@ module Gitlab { name: 'configuration-volume', configMap: { - name: 'values-content-configuration', + name: "values-content-configuration-#{command.name}", items: [{ key: 'values', path: 'values.yaml' }] } } @@ -81,7 +81,11 @@ module Gitlab def create_config_map resource = ::Kubeclient::Resource.new - resource.metadata = { name: 'values-content-configuration', namespace: namespace_name, labels: { name: 'values-content-configuration' } } + resource.metadata = { + name: "values-content-configuration-#{command.name}", + namespace: namespace_name, + labels: { name: "values-content-configuration-#{command.name}" } + } resource.data = { values: File.read(command.chart_values_file) } kubeclient.create_config_map(resource) end diff --git a/lib/gitlab/ldap/auth_hash.rb b/lib/gitlab/ldap/auth_hash.rb index 1bd0965679a..96171dc26c4 100644 --- a/lib/gitlab/ldap/auth_hash.rb +++ b/lib/gitlab/ldap/auth_hash.rb @@ -7,6 +7,12 @@ module Gitlab @uid ||= Gitlab::LDAP::Person.normalize_dn(super) end + def username + super.tap do |username| + username.downcase! if ldap_config.lowercase_usernames + end + end + private def get_info(key) diff --git a/lib/gitlab/ldap/config.rb b/lib/gitlab/ldap/config.rb index cde60addcf7..47b3fce3e7a 100644 --- a/lib/gitlab/ldap/config.rb +++ b/lib/gitlab/ldap/config.rb @@ -139,6 +139,10 @@ module Gitlab options['allow_username_or_email_login'] end + def lowercase_usernames + options['lowercase_usernames'] + end + def name_proc if allow_username_or_email_login proc { |name| name.gsub(/@.*\z/, '') } diff --git a/lib/gitlab/ldap/person.rb b/lib/gitlab/ldap/person.rb index e81cec6ba1a..b91757c2a4b 100644 --- a/lib/gitlab/ldap/person.rb +++ b/lib/gitlab/ldap/person.rb @@ -82,7 +82,9 @@ module Gitlab # be returned. We need only one for username. # Ex. `uid` returns only one value but `mail` may # return an array of multiple email addresses. - [username].flatten.first + [username].flatten.first.tap do |username| + username.downcase! if config.lowercase_usernames + end end def email diff --git a/lib/gitlab/legacy_github_import/project_creator.rb b/lib/gitlab/legacy_github_import/project_creator.rb index 41e7eac4d08..cbabe5454ca 100644 --- a/lib/gitlab/legacy_github_import/project_creator.rb +++ b/lib/gitlab/legacy_github_import/project_creator.rb @@ -1,8 +1,6 @@ module Gitlab module LegacyGithubImport class ProjectCreator - include Gitlab::CurrentSettings - attr_reader :repo, :name, :namespace, :current_user, :session_data, :type def initialize(repo, name, namespace, current_user, session_data, type: 'github') @@ -36,7 +34,7 @@ module Gitlab end def visibility_level - repo.private ? Gitlab::VisibilityLevel::PRIVATE : current_application_settings.default_project_visibility + repo.private ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::CurrentSettings.default_project_visibility end # diff --git a/lib/gitlab/metrics/prometheus.rb b/lib/gitlab/metrics/prometheus.rb index f07ea3560ff..d12ba0ec176 100644 --- a/lib/gitlab/metrics/prometheus.rb +++ b/lib/gitlab/metrics/prometheus.rb @@ -71,8 +71,7 @@ module Gitlab end def prometheus_metrics_enabled_unmemoized - metrics_folder_present? && - Gitlab::CurrentSettings.current_application_settings[:prometheus_metrics_enabled] || false + metrics_folder_present? && Gitlab::CurrentSettings.prometheus_metrics_enabled || false end end end diff --git a/lib/gitlab/metrics/subscribers/action_view.rb b/lib/gitlab/metrics/subscribers/action_view.rb index 274436ca2b4..b600e8a2a50 100644 --- a/lib/gitlab/metrics/subscribers/action_view.rb +++ b/lib/gitlab/metrics/subscribers/action_view.rb @@ -34,7 +34,7 @@ module Gitlab end def relative_path(path) - path.gsub(/^#{Rails.root.to_s}\/?/, '') + path.gsub(%r{^#{Rails.root.to_s}/?}, '') end def values_for(event) diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb index c6a56277922..1a570f480c6 100644 --- a/lib/gitlab/middleware/go.rb +++ b/lib/gitlab/middleware/go.rb @@ -4,7 +4,6 @@ module Gitlab module Middleware class Go include ActionView::Helpers::TagHelper - include Gitlab::CurrentSettings PROJECT_PATH_REGEX = %r{\A(#{Gitlab::PathRegex.full_namespace_route_regex}/#{Gitlab::PathRegex.project_route_regex})/}.freeze @@ -42,7 +41,7 @@ module Gitlab project_url = URI.join(config.gitlab.url, path) import_prefix = strip_url(project_url.to_s) - repository_url = if current_application_settings.enabled_git_access_protocol == 'ssh' + repository_url = if Gitlab::CurrentSettings.enabled_git_access_protocol == 'ssh' shell = config.gitlab_shell port = ":#{shell.ssh_port}" unless shell.ssh_port == 22 "ssh://#{shell.ssh_user}@#{shell.ssh_host}#{port}/#{path}.git" @@ -56,12 +55,12 @@ module Gitlab end def strip_url(url) - url.gsub(/\Ahttps?:\/\//, '') + url.gsub(%r{\Ahttps?://}, '') end def project_path(request) path_info = request.env["PATH_INFO"] - path_info.sub!(/^\//, '') + path_info.sub!(%r{^/}, '') project_path_match = "#{path_info}/".match(PROJECT_PATH_REGEX) return unless project_path_match diff --git a/lib/gitlab/middleware/static.rb b/lib/gitlab/middleware/static.rb index 85ffa8aca68..aa1e9dc0fdb 100644 --- a/lib/gitlab/middleware/static.rb +++ b/lib/gitlab/middleware/static.rb @@ -1,7 +1,7 @@ module Gitlab module Middleware class Static < ActionDispatch::Static - UPLOADS_REGEX = /\A\/uploads(\/|\z)/.freeze + UPLOADS_REGEX = %r{\A/uploads(/|\z)}.freeze def call(env) return @app.call(env) if env['PATH_INFO'] =~ UPLOADS_REGEX diff --git a/lib/gitlab/performance_bar.rb b/lib/gitlab/performance_bar.rb index e29e168fc5a..6c2b2036074 100644 --- a/lib/gitlab/performance_bar.rb +++ b/lib/gitlab/performance_bar.rb @@ -1,7 +1,5 @@ module Gitlab module PerformanceBar - extend Gitlab::CurrentSettings - ALLOWED_USER_IDS_KEY = 'performance_bar_allowed_user_ids:v2'.freeze EXPIRY_TIME = 5.minutes @@ -13,7 +11,7 @@ module Gitlab end def self.allowed_group_id - current_application_settings.performance_bar_allowed_group_id + Gitlab::CurrentSettings.performance_bar_allowed_group_id end def self.allowed_user_ids diff --git a/lib/gitlab/polling_interval.rb b/lib/gitlab/polling_interval.rb index 4780675a492..fe4bdfe3831 100644 --- a/lib/gitlab/polling_interval.rb +++ b/lib/gitlab/polling_interval.rb @@ -1,12 +1,10 @@ module Gitlab class PollingInterval - extend Gitlab::CurrentSettings - HEADER_NAME = 'Poll-Interval'.freeze def self.set_header(response, interval:) if polling_enabled? - multiplier = current_application_settings.polling_interval_multiplier + multiplier = Gitlab::CurrentSettings.polling_interval_multiplier value = (interval * multiplier).to_i else value = -1 @@ -16,7 +14,7 @@ module Gitlab end def self.polling_enabled? - !current_application_settings.polling_interval_multiplier.zero? + !Gitlab::CurrentSettings.polling_interval_multiplier.zero? end end end diff --git a/lib/gitlab/protocol_access.rb b/lib/gitlab/protocol_access.rb index 09fa14764e6..2819c7d062c 100644 --- a/lib/gitlab/protocol_access.rb +++ b/lib/gitlab/protocol_access.rb @@ -1,14 +1,12 @@ module Gitlab module ProtocolAccess - extend Gitlab::CurrentSettings - def self.allowed?(protocol) if protocol == 'web' true - elsif current_application_settings.enabled_git_access_protocol.blank? + elsif Gitlab::CurrentSettings.enabled_git_access_protocol.blank? true else - protocol == current_application_settings.enabled_git_access_protocol + protocol == Gitlab::CurrentSettings.enabled_git_access_protocol end end end diff --git a/lib/gitlab/query_limiting.rb b/lib/gitlab/query_limiting.rb new file mode 100644 index 00000000000..f64f1757144 --- /dev/null +++ b/lib/gitlab/query_limiting.rb @@ -0,0 +1,36 @@ +module Gitlab + module QueryLimiting + # Returns true if we should enable tracking of query counts. + # + # This is only enabled in production/staging if we're running on GitLab.com. + # This ensures we don't produce any errors that users can't do anything + # about themselves. + def self.enable? + Gitlab.com? || Rails.env.development? || Rails.env.test? + end + + # Allows the current request to execute any number of SQL queries. + # + # This method should _only_ be used when there's a corresponding issue to + # reduce the number of queries. + # + # The issue URL is only meant to push developers into creating an issue + # instead of blindly whitelisting offending blocks of code. + def self.whitelist(issue_url) + return unless enable_whitelist? + + unless issue_url.start_with?('https://') + raise( + ArgumentError, + 'You must provide a valid issue URL in order to whitelist a block of code' + ) + end + + Transaction&.current&.whitelisted = true + end + + def self.enable_whitelist? + Rails.env.development? || Rails.env.test? + end + end +end diff --git a/lib/gitlab/query_limiting/active_support_subscriber.rb b/lib/gitlab/query_limiting/active_support_subscriber.rb new file mode 100644 index 00000000000..66049c94ec6 --- /dev/null +++ b/lib/gitlab/query_limiting/active_support_subscriber.rb @@ -0,0 +1,11 @@ +module Gitlab + module QueryLimiting + class ActiveSupportSubscriber < ActiveSupport::Subscriber + attach_to :active_record + + def sql(*) + Transaction.current&.increment + end + end + end +end diff --git a/lib/gitlab/query_limiting/middleware.rb b/lib/gitlab/query_limiting/middleware.rb new file mode 100644 index 00000000000..949ae79a047 --- /dev/null +++ b/lib/gitlab/query_limiting/middleware.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Gitlab + module QueryLimiting + # Middleware for reporting (or raising) when a request performs more than a + # certain amount of database queries. + class Middleware + CONTROLLER_KEY = 'action_controller.instance'.freeze + ENDPOINT_KEY = 'api.endpoint'.freeze + + def initialize(app) + @app = app + end + + def call(env) + transaction, retval = Transaction.run do + @app.call(env) + end + + transaction.action = action_name(env) + transaction.act_upon_results + + retval + end + + def action_name(env) + if env[CONTROLLER_KEY] + action_for_rails(env) + elsif env[ENDPOINT_KEY] + action_for_grape(env) + end + end + + private + + def action_for_rails(env) + controller = env[CONTROLLER_KEY] + action = "#{controller.class.name}##{controller.action_name}" + + if controller.content_type == 'text/html' + action + else + "#{action} (#{controller.content_type})" + end + end + + def action_for_grape(env) + endpoint = env[ENDPOINT_KEY] + route = endpoint.route rescue nil + + "#{route.request_method} #{route.path}" if route + end + end + end +end diff --git a/lib/gitlab/query_limiting/transaction.rb b/lib/gitlab/query_limiting/transaction.rb new file mode 100644 index 00000000000..3cbafadb0d0 --- /dev/null +++ b/lib/gitlab/query_limiting/transaction.rb @@ -0,0 +1,83 @@ +module Gitlab + module QueryLimiting + class Transaction + THREAD_KEY = :__gitlab_query_counts_transaction + + attr_accessor :count, :whitelisted + + # The name of the action (e.g. `UsersController#show`) that is being + # executed. + attr_accessor :action + + # The maximum number of SQL queries that can be executed in a request. For + # the sake of keeping things simple we hardcode this value here, it's not + # supposed to be changed very often anyway. + THRESHOLD = 100 + + # Error that is raised whenever exceeding the maximum number of queries. + ThresholdExceededError = Class.new(StandardError) + + def self.current + Thread.current[THREAD_KEY] + end + + # Starts a new transaction and returns it and the blocks' return value. + # + # Example: + # + # transaction, retval = Transaction.run do + # 10 + # end + # + # retval # => 10 + def self.run + transaction = new + Thread.current[THREAD_KEY] = transaction + + [transaction, yield] + ensure + Thread.current[THREAD_KEY] = nil + end + + def initialize + @action = nil + @count = 0 + @whitelisted = false + end + + # Sends a notification based on the number of executed SQL queries. + def act_upon_results + return unless threshold_exceeded? + + error = ThresholdExceededError.new(error_message) + + if raise_error? + raise(error) + else + # Raven automatically logs to the Rails log if disabled, thus we don't + # need to manually log anything in case Sentry support is not enabled. + Raven.capture_exception(error) + end + end + + def increment + @count += 1 unless whitelisted + end + + def raise_error? + Rails.env.test? + end + + def threshold_exceeded? + count > THRESHOLD + end + + def error_message + header = 'Too many SQL queries were executed' + header += " in #{action}" if action + + "#{header}: a maximum of #{THRESHOLD} is allowed but #{count} SQL queries were executed" + end + end + end +end diff --git a/lib/gitlab/quick_actions/spend_time_and_date_separator.rb b/lib/gitlab/quick_actions/spend_time_and_date_separator.rb index 3f52402b31f..7328c517a30 100644 --- a/lib/gitlab/quick_actions/spend_time_and_date_separator.rb +++ b/lib/gitlab/quick_actions/spend_time_and_date_separator.rb @@ -9,7 +9,7 @@ module Gitlab # if date doesn't present return time with current date # in other cases return nil class SpendTimeAndDateSeparator - DATE_REGEX = /(\d{2,4}[\/\-.]\d{1,2}[\/\-.]\d{1,2})/ + DATE_REGEX = %r{(\d{2,4}[/\-.]\d{1,2}[/\-.]\d{1,2})} def initialize(spend_command_arg) @spend_arg = spend_command_arg diff --git a/lib/gitlab/recaptcha.rb b/lib/gitlab/recaptcha.rb index c463dd487a0..c9efa28d7e7 100644 --- a/lib/gitlab/recaptcha.rb +++ b/lib/gitlab/recaptcha.rb @@ -1,12 +1,10 @@ module Gitlab module Recaptcha - extend Gitlab::CurrentSettings - def self.load_configurations! - if current_application_settings.recaptcha_enabled + if Gitlab::CurrentSettings.recaptcha_enabled ::Recaptcha.configure do |config| - config.public_key = current_application_settings.recaptcha_site_key - config.private_key = current_application_settings.recaptcha_private_key + config.public_key = Gitlab::CurrentSettings.recaptcha_site_key + config.private_key = Gitlab::CurrentSettings.recaptcha_private_key end true @@ -14,7 +12,7 @@ module Gitlab end def self.enabled? - current_application_settings.recaptcha_enabled + Gitlab::CurrentSettings.recaptcha_enabled end end end diff --git a/lib/gitlab/repo_path.rb b/lib/gitlab/repo_path.rb index 3591fa9145e..79265cf952d 100644 --- a/lib/gitlab/repo_path.rb +++ b/lib/gitlab/repo_path.rb @@ -30,7 +30,7 @@ module Gitlab raise NotFoundError.new("No known storage path matches #{repo_path.inspect}") end - result.sub(/\A\/*/, '') + result.sub(%r{\A/*}, '') end def self.find_project(project_path) diff --git a/lib/gitlab/sentry.rb b/lib/gitlab/sentry.rb index 159d0e7952e..4a22fc80f75 100644 --- a/lib/gitlab/sentry.rb +++ b/lib/gitlab/sentry.rb @@ -1,9 +1,7 @@ module Gitlab module Sentry - extend Gitlab::CurrentSettings - def self.enabled? - Rails.env.production? && current_application_settings.sentry_enabled? + Rails.env.production? && Gitlab::CurrentSettings.sentry_enabled? end def self.context(current_user = nil) diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb index d01213bb6e0..e90a90508a2 100644 --- a/lib/gitlab/setup_helper.rb +++ b/lib/gitlab/setup_helper.rb @@ -31,7 +31,7 @@ module Gitlab storages << { name: 'test_second_storage', path: Rails.root.join('tmp', 'tests', 'second_storage').to_s } end - config = { socket_path: address.sub(%r{\Aunix:}, ''), storage: storages } + config = { socket_path: address.sub(/\Aunix:/, ''), storage: storages } config[:auth] = { token: 'secret' } if Rails.env.test? config[:'gitaly-ruby'] = { dir: File.join(gitaly_dir, 'ruby') } if gitaly_ruby config[:'gitlab-shell'] = { dir: Gitlab.config.gitlab_shell.path } diff --git a/lib/gitlab/sherlock/file_sample.rb b/lib/gitlab/sherlock/file_sample.rb index 8a3e1a5e5bf..89072b01f2e 100644 --- a/lib/gitlab/sherlock/file_sample.rb +++ b/lib/gitlab/sherlock/file_sample.rb @@ -16,7 +16,7 @@ module Gitlab end def relative_path - @relative_path ||= @file.gsub(/^#{Rails.root.to_s}\/?/, '') + @relative_path ||= @file.gsub(%r{^#{Rails.root.to_s}/?}, '') end def to_param diff --git a/lib/gitlab/sherlock/middleware.rb b/lib/gitlab/sherlock/middleware.rb index 687332fc5fc..4c88e33699a 100644 --- a/lib/gitlab/sherlock/middleware.rb +++ b/lib/gitlab/sherlock/middleware.rb @@ -2,7 +2,7 @@ module Gitlab module Sherlock # Rack middleware used for tracking request metrics. class Middleware - CONTENT_TYPES = /text\/html|application\/json/i + CONTENT_TYPES = %r{text/html|application/json}i IGNORE_PATHS = %r{^/sherlock} diff --git a/lib/gitlab/sherlock/query.rb b/lib/gitlab/sherlock/query.rb index 948bf5e6528..02ddc3f47eb 100644 --- a/lib/gitlab/sherlock/query.rb +++ b/lib/gitlab/sherlock/query.rb @@ -4,7 +4,7 @@ module Gitlab attr_reader :id, :query, :started_at, :finished_at, :backtrace # SQL identifiers that should be prefixed with newlines. - PREFIX_NEWLINE = / + PREFIX_NEWLINE = %r{ \s+(FROM |(LEFT|RIGHT)?INNER\s+JOIN |(LEFT|RIGHT)?OUTER\s+JOIN @@ -13,7 +13,7 @@ module Gitlab |GROUP\s+BY |ORDER\s+BY |LIMIT - |OFFSET)\s+/ix # Vim indent breaks when this is on a newline :< + |OFFSET)\s+}ix # Vim indent breaks when this is on a newline :< # Creates a new Query using a String and a separate Array of bindings. # diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb index 2bfb7caefd9..b89ae2505c9 100644 --- a/lib/gitlab/sidekiq_middleware/memory_killer.rb +++ b/lib/gitlab/sidekiq_middleware/memory_killer.rb @@ -45,7 +45,7 @@ module Gitlab private def get_rss - output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{pid})) + output, status = Gitlab::Popen.popen(%W(ps -o rss= -p #{pid}), Rails.root.to_s) return 0 unless status.zero? output.to_i diff --git a/lib/gitlab/ssh_public_key.rb b/lib/gitlab/ssh_public_key.rb index 89ca1298120..545e7c74f7e 100644 --- a/lib/gitlab/ssh_public_key.rb +++ b/lib/gitlab/ssh_public_key.rb @@ -21,6 +21,22 @@ module Gitlab technology(name)&.supported_sizes end + def self.sanitize(key_content) + ssh_type, *parts = key_content.strip.split + + return key_content if parts.empty? + + parts.each_with_object("#{ssh_type} ").with_index do |(part, content), index| + content << part + + if Gitlab::SSHPublicKey.new(content).valid? + break [content, parts[index + 1]].compact.join(' ') # Add the comment part if present + elsif parts.size == index + 1 # return original content if we've reached the last element + break key_content + end + end + end + attr_reader :key_text, :key # Unqualified MD5 fingerprint for compatibility @@ -37,23 +53,23 @@ module Gitlab end def valid? - key.present? + key.present? && bits && technology.supported_sizes.include?(bits) end def type - technology.name if valid? + technology.name if key.present? end def bits - return unless valid? + return if key.blank? case type when :rsa - key.n.num_bits + key.n&.num_bits when :dsa - key.p.num_bits + key.p&.num_bits when :ecdsa - key.group.order.num_bits + key.group.order&.num_bits when :ed25519 256 else diff --git a/lib/gitlab/upgrader.rb b/lib/gitlab/upgrader.rb index d545f2f95f1..024be6aca44 100644 --- a/lib/gitlab/upgrader.rb +++ b/lib/gitlab/upgrader.rb @@ -49,7 +49,7 @@ module Gitlab def fetch_git_tags remote_tags, _ = Gitlab::Popen.popen(%W(#{Gitlab.config.git.bin_path} ls-remote --tags https://gitlab.com/gitlab-org/gitlab-ce.git)) - remote_tags.split("\n").grep(/tags\/v#{current_version.major}/) + remote_tags.split("\n").grep(%r{tags/v#{current_version.major}}) end def update_commands diff --git a/lib/gitlab/uploads_transfer.rb b/lib/gitlab/uploads_transfer.rb index b5f41240529..7d7400bdabf 100644 --- a/lib/gitlab/uploads_transfer.rb +++ b/lib/gitlab/uploads_transfer.rb @@ -1,7 +1,7 @@ module Gitlab class UploadsTransfer < ProjectTransfer def root_dir - File.join(CarrierWave.root, FileUploader.base_dir) + FileUploader.root end end end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 2adcc9809b3..9d13d1d781f 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -1,8 +1,6 @@ module Gitlab class UsageData class << self - include Gitlab::CurrentSettings - def data(force_refresh: false) Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) { uncached_data } end @@ -19,7 +17,7 @@ module Gitlab def license_usage_data usage_data = { - uuid: current_application_settings.uuid, + uuid: Gitlab::CurrentSettings.uuid, hostname: Gitlab.config.gitlab.host, version: Gitlab::VERSION, active_user_count: User.active.count, @@ -79,9 +77,9 @@ module Gitlab def features_usage_data_ce { - signup: current_application_settings.allow_signup?, + signup: Gitlab::CurrentSettings.allow_signup?, ldap: Gitlab.config.ldap.enabled, - gravatar: current_application_settings.gravatar_enabled?, + gravatar: Gitlab::CurrentSettings.gravatar_enabled?, omniauth: Gitlab.config.omniauth.enabled, reply_by_email: Gitlab::IncomingEmail.enabled?, container_registry: Gitlab.config.registry.enabled, diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index 6ced06a863d..2612208a927 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -5,7 +5,6 @@ # module Gitlab module VisibilityLevel - extend CurrentSettings extend ActiveSupport::Concern included do @@ -58,9 +57,9 @@ module Gitlab end def allowed_levels - restricted_levels = current_application_settings.restricted_visibility_levels + restricted_levels = Gitlab::CurrentSettings.restricted_visibility_levels - self.values - restricted_levels + self.values - Array(restricted_levels) end def closest_allowed_level(target_level) @@ -81,7 +80,7 @@ module Gitlab end def non_restricted_level?(level) - restricted_levels = current_application_settings.restricted_visibility_levels + restricted_levels = Gitlab::CurrentSettings.restricted_visibility_levels if restricted_levels.nil? true diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 633da44b22d..823df67ea39 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -55,14 +55,14 @@ module Gitlab def lfs_upload_ok(oid, size) { - StoreLFSPath: "#{Gitlab.config.lfs.storage_path}/tmp/upload", + StoreLFSPath: LfsObjectUploader.workhorse_upload_path, LfsOid: oid, LfsSize: size } end def artifact_upload_ok - { TempPath: JobArtifactUploader.artifacts_upload_path } + { TempPath: JobArtifactUploader.workhorse_upload_path } end def send_git_blob(repository, blob) @@ -147,8 +147,11 @@ module Gitlab end def send_artifacts_entry(build, entry) + file = build.artifacts_file + archive = file.file_storage? ? file.path : file.url + params = { - 'Archive' => build.artifacts_file.path, + 'Archive' => archive, 'Entry' => Base64.encode64(entry.to_s) } @@ -158,6 +161,18 @@ module Gitlab ] end + def send_url(url, allow_redirects: false) + params = { + 'URL' => url, + 'AllowRedirects' => allow_redirects + } + + [ + SEND_DATA_HEADER, + "send-url:#{encode(params)}" + ] + end + def terminal_websocket(terminal) details = { 'Terminal' => { diff --git a/lib/tasks/flay.rake b/lib/tasks/flay.rake index b1e012e70c5..4b4881cecb8 100644 --- a/lib/tasks/flay.rake +++ b/lib/tasks/flay.rake @@ -2,7 +2,7 @@ desc 'Code duplication analyze via flay' task :flay do output = `bundle exec flay --mass 35 app/ lib/gitlab/ 2> #{File::NULL}` - if output.include? "Similar code found" + if output.include?("Similar code found") || output.include?("IDENTICAL code found") puts output exit 1 end |