diff options
Diffstat (limited to 'lib/gitlab')
31 files changed, 367 insertions, 128 deletions
diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 86393ee254d..f5ccf952cf9 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -40,8 +40,8 @@ module Gitlab end def find_with_user_password(login, password) - # Avoid resource intensive login checks if password is not provided - return unless password.present? + # Avoid resource intensive checks if login credentials are not provided + return unless login.present? && password.present? # Nothing to do here if internal auth is disabled and LDAP is # not configured @@ -50,14 +50,26 @@ module Gitlab Gitlab::Auth::UniqueIpsLimiter.limit_user! do user = User.by_login(login) - # If no user is found, or it's an LDAP server, try LDAP. - # LDAP users are only authenticated via LDAP - if user.nil? || user.ldap_user? - # Second chance - try LDAP authentication - Gitlab::Auth::LDAP::Authentication.login(login, password) - elsif Gitlab::CurrentSettings.password_authentication_enabled_for_git? - user if user.active? && user.valid_password?(password) + return if user && !user.active? + + authenticators = [] + + if user + authenticators << Gitlab::Auth::OAuth::Provider.authentication(user, 'database') + + # Add authenticators for all identities if user is not nil + user&.identities&.each do |identity| + authenticators << Gitlab::Auth::OAuth::Provider.authentication(user, identity.provider) + end + else + # If no user is provided, try LDAP. + # LDAP users are only authenticated via LDAP + authenticators << Gitlab::Auth::LDAP::Authentication end + + authenticators.compact! + + user if authenticators.find { |auth| auth.login(login, password) } end end diff --git a/lib/gitlab/auth/database/authentication.rb b/lib/gitlab/auth/database/authentication.rb new file mode 100644 index 00000000000..260a77058a4 --- /dev/null +++ b/lib/gitlab/auth/database/authentication.rb @@ -0,0 +1,16 @@ +# These calls help to authenticate to OAuth provider by providing username and password +# + +module Gitlab + module Auth + module Database + class Authentication < Gitlab::Auth::OAuth::Authentication + def login(login, password) + return false unless Gitlab::CurrentSettings.password_authentication_enabled_for_git? + + user&.valid_password?(password) + end + end + end + end +end diff --git a/lib/gitlab/auth/ldap/authentication.rb b/lib/gitlab/auth/ldap/authentication.rb index cbb9cf4bb9c..e70c3ab6b46 100644 --- a/lib/gitlab/auth/ldap/authentication.rb +++ b/lib/gitlab/auth/ldap/authentication.rb @@ -7,7 +7,7 @@ module Gitlab module Auth module LDAP - class Authentication + class Authentication < Gitlab::Auth::OAuth::Authentication def self.login(login, password) return unless Gitlab::Auth::LDAP::Config.enabled? return unless login.present? && password.present? @@ -28,11 +28,7 @@ module Gitlab Gitlab::Auth::LDAP::Config.providers end - attr_accessor :provider, :ldap_user - - def initialize(provider) - @provider = provider - end + attr_accessor :ldap_user def login(login, password) @ldap_user = adapter.bind_as( @@ -62,7 +58,7 @@ module Gitlab end def user - return nil unless ldap_user + return unless ldap_user Gitlab::Auth::LDAP::User.find_by_uid_and_provider(ldap_user.dn, provider) end diff --git a/lib/gitlab/auth/o_auth/authentication.rb b/lib/gitlab/auth/o_auth/authentication.rb new file mode 100644 index 00000000000..ed03b9f8b40 --- /dev/null +++ b/lib/gitlab/auth/o_auth/authentication.rb @@ -0,0 +1,21 @@ +# These calls help to authenticate to OAuth provider by providing username and password +# + +module Gitlab + module Auth + module OAuth + class Authentication + attr_reader :provider, :user + + def initialize(provider, user = nil) + @provider = provider + @user = user + end + + def login(login, password) + raise NotImplementedError + end + end + end + end +end diff --git a/lib/gitlab/auth/o_auth/provider.rb b/lib/gitlab/auth/o_auth/provider.rb index f8ab8ee1388..5fb61ffe00d 100644 --- a/lib/gitlab/auth/o_auth/provider.rb +++ b/lib/gitlab/auth/o_auth/provider.rb @@ -8,11 +8,28 @@ module Gitlab "google_oauth2" => "Google" }.freeze + def self.authentication(user, provider) + return unless user + return unless enabled?(provider) + + authenticator = + case provider + when /^ldap/ + Gitlab::Auth::LDAP::Authentication + when 'database' + Gitlab::Auth::Database::Authentication + end + + authenticator&.new(provider, user) + end + def self.providers Devise.omniauth_providers end def self.enabled?(name) + return true if name == 'database' + providers.include?(name.to_sym) end diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index acd785bb02d..b6a96081278 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -161,7 +161,7 @@ module Gitlab def find_by_uid_and_provider identity = Identity.with_extern_uid(auth_hash.provider, auth_hash.uid).take - identity && identity.user + identity&.user end def build_new_user diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index f2e5124c8a8..cedf4171ab1 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -1,6 +1,8 @@ module Gitlab module Ci class Trace + ArchiveError = Class.new(StandardError) + attr_reader :job delegate :old_trace, to: :job @@ -93,8 +95,53 @@ module Gitlab job.erase_old_trace! end + def archive! + raise ArchiveError, 'Already archived' if trace_artifact + raise ArchiveError, 'Job is not finished yet' unless job.complete? + + if current_path + File.open(current_path) do |stream| + archive_stream!(stream) + FileUtils.rm(current_path) + end + elsif old_trace + StringIO.new(old_trace, 'rb').tap do |stream| + archive_stream!(stream) + job.erase_old_trace! + end + end + end + private + def archive_stream!(stream) + clone_file!(stream, JobArtifactUploader.workhorse_upload_path) do |clone_path| + create_job_trace!(job, clone_path) + end + end + + def clone_file!(src_stream, temp_dir) + FileUtils.mkdir_p(temp_dir) + Dir.mktmpdir('tmp-trace', temp_dir) do |dir_path| + temp_path = File.join(dir_path, "job.log") + FileUtils.touch(temp_path) + size = IO.copy_stream(src_stream, temp_path) + raise ArchiveError, 'Failed to copy stream' unless size == src_stream.size + + yield(temp_path) + end + end + + def create_job_trace!(job, path) + File.open(path) do |stream| + job.create_job_artifacts_trace!( + project: job.project, + file_type: :trace, + file: stream, + file_sha256: Digest::SHA256.file(path).hexdigest) + end + end + def ensure_path return current_path if current_path diff --git a/lib/gitlab/git/branch.rb b/lib/gitlab/git/branch.rb index ae7e88f0503..6351cfb83e3 100644 --- a/lib/gitlab/git/branch.rb +++ b/lib/gitlab/git/branch.rb @@ -1,6 +1,8 @@ module Gitlab module Git class Branch < Ref + STALE_BRANCH_THRESHOLD = 3.months + def self.find(repo, branch_name) if branch_name.is_a?(Gitlab::Git::Branch) branch_name @@ -12,6 +14,18 @@ module Gitlab def initialize(repository, name, target, target_commit) super(repository, name, target, target_commit) end + + def active? + self.dereferenced_target.committed_date >= STALE_BRANCH_THRESHOLD.ago + end + + def stale? + !active? + end + + def state + active? ? :active : :stale + end end end end diff --git a/lib/gitlab/git/gitlab_projects.rb b/lib/gitlab/git/gitlab_projects.rb index e5a747cb987..5e1e22ae65c 100644 --- a/lib/gitlab/git/gitlab_projects.rb +++ b/lib/gitlab/git/gitlab_projects.rb @@ -63,11 +63,12 @@ module Gitlab end end - def fetch_remote(name, timeout, force:, tags:, ssh_key: nil, known_hosts: nil) + def fetch_remote(name, timeout, force:, tags:, ssh_key: nil, known_hosts: nil, prune: true) tags_option = tags ? '--tags' : '--no-tags' logger.info "Fetching remote #{name} for repository #{repository_absolute_path}." - cmd = %W(git fetch #{name} --prune --quiet) + cmd = %W(git fetch #{name} --quiet) + cmd << '--prune' if prune cmd << '--force' if force cmd << tags_option diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index fdb3247cf4d..e1bc2f9ab61 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -45,10 +45,10 @@ module Gitlab GitalyClient.call(@storage, :repository_service, :apply_gitattributes, request) end - def fetch_remote(remote, ssh_auth:, forced:, no_tags:, timeout:) + def fetch_remote(remote, ssh_auth:, forced:, no_tags:, timeout:, prune: true) request = Gitaly::FetchRemoteRequest.new( repository: @gitaly_repo, remote: remote, force: forced, - no_tags: no_tags, timeout: timeout + no_tags: no_tags, timeout: timeout, no_prune: !prune ) if ssh_auth&.ssh_import? diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 92f0e0402a8..a7e055ac444 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -20,6 +20,7 @@ module Gitlab gon.sprite_icons = IconsHelper.sprite_icon_path gon.sprite_file_icons = IconsHelper.sprite_file_icons_path gon.test_env = Rails.env.test? + gon.suggested_label_colors = LabelsHelper.suggested_colors if current_user gon.current_user_id = current_user.id diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index bdc0f04b56b..3772ef11c7f 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -18,7 +18,10 @@ module Gitlab 'uk' => 'Українська', 'ja' => '日本語', 'ko' => '한국어', - 'nl_NL' => 'Nederlands' + 'nl_NL' => 'Nederlands', + 'tr_TR' => 'Türkçe', + 'id_ID' => 'Bahasa Indonesia', + 'fil_PH' => 'Filipino' }.freeze def available_locales diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index af203ff711d..b713fa7e1cd 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -3,7 +3,7 @@ module Gitlab extend self # For every version update, the version history in import_export.md has to be kept up to date. - VERSION = '0.2.2'.freeze + VERSION = '0.2.3'.freeze FILENAME_LIMIT = 50 def export_path(relative_path:) diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 9f404003125..4bdd01f5e94 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -65,6 +65,7 @@ project_tree: - :create_access_levels - :project_feature - :custom_attributes + - :project_badges # Only include the following attributes for the models specified. included_attributes: @@ -125,6 +126,8 @@ excluded_attributes: - :when push_event_payload: - :event_id + project_badges: + - :group_id methods: labels: @@ -147,3 +150,5 @@ methods: - :action push_event_payload: - :action + project_badges: + - :type diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index a00795f553e..c38df9102eb 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -9,7 +9,7 @@ module Gitlab @archive_file = project.import_source @current_user = project.creator @project = project - @shared = Gitlab::ImportExport::Shared.new(relative_path: path_with_namespace) + @shared = project.import_export_shared end def execute diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 759833a5ee5..cf6b7e306dd 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -16,7 +16,8 @@ module Gitlab priorities: :label_priorities, auto_devops: :project_auto_devops, label: :project_label, - custom_attributes: 'ProjectCustomAttribute' }.freeze + custom_attributes: 'ProjectCustomAttribute', + project_badges: 'Badge' }.freeze USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id created_by_id last_edited_by_id merge_user_id resolved_by_id].freeze diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb index b34cafc6876..3d3d998a6a3 100644 --- a/lib/gitlab/import_export/shared.rb +++ b/lib/gitlab/import_export/shared.rb @@ -1,13 +1,17 @@ module Gitlab module ImportExport class Shared - attr_reader :errors, :opts + attr_reader :errors, :project - def initialize(opts) - @opts = opts + def initialize(project) + @project = project @errors = [] end + def active_export_count + Dir[File.join(archive_path, '*')].count { |name| File.directory?(name) } + end + def export_path @export_path ||= Gitlab::ImportExport.export_path(relative_path: relative_path) end @@ -31,11 +35,11 @@ module Gitlab private def relative_path - File.join(opts[:relative_path], SecureRandom.hex) + File.join(relative_archive_path, SecureRandom.hex) end def relative_archive_path - File.join(opts[:relative_path], '..') + @project.disk_path end def error_out(message, caller) diff --git a/lib/gitlab/middleware/read_only.rb b/lib/gitlab/middleware/read_only.rb index c26656704d7..d9d5f90596f 100644 --- a/lib/gitlab/middleware/read_only.rb +++ b/lib/gitlab/middleware/read_only.rb @@ -1,90 +1,19 @@ module Gitlab module Middleware class ReadOnly - DISALLOWED_METHODS = %w(POST PATCH PUT DELETE).freeze - APPLICATION_JSON = 'application/json'.freeze API_VERSIONS = (3..4) + def self.internal_routes + @internal_routes ||= + API_VERSIONS.map { |version| "api/v#{version}/internal" } + end + def initialize(app) @app = app - @whitelisted = internal_routes end def call(env) - @env = env - @route_hash = nil - - if disallowed_request? && Gitlab::Database.read_only? - Rails.logger.debug('GitLab ReadOnly: preventing possible non read-only operation') - error_message = 'You cannot do writing operations on a read-only GitLab instance' - - if json_request? - return [403, { 'Content-Type' => 'application/json' }, [{ 'message' => error_message }.to_json]] - else - rack_flash.alert = error_message - rack_session['flash'] = rack_flash.to_session_value - - return [301, { 'Location' => last_visited_url }, []] - end - end - - @app.call(env) - end - - private - - def internal_routes - API_VERSIONS.flat_map { |version| "api/v#{version}/internal" } - end - - def disallowed_request? - DISALLOWED_METHODS.include?(@env['REQUEST_METHOD']) && !whitelisted_routes - end - - def json_request? - request.media_type == APPLICATION_JSON - end - - def rack_flash - @rack_flash ||= ActionDispatch::Flash::FlashHash.from_session_value(rack_session) - end - - def rack_session - @env['rack.session'] - end - - def request - @env['rack.request'] ||= Rack::Request.new(@env) - end - - def last_visited_url - @env['HTTP_REFERER'] || rack_session['user_return_to'] || Gitlab::Routing.url_helpers.root_url - end - - def route_hash - @route_hash ||= Rails.application.routes.recognize_path(request.url, { method: request.request_method }) rescue {} - end - - def whitelisted_routes - grack_route || @whitelisted.any? { |path| request.path.include?(path) } || lfs_route || sidekiq_route - end - - def sidekiq_route - request.path.start_with?('/admin/sidekiq') - end - - def grack_route - # Calling route_hash may be expensive. Only do it if we think there's a possible match - return false unless request.path.end_with?('.git/git-upload-pack') - - route_hash[:controller] == 'projects/git_http' && route_hash[:action] == 'git_upload_pack' - end - - def lfs_route - # Calling route_hash may be expensive. Only do it if we think there's a possible match - return false unless request.path.end_with?('/info/lfs/objects/batch') - - route_hash[:controller] == 'projects/lfs_api' && route_hash[:action] == 'batch' + ReadOnly::Controller.new(@app, env).call end end end diff --git a/lib/gitlab/middleware/read_only/controller.rb b/lib/gitlab/middleware/read_only/controller.rb new file mode 100644 index 00000000000..45b644e6510 --- /dev/null +++ b/lib/gitlab/middleware/read_only/controller.rb @@ -0,0 +1,86 @@ +module Gitlab + module Middleware + class ReadOnly + class Controller + DISALLOWED_METHODS = %w(POST PATCH PUT DELETE).freeze + APPLICATION_JSON = 'application/json'.freeze + ERROR_MESSAGE = 'You cannot perform write operations on a read-only instance'.freeze + + def initialize(app, env) + @app = app + @env = env + end + + def call + if disallowed_request? && Gitlab::Database.read_only? + Rails.logger.debug('GitLab ReadOnly: preventing possible non read-only operation') + + if json_request? + return [403, { 'Content-Type' => APPLICATION_JSON }, [{ 'message' => ERROR_MESSAGE }.to_json]] + else + rack_flash.alert = ERROR_MESSAGE + rack_session['flash'] = rack_flash.to_session_value + + return [301, { 'Location' => last_visited_url }, []] + end + end + + @app.call(@env) + end + + private + + def disallowed_request? + DISALLOWED_METHODS.include?(@env['REQUEST_METHOD']) && + !whitelisted_routes + end + + def json_request? + request.media_type == APPLICATION_JSON + end + + def rack_flash + @rack_flash ||= ActionDispatch::Flash::FlashHash.from_session_value(rack_session) + end + + def rack_session + @env['rack.session'] + end + + def request + @env['rack.request'] ||= Rack::Request.new(@env) + end + + def last_visited_url + @env['HTTP_REFERER'] || rack_session['user_return_to'] || Gitlab::Routing.url_helpers.root_url + end + + def route_hash + @route_hash ||= Rails.application.routes.recognize_path(request.url, { method: request.request_method }) rescue {} + end + + def whitelisted_routes + grack_route || ReadOnly.internal_routes.any? { |path| request.path.include?(path) } || lfs_route || sidekiq_route + end + + def sidekiq_route + request.path.start_with?('/admin/sidekiq') + end + + def grack_route + # Calling route_hash may be expensive. Only do it if we think there's a possible match + return false unless request.path.end_with?('.git/git-upload-pack') + + route_hash[:controller] == 'projects/git_http' && route_hash[:action] == 'git_upload_pack' + end + + def lfs_route + # Calling route_hash may be expensive. Only do it if we think there's a possible match + return false unless request.path.end_with?('/info/lfs/objects/batch') + + route_hash[:controller] == 'projects/lfs_api' && route_hash[:action] == 'batch' + end + end + end + end +end diff --git a/lib/gitlab/middleware/release_env.rb b/lib/gitlab/middleware/release_env.rb new file mode 100644 index 00000000000..f8d0a135965 --- /dev/null +++ b/lib/gitlab/middleware/release_env.rb @@ -0,0 +1,14 @@ +module Gitlab + module Middleware + # Some of middleware would hold env for no good reason even after the + # request had already been processed, and we could not garbage collect + # them due to this. Put this middleware as the first middleware so that + # it would clear the env after the request is done, allowing GC gets a + # chance to release memory for the last request. + ReleaseEnv = Struct.new(:app) do + def call(env) + app.call(env).tap { env.clear } + end + end + end +end diff --git a/lib/gitlab/prometheus/additional_metrics_parser.rb b/lib/gitlab/prometheus/additional_metrics_parser.rb index cb95daf2260..bb1172f82a1 100644 --- a/lib/gitlab/prometheus/additional_metrics_parser.rb +++ b/lib/gitlab/prometheus/additional_metrics_parser.rb @@ -1,10 +1,12 @@ module Gitlab module Prometheus module AdditionalMetricsParser + CONFIG_ROOT = 'config/prometheus'.freeze + MUTEX = Mutex.new extend self - def load_groups_from_yaml - additional_metrics_raw.map(&method(:group_from_entry)) + def load_groups_from_yaml(file_name = 'additional_metrics.yml') + yaml_metrics_raw(file_name).map(&method(:group_from_entry)) end private @@ -22,13 +24,20 @@ module Gitlab MetricGroup.new(entry).tap(&method(:validate!)) end - def additional_metrics_raw - load_yaml_file&.map(&:deep_symbolize_keys).freeze + def yaml_metrics_raw(file_name) + load_yaml_file(file_name)&.map(&:deep_symbolize_keys).freeze end - def load_yaml_file - @loaded_yaml_file ||= YAML.load_file(Rails.root.join('config/prometheus/additional_metrics.yml')) + # rubocop:disable Gitlab/ModuleWithInstanceVariables + def load_yaml_file(file_name) + return YAML.load_file(Rails.root.join(CONFIG_ROOT, file_name)) if Rails.env.development? + + MUTEX.synchronize do + @loaded_yaml_cache ||= {} + @loaded_yaml_cache[file_name] ||= YAML.load_file(Rails.root.join(CONFIG_ROOT, file_name)) + end end + # rubocop:enable Gitlab/ModuleWithInstanceVariables end end end diff --git a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb index 972ab75d1d5..e677ec84cd4 100644 --- a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb +++ b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb @@ -4,7 +4,7 @@ module Gitlab class AdditionalMetricsDeploymentQuery < BaseQuery include QueryAdditionalMetrics - def query(environment_id, deployment_id) + def query(deployment_id) Deployment.find_by(id: deployment_id).try do |deployment| query_metrics( deployment.project, diff --git a/lib/gitlab/prometheus/queries/base_query.rb b/lib/gitlab/prometheus/queries/base_query.rb index c60828165bd..29cab6e9c15 100644 --- a/lib/gitlab/prometheus/queries/base_query.rb +++ b/lib/gitlab/prometheus/queries/base_query.rb @@ -20,6 +20,10 @@ module Gitlab def query(*args) raise NotImplementedError end + + def self.transform_reactive_result(result) + result + end end end end diff --git a/lib/gitlab/prometheus/queries/deployment_query.rb b/lib/gitlab/prometheus/queries/deployment_query.rb index 6e6da593178..c2626581897 100644 --- a/lib/gitlab/prometheus/queries/deployment_query.rb +++ b/lib/gitlab/prometheus/queries/deployment_query.rb @@ -2,7 +2,7 @@ module Gitlab module Prometheus module Queries class DeploymentQuery < BaseQuery - def query(environment_id, deployment_id) + def query(deployment_id) Deployment.find_by(id: deployment_id).try do |deployment| environment_slug = deployment.environment.slug @@ -25,6 +25,11 @@ module Gitlab } end end + + def self.transform_reactive_result(result) + result[:metrics] = result.delete :data + result + end end end end diff --git a/lib/gitlab/prometheus/queries/environment_query.rb b/lib/gitlab/prometheus/queries/environment_query.rb index 1d17d3cfd56..b62910c8de6 100644 --- a/lib/gitlab/prometheus/queries/environment_query.rb +++ b/lib/gitlab/prometheus/queries/environment_query.rb @@ -19,6 +19,11 @@ module Gitlab } end end + + def self.transform_reactive_result(result) + result[:metrics] = result.delete :data + result + end end end end diff --git a/lib/gitlab/prometheus/queries/matched_metrics_query.rb b/lib/gitlab/prometheus/queries/matched_metric_query.rb index 5710ad47c1a..d920e9a749f 100644 --- a/lib/gitlab/prometheus/queries/matched_metrics_query.rb +++ b/lib/gitlab/prometheus/queries/matched_metric_query.rb @@ -1,7 +1,7 @@ module Gitlab module Prometheus module Queries - class MatchedMetricsQuery < BaseQuery + class MatchedMetricQuery < BaseQuery MAX_QUERY_ITEMS = 40.freeze def query diff --git a/lib/gitlab/prometheus/queries/query_additional_metrics.rb b/lib/gitlab/prometheus/queries/query_additional_metrics.rb index 0c280dc9a3c..aad76e335af 100644 --- a/lib/gitlab/prometheus/queries/query_additional_metrics.rb +++ b/lib/gitlab/prometheus/queries/query_additional_metrics.rb @@ -3,9 +3,16 @@ module Gitlab module Queries module QueryAdditionalMetrics def query_metrics(project, query_context) + matched_metrics(project).map(&query_group(query_context)) + .select(&method(:group_with_any_metrics)) + end + + protected + + def query_group(query_context) query_processor = method(:process_query).curry[query_context] - groups = matched_metrics(project).map do |group| + lambda do |group| metrics = group.metrics.map do |metric| { title: metric.title, @@ -21,8 +28,6 @@ module Gitlab metrics: metrics.select(&method(:metric_with_any_queries)) } end - - groups.select(&method(:group_with_any_metrics)) end private @@ -72,12 +77,17 @@ module Gitlab end def common_query_context(environment, timeframe_start:, timeframe_end:) - { - timeframe_start: timeframe_start, - timeframe_end: timeframe_end, + base_query_context(timeframe_start, timeframe_end).merge({ ci_environment_slug: environment.slug, kube_namespace: environment.project.deployment_platform&.actual_namespace || '', environment_filter: %{container_name!="POD",environment="#{environment.slug}"} + }) + end + + def base_query_context(timeframe_start, timeframe_end) + { + timeframe_start: timeframe_start, + timeframe_end: timeframe_end } end end diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb index 659021c9ac9..b66253a10e0 100644 --- a/lib/gitlab/prometheus_client.rb +++ b/lib/gitlab/prometheus_client.rb @@ -57,7 +57,11 @@ module Gitlab rescue OpenSSL::SSL::SSLError raise PrometheusClient::Error, "#{rest_client.url} contains invalid SSL data" rescue RestClient::ExceptionWithResponse => ex - handle_exception_response(ex.response) + if ex.response + handle_exception_response(ex.response) + else + raise PrometheusClient::Error, "Network connection error" + end rescue RestClient::Exception raise PrometheusClient::Error, "Network connection error" end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 4ba44e0feef..dda7afc0999 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -125,13 +125,13 @@ module Gitlab # Ex. # fetch_remote(my_repo, "upstream") # - def fetch_remote(repository, remote, ssh_auth: nil, forced: false, no_tags: false) + def fetch_remote(repository, remote, ssh_auth: nil, forced: false, no_tags: false, prune: true) gitaly_migrate(:fetch_remote) do |is_enabled| if is_enabled - repository.gitaly_repository_client.fetch_remote(remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, timeout: git_timeout) + repository.gitaly_repository_client.fetch_remote(remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, timeout: git_timeout, prune: prune) else storage_path = Gitlab.config.repositories.storages[repository.storage]["path"] - local_fetch_remote(storage_path, repository.relative_path, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags) + local_fetch_remote(storage_path, repository.relative_path, remote, ssh_auth: ssh_auth, forced: forced, no_tags: no_tags, prune: prune) end end end @@ -428,8 +428,8 @@ module Gitlab ) end - def local_fetch_remote(storage_path, repository_relative_path, remote, ssh_auth: nil, forced: false, no_tags: false) - vars = { force: forced, tags: !no_tags } + def local_fetch_remote(storage_path, repository_relative_path, remote, ssh_auth: nil, forced: false, no_tags: false, prune: true) + vars = { force: forced, tags: !no_tags, prune: prune } if ssh_auth&.ssh_import? if ssh_auth.ssh_key_auth? && ssh_auth.ssh_private_key.present? diff --git a/lib/gitlab/string_placeholder_replacer.rb b/lib/gitlab/string_placeholder_replacer.rb new file mode 100644 index 00000000000..9a2219b7d77 --- /dev/null +++ b/lib/gitlab/string_placeholder_replacer.rb @@ -0,0 +1,27 @@ +module Gitlab + class StringPlaceholderReplacer + # This method accepts the following paras + # - string: the string to be analyzed + # - placeholder_regex: i.e. /%{project_path|project_id|default_branch|commit_sha}/ + # - block: this block will be called with each placeholder found in the string using + # the placeholder regex. If the result of the block is nil, the original + # placeholder will be returned. + + def self.replace_string_placeholders(string, placeholder_regex = nil, &block) + return string if string.blank? || placeholder_regex.blank? || !block_given? + + replace_placeholders(string, placeholder_regex, &block) + end + + class << self + private + + # If the result of the block is nil, then the placeholder is returned + def replace_placeholders(string, placeholder_regex, &block) + string.gsub(/%{(#{placeholder_regex})}/) do |arg| + yield($~[1]) || arg + end + end + end + end +end diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index fa22f0e37b2..dc9391f32cf 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -67,5 +67,13 @@ module Gitlab nil end + + # Used in EE + # Accepts either an Array or a String and returns an array + def ensure_array_from_string(string_or_array) + return string_or_array if string_or_array.is_a?(Array) + + string_or_array.split(',').map(&:strip) + end end end |