diff options
Diffstat (limited to 'lib/gitlab/gitaly_client.rb')
| -rw-r--r-- | lib/gitlab/gitaly_client.rb | 121 |
1 files changed, 74 insertions, 47 deletions
diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 5aeedb0f50d..e683d4e5bbe 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -26,18 +26,16 @@ module Gitlab end end - PEM_REGEX = /\-+BEGIN CERTIFICATE\-+.+?\-+END CERTIFICATE\-+/m + PEM_REGEX = /\-+BEGIN CERTIFICATE\-+.+?\-+END CERTIFICATE\-+/m.freeze SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION' MAXIMUM_GITALY_CALLS = 30 CLIENT_NAME = (Sidekiq.server? ? 'gitlab-sidekiq' : 'gitlab-web').freeze - MUTEX = Mutex.new - - class << self - attr_accessor :query_time - end + SERVER_FEATURE_CATFILE_CACHE = 'catfile-cache'.freeze + # Server feature flags should use '_' to separate words. + SERVER_FEATURE_FLAGS = [SERVER_FEATURE_CATFILE_CACHE].freeze - self.query_time = 0 + MUTEX = Mutex.new define_histogram :gitaly_controller_action_duration_seconds do docstring "Gitaly endpoint histogram by controller and action combination" @@ -58,9 +56,9 @@ module Gitlab end def self.interceptors - return [] unless Gitlab::Tracing.enabled? + return [] unless Labkit::Tracing.enabled? - [Gitlab::Tracing::GRPCInterceptor.instance] + [Labkit::Tracing::GRPCInterceptor.instance] end private_class_method :interceptors @@ -75,13 +73,11 @@ module Gitlab @certs = stub_cert_paths.flat_map do |cert_file| File.read(cert_file).scan(PEM_REGEX).map do |cert| - begin - OpenSSL::X509::Certificate.new(cert).to_pem - rescue OpenSSL::OpenSSLError => e - Rails.logger.error "Could not load certificate #{cert_file} #{e}" - Gitlab::Sentry.track_exception(e, extra: { cert_file: cert_file }) - nil - end + OpenSSL::X509::Certificate.new(cert).to_pem + rescue OpenSSL::OpenSSLError => e + Rails.logger.error "Could not load certificate #{cert_file} #{e}" + Gitlab::Sentry.track_exception(e, extra: { cert_file: cert_file }) + nil end.compact end.uniq.join("\n") end @@ -164,8 +160,6 @@ module Gitlab kwargs = yield(kwargs) if block_given? stub(service, storage).__send__(rpc, request, kwargs) # rubocop:disable GitlabSecurity/PublicSend - rescue GRPC::Unavailable => ex - handle_grpc_unavailable!(ex) ensure duration = Gitlab::Metrics::System.monotonic_time - start @@ -175,29 +169,23 @@ module Gitlab current_transaction_labels.merge(gitaly_service: service.to_s, rpc: rpc.to_s), duration) - add_call_details(feature: "#{service}##{rpc}", duration: duration, request: request_hash, rpc: rpc) + if peek_enabled? + add_call_details(feature: "#{service}##{rpc}", duration: duration, request: request_hash, rpc: rpc, + backtrace: Gitlab::Profiler.clean_backtrace(caller)) + end end - def self.handle_grpc_unavailable!(ex) - status = ex.to_status - raise ex unless status.details == 'Endpoint read failed' - - # There is a bug in grpc 1.8.x that causes a client process to get stuck - # always raising '14:Endpoint read failed'. The only thing that we can - # do to recover is to restart the process. - # - # See https://gitlab.com/gitlab-org/gitaly/issues/1029 + def self.query_time + SafeRequestStore[:gitaly_query_time] ||= 0 + end - if Sidekiq.server? - raise Gitlab::SidekiqMiddleware::Shutdown::WantShutdown.new(ex.to_s) - else - # SIGQUIT requests a Unicorn worker to shut down gracefully after the current request. - Process.kill('QUIT', Process.pid) - end + def self.query_time=(duration) + SafeRequestStore[:gitaly_query_time] = duration + end - raise ex + def self.query_time_ms + (self.query_time * 1000).round(2) end - private_class_method :handle_grpc_unavailable! def self.current_transaction_labels Gitlab::Metrics::Transaction.current&.labels || {} @@ -234,7 +222,8 @@ module Gitlab feature = feature_stack && feature_stack[0] metadata['call_site'] = feature.to_s if feature metadata['gitaly-servers'] = address_metadata(remote_storage) if remote_storage - metadata['x-gitlab-correlation-id'] = Gitlab::CorrelationId.current_id if Gitlab::CorrelationId.current_id + metadata['x-gitlab-correlation-id'] = Labkit::Correlation::CorrelationId.current_id if Labkit::Correlation::CorrelationId.current_id + metadata['gitaly-session-id'] = session_id if feature_enabled?(SERVER_FEATURE_CATFILE_CACHE) metadata.merge!(server_feature_flags) @@ -251,7 +240,9 @@ module Gitlab result end - SERVER_FEATURE_FLAGS = %w[].freeze + def self.session_id + Gitlab::SafeRequestStore[:gitaly_session_id] ||= SecureRandom.uuid + end def self.server_feature_flags SERVER_FEATURE_FLAGS.map do |f| @@ -267,7 +258,9 @@ module Gitlab end def self.feature_enabled?(feature_name) - Feature.enabled?("gitaly_#{feature_name}") + Feature::FlipperFeature.table_exists? && Feature.enabled?("gitaly_#{feature_name}") + rescue ActiveRecord::NoDatabaseError + false end # Ensures that Gitaly is not being abuse through n+1 misuse etc @@ -278,8 +271,7 @@ module Gitlab # This is this actual number of times this call was made. Used for information purposes only actual_call_count = increment_call_count("gitaly_#{call_site}_actual") - # Do no enforce limits in production - return if Rails.env.production? || ENV["GITALY_DISABLE_REQUEST_LIMITS"] + return unless enforce_gitaly_request_limits? # Check if this call is nested within a allow_n_plus_1_calls # block and skip check if it is @@ -296,6 +288,19 @@ module Gitlab raise TooManyInvocationsError.new(call_site, actual_call_count, max_call_count, max_stacks) end + def self.enforce_gitaly_request_limits? + # We typically don't want to enforce request limits in production + # However, we have some production-like test environments, i.e., ones + # where `Rails.env.production?` returns `true`. We do want to be able to + # check if the limit is being exceeded while testing in those environments + # In that case we can use a feature flag to indicate that we do want to + # enforce request limits. + return true if feature_enabled?('enforce_requests_limits') + + !(Rails.env.production? || ENV["GITALY_DISABLE_REQUEST_LIMITS"]) + end + private_class_method :enforce_gitaly_request_limits? + def self.allow_n_plus_1_calls return yield unless Gitlab::SafeRequestStore.active? @@ -307,6 +312,26 @@ module Gitlab end end + # Normally a FindCommit RPC will cache the commit with its SHA + # instead of a ref name, since it's possible the branch is mutated + # afterwards. However, for read-only requests that never mutate the + # branch, this method allows caching of the ref name directly. + def self.allow_ref_name_caching + return yield unless Gitlab::SafeRequestStore.active? + return yield if ref_name_caching_allowed? + + begin + Gitlab::SafeRequestStore[:allow_ref_name_caching] = true + yield + ensure + Gitlab::SafeRequestStore[:allow_ref_name_caching] = false + end + end + + def self.ref_name_caching_allowed? + Gitlab::SafeRequestStore[:allow_ref_name_caching] + end + def self.get_call_count(key) Gitlab::SafeRequestStore[key] || 0 end @@ -335,15 +360,17 @@ module Gitlab Gitlab::SafeRequestStore["gitaly_call_permitted"] = 0 end - def self.add_call_details(details) - return unless Gitlab::SafeRequestStore[:peek_enabled] + def self.peek_enabled? + Gitlab::SafeRequestStore[:peek_enabled] + end + def self.add_call_details(details) Gitlab::SafeRequestStore['gitaly_call_details'] ||= [] Gitlab::SafeRequestStore['gitaly_call_details'] << details end def self.list_call_details - return [] unless Gitlab::SafeRequestStore[:peek_enabled] + return [] unless peek_enabled? Gitlab::SafeRequestStore['gitaly_call_details'] || [] end @@ -407,13 +434,13 @@ module Gitlab # Returns the stacks that calls Gitaly the most times. Used for n+1 detection def self.max_stacks - return nil unless Gitlab::SafeRequestStore.active? + return unless Gitlab::SafeRequestStore.active? stack_counter = Gitlab::SafeRequestStore[:stack_counter] - return nil unless stack_counter + return unless stack_counter max = max_call_count - return nil if max.zero? + return if max.zero? stack_counter.select { |_, v| v == max }.keys end |