diff options
Diffstat (limited to 'lib/gitlab/utils/usage_data.rb')
| -rw-r--r-- | lib/gitlab/utils/usage_data.rb | 94 |
1 files changed, 86 insertions, 8 deletions
diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb index 28dc66e19f8..854fc5c917d 100644 --- a/lib/gitlab/utils/usage_data.rb +++ b/lib/gitlab/utils/usage_data.rb @@ -24,7 +24,8 @@ # alt_usage_data(fallback: nil) { Gitlab.config.registry.enabled } # # * redis_usage_data method -# handles ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent +# handles ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent, +# Gitlab::UsageDataCounters::HLLRedisCounter::EventError # returns -1 when a block is sent or hash with all values -1 when a counter is sent # different behaviour due to 2 different implementations of redis counter # @@ -38,10 +39,12 @@ module Gitlab extend self FALLBACK = -1 + HISTOGRAM_FALLBACK = { '-1' => -1 }.freeze DISTRIBUTED_HLL_FALLBACK = -2 - ALL_TIME_PERIOD_HUMAN_NAME = "all_time" - WEEKLY_PERIOD_HUMAN_NAME = "weekly" - MONTHLY_PERIOD_HUMAN_NAME = "monthly" + ALL_TIME_TIME_FRAME_NAME = "all" + SEVEN_DAYS_TIME_FRAME_NAME = "7d" + TWENTY_EIGHT_DAYS_TIME_FRAME_NAME = "28d" + MAX_BUCKET_SIZE = 100 def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil) if batch @@ -86,6 +89,81 @@ module Gitlab FALLBACK end + # We don't support batching with histograms. + # Please avoid using this method on large tables. + # See https://gitlab.com/gitlab-org/gitlab/-/issues/323949. + # + # rubocop: disable CodeReuse/ActiveRecord + def histogram(relation, column, buckets:, bucket_size: buckets.size) + # Using lambda to avoid exposing histogram specific methods + parameters_valid = lambda do + error_message = + if buckets.first == buckets.last + 'Lower bucket bound cannot equal to upper bucket bound' + elsif bucket_size == 0 + 'Bucket size cannot be zero' + elsif bucket_size > MAX_BUCKET_SIZE + "Bucket size #{bucket_size} exceeds the limit of #{MAX_BUCKET_SIZE}" + end + + return true unless error_message + + exception = ArgumentError.new(error_message) + exception.set_backtrace(caller) + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(exception) + + false + end + + return HISTOGRAM_FALLBACK unless parameters_valid.call + + count_grouped = relation.group(column).select(Arel.star.count.as('count_grouped')) + cte = Gitlab::SQL::CTE.new(:count_cte, count_grouped) + + # For example, 9 segements gives 10 buckets + bucket_segments = bucket_size - 1 + + width_bucket = Arel::Nodes::NamedFunction + .new('WIDTH_BUCKET', [cte.table[:count_grouped], buckets.first, buckets.last, bucket_segments]) + .as('buckets') + + query = cte + .table + .project(width_bucket, cte.table[:count]) + .group('buckets') + .order('buckets') + .with(cte.to_arel) + + # Return the histogram as a Hash because buckets are unique. + relation + .connection + .exec_query(query.to_sql) + .rows + .to_h + # Keys are converted to strings in Usage Ping JSON + .stringify_keys + rescue ActiveRecord::StatementInvalid => e + Gitlab::AppJsonLogger.error( + event: 'histogram', + relation: relation.table_name, + operation: 'histogram', + operation_args: [column, buckets.first, buckets.last, bucket_segments], + query: query.to_sql, + message: e.message + ) + + HISTOGRAM_FALLBACK + end + # rubocop: enable CodeReuse/ActiveRecord + + def add(*args) + return -1 if args.any?(&:negative?) + + args.sum + rescue StandardError + FALLBACK + end + def alt_usage_data(value = nil, fallback: FALLBACK, &block) if block_given? yield @@ -104,11 +182,13 @@ module Gitlab end end - def with_prometheus_client(fallback: nil, verify: true) + def with_prometheus_client(fallback: {}, verify: true) client = prometheus_client(verify: verify) return fallback unless client yield client + rescue + fallback end def measure_duration @@ -126,8 +206,6 @@ module Gitlab # @param event_name [String] the event name # @param values [Array|String] the values counted def track_usage_event(event_name, values) - return unless Feature.enabled?(:"usage_data_#{event_name}", default_enabled: true) - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event_name.to_s, values: values) end @@ -160,7 +238,7 @@ module Gitlab def redis_usage_counter yield - rescue ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent + rescue ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent, Gitlab::UsageDataCounters::HLLRedisCounter::EventError FALLBACK end |