diff options
Diffstat (limited to 'lib/gitlab/usage_data_counters')
| -rw-r--r-- | lib/gitlab/usage_data_counters/aggregated_metrics/common.yml | 17 | ||||
| -rw-r--r-- | lib/gitlab/usage_data_counters/designs_counter.rb | 38 | ||||
| -rw-r--r-- | lib/gitlab/usage_data_counters/hll_redis_counter.rb | 244 | ||||
| -rw-r--r-- | lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb | 42 | ||||
| -rw-r--r-- | lib/gitlab/usage_data_counters/known_events/common.yml (renamed from lib/gitlab/usage_data_counters/known_events.yml) | 99 | ||||
| -rw-r--r-- | lib/gitlab/usage_data_counters/known_events/package_events.yml | 265 | ||||
| -rw-r--r-- | lib/gitlab/usage_data_counters/static_site_editor_counter.rb | 2 | ||||
| -rw-r--r-- | lib/gitlab/usage_data_counters/track_unique_events.rb | 11 | ||||
| -rw-r--r-- | lib/gitlab/usage_data_counters/web_ide_counter.rb | 31 |
9 files changed, 623 insertions, 126 deletions
diff --git a/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml b/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml new file mode 100644 index 00000000000..97ec8423b95 --- /dev/null +++ b/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml @@ -0,0 +1,17 @@ +#- name: unique name of aggregated metric +# operator: aggregation operator. Valid values are: +# - "OR": counts unique elements that were observed triggering any of following events +# - "AND": counts unique elements that were observed triggering all of following events +# events: list of events names to aggregate into metric. All events in this list must have the same 'redis_slot' and 'aggregation' attributes +# see from lib/gitlab/usage_data_counters/known_events/ for the list of valid events. +# feature_flag: name of development feature flag that will be checked before metrics aggregation is performed. +# Corresponding feature flag should have `default_enabled` attribute set to `false`. +# This attribute is OPTIONAL and can be omitted, when `feature_flag` is missing no feature flag will be checked. +--- +- name: product_analytics_test_metrics_union + operator: OR + events: ['i_search_total', 'i_search_advanced', 'i_search_paid'] + feature_flag: product_analytics_aggregated_metrics +- name: product_analytics_test_metrics_intersection + operator: AND + events: ['i_search_total', 'i_search_advanced', 'i_search_paid'] diff --git a/lib/gitlab/usage_data_counters/designs_counter.rb b/lib/gitlab/usage_data_counters/designs_counter.rb index 22188b555d2..07e1963f9fb 100644 --- a/lib/gitlab/usage_data_counters/designs_counter.rb +++ b/lib/gitlab/usage_data_counters/designs_counter.rb @@ -1,42 +1,8 @@ # frozen_string_literal: true module Gitlab::UsageDataCounters - class DesignsCounter - extend Gitlab::UsageDataCounters::RedisCounter - + class DesignsCounter < BaseCounter KNOWN_EVENTS = %w[create update delete].freeze - - UnknownEvent = Class.new(StandardError) - - class << self - # Each event gets a unique Redis key - def redis_key(event) - raise UnknownEvent, event unless KNOWN_EVENTS.include?(event.to_s) - - "USAGE_DESIGN_MANAGEMENT_DESIGNS_#{event}".upcase - end - - def count(event) - increment(redis_key(event)) - end - - def read(event) - total_count(redis_key(event)) - end - - def totals - KNOWN_EVENTS.map { |event| [counter_key(event), read(event)] }.to_h - end - - def fallback_totals - KNOWN_EVENTS.map { |event| [counter_key(event), -1] }.to_h - end - - private - - def counter_key(event) - "design_management_designs_#{event}".to_sym - end - end + PREFIX = 'design_management_designs' end end diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index eb132ef0967..573ad1dce35 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -5,18 +5,28 @@ module Gitlab module HLLRedisCounter DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH = 6.weeks DEFAULT_DAILY_KEY_EXPIRY_LENGTH = 29.days - DEFAULT_REDIS_SLOT = ''.freeze - - UnknownEvent = Class.new(StandardError) - UnknownAggregation = Class.new(StandardError) - - KNOWN_EVENTS_PATH = 'lib/gitlab/usage_data_counters/known_events.yml'.freeze + DEFAULT_REDIS_SLOT = '' + + EventError = Class.new(StandardError) + UnknownEvent = Class.new(EventError) + UnknownAggregation = Class.new(EventError) + AggregationMismatch = Class.new(EventError) + SlotMismatch = Class.new(EventError) + CategoryMismatch = Class.new(EventError) + UnknownAggregationOperator = Class.new(EventError) + InvalidContext = Class.new(EventError) + + KNOWN_EVENTS_PATH = File.expand_path('known_events/*.yml', __dir__) ALLOWED_AGGREGATIONS = %i(daily weekly).freeze + UNION_OF_AGGREGATED_METRICS = 'OR' + INTERSECTION_OF_AGGREGATED_METRICS = 'AND' + ALLOWED_METRICS_AGGREGATIONS = [UNION_OF_AGGREGATED_METRICS, INTERSECTION_OF_AGGREGATED_METRICS].freeze + AGGREGATED_METRICS_PATH = File.expand_path('aggregated_metrics/*.yml', __dir__) # Track event on entity_id # Increment a Redis HLL counter for unique event_name and entity_id # - # All events should be added to know_events file lib/gitlab/usage_data_counters/known_events.yml + # All events should be added to known_events yml files lib/gitlab/usage_data_counters/known_events/ # # Event example: # @@ -25,6 +35,7 @@ module Gitlab # category: compliance # Group events in categories # expiry: 29 # Optional expiration time in days, default value 29 days for daily and 6.weeks for weekly # aggregation: daily # Aggregation level, keys are stored daily or weekly + # feature_flag: # The event feature flag # # Usage: # @@ -33,28 +44,24 @@ module Gitlab class << self include Gitlab::Utils::UsageData - def track_event(entity_id, event_name, time = Time.zone.now) - return unless Gitlab::CurrentSettings.usage_ping_enabled? - - event = event_for(event_name) - - raise UnknownEvent.new("Unknown event #{event_name}") unless event.present? - - Gitlab::Redis::HLL.add(key: redis_key(event, time), value: entity_id, expiry: expiry(event)) + def track_event(value, event_name, time = Time.zone.now) + track(value, event_name, time: time) end - def unique_events(event_names:, start_date:, end_date:) - events = events_for(Array(event_names)) - - raise 'Events should be in same slot' unless events_in_same_slot?(events) - raise 'Events should be in same category' unless events_in_same_category?(events) - raise 'Events should have same aggregation level' unless events_same_aggregation?(events) - - aggregation = events.first[:aggregation] + def track_event_in_context(value, event_name, context, time = Time.zone.now) + return if context.blank? + return unless context.in?(valid_context_list) - keys = keys_for_aggregation(aggregation, events: events, start_date: start_date, end_date: end_date) + track(value, event_name, context: context, time: time) + end - redis_usage_data { Gitlab::Redis::HLL.count(keys: keys) } + def unique_events(event_names:, start_date:, end_date:, context: '') + count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date, context: context) do |events| + raise SlotMismatch, events unless events_in_same_slot?(events) + raise CategoryMismatch, events unless events_in_same_category?(events) + raise AggregationMismatch, events unless events_same_aggregation?(events) + raise InvalidContext if context.present? && !context.in?(valid_context_list) + end end def categories @@ -72,8 +79,8 @@ module Gitlab events_names = events_for_category(category) event_results = events_names.each_with_object({}) do |event, hash| - hash["#{event}_weekly"] = unique_events(event_names: event, start_date: 7.days.ago.to_date, end_date: Date.current) - hash["#{event}_monthly"] = unique_events(event_names: event, start_date: 4.weeks.ago.to_date, end_date: Date.current) + hash["#{event}_weekly"] = unique_events(event_names: [event], start_date: 7.days.ago.to_date, end_date: Date.current) + hash["#{event}_monthly"] = unique_events(event_names: [event], start_date: 4.weeks.ago.to_date, end_date: Date.current) end if eligible_for_totals?(events_names) @@ -89,8 +96,136 @@ module Gitlab event_for(event_name).present? end + def aggregated_metrics_monthly_data + aggregated_metrics_data(4.weeks.ago.to_date) + end + + def aggregated_metrics_weekly_data + aggregated_metrics_data(7.days.ago.to_date) + end + + def known_events + @known_events ||= load_events(KNOWN_EVENTS_PATH) + end + + def aggregated_metrics + @aggregated_metrics ||= load_events(AGGREGATED_METRICS_PATH) + end + private + def track(value, event_name, context: '', time: Time.zone.now) + return unless Gitlab::CurrentSettings.usage_ping_enabled? + + event = event_for(event_name) + raise UnknownEvent, "Unknown event #{event_name}" unless event.present? + + Gitlab::Redis::HLL.add(key: redis_key(event, time, context), value: value, expiry: expiry(event)) + end + + # The aray of valid context on which we allow tracking + def valid_context_list + Plan.all_plans + end + + def aggregated_metrics_data(start_date) + aggregated_metrics.each_with_object({}) do |aggregation, weekly_data| + next if aggregation[:feature_flag] && Feature.disabled?(aggregation[:feature_flag], default_enabled: false, type: :development) + + weekly_data[aggregation[:name]] = calculate_count_for_aggregation(aggregation, start_date: start_date, end_date: Date.current) + end + end + + def calculate_count_for_aggregation(aggregation, start_date:, end_date:) + case aggregation[:operator] + when UNION_OF_AGGREGATED_METRICS + calculate_events_union(event_names: aggregation[:events], start_date: start_date, end_date: end_date) + when INTERSECTION_OF_AGGREGATED_METRICS + calculate_events_intersections(event_names: aggregation[:events], start_date: start_date, end_date: end_date) + else + raise UnknownAggregationOperator, "Events should be aggregated with one of operators #{ALLOWED_METRICS_AGGREGATIONS}" + end + end + + # calculate intersection of 'n' sets based on inclusion exclusion principle https://en.wikipedia.org/wiki/Inclusion%E2%80%93exclusion_principle + # this method will be extracted to dedicated module with https://gitlab.com/gitlab-org/gitlab/-/issues/273391 + def calculate_events_intersections(event_names:, start_date:, end_date:, subset_powers_cache: Hash.new({})) + # calculate power of intersection of all given metrics from inclusion exclusion principle + # |A + B + C| = (|A| + |B| + |C|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C|) => + # |A & B & C| = - (|A| + |B| + |C|) + (|A & B| + |A & C| + .. + |C & D|) + |A + B + C| + # |A + B + C + D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A & B & C & D| => + # |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A + B + C + D| + + # calculate each components of equation except for the last one |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - ... + subset_powers_data = subsets_intersection_powers(event_names, start_date, end_date, subset_powers_cache) + + # calculate last component of the equation |A & B & C & D| = .... - |A + B + C + D| + power_of_union_of_all_events = begin + subset_powers_cache[event_names.size][event_names.join('_+_')] ||= \ + calculate_events_union(event_names: event_names, start_date: start_date, end_date: end_date) + end + + # in order to determine if part of equation (|A & B & C|, |A & B & C & D|), that represents the intersection that we need to calculate, + # is positive or negative in particular equation we need to determine if number of subsets is even or odd. Please take a look at two examples below + # |A + B + C| = (|A| + |B| + |C|) - (|A & B| + |A & C| + .. + |C & D|) + |A & B & C| => + # |A & B & C| = - (|A| + |B| + |C|) + (|A & B| + |A & C| + .. + |C & D|) + |A + B + C| + # |A + B + C + D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A & B & C & D| => + # |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A + B + C + D| + subset_powers_size_even = subset_powers_data.size.even? + + # sum all components of equation except for the last one |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - ... => + sum_of_all_subset_powers = sum_subset_powers(subset_powers_data, subset_powers_size_even) + + # add last component of the equation |A & B & C & D| = sum_of_all_subset_powers - |A + B + C + D| + sum_of_all_subset_powers + (subset_powers_size_even ? power_of_union_of_all_events : -power_of_union_of_all_events) + end + + def sum_subset_powers(subset_powers_data, subset_powers_size_even) + sum_without_sign = subset_powers_data.to_enum.with_index.sum do |value, index| + (index + 1).odd? ? value : -value + end + + (subset_powers_size_even ? -1 : 1) * sum_without_sign + end + + def subsets_intersection_powers(event_names, start_date, end_date, subset_powers_cache) + subset_sizes = (1..(event_names.size - 1)) + + subset_sizes.map do |subset_size| + if subset_size > 1 + # calculate sum of powers of intersection between each subset (with given size) of metrics: #|A + B + C + D| = ... - (|A & B| + |A & C| + .. + |C & D|) + event_names.combination(subset_size).sum do |events_subset| + subset_powers_cache[subset_size][events_subset.join('_&_')] ||= \ + calculate_events_intersections(event_names: events_subset, start_date: start_date, end_date: end_date, subset_powers_cache: subset_powers_cache) + end + else + # calculate sum of powers of each set (metric) alone #|A + B + C + D| = (|A| + |B| + |C| + |D|) - ... + event_names.sum do |event| + subset_powers_cache[subset_size][event] ||= \ + unique_events(event_names: event, start_date: start_date, end_date: end_date) + end + end + end + end + + def calculate_events_union(event_names:, start_date:, end_date:) + count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date) do |events| + raise SlotMismatch, events unless events_in_same_slot?(events) + raise AggregationMismatch, events unless events_same_aggregation?(events) + end + end + + def count_unique_events(event_names:, start_date:, end_date:, context: '') + events = events_for(Array(event_names).map(&:to_s)) + + yield events if block_given? + + aggregation = events.first[:aggregation] + + keys = keys_for_aggregation(aggregation, events: events, start_date: start_date, end_date: end_date, context: context) + redis_usage_data { Gitlab::Redis::HLL.count(keys: keys) } + end + # Allow to add totals for events that are in the same redis slot, category and have the same aggregation level # and if there are more than 1 event def eligible_for_totals?(events_names) @@ -100,16 +235,22 @@ module Gitlab events_in_same_slot?(events) && events_in_same_category?(events) && events_same_aggregation?(events) end - def keys_for_aggregation(aggregation, events:, start_date:, end_date:) + def keys_for_aggregation(aggregation, events:, start_date:, end_date:, context: '') if aggregation.to_sym == :daily - daily_redis_keys(events: events, start_date: start_date, end_date: end_date) + daily_redis_keys(events: events, start_date: start_date, end_date: end_date, context: context) else - weekly_redis_keys(events: events, start_date: start_date, end_date: end_date) + weekly_redis_keys(events: events, start_date: start_date, end_date: end_date, context: context) end end - def known_events - @known_events ||= YAML.load_file(Rails.root.join(KNOWN_EVENTS_PATH)).map(&:with_indifferent_access) + def load_events(wildcard) + Dir[wildcard].each_with_object([]) do |path, events| + events.push(*load_yaml_from_path(path)) + end + end + + def load_yaml_from_path(path) + YAML.safe_load(File.read(path))&.map(&:with_indifferent_access) end def known_events_names @@ -141,7 +282,7 @@ module Gitlab end def event_for(event_name) - known_events.find { |event| event[:name] == event_name } + known_events.find { |event| event[:name] == event_name.to_s } end def events_for(event_names) @@ -153,17 +294,26 @@ module Gitlab end # Compose the key in order to store events daily or weekly - def redis_key(event, time) + def redis_key(event, time, context = '') raise UnknownEvent.new("Unknown event #{event[:name]}") unless known_events_names.include?(event[:name].to_s) raise UnknownAggregation.new("Use :daily or :weekly aggregation") unless ALLOWED_AGGREGATIONS.include?(event[:aggregation].to_sym) + key = apply_slot(event) + key = apply_time_aggregation(key, time, event) + key = "#{context}_#{key}" if context.present? + key + end + + def apply_slot(event) slot = redis_slot(event) - key = if slot.present? - event[:name].to_s.gsub(slot, "{#{slot}}") - else - "{#{event[:name]}}" - end + if slot.present? + event[:name].to_s.gsub(slot, "{#{slot}}") + else + "{#{event[:name]}}" + end + end + def apply_time_aggregation(key, time, event) if event[:aggregation].to_sym == :daily year_day = time.strftime('%G-%j') "#{year_day}-#{key}" @@ -173,21 +323,29 @@ module Gitlab end end - def daily_redis_keys(events:, start_date:, end_date:) + def daily_redis_keys(events:, start_date:, end_date:, context: '') (start_date.to_date..end_date.to_date).map do |date| - events.map { |event| redis_key(event, date) } + events.map { |event| redis_key(event, date, context) } end.flatten end - def weekly_redis_keys(events:, start_date:, end_date:) + def validate_aggregation_operator!(operator) + return true if ALLOWED_METRICS_AGGREGATIONS.include?(operator) + + raise UnknownAggregationOperator.new("Events should be aggregated with one of operators #{ALLOWED_METRICS_AGGREGATIONS}") + end + + def weekly_redis_keys(events:, start_date:, end_date:, context: '') weeks = end_date.to_date.cweek - start_date.to_date.cweek weeks = 1 if weeks == 0 (0..(weeks - 1)).map do |week_increment| - events.map { |event| redis_key(event, start_date + week_increment * 7.days) } + events.map { |event| redis_key(event, start_date + week_increment * 7.days, context) } end.flatten end end end end end + +Gitlab::UsageDataCounters::HLLRedisCounter.prepend_if_ee('EE::Gitlab::UsageDataCounters::HLLRedisCounter') diff --git a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb index e8839875109..da013a06777 100644 --- a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb @@ -9,14 +9,12 @@ module Gitlab ISSUE_CREATED = 'g_project_management_issue_created' ISSUE_CLOSED = 'g_project_management_issue_closed' ISSUE_DESCRIPTION_CHANGED = 'g_project_management_issue_description_changed' - ISSUE_ITERATION_CHANGED = 'g_project_management_issue_iteration_changed' ISSUE_LABEL_CHANGED = 'g_project_management_issue_label_changed' ISSUE_MADE_CONFIDENTIAL = 'g_project_management_issue_made_confidential' ISSUE_MADE_VISIBLE = 'g_project_management_issue_made_visible' ISSUE_MILESTONE_CHANGED = 'g_project_management_issue_milestone_changed' ISSUE_REOPENED = 'g_project_management_issue_reopened' ISSUE_TITLE_CHANGED = 'g_project_management_issue_title_changed' - ISSUE_WEIGHT_CHANGED = 'g_project_management_issue_weight_changed' ISSUE_CROSS_REFERENCED = 'g_project_management_issue_cross_referenced' ISSUE_MOVED = 'g_project_management_issue_moved' ISSUE_RELATED = 'g_project_management_issue_related' @@ -24,15 +22,15 @@ module Gitlab ISSUE_MARKED_AS_DUPLICATE = 'g_project_management_issue_marked_as_duplicate' ISSUE_LOCKED = 'g_project_management_issue_locked' ISSUE_UNLOCKED = 'g_project_management_issue_unlocked' - ISSUE_ADDED_TO_EPIC = 'g_project_management_issue_added_to_epic' - ISSUE_REMOVED_FROM_EPIC = 'g_project_management_issue_removed_from_epic' - ISSUE_CHANGED_EPIC = 'g_project_management_issue_changed_epic' ISSUE_DESIGNS_ADDED = 'g_project_management_issue_designs_added' ISSUE_DESIGNS_MODIFIED = 'g_project_management_issue_designs_modified' ISSUE_DESIGNS_REMOVED = 'g_project_management_issue_designs_removed' ISSUE_DUE_DATE_CHANGED = 'g_project_management_issue_due_date_changed' ISSUE_TIME_ESTIMATE_CHANGED = 'g_project_management_issue_time_estimate_changed' ISSUE_TIME_SPENT_CHANGED = 'g_project_management_issue_time_spent_changed' + ISSUE_COMMENT_ADDED = 'g_project_management_issue_comment_added' + ISSUE_COMMENT_EDITED = 'g_project_management_issue_comment_edited' + ISSUE_COMMENT_REMOVED = 'g_project_management_issue_comment_removed' class << self def track_issue_created_action(author:, time: Time.zone.now) @@ -75,14 +73,6 @@ module Gitlab track_unique_action(ISSUE_MILESTONE_CHANGED, author, time) end - def track_issue_iteration_changed_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_ITERATION_CHANGED, author, time) - end - - def track_issue_weight_changed_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_WEIGHT_CHANGED, author, time) - end - def track_issue_cross_referenced_action(author:, time: Time.zone.now) track_unique_action(ISSUE_CROSS_REFERENCED, author, time) end @@ -111,18 +101,6 @@ module Gitlab track_unique_action(ISSUE_UNLOCKED, author, time) end - def track_issue_added_to_epic_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_ADDED_TO_EPIC, author, time) - end - - def track_issue_removed_from_epic_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_REMOVED_FROM_EPIC, author, time) - end - - def track_issue_changed_epic_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_CHANGED_EPIC, author, time) - end - def track_issue_designs_added_action(author:, time: Time.zone.now) track_unique_action(ISSUE_DESIGNS_ADDED, author, time) end @@ -147,6 +125,18 @@ module Gitlab track_unique_action(ISSUE_TIME_SPENT_CHANGED, author, time) end + def track_issue_comment_added_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_COMMENT_ADDED, author, time) + end + + def track_issue_comment_edited_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_COMMENT_EDITED, author, time) + end + + def track_issue_comment_removed_action(author:, time: Time.zone.now) + track_unique_action(ISSUE_COMMENT_REMOVED, author, time) + end + private def track_unique_action(action, author, time) @@ -159,3 +149,5 @@ module Gitlab end end end + +Gitlab::UsageDataCounters::IssueActivityUniqueCounter.prepend_if_ee('EE::Gitlab::UsageDataCounters::IssueActivityUniqueCounter') diff --git a/lib/gitlab/usage_data_counters/known_events.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index bc56c5d6d9b..85f16ea807b 100644 --- a/lib/gitlab/usage_data_counters/known_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -4,114 +4,141 @@ redis_slot: compliance category: compliance aggregation: weekly + feature_flag: track_unique_visits - name: g_compliance_audit_events category: compliance redis_slot: compliance aggregation: weekly + feature_flag: track_unique_visits - name: i_compliance_audit_events category: compliance redis_slot: compliance aggregation: weekly + feature_flag: track_unique_visits - name: i_compliance_credential_inventory category: compliance redis_slot: compliance aggregation: weekly + feature_flag: track_unique_visits - name: a_compliance_audit_events_api category: compliance redis_slot: compliance aggregation: weekly + feature_flag: usage_data_a_compliance_audit_events_api # Analytics category - name: g_analytics_contribution category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: g_analytics_insights category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: g_analytics_issues category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: g_analytics_productivity category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: g_analytics_valuestream category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: p_analytics_pipelines category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: p_analytics_code_reviews category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: p_analytics_valuestream category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: p_analytics_insights category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: p_analytics_issues category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: p_analytics_repo category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: i_analytics_cohorts category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: i_analytics_dev_ops_score category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: g_analytics_merge_request category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: p_analytics_merge_request category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: i_analytics_instance_statistics category: analytics redis_slot: analytics aggregation: weekly + feature_flag: track_unique_visits - name: g_edit_by_web_ide category: ide_edit redis_slot: edit expiry: 29 aggregation: daily + feature_flag: track_editor_edit_actions - name: g_edit_by_sfe category: ide_edit redis_slot: edit expiry: 29 aggregation: daily + feature_flag: track_editor_edit_actions - name: g_edit_by_snippet_ide category: ide_edit redis_slot: edit expiry: 29 aggregation: daily + feature_flag: track_editor_edit_actions - name: i_search_total category: search redis_slot: search aggregation: weekly + feature_flag: search_track_unique_users - name: i_search_advanced category: search redis_slot: search aggregation: weekly + feature_flag: search_track_unique_users - name: i_search_paid category: search redis_slot: search aggregation: weekly + feature_flag: search_track_unique_users - name: wiki_action category: source_code aggregation: daily @@ -121,6 +148,9 @@ - name: project_action category: source_code aggregation: daily +- name: git_write_action + category: source_code + aggregation: daily - name: merge_request_action category: source_code aggregation: daily @@ -133,173 +163,242 @@ redis_slot: incident_management category: incident_management aggregation: weekly + feature_flag: usage_data_incident_management_alert_status_changed - name: incident_management_alert_assigned redis_slot: incident_management category: incident_management aggregation: weekly + feature_flag: usage_data_incident_management_alert_assigned - name: incident_management_alert_todo redis_slot: incident_management category: incident_management aggregation: weekly + feature_flag: usage_data_incident_management_alert_todo - name: incident_management_incident_created redis_slot: incident_management category: incident_management aggregation: weekly + feature_flag: usage_data_incident_management_incident_created - name: incident_management_incident_reopened redis_slot: incident_management category: incident_management aggregation: weekly + feature_flag: usage_data_incident_management_incident_reopened - name: incident_management_incident_closed redis_slot: incident_management category: incident_management aggregation: weekly + feature_flag: usage_data_incident_management_incident_closed - name: incident_management_incident_assigned redis_slot: incident_management category: incident_management aggregation: weekly + feature_flag: usage_data_incident_management_incident_assigned - name: incident_management_incident_todo redis_slot: incident_management category: incident_management aggregation: weekly + feature_flag: usage_data_incident_management_incident_todo - name: incident_management_incident_comment redis_slot: incident_management category: incident_management aggregation: weekly + feature_flag: usage_data_incident_management_incident_comment - name: incident_management_incident_zoom_meeting redis_slot: incident_management category: incident_management aggregation: weekly + feature_flag: usage_data_incident_management_incident_zoom_meeting - name: incident_management_incident_published redis_slot: incident_management category: incident_management aggregation: weekly + feature_flag: usage_data_incident_management_incident_published - name: incident_management_incident_relate redis_slot: incident_management category: incident_management aggregation: weekly + feature_flag: usage_data_incident_management_incident_relate - name: incident_management_incident_unrelate redis_slot: incident_management category: incident_management aggregation: weekly + feature_flag: usage_data_incident_management_incident_unrelate - name: incident_management_incident_change_confidential redis_slot: incident_management category: incident_management aggregation: weekly + feature_flag: usage_data_incident_management_incident_change_confidential # Testing category - name: i_testing_test_case_parsed category: testing redis_slot: testing aggregation: weekly + feature_flag: usage_data_i_testing_test_case_parsed # Project Management group - name: g_project_management_issue_title_changed category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_description_changed category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_assignee_changed category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_made_confidential category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_made_visible category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_created category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_closed category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_reopened category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_label_changed category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_milestone_changed category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_iteration_changed category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_weight_changed category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_cross_referenced category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_moved category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_related category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_unrelated category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_marked_as_duplicate category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_locked category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_unlocked category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_added_to_epic category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_removed_from_epic category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_changed_epic category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_designs_added category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_designs_modified category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_designs_removed category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_due_date_changed category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_time_estimate_changed category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions - name: g_project_management_issue_time_spent_changed category: issues_edit redis_slot: project_management aggregation: daily + feature_flag: track_issue_activity_actions +- name: g_project_management_issue_comment_added + category: issues_edit + redis_slot: project_management + aggregation: daily + feature_flag: track_issue_activity_actions +- name: g_project_management_issue_comment_edited + category: issues_edit + redis_slot: project_management + aggregation: daily + feature_flag: track_issue_activity_actions +- name: g_project_management_issue_comment_removed + category: issues_edit + redis_slot: project_management + aggregation: daily + feature_flag: track_issue_activity_actions +- name: g_project_management_issue_health_status_changed + category: issues_edit + redis_slot: project_management + aggregation: daily + feature_flag: track_issue_activity_actions +# Secrets Management +- name: i_ci_secrets_management_vault_build_created + category: ci_secrets_management + redis_slot: ci_secrets_management + aggregation: weekly + feature_flag: usage_data_i_ci_secrets_management_vault_build_created diff --git a/lib/gitlab/usage_data_counters/known_events/package_events.yml b/lib/gitlab/usage_data_counters/known_events/package_events.yml new file mode 100644 index 00000000000..7ed02aa2a85 --- /dev/null +++ b/lib/gitlab/usage_data_counters/known_events/package_events.yml @@ -0,0 +1,265 @@ +--- +- name: i_package_maven_user_push + category: maven_packages + aggregation: weekly + redis_slot: package +- name: i_package_maven_deploy_token_push + category: maven_packages + aggregation: weekly + redis_slot: package +- name: i_package_maven_user_delete + category: maven_packages + aggregation: weekly + redis_slot: package +- name: i_package_maven_deploy_token_delete + category: maven_packages + aggregation: weekly + redis_slot: package +- name: i_package_maven_user_pull + category: maven_packages + aggregation: weekly + redis_slot: package +- name: i_package_maven_deploy_token_pull + category: maven_packages + aggregation: weekly + redis_slot: package +- name: i_package_npm_user_push + category: npm_packages + aggregation: weekly + redis_slot: package +- name: i_package_npm_deploy_token_push + category: npm_packages + aggregation: weekly + redis_slot: package +- name: i_package_npm_user_delete + category: npm_packages + aggregation: weekly + redis_slot: package +- name: i_package_npm_deploy_token_delete + category: npm_packages + aggregation: weekly + redis_slot: package +- name: i_package_npm_user_pull + category: npm_packages + aggregation: weekly + redis_slot: package +- name: i_package_npm_deploy_token_pull + category: npm_packages + aggregation: weekly + redis_slot: package +- name: i_package_conan_user_push + category: conan_packages + aggregation: weekly + redis_slot: package +- name: i_package_conan_deploy_token_push + category: conan_packages + aggregation: weekly + redis_slot: package +- name: i_package_conan_user_delete + category: conan_packages + aggregation: weekly + redis_slot: package +- name: i_package_conan_deploy_token_delete + category: conan_packages + aggregation: weekly + redis_slot: package +- name: i_package_conan_user_pull + category: conan_packages + aggregation: weekly + redis_slot: package +- name: i_package_conan_deploy_token_pull + category: conan_packages + aggregation: weekly + redis_slot: package +- name: i_package_nuget_user_push + category: nuget_packages + aggregation: weekly + redis_slot: package +- name: i_package_nuget_deploy_token_push + category: nuget_packages + aggregation: weekly + redis_slot: package +- name: i_package_nuget_user_delete + category: nuget_packages + aggregation: weekly + redis_slot: package +- name: i_package_nuget_deploy_token_delete + category: nuget_packages + aggregation: weekly + redis_slot: package +- name: i_package_nuget_user_pull + category: nuget_packages + aggregation: weekly + redis_slot: package +- name: i_package_nuget_deploy_token_pull + category: nuget_packages + aggregation: weekly + redis_slot: package +- name: i_package_pypi_user_push + category: pypi_packages + aggregation: weekly + redis_slot: package +- name: i_package_pypi_deploy_token_push + category: pypi_packages + aggregation: weekly + redis_slot: package +- name: i_package_pypi_user_delete + category: pypi_packages + aggregation: weekly + redis_slot: package +- name: i_package_pypi_deploy_token_delete + category: pypi_packages + aggregation: weekly + redis_slot: package +- name: i_package_pypi_user_pull + category: pypi_packages + aggregation: weekly + redis_slot: package +- name: i_package_pypi_deploy_token_pull + category: pypi_packages + aggregation: weekly + redis_slot: package +- name: i_package_composer_user_push + category: composer_packages + aggregation: weekly + redis_slot: package +- name: i_package_composer_deploy_token_push + category: composer_packages + aggregation: weekly + redis_slot: package +- name: i_package_composer_user_delete + category: composer_packages + aggregation: weekly + redis_slot: package +- name: i_package_composer_deploy_token_delete + category: composer_packages + aggregation: weekly + redis_slot: package +- name: i_package_composer_user_pull + category: composer_packages + aggregation: weekly + redis_slot: package +- name: i_package_composer_deploy_token_pull + category: composer_packages + aggregation: weekly + redis_slot: package +- name: i_package_generic_user_push + category: generic_packages + aggregation: weekly + redis_slot: package +- name: i_package_generic_deploy_token_push + category: generic_packages + aggregation: weekly + redis_slot: package +- name: i_package_generic_user_delete + category: generic_packages + aggregation: weekly + redis_slot: package +- name: i_package_generic_deploy_token_delete + category: generic_packages + aggregation: weekly + redis_slot: package +- name: i_package_generic_user_pull + category: generic_packages + aggregation: weekly + redis_slot: package +- name: i_package_generic_deploy_token_pull + category: generic_packages + aggregation: weekly + redis_slot: package +- name: i_package_golang_user_push + category: golang_packages + aggregation: weekly + redis_slot: package +- name: i_package_golang_deploy_token_push + category: golang_packages + aggregation: weekly + redis_slot: package +- name: i_package_golang_user_delete + category: golang_packages + aggregation: weekly + redis_slot: package +- name: i_package_golang_deploy_token_delete + category: golang_packages + aggregation: weekly + redis_slot: package +- name: i_package_golang_user_pull + category: golang_packages + aggregation: weekly + redis_slot: package +- name: i_package_golang_deploy_token_pull + category: golang_packages + aggregation: weekly + redis_slot: package +- name: i_package_debian_user_push + category: debian_packages + aggregation: weekly + redis_slot: package +- name: i_package_debian_deploy_token_push + category: debian_packages + aggregation: weekly + redis_slot: package +- name: i_package_debian_user_delete + category: debian_packages + aggregation: weekly + redis_slot: package +- name: i_package_debian_deploy_token_delete + category: debian_packages + aggregation: weekly + redis_slot: package +- name: i_package_debian_user_pull + category: debian_packages + aggregation: weekly + redis_slot: package +- name: i_package_debian_deploy_token_pull + category: debian_packages + aggregation: weekly + redis_slot: package +- name: i_package_container_user_push + category: container_packages + aggregation: weekly + redis_slot: package +- name: i_package_container_deploy_token_push + category: container_packages + aggregation: weekly + redis_slot: package +- name: i_package_container_user_delete + category: container_packages + aggregation: weekly + redis_slot: package +- name: i_package_container_deploy_token_delete + category: container_packages + aggregation: weekly + redis_slot: package +- name: i_package_container_user_pull + category: container_packages + aggregation: weekly + redis_slot: package +- name: i_package_container_deploy_token_pull + category: container_packages + aggregation: weekly + redis_slot: package +- name: i_package_tag_user_push + category: tag_packages + aggregation: weekly + redis_slot: package +- name: i_package_tag_deploy_token_push + category: tag_packages + aggregation: weekly + redis_slot: package +- name: i_package_tag_user_delete + category: tag_packages + aggregation: weekly + redis_slot: package +- name: i_package_tag_deploy_token_delete + category: tag_packages + aggregation: weekly + redis_slot: package +- name: i_package_tag_user_pull + category: tag_packages + aggregation: weekly + redis_slot: package +- name: i_package_tag_deploy_token_pull + category: tag_packages + aggregation: weekly + redis_slot: package diff --git a/lib/gitlab/usage_data_counters/static_site_editor_counter.rb b/lib/gitlab/usage_data_counters/static_site_editor_counter.rb index 8886a106da8..3c5989d1e11 100644 --- a/lib/gitlab/usage_data_counters/static_site_editor_counter.rb +++ b/lib/gitlab/usage_data_counters/static_site_editor_counter.rb @@ -3,7 +3,7 @@ module Gitlab module UsageDataCounters class StaticSiteEditorCounter < BaseCounter - KNOWN_EVENTS = %w[views].freeze + KNOWN_EVENTS = %w[views commits merge_requests].freeze PREFIX = 'static_site_editor' class << self diff --git a/lib/gitlab/usage_data_counters/track_unique_events.rb b/lib/gitlab/usage_data_counters/track_unique_events.rb index 7053744b665..95380ae0b1d 100644 --- a/lib/gitlab/usage_data_counters/track_unique_events.rb +++ b/lib/gitlab/usage_data_counters/track_unique_events.rb @@ -8,6 +8,9 @@ module Gitlab PUSH_ACTION = :project_action MERGE_REQUEST_ACTION = :merge_request_action + GIT_WRITE_ACTIONS = [WIKI_ACTION, DESIGN_ACTION, PUSH_ACTION].freeze + GIT_WRITE_ACTION = :git_write_action + ACTION_TRANSFORMATIONS = HashWithIndifferentAccess.new({ wiki: { created: WIKI_ACTION, @@ -41,6 +44,8 @@ module Gitlab return unless Gitlab::UsageDataCounters::HLLRedisCounter.known_event?(transformed_action.to_s) Gitlab::UsageDataCounters::HLLRedisCounter.track_event(author_id, transformed_action.to_s, time) + + track_git_write_action(author_id, transformed_action, time) end def count_unique_events(event_action:, date_from:, date_to:) @@ -64,6 +69,12 @@ module Gitlab def valid_action?(action) Event.actions.key?(action) end + + def track_git_write_action(author_id, transformed_action, time) + return unless GIT_WRITE_ACTIONS.include?(transformed_action) + + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(author_id, GIT_WRITE_ACTION, time) + end end end end diff --git a/lib/gitlab/usage_data_counters/web_ide_counter.rb b/lib/gitlab/usage_data_counters/web_ide_counter.rb index 00fcd42a9af..9f2f4ac3971 100644 --- a/lib/gitlab/usage_data_counters/web_ide_counter.rb +++ b/lib/gitlab/usage_data_counters/web_ide_counter.rb @@ -2,54 +2,43 @@ module Gitlab module UsageDataCounters - class WebIdeCounter - extend RedisCounter - KNOWN_EVENTS = %i[commits views merge_requests previews terminals pipelines].freeze + class WebIdeCounter < BaseCounter + KNOWN_EVENTS = %w[commits views merge_requests previews terminals pipelines].freeze PREFIX = 'web_ide' class << self def increment_commits_count - increment(redis_key('commits')) + count('commits') end def increment_merge_requests_count - increment(redis_key('merge_requests')) + count('merge_requests') end def increment_views_count - increment(redis_key('views')) + count('views') end def increment_terminals_count - increment(redis_key('terminals')) + count('terminals') end def increment_pipelines_count - increment(redis_key('pipelines')) + count('pipelines') end def increment_previews_count return unless Gitlab::CurrentSettings.web_ide_clientside_preview_enabled? - increment(redis_key('previews')) - end - - def totals - KNOWN_EVENTS.map { |event| [counter_key(event), total_count(redis_key(event))] }.to_h - end - - def fallback_totals - KNOWN_EVENTS.map { |event| [counter_key(event), -1] }.to_h + count('previews') end private def redis_key(event) - "#{PREFIX}_#{event}_count".upcase - end + require_known_event(event) - def counter_key(event) - "#{PREFIX}_#{event}".to_sym + "#{prefix}_#{event}_count".upcase end end end |