1 files changed, 308 insertions, 16 deletions

diff --git a/spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb b/spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb
index e84c3c17274..93704a39555 100644
--- a/spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb
+++ b/spec/lib/gitlab/usage_data_counters/hll_redis_counter_spec.rb
@@ -8,6 +8,9 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s
   let(:entity3) { '34rfjuuy-ce56-sa35-ds34-dfer567dfrf2' }
   let(:entity4) { '8b9a2671-2abf-4bec-a682-22f6a8f7bf31' }
 
+  let(:default_context) { 'default' }
+  let(:invalid_context) { 'invalid' }
+
   around do |example|
     # We need to freeze to a reference time
     # because visits are grouped by the week number in the year
@@ -20,7 +23,28 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s
 
   describe '.categories' do
     it 'gets all unique category names' do
-      expect(described_class.categories).to contain_exactly('analytics', 'compliance', 'ide_edit', 'search', 'source_code', 'incident_management', 'issues_edit', 'testing')
+      expect(described_class.categories).to contain_exactly(
+        'compliance',
+        'analytics',
+        'ide_edit',
+        'search',
+        'source_code',
+        'incident_management',
+        'testing',
+        'issues_edit',
+        'ci_secrets_management',
+        'maven_packages',
+        'npm_packages',
+        'conan_packages',
+        'nuget_packages',
+        'pypi_packages',
+        'composer_packages',
+        'generic_packages',
+        'golang_packages',
+        'debian_packages',
+        'container_packages',
+        'tag_packages'
+      )
     end
   end
 
@@ -34,11 +58,13 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s
     let(:no_slot) { 'no_slot' }
     let(:different_aggregation) { 'different_aggregation' }
     let(:custom_daily_event) { 'g_analytics_custom' }
+    let(:context_event) { 'context_event' }
 
     let(:global_category) { 'global' }
-    let(:compliance_category) {'compliance' }
-    let(:productivity_category) {'productivity' }
+    let(:compliance_category) { 'compliance' }
+    let(:productivity_category) { 'productivity' }
     let(:analytics_category) { 'analytics' }
+    let(:other_category) { 'other' }
 
     let(:known_events) do
       [
@@ -47,7 +73,8 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s
         { name: category_productivity_event, redis_slot: "analytics", category: productivity_category, aggregation: "weekly" },
         { name: compliance_slot_event, redis_slot: "compliance", category: compliance_category, aggregation: "weekly" },
         { name: no_slot, category: global_category, aggregation: "daily" },
-        { name: different_aggregation, category: global_category, aggregation: "monthly" }
+        { name: different_aggregation, category: global_category, aggregation: "monthly" },
+        { name: context_event, category: other_category, expiry: 6, aggregation: 'weekly' }
       ].map(&:with_indifferent_access)
     end
 
@@ -77,12 +104,18 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s
           stub_application_setting(usage_ping_enabled: true)
         end
 
+        it 'tracks event when using symbol' do
+          expect(Gitlab::Redis::HLL).to receive(:add)
+
+          described_class.track_event(entity1, :g_analytics_contribution)
+        end
+
         it "raise error if metrics don't have same aggregation" do
-          expect { described_class.track_event(entity1, different_aggregation, Date.current) } .to raise_error(Gitlab::UsageDataCounters::HLLRedisCounter::UnknownAggregation)
+          expect { described_class.track_event(entity1, different_aggregation, Date.current) }.to raise_error(Gitlab::UsageDataCounters::HLLRedisCounter::UnknownAggregation)
         end
 
         it 'raise error if metrics of unknown aggregation' do
-          expect { described_class.track_event(entity1, 'unknown', Date.current) } .to raise_error(Gitlab::UsageDataCounters::HLLRedisCounter::UnknownEvent)
+          expect { described_class.track_event(entity1, 'unknown', Date.current) }.to raise_error(Gitlab::UsageDataCounters::HLLRedisCounter::UnknownEvent)
         end
 
         context 'for weekly events' do
@@ -143,6 +176,34 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s
       end
     end
 
+    describe '.track_event_in_context' do
+      context 'with valid contex' do
+        it 'increments conext event counte' do
+          expect(Gitlab::Redis::HLL).to receive(:add) do |kwargs|
+            expect(kwargs[:key]).to match(/^#{default_context}\_.*/)
+          end
+
+          described_class.track_event_in_context(entity1, context_event, default_context)
+        end
+      end
+
+      context 'with empty context' do
+        it 'does not increment a counter' do
+          expect(Gitlab::Redis::HLL).not_to receive(:add)
+
+          described_class.track_event_in_context(entity1, context_event, '')
+        end
+      end
+
+      context 'when sending invalid context' do
+        it 'does not increment a counter' do
+          expect(Gitlab::Redis::HLL).not_to receive(:add)
+
+          described_class.track_event_in_context(entity1, context_event, invalid_context)
+        end
+      end
+    end
+
     describe '.unique_events' do
       before do
         # events in current week, should not be counted as week is not complete
@@ -178,37 +239,89 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s
       end
 
       it 'raise error if metrics are not in the same slot' do
-        expect { described_class.unique_events(event_names: [compliance_slot_event, analytics_slot_event], start_date: 4.weeks.ago, end_date: Date.current) }.to raise_error('Events should be in same slot')
+        expect do
+          described_class.unique_events(event_names: [compliance_slot_event, analytics_slot_event], start_date: 4.weeks.ago, end_date: Date.current)
+        end.to raise_error(Gitlab::UsageDataCounters::HLLRedisCounter::SlotMismatch)
       end
 
       it 'raise error if metrics are not in the same category' do
-        expect { described_class.unique_events(event_names: [category_analytics_event, category_productivity_event], start_date: 4.weeks.ago, end_date: Date.current) }.to raise_error('Events should be in same category')
+        expect do
+          described_class.unique_events(event_names: [category_analytics_event, category_productivity_event], start_date: 4.weeks.ago, end_date: Date.current)
+        end.to raise_error(Gitlab::UsageDataCounters::HLLRedisCounter::CategoryMismatch)
       end
 
       it "raise error if metrics don't have same aggregation" do
-        expect { described_class.unique_events(event_names: [daily_event, weekly_event], start_date: 4.weeks.ago, end_date: Date.current) }.to raise_error('Events should have same aggregation level')
+        expect do
+          described_class.unique_events(event_names: [daily_event, weekly_event], start_date: 4.weeks.ago, end_date: Date.current)
+        end.to raise_error(Gitlab::UsageDataCounters::HLLRedisCounter::AggregationMismatch)
       end
 
       context 'when data for the last complete week' do
-        it { expect(described_class.unique_events(event_names: weekly_event, start_date: 1.week.ago, end_date: Date.current)).to eq(1) }
+        it { expect(described_class.unique_events(event_names: [weekly_event], start_date: 1.week.ago, end_date: Date.current)).to eq(1) }
       end
 
       context 'when data for the last 4 complete weeks' do
-        it { expect(described_class.unique_events(event_names: weekly_event, start_date: 4.weeks.ago, end_date: Date.current)).to eq(2) }
+        it { expect(described_class.unique_events(event_names: [weekly_event], start_date: 4.weeks.ago, end_date: Date.current)).to eq(2) }
       end
 
       context 'when data for the week 4 weeks ago' do
-        it { expect(described_class.unique_events(event_names: weekly_event, start_date: 4.weeks.ago, end_date: 3.weeks.ago)).to eq(1) }
+        it { expect(described_class.unique_events(event_names: [weekly_event], start_date: 4.weeks.ago, end_date: 3.weeks.ago)).to eq(1) }
+      end
+
+      context 'when using symbol as parameter' do
+        it { expect(described_class.unique_events(event_names: [weekly_event.to_sym], start_date: 4.weeks.ago, end_date: 3.weeks.ago)).to eq(1) }
       end
 
       context 'when using daily aggregation' do
-        it { expect(described_class.unique_events(event_names: daily_event, start_date: 7.days.ago, end_date: Date.current)).to eq(2) }
-        it { expect(described_class.unique_events(event_names: daily_event, start_date: 28.days.ago, end_date: Date.current)).to eq(3) }
-        it { expect(described_class.unique_events(event_names: daily_event, start_date: 28.days.ago, end_date: 21.days.ago)).to eq(1) }
+        it { expect(described_class.unique_events(event_names: [daily_event], start_date: 7.days.ago, end_date: Date.current)).to eq(2) }
+        it { expect(described_class.unique_events(event_names: [daily_event], start_date: 28.days.ago, end_date: Date.current)).to eq(3) }
+        it { expect(described_class.unique_events(event_names: [daily_event], start_date: 28.days.ago, end_date: 21.days.ago)).to eq(1) }
       end
 
       context 'when no slot is set' do
-        it { expect(described_class.unique_events(event_names: no_slot, start_date: 7.days.ago, end_date: Date.current)).to eq(1) }
+        it { expect(described_class.unique_events(event_names: [no_slot], start_date: 7.days.ago, end_date: Date.current)).to eq(1) }
+      end
+    end
+  end
+
+  describe 'context level tracking' do
+    using RSpec::Parameterized::TableSyntax
+
+    let(:known_events) do
+      [
+        { name: 'event_name_1', redis_slot: 'event', category: 'category1', aggregation: "weekly" },
+        { name: 'event_name_2', redis_slot: 'event', category: 'category1', aggregation: "weekly" },
+        { name: 'event_name_3', redis_slot: 'event', category: 'category1', aggregation: "weekly" }
+      ].map(&:with_indifferent_access)
+    end
+
+    before do
+      allow(described_class).to receive(:known_events).and_return(known_events)
+      allow(described_class).to receive(:categories).and_return(%w(category1 category2))
+
+      described_class.track_event_in_context([entity1, entity3], 'event_name_1', default_context, 2.days.ago)
+      described_class.track_event_in_context(entity3, 'event_name_1', default_context, 2.days.ago)
+      described_class.track_event_in_context(entity3, 'event_name_1', invalid_context, 2.days.ago)
+      described_class.track_event_in_context([entity1, entity2], 'event_name_2', '', 2.weeks.ago)
+    end
+
+    subject(:unique_events) { described_class.unique_events(event_names: event_names, start_date: 4.weeks.ago, end_date: Date.current, context: context) }
+
+    context 'with correct arguments' do
+      where(:event_names, :context, :value) do
+        ['event_name_1'] | 'default' | 2
+        ['event_name_1'] | ''        | 0
+        ['event_name_2'] | ''        | 0
+      end
+
+      with_them do
+        it { is_expected.to eq value }
+      end
+    end
+
+    context 'with invalid context' do
+      it 'raise error' do
+        expect { described_class.unique_events(event_names: 'event_name_1', start_date: 4.weeks.ago, end_date: Date.current, context: invalid_context) }.to raise_error(Gitlab::UsageDataCounters::HLLRedisCounter::InvalidContext)
       end
     end
   end
@@ -257,4 +370,183 @@ RSpec.describe Gitlab::UsageDataCounters::HLLRedisCounter, :clean_gitlab_redis_s
       expect(subject.unique_events_data).to eq(results)
     end
   end
+
+  context 'aggregated_metrics_data' do
+    let(:known_events) do
+      [
+        { name: 'event1_slot', redis_slot: "slot", category: 'category1', aggregation: "weekly" },
+        { name: 'event2_slot', redis_slot: "slot", category: 'category2', aggregation: "weekly" },
+        { name: 'event3_slot', redis_slot: "slot", category: 'category3', aggregation: "weekly" },
+        { name: 'event5_slot', redis_slot: "slot", category: 'category4', aggregation: "weekly" },
+        { name: 'event4', category: 'category2', aggregation: "weekly" }
+      ].map(&:with_indifferent_access)
+    end
+
+    before do
+      allow(described_class).to receive(:known_events).and_return(known_events)
+    end
+
+    shared_examples 'aggregated_metrics_data' do
+      context 'no aggregated metrics is defined' do
+        it 'returns empty hash' do
+          allow(described_class).to receive(:aggregated_metrics).and_return([])
+
+          expect(aggregated_metrics_data).to eq({})
+        end
+      end
+
+      context 'there are aggregated metrics defined' do
+        before do
+          allow(described_class).to receive(:aggregated_metrics).and_return(aggregated_metrics)
+        end
+
+        context 'with AND operator' do
+          let(:aggregated_metrics) do
+            [
+              { name: 'gmau_1', events: %w[event1_slot event2_slot], operator: "AND" },
+              { name: 'gmau_2', events: %w[event1_slot event2_slot event3_slot], operator: "AND" },
+              { name: 'gmau_3', events: %w[event1_slot event2_slot event3_slot event5_slot], operator: "AND" },
+              { name: 'gmau_4', events: %w[event4], operator: "AND" }
+            ].map(&:with_indifferent_access)
+          end
+
+          it 'returns the number of unique events for all known events' do
+            results = {
+              'gmau_1' => 3,
+              'gmau_2' => 2,
+              'gmau_3' => 1,
+              'gmau_4' => 3
+            }
+
+            expect(aggregated_metrics_data).to eq(results)
+          end
+        end
+
+        context 'with OR operator' do
+          let(:aggregated_metrics) do
+            [
+              { name: 'gmau_1', events: %w[event3_slot event5_slot], operator: "OR" },
+              { name: 'gmau_2', events: %w[event1_slot event2_slot event3_slot event5_slot], operator: "OR" },
+              { name: 'gmau_3', events: %w[event4], operator: "OR" }
+            ].map(&:with_indifferent_access)
+          end
+
+          it 'returns the number of unique events for all known events' do
+            results = {
+              'gmau_1' => 2,
+              'gmau_2' => 3,
+              'gmau_3' => 3
+            }
+
+            expect(aggregated_metrics_data).to eq(results)
+          end
+        end
+
+        context 'hidden behind feature flag' do
+          let(:enabled_feature_flag) { 'test_ff_enabled' }
+          let(:disabled_feature_flag) { 'test_ff_disabled' }
+          let(:aggregated_metrics) do
+            [
+              # represents stable aggregated metrics that has been fully released
+              { name: 'gmau_without_ff', events: %w[event3_slot event5_slot], operator: "OR" },
+              # represents new aggregated metric that is under performance testing on gitlab.com
+              { name: 'gmau_enabled', events: %w[event4], operator: "AND", feature_flag: enabled_feature_flag },
+              # represents aggregated metric that is under development and shouldn't be yet collected even on gitlab.com
+              { name: 'gmau_disabled', events: %w[event4], operator: "AND", feature_flag: disabled_feature_flag }
+            ].map(&:with_indifferent_access)
+          end
+
+          it 'returns the number of unique events for all known events' do
+            skip_feature_flags_yaml_validation
+            stub_feature_flags(enabled_feature_flag => true, disabled_feature_flag => false)
+
+            expect(aggregated_metrics_data).to eq('gmau_without_ff' => 2, 'gmau_enabled' => 3)
+          end
+        end
+      end
+    end
+
+    describe '.aggregated_metrics_weekly_data' do
+      subject(:aggregated_metrics_data) { described_class.aggregated_metrics_weekly_data }
+
+      before do
+        described_class.track_event(entity1, 'event1_slot', 2.days.ago)
+        described_class.track_event(entity2, 'event1_slot', 2.days.ago)
+        described_class.track_event(entity3, 'event1_slot', 2.days.ago)
+        described_class.track_event(entity1, 'event2_slot', 2.days.ago)
+        described_class.track_event(entity2, 'event2_slot', 3.days.ago)
+        described_class.track_event(entity3, 'event2_slot', 3.days.ago)
+        described_class.track_event(entity1, 'event3_slot', 3.days.ago)
+        described_class.track_event(entity2, 'event3_slot', 3.days.ago)
+        described_class.track_event(entity2, 'event5_slot', 3.days.ago)
+
+        # events out of time scope
+        described_class.track_event(entity3, 'event2_slot', 8.days.ago)
+
+        # events in different slots
+        described_class.track_event(entity1, 'event4', 2.days.ago)
+        described_class.track_event(entity2, 'event4', 2.days.ago)
+        described_class.track_event(entity4, 'event4', 2.days.ago)
+      end
+
+      it_behaves_like 'aggregated_metrics_data'
+    end
+
+    describe '.aggregated_metrics_monthly_data' do
+      subject(:aggregated_metrics_data) { described_class.aggregated_metrics_monthly_data }
+
+      it_behaves_like 'aggregated_metrics_data' do
+        before do
+          described_class.track_event(entity1, 'event1_slot', 2.days.ago)
+          described_class.track_event(entity2, 'event1_slot', 2.days.ago)
+          described_class.track_event(entity3, 'event1_slot', 2.days.ago)
+          described_class.track_event(entity1, 'event2_slot', 2.days.ago)
+          described_class.track_event(entity2, 'event2_slot', 3.days.ago)
+          described_class.track_event(entity3, 'event2_slot', 3.days.ago)
+          described_class.track_event(entity1, 'event3_slot', 3.days.ago)
+          described_class.track_event(entity2, 'event3_slot', 10.days.ago)
+          described_class.track_event(entity2, 'event5_slot', 4.weeks.ago.advance(days: 1))
+
+          # events out of time scope
+          described_class.track_event(entity1, 'event5_slot', 4.weeks.ago.advance(days: -1))
+
+          # events in different slots
+          described_class.track_event(entity1, 'event4', 2.days.ago)
+          described_class.track_event(entity2, 'event4', 2.days.ago)
+          described_class.track_event(entity4, 'event4', 2.days.ago)
+        end
+      end
+
+      context 'Redis calls' do
+        let(:aggregated_metrics) do
+          [
+            { name: 'gmau_3', events: %w[event1_slot event2_slot event3_slot event5_slot], operator: "AND" }
+          ].map(&:with_indifferent_access)
+        end
+
+        let(:known_events) do
+          [
+            { name: 'event1_slot', redis_slot: "slot", category: 'category1', aggregation: "weekly" },
+            { name: 'event2_slot', redis_slot: "slot", category: 'category2', aggregation: "weekly" },
+            { name: 'event3_slot', redis_slot: "slot", category: 'category3', aggregation: "weekly" },
+            { name: 'event5_slot', redis_slot: "slot", category: 'category4', aggregation: "weekly" }
+          ].map(&:with_indifferent_access)
+        end
+
+        it 'caches intermediate operations' do
+          allow(described_class).to receive(:known_events).and_return(known_events)
+          allow(described_class).to receive(:aggregated_metrics).and_return(aggregated_metrics)
+
+          4.downto(1) do |subset_size|
+            known_events.combination(subset_size).each do |events|
+              keys = described_class.send(:weekly_redis_keys, events: events, start_date: 4.weeks.ago.to_date, end_date: Date.current)
+              expect(Gitlab::Redis::HLL).to receive(:count).with(keys: keys).once.and_return(0)
+            end
+          end
+
+          subject
+        end
+      end
+    end
+  end
 end