Includes logic to persist namespace statistics

- Add two new ActiveRecord models:
  - RootNamespaceStoragestatistics will persist root namespace statistics
  - NamespaceAggregationSchedule will save information when a new update
to the namespace statistics needs to be scheduled
- Inject into UpdateProjectStatistics concern a new callback that will
call an async job to insert a new row onto NamespaceAggregationSchedule
table
- When a new row is inserted a new job is scheduled. This job will
update call an specific service to update the statistics and after that
it will delete thee aggregated scheduled row
- The RefresherServices makes heavy use of arel to build composable
queries to update Namespace::RootStorageStatistics attributes.
- Add an extra worker to traverse pending rows on
NAmespace::AggregationSchedule table and schedule a worker for each one
of this rows.
- Add an extra worker to traverse pending rows on
NAmespace::AggregationSchedule table and schedule a worker for each one
of this rows

2 files changed, 137 insertions, 1 deletions

diff --git a/spec/models/namespace/aggregation_schedule_spec.rb b/spec/models/namespace/aggregation_schedule_spec.rb
index 5ba7547ff4d..8ed0248e1b2 100644
--- a/spec/models/namespace/aggregation_schedule_spec.rb
+++ b/spec/models/namespace/aggregation_schedule_spec.rb
@@ -2,6 +2,77 @@
 
 require 'spec_helper'
 
-RSpec.describe Namespace::AggregationSchedule, type: :model do
+RSpec.describe Namespace::AggregationSchedule, :clean_gitlab_redis_shared_state, type: :model do
+  include ExclusiveLeaseHelpers
+
   it { is_expected.to belong_to :namespace }
+
+  describe '.delay_timeout' do
+    context 'when timeout is set on redis' do
+      it 'uses personalized timeout' do
+        Gitlab::Redis::SharedState.with do |redis|
+          redis.set(described_class::REDIS_SHARED_KEY, 1.hour)
+        end
+
+        expect(described_class.delay_timeout).to eq(1.hour)
+      end
+    end
+
+    context 'when timeout is not set on redis' do
+      it 'uses default timeout' do
+        expect(described_class.delay_timeout).to eq(3.hours)
+      end
+    end
+  end
+
+  describe '#schedule_root_storage_statistics' do
+    let(:namespace) { create(:namespace) }
+    let(:aggregation_schedule) { namespace.build_aggregation_schedule }
+    let(:lease_key) { "namespace:namespaces_root_statistics:#{namespace.id}" }
+
+    context "when we can't obtain the lease" do
+      it 'does not schedule the workers' do
+        stub_exclusive_lease_taken(lease_key, timeout: described_class::DEFAULT_LEASE_TIMEOUT)
+
+        expect(Namespaces::RootStatisticsWorker)
+          .not_to receive(:perform_async)
+
+        expect(Namespaces::RootStatisticsWorker)
+          .not_to receive(:perform_in)
+
+        aggregation_schedule.save!
+      end
+    end
+
+    context 'when we can obtain the lease' do
+      it 'schedules a root storage statistics after create' do
+        stub_exclusive_lease(lease_key, timeout: described_class::DEFAULT_LEASE_TIMEOUT)
+
+        expect(Namespaces::RootStatisticsWorker)
+          .to receive(:perform_async).once
+
+        expect(Namespaces::RootStatisticsWorker)
+          .to receive(:perform_in).once
+          .with(described_class::DEFAULT_LEASE_TIMEOUT, aggregation_schedule.namespace_id )
+
+        aggregation_schedule.save!
+      end
+    end
+
+    context 'with a personalized lease timeout' do
+      before do
+        Gitlab::Redis::SharedState.with do |redis|
+          redis.set(described_class::REDIS_SHARED_KEY, 1.hour)
+        end
+      end
+
+      it 'uses a personalized time' do
+        expect(Namespaces::RootStatisticsWorker)
+          .to receive(:perform_in)
+          .with(1.hour, aggregation_schedule.namespace_id)
+
+        aggregation_schedule.save!
+      end
+    end
+  end
 end
diff --git a/spec/models/namespace/root_storage_statistics_spec.rb b/spec/models/namespace/root_storage_statistics_spec.rb
index f6fb5af5aae..3229a32234e 100644
--- a/spec/models/namespace/root_storage_statistics_spec.rb
+++ b/spec/models/namespace/root_storage_statistics_spec.rb
@@ -7,4 +7,69 @@ RSpec.describe Namespace::RootStorageStatistics, type: :model do
   it { is_expected.to have_one(:route).through(:namespace) }
 
   it { is_expected.to delegate_method(:all_projects).to(:namespace) }
+
+  describe '#recalculate!' do
+    let(:namespace) { create(:group) }
+    let(:root_storage_statistics) { create(:namespace_root_storage_statistics, namespace: namespace) }
+
+    let(:project1) { create(:project, namespace: namespace) }
+    let(:project2) { create(:project, namespace: namespace) }
+
+    let!(:stat1) { create(:project_statistics, project: project1, with_data: true, size_multiplier: 100) }
+    let!(:stat2) { create(:project_statistics, project: project2, with_data: true, size_multiplier: 200) }
+
+    shared_examples 'data refresh' do
+      it 'aggregates project statistics' do
+        root_storage_statistics.recalculate!
+
+        root_storage_statistics.reload
+
+        total_repository_size = stat1.repository_size + stat2.repository_size
+        total_wiki_size = stat1.wiki_size + stat2.wiki_size
+        total_lfs_objects_size = stat1.lfs_objects_size + stat2.lfs_objects_size
+        total_build_artifacts_size = stat1.build_artifacts_size + stat2.build_artifacts_size
+        total_packages_size = stat1.packages_size + stat2.packages_size
+        total_storage_size = stat1.storage_size + stat2.storage_size
+
+        expect(root_storage_statistics.repository_size).to eq(total_repository_size)
+        expect(root_storage_statistics.wiki_size).to eq(total_wiki_size)
+        expect(root_storage_statistics.lfs_objects_size).to eq(total_lfs_objects_size)
+        expect(root_storage_statistics.build_artifacts_size).to eq(total_build_artifacts_size)
+        expect(root_storage_statistics.packages_size).to eq(total_packages_size)
+        expect(root_storage_statistics.storage_size).to eq(total_storage_size)
+      end
+
+      it 'works when there are no projects' do
+        Project.delete_all
+
+        root_storage_statistics.recalculate!
+
+        root_storage_statistics.reload
+        expect(root_storage_statistics.repository_size).to eq(0)
+        expect(root_storage_statistics.wiki_size).to eq(0)
+        expect(root_storage_statistics.lfs_objects_size).to eq(0)
+        expect(root_storage_statistics.build_artifacts_size).to eq(0)
+        expect(root_storage_statistics.packages_size).to eq(0)
+        expect(root_storage_statistics.storage_size).to eq(0)
+      end
+    end
+
+    it_behaves_like 'data refresh'
+
+    context 'with subgroups', :nested_groups do
+      let(:subgroup1) { create(:group, parent: namespace)}
+      let(:subgroup2) { create(:group, parent: subgroup1)}
+
+      let(:project1) { create(:project, namespace: subgroup1) }
+      let(:project2) { create(:project, namespace: subgroup2) }
+
+      it_behaves_like 'data refresh'
+    end
+
+    context 'with a personal namespace' do
+      let(:namespace) { create(:user).namespace }
+
+      it_behaves_like 'data refresh'
+    end
+  end
 end