Includes logic to persist namespace statistics

- Add two new ActiveRecord models:
  - RootNamespaceStoragestatistics will persist root namespace statistics
  - NamespaceAggregationSchedule will save information when a new update
to the namespace statistics needs to be scheduled
- Inject into UpdateProjectStatistics concern a new callback that will
call an async job to insert a new row onto NamespaceAggregationSchedule
table
- When a new row is inserted a new job is scheduled. This job will
update call an specific service to update the statistics and after that
it will delete thee aggregated scheduled row
- The RefresherServices makes heavy use of arel to build composable
queries to update Namespace::RootStorageStatistics attributes.
- Add an extra worker to traverse pending rows on
NAmespace::AggregationSchedule table and schedule a worker for each one
of this rows.
- Add an extra worker to traverse pending rows on
NAmespace::AggregationSchedule table and schedule a worker for each one
of this rows

2 files changed, 68 insertions, 0 deletions

diff --git a/app/models/namespace/aggregation_schedule.rb b/app/models/namespace/aggregation_schedule.rb
index 43afd0b954c..355593597c6 100644
--- a/app/models/namespace/aggregation_schedule.rb
+++ b/app/models/namespace/aggregation_schedule.rb
@@ -1,7 +1,47 @@
 # frozen_string_literal: true
 
 class Namespace::AggregationSchedule < ApplicationRecord
+  include AfterCommitQueue
+  include ExclusiveLeaseGuard
+
   self.primary_key = :namespace_id
 
+  DEFAULT_LEASE_TIMEOUT = 3.hours
+  REDIS_SHARED_KEY = 'gitlab:update_namespace_statistics_delay'.freeze
+
   belongs_to :namespace
+
+  after_create :schedule_root_storage_statistics
+
+  def self.delay_timeout
+    redis_timeout = Gitlab::Redis::SharedState.with do |redis|
+      redis.get(REDIS_SHARED_KEY)
+    end
+
+    redis_timeout.nil? ? DEFAULT_LEASE_TIMEOUT : redis_timeout.to_i
+  end
+
+  def schedule_root_storage_statistics
+    run_after_commit_or_now do
+      try_obtain_lease do
+        Namespaces::RootStatisticsWorker
+          .perform_async(namespace_id)
+
+        Namespaces::RootStatisticsWorker
+          .perform_in(self.class.delay_timeout, namespace_id)
+      end
+    end
+  end
+
+  private
+
+  # Used by ExclusiveLeaseGuard
+  def lease_timeout
+    self.class.delay_timeout
+  end
+
+  # Used by ExclusiveLeaseGuard
+  def lease_key
+    "namespace:namespaces_root_statistics:#{namespace_id}"
+  end
 end
diff --git a/app/models/namespace/root_storage_statistics.rb b/app/models/namespace/root_storage_statistics.rb
index de28eb6b37f..56c430013ee 100644
--- a/app/models/namespace/root_storage_statistics.rb
+++ b/app/models/namespace/root_storage_statistics.rb
@@ -1,10 +1,38 @@
 # frozen_string_literal: true
 
 class Namespace::RootStorageStatistics < ApplicationRecord
+  STATISTICS_ATTRIBUTES = %w(storage_size repository_size wiki_size lfs_objects_size build_artifacts_size packages_size).freeze
+
   self.primary_key = :namespace_id
 
   belongs_to :namespace
   has_one :route, through: :namespace
 
   delegate :all_projects, to: :namespace
+
+  def recalculate!
+    update!(attributes_from_project_statistics)
+  end
+
+  private
+
+  def attributes_from_project_statistics
+    from_project_statistics
+      .take
+      .attributes
+      .slice(*STATISTICS_ATTRIBUTES)
+  end
+
+  def from_project_statistics
+    all_projects
+      .joins('INNER JOIN project_statistics ps ON ps.project_id  = projects.id')
+      .select(
+        'COALESCE(SUM(ps.storage_size), 0) AS storage_size',
+        'COALESCE(SUM(ps.repository_size), 0) AS repository_size',
+        'COALESCE(SUM(ps.wiki_size), 0) AS wiki_size',
+        'COALESCE(SUM(ps.lfs_objects_size), 0) AS lfs_objects_size',
+        'COALESCE(SUM(ps.build_artifacts_size), 0) AS build_artifacts_size',
+        'COALESCE(SUM(ps.packages_size), 0) AS packages_size'
+      )
+  end
 end