Merge branch 'ce-6064-geo-sql-query-for-counting-projects-with-wikis-is-very-slow' into 'master'

[CE Backport] Geo: Cache projects_count and wikis_count in SiteStatistic

See merge request gitlab-org/gitlab-ce!20413

3 files changed, 105 insertions, 0 deletions

diff --git a/app/models/project.rb b/app/models/project.rb
index f880d728839..32315dfaa01 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -31,6 +31,7 @@ class Project < ActiveRecord::Base
 
   BoardLimitExceeded = Class.new(StandardError)
 
+  STATISTICS_ATTRIBUTE = 'repositories_count'.freeze
   NUMBER_OF_PERMITTED_BOARDS = 1
   UNKNOWN_IMPORT_URL = 'http://unknown.git'.freeze
   # Hashed Storage versions handle rolling out new storage to project and dependents models:
@@ -79,6 +80,10 @@ class Project < ActiveRecord::Base
 
   after_create :create_project_feature, unless: :project_feature
 
+  after_create -> { SiteStatistic.track(STATISTICS_ATTRIBUTE) }
+  before_destroy ->(project) { project.project_feature.untrack_statistics_for_deletion! }
+  after_destroy -> { SiteStatistic.untrack(STATISTICS_ATTRIBUTE) }
+
   after_create :create_ci_cd_settings,
     unless: :ci_cd_settings,
     if: proc { ProjectCiCdSetting.available? }
diff --git a/app/models/project_feature.rb b/app/models/project_feature.rb
index bfb8d703ec9..9c768b13f78 100644
--- a/app/models/project_feature.rb
+++ b/app/models/project_feature.rb
@@ -19,6 +19,7 @@ class ProjectFeature < ActiveRecord::Base
   ENABLED  = 20
 
   FEATURES = %i(issues merge_requests wiki snippets builds repository).freeze
+  STATISTICS_ATTRIBUTE = 'wikis_count'.freeze
 
   class << self
     def access_level_attribute(feature)
@@ -52,6 +53,9 @@ class ProjectFeature < ActiveRecord::Base
   default_value_for :wiki_access_level,           value: ENABLED, allows_nil: false
   default_value_for :repository_access_level,     value: ENABLED, allows_nil: false
 
+  after_create ->(model) { SiteStatistic.track(STATISTICS_ATTRIBUTE) if model.wiki_enabled? }
+  after_update :update_site_statistics
+
   def feature_available?(feature, user)
     get_permission(user, access_level(feature))
   end
@@ -76,8 +80,30 @@ class ProjectFeature < ActiveRecord::Base
     issues_access_level > DISABLED
   end
 
+  # This is a workaround for the removal hooks not been triggered when removing a Project.
+  #
+  # ProjectFeature is removed using database cascade index rule.
+  # This method is called by Project model when deletion starts.
+  def untrack_statistics_for_deletion!
+    return unless wiki_enabled?
+
+    SiteStatistic.untrack(STATISTICS_ATTRIBUTE)
+  end
+
   private
 
+  def update_site_statistics
+    return unless wiki_access_level_changed?
+
+    if self.wiki_access_level_was == DISABLED
+      # possible new states are PRIVATE / ENABLED, both should be tracked
+      SiteStatistic.track(STATISTICS_ATTRIBUTE)
+    elsif self.wiki_access_level == DISABLED
+      # old state was either PRIVATE / ENABLED, only untrack if new state is DISABLED
+      SiteStatistic.untrack(STATISTICS_ATTRIBUTE)
+    end
+  end
+
   # Validates builds and merge requests access level
   # which cannot be higher than repository access level
   def repository_children_level
diff --git a/app/models/site_statistic.rb b/app/models/site_statistic.rb
new file mode 100644
index 00000000000..9c9c3172fe6
--- /dev/null
+++ b/app/models/site_statistic.rb
@@ -0,0 +1,74 @@
+class SiteStatistic < ActiveRecord::Base
+  # prevents the creation of multiple rows
+  default_value_for :id, 1
+
+  COUNTER_ATTRIBUTES = %w(repositories_count wikis_count).freeze
+  REQUIRED_SCHEMA_VERSION = 20180629153018
+
+  # Tracks specific attribute
+  #
+  # @param [String] raw_attribute must be one of the values listed in COUNTER_ATTRIBUTES
+  def self.track(raw_attribute)
+    with_statistics_available(raw_attribute) do |attribute|
+      SiteStatistic.update_all(["#{attribute} = #{attribute}+1"])
+    end
+  end
+
+  # Untracks specific attribute
+  #
+  # @param [String] raw_attribute must be one of the values listed in COUNTER_ATTRIBUTES
+  def self.untrack(raw_attribute)
+    with_statistics_available(raw_attribute) do |attribute|
+      SiteStatistic.update_all(["#{attribute} = #{attribute}-1 WHERE #{attribute} > 0"])
+    end
+  end
+
+  # Wrapper for track/untrack operations with basic validations and enforced requirements
+  #
+  # @param [String] raw_attribute must be one of the values listed in COUNTER_ATTRIBUTES
+  # @yield [String] attribute quoted to be used inside SQL / Arel query
+  def self.with_statistics_available(raw_attribute)
+    unless raw_attribute.in?(COUNTER_ATTRIBUTES)
+      raise ArgumentError, "Invalid attribute: '#{raw_attribute}' to '#{caller_locations(1, 1)[0].label}' method. " \
+                           "Valid attributes are: #{COUNTER_ATTRIBUTES.join(', ')}"
+    end
+
+    return unless available?
+
+    self.fetch # make sure record exists
+
+    attribute = self.connection.quote_column_name(raw_attribute)
+
+    # will be running on its own transaction context
+    yield(attribute)
+  end
+
+  # Returns a site statistic record with tracked information
+  #
+  # @return [SiteStatistic] record with tracked information
+  def self.fetch
+    SiteStatistic.transaction(requires_new: true) do
+      SiteStatistic.first_or_create!
+    end
+  rescue ActiveRecord::RecordNotUnique
+    retry
+  end
+
+  # Return whether required schema change is available
+  #
+  # This is needed in order to degrade gracefully when testing schema migrations
+  #
+  # @return [Boolean] whether schema is available
+  def self.available?
+    @available_flag ||= ActiveRecord::Migrator.current_version >= REQUIRED_SCHEMA_VERSION
+  end
+
+  # Resets cached column information
+  #
+  # This is called during schema migration specs, in order to reset internal cache state
+  def self.reset_column_information
+    @available_flag = nil
+
+    super
+  end
+end