Add latest changes from gitlab-org/gitlab@13-1-stable-ee

1 files changed, 94 insertions, 75 deletions

diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb
index e60c786b52c..7b6f5e69ee1 100644
--- a/lib/gitlab/usage_data.rb
+++ b/lib/gitlab/usage_data.rb
@@ -1,27 +1,25 @@
 # frozen_string_literal: true
 
-# For hardening usage ping and make it easier to add measures there is in place
-#   * alt_usage_data method
-#     handles StandardError and fallbacks into -1 this way not all measures fail if we encounter one exception
+# When developing usage data metrics use the below usage data interface methods
+# unless you have good reasons to implement custom usage data
+# See `lib/gitlab/utils/usage_data.rb`
 #
-#     Examples:
-#     alt_usage_data { Gitlab::VERSION }
-#     alt_usage_data { Gitlab::CurrentSettings.uuid }
-#
-#   * redis_usage_data method
-#     handles ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent
-#     returns -1 when a block is sent or hash with all values -1 when a counter is sent
-#     different behaviour due to 2 different implementations of redis counter
-#
-#     Examples:
-#     redis_usage_data(Gitlab::UsageDataCounters::WikiPageCounter)
-#     redis_usage_data { ::Gitlab::UsageCounters::PodLogs.usage_totals[:total] }
+# Examples
+#   issues_using_zoom_quick_actions: distinct_count(ZoomMeeting, :issue_id),
+#   active_user_count: count(User.active)
+#   alt_usage_data { Gitlab::VERSION }
+#   redis_usage_data(Gitlab::UsageDataCounters::WikiPageCounter)
+#   redis_usage_data { ::Gitlab::UsageCounters::PodLogs.usage_totals[:total] }
+
 module Gitlab
   class UsageData
     BATCH_SIZE = 100
-    FALLBACK = -1
 
     class << self
+      include Gitlab::Utils::UsageData
+      include Gitlab::Utils::StrongMemoize
+      include Gitlab::UsageDataConcerns::Topology
+
       def data(force_refresh: false)
         Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) do
           uncached_data
@@ -29,12 +27,15 @@ module Gitlab
       end
 
       def uncached_data
+        clear_memoized_limits
+
         license_usage_data
           .merge(system_usage_data)
           .merge(features_usage_data)
           .merge(components_usage_data)
           .merge(cycle_analytics_usage_data)
           .merge(object_store_usage_data)
+          .merge(topology_usage_data)
           .merge(recording_ce_finish_data)
       end
 
@@ -44,7 +45,7 @@ module Gitlab
 
       def license_usage_data
         {
-          recorded_at: Time.now, # should be calculated very first
+          recorded_at: recorded_at,
           uuid: alt_usage_data { Gitlab::CurrentSettings.uuid },
           hostname: alt_usage_data { Gitlab.config.gitlab.host },
           version: alt_usage_data { Gitlab::VERSION },
@@ -54,6 +55,10 @@ module Gitlab
         }
       end
 
+      def recorded_at
+        Time.now
+      end
+
       def recording_ce_finish_data
         {
           recording_ce_finished_at: Time.now
@@ -64,6 +69,7 @@ module Gitlab
       # rubocop: disable CodeReuse/ActiveRecord
       def system_usage_data
         alert_bot_incident_count = count(::Issue.authored(::User.alert_bot))
+        issues_created_manually_from_alerts = count(Issue.with_alert_management_alerts.not_authored_by(::User.alert_bot))
 
         {
           counts: {
@@ -114,7 +120,9 @@ module Gitlab
             issues_with_associated_zoom_link: count(ZoomMeeting.added_to_issue),
             issues_using_zoom_quick_actions: distinct_count(ZoomMeeting, :issue_id),
             issues_with_embedded_grafana_charts_approx: grafana_embed_usage_data,
-            issues_created_gitlab_alerts: count(Issue.with_alert_management_alerts.not_authored_by(::User.alert_bot)),
+            issues_created_from_alerts: total_alert_issues,
+            issues_created_gitlab_alerts: issues_created_manually_from_alerts,
+            issues_created_manually_from_alerts: issues_created_manually_from_alerts,
             incident_issues: alert_bot_incident_count,
             alert_bot_incident_issues: alert_bot_incident_count,
             incident_labeled_issues: count(::Issue.with_label_attributes(IncidentManagement::CreateIssueService::INCIDENT_LABEL)),
@@ -131,11 +139,17 @@ module Gitlab
             projects_with_error_tracking_enabled: count(::ErrorTracking::ProjectErrorTrackingSetting.where(enabled: true)),
             projects_with_alerts_service_enabled: count(AlertsService.active),
             projects_with_prometheus_alerts: distinct_count(PrometheusAlert, :project_id),
+            projects_with_terraform_reports: distinct_count(::Ci::JobArtifact.terraform_reports, :project_id),
+            projects_with_terraform_states: distinct_count(::Terraform::State, :project_id),
             protected_branches: count(ProtectedBranch),
             releases: count(Release),
             remote_mirrors: count(RemoteMirror),
             snippets: count(Snippet),
+            personal_snippets: count(PersonalSnippet),
+            project_snippets: count(ProjectSnippet),
             suggestions: count(Suggestion),
+            terraform_reports: count(::Ci::JobArtifact.terraform_reports),
+            terraform_states: count(::Terraform::State),
             todos: count(Todo),
             uploads: count(Upload),
             web_hooks: count(WebHook),
@@ -147,7 +161,8 @@ module Gitlab
             usage_counters,
             user_preferences_usage,
             ingress_modsecurity_usage,
-            container_expiration_policies_usage
+            container_expiration_policies_usage,
+            merge_requests_usage(default_time_period)
           )
         }
       end
@@ -174,19 +189,20 @@ module Gitlab
 
       def features_usage_data_ce
         {
-          container_registry_enabled: alt_usage_data { Gitlab.config.registry.enabled },
+          instance_auto_devops_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.auto_devops_enabled? },
+          container_registry_enabled: alt_usage_data(fallback: nil) { Gitlab.config.registry.enabled },
           dependency_proxy_enabled: Gitlab.config.try(:dependency_proxy)&.enabled,
-          gitlab_shared_runners_enabled: alt_usage_data { Gitlab.config.gitlab_ci.shared_runners_enabled },
-          gravatar_enabled: alt_usage_data { Gitlab::CurrentSettings.gravatar_enabled? },
-          ldap_enabled: alt_usage_data { Gitlab.config.ldap.enabled },
-          mattermost_enabled: alt_usage_data { Gitlab.config.mattermost.enabled },
-          omniauth_enabled: alt_usage_data { Gitlab::Auth.omniauth_enabled? },
-          prometheus_metrics_enabled: alt_usage_data { Gitlab::Metrics.prometheus_metrics_enabled? },
-          reply_by_email_enabled: alt_usage_data { Gitlab::IncomingEmail.enabled? },
-          signup_enabled: alt_usage_data { Gitlab::CurrentSettings.allow_signup? },
-          web_ide_clientside_preview_enabled: alt_usage_data { Gitlab::CurrentSettings.web_ide_clientside_preview_enabled? },
+          gitlab_shared_runners_enabled: alt_usage_data(fallback: nil) { Gitlab.config.gitlab_ci.shared_runners_enabled },
+          gravatar_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.gravatar_enabled? },
+          ldap_enabled: alt_usage_data(fallback: nil) { Gitlab.config.ldap.enabled },
+          mattermost_enabled: alt_usage_data(fallback: nil) { Gitlab.config.mattermost.enabled },
+          omniauth_enabled: alt_usage_data(fallback: nil) { Gitlab::Auth.omniauth_enabled? },
+          prometheus_metrics_enabled: alt_usage_data(fallback: nil) { Gitlab::Metrics.prometheus_metrics_enabled? },
+          reply_by_email_enabled: alt_usage_data(fallback: nil) { Gitlab::IncomingEmail.enabled? },
+          signup_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.allow_signup? },
+          web_ide_clientside_preview_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.web_ide_clientside_preview_enabled? },
           ingress_modsecurity_enabled: Feature.enabled?(:ingress_modsecurity),
-          grafana_link_enabled: alt_usage_data { Gitlab::CurrentSettings.grafana_enabled? }
+          grafana_link_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.grafana_enabled? }
         }
       end
 
@@ -213,14 +229,15 @@ module Gitlab
 
       def components_usage_data
         {
-          git: { version: alt_usage_data { Gitlab::Git.version } },
+          git: { version: alt_usage_data(fallback: { major: -1 }) { Gitlab::Git.version } },
           gitaly: {
             version: alt_usage_data { Gitaly::Server.all.first.server_version },
             servers: alt_usage_data { Gitaly::Server.count },
-            filesystems: alt_usage_data { Gitaly::Server.filesystems }
+            clusters: alt_usage_data { Gitaly::Server.gitaly_clusters },
+            filesystems: alt_usage_data(fallback: ["-1"]) { Gitaly::Server.filesystems }
           },
           gitlab_pages: {
-            enabled: alt_usage_data { Gitlab.config.pages.enabled },
+            enabled: alt_usage_data(fallback: nil) { Gitlab.config.pages.enabled },
             version: alt_usage_data { Gitlab::Pages::VERSION }
           },
           database: {
@@ -382,65 +399,67 @@ module Gitlab
         {} # augmented in EE
       end
 
-      def count(relation, column = nil, batch: true, start: nil, finish: nil)
-        if batch && Feature.enabled?(:usage_ping_batch_counter, default_enabled: true)
-          Gitlab::Database::BatchCount.batch_count(relation, column, start: start, finish: finish)
-        else
-          relation.count
-        end
-      rescue ActiveRecord::StatementInvalid
-        FALLBACK
-      end
+      # rubocop: disable CodeReuse/ActiveRecord
+      def merge_requests_usage(time_period)
+        query =
+          Event
+            .where(target_type: Event::TARGET_TYPES[:merge_request].to_s)
+            .where(time_period)
+
+        merge_request_users = distinct_count(
+          query,
+          :author_id,
+          batch_size: 5_000, # Based on query performance, this is the optimal batch size.
+          start: User.minimum(:id),
+          finish: User.maximum(:id)
+        )
 
-      def distinct_count(relation, column = nil, batch: true, start: nil, finish: nil)
-        if batch && Feature.enabled?(:usage_ping_batch_counter, default_enabled: true)
-          Gitlab::Database::BatchCount.batch_distinct_count(relation, column, start: start, finish: finish)
-        else
-          relation.distinct_count_by(column)
-        end
-      rescue ActiveRecord::StatementInvalid
-        FALLBACK
+        {
+          merge_requests_users: merge_request_users
+        }
       end
+      # rubocop: enable CodeReuse/ActiveRecord
 
-      def alt_usage_data(value = nil, fallback: FALLBACK, &block)
-        if block_given?
-          yield
+      def installation_type
+        if Rails.env.production?
+          Gitlab::INSTALLATION_TYPE
         else
-          value
+          "gitlab-development-kit"
         end
-      rescue
-        fallback
       end
 
-      def redis_usage_data(counter = nil, &block)
-        if block_given?
-          redis_usage_counter(&block)
-        elsif counter.present?
-          redis_usage_data_totals(counter)
-        end
+      def default_time_period
+        { created_at: 28.days.ago..Time.current }
       end
 
       private
 
-      def redis_usage_counter
-        yield
-      rescue ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent
-        FALLBACK
+      def total_alert_issues
+        # Remove prometheus table queries once they are deprecated
+        # To be removed with https://gitlab.com/gitlab-org/gitlab/-/issues/217407.
+        [
+          count(Issue.with_alert_management_alerts),
+          count(::Issue.with_self_managed_prometheus_alert_events),
+          count(::Issue.with_prometheus_alert_events)
+        ].reduce(:+)
       end
 
-      def redis_usage_data_totals(counter)
-        counter.totals
-      rescue ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent
-        counter.fallback_totals
+      def user_minimum_id
+        strong_memoize(:user_minimum_id) do
+          ::User.minimum(:id)
+        end
       end
 
-      def installation_type
-        if Rails.env.production?
-          Gitlab::INSTALLATION_TYPE
-        else
-          "gitlab-development-kit"
+      def user_maximum_id
+        strong_memoize(:user_maximum_id) do
+          ::User.maximum(:id)
         end
       end
+
+      def clear_memoized_limits
+        clear_memoization(:user_minimum_id)
+        clear_memoization(:user_maximum_id)
+      end
     end
   end
 end