Add latest changes from gitlab-org/gitlab@13-5-stable-eev13.5.0-rc42

1 files changed, 76 insertions, 28 deletions

diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb
index 89605ce5d07..68f24559b1f 100644
--- a/lib/gitlab/usage_data.rb
+++ b/lib/gitlab/usage_data.rb
@@ -12,6 +12,19 @@
 #   redis_usage_data { ::Gitlab::UsageCounters::PodLogs.usage_totals[:total] }
 module Gitlab
   class UsageData
+    CE_MEMOIZED_VALUES = %i(
+        issue_minimum_id
+        issue_maximum_id
+        project_minimum_id
+        project_maximum_id
+        user_minimum_id
+        user_maximum_id
+        unique_visit_service
+        deployment_minimum_id
+        deployment_maximum_id
+        auth_providers
+      ).freeze
+
     class << self
       include Gitlab::Utils::UsageData
       include Gitlab::Utils::StrongMemoize
@@ -138,8 +151,10 @@ module Gitlab
             pages_domains: count(PagesDomain),
             pool_repositories: count(PoolRepository),
             projects: count(Project),
+            projects_creating_incidents: distinct_count(Issue.incident, :project_id),
             projects_imported_from_github: count(Project.where(import_type: 'github')),
             projects_with_repositories_enabled: count(ProjectFeature.where('repository_access_level > ?', ProjectFeature::DISABLED)),
+            projects_with_tracing_enabled: count(ProjectTracingSetting),
             projects_with_error_tracking_enabled: count(::ErrorTracking::ProjectErrorTrackingSetting.where(enabled: true)),
             projects_with_alerts_service_enabled: count(AlertsService.active),
             projects_with_prometheus_alerts: distinct_count(PrometheusAlert, :project_id),
@@ -166,8 +181,7 @@ module Gitlab
             user_preferences_usage,
             ingress_modsecurity_usage,
             container_expiration_policies_usage,
-            service_desk_counts,
-            snowplow_event_counts
+            service_desk_counts
           ).tap do |data|
             data[:snippets] = data[:personal_snippets] + data[:project_snippets]
           end
@@ -175,7 +189,7 @@ module Gitlab
       end
       # rubocop: enable Metrics/AbcSize
 
-      def snowplow_event_counts(time_period: {})
+      def snowplow_event_counts(time_period)
         return {} unless report_snowplow_events?
 
         {
@@ -200,7 +214,7 @@ module Gitlab
             personal_snippets: count(PersonalSnippet.where(last_28_days_time_period)),
             project_snippets: count(ProjectSnippet.where(last_28_days_time_period))
           }.merge(
-            snowplow_event_counts(time_period: last_28_days_time_period(column: :collector_tstamp))
+            snowplow_event_counts(last_28_days_time_period(column: :collector_tstamp))
           ).tap do |data|
             data[:snippets] = data[:personal_snippets] + data[:project_snippets]
           end
@@ -242,7 +256,8 @@ module Gitlab
           signup_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.allow_signup? },
           web_ide_clientside_preview_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.web_ide_clientside_preview_enabled? },
           ingress_modsecurity_enabled: Feature.enabled?(:ingress_modsecurity),
-          grafana_link_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.grafana_enabled? }
+          grafana_link_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.grafana_enabled? },
+          gitpod_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.gitpod_enabled? }
         }
       end
 
@@ -264,7 +279,8 @@ module Gitlab
           Gitlab::UsageDataCounters::SourceCodeCounter,
           Gitlab::UsageDataCounters::MergeRequestCounter,
           Gitlab::UsageDataCounters::DesignsCounter,
-          Gitlab::UsageDataCounters::KubernetesAgentCounter
+          Gitlab::UsageDataCounters::KubernetesAgentCounter,
+          Gitlab::UsageDataCounters::StaticSiteEditorCounter
         ]
       end
 
@@ -287,7 +303,8 @@ module Gitlab
           },
           database: {
             adapter: alt_usage_data { Gitlab::Database.adapter_name },
-            version: alt_usage_data { Gitlab::Database.version }
+            version: alt_usage_data { Gitlab::Database.version },
+            pg_system_id: alt_usage_data { Gitlab::Database.system_id }
           },
           mail: {
             smtp_server: alt_usage_data { ActionMailer::Base.smtp_settings[:address] }
@@ -385,10 +402,12 @@ module Gitlab
       def services_usage
         # rubocop: disable UsageData/LargeTable:
         Service.available_services_names.each_with_object({}) do |service_name, response|
-          response["projects_#{service_name}_active".to_sym] = count(Service.active.where(template: false, instance: false, type: "#{service_name}_service".camelize))
+          response["projects_#{service_name}_active".to_sym] = count(Service.active.where.not(project: nil).where(type: "#{service_name}_service".camelize))
+          response["groups_#{service_name}_active".to_sym] = count(Service.active.where.not(group: nil).where(type: "#{service_name}_service".camelize))
           response["templates_#{service_name}_active".to_sym] = count(Service.active.where(template: true, type: "#{service_name}_service".camelize))
           response["instances_#{service_name}_active".to_sym] = count(Service.active.where(instance: true, type: "#{service_name}_service".camelize))
-          response["projects_inheriting_instance_#{service_name}_active".to_sym] = count(Service.active.where.not(inherit_from_id: nil).where(type: "#{service_name}_service".camelize))
+          response["projects_inheriting_#{service_name}_active".to_sym] = count(Service.active.where.not(project: nil).where.not(inherit_from_id: nil).where(type: "#{service_name}_service".camelize))
+          response["groups_inheriting_#{service_name}_active".to_sym] = count(Service.active.where.not(group: nil).where.not(inherit_from_id: nil).where(type: "#{service_name}_service".camelize))
         end.merge(jira_usage, jira_import_usage)
         # rubocop: enable UsageData/LargeTable:
       end
@@ -444,8 +463,11 @@ module Gitlab
       # rubocop: enable UsageData/LargeTable
       # rubocop: enable CodeReuse/ActiveRecord
 
+      # augmented in EE
       def user_preferences_usage
-        {} # augmented in EE
+        {
+          user_preferences_user_gitpod_enabled: count(UserPreference.with_user.gitpod_enabled.merge(User.active))
+        }
       end
 
       def merge_requests_users(time_period)
@@ -469,7 +491,7 @@ module Gitlab
       end
 
       def last_28_days_time_period(column: :created_at)
-        { column => 28.days.ago..Time.current }
+        { column => 30.days.ago..2.days.ago }
       end
 
       # Source: https://gitlab.com/gitlab-data/analytics/blob/master/transform/snowflake-dbt/data/ping_metrics_to_stage_mapping_data.csv
@@ -541,6 +563,7 @@ module Gitlab
           groups: distinct_count(::GroupMember.where(time_period), :user_id),
           users_created: count(::User.where(time_period), start: user_minimum_id, finish: user_maximum_id),
           omniauth_providers: filtered_omniauth_provider_names.reject { |name| name == 'group_saml' },
+          user_auth_by_provider: distinct_count_user_auth_by_provider(time_period),
           projects_imported: {
             gitlab_project: projects_imported_count('gitlab_project', time_period),
             gitlab: projects_imported_count('gitlab', time_period),
@@ -555,7 +578,8 @@ module Gitlab
             jira: distinct_count(::JiraImportState.where(time_period), :user_id),
             fogbugz: projects_imported_count('fogbugz', time_period),
             phabricator: projects_imported_count('phabricator', time_period)
-          }
+          },
+          groups_imported: distinct_count(::GroupImportState.where(time_period), :user_id)
         }
       end
       # rubocop: enable CodeReuse/ActiveRecord
@@ -567,7 +591,8 @@ module Gitlab
           clusters_applications_prometheus: cluster_applications_user_distinct_count(::Clusters::Applications::Prometheus, time_period),
           operations_dashboard_default_dashboard: count(::User.active.with_dashboard('operations').where(time_period),
                                                         start: user_minimum_id,
-                                                        finish: user_maximum_id)
+                                                        finish: user_maximum_id),
+          projects_with_tracing_enabled: distinct_count(::Project.with_tracing_enabled.where(time_period), :creator_id)
         }
       end
       # rubocop: enable CodeReuse/ActiveRecord
@@ -696,10 +721,10 @@ module Gitlab
         counter = Gitlab::UsageDataCounters::EditorUniqueCounter
 
         {
-          action_monthly_active_users_web_ide_edit: redis_usage_data { counter.count_web_ide_edit_actions(date_range) },
-          action_monthly_active_users_sfe_edit: redis_usage_data { counter.count_sfe_edit_actions(date_range) },
-          action_monthly_active_users_snippet_editor_edit: redis_usage_data { counter.count_snippet_editor_edit_actions(date_range) },
-          action_monthly_active_users_ide_edit: redis_usage_data { counter.count_edit_using_editor(date_range) }
+          action_monthly_active_users_web_ide_edit: redis_usage_data { counter.count_web_ide_edit_actions(**date_range) },
+          action_monthly_active_users_sfe_edit: redis_usage_data { counter.count_sfe_edit_actions(**date_range) },
+          action_monthly_active_users_snippet_editor_edit: redis_usage_data { counter.count_snippet_editor_edit_actions(**date_range) },
+          action_monthly_active_users_ide_edit: redis_usage_data { counter.count_edit_using_editor(**date_range) }
         }
       end
 
@@ -801,17 +826,7 @@ module Gitlab
       end
 
       def clear_memoized
-        clear_memoization(:issue_minimum_id)
-        clear_memoization(:issue_maximum_id)
-        clear_memoization(:user_minimum_id)
-        clear_memoization(:user_maximum_id)
-        clear_memoization(:unique_visit_service)
-        clear_memoization(:deployment_minimum_id)
-        clear_memoization(:deployment_maximum_id)
-        clear_memoization(:approval_merge_request_rule_minimum_id)
-        clear_memoization(:approval_merge_request_rule_maximum_id)
-        clear_memoization(:project_minimum_id)
-        clear_memoization(:project_maximum_id)
+        CE_MEMOIZED_VALUES.each { |v| clear_memoization(v) }
       end
 
       # rubocop: disable CodeReuse/ActiveRecord
@@ -843,6 +858,39 @@ module Gitlab
       def projects_imported_count(from, time_period)
         distinct_count(::Project.imported_from(from).where(time_period), :creator_id) # rubocop: disable CodeReuse/ActiveRecord
       end
+
+      # rubocop:disable CodeReuse/ActiveRecord
+      def distinct_count_user_auth_by_provider(time_period)
+        counts = auth_providers_except_ldap.each_with_object({}) do |provider, hash|
+          hash[provider] = distinct_count(
+            ::AuthenticationEvent.success.for_provider(provider).where(time_period), :user_id)
+        end
+
+        if any_ldap_auth_providers?
+          counts['ldap'] = distinct_count(
+            ::AuthenticationEvent.success.ldap.where(time_period), :user_id
+          )
+        end
+
+        counts
+      end
+      # rubocop:enable CodeReuse/ActiveRecord
+
+      # rubocop:disable UsageData/LargeTable
+      def auth_providers
+        strong_memoize(:auth_providers) do
+          ::AuthenticationEvent.providers
+        end
+      end
+      # rubocop:enable UsageData/LargeTable
+
+      def auth_providers_except_ldap
+        auth_providers.reject { |provider| provider.starts_with?('ldap') }
+      end
+
+      def any_ldap_auth_providers?
+        auth_providers.any? { |provider| provider.starts_with?('ldap') }
+      end
     end
   end
 end