Add latest changes from gitlab-org/gitlab@13-2-stable-ee

1 files changed, 306 insertions, 57 deletions

diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb
index 7b6f5e69ee1..9d7e6536608 100644
--- a/lib/gitlab/usage_data.rb
+++ b/lib/gitlab/usage_data.rb
@@ -18,7 +18,6 @@ module Gitlab
     class << self
       include Gitlab::Utils::UsageData
       include Gitlab::Utils::StrongMemoize
-      include Gitlab::UsageDataConcerns::Topology
 
       def data(force_refresh: false)
         Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) do
@@ -27,16 +26,21 @@ module Gitlab
       end
 
       def uncached_data
-        clear_memoized_limits
-
-        license_usage_data
-          .merge(system_usage_data)
-          .merge(features_usage_data)
-          .merge(components_usage_data)
-          .merge(cycle_analytics_usage_data)
-          .merge(object_store_usage_data)
-          .merge(topology_usage_data)
-          .merge(recording_ce_finish_data)
+        clear_memoized
+
+        with_finished_at(:recording_ce_finished_at) do
+          license_usage_data
+            .merge(system_usage_data)
+            .merge(system_usage_data_monthly)
+            .merge(features_usage_data)
+            .merge(components_usage_data)
+            .merge(cycle_analytics_usage_data)
+            .merge(object_store_usage_data)
+            .merge(topology_usage_data)
+            .merge(usage_activity_by_stage)
+            .merge(usage_activity_by_stage(:usage_activity_by_stage_monthly, last_28_days_time_period))
+            .merge(analytics_unique_visits_data)
+        end
       end
 
       def to_json(force_refresh: false)
@@ -59,17 +63,11 @@ module Gitlab
         Time.now
       end
 
-      def recording_ce_finish_data
-        {
-          recording_ce_finished_at: Time.now
-        }
-      end
-
       # rubocop: disable Metrics/AbcSize
       # rubocop: disable CodeReuse/ActiveRecord
       def system_usage_data
-        alert_bot_incident_count = count(::Issue.authored(::User.alert_bot))
-        issues_created_manually_from_alerts = count(Issue.with_alert_management_alerts.not_authored_by(::User.alert_bot))
+        alert_bot_incident_count = count(::Issue.authored(::User.alert_bot), start: issue_minimum_id, finish: issue_maximum_id)
+        issues_created_manually_from_alerts = count(Issue.with_alert_management_alerts.not_authored_by(::User.alert_bot), start: issue_minimum_id, finish: issue_maximum_id)
 
         {
           counts: {
@@ -86,9 +84,9 @@ module Gitlab
             auto_devops_enabled: count(::ProjectAutoDevops.enabled),
             auto_devops_disabled: count(::ProjectAutoDevops.disabled),
             deploy_keys: count(DeployKey),
-            deployments: count(Deployment),
-            successful_deployments: count(Deployment.success),
-            failed_deployments: count(Deployment.failed),
+            deployments: deployment_count(Deployment),
+            successful_deployments: deployment_count(Deployment.success),
+            failed_deployments: deployment_count(Deployment.failed),
             environments: count(::Environment),
             clusters: count(::Clusters::Cluster),
             clusters_enabled: count(::Clusters::Cluster.enabled),
@@ -111,11 +109,12 @@ module Gitlab
             clusters_applications_knative: count(::Clusters::Applications::Knative.available),
             clusters_applications_elastic_stack: count(::Clusters::Applications::ElasticStack.available),
             clusters_applications_jupyter: count(::Clusters::Applications::Jupyter.available),
+            clusters_applications_cilium: count(::Clusters::Applications::Cilium.available),
             clusters_management_project: count(::Clusters::Cluster.with_management_project),
             in_review_folder: count(::Environment.in_review_folder),
             grafana_integrated_projects: count(GrafanaIntegration.enabled),
             groups: count(Group),
-            issues: count(Issue),
+            issues: count(Issue, start: issue_minimum_id, finish: issue_maximum_id),
             issues_created_from_gitlab_error_tracking_ui: count(SentryIssue),
             issues_with_associated_zoom_link: count(ZoomMeeting.added_to_issue),
             issues_using_zoom_quick_actions: distinct_count(ZoomMeeting, :issue_id),
@@ -125,7 +124,7 @@ module Gitlab
             issues_created_manually_from_alerts: issues_created_manually_from_alerts,
             incident_issues: alert_bot_incident_count,
             alert_bot_incident_issues: alert_bot_incident_count,
-            incident_labeled_issues: count(::Issue.with_label_attributes(IncidentManagement::CreateIssueService::INCIDENT_LABEL)),
+            incident_labeled_issues: count(::Issue.with_label_attributes(::IncidentManagement::CreateIncidentLabelService::LABEL_PROPERTIES), start: issue_minimum_id, finish: issue_maximum_id),
             keys: count(Key),
             label_lists: count(List.label),
             lfs_objects: count(LfsObject),
@@ -144,7 +143,6 @@ module Gitlab
             protected_branches: count(ProtectedBranch),
             releases: count(Release),
             remote_mirrors: count(RemoteMirror),
-            snippets: count(Snippet),
             personal_snippets: count(PersonalSnippet),
             project_snippets: count(ProjectSnippet),
             suggestions: count(Suggestion),
@@ -161,14 +159,29 @@ module Gitlab
             usage_counters,
             user_preferences_usage,
             ingress_modsecurity_usage,
-            container_expiration_policies_usage,
-            merge_requests_usage(default_time_period)
-          )
+            container_expiration_policies_usage
+          ).tap do |data|
+            data[:snippets] = data[:personal_snippets] + data[:project_snippets]
+          end
         }
       end
-      # rubocop: enable CodeReuse/ActiveRecord
       # rubocop: enable Metrics/AbcSize
 
+      def system_usage_data_monthly
+        {
+          counts_monthly: {
+            deployments: deployment_count(Deployment.where(last_28_days_time_period)),
+            successful_deployments: deployment_count(Deployment.success.where(last_28_days_time_period)),
+            failed_deployments: deployment_count(Deployment.failed.where(last_28_days_time_period)),
+            personal_snippets: count(PersonalSnippet.where(last_28_days_time_period)),
+            project_snippets: count(ProjectSnippet.where(last_28_days_time_period))
+          }.tap do |data|
+            data[:snippets] = data[:personal_snippets] + data[:project_snippets]
+          end
+        }
+      end
+      # rubocop: enable CodeReuse/ActiveRecord
+
       def cycle_analytics_usage_data
         Gitlab::CycleAnalytics::UsageData.new.to_json
       rescue ActiveRecord::StatementInvalid
@@ -197,6 +210,7 @@ module Gitlab
           ldap_enabled: alt_usage_data(fallback: nil) { Gitlab.config.ldap.enabled },
           mattermost_enabled: alt_usage_data(fallback: nil) { Gitlab.config.mattermost.enabled },
           omniauth_enabled: alt_usage_data(fallback: nil) { Gitlab::Auth.omniauth_enabled? },
+          prometheus_enabled: alt_usage_data(fallback: nil) { Gitlab::Prometheus::Internal.prometheus_enabled? },
           prometheus_metrics_enabled: alt_usage_data(fallback: nil) { Gitlab::Metrics.prometheus_metrics_enabled? },
           reply_by_email_enabled: alt_usage_data(fallback: nil) { Gitlab::IncomingEmail.enabled? },
           signup_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.allow_signup? },
@@ -290,6 +304,10 @@ module Gitlab
         }
       end
 
+      def topology_usage_data
+        Gitlab::UsageData::Topology.new.topology_usage_data
+      end
+
       def ingress_modsecurity_usage
         ##
         # This method measures usage of the Modsecurity Web Application Firewall across the entire
@@ -336,15 +354,9 @@ module Gitlab
 
       # rubocop: disable CodeReuse/ActiveRecord
       def services_usage
-        results = Service.available_services_names.without('jira').each_with_object({}) do |service_name, response|
+        Service.available_services_names.without('jira').each_with_object({}) do |service_name, response|
           response["projects_#{service_name}_active".to_sym] = count(Service.active.where(template: false, type: "#{service_name}_service".camelize))
-        end
-
-        # Keep old Slack keys for backward compatibility, https://gitlab.com/gitlab-data/analytics/issues/3241
-        results[:projects_slack_notifications_active] = results[:projects_slack_active]
-        results[:projects_slack_slash_active] = results[:projects_slack_slash_commands_active]
-
-        results.merge(jira_usage).merge(jira_import_usage)
+        end.merge(jira_usage).merge(jira_import_usage)
       end
 
       def jira_usage
@@ -357,18 +369,15 @@ module Gitlab
           projects_jira_active: 0
         }
 
-        Service.active
-          .by_type(:JiraService)
-          .includes(:jira_tracker_data)
-          .find_in_batches(batch_size: BATCH_SIZE) do |services|
+        JiraService.active.includes(:jira_tracker_data).find_in_batches(batch_size: BATCH_SIZE) do |services|
           counts = services.group_by do |service|
             # TODO: Simplify as part of https://gitlab.com/gitlab-org/gitlab/issues/29404
             service_url = service.data_fields&.url || (service.properties && service.properties['url'])
             service_url&.include?('.atlassian.net') ? :cloud : :server
           end
 
-          results[:projects_jira_server_active] += counts[:server].count if counts[:server]
-          results[:projects_jira_cloud_active] += counts[:cloud].count if counts[:cloud]
+          results[:projects_jira_server_active] += counts[:server].size if counts[:server]
+          results[:projects_jira_cloud_active] += counts[:cloud].size if counts[:cloud]
           results[:projects_jira_active] += services.size
         end
 
@@ -400,23 +409,18 @@ module Gitlab
       end
 
       # rubocop: disable CodeReuse/ActiveRecord
-      def merge_requests_usage(time_period)
+      def merge_requests_users(time_period)
         query =
           Event
             .where(target_type: Event::TARGET_TYPES[:merge_request].to_s)
             .where(time_period)
 
-        merge_request_users = distinct_count(
+        distinct_count(
           query,
           :author_id,
-          batch_size: 5_000, # Based on query performance, this is the optimal batch size.
-          start: User.minimum(:id),
-          finish: User.maximum(:id)
+          start: user_minimum_id,
+          finish: user_maximum_id
         )
-
-        {
-          merge_requests_users: merge_request_users
-        }
       end
       # rubocop: enable CodeReuse/ActiveRecord
 
@@ -428,19 +432,207 @@ module Gitlab
         end
       end
 
-      def default_time_period
+      def last_28_days_time_period
         { created_at: 28.days.ago..Time.current }
       end
 
+      # Source: https://gitlab.com/gitlab-data/analytics/blob/master/transform/snowflake-dbt/data/ping_metrics_to_stage_mapping_data.csv
+      def usage_activity_by_stage(key = :usage_activity_by_stage, time_period = {})
+        {
+          key => {
+            configure: usage_activity_by_stage_configure(time_period),
+            create: usage_activity_by_stage_create(time_period),
+            manage: usage_activity_by_stage_manage(time_period),
+            monitor: usage_activity_by_stage_monitor(time_period),
+            package: usage_activity_by_stage_package(time_period),
+            plan: usage_activity_by_stage_plan(time_period),
+            release: usage_activity_by_stage_release(time_period),
+            secure: usage_activity_by_stage_secure(time_period),
+            verify: usage_activity_by_stage_verify(time_period)
+          }
+        }
+      end
+
+      # rubocop: disable CodeReuse/ActiveRecord
+      def usage_activity_by_stage_configure(time_period)
+        {
+          clusters_applications_cert_managers: cluster_applications_user_distinct_count(::Clusters::Applications::CertManager, time_period),
+          clusters_applications_helm: cluster_applications_user_distinct_count(::Clusters::Applications::Helm, time_period),
+          clusters_applications_ingress: cluster_applications_user_distinct_count(::Clusters::Applications::Ingress, time_period),
+          clusters_applications_knative: cluster_applications_user_distinct_count(::Clusters::Applications::Knative, time_period),
+          clusters_management_project: clusters_user_distinct_count(::Clusters::Cluster.with_management_project, time_period),
+          clusters_disabled: clusters_user_distinct_count(::Clusters::Cluster.disabled, time_period),
+          clusters_enabled: clusters_user_distinct_count(::Clusters::Cluster.enabled, time_period),
+          clusters_platforms_gke: clusters_user_distinct_count(::Clusters::Cluster.gcp_installed.enabled, time_period),
+          clusters_platforms_eks: clusters_user_distinct_count(::Clusters::Cluster.aws_installed.enabled, time_period),
+          clusters_platforms_user: clusters_user_distinct_count(::Clusters::Cluster.user_provided.enabled, time_period),
+          instance_clusters_disabled: clusters_user_distinct_count(::Clusters::Cluster.disabled.instance_type, time_period),
+          instance_clusters_enabled: clusters_user_distinct_count(::Clusters::Cluster.enabled.instance_type, time_period),
+          group_clusters_disabled: clusters_user_distinct_count(::Clusters::Cluster.disabled.group_type, time_period),
+          group_clusters_enabled: clusters_user_distinct_count(::Clusters::Cluster.enabled.group_type, time_period),
+          project_clusters_disabled: clusters_user_distinct_count(::Clusters::Cluster.disabled.project_type, time_period),
+          project_clusters_enabled: clusters_user_distinct_count(::Clusters::Cluster.enabled.project_type, time_period)
+        }
+      end
+      # rubocop: enable CodeReuse/ActiveRecord
+
+      # rubocop: disable CodeReuse/ActiveRecord
+      def usage_activity_by_stage_create(time_period)
+        {
+          deploy_keys: distinct_count(::DeployKey.where(time_period), :user_id),
+          keys: distinct_count(::Key.regular_keys.where(time_period), :user_id),
+          merge_requests: distinct_count(::MergeRequest.where(time_period), :author_id),
+          projects_with_disable_overriding_approvers_per_merge_request: count(::Project.where(time_period.merge(disable_overriding_approvers_per_merge_request: true))),
+          projects_without_disable_overriding_approvers_per_merge_request: count(::Project.where(time_period.merge(disable_overriding_approvers_per_merge_request: [false, nil]))),
+          remote_mirrors: distinct_count(::Project.with_remote_mirrors.where(time_period), :creator_id),
+          snippets: distinct_count(::Snippet.where(time_period), :author_id)
+        }.tap do |h|
+          if time_period.present?
+            h[:merge_requests_users] = merge_requests_users(time_period)
+            h.merge!(action_monthly_active_users(time_period))
+          end
+        end
+      end
+      # rubocop: enable CodeReuse/ActiveRecord
+
+      # Omitted because no user, creator or author associated: `campaigns_imported_from_github`, `ldap_group_links`
+      # rubocop: disable CodeReuse/ActiveRecord
+      def usage_activity_by_stage_manage(time_period)
+        {
+          events: distinct_count(::Event.where(time_period), :author_id),
+          groups: distinct_count(::GroupMember.where(time_period), :user_id),
+          users_created: count(::User.where(time_period), start: user_minimum_id, finish: user_maximum_id),
+          omniauth_providers: filtered_omniauth_provider_names.reject { |name| name == 'group_saml' }
+        }
+      end
+      # rubocop: enable CodeReuse/ActiveRecord
+
+      # rubocop: disable CodeReuse/ActiveRecord
+      def usage_activity_by_stage_monitor(time_period)
+        {
+          clusters: distinct_count(::Clusters::Cluster.where(time_period), :user_id),
+          clusters_applications_prometheus: cluster_applications_user_distinct_count(::Clusters::Applications::Prometheus, time_period),
+          operations_dashboard_default_dashboard: count(::User.active.with_dashboard('operations').where(time_period),
+                                                        start: user_minimum_id,
+                                                        finish: user_maximum_id)
+        }
+      end
+      # rubocop: enable CodeReuse/ActiveRecord
+
+      def usage_activity_by_stage_package(time_period)
+        {}
+      end
+
+      # Omitted because no user, creator or author associated: `boards`, `labels`, `milestones`, `uploads`
+      # Omitted because too expensive: `epics_deepest_relationship_level`
+      # Omitted because of encrypted properties: `projects_jira_cloud_active`, `projects_jira_server_active`
+      # rubocop: disable CodeReuse/ActiveRecord
+      def usage_activity_by_stage_plan(time_period)
+        {
+          issues: distinct_count(::Issue.where(time_period), :author_id),
+          notes: distinct_count(::Note.where(time_period), :author_id),
+          projects: distinct_count(::Project.where(time_period), :creator_id),
+          todos: distinct_count(::Todo.where(time_period), :author_id)
+        }
+      end
+      # rubocop: enable CodeReuse/ActiveRecord
+
+      # Omitted because no user, creator or author associated: `environments`, `feature_flags`, `in_review_folder`, `pages_domains`
+      # rubocop: disable CodeReuse/ActiveRecord
+      def usage_activity_by_stage_release(time_period)
+        {
+          deployments: distinct_count(::Deployment.where(time_period), :user_id),
+          failed_deployments: distinct_count(::Deployment.failed.where(time_period), :user_id),
+          releases: distinct_count(::Release.where(time_period), :author_id),
+          successful_deployments: distinct_count(::Deployment.success.where(time_period), :user_id)
+        }
+      end
+      # rubocop: enable CodeReuse/ActiveRecord
+
+      # Omitted because no user, creator or author associated: `ci_runners`
+      # rubocop: disable CodeReuse/ActiveRecord
+      def usage_activity_by_stage_verify(time_period)
+        {
+          ci_builds: distinct_count(::Ci::Build.where(time_period), :user_id),
+          ci_external_pipelines: distinct_count(::Ci::Pipeline.external.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id),
+          ci_internal_pipelines: distinct_count(::Ci::Pipeline.internal.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id),
+          ci_pipeline_config_auto_devops: distinct_count(::Ci::Pipeline.auto_devops_source.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id),
+          ci_pipeline_config_repository: distinct_count(::Ci::Pipeline.repository_source.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id),
+          ci_pipeline_schedules: distinct_count(::Ci::PipelineSchedule.where(time_period), :owner_id),
+          ci_pipelines: distinct_count(::Ci::Pipeline.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id),
+          ci_triggers: distinct_count(::Ci::Trigger.where(time_period), :owner_id),
+          clusters_applications_runner: cluster_applications_user_distinct_count(::Clusters::Applications::Runner, time_period)
+        }
+      end
+      # rubocop: enable CodeReuse/ActiveRecord
+
+      # Currently too complicated and to get reliable counts for these stats:
+      # container_scanning_jobs, dast_jobs, dependency_scanning_jobs, license_management_jobs, sast_jobs, secret_detection_jobs
+      # Once https://gitlab.com/gitlab-org/gitlab/merge_requests/17568 is merged, this might be doable
+      def usage_activity_by_stage_secure(time_period)
+        {}
+      end
+
+      def analytics_unique_visits_data
+        results = ::Gitlab::Analytics::UniqueVisits::TARGET_IDS.each_with_object({}) do |target_id, hash|
+          hash[target_id] = redis_usage_data { unique_visit_service.weekly_unique_visits_for_target(target_id) }
+        end
+        results['analytics_unique_visits_for_any_target'] = redis_usage_data { unique_visit_service.weekly_unique_visits_for_any_target }
+
+        { analytics_unique_visits: results }
+      end
+
+      def action_monthly_active_users(time_period)
+        return {} unless Feature.enabled?(Gitlab::UsageDataCounters::TrackUniqueActions::FEATURE_FLAG)
+
+        counter = Gitlab::UsageDataCounters::TrackUniqueActions
+
+        project_count = redis_usage_data do
+          counter.count_unique_events(
+            event_action: Gitlab::UsageDataCounters::TrackUniqueActions::PUSH_ACTION,
+            date_from: time_period[:created_at].first,
+            date_to: time_period[:created_at].last
+          )
+        end
+
+        design_count = redis_usage_data do
+          counter.count_unique_events(
+            event_action: Gitlab::UsageDataCounters::TrackUniqueActions::DESIGN_ACTION,
+            date_from: time_period[:created_at].first,
+            date_to: time_period[:created_at].last
+          )
+        end
+
+        wiki_count = redis_usage_data do
+          counter.count_unique_events(
+            event_action: Gitlab::UsageDataCounters::TrackUniqueActions::WIKI_ACTION,
+            date_from: time_period[:created_at].first,
+            date_to: time_period[:created_at].last
+          )
+        end
+
+        {
+          action_monthly_active_users_project_repo: project_count,
+          action_monthly_active_users_design_management: design_count,
+          action_monthly_active_users_wiki_repo: wiki_count
+        }
+      end
+
       private
 
+      def unique_visit_service
+        strong_memoize(:unique_visit_service) do
+          ::Gitlab::Analytics::UniqueVisits.new
+        end
+      end
+
       def total_alert_issues
         # Remove prometheus table queries once they are deprecated
         # To be removed with https://gitlab.com/gitlab-org/gitlab/-/issues/217407.
         [
-          count(Issue.with_alert_management_alerts),
-          count(::Issue.with_self_managed_prometheus_alert_events),
-          count(::Issue.with_prometheus_alert_events)
+          count(Issue.with_alert_management_alerts, start: issue_minimum_id, finish: issue_maximum_id),
+          count(::Issue.with_self_managed_prometheus_alert_events, start: issue_minimum_id, finish: issue_maximum_id),
+          count(::Issue.with_prometheus_alert_events, start: issue_minimum_id, finish: issue_maximum_id)
         ].reduce(:+)
       end
 
@@ -456,9 +648,66 @@ module Gitlab
         end
       end
 
-      def clear_memoized_limits
+      def issue_minimum_id
+        strong_memoize(:issue_minimum_id) do
+          ::Issue.minimum(:id)
+        end
+      end
+
+      def issue_maximum_id
+        strong_memoize(:issue_maximum_id) do
+          ::Issue.maximum(:id)
+        end
+      end
+
+      def deployment_minimum_id
+        strong_memoize(:deployment_minimum_id) do
+          ::Deployment.minimum(:id)
+        end
+      end
+
+      def deployment_maximum_id
+        strong_memoize(:deployment_maximum_id) do
+          ::Deployment.maximum(:id)
+        end
+      end
+
+      def clear_memoized
+        clear_memoization(:issue_minimum_id)
+        clear_memoization(:issue_maximum_id)
         clear_memoization(:user_minimum_id)
         clear_memoization(:user_maximum_id)
+        clear_memoization(:unique_visit_service)
+        clear_memoization(:deployment_minimum_id)
+        clear_memoization(:deployment_maximum_id)
+        clear_memoization(:approval_merge_request_rule_minimum_id)
+        clear_memoization(:approval_merge_request_rule_maximum_id)
+      end
+
+      # rubocop: disable CodeReuse/ActiveRecord
+      def cluster_applications_user_distinct_count(applications, time_period)
+        distinct_count(applications.where(time_period).available.joins(:cluster), 'clusters.user_id')
+      end
+
+      def clusters_user_distinct_count(clusters, time_period)
+        distinct_count(clusters.where(time_period), :user_id)
+      end
+      # rubocop: enable CodeReuse/ActiveRecord
+
+      def omniauth_provider_names
+        ::Gitlab.config.omniauth.providers.map(&:name)
+      end
+
+      # LDAP provider names are set by customers and could include
+      # sensitive info (server names, etc). LDAP providers normally
+      # don't appear in omniauth providers but filter to ensure
+      # no internal details leak via usage ping.
+      def filtered_omniauth_provider_names
+        omniauth_provider_names.reject { |name| name.starts_with?('ldap') }
+      end
+
+      def deployment_count(relation)
+        count relation, start: deployment_minimum_id, finish: deployment_maximum_id
       end
     end
   end