diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-01-13 03:10:20 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-01-13 03:10:20 +0000 |
| commit | 0e06a8a602707451ea6b60bcaa7ca6bb4953db4d (patch) | |
| tree | dc98e90c48915a90e8895f97b03e31331bd88e72 | |
| parent | 033182c9f14e10ed6a45632054336a92f5d88af9 (diff) | |
| download | gitlab-ce-0e06a8a602707451ea6b60bcaa7ca6bb4953db4d.tar.gz | |
Add latest changes from gitlab-org/gitlab@master
61 files changed, 2102 insertions, 1835 deletions
diff --git a/GITLAB_KAS_VERSION b/GITLAB_KAS_VERSION index d6ef99820ff..44726955203 100644 --- a/GITLAB_KAS_VERSION +++ b/GITLAB_KAS_VERSION @@ -1 +1 @@ -13.7.0 +13.8.0 diff --git a/app/views/admin/application_settings/_usage.html.haml b/app/views/admin/application_settings/_usage.html.haml index 2ba7dcefd44..fe83d4b807c 100644 --- a/app/views/admin/application_settings/_usage.html.haml +++ b/app/views/admin/application_settings/_usage.html.haml @@ -33,7 +33,7 @@ %pre.usage-data.js-syntax-highlight.code.highlight.mt-2.d-none{ class: payload_class, data: { endpoint: usage_data_admin_application_settings_path(format: :html) } } - else = _('The usage ping is disabled, and cannot be configured through this form.') - - deactivating_usage_ping_path = help_page_path('development/product_analytics/usage_ping', anchor: 'disable-usage-ping') + - deactivating_usage_ping_path = help_page_path('development/usage_ping', anchor: 'disable-usage-ping') - deactivating_usage_ping_link_start = '<a href="%{url}" target="_blank" rel="noopener noreferrer">'.html_safe % { url: deactivating_usage_ping_path } = s_('For more information, see the documentation on %{deactivating_usage_ping_link_start}deactivating the usage ping%{deactivating_usage_ping_link_end}.').html_safe % { deactivating_usage_ping_link_start: deactivating_usage_ping_link_start, deactivating_usage_ping_link_end: '</a>'.html_safe } diff --git a/app/views/admin/dev_ops_report/_report.html.haml b/app/views/admin/dev_ops_report/_report.html.haml index 24c805d273a..5faadd15ef8 100644 --- a/app/views/admin/dev_ops_report/_report.html.haml +++ b/app/views/admin/dev_ops_report/_report.html.haml @@ -4,7 +4,7 @@ = render 'callout' - if !usage_ping_enabled - #js-devops-empty-state{ data: { is_admin: current_user&.admin.to_s, empty_state_svg_path: image_path('illustrations/convdev/convdev_no_index.svg'), enable_usage_ping_link: metrics_and_profiling_admin_application_settings_path(anchor: 'js-usage-settings'), docs_link: help_page_path('development/product_analytics/usage_ping') } } + #js-devops-empty-state{ data: { is_admin: current_user&.admin.to_s, empty_state_svg_path: image_path('illustrations/convdev/convdev_no_index.svg'), enable_usage_ping_link: metrics_and_profiling_admin_application_settings_path(anchor: 'js-usage-settings'), docs_link: help_page_path('development/usage_ping') } } - elsif @metric.blank? = render 'no_data' - else diff --git a/changelogs/unreleased/23531-change-needs-failure-message.yml b/changelogs/unreleased/23531-change-needs-failure-message.yml new file mode 100644 index 00000000000..a9042a7b0ee --- /dev/null +++ b/changelogs/unreleased/23531-change-needs-failure-message.yml @@ -0,0 +1,5 @@ +--- +title: Change failure message of missing job needs +merge_request: 50492 +author: +type: other diff --git a/config/feature_flags/development/usage_data_i_testing_group_code_coverage_visit_total.yml b/config/feature_flags/development/usage_data_i_testing_group_code_coverage_visit_total.yml new file mode 100644 index 00000000000..720b94fcf6e --- /dev/null +++ b/config/feature_flags/development/usage_data_i_testing_group_code_coverage_visit_total.yml @@ -0,0 +1,8 @@ +--- +name: usage_data_i_testing_group_code_coverage_visit_total +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/51382 +rollout_issue_url: +milestone: '13.8' +type: development +group: group::testing +default_enabled: true diff --git a/doc/administration/object_storage.md b/doc/administration/object_storage.md index a89c50a8412..999d9b87363 100644 --- a/doc/administration/object_storage.md +++ b/doc/administration/object_storage.md @@ -61,10 +61,10 @@ must be enabled, only the following providers can be used: - [Google Cloud Storage](#google-cloud-storage-gcs) - [Azure Blob storage](#azure-blob-storage) -Background upload isn't supported with the consolidated object storage -configuration. We recommend enabling direct upload mode because it doesn't -require a shared folder, and [this setting may become the -default](https://gitlab.com/gitlab-org/gitlab/-/issues/27331). +When consolidated object storage is used, direct upload is enabled +automatically. Background upload is not supported. For storage-specific +configuration, [direct upload may become the default](https://gitlab.com/gitlab-org/gitlab/-/issues/27331) +because it does not require a shared folder. Consolidated object storage configuration can't be used for backups or Mattermost. See the [full table for a complete list](#storage-specific-configuration). @@ -232,7 +232,7 @@ The connection settings match those provided by [fog-aws](https://github.com/fog | `aws_secret_access_key` | AWS credentials, or compatible | | | `aws_signature_version` | AWS signature version to use. `2` or `4` are valid options. Digital Ocean Spaces and other providers may need `2`. | `4` | | `enable_signature_v4_streaming` | Set to `true` to enable HTTP chunked transfers with [AWS v4 signatures](https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html). Oracle Cloud S3 needs this to be `false`. | `true` | -| `region` | AWS region | us-east-1 | +| `region` | AWS region. | | | `host` | S3 compatible host for when not using AWS, e.g. `localhost` or `storage.example.com`. HTTPS and port 443 is assumed. | `s3.amazonaws.com` | | `endpoint` | Can be used when configuring an S3 compatible service such as [MinIO](https://min.io), by entering a URL such as `http://127.0.0.1:9000`. This takes precedence over `host`. | (optional) | | `path_style` | Set to `true` to use `host/bucket_name/object` style paths instead of `bucket_name.host/object`. Leave as `false` for AWS S3. | `false` | diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index 7381eba53ef..8c881460606 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -3483,6 +3483,10 @@ If there is more than one matched line in the job output, the last line is used. For the matched line, the first occurence of `\d+(\.\d+)?` is the code coverage. Leading zeros are removed. +Coverage output from [child pipelines](../parent_child_pipelines.md) is not recorded +or displayed. Check [the related issue](https://gitlab.com/gitlab-org/gitlab/-/issues/280818) +for more details. + ### `retry` > - [Introduced](https://gitlab.com/gitlab-org/gitlab-foss/-/issues/3442) in GitLab 9.5. diff --git a/doc/development/README.md b/doc/development/README.md index 4b54f11db0a..0d3c1b3cbe9 100644 --- a/doc/development/README.md +++ b/doc/development/README.md @@ -256,11 +256,11 @@ See [database guidelines](database/index.md). - [Externalization](i18n/externalization.md) - [Translation](i18n/translation.md) -## Product Analytics guides +## Product Intelligence guides -- [Product Analytics guide](https://about.gitlab.com/handbook/product/product-analytics-guide/) -- [Usage Ping guide](product_analytics/usage_ping.md) -- [Snowplow guide](product_analytics/snowplow.md) +- [Product Intelligence guide](https://about.gitlab.com/handbook/product/product-intelligence-guide/) +- [Usage Ping guide](usage_ping.md) +- [Snowplow guide](snowplow.md) ## Experiment guide diff --git a/doc/development/changelog.md b/doc/development/changelog.md index 894ae5a1893..8fad32ed163 100644 --- a/doc/development/changelog.md +++ b/doc/development/changelog.md @@ -47,7 +47,7 @@ the `author` field. GitLab team members **should not**. - Any user-facing change **must** have a changelog entry. This includes both visual changes (regardless of how minor), and changes to the rendered DOM which impact how a screen reader may announce the content. - Any client-facing change to our REST and GraphQL APIs **must** have a changelog entry. - Performance improvements **should** have a changelog entry. -- Changes that need to be documented in the Product Analytics [Event Dictionary](https://about.gitlab.com/handbook/product/product-analytics-guide/#event-dictionary) +- Changes that need to be documented in the Product Intelligence [Event Dictionary](https://about.gitlab.com/handbook/product/product-intelligence-guide/#event-dictionary) also require a changelog entry. - _Any_ contribution from a community member, no matter how small, **may** have a changelog entry regardless of these guidelines if the contributor wants one. @@ -55,7 +55,7 @@ the `author` field. GitLab team members **should not**. - Any docs-only changes **should not** have a changelog entry. - Any change behind a disabled feature flag **should not** have a changelog entry. - Any change behind an enabled feature flag **should** have a changelog entry. -- Any change that adds new usage data metrics and changes that needs to be documented in Product Analytics [Event Dictionary](https://about.gitlab.com/handbook/product/product-analytics-guide/#event-dictionary) **should** have a changelog entry. +- Any change that adds new usage data metrics and changes that needs to be documented in Product Intelligence [Event Dictionary](https://about.gitlab.com/handbook/product/product-intelligence-guide/#event-dictionary) **should** have a changelog entry. - A change that adds snowplow events **should** have a changelog entry - - A change that [removes a feature flag](feature_flags/development.md) **should** have a changelog entry - only if the feature flag did not default to true already. diff --git a/doc/development/code_review.md b/doc/development/code_review.md index b270c7c9807..4cee364b209 100644 --- a/doc/development/code_review.md +++ b/doc/development/code_review.md @@ -119,7 +119,7 @@ with [domain expertise](#domain-experts). by a [Software Engineer in Test](https://about.gitlab.com/handbook/engineering/quality/#individual-contributors)**. 1. If your merge request only includes end-to-end changes (*3*) **or** if the MR author is a [Software Engineer in Test](https://about.gitlab.com/handbook/engineering/quality/#individual-contributors), it must be **approved by a [Quality maintainer](https://about.gitlab.com/handbook/engineering/projects/#gitlab_maintainers_qa)** 1. If your merge request includes a new or updated [application limit](https://about.gitlab.com/handbook/product/product-processes/#introducing-application-limits), it must be **approved by a [product manager](https://about.gitlab.com/company/team/)**. -1. If your merge request includes Product Analytics (telemetry) changes, it should be reviewed and approved by a [Product analytics engineer](https://gitlab.com/gitlab-org/growth/product-analytics/engineers). +1. If your merge request includes Product Intelligence (telemetry or analytics) changes, it should be reviewed and approved by a [Product Intelligence engineer](https://gitlab.com/gitlab-org/growth/product_intelligence/engineers). - (*1*): Please note that specs other than JavaScript specs are considered backend code. - (*2*): We encourage you to seek guidance from a database maintainer if your merge diff --git a/doc/development/database_review.md b/doc/development/database_review.md index 68a0fe1670a..da2c93cc1fd 100644 --- a/doc/development/database_review.md +++ b/doc/development/database_review.md @@ -27,7 +27,7 @@ A database review is required for: database review. - Changes in usage data metrics that use `count`, `distinct_count` and `estimate_batch_distinct_count`. These metrics could have complex queries over large tables. - See the [Product Analytics Guide](https://about.gitlab.com/handbook/product/product-analytics-guide/) + See the [Product Intelligence Guide](https://about.gitlab.com/handbook/product/product-intelligence-guide/) for implementation details. A database reviewer is expected to look out for obviously complex diff --git a/doc/development/documentation/index.md b/doc/development/documentation/index.md index 1d412e32e69..9ca37d1b383 100644 --- a/doc/development/documentation/index.md +++ b/doc/development/documentation/index.md @@ -173,7 +173,7 @@ There are two types of redirects: - Redirect files added into the docs themselves, for users who view the docs in `/help` on self-managed instances. For example, [`/help` on GitLab.com](https://gitlab.com/help). - Redirects in a [`_redirects`](../../user/project/pages/redirects.md) file, for users - who view the docs on <http://docs.gitlab.com>. + who view the docs on <https://docs.gitlab.com>. To add a redirect: diff --git a/doc/development/documentation/styleguide/index.md b/doc/development/documentation/styleguide/index.md index a1c740da065..baa85a78db6 100644 --- a/doc/development/documentation/styleguide/index.md +++ b/doc/development/documentation/styleguide/index.md @@ -332,7 +332,7 @@ GitLab documentation should be clear and easy to understand. ### Trademark Only use the GitLab name and trademarks in accordance with -[GitLab Brand Guidelines](https://about.gitlab.com/handbook/marketing/inbound-marketing/digital-experience/brand-guidelines/#trademark). +[GitLab Brand Guidelines](https://about.gitlab.com/handbook/marketing/corporate-marketing/brand-activation/brand-guidelines/#trademark). Don't use the possessive form of the word GitLab (`GitLab's`). diff --git a/doc/development/event_tracking/backend.md b/doc/development/event_tracking/backend.md index 24e83ffc524..e8b8e0c4885 100644 --- a/doc/development/event_tracking/backend.md +++ b/doc/development/event_tracking/backend.md @@ -1,8 +1,8 @@ --- -redirect_to: '../product_analytics/index.md' +redirect_to: 'https://about.gitlab.com/handbook/product/product-intelligence-guide/' --- -This document was moved to [another location](../product_analytics/index.md). +This document was moved to [another location](https://about.gitlab.com/handbook/product/product-intelligence-guide/). -<!-- This redirect file can be deleted after February 1, 2021. --> +<!-- This redirect file can be deleted after December 1, 2021. --> <!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/#move-or-rename-a-page --> diff --git a/doc/development/event_tracking/frontend.md b/doc/development/event_tracking/frontend.md index 24e83ffc524..e8b8e0c4885 100644 --- a/doc/development/event_tracking/frontend.md +++ b/doc/development/event_tracking/frontend.md @@ -1,8 +1,8 @@ --- -redirect_to: '../product_analytics/index.md' +redirect_to: 'https://about.gitlab.com/handbook/product/product-intelligence-guide/' --- -This document was moved to [another location](../product_analytics/index.md). +This document was moved to [another location](https://about.gitlab.com/handbook/product/product-intelligence-guide/). -<!-- This redirect file can be deleted after February 1, 2021. --> +<!-- This redirect file can be deleted after December 1, 2021. --> <!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/#move-or-rename-a-page --> diff --git a/doc/development/event_tracking/index.md b/doc/development/event_tracking/index.md index 24e83ffc524..e8b8e0c4885 100644 --- a/doc/development/event_tracking/index.md +++ b/doc/development/event_tracking/index.md @@ -1,8 +1,8 @@ --- -redirect_to: '../product_analytics/index.md' +redirect_to: 'https://about.gitlab.com/handbook/product/product-intelligence-guide/' --- -This document was moved to [another location](../product_analytics/index.md). +This document was moved to [another location](https://about.gitlab.com/handbook/product/product-intelligence-guide/). -<!-- This redirect file can be deleted after February 1, 2021. --> +<!-- This redirect file can be deleted after December 1, 2021. --> <!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/#move-or-rename-a-page --> diff --git a/doc/development/experiment_guide/index.md b/doc/development/experiment_guide/index.md index 8e9e7b25042..a1899ab5f18 100644 --- a/doc/development/experiment_guide/index.md +++ b/doc/development/experiment_guide/index.md @@ -145,7 +145,7 @@ addressed. To determine whether the experiment is a success or not, we must implement tracking events to acquire data for analyzing. We can send events to Snowplow via either the backend or frontend. -Read the [product analytics guide](https://about.gitlab.com/handbook/product/product-analytics-guide/) for more details. +Read the [product intelligence guide](https://about.gitlab.com/handbook/product/product-intelligence-guide/) for more details. #### Track backend events diff --git a/doc/development/fe_guide/event_tracking.md b/doc/development/fe_guide/event_tracking.md index 24e83ffc524..e8b8e0c4885 100644 --- a/doc/development/fe_guide/event_tracking.md +++ b/doc/development/fe_guide/event_tracking.md @@ -1,8 +1,8 @@ --- -redirect_to: '../product_analytics/index.md' +redirect_to: 'https://about.gitlab.com/handbook/product/product-intelligence-guide/' --- -This document was moved to [another location](../product_analytics/index.md). +This document was moved to [another location](https://about.gitlab.com/handbook/product/product-intelligence-guide/). -<!-- This redirect file can be deleted after February 1, 2021. --> +<!-- This redirect file can be deleted after December 1, 2021. --> <!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/#move-or-rename-a-page --> diff --git a/doc/development/gotchas.md b/doc/development/gotchas.md index 2b34aedddf6..a506b67d89d 100644 --- a/doc/development/gotchas.md +++ b/doc/development/gotchas.md @@ -270,7 +270,7 @@ This problem disappears as soon as we upgrade to Rails 6 and use the Zeitwerk au ### Further reading - Rails Guides: [Autoloading and Reloading Constants (Classic Mode)](https://guides.rubyonrails.org/autoloading_and_reloading_constants_classic_mode.html) -- Ruby Constant lookup: [Everything you ever wanted to know about constant lookup in Ruby](http://cirw.in/blog/constant-lookup) +- Ruby Constant lookup: [Everything you ever wanted to know about constant lookup in Ruby](https://cirw.in/blog/constant-lookup) - Rails 6 and Zeitwerk autoloader: [Understanding Zeitwerk in Rails 6](https://medium.com/cedarcode/understanding-zeitwerk-in-rails-6-f168a9f09a1f) ## Storing assets that do not require pre-compiling diff --git a/doc/development/instrumentation.md b/doc/development/instrumentation.md index 8fb7f29c86c..94b56e10d9e 100644 --- a/doc/development/instrumentation.md +++ b/doc/development/instrumentation.md @@ -11,7 +11,7 @@ blocks of Ruby code. Method instrumentation is the primary form of instrumentation with block-based instrumentation only being used when we want to drill down to specific regions of code within a method. -Please refer to [Product Analytics](https://about.gitlab.com/handbook/product/product-analytics-guide/) if you are tracking product usage patterns. +Please refer to [Product Intelligence](https://about.gitlab.com/handbook/product/product-intelligence-guide/) if you are tracking product usage patterns. ## Instrumenting Methods diff --git a/doc/development/new_fe_guide/development/accessibility.md b/doc/development/new_fe_guide/development/accessibility.md index 81f3773dd5c..65485104efe 100644 --- a/doc/development/new_fe_guide/development/accessibility.md +++ b/doc/development/new_fe_guide/development/accessibility.md @@ -42,7 +42,7 @@ In forms we should use the `for` attribute in the label statement: ## Testing -1. On MacOS you can use [VoiceOver](http://www.apple.com/accessibility/vision/) by pressing `cmd+F5`. +1. On MacOS you can use [VoiceOver](https://www.apple.com/accessibility/vision/) by pressing `cmd+F5`. 1. On Windows you can use [Narrator](https://www.microsoft.com/en-us/accessibility/windows) by pressing Windows logo key + Control + Enter. ## Online resources diff --git a/doc/development/product_analytics/event_dictionary.md b/doc/development/product_analytics/event_dictionary.md index 9c363f08cb4..e8b8e0c4885 100644 --- a/doc/development/product_analytics/event_dictionary.md +++ b/doc/development/product_analytics/event_dictionary.md @@ -1,8 +1,8 @@ --- -redirect_to: 'https://about.gitlab.com/handbook/product/product-analytics-guide/' +redirect_to: 'https://about.gitlab.com/handbook/product/product-intelligence-guide/' --- -This document was moved to [another location](https://about.gitlab.com/handbook/product/product-analytics-guide/). +This document was moved to [another location](https://about.gitlab.com/handbook/product/product-intelligence-guide/). -<!-- This redirect file can be deleted after February 1, 2021. --> +<!-- This redirect file can be deleted after December 1, 2021. --> <!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/#move-or-rename-a-page --> diff --git a/doc/development/product_analytics/index.md b/doc/development/product_analytics/index.md index a8f3bb885cd..4d2168cf304 100644 --- a/doc/development/product_analytics/index.md +++ b/doc/development/product_analytics/index.md @@ -1,8 +1,8 @@ --- -redirect_to: 'https://about.gitlab.com/handbook/product/product-analytics-guide/' +redirect_to: 'https://about.gitlab.com/handbook/product/product-intelligence-guide/' --- -This document was moved to [another location](https://about.gitlab.com/handbook/product/product-analytics-guide/). +This document was moved to [another location](https://about.gitlab.com/handbook/product/product-intelligence-guide/). <!-- Needed by the Product Intelligence group diff --git a/doc/development/product_analytics/snowplow.md b/doc/development/product_analytics/snowplow.md index 22da5a53c2f..bb056ffddfe 100644 --- a/doc/development/product_analytics/snowplow.md +++ b/doc/development/product_analytics/snowplow.md @@ -1,623 +1,8 @@ --- -stage: Growth -group: Product Analytics -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: '../snowplow.md' --- -# Snowplow Guide +This document was moved to [another location](../snowplow.md). -This guide provides an overview of how Snowplow works, and implementation details. - -For more information about Product Analytics, see: - -- [Product Analytics Guide](https://about.gitlab.com/handbook/product/product-analytics-guide/) -- [Usage Ping Guide](usage_ping.md) - -More useful links: - -- [Product Analytics Direction](https://about.gitlab.com/direction/product-analytics/) -- [Data Analysis Process](https://about.gitlab.com/handbook/business-ops/data-team/#data-analysis-process/) -- [Data for Product Managers](https://about.gitlab.com/handbook/business-ops/data-team/programs/data-for-product-managers/) -- [Data Infrastructure](https://about.gitlab.com/handbook/business-ops/data-team/platform/infrastructure/) - -## What is Snowplow - -Snowplow is an enterprise-grade marketing and product analytics platform which helps track the way users engage with our website and application. - -[Snowplow](https://github.com/snowplow/snowplow) consists of the following loosely-coupled sub-systems: - -- **Trackers** fire Snowplow events. Snowplow has 12 trackers, covering web, mobile, desktop, server, and IoT. -- **Collectors** receive Snowplow events from trackers. We have three different event collectors, synchronizing events either to Amazon S3, Apache Kafka, or Amazon Kinesis. -- **Enrich** cleans up the raw Snowplow events, enriches them and puts them into storage. We have an Hadoop-based enrichment process, and a Kinesis-based or Kafka-based process. -- **Storage** is where the Snowplow events live. We store the Snowplow events in a flat file structure on S3, and in the Redshift and PostgreSQL databases. -- **Data modeling** is where event-level data is joined with other data sets and aggregated into smaller data sets, and business logic is applied. This produces a clean set of tables which make it easier to perform analysis on the data. We have data models for Redshift and Looker. -- **Analytics** are performed on the Snowplow events or on the aggregate tables. - - - -## Snowplow schema - -We have many definitions of Snowplow's schema. We have an active issue to [standardize this schema](https://gitlab.com/gitlab-org/gitlab/-/issues/207930) including the following definitions: - -- Frontend and backend taxonomy as listed below -- [Structured event taxonomy](#structured-event-taxonomy) -- [Self describing events](https://github.com/snowplow/snowplow/wiki/Custom-events#self-describing-events) -- [Iglu schema](https://gitlab.com/gitlab-org/iglu/) -- [Snowplow authored events](https://github.com/snowplow/snowplow/wiki/Snowplow-authored-events) - -## Enabling Snowplow - -Tracking can be enabled at: - -- The instance level, which enables tracking on both the frontend and backend layers. -- User level, though user tracking can be disabled on a per-user basis. GitLab tracking respects the [Do Not Track](https://www.eff.org/issues/do-not-track) standard, so any user who has enabled the Do Not Track option in their browser is not tracked at a user level. - -We use Snowplow for the majority of our tracking strategy and it is enabled on GitLab.com. On a self-managed instance, Snowplow can be enabled by navigating to: - -- **Admin Area > Settings > General** in the UI. -- `admin/application_settings/integrations` in your browser. - -The following configuration is required: - -| Name | Value | -|---------------|---------------------------| -| Collector | `snowplow.trx.gitlab.net` | -| Site ID | `gitlab` | -| Cookie domain | `.gitlab.com` | - -## Snowplow request flow - -The following example shows a basic request/response flow between the following components: - -- Snowplow JS / Ruby Trackers on GitLab.com -- [GitLab.com Snowplow Collector](https://gitlab.com/gitlab-com/gl-infra/readiness/-/blob/master/library/snowplow/index.md) -- The GitLab S3 Bucket -- The GitLab Snowflake Data Warehouse -- Sisense: - -```mermaid -sequenceDiagram - participant Snowplow JS (Frontend) - participant Snowplow Ruby (Backend) - participant GitLab.com Snowplow Collector - participant S3 Bucket - participant Snowflake DW - participant Sisense Dashboards - Snowplow JS (Frontend) ->> GitLab.com Snowplow Collector: FE Tracking event - Snowplow Ruby (Backend) ->> GitLab.com Snowplow Collector: BE Tracking event - loop Process using Kinesis Stream - GitLab.com Snowplow Collector ->> GitLab.com Snowplow Collector: Log raw events - GitLab.com Snowplow Collector ->> GitLab.com Snowplow Collector: Enrich events - GitLab.com Snowplow Collector ->> GitLab.com Snowplow Collector: Write to disk - end - GitLab.com Snowplow Collector ->> S3 Bucket: Kinesis Firehose - S3 Bucket->>Snowflake DW: Import data - Snowflake DW->>Snowflake DW: Transform data using dbt - Snowflake DW->>Sisense Dashboards: Data available for querying -``` - -## Structured event taxonomy - -When adding new click events, we should add them in a way that's internally consistent. If we don't, it is very painful to perform analysis across features since each feature captures events differently. - -The current method provides several attributes that are sent on each click event. Please try to follow these guidelines when specifying events to capture: - -| attribute | type | required | description | -| --------- | ------- | -------- | ----------- | -| category | text | true | The page or backend area of the application. Unless infeasible, please use the Rails page attribute by default in the frontend, and namespace + classname on the backend. | -| action | text | true | The action the user is taking, or aspect that's being instrumented. The first word should always describe the action or aspect: clicks should be `click`, activations should be `activate`, creations should be `create`, etc. Use underscores to describe what was acted on; for example, activating a form field would be `activate_form_input`. An interface action like clicking on a dropdown would be `click_dropdown`, while a behavior like creating a project record from the backend would be `create_project` | -| label | text | false | The specific element, or object that's being acted on. This is either the label of the element (e.g. a tab labeled 'Create from template' may be `create_from_template`) or a unique identifier if no text is available (e.g. closing the Groups dropdown in the top navbar might be `groups_dropdown_close`), or it could be the name or title attribute of a record being created. | -| property | text | false | Any additional property of the element, or object being acted on. | -| value | decimal | false | Describes a numeric value or something directly related to the event. This could be the value of an input (e.g. `10` when clicking `internal` visibility). | - -### Web-specific parameters - -Snowplow JS adds many [web-specific parameters](https://docs.snowplowanalytics.com/docs/collecting-data/collecting-from-own-applications/snowplow-tracker-protocol/#Web-specific_parameters) to all web events by default. - -## Implementing Snowplow JS (Frontend) tracking - -GitLab provides `Tracking`, an interface that wraps the [Snowplow JavaScript Tracker](https://github.com/snowplow/snowplow/wiki/javascript-tracker) for tracking custom events. There are a few ways to use tracking, but each generally requires at minimum, a `category` and an `action`. Additional data can be provided that adheres to our [Structured event taxonomy](#structured-event-taxonomy). - -| field | type | default value | description | -|:-----------|:-------|:---------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `category` | string | document.body.dataset.page | Page or subsection of a page that events are being captured within. | -| `action` | string | 'generic' | Action the user is taking. Clicks should be `click` and activations should be `activate`, so for example, focusing a form field would be `activate_form_input`, and clicking a button would be `click_button`. | -| `data` | object | {} | Additional data such as `label`, `property`, `value`, and `context` as described in our [Structured event taxonomy](#structured-event-taxonomy). | - -### Tracking in HAML (or Vue Templates) - -When working within HAML (or Vue templates) we can add `data-track-*` attributes to elements of interest. All elements that have a `data-track-event` attribute automatically have event tracking bound on clicks. - -Below is an example of `data-track-*` attributes assigned to a button: - -```haml -%button.btn{ data: { track: { event: "click_button", label: "template_preview", property: "my-template" } } } -``` - -```html -<button class="btn" - data-track-event="click_button" - data-track-label="template_preview" - data-track-property="my-template" -/> -``` - -Event listeners are bound at the document level to handle click events on or within elements with these data attributes. This allows them to be properly handled on re-rendering and changes to the DOM. Note that because of the way these events are bound, click events should not be stopped from propagating up the DOM tree. If for any reason click events are being stopped from propagating, you need to implement your own listeners and follow the instructions in [Tracking in raw JavaScript](#tracking-in-raw-javascript). - -Below is a list of supported `data-track-*` attributes: - -| attribute | required | description | -|:----------------------|:---------|:------------| -| `data-track-event` | true | Action the user is taking. Clicks must be prepended with `click` and activations must be prepended with `activate`. For example, focusing a form field would be `activate_form_input` and clicking a button would be `click_button`. | -| `data-track-label` | false | The `label` as described in our [Structured event taxonomy](#structured-event-taxonomy). | -| `data-track-property` | false | The `property` as described in our [Structured event taxonomy](#structured-event-taxonomy). | -| `data-track-value` | false | The `value` as described in our [Structured event taxonomy](#structured-event-taxonomy). If omitted, this is the element's `value` property or an empty string. For checkboxes, the default value is the element's checked attribute or `false` when unchecked. | -| `data-track-context` | false | The `context` as described in our [Structured event taxonomy](#structured-event-taxonomy). | - -#### Caveats - -When using the GitLab helper method [`nav_link`](https://gitlab.com/gitlab-org/gitlab/-/blob/898b286de322e5df6a38d257b10c94974d580df8/app/helpers/tab_helper.rb#L69) be sure to wrap `html_options` under the `html_options` keyword argument. -Be careful, as this behavior can be confused with the `ActionView` helper method [`link_to`](https://api.rubyonrails.org/v5.2.3/classes/ActionView/Helpers/UrlHelper.html#method-i-link_to) that does not require additional wrapping of `html_options` - -`nav_link(controller: ['dashboard/groups', 'explore/groups'], html_options: { data: { track_label: "groups_dropdown", track_event: "click_dropdown" } })` - -vs - -`link_to assigned_issues_dashboard_path, title: _('Issues'), data: { track_label: 'main_navigation', track_event: 'click_issues_link' }` - -### Tracking within Vue components - -There's a tracking Vue mixin that can be used in components if more complex tracking is required. To use it, first import the `Tracking` library and request a mixin. - -```javascript -import Tracking from '~/tracking'; -const trackingMixin = Tracking.mixin({ label: 'right_sidebar' }); -``` - -You can provide default options that are passed along whenever an event is tracked from within your component. For instance, if all events within a component should be tracked with a given `label`, you can provide one at this time. Available defaults are `category`, `label`, `property`, and `value`. If no category is specified, `document.body.dataset.page` is used as the default. - -You can then use the mixin normally in your component with the `mixin` Vue declaration. The mixin also provides the ability to specify tracking options in `data` or `computed`. These override any defaults and allow the values to be dynamic from props, or based on state. - -```javascript -export default { - mixins: [trackingMixin], - // ...[component implementation]... - data() { - return { - expanded: false, - tracking: { - label: 'left_sidebar' - } - }; - }, -} -``` - -The mixin provides a `track` method that can be called within the template, or from component methods. An example of the whole implementation might look like the following. - -```javascript -export default { - mixins: [Tracking.mixin({ label: 'right_sidebar' })], - data() { - return { - expanded: false, - }; - }, - methods: { - toggle() { - this.expanded = !this.expanded; - this.track('click_toggle', { value: this.expanded }) - } - } -}; -``` - -And if needed within the template, you can use the `track` method directly as well. - -```html -<template> - <div> - <a class="toggle" @click.prevent="toggle">Toggle</a> - <div v-if="expanded"> - <p>Hello world!</p> - <a @click.prevent="track('click_action')">Track an event</a> - </div> - </div> -</template> -``` - -### Tracking in raw JavaScript - -Custom event tracking and instrumentation can be added by directly calling the `Tracking.event` static function. The following example demonstrates tracking a click on a button by calling `Tracking.event` manually. - -```javascript -import Tracking from '~/tracking'; - -const button = document.getElementById('create_from_template_button'); -button.addEventListener('click', () => { - Tracking.event('dashboard:projects:index', 'click_button', { - label: 'create_from_template', - property: 'template_preview', - value: 'rails', - }); -}) -``` - -### Tests and test helpers - -In Jest particularly in Vue tests, you can use the following: - -```javascript -import { mockTracking } from 'helpers/tracking_helper'; - -describe('MyTracking', () => { - let spy; - - beforeEach(() => { - spy = mockTracking('_category_', wrapper.element, jest.spyOn); - }); - - it('tracks an event when clicked on feedback', () => { - wrapper.find('.discover-feedback-icon').trigger('click'); - - expect(spy).toHaveBeenCalledWith('_category_', 'click_button', { - label: 'security-discover-feedback-cta', - property: '0', - }); - }); -}); -``` - -In obsolete Karma tests it's used as below: - -```javascript -import { mockTracking, triggerEvent } from 'spec/helpers/tracking_helper'; - -describe('my component', () => { - let trackingSpy; - - beforeEach(() => { - trackingSpy = mockTracking('_category_', vm.$el, spyOn); - }); - - const triggerEvent = () => { - // action which should trigger a event - }; - - it('tracks an event when toggled', () => { - expect(trackingSpy).not.toHaveBeenCalled(); - - triggerEvent('a.toggle'); - - expect(trackingSpy).toHaveBeenCalledWith('_category_', 'click_edit_button', { - label: 'right_sidebar', - property: 'confidentiality', - }); - }); -}); -``` - -## Implementing Snowplow Ruby (Backend) tracking - -GitLab provides `Gitlab::Tracking`, an interface that wraps the [Snowplow Ruby Tracker](https://github.com/snowplow/snowplow/wiki/ruby-tracker) for tracking custom events. - -Custom event tracking and instrumentation can be added by directly calling the `GitLab::Tracking.event` class method, which accepts the following arguments: - -| argument | type | default value | description | -|:-----------|:-------|:--------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `category` | string | 'application' | Area or aspect of the application. This could be `HealthCheckController` or `Lfs::FileTransformer` for instance. | -| `action` | string | 'generic' | The action being taken, which can be anything from a controller action like `create` to something like an Active Record callback. | -| `data` | object | {} | Additional data such as `label`, `property`, `value`, and `context` as described in [Structured event taxonomy](#structured-event-taxonomy). These are set as empty strings if you don't provide them. | - -Tracking can be viewed as either tracking user behavior, or can be used for instrumentation to monitor and visualize performance over time in an area or aspect of code. - -For example: - -```ruby -class Projects::CreateService < BaseService - def execute - project = Project.create(params) - - Gitlab::Tracking.event('Projects::CreateService', 'create_project', - label: project.errors.full_messages.to_sentence, - value: project.valid? - ) - end -end -``` - -### Unit testing - -Use the `expect_snowplow_event` helper when testing backend Snowplow events. See [testing best practices]( -https://docs.gitlab.com/ee/development/testing_guide/best_practices.html#test-snowplow-events) for details. - -### Performance - -We use the [AsyncEmitter](https://github.com/snowplow/snowplow/wiki/Ruby-Tracker#52-the-asyncemitter-class) when tracking events, which allows for instrumentation calls to be run in a background thread. This is still an active area of development. - -## Developing and testing Snowplow - -There are several tools for developing and testing Snowplow Event - -| Testing Tool | Frontend Tracking | Backend Tracking | Local Development Environment | Production Environment | Production Environment | -|----------------------------------------------|--------------------|---------------------|-------------------------------|------------------------|------------------------| -| Snowplow Analytics Debugger Chrome Extension | **{check-circle}** | **{dotted-circle}** | **{check-circle}** | **{check-circle}** | **{check-circle}** | -| Snowplow Inspector Chrome Extension | **{check-circle}** | **{dotted-circle}** | **{check-circle}** | **{check-circle}** | **{check-circle}** | -| Snowplow Micro | **{check-circle}** | **{check-circle}** | **{check-circle}** | **{dotted-circle}** | **{dotted-circle}** | -| Snowplow Mini | **{check-circle}** | **{check-circle}** | **{dotted-circle}** | **{status_preparing}** | **{status_preparing}** | - -**Legend** - -**{check-circle}** Available, **{status_preparing}** In progress, **{dotted-circle}** Not Planned - -### Preparing your MR for Review - -1. For frontend events, in the MR description section, add a screenshot of the event's relevant section using the [Snowplow Analytics Debugger](https://chrome.google.com/webstore/detail/snowplow-analytics-debugg/jbnlcgeengmijcghameodeaenefieedm) Chrome browser extension. -1. For backend events, please use Snowplow Micro and add the output of the Snowplow Micro good events `GET http://localhost:9090/micro/good`. - -### Snowplow Analytics Debugger Chrome Extension - -Snowplow Analytics Debugger is a browser extension for testing frontend events. This works on production, staging and local development environments. - -1. Install the [Snowplow Analytics Debugger](https://chrome.google.com/webstore/detail/snowplow-analytics-debugg/jbnlcgeengmijcghameodeaenefieedm) Chrome browser extension. -1. Open Chrome DevTools to the Snowplow Analytics Debugger tab. -1. Learn more at [Igloo Analytics](https://www.iglooanalytics.com/blog/snowplow-analytics-debugger-chrome-extension.html). - -### Snowplow Inspector Chrome Extension - -Snowplow Inspector Chrome Extension is a browser extension for testing frontend events. This works on production, staging and local development environments. - -1. Install [Snowplow Inspector](https://chrome.google.com/webstore/detail/snowplow-inspector/maplkdomeamdlngconidoefjpogkmljm?hl=en). -1. Open the Chrome extension by pressing the Snowplow Inspector icon beside the address bar. -1. Click around on a webpage with Snowplow and you should see JavaScript events firing in the inspector window. - -### Snowplow Micro - -Snowplow Micro is a very small version of a full Snowplow data collection pipeline: small enough that it can be launched by a test suite. Events can be recorded into Snowplow Micro just as they can a full Snowplow pipeline. Micro then exposes an API that can be queried. - -Snowplow Micro is a Docker-based solution for testing frontend and backend events in a local development environment. You need to modify GDK using the instructions below to set this up. - -- Read [Introducing Snowplow Micro](https://snowplowanalytics.com/blog/2019/07/17/introducing-snowplow-micro/) -- Look at the [Snowplow Micro repository](https://github.com/snowplow-incubator/snowplow-micro) -- Watch our [installation guide recording](https://www.youtube.com/watch?v=OX46fo_A0Ag) - -1. Ensure Docker is installed and running. - -1. Install [Snowplow Micro](https://github.com/snowplow-incubator/snowplow-micro) by cloning the settings in [this project](https://gitlab.com/gitlab-org/snowplow-micro-configuration): -1. Navigate to the directory with the cloned project, and start the appropriate Docker - container with the following script: - - ```shell - ./snowplow-micro.sh - ``` - -1. Update your instance's settings to enable Snowplow events and point to the Snowplow Micro collector: - - ```shell - gdk psql -d gitlabhq_development - update application_settings set snowplow_collector_hostname='localhost:9090', snowplow_enabled=true, snowplow_cookie_domain='.gitlab.com'; - ``` - -1. Update `DEFAULT_SNOWPLOW_OPTIONS` in `app/assets/javascripts/tracking.js` to remove `forceSecureTracker: true`: - - ```diff - diff --git a/app/assets/javascripts/tracking.js b/app/assets/javascripts/tracking.js - index 0a1211d0a76..3b98c8f28f2 100644 - --- a/app/assets/javascripts/tracking.js - +++ b/app/assets/javascripts/tracking.js - @@ -7,7 +7,6 @@ const DEFAULT_SNOWPLOW_OPTIONS = { - appId: '', - userFingerprint: false, - respectDoNotTrack: true, - - forceSecureTracker: true, - eventMethod: 'post', - contexts: { webPage: true, performanceTiming: true }, - formTracking: false, - - ``` - -1. Update `snowplow_options` in `lib/gitlab/tracking.rb` to add `protocol` and `port`: - - ```diff - diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb - index 618e359211b..e9084623c43 100644 - --- a/lib/gitlab/tracking.rb - +++ b/lib/gitlab/tracking.rb - @@ -41,7 +41,9 @@ def snowplow_options(group) - cookie_domain: Gitlab::CurrentSettings.snowplow_cookie_domain, - app_id: Gitlab::CurrentSettings.snowplow_app_id, - form_tracking: additional_features, - - link_click_tracking: additional_features - + link_click_tracking: additional_features, - + protocol: 'http', - + port: 9090 - }.transform_keys! { |key| key.to_s.camelize(:lower).to_sym } - end - ``` - -1. Update `emitter` in `lib/gitlab/tracking/destinations/snowplow.rb` to change `protocol`: - - ```diff - diff --git a/lib/gitlab/tracking/destinations/snowplow.rb b/lib/gitlab/tracking/destinations/snowplow.rb - index 4fa844de325..5dd9d0eacfb 100644 - --- a/lib/gitlab/tracking/destinations/snowplow.rb - +++ b/lib/gitlab/tracking/destinations/snowplow.rb - @@ -40,7 +40,7 @@ def tracker - def emitter - SnowplowTracker::AsyncEmitter.new( - Gitlab::CurrentSettings.snowplow_collector_hostname, - - protocol: 'https' - + protocol: 'http' - ) - end - end - - ``` - -1. Restart GDK: - - ```shell - `gdk restart` - ``` - -1. Send a test Snowplow event from the Rails console: - - ```ruby - Gitlab::Tracking.self_describing_event('iglu:com.gitlab/pageview_context/jsonschema/1-0-0', data: { page_type: 'MY_TYPE' }, context: nil) - ``` - -1. Navigate to `localhost:9090/micro/good` to see the event. - -### Snowplow Mini - -[Snowplow Mini](https://github.com/snowplow/snowplow-mini) is an easily-deployable, single-instance version of Snowplow. - -Snowplow Mini can be used for testing frontend and backend events on a production, staging and local development environment. - -For GitLab.com, we're setting up a [QA and Testing environment](https://gitlab.com/gitlab-org/telemetry/-/issues/266) using Snowplow Mini. - -## Snowplow Schemas - -### [gitlab_standard](https://gitlab.com/gitlab-org/iglu/-/blob/master/public/schemas/com.gitlab/gitlab_standard/jsonschema/1-0-0) Schema - -| Field Name | Required | Type | Description | -|--------------|---------------------|---------|--------------------------------| -| project_id | **{dotted-circle}** | integer | ID of the associated project | -| namespace_id | **{dotted-circle}** | integer | ID of the associated namespace | - -### Default Schema - -| Field Name | Required | Type | Description | -|--------------------------|---------------------|-----------|----------------------------------------------------------------------------------------------------------------------------------| -| app_id | **{check-circle}** | string | Unique identifier for website / application | -| base_currency | **{dotted-circle}** | string | Reporting currency | -| br_colordepth | **{dotted-circle}** | integer | Browser color depth | -| br_cookies | **{dotted-circle}** | boolean | Does the browser permit cookies? | -| br_family | **{dotted-circle}** | string | Browser family | -| br_features_director | **{dotted-circle}** | boolean | Director plugin installed? | -| br_features_flash | **{dotted-circle}** | boolean | Flash plugin installed? | -| br_features_gears | **{dotted-circle}** | boolean | Google gears installed? | -| br_features_java | **{dotted-circle}** | boolean | Java plugin installed? | -| br_features_pdf | **{dotted-circle}** | boolean | Adobe PDF plugin installed? | -| br_features_quicktime | **{dotted-circle}** | boolean | Quicktime plugin installed? | -| br_features_realplayer | **{dotted-circle}** | boolean | Realplayer plugin installed? | -| br_features_silverlight | **{dotted-circle}** | boolean | Silverlight plugin installed? | -| br_features_windowsmedia | **{dotted-circle}** | boolean | Windows media plugin installed? | -| br_lang | **{dotted-circle}** | string | Language the browser is set to | -| br_name | **{dotted-circle}** | string | Browser name | -| br_renderengine | **{dotted-circle}** | string | Browser rendering engine | -| br_type | **{dotted-circle}** | string | Browser type | -| br_version | **{dotted-circle}** | string | Browser version | -| br_viewheight | **{dotted-circle}** | string | Browser viewport height | -| br_viewwidth | **{dotted-circle}** | string | Browser viewport width | -| collector_tstamp | **{dotted-circle}** | timestamp | Time stamp for the event recorded by the collector | -| contexts | **{dotted-circle}** | | | -| derived_contexts | **{dotted-circle}** | | Contexts derived in the Enrich process | -| derived_tstamp | **{dotted-circle}** | timestamp | Timestamp making allowance for innaccurate device clock | -| doc_charset | **{dotted-circle}** | string | Web page’s character encoding | -| doc_height | **{dotted-circle}** | string | Web page height | -| doc_width | **{dotted-circle}** | string | Web page width | -| domain_sessionid | **{dotted-circle}** | string | Unique identifier (UUID) for this visit of this user_id to this domain | -| domain_sessionidx | **{dotted-circle}** | integer | Index of number of visits that this user_id has made to this domain (The first visit is `1`) | -| domain_userid | **{dotted-circle}** | string | Unique identifier for a user, based on a first party cookie (so domain specific) | -| dvce_created_tstamp | **{dotted-circle}** | timestamp | Timestamp when event occurred, as recorded by client device | -| dvce_ismobile | **{dotted-circle}** | boolean | Indicates whether device is mobile | -| dvce_screenheight | **{dotted-circle}** | string | Screen / monitor resolution | -| dvce_screenwidth | **{dotted-circle}** | string | Screen / monitor resolution | -| dvce_sent_tstamp | **{dotted-circle}** | timestamp | Timestamp when event was sent by client device to collector | -| dvce_type | **{dotted-circle}** | string | Type of device | -| etl_tags | **{dotted-circle}** | string | JSON of tags for this ETL run | -| etl_tstamp | **{dotted-circle}** | timestamp | Timestamp event began ETL | -| event | **{dotted-circle}** | string | Event type | -| event_fingerprint | **{dotted-circle}** | string | Hash client-set event fields | -| event_format | **{dotted-circle}** | string | Format for event | -| event_id | **{dotted-circle}** | string | Event UUID | -| event_name | **{dotted-circle}** | string | Event name | -| event_vendor | **{dotted-circle}** | string | The company who developed the event model | -| event_version | **{dotted-circle}** | string | Version of event schema | -| geo_city | **{dotted-circle}** | string | City of IP origin | -| geo_country | **{dotted-circle}** | string | Country of IP origin | -| geo_latitude | **{dotted-circle}** | string | An approximate latitude | -| geo_longitude | **{dotted-circle}** | string | An approximate longitude | -| geo_region | **{dotted-circle}** | string | Region of IP origin | -| geo_region_name | **{dotted-circle}** | string | Region of IP origin | -| geo_timezone | **{dotted-circle}** | string | Timezone of IP origin | -| geo_zipcode | **{dotted-circle}** | string | Zip (postal) code of IP origin | -| ip_domain | **{dotted-circle}** | string | Second level domain name associated with the visitor’s IP address | -| ip_isp | **{dotted-circle}** | string | Visitor’s ISP | -| ip_netspeed | **{dotted-circle}** | string | Visitor’s connection type | -| ip_organization | **{dotted-circle}** | string | Organization associated with the visitor’s IP address – defaults to ISP name if none is found | -| mkt_campaign | **{dotted-circle}** | string | The campaign ID | -| mkt_clickid | **{dotted-circle}** | string | The click ID | -| mkt_content | **{dotted-circle}** | string | The content or ID of the ad. | -| mkt_medium | **{dotted-circle}** | string | Type of traffic source | -| mkt_network | **{dotted-circle}** | string | The ad network to which the click ID belongs | -| mkt_source | **{dotted-circle}** | string | The company / website where the traffic came from | -| mkt_term | **{dotted-circle}** | string | Keywords associated with the referrer | -| name_tracker | **{dotted-circle}** | string | The tracker namespace | -| network_userid | **{dotted-circle}** | string | Unique identifier for a user, based on a cookie from the collector (so set at a network level and shouldn’t be set by a tracker) | -| os_family | **{dotted-circle}** | string | Operating system family | -| os_manufacturer | **{dotted-circle}** | string | Manufacturers of operating system | -| os_name | **{dotted-circle}** | string | Name of operating system | -| os_timezone | **{dotted-circle}** | string | Client operating system timezone | -| page_referrer | **{dotted-circle}** | string | Referrer URL | -| page_title | **{dotted-circle}** | string | Page title | -| page_url | **{dotted-circle}** | string | Page URL | -| page_urlfragment | **{dotted-circle}** | string | Fragment aka anchor | -| page_urlhost | **{dotted-circle}** | string | Host aka domain | -| page_urlpath | **{dotted-circle}** | string | Path to page | -| page_urlport | **{dotted-circle}** | integer | Port if specified, 80 if not | -| page_urlquery | **{dotted-circle}** | string | Query string | -| page_urlscheme | **{dotted-circle}** | string | Scheme (protocol name) | -| platform | **{dotted-circle}** | string | The platform the app runs on | -| pp_xoffset_max | **{dotted-circle}** | integer | Maximum page x offset seen in the last ping period | -| pp_xoffset_min | **{dotted-circle}** | integer | Minimum page x offset seen in the last ping period | -| pp_yoffset_max | **{dotted-circle}** | integer | Maximum page y offset seen in the last ping period | -| pp_yoffset_min | **{dotted-circle}** | integer | Minimum page y offset seen in the last ping period | -| refr_domain_userid | **{dotted-circle}** | string | The Snowplow domain_userid of the referring website | -| refr_dvce_tstamp | **{dotted-circle}** | timestamp | The time of attaching the domain_userid to the inbound link | -| refr_medium | **{dotted-circle}** | string | Type of referer | -| refr_source | **{dotted-circle}** | string | Name of referer if recognised | -| refr_term | **{dotted-circle}** | string | Keywords if source is a search engine | -| refr_urlfragment | **{dotted-circle}** | string | Referer URL fragment | -| refr_urlhost | **{dotted-circle}** | string | Referer host | -| refr_urlpath | **{dotted-circle}** | string | Referer page path | -| refr_urlport | **{dotted-circle}** | integer | Referer port | -| refr_urlquery | **{dotted-circle}** | string | Referer URL querystring | -| refr_urlscheme | **{dotted-circle}** | string | Referer scheme | -| se_action | **{dotted-circle}** | string | The action / event itself | -| se_category | **{dotted-circle}** | string | The category of event | -| se_label | **{dotted-circle}** | string | A label often used to refer to the ‘object’ the action is performed on | -| se_property | **{dotted-circle}** | string | A property associated with either the action or the object | -| se_value | **{dotted-circle}** | decimal | A value associated with the user action | -| ti_category | **{dotted-circle}** | string | Item category | -| ti_currency | **{dotted-circle}** | string | Currency | -| ti_name | **{dotted-circle}** | string | Item name | -| ti_orderid | **{dotted-circle}** | string | Order ID | -| ti_price | **{dotted-circle}** | decimal | Item price | -| ti_price_base | **{dotted-circle}** | decimal | Item price in base currency | -| ti_quantity | **{dotted-circle}** | integer | Item quantity | -| ti_sku | **{dotted-circle}** | string | Item SKU | -| tr_affiliation | **{dotted-circle}** | string | Transaction affiliation (such as channel) | -| tr_city | **{dotted-circle}** | string | Delivery address: city | -| tr_country | **{dotted-circle}** | string | Delivery address: country | -| tr_currency | **{dotted-circle}** | string | Transaction Currency | -| tr_orderid | **{dotted-circle}** | string | Order ID | -| tr_shipping | **{dotted-circle}** | decimal | Delivery cost charged | -| tr_shipping_base | **{dotted-circle}** | decimal | Shipping cost in base currency | -| tr_state | **{dotted-circle}** | string | Delivery address: state | -| tr_tax | **{dotted-circle}** | decimal | Transaction tax value (such as amount of VAT included) | -| tr_tax_base | **{dotted-circle}** | decimal | Tax applied in base currency | -| tr_total | **{dotted-circle}** | decimal | Transaction total value | -| tr_total_base | **{dotted-circle}** | decimal | Total amount of transaction in base currency | -| true_tstamp | **{dotted-circle}** | timestamp | User-set exact timestamp | -| txn_id | **{dotted-circle}** | string | Transaction ID | -| unstruct_event | **{dotted-circle}** | JSON | The properties of the event | -| uploaded_at | **{dotted-circle}** | | | -| user_fingerprint | **{dotted-circle}** | integer | User identifier based on (hopefully unique) browser features | -| user_id | **{dotted-circle}** | string | Unique identifier for user, set by the business using setUserId | -| user_ipaddress | **{dotted-circle}** | string | IP address | -| useragent | **{dotted-circle}** | string | User agent (expressed as a browser string) | -| v_collector | **{dotted-circle}** | string | Collector version | -| v_etl | **{dotted-circle}** | string | ETL version | -| v_tracker | **{dotted-circle}** | string | Identifier for Snowplow tracker | +<!-- This redirect file can be deleted after February 1, 2021. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/#move-or-rename-a-page --> diff --git a/doc/development/product_analytics/usage_ping.md b/doc/development/product_analytics/usage_ping.md index cb4d48ba457..5fbdb508bb1 100644 --- a/doc/development/product_analytics/usage_ping.md +++ b/doc/development/product_analytics/usage_ping.md @@ -1,1145 +1,8 @@ --- -stage: Growth -group: Product Analytics -info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +redirect_to: '../usage_ping.md' --- -# Usage Ping Guide +This document was moved to [another location](../usage_ping.md). -> - Introduced in GitLab Enterprise Edition 8.10. -> - More statistics were added in GitLab Enterprise Edition 8.12. -> - Moved to GitLab Core in 9.1. -> - More statistics were added in GitLab Ultimate 11.2. - -This guide describes Usage Ping's purpose and how it's implemented. - -For more information about Product Analytics, see: - -- [Product Analytics Guide](https://about.gitlab.com/handbook/product/product-analytics-guide/) -- [Snowplow Guide](snowplow.md) - -More useful links: - -- [Product Analytics Direction](https://about.gitlab.com/direction/product-analytics/) -- [Data Analysis Process](https://about.gitlab.com/handbook/business-ops/data-team/#data-analysis-process/) -- [Data for Product Managers](https://about.gitlab.com/handbook/business-ops/data-team/programs/data-for-product-managers/) -- [Data Infrastructure](https://about.gitlab.com/handbook/business-ops/data-team/platform/infrastructure/) - -## What is Usage Ping? - -- GitLab sends a weekly payload containing usage data to GitLab Inc. Usage Ping provides high-level data to help our product, support, and sales teams. It does not send any project names, usernames, or any other specific data. The information from the usage ping is not anonymous, it is linked to the hostname of the instance. Sending usage ping is optional, and any instance can disable analytics. -- The usage data is primarily composed of row counts for different tables in the instance’s database. By comparing these counts month over month (or week over week), we can get a rough sense for how an instance is using the different features within the product. In addition to counts, other facts - that help us classify and understand GitLab installations are collected. -- Usage ping is important to GitLab as we use it to calculate our Stage Monthly Active Users (SMAU) which helps us measure the success of our stages and features. -- While usage ping is enabled, GitLab gathers data from the other instances and can show usage statistics of your instance to your users. - -### Why should we enable Usage Ping? - -- The main purpose of Usage Ping is to build a better GitLab. Data about how GitLab is used is collected to better understand feature/stage adoption and usage, which helps us understand how GitLab is adding value and helps our team better understand the reasons why people use GitLab and with this knowledge we're able to make better product decisions. -- As a benefit of having the usage ping active, GitLab lets you analyze the users’ activities over time of your GitLab installation. -- As a benefit of having the usage ping active, GitLab provides you with The DevOps Report,which gives you an overview of your entire instance’s adoption of Concurrent DevOps from planning to monitoring. -- You get better, more proactive support. (assuming that our TAMs and support organization used the data to deliver more value) -- You get insight and advice into how to get the most value out of your investment in GitLab. Wouldn't you want to know that a number of features or values are not being adopted in your organization? -- You get a report that illustrates how you compare against other similar organizations (anonymized), with specific advice and recommendations on how to improve your DevOps processes. -- Usage Ping is enabled by default. To disable it, see [Disable Usage Ping](#disable-usage-ping). - -### Limitations - -- Usage Ping does not track frontend events things like page views, link clicks, or user sessions, and only focuses on aggregated backend events. -- Because of these limitations we recommend instrumenting your products with Snowplow for more detailed analytics on GitLab.com and use Usage Ping to track aggregated backend events on self-managed. - -## Usage Ping payload - -You can view the exact JSON payload sent to GitLab Inc. in the administration panel. To view the payload: - -1. Navigate to **Admin Area > Settings > Metrics and profiling**. -1. Expand the **Usage statistics** section. -1. Click the **Preview payload** button. - -For an example payload, see [Example Usage Ping payload](#example-usage-ping-payload). - -## Disable Usage Ping - -To disable Usage Ping in the GitLab UI, go to the **Settings** page of your administration panel and uncheck the **Usage Ping** checkbox. - -To disable Usage Ping and prevent it from being configured in the future through the administration panel, Omnibus installs can set the following in [`gitlab.rb`](https://docs.gitlab.com/omnibus/settings/configuration.html#configuration-options): - -```ruby -gitlab_rails['usage_ping_enabled'] = false -``` - -Source installations can set the following in `gitlab.yml`: - -```yaml -production: &base - # ... - gitlab: - # ... - usage_ping_enabled: false -``` - -## Usage Ping request flow - -The following example shows a basic request/response flow between a GitLab instance, the Versions Application, the License Application, Salesforce, the GitLab S3 Bucket, the GitLab Snowflake Data Warehouse, and Sisense: - -```mermaid -sequenceDiagram - participant GitLab Instance - participant Versions Application - participant Licenses Application - participant Salesforce - participant S3 Bucket - participant Snowflake DW - participant Sisense Dashboards - GitLab Instance->>Versions Application: Send Usage Ping - loop Process usage data - Versions Application->>Versions Application: Parse usage data - Versions Application->>Versions Application: Write to database - Versions Application->>Versions Application: Update license ping time - end - loop Process data for Salesforce - Versions Application-xLicenses Application: Request Zuora subscription id - Licenses Application-xVersions Application: Zuora subscription id - Versions Application-xSalesforce: Request Zuora account id by Zuora subscription id - Salesforce-xVersions Application: Zuora account id - Versions Application-xSalesforce: Usage data for the Zuora account - end - Versions Application->>S3 Bucket: Export Versions database - S3 Bucket->>Snowflake DW: Import data - Snowflake DW->>Snowflake DW: Transform data using dbt - Snowflake DW->>Sisense Dashboards: Data available for querying - Versions Application->>GitLab Instance: DevOps Report (Conversational Development Index) -``` - -## How Usage Ping works - -1. The Usage Ping [cron job](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/workers/gitlab_usage_ping_worker.rb#L30) is set in Sidekiq to run weekly. -1. When the cron job runs, it calls [`Gitlab::UsageData.to_json`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/services/submit_usage_ping_service.rb#L22). -1. `Gitlab::UsageData.to_json` [cascades down](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data.rb#L22) to ~400+ other counter method calls. -1. The response of all methods calls are [merged together](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data.rb#L14) into a single JSON payload in `Gitlab::UsageData.to_json`. -1. The JSON payload is then [posted to the Versions application]( https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/services/submit_usage_ping_service.rb#L20) - If a firewall exception is needed, the required URL depends on several things. If - the hostname is `version.gitlab.com`, the protocol is `TCP`, and the port number is `443`, - the required URL is <https://version.gitlab.com/>. - -## Implementing Usage Ping - -Usage Ping consists of two kinds of data, counters and observations. Counters track how often a certain event -happened over time, such as how many CI pipelines have run. They are monotonic and always trend up. -Observations are facts collected from one or more GitLab instances and can carry arbitrary data. There are no -general guidelines around how to collect those, due to the individual nature of that data. - -There are several types of counters which are all found in `usage_data.rb`: - -- **Ordinary Batch Counters:** Simple count of a given ActiveRecord_Relation -- **Distinct Batch Counters:** Distinct count of a given ActiveRecord_Relation in a given column -- **Sum Batch Counters:** Sum the values of a given ActiveRecord_Relation in a given column -- **Alternative Counters:** Used for settings and configurations -- **Redis Counters:** Used for in-memory counts. - -NOTE: -Only use the provided counter methods. Each counter method contains a built in fail safe to isolate each counter to avoid breaking the entire Usage Ping. - -### Why batch counting - -For large tables, PostgreSQL can take a long time to count rows due to MVCC [(Multi-version Concurrency Control)](https://en.wikipedia.org/wiki/Multiversion_concurrency_control). Batch counting is a counting method where a single large query is broken into multiple smaller queries. For example, instead of a single query querying 1,000,000 records, with batch counting, you can execute 100 queries of 10,000 records each. Batch counting is useful for avoiding database timeouts as each batch query is significantly shorter than one single long running query. - -For GitLab.com, there are extremely large tables with 15 second query timeouts, so we use batch counting to avoid encountering timeouts. Here are the sizes of some GitLab.com tables: - -| Table | Row counts in millions | -|------------------------------|------------------------| -| `merge_request_diff_commits` | 2280 | -| `ci_build_trace_sections` | 1764 | -| `merge_request_diff_files` | 1082 | -| `events` | 514 | - -We have several batch counting methods available: - -- `Ordinary Batch Counters` -- `Distinct Batch Counters` -- `Sum Batch Counters` -- `Estimated Batch Counters` - -Batch counting requires indexes on columns to calculate max, min, and range queries. In some cases, -you may need to add a specialized index on the columns involved in a counter. - -### Ordinary Batch Counters - -Handles `ActiveRecord::StatementInvalid` error - -Simple count of a given ActiveRecord_Relation, does a non-distinct batch count, smartly reduces batch_size and handles errors. - -Method: `count(relation, column = nil, batch: true, start: nil, finish: nil)` - -Arguments: - -- `relation` the ActiveRecord_Relation to perform the count -- `column` the column to perform the count on, by default is the primary key -- `batch`: default `true` in order to use batch counting -- `start`: custom start of the batch counting in order to avoid complex min calculations -- `end`: custom end of the batch counting in order to avoid complex min calculations - -Examples: - -```ruby -count(User.active) -count(::Clusters::Cluster.aws_installed.enabled, :cluster_id) -count(::Clusters::Cluster.aws_installed.enabled, :cluster_id, start: ::Clusters::Cluster.minimum(:id), finish: ::Clusters::Cluster.maximum(:id)) -``` - -### Distinct Batch Counters - -Handles `ActiveRecord::StatementInvalid` error - -Distinct count of a given ActiveRecord_Relation on given column, a distinct batch count, smartly reduces batch_size and handles errors. - -Method: `distinct_count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil)` - -Arguments: - -- `relation` the ActiveRecord_Relation to perform the count -- `column` the column to perform the distinct count, by default is the primary key -- `batch`: default `true` in order to use batch counting -- `batch_size`: if none set it uses default value 10000 from `Gitlab::Database::BatchCounter` -- `start`: custom start of the batch counting in order to avoid complex min calculations -- `end`: custom end of the batch counting in order to avoid complex min calculations - -WARNING: -Counting over non-unique columns can lead to performance issues. Take a look at the [iterating tables in batches](../iterating_tables_in_batches.md) guide for more details. - -Examples: - -```ruby -distinct_count(::Project, :creator_id) -distinct_count(::Note.with_suggestions.where(time_period), :author_id, start: ::User.minimum(:id), finish: ::User.maximum(:id)) -distinct_count(::Clusters::Applications::CertManager.where(time_period).available.joins(:cluster), 'clusters.user_id') -``` - -### Sum Batch Counters - -Handles `ActiveRecord::StatementInvalid` error - -Sum the values of a given ActiveRecord_Relation on given column and handles errors. - -Method: `sum(relation, column, batch_size: nil, start: nil, finish: nil)` - -Arguments: - -- `relation` the ActiveRecord_Relation to perform the operation -- `column` the column to sum on -- `batch_size`: if none set it uses default value 1000 from `Gitlab::Database::BatchCounter` -- `start`: custom start of the batch counting in order to avoid complex min calculations -- `end`: custom end of the batch counting in order to avoid complex min calculations - -Examples: - -```ruby -sum(JiraImportState.finished, :imported_issues_count) -``` - -### Grouping & Batch Operations - -The `count`, `distinct_count`, and `sum` batch counters can accept an `ActiveRecord::Relation` -object, which groups by a specified column. With a grouped relation, the methods do batch counting, -handle errors, and returns a hash table of key-value pairs. - -Examples: - -```ruby -count(Namespace.group(:type)) -# returns => {nil=>179, "Group"=>54} - -distinct_count(Project.group(:visibility_level), :creator_id) -# returns => {0=>1, 10=>1, 20=>11} - -sum(Issue.group(:state_id), :weight)) -# returns => {1=>3542, 2=>6820} -``` - -### Estimated Batch Counters - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/48233) in GitLab 13.7. - -Estimated batch counter functionality handles `ActiveRecord::StatementInvalid` errors -when used through the provided `estimate_batch_distinct_count` method. -Errors return a value of `-1`. - -WARNING: -This functionality estimates a distinct count of a specific ActiveRecord_Relation in a given column, -which uses the [HyperLogLog](http://algo.inria.fr/flajolet/Publications/FlFuGaMe07.pdf) algorithm. -As the HyperLogLog algorithm is probabilistic, the **results always includes error**. -The highest encountered error rate is 4.9%. - -When correctly used, the `estimate_batch_distinct_count` method enables efficient counting over -columns that contain non-unique values, which can not be assured by other counters. - -Method: [`estimate_batch_distinct_count(relation, column = nil, batch_size: nil, start: nil, finish: nil)`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/utils/usage_data.rb#L63) - -The method includes the following arguments: - -- `relation`: The ActiveRecord_Relation to perform the count. -- `column`: The column to perform the distinct count. The default is the primary key. -- `batch_size`: The default is 10,000, from `Gitlab::Database::PostgresHll::BatchDistinctCounter::DEFAULT_BATCH_SIZE`. -- `start`: The custom start of the batch count, to avoid complex minimum calculations. -- `finish`: The custom end of the batch count in order to avoid complex maximum calculations. - -The method includes the following prerequisites: - -1. The supplied `relation` must include the primary key defined as the numeric column. - For example: `id bigint NOT NULL`. -1. The `estimate_batch_distinct_count` can handle a joined relation. To utilize its ability to - count non-unique columns, the joined relation **must NOT** have a one-to-many relationship, - such as `has_many :boards`. -1. Both `start` and `finish` arguments should always represent primary key relationship values, - even if the estimated count refers to another column, for example: - - ```ruby - estimate_batch_distinct_count(::Note, :author_id, start: ::Note.minimum(:id), finish: ::Note.maximum(:id)) - ``` - -Examples: - -1. Simple execution of estimated batch counter, with only relation provided, returned value will represent estimated - number of unique values in `id` column (which is the primary key) of `Project` relation: - - ```ruby - estimate_batch_distinct_count(::Project) - ``` - -1. Execution of estimated batch counter, where provided relation has applied additional filter (`.where(time_period)`), number of unique values is going to be estimated in custom column (`:author_id`), and parameters: `start` and `finish` together apply boundaries that defines range of provided relation that is going to be analyzed - - ```ruby - estimate_batch_distinct_count(::Note.with_suggestions.where(time_period), :author_id, start: ::Note.minimum(:id), finish: ::Note.maximum(:id)) - ``` - -1. Execution of estimated batch counter with joined relation (`joins(:cluster)`), for a custom column (`'clusters.user_id'`): - - ```ruby - estimate_batch_distinct_count(::Clusters::Applications::CertManager.where(time_period).available.joins(:cluster), 'clusters.user_id') - ``` - -When instrumenting metric with usage of estimated batch counter please add `_estimated` suffix to its name, for example: - -```ruby - "counts": { - "ci_builds_estimated": estimate_batch_distinct_count(Ci::Build), - ... -``` - -### Redis Counters - -Handles `::Redis::CommandError` and `Gitlab::UsageDataCounters::BaseCounter::UnknownEvent` -returns -1 when a block is sent or hash with all values -1 when a `counter(Gitlab::UsageDataCounters)` is sent -different behavior due to 2 different implementations of Redis counter - -Method: `redis_usage_data(counter, &block)` - -Arguments: - -- `counter`: a counter from `Gitlab::UsageDataCounters`, that has `fallback_totals` method implemented -- or a `block`: which is evaluated - -#### Ordinary Redis Counters - -Examples of implementation: - -- Using Redis methods [`INCR`](https://redis.io/commands/incr), [`GET`](https://redis.io/commands/get), and [`Gitlab::UsageDataCounters::WikiPageCounter`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/wiki_page_counter.rb) -- Using Redis methods [`HINCRBY`](https://redis.io/commands/hincrby), [`HGETALL`](https://redis.io/commands/hgetall), and [`Gitlab::UsageCounters::PodLogs`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_counters/pod_logs.rb) - -##### UsageData API Tracking - -<!-- There's nearly identical content in `##### Adding new events`. If you fix errors here, you may need to fix the same errors in the other location. --> - -1. Track event using `UsageData` API - - Increment event count using ordinary Redis counter, for given event name. - - Tracking events using the `UsageData` API requires the `usage_data_api` feature flag to be enabled, which is enabled by default. - - API requests are protected by checking for a valid CSRF token. - - In order to be able to increment the values the related feature `usage_data_<event_name>` should be enabled. - - ```plaintext - POST /usage_data/increment_counter - ``` - - | Attribute | Type | Required | Description | - | :-------- | :--- | :------- | :---------- | - | `event` | string | yes | The event name it should be tracked | - - Response - - - `200` if event was tracked - - `400 Bad request` if event parameter is missing - - `401 Unauthorized` if user is not authenticated - - `403 Forbidden` for invalid CSRF token provided - -1. Track events using JavaScript/Vue API helper which calls the API above - - Note that `usage_data_api` and `usage_data_#{event_name}` should be enabled in order to be able to track events - - ```javascript - import api from '~/api'; - - api.trackRedisCounterEvent('my_already_defined_event_name'), - ``` - -#### Redis HLL Counters - -WARNING: -HyperLogLog (HLL) is a probabilistic algorithm and its **results always includes some small error**. According to [Redis documentation](https://redis.io/commands/pfcount), data from -used HLL implementation is "approximated with a standard error of 0.81%". - -With `Gitlab::UsageDataCounters::HLLRedisCounter` we have available data structures used to count unique values. - -Implemented using Redis methods [PFADD](https://redis.io/commands/pfadd) and [PFCOUNT](https://redis.io/commands/pfcount). - -##### Adding new events - -1. Define events in [`known_events`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/known_events/). - - Example event: - - ```yaml - - name: i_compliance_credential_inventory - category: compliance - redis_slot: compliance - expiry: 42 # 6 weeks - aggregation: weekly - ``` - - Keys: - - - `name`: unique event name. - - Name format `<prefix>_<redis_slot>_name`. - - Use one of the following prefixes for the event's name: - - - `g_` for group, as an event which is tracked for group. - - `p_` for project, as an event which is tracked for project. - - `i_` for instance, as an event which is tracked for instance. - - `a_` for events encompassing all `g_`, `p_`, `i_`. - - `o_` for other. - - Consider including in the event's name the Redis slot in order to be able to count totals for a specific category. - - Example names: `i_compliance_credential_inventory`, `g_analytics_contribution`. - - - `category`: event category. Used for getting total counts for events in a category, for easier - access to a group of events. - - `redis_slot`: optional Redis slot; default value: event name. Used if needed to calculate totals - for a group of metrics. Ensure keys are in the same slot. For example: - `i_compliance_credential_inventory` with `redis_slot: 'compliance'` builds Redis key - `i_{compliance}_credential_inventory-2020-34`. If `redis_slot` is not defined the Redis key will - be `{i_compliance_credential_inventory}-2020-34`. - - `expiry`: expiry time in days. Default: 29 days for daily aggregation and 6 weeks for weekly - aggregation. - - `aggregation`: may be set to a `:daily` or `:weekly` key. Defines how counting data is stored in Redis. - Aggregation on a `daily` basis does not pull more fine grained data. - - `feature_flag`: optional. For details, see our [GitLab internal Feature flags](../feature_flags/) documentation. - -1. Track event in controller using `RedisTracking` module with `track_redis_hll_event(*controller_actions, name:, feature:, feature_default_enabled: false)`. - - Arguments: - - - `controller_actions`: controller actions we want to track. - - `name`: event name. - - `feature`: feature name, all metrics we track should be under feature flag. - - `feature_default_enabled`: feature flag is disabled by default, set to `true` for it to be enabled by default. - - Example usage: - - ```ruby - # controller - class ProjectsController < Projects::ApplicationController - include RedisTracking - - skip_before_action :authenticate_user!, only: :show - track_redis_hll_event :index, :show, name: 'g_compliance_example_feature_visitors', feature: :compliance_example_feature, feature_default_enabled: true - - def index - render html: 'index' - end - - def new - render html: 'new' - end - - def show - render html: 'show' - end - end - ``` - -1. Track event in API using `increment_unique_values(event_name, values)` helper method. - - In order to be able to track the event, Usage Ping must be enabled and the event feature `usage_data_<event_name>` must be enabled. - - Arguments: - - - `event_name`: event name. - - `values`: values counted, one value or array of values. - - Example usage: - - ```ruby - get ':id/registry/repositories' do - repositories = ContainerRepositoriesFinder.new( - user: current_user, subject: user_group - ).execute - - increment_unique_values('i_list_repositories', current_user.id) - - present paginate(repositories), with: Entities::ContainerRegistry::Repository, tags: params[:tags], tags_count: params[:tags_count] - end - ``` - -1. Track event using `track_usage_event(event_name, values) in services and graphql - - Increment unique values count using Redis HLL, for given event name. - - Example: - - [Track usage event for incident created in service](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/services/issues/update_service.rb) - - [Track usage event for incident created in graphql](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/graphql/mutations/alert_management/update_alert_status.rb) - - ```ruby - track_usage_event(:incident_management_incident_created, current_user.id) - ``` - -<!-- There's nearly identical content in `##### UsageData API Tracking`. If you find / fix errors here, you may need to fix errors in that section too. --> - -1. Track event using `UsageData` API - - Increment unique users count using Redis HLL, for given event name. - - Tracking events using the `UsageData` API requires the `usage_data_api` feature flag to be enabled, which is enabled by default. - - API requests are protected by checking for a valid CSRF token. - - In order to increment the values, the related feature `usage_data_<event_name>` should be - set to `default_enabled: true`. For more information, see - [Feature flags in development of GitLab](../feature_flags/index.md). - - ```plaintext - POST /usage_data/increment_unique_users - ``` - - | Attribute | Type | Required | Description | - | :-------- | :--- | :------- | :---------- | - | `event` | string | yes | The event name it should be tracked | - - Response - - Return 200 if tracking failed for any reason. - - - `200` if event was tracked or any errors - - `400 Bad request` if event parameter is missing - - `401 Unauthorized` if user is not authenticated - - `403 Forbidden` for invalid CSRF token provided - -1. Track events using JavaScript/Vue API helper which calls the API above - - Example usage for an existing event already defined in [known events](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/known_events/): - - Usage Data API is behind `usage_data_api` feature flag which, as of GitLab 13.7, is - now set to `default_enabled: true`. - - Each event tracked using Usage Data API is behind a feature flag `usage_data_#{event_name}` which should be `default_enabled: true` - - ```javascript - import api from '~/api'; - - api.trackRedisHllUserEvent('my_already_defined_event_name'), - ``` - -1. Track event using base module `Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event_name, values:)`. - - Arguments: - - - `event_name`: event name. - - `values`: One value or array of values we count. For example: user_id, visitor_id, user_ids. - -1. Track event on context level using base module `Gitlab::UsageDataCounters::HLLRedisCounter.track_event_in_context(event_name, values:, context:)`. - - Arguments: - - - `event_name`: event name. - - `values`: values we count. For example: user_id, visitor_id. - - `context`: context value. Allowed values are `default`, `free`, `bronze`, `silver`, `gold`, `starter`, `premium`, `ultimate` - -1. Get event data using `Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names:, start_date:, end_date:, context: '')`. - - Arguments: - - - `event_names`: the list of event names. - - `start_date`: start date of the period for which we want to get event data. - - `end_date`: end date of the period for which we want to get event data. - - `context`: context of the event. Allowed values are `default`, `free`, `bronze`, `silver`, `gold`, `starter`, `premium`, `ultimate`. - -1. Testing tracking and getting unique events - -Trigger events in rails console by using `track_event` method - - ```ruby - Gitlab::UsageDataCounters::HLLRedisCounter.track_event('g_compliance_audit_events', values: 1) - Gitlab::UsageDataCounters::HLLRedisCounter.track_event('g_compliance_audit_events', values: [2, 3]) - ``` - -Next, get the unique events for the current week. - - ```ruby - # Get unique events for metric for current_week - Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: 'g_compliance_audit_events', - start_date: Date.current.beginning_of_week, end_date: Date.current.end_of_week) - ``` - -##### Recommendations - -We have the following recommendations for [Adding new events](#adding-new-events): - -- Event aggregation: weekly. -- Key expiry time: - - Daily: 29 days. - - Weekly: 42 days. -- When adding new metrics, use a [feature flag](../../operations/feature_flags.md) to control the impact. -- For feature flags triggered by another service, set `default_enabled: false`, - - Events can be triggered using the `UsageData` API, which helps when there are > 10 events per change - -##### Enable/Disable Redis HLL tracking - -Events are tracked behind [feature flags](../feature_flags/index.md) due to concerns for Redis performance and scalability. - -For a full list of events and corresponding feature flags see, [known_events](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/known_events/) files. - -To enable or disable tracking for specific event within <https://gitlab.com> or <https://about.staging.gitlab.com>, run commands such as the following to -[enable or disable the corresponding feature](../feature_flags/index.md). - -```shell -/chatops run feature set <feature_name> true -/chatops run feature set <feature_name> false -``` - -##### Known events are added automatically in usage data payload - -All events added in [`known_events/common.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/known_events/common.yml) are automatically added to usage data generation under the `redis_hll_counters` key. This column is stored in [version-app as a JSON](https://gitlab.com/gitlab-services/version-gitlab-com/-/blob/master/db/schema.rb#L209). -For each event we add metrics for the weekly and monthly time frames, and totals for each where applicable: - -- `#{event_name}_weekly`: Data for 7 days for daily [aggregation](#adding-new-events) events and data for the last complete week for weekly [aggregation](#adding-new-events) events. -- `#{event_name}_monthly`: Data for 28 days for daily [aggregation](#adding-new-events) events and data for the last 4 complete weeks for weekly [aggregation](#adding-new-events) events. - -Redis HLL implementation calculates automatic total metrics, if there are more than one metric for the same category, aggregation and Redis slot. - -- `#{category}_total_unique_counts_weekly`: Total unique counts for events in the same category for the last 7 days or the last complete week, if events are in the same Redis slot and we have more than one metric. -- `#{category}_total_unique_counts_monthly`: Total unique counts for events in same category for the last 28 days or the last 4 complete weeks, if events are in the same Redis slot and we have more than one metric. - -Example of `redis_hll_counters` data: - -```ruby -{:redis_hll_counters=> - {"compliance"=> - {"g_compliance_dashboard_weekly"=>0, - "g_compliance_dashboard_monthly"=>0, - "g_compliance_audit_events_weekly"=>0, - "g_compliance_audit_events_monthly"=>0, - "compliance_total_unique_counts_weekly"=>0, - "compliance_total_unique_counts_monthly"=>0}, - "analytics"=> - {"g_analytics_contribution_weekly"=>0, - "g_analytics_contribution_monthly"=>0, - "g_analytics_insights_weekly"=>0, - "g_analytics_insights_monthly"=>0, - "analytics_total_unique_counts_weekly"=>0, - "analytics_total_unique_counts_monthly"=>0}, - "ide_edit"=> - {"g_edit_by_web_ide_weekly"=>0, - "g_edit_by_web_ide_monthly"=>0, - "g_edit_by_sfe_weekly"=>0, - "g_edit_by_sfe_monthly"=>0, - "ide_edit_total_unique_counts_weekly"=>0, - "ide_edit_total_unique_counts_monthly"=>0}, - "search"=> - {"i_search_total_weekly"=>0, "i_search_total_monthly"=>0, "i_search_advanced_weekly"=>0, "i_search_advanced_monthly"=>0, "i_search_paid_weekly"=>0, "i_search_paid_monthly"=>0, "search_total_unique_counts_weekly"=>0, "search_total_unique_counts_monthly"=>0}, - "source_code"=>{"wiki_action_weekly"=>0, "wiki_action_monthly"=>0} - } -``` - -Example usage: - -```ruby -# Redis Counters -redis_usage_data(Gitlab::UsageDataCounters::WikiPageCounter) -redis_usage_data { ::Gitlab::UsageCounters::PodLogs.usage_totals[:total] } - -# Define events in common.yml https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/known_events/common.yml - -# Tracking events -Gitlab::UsageDataCounters::HLLRedisCounter.track_event('expand_vulnerabilities', values: visitor_id) - -# Get unique events for metric -redis_usage_data { Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: 'expand_vulnerabilities', start_date: 28.days.ago, end_date: Date.current) } -``` - -### Alternative Counters - -Handles `StandardError` and fallbacks into -1 this way not all measures fail if we encounter one exception. -Mainly used for settings and configurations. - -Method: `alt_usage_data(value = nil, fallback: -1, &block)` - -Arguments: - -- `value`: a simple static value in which case the value is simply returned. -- or a `block`: which is evaluated -- `fallback: -1`: the common value used for any metrics that are failing. - -Example of usage: - -```ruby -alt_usage_data { Gitlab::VERSION } -alt_usage_data { Gitlab::CurrentSettings.uuid } -alt_usage_data(999) -``` - -### Prometheus Queries - -In those cases where operational metrics should be part of Usage Ping, a database or Redis query is unlikely -to provide useful data. Instead, Prometheus might be more appropriate, since most GitLab architectural -components publish metrics to it that can be queried back, aggregated, and included as usage data. - -NOTE: -Prometheus as a data source for Usage Ping is currently only available for single-node Omnibus installations -that are running the [bundled Prometheus](../../administration/monitoring/prometheus/index.md) instance. - -To query Prometheus for metrics, a helper method is available to `yield` a fully configured -`PrometheusClient`, given it is available as per the note above: - -```ruby -with_prometheus_client do |client| - response = client.query('<your query>') - ... -end -``` - -Please refer to [the `PrometheusClient` definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/prometheus_client.rb) -for how to use its API to query for data. - -## Developing and testing Usage Ping - -### 1. Naming and placing the metrics - -Add the metric in one of the top level keys - -- `license`: for license related metrics. -- `settings`: for settings related metrics. -- `counts_weekly`: for counters that have data for the most recent 7 days. -- `counts_monthly`: for counters that have data for the most recent 28 days. -- `counts`: for counters that have data for all time. - -### 2. Use your Rails console to manually test counters - -```ruby -# count -Gitlab::UsageData.count(User.active) -Gitlab::UsageData.count(::Clusters::Cluster.aws_installed.enabled, :cluster_id) - -# count distinct -Gitlab::UsageData.distinct_count(::Project, :creator_id) -Gitlab::UsageData.distinct_count(::Note.with_suggestions.where(time_period), :author_id, start: ::User.minimum(:id), finish: ::User.maximum(:id)) -``` - -### 3. Generate the SQL query - -Your Rails console returns the generated SQL queries. - -Example: - -```ruby -pry(main)> Gitlab::UsageData.count(User.active) - (2.6ms) SELECT "features"."key" FROM "features" - (15.3ms) SELECT MIN("users"."id") FROM "users" WHERE ("users"."state" IN ('active')) AND ("users"."user_type" IS NULL OR "users"."user_type" IN (6, 4)) - (2.4ms) SELECT MAX("users"."id") FROM "users" WHERE ("users"."state" IN ('active')) AND ("users"."user_type" IS NULL OR "users"."user_type" IN (6, 4)) - (1.9ms) SELECT COUNT("users"."id") FROM "users" WHERE ("users"."state" IN ('active')) AND ("users"."user_type" IS NULL OR "users"."user_type" IN (6, 4)) AND "users"."id" BETWEEN 1 AND 100000 -``` - -### 4. Optimize queries with #database-lab - -Paste the SQL query into `#database-lab` to see how the query performs at scale. - -- `#database-lab` is a Slack channel which uses a production-sized environment to test your queries. -- GitLab.com’s production database has a 15 second timeout. -- Any single query must stay below [1 second execution time](../query_performance.md#timing-guidelines-for-queries) with cold caches. -- Add a specialized index on columns involved to reduce the execution time. - -In order to have an understanding of the query's execution we add in the MR description the following information: - -- For counters that have a `time_period` test we add information for both cases: - - `time_period = {}` for all time periods - - `time_period = { created_at: 28.days.ago..Time.current }` for last 28 days period -- Execution plan and query time before and after optimization -- Query generated for the index and time -- Migration output for up and down execution - -We also use `#database-lab` and [explain.depesz.com](https://explain.depesz.com/). For more details, see the [database review guide](../database_review.md#preparation-when-adding-or-modifying-queries). - -#### Optimization recommendations and examples - -- Use specialized indexes [example 1](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26871), [example 2](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26445). -- Use defined `start` and `finish`, and simple queries, because these values can be memoized and reused, [example](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/37155). -- Avoid joins and write the queries as simply as possible, [example](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/36316). -- Set a custom `batch_size` for `distinct_count`, [example](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38000). - -### 5. Add the metric definition - -When adding, changing, or updating metrics, please update the [Event Dictionary's **Usage Ping** table](https://about.gitlab.com/handbook/product/product-analytics-guide/#event-dictionary). - -### 6. Add new metric to Versions Application - -Check if new metrics need to be added to the Versions Application. See `usage_data` [schema](https://gitlab.com/gitlab-services/version-gitlab-com/-/blob/master/db/schema.rb#L147) and usage data [parameters accepted](https://gitlab.com/gitlab-services/version-gitlab-com/-/blob/master/app/services/usage_ping.rb). Any metrics added under the `counts` key are saved in the `stats` column. - -### 7. Add the feature label - -Add the `feature` label to the Merge Request for new Usage Ping metrics. These are user-facing changes and are part of expanding the Usage Ping feature. - -### 8. Add a changelog file - -Ensure you comply with the [Changelog entries guide](../changelog.md). - -### 9. Ask for a Product Analytics Review - -On GitLab.com, we have DangerBot setup to monitor Product Analytics related files and DangerBot recommends a Product Analytics review. Mention `@gitlab-org/growth/product_analytics/engineers` in your MR for a review. - -### 10. Verify your metric - -On GitLab.com, the Product Analytics team regularly monitors Usage Ping. They may alert you that your metrics need further optimization to run quicker and with greater success. You may also use the [Usage Ping QA dashboard](https://app.periscopedata.com/app/gitlab/632033/Usage-Ping-QA) to check how well your metric performs. The dashboard allows filtering by GitLab version, by "Self-managed" & "Saas" and shows you how many failures have occurred for each metric. Whenever you notice a high failure rate, you may re-optimize your metric. - -### Optional: Test Prometheus based Usage Ping - -If the data submitted includes metrics [queried from Prometheus](#prometheus-queries) that you would like to inspect and verify, -then you need to ensure that a Prometheus server is running locally, and that furthermore the respective GitLab components -are exporting metrics to it. If you do not need to test data coming from Prometheus, no further action -is necessary, since Usage Ping should degrade gracefully in the absence of a running Prometheus server. - -There are currently three kinds of components that may export data to Prometheus, and which are included in Usage Ping: - -- [`node_exporter`](https://github.com/prometheus/node_exporter) - Exports node metrics from the host machine -- [`gitlab-exporter`](https://gitlab.com/gitlab-org/gitlab-exporter) - Exports process metrics from various GitLab components -- various GitLab services such as Sidekiq and the Rails server that export their own metrics - -#### Test with an Omnibus container - -This is the recommended approach to test Prometheus based Usage Ping. - -The easiest way to verify your changes is to build a new Omnibus image from your code branch via CI, then download the image -and run a local container instance: - -1. From your merge request, click on the `qa` stage, then trigger the `package-and-qa` job. This job triggers an Omnibus -build in a [downstream pipeline of the `omnibus-gitlab-mirror` project](https://gitlab.com/gitlab-org/build/omnibus-gitlab-mirror/-/pipelines). -1. In the downstream pipeline, wait for the `gitlab-docker` job to finish. -1. Open the job logs and locate the full container name including the version. It takes the following form: `registry.gitlab.com/gitlab-org/build/omnibus-gitlab-mirror/gitlab-ee:<VERSION>`. -1. On your local machine, make sure you are logged in to the GitLab Docker registry. You can find the instructions for this in -[Authenticate to the GitLab Container Registry](../../user/packages/container_registry/index.md#authenticate-with-the-container-registry). -1. Once logged in, download the new image via `docker pull registry.gitlab.com/gitlab-org/build/omnibus-gitlab-mirror/gitlab-ee:<VERSION>` -1. For more information about working with and running Omnibus GitLab containers in Docker, please refer to [GitLab Docker images](https://docs.gitlab.com/omnibus/docker/README.html) in the Omnibus documentation. - -#### Test with GitLab development toolkits - -This is the less recommended approach, since it comes with a number of difficulties when emulating a real GitLab deployment. - -The [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit) is not currently set up to run a Prometheus server or `node_exporter` alongside other GitLab components. If you would -like to do so, [Monitoring the GDK with Prometheus](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/master/doc/howto/prometheus/index.md#monitoring-the-gdk-with-prometheus) is a good start. - -The [GCK](https://gitlab.com/gitlab-org/gitlab-compose-kit) has limited support for testing Prometheus based Usage Ping. -By default, it already comes with a fully configured Prometheus service that is set up to scrape a number of components, -but with the following limitations: - -- It does not currently run a `gitlab-exporter` instance, so several `process_*` metrics from services such as Gitaly may be missing. -- While it runs a `node_exporter`, `docker-compose` services emulate hosts, meaning that it would normally report itself to not be associated -with any of the other services that are running. That is not how node metrics are reported in a production setup, where `node_exporter` -always runs as a process alongside other GitLab components on any given node. From Usage Ping's perspective none of the node data would therefore -appear to be associated to any of the services running, since they all appear to be running on different hosts. To alleviate this problem, the `node_exporter` in GCK was arbitrarily "assigned" to the `web` service, meaning only for this service `node_*` metrics appears in Usage Ping. - -## Aggregated metrics - -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/45979) in GitLab 13.6. - -WARNING: -This feature is intended solely for internal GitLab use. - -In order to add data for aggregated metrics into Usage Ping payload you should add corresponding definition in [`aggregated_metrics`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/aggregated_metrics/). Each aggregate definition includes following parts: - -- name: unique name under which aggregate metric is added to Usage Ping payload -- operator: operator that defines how aggregated metric data is counted. Available operators are: - - `OR`: removes duplicates and counts all entries that triggered any of listed events - - `AND`: removes duplicates and counts all elements that were observed triggering all of following events -- events: list of events names (from [`known_events/`](#known-events-are-added-automatically-in-usage-data-payload)) to aggregate into metric. All events in this list must have the same `redis_slot` and `aggregation` attributes. -- feature_flag: name of [development feature flag](../feature_flags/development.md#development-type) that is checked before -metrics aggregation is performed. Corresponding feature flag should have `default_enabled` attribute set to `false`. -`feature_flag` attribute is **OPTIONAL** and can be omitted, when `feature_flag` is missing no feature flag is checked. - -Example aggregated metric entries: - -```yaml -- name: product_analytics_test_metrics_union - operator: OR - events: ['i_search_total', 'i_search_advanced', 'i_search_paid'] -- name: product_analytics_test_metrics_intersection_with_feautre_flag - operator: AND - events: ['i_search_total', 'i_search_advanced', 'i_search_paid'] - feature_flag: example_aggregated_metric -``` - -Aggregated metrics are added under `aggregated_metrics` key in both `counts_weekly` and `counts_monthly` top level keys in Usage Ping payload. - -```ruby -{ - :counts_monthly => { - :deployments => 1003, - :successful_deployments => 78, - :failed_deployments => 275, - :packages => 155, - :personal_snippets => 2106, - :project_snippets => 407, - :promoted_issues => 719, - :aggregated_metrics => { - :product_analytics_test_metrics_union => 7, - :product_analytics_test_metrics_intersection_with_feautre_flag => 2 - }, - :snippets => 2513 - } -} -``` - -## Example Usage Ping payload - -The following is example content of the Usage Ping payload. - -```json -{ - "uuid": "0000000-0000-0000-0000-000000000000", - "hostname": "example.com", - "version": "12.10.0-pre", - "installation_type": "omnibus-gitlab", - "active_user_count": 999, - "recorded_at": "2020-04-17T07:43:54.162+00:00", - "edition": "EEU", - "license_md5": "00000000000000000000000000000000", - "license_id": null, - "historical_max_users": 999, - "licensee": { - "Name": "ABC, Inc.", - "Email": "email@example.com", - "Company": "ABC, Inc." - }, - "license_user_count": 999, - "license_starts_at": "2020-01-01", - "license_expires_at": "2021-01-01", - "license_plan": "ultimate", - "license_add_ons": { - }, - "license_trial": false, - "counts": { - "assignee_lists": 999, - "boards": 999, - "ci_builds": 999, - ... - }, - "container_registry_enabled": true, - "dependency_proxy_enabled": false, - "gitlab_shared_runners_enabled": true, - "gravatar_enabled": true, - "influxdb_metrics_enabled": true, - "ldap_enabled": false, - "mattermost_enabled": false, - "omniauth_enabled": true, - "prometheus_enabled": false, - "prometheus_metrics_enabled": false, - "reply_by_email_enabled": "incoming+%{key}@incoming.gitlab.com", - "signup_enabled": true, - "web_ide_clientside_preview_enabled": true, - "ingress_modsecurity_enabled": true, - "projects_with_expiration_policy_disabled": 999, - "projects_with_expiration_policy_enabled": 999, - ... - "elasticsearch_enabled": true, - "license_trial_ends_on": null, - "geo_enabled": false, - "git": { - "version": { - "major": 2, - "minor": 26, - "patch": 1 - } - }, - "gitaly": { - "version": "12.10.0-rc1-93-g40980d40", - "servers": 56, - "clusters": 14, - "filesystems": [ - "EXT_2_3_4" - ] - }, - "gitlab_pages": { - "enabled": true, - "version": "1.17.0" - }, - "container_registry_server": { - "vendor": "gitlab", - "version": "2.9.1-gitlab" - }, - "database": { - "adapter": "postgresql", - "version": "9.6.15", - "pg_system_id": 6842684531675334351 - }, - "analytics_unique_visits": { - "g_analytics_contribution": 999, - ... - }, - "usage_activity_by_stage": { - "configure": { - "project_clusters_enabled": 999, - ... - }, - "create": { - "merge_requests": 999, - ... - }, - "manage": { - "events": 999, - ... - }, - "monitor": { - "clusters": 999, - ... - }, - "package": { - "projects_with_packages": 999 - }, - "plan": { - "issues": 999, - ... - }, - "release": { - "deployments": 999, - ... - }, - "secure": { - "user_container_scanning_jobs": 999, - ... - }, - "verify": { - "ci_builds": 999, - ... - } - }, - "usage_activity_by_stage_monthly": { - "configure": { - "project_clusters_enabled": 999, - ... - }, - "create": { - "merge_requests": 999, - ... - }, - "manage": { - "events": 999, - ... - }, - "monitor": { - "clusters": 999, - ... - }, - "package": { - "projects_with_packages": 999 - }, - "plan": { - "issues": 999, - ... - }, - "release": { - "deployments": 999, - ... - }, - "secure": { - "user_container_scanning_jobs": 999, - ... - }, - "verify": { - "ci_builds": 999, - ... - } - }, - "topology": { - "duration_s": 0.013836685999194742, - "application_requests_per_hour": 4224, - "query_apdex_weekly_average": 0.996, - "failures": [], - "nodes": [ - { - "node_memory_total_bytes": 33269903360, - "node_memory_utilization": 0.35, - "node_cpus": 16, - "node_cpu_utilization": 0.2, - "node_uname_info": { - "machine": "x86_64", - "sysname": "Linux", - "release": "4.19.76-linuxkit" - }, - "node_services": [ - { - "name": "web", - "process_count": 16, - "process_memory_pss": 233349888, - "process_memory_rss": 788220927, - "process_memory_uss": 195295487, - "server": "puma" - }, - { - "name": "sidekiq", - "process_count": 1, - "process_memory_pss": 734080000, - "process_memory_rss": 750051328, - "process_memory_uss": 731533312 - }, - ... - ], - ... - }, - ... - ] - } -} -``` - -## Notable changes - -In GitLab 13.5, `pg_system_id` was added to send the [PostgreSQL system identifier](https://www.2ndquadrant.com/en/blog/support-for-postgresqls-system-identifier-in-barman/). - -## Exporting Usage Ping SQL queries and definitions - -Two Rake tasks exist to export Usage Ping definitions. - -- The Rake tasks export the raw SQL queries for `count`, `distinct_count`, `sum`. -- The Rake tasks export the Redis counter class or the line of the Redis block for `redis_usage_data`. -- The Rake tasks calculate the `alt_usage_data` metrics. - -In the home directory of your local GitLab installation run the following Rake tasks for the YAML and JSON versions respectively: - -```shell -# for YAML export -bin/rake gitlab:usage_data:dump_sql_in_yaml - -# for JSON export -bin/rake gitlab:usage_data:dump_sql_in_json - -# You may pipe the output into a file -bin/rake gitlab:usage_data:dump_sql_in_yaml > ~/Desktop/usage-metrics-2020-09-02.yaml -``` - -## Generating and troubleshooting usage ping - -To get a usage ping, or to troubleshoot caching issues on your GitLab instance, please follow [instructions to generate usage ping](../../administration/troubleshooting/gitlab_rails_cheat_sheet.md#generate-usage-ping). +<!-- This redirect file can be deleted after February 1, 2021. --> +<!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/#move-or-rename-a-page --> diff --git a/doc/development/query_performance.md b/doc/development/query_performance.md index c61d2a0864f..3cb1b10c417 100644 --- a/doc/development/query_performance.md +++ b/doc/development/query_performance.md @@ -21,7 +21,7 @@ When you are optimizing your SQL queries, there are two dimensions to pay attent | Queries in a migration | `100ms` | This is different than the total [migration time](database_review.md#timing-guidelines-for-migrations). | | Concurrent operations in a migration | `5min` | Concurrent operations do not block the database, but they block the GitLab update. This includes operations such as `add_concurrent_index` and `add_concurrent_foreign_key`. | | Background migrations | `1s` | | -| Usage Ping | `1s` | See the [usage ping docs](product_analytics/usage_ping.md#developing-and-testing-usage-ping) for more details. | +| Usage Ping | `1s` | See the [usage ping docs](usage_ping.md#developing-and-testing-usage-ping) for more details. | - When analyzing your query's performance, pay attention to if the time you are seeing is on a [cold or warm cache](#cold-and-warm-cache). These guidelines apply for both cache types. - When working with batched queries, change the range and batch size to see how it effects the query timing and caching. diff --git a/doc/development/secure_coding_guidelines.md b/doc/development/secure_coding_guidelines.md index 44a95f6e820..bd98ea170e5 100644 --- a/doc/development/secure_coding_guidelines.md +++ b/doc/development/secure_coding_guidelines.md @@ -194,7 +194,7 @@ Go's [`regexp`](https://golang.org/pkg/regexp/) package uses `re2` and isn't vul - [Rubular](https://rubular.com/) is a nice online tool to fiddle with Ruby Regexps. - [Runaway Regular Expressions](https://www.regular-expressions.info/catastrophic.html) -- [The impact of regular expression denial of service (ReDoS) in practice: an empirical study at the ecosystem scale](http://people.cs.vt.edu/~davisjam/downloads/publications/DavisCoghlanServantLee-EcosystemREDOS-ESECFSE18.pdf). This research paper discusses approaches to automatically detect ReDoS vulnerabilities. +- [The impact of regular expression denial of service (ReDoS) in practice: an empirical study at the ecosystem scale](https://people.cs.vt.edu/~davisjam/downloads/publications/DavisCoghlanServantLee-EcosystemREDOS-ESECFSE18.pdf). This research paper discusses approaches to automatically detect ReDoS vulnerabilities. - [Freezing the web: A study of redos vulnerabilities in JavaScript-based web servers](https://www.usenix.org/system/files/conference/usenixsecurity18/sec18-staicu.pdf). Another research paper about detecting ReDoS vulnerabilities. ## Server Side Request Forgery (SSRF) diff --git a/doc/development/snowplow.md b/doc/development/snowplow.md new file mode 100644 index 00000000000..6b37936cd93 --- /dev/null +++ b/doc/development/snowplow.md @@ -0,0 +1,623 @@ +--- +stage: Growth +group: Product Intelligence +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Snowplow Guide + +This guide provides an overview of how Snowplow works, and implementation details. + +For more information about Product Intelligence, see: + +- [Product Intelligence Guide](https://about.gitlab.com/handbook/product/product-intelligence-guide/) +- [Usage Ping Guide](usage_ping.md) + +More useful links: + +- [Product Intelligence Direction](https://about.gitlab.com/direction/product-intelligence/) +- [Data Analysis Process](https://about.gitlab.com/handbook/business-ops/data-team/#data-analysis-process/) +- [Data for Product Managers](https://about.gitlab.com/handbook/business-ops/data-team/programs/data-for-product-managers/) +- [Data Infrastructure](https://about.gitlab.com/handbook/business-ops/data-team/platform/infrastructure/) + +## What is Snowplow + +Snowplow is an enterprise-grade marketing and Product Intelligence platform which helps track the way users engage with our website and application. + +[Snowplow](https://github.com/snowplow/snowplow) consists of the following loosely-coupled sub-systems: + +- **Trackers** fire Snowplow events. Snowplow has 12 trackers, covering web, mobile, desktop, server, and IoT. +- **Collectors** receive Snowplow events from trackers. We have three different event collectors, synchronizing events either to Amazon S3, Apache Kafka, or Amazon Kinesis. +- **Enrich** cleans up the raw Snowplow events, enriches them and puts them into storage. We have an Hadoop-based enrichment process, and a Kinesis-based or Kafka-based process. +- **Storage** is where the Snowplow events live. We store the Snowplow events in a flat file structure on S3, and in the Redshift and PostgreSQL databases. +- **Data modeling** is where event-level data is joined with other data sets and aggregated into smaller data sets, and business logic is applied. This produces a clean set of tables which make it easier to perform analysis on the data. We have data models for Redshift and Looker. +- **Analytics** are performed on the Snowplow events or on the aggregate tables. + + + +## Snowplow schema + +We have many definitions of Snowplow's schema. We have an active issue to [standardize this schema](https://gitlab.com/gitlab-org/gitlab/-/issues/207930) including the following definitions: + +- Frontend and backend taxonomy as listed below +- [Structured event taxonomy](#structured-event-taxonomy) +- [Self describing events](https://github.com/snowplow/snowplow/wiki/Custom-events#self-describing-events) +- [Iglu schema](https://gitlab.com/gitlab-org/iglu/) +- [Snowplow authored events](https://github.com/snowplow/snowplow/wiki/Snowplow-authored-events) + +## Enabling Snowplow + +Tracking can be enabled at: + +- The instance level, which enables tracking on both the frontend and backend layers. +- User level, though user tracking can be disabled on a per-user basis. GitLab tracking respects the [Do Not Track](https://www.eff.org/issues/do-not-track) standard, so any user who has enabled the Do Not Track option in their browser is not tracked at a user level. + +We use Snowplow for the majority of our tracking strategy and it is enabled on GitLab.com. On a self-managed instance, Snowplow can be enabled by navigating to: + +- **Admin Area > Settings > General** in the UI. +- `admin/application_settings/integrations` in your browser. + +The following configuration is required: + +| Name | Value | +|---------------|---------------------------| +| Collector | `snowplow.trx.gitlab.net` | +| Site ID | `gitlab` | +| Cookie domain | `.gitlab.com` | + +## Snowplow request flow + +The following example shows a basic request/response flow between the following components: + +- Snowplow JS / Ruby Trackers on GitLab.com +- [GitLab.com Snowplow Collector](https://gitlab.com/gitlab-com/gl-infra/readiness/-/blob/master/library/snowplow/index.md) +- The GitLab S3 Bucket +- The GitLab Snowflake Data Warehouse +- Sisense: + +```mermaid +sequenceDiagram + participant Snowplow JS (Frontend) + participant Snowplow Ruby (Backend) + participant GitLab.com Snowplow Collector + participant S3 Bucket + participant Snowflake DW + participant Sisense Dashboards + Snowplow JS (Frontend) ->> GitLab.com Snowplow Collector: FE Tracking event + Snowplow Ruby (Backend) ->> GitLab.com Snowplow Collector: BE Tracking event + loop Process using Kinesis Stream + GitLab.com Snowplow Collector ->> GitLab.com Snowplow Collector: Log raw events + GitLab.com Snowplow Collector ->> GitLab.com Snowplow Collector: Enrich events + GitLab.com Snowplow Collector ->> GitLab.com Snowplow Collector: Write to disk + end + GitLab.com Snowplow Collector ->> S3 Bucket: Kinesis Firehose + S3 Bucket->>Snowflake DW: Import data + Snowflake DW->>Snowflake DW: Transform data using dbt + Snowflake DW->>Sisense Dashboards: Data available for querying +``` + +## Structured event taxonomy + +When adding new click events, we should add them in a way that's internally consistent. If we don't, it is very painful to perform analysis across features since each feature captures events differently. + +The current method provides several attributes that are sent on each click event. Please try to follow these guidelines when specifying events to capture: + +| attribute | type | required | description | +| --------- | ------- | -------- | ----------- | +| category | text | true | The page or backend area of the application. Unless infeasible, please use the Rails page attribute by default in the frontend, and namespace + classname on the backend. | +| action | text | true | The action the user is taking, or aspect that's being instrumented. The first word should always describe the action or aspect: clicks should be `click`, activations should be `activate`, creations should be `create`, etc. Use underscores to describe what was acted on; for example, activating a form field would be `activate_form_input`. An interface action like clicking on a dropdown would be `click_dropdown`, while a behavior like creating a project record from the backend would be `create_project` | +| label | text | false | The specific element, or object that's being acted on. This is either the label of the element (e.g. a tab labeled 'Create from template' may be `create_from_template`) or a unique identifier if no text is available (e.g. closing the Groups dropdown in the top navbar might be `groups_dropdown_close`), or it could be the name or title attribute of a record being created. | +| property | text | false | Any additional property of the element, or object being acted on. | +| value | decimal | false | Describes a numeric value or something directly related to the event. This could be the value of an input (e.g. `10` when clicking `internal` visibility). | + +### Web-specific parameters + +Snowplow JS adds many [web-specific parameters](https://docs.snowplowanalytics.com/docs/collecting-data/collecting-from-own-applications/snowplow-tracker-protocol/#Web-specific_parameters) to all web events by default. + +## Implementing Snowplow JS (Frontend) tracking + +GitLab provides `Tracking`, an interface that wraps the [Snowplow JavaScript Tracker](https://github.com/snowplow/snowplow/wiki/javascript-tracker) for tracking custom events. There are a few ways to use tracking, but each generally requires at minimum, a `category` and an `action`. Additional data can be provided that adheres to our [Structured event taxonomy](#structured-event-taxonomy). + +| field | type | default value | description | +|:-----------|:-------|:---------------------------|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `category` | string | document.body.dataset.page | Page or subsection of a page that events are being captured within. | +| `action` | string | 'generic' | Action the user is taking. Clicks should be `click` and activations should be `activate`, so for example, focusing a form field would be `activate_form_input`, and clicking a button would be `click_button`. | +| `data` | object | {} | Additional data such as `label`, `property`, `value`, and `context` as described in our [Structured event taxonomy](#structured-event-taxonomy). | + +### Tracking in HAML (or Vue Templates) + +When working within HAML (or Vue templates) we can add `data-track-*` attributes to elements of interest. All elements that have a `data-track-event` attribute automatically have event tracking bound on clicks. + +Below is an example of `data-track-*` attributes assigned to a button: + +```haml +%button.btn{ data: { track: { event: "click_button", label: "template_preview", property: "my-template" } } } +``` + +```html +<button class="btn" + data-track-event="click_button" + data-track-label="template_preview" + data-track-property="my-template" +/> +``` + +Event listeners are bound at the document level to handle click events on or within elements with these data attributes. This allows them to be properly handled on re-rendering and changes to the DOM. Note that because of the way these events are bound, click events should not be stopped from propagating up the DOM tree. If for any reason click events are being stopped from propagating, you need to implement your own listeners and follow the instructions in [Tracking in raw JavaScript](#tracking-in-raw-javascript). + +Below is a list of supported `data-track-*` attributes: + +| attribute | required | description | +|:----------------------|:---------|:------------| +| `data-track-event` | true | Action the user is taking. Clicks must be prepended with `click` and activations must be prepended with `activate`. For example, focusing a form field would be `activate_form_input` and clicking a button would be `click_button`. | +| `data-track-label` | false | The `label` as described in our [Structured event taxonomy](#structured-event-taxonomy). | +| `data-track-property` | false | The `property` as described in our [Structured event taxonomy](#structured-event-taxonomy). | +| `data-track-value` | false | The `value` as described in our [Structured event taxonomy](#structured-event-taxonomy). If omitted, this is the element's `value` property or an empty string. For checkboxes, the default value is the element's checked attribute or `false` when unchecked. | +| `data-track-context` | false | The `context` as described in our [Structured event taxonomy](#structured-event-taxonomy). | + +#### Caveats + +When using the GitLab helper method [`nav_link`](https://gitlab.com/gitlab-org/gitlab/-/blob/898b286de322e5df6a38d257b10c94974d580df8/app/helpers/tab_helper.rb#L69) be sure to wrap `html_options` under the `html_options` keyword argument. +Be careful, as this behavior can be confused with the `ActionView` helper method [`link_to`](https://api.rubyonrails.org/v5.2.3/classes/ActionView/Helpers/UrlHelper.html#method-i-link_to) that does not require additional wrapping of `html_options` + +`nav_link(controller: ['dashboard/groups', 'explore/groups'], html_options: { data: { track_label: "groups_dropdown", track_event: "click_dropdown" } })` + +vs + +`link_to assigned_issues_dashboard_path, title: _('Issues'), data: { track_label: 'main_navigation', track_event: 'click_issues_link' }` + +### Tracking within Vue components + +There's a tracking Vue mixin that can be used in components if more complex tracking is required. To use it, first import the `Tracking` library and request a mixin. + +```javascript +import Tracking from '~/tracking'; +const trackingMixin = Tracking.mixin({ label: 'right_sidebar' }); +``` + +You can provide default options that are passed along whenever an event is tracked from within your component. For instance, if all events within a component should be tracked with a given `label`, you can provide one at this time. Available defaults are `category`, `label`, `property`, and `value`. If no category is specified, `document.body.dataset.page` is used as the default. + +You can then use the mixin normally in your component with the `mixin` Vue declaration. The mixin also provides the ability to specify tracking options in `data` or `computed`. These override any defaults and allow the values to be dynamic from props, or based on state. + +```javascript +export default { + mixins: [trackingMixin], + // ...[component implementation]... + data() { + return { + expanded: false, + tracking: { + label: 'left_sidebar' + } + }; + }, +} +``` + +The mixin provides a `track` method that can be called within the template, or from component methods. An example of the whole implementation might look like the following. + +```javascript +export default { + mixins: [Tracking.mixin({ label: 'right_sidebar' })], + data() { + return { + expanded: false, + }; + }, + methods: { + toggle() { + this.expanded = !this.expanded; + this.track('click_toggle', { value: this.expanded }) + } + } +}; +``` + +And if needed within the template, you can use the `track` method directly as well. + +```html +<template> + <div> + <a class="toggle" @click.prevent="toggle">Toggle</a> + <div v-if="expanded"> + <p>Hello world!</p> + <a @click.prevent="track('click_action')">Track an event</a> + </div> + </div> +</template> +``` + +### Tracking in raw JavaScript + +Custom event tracking and instrumentation can be added by directly calling the `Tracking.event` static function. The following example demonstrates tracking a click on a button by calling `Tracking.event` manually. + +```javascript +import Tracking from '~/tracking'; + +const button = document.getElementById('create_from_template_button'); +button.addEventListener('click', () => { + Tracking.event('dashboard:projects:index', 'click_button', { + label: 'create_from_template', + property: 'template_preview', + value: 'rails', + }); +}) +``` + +### Tests and test helpers + +In Jest particularly in Vue tests, you can use the following: + +```javascript +import { mockTracking } from 'helpers/tracking_helper'; + +describe('MyTracking', () => { + let spy; + + beforeEach(() => { + spy = mockTracking('_category_', wrapper.element, jest.spyOn); + }); + + it('tracks an event when clicked on feedback', () => { + wrapper.find('.discover-feedback-icon').trigger('click'); + + expect(spy).toHaveBeenCalledWith('_category_', 'click_button', { + label: 'security-discover-feedback-cta', + property: '0', + }); + }); +}); +``` + +In obsolete Karma tests it's used as below: + +```javascript +import { mockTracking, triggerEvent } from 'spec/helpers/tracking_helper'; + +describe('my component', () => { + let trackingSpy; + + beforeEach(() => { + trackingSpy = mockTracking('_category_', vm.$el, spyOn); + }); + + const triggerEvent = () => { + // action which should trigger a event + }; + + it('tracks an event when toggled', () => { + expect(trackingSpy).not.toHaveBeenCalled(); + + triggerEvent('a.toggle'); + + expect(trackingSpy).toHaveBeenCalledWith('_category_', 'click_edit_button', { + label: 'right_sidebar', + property: 'confidentiality', + }); + }); +}); +``` + +## Implementing Snowplow Ruby (Backend) tracking + +GitLab provides `Gitlab::Tracking`, an interface that wraps the [Snowplow Ruby Tracker](https://github.com/snowplow/snowplow/wiki/ruby-tracker) for tracking custom events. + +Custom event tracking and instrumentation can be added by directly calling the `GitLab::Tracking.event` class method, which accepts the following arguments: + +| argument | type | default value | description | +|:-----------|:-------|:--------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `category` | string | 'application' | Area or aspect of the application. This could be `HealthCheckController` or `Lfs::FileTransformer` for instance. | +| `action` | string | 'generic' | The action being taken, which can be anything from a controller action like `create` to something like an Active Record callback. | +| `data` | object | {} | Additional data such as `label`, `property`, `value`, and `context` as described in [Structured event taxonomy](#structured-event-taxonomy). These are set as empty strings if you don't provide them. | + +Tracking can be viewed as either tracking user behavior, or can be used for instrumentation to monitor and visualize performance over time in an area or aspect of code. + +For example: + +```ruby +class Projects::CreateService < BaseService + def execute + project = Project.create(params) + + Gitlab::Tracking.event('Projects::CreateService', 'create_project', + label: project.errors.full_messages.to_sentence, + value: project.valid? + ) + end +end +``` + +### Unit testing + +Use the `expect_snowplow_event` helper when testing backend Snowplow events. See [testing best practices]( +https://docs.gitlab.com/ee/development/testing_guide/best_practices.html#test-snowplow-events) for details. + +### Performance + +We use the [AsyncEmitter](https://github.com/snowplow/snowplow/wiki/Ruby-Tracker#52-the-asyncemitter-class) when tracking events, which allows for instrumentation calls to be run in a background thread. This is still an active area of development. + +## Developing and testing Snowplow + +There are several tools for developing and testing Snowplow Event + +| Testing Tool | Frontend Tracking | Backend Tracking | Local Development Environment | Production Environment | Production Environment | +|----------------------------------------------|--------------------|---------------------|-------------------------------|------------------------|------------------------| +| Snowplow Analytics Debugger Chrome Extension | **{check-circle}** | **{dotted-circle}** | **{check-circle}** | **{check-circle}** | **{check-circle}** | +| Snowplow Inspector Chrome Extension | **{check-circle}** | **{dotted-circle}** | **{check-circle}** | **{check-circle}** | **{check-circle}** | +| Snowplow Micro | **{check-circle}** | **{check-circle}** | **{check-circle}** | **{dotted-circle}** | **{dotted-circle}** | +| Snowplow Mini | **{check-circle}** | **{check-circle}** | **{dotted-circle}** | **{status_preparing}** | **{status_preparing}** | + +**Legend** + +**{check-circle}** Available, **{status_preparing}** In progress, **{dotted-circle}** Not Planned + +### Preparing your MR for Review + +1. For frontend events, in the MR description section, add a screenshot of the event's relevant section using the [Snowplow Analytics Debugger](https://chrome.google.com/webstore/detail/snowplow-analytics-debugg/jbnlcgeengmijcghameodeaenefieedm) Chrome browser extension. +1. For backend events, please use Snowplow Micro and add the output of the Snowplow Micro good events `GET http://localhost:9090/micro/good`. + +### Snowplow Analytics Debugger Chrome Extension + +Snowplow Analytics Debugger is a browser extension for testing frontend events. This works on production, staging and local development environments. + +1. Install the [Snowplow Analytics Debugger](https://chrome.google.com/webstore/detail/snowplow-analytics-debugg/jbnlcgeengmijcghameodeaenefieedm) Chrome browser extension. +1. Open Chrome DevTools to the Snowplow Analytics Debugger tab. +1. Learn more at [Igloo Analytics](https://www.iglooanalytics.com/blog/snowplow-analytics-debugger-chrome-extension.html). + +### Snowplow Inspector Chrome Extension + +Snowplow Inspector Chrome Extension is a browser extension for testing frontend events. This works on production, staging and local development environments. + +1. Install [Snowplow Inspector](https://chrome.google.com/webstore/detail/snowplow-inspector/maplkdomeamdlngconidoefjpogkmljm?hl=en). +1. Open the Chrome extension by pressing the Snowplow Inspector icon beside the address bar. +1. Click around on a webpage with Snowplow and you should see JavaScript events firing in the inspector window. + +### Snowplow Micro + +Snowplow Micro is a very small version of a full Snowplow data collection pipeline: small enough that it can be launched by a test suite. Events can be recorded into Snowplow Micro just as they can a full Snowplow pipeline. Micro then exposes an API that can be queried. + +Snowplow Micro is a Docker-based solution for testing frontend and backend events in a local development environment. You need to modify GDK using the instructions below to set this up. + +- Read [Introducing Snowplow Micro](https://snowplowanalytics.com/blog/2019/07/17/introducing-snowplow-micro/) +- Look at the [Snowplow Micro repository](https://github.com/snowplow-incubator/snowplow-micro) +- Watch our [installation guide recording](https://www.youtube.com/watch?v=OX46fo_A0Ag) + +1. Ensure Docker is installed and running. + +1. Install [Snowplow Micro](https://github.com/snowplow-incubator/snowplow-micro) by cloning the settings in [this project](https://gitlab.com/gitlab-org/snowplow-micro-configuration): +1. Navigate to the directory with the cloned project, and start the appropriate Docker + container with the following script: + + ```shell + ./snowplow-micro.sh + ``` + +1. Update your instance's settings to enable Snowplow events and point to the Snowplow Micro collector: + + ```shell + gdk psql -d gitlabhq_development + update application_settings set snowplow_collector_hostname='localhost:9090', snowplow_enabled=true, snowplow_cookie_domain='.gitlab.com'; + ``` + +1. Update `DEFAULT_SNOWPLOW_OPTIONS` in `app/assets/javascripts/tracking.js` to remove `forceSecureTracker: true`: + + ```diff + diff --git a/app/assets/javascripts/tracking.js b/app/assets/javascripts/tracking.js + index 0a1211d0a76..3b98c8f28f2 100644 + --- a/app/assets/javascripts/tracking.js + +++ b/app/assets/javascripts/tracking.js + @@ -7,7 +7,6 @@ const DEFAULT_SNOWPLOW_OPTIONS = { + appId: '', + userFingerprint: false, + respectDoNotTrack: true, + - forceSecureTracker: true, + eventMethod: 'post', + contexts: { webPage: true, performanceTiming: true }, + formTracking: false, + + ``` + +1. Update `snowplow_options` in `lib/gitlab/tracking.rb` to add `protocol` and `port`: + + ```diff + diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb + index 618e359211b..e9084623c43 100644 + --- a/lib/gitlab/tracking.rb + +++ b/lib/gitlab/tracking.rb + @@ -41,7 +41,9 @@ def snowplow_options(group) + cookie_domain: Gitlab::CurrentSettings.snowplow_cookie_domain, + app_id: Gitlab::CurrentSettings.snowplow_app_id, + form_tracking: additional_features, + - link_click_tracking: additional_features + + link_click_tracking: additional_features, + + protocol: 'http', + + port: 9090 + }.transform_keys! { |key| key.to_s.camelize(:lower).to_sym } + end + ``` + +1. Update `emitter` in `lib/gitlab/tracking/destinations/snowplow.rb` to change `protocol`: + + ```diff + diff --git a/lib/gitlab/tracking/destinations/snowplow.rb b/lib/gitlab/tracking/destinations/snowplow.rb + index 4fa844de325..5dd9d0eacfb 100644 + --- a/lib/gitlab/tracking/destinations/snowplow.rb + +++ b/lib/gitlab/tracking/destinations/snowplow.rb + @@ -40,7 +40,7 @@ def tracker + def emitter + SnowplowTracker::AsyncEmitter.new( + Gitlab::CurrentSettings.snowplow_collector_hostname, + - protocol: 'https' + + protocol: 'http' + ) + end + end + + ``` + +1. Restart GDK: + + ```shell + `gdk restart` + ``` + +1. Send a test Snowplow event from the Rails console: + + ```ruby + Gitlab::Tracking.self_describing_event('iglu:com.gitlab/pageview_context/jsonschema/1-0-0', data: { page_type: 'MY_TYPE' }, context: nil) + ``` + +1. Navigate to `localhost:9090/micro/good` to see the event. + +### Snowplow Mini + +[Snowplow Mini](https://github.com/snowplow/snowplow-mini) is an easily-deployable, single-instance version of Snowplow. + +Snowplow Mini can be used for testing frontend and backend events on a production, staging and local development environment. + +For GitLab.com, we're setting up a [QA and Testing environment](https://gitlab.com/gitlab-org/telemetry/-/issues/266) using Snowplow Mini. + +## Snowplow Schemas + +### [gitlab_standard](https://gitlab.com/gitlab-org/iglu/-/blob/master/public/schemas/com.gitlab/gitlab_standard/jsonschema/1-0-0) Schema + +| Field Name | Required | Type | Description | +|--------------|---------------------|---------|--------------------------------| +| project_id | **{dotted-circle}** | integer | ID of the associated project | +| namespace_id | **{dotted-circle}** | integer | ID of the associated namespace | + +### Default Schema + +| Field Name | Required | Type | Description | +|--------------------------|---------------------|-----------|----------------------------------------------------------------------------------------------------------------------------------| +| app_id | **{check-circle}** | string | Unique identifier for website / application | +| base_currency | **{dotted-circle}** | string | Reporting currency | +| br_colordepth | **{dotted-circle}** | integer | Browser color depth | +| br_cookies | **{dotted-circle}** | boolean | Does the browser permit cookies? | +| br_family | **{dotted-circle}** | string | Browser family | +| br_features_director | **{dotted-circle}** | boolean | Director plugin installed? | +| br_features_flash | **{dotted-circle}** | boolean | Flash plugin installed? | +| br_features_gears | **{dotted-circle}** | boolean | Google gears installed? | +| br_features_java | **{dotted-circle}** | boolean | Java plugin installed? | +| br_features_pdf | **{dotted-circle}** | boolean | Adobe PDF plugin installed? | +| br_features_quicktime | **{dotted-circle}** | boolean | Quicktime plugin installed? | +| br_features_realplayer | **{dotted-circle}** | boolean | Realplayer plugin installed? | +| br_features_silverlight | **{dotted-circle}** | boolean | Silverlight plugin installed? | +| br_features_windowsmedia | **{dotted-circle}** | boolean | Windows media plugin installed? | +| br_lang | **{dotted-circle}** | string | Language the browser is set to | +| br_name | **{dotted-circle}** | string | Browser name | +| br_renderengine | **{dotted-circle}** | string | Browser rendering engine | +| br_type | **{dotted-circle}** | string | Browser type | +| br_version | **{dotted-circle}** | string | Browser version | +| br_viewheight | **{dotted-circle}** | string | Browser viewport height | +| br_viewwidth | **{dotted-circle}** | string | Browser viewport width | +| collector_tstamp | **{dotted-circle}** | timestamp | Time stamp for the event recorded by the collector | +| contexts | **{dotted-circle}** | | | +| derived_contexts | **{dotted-circle}** | | Contexts derived in the Enrich process | +| derived_tstamp | **{dotted-circle}** | timestamp | Timestamp making allowance for innaccurate device clock | +| doc_charset | **{dotted-circle}** | string | Web page’s character encoding | +| doc_height | **{dotted-circle}** | string | Web page height | +| doc_width | **{dotted-circle}** | string | Web page width | +| domain_sessionid | **{dotted-circle}** | string | Unique identifier (UUID) for this visit of this user_id to this domain | +| domain_sessionidx | **{dotted-circle}** | integer | Index of number of visits that this user_id has made to this domain (The first visit is `1`) | +| domain_userid | **{dotted-circle}** | string | Unique identifier for a user, based on a first party cookie (so domain specific) | +| dvce_created_tstamp | **{dotted-circle}** | timestamp | Timestamp when event occurred, as recorded by client device | +| dvce_ismobile | **{dotted-circle}** | boolean | Indicates whether device is mobile | +| dvce_screenheight | **{dotted-circle}** | string | Screen / monitor resolution | +| dvce_screenwidth | **{dotted-circle}** | string | Screen / monitor resolution | +| dvce_sent_tstamp | **{dotted-circle}** | timestamp | Timestamp when event was sent by client device to collector | +| dvce_type | **{dotted-circle}** | string | Type of device | +| etl_tags | **{dotted-circle}** | string | JSON of tags for this ETL run | +| etl_tstamp | **{dotted-circle}** | timestamp | Timestamp event began ETL | +| event | **{dotted-circle}** | string | Event type | +| event_fingerprint | **{dotted-circle}** | string | Hash client-set event fields | +| event_format | **{dotted-circle}** | string | Format for event | +| event_id | **{dotted-circle}** | string | Event UUID | +| event_name | **{dotted-circle}** | string | Event name | +| event_vendor | **{dotted-circle}** | string | The company who developed the event model | +| event_version | **{dotted-circle}** | string | Version of event schema | +| geo_city | **{dotted-circle}** | string | City of IP origin | +| geo_country | **{dotted-circle}** | string | Country of IP origin | +| geo_latitude | **{dotted-circle}** | string | An approximate latitude | +| geo_longitude | **{dotted-circle}** | string | An approximate longitude | +| geo_region | **{dotted-circle}** | string | Region of IP origin | +| geo_region_name | **{dotted-circle}** | string | Region of IP origin | +| geo_timezone | **{dotted-circle}** | string | Timezone of IP origin | +| geo_zipcode | **{dotted-circle}** | string | Zip (postal) code of IP origin | +| ip_domain | **{dotted-circle}** | string | Second level domain name associated with the visitor’s IP address | +| ip_isp | **{dotted-circle}** | string | Visitor’s ISP | +| ip_netspeed | **{dotted-circle}** | string | Visitor’s connection type | +| ip_organization | **{dotted-circle}** | string | Organization associated with the visitor’s IP address – defaults to ISP name if none is found | +| mkt_campaign | **{dotted-circle}** | string | The campaign ID | +| mkt_clickid | **{dotted-circle}** | string | The click ID | +| mkt_content | **{dotted-circle}** | string | The content or ID of the ad. | +| mkt_medium | **{dotted-circle}** | string | Type of traffic source | +| mkt_network | **{dotted-circle}** | string | The ad network to which the click ID belongs | +| mkt_source | **{dotted-circle}** | string | The company / website where the traffic came from | +| mkt_term | **{dotted-circle}** | string | Keywords associated with the referrer | +| name_tracker | **{dotted-circle}** | string | The tracker namespace | +| network_userid | **{dotted-circle}** | string | Unique identifier for a user, based on a cookie from the collector (so set at a network level and shouldn’t be set by a tracker) | +| os_family | **{dotted-circle}** | string | Operating system family | +| os_manufacturer | **{dotted-circle}** | string | Manufacturers of operating system | +| os_name | **{dotted-circle}** | string | Name of operating system | +| os_timezone | **{dotted-circle}** | string | Client operating system timezone | +| page_referrer | **{dotted-circle}** | string | Referrer URL | +| page_title | **{dotted-circle}** | string | Page title | +| page_url | **{dotted-circle}** | string | Page URL | +| page_urlfragment | **{dotted-circle}** | string | Fragment aka anchor | +| page_urlhost | **{dotted-circle}** | string | Host aka domain | +| page_urlpath | **{dotted-circle}** | string | Path to page | +| page_urlport | **{dotted-circle}** | integer | Port if specified, 80 if not | +| page_urlquery | **{dotted-circle}** | string | Query string | +| page_urlscheme | **{dotted-circle}** | string | Scheme (protocol name) | +| platform | **{dotted-circle}** | string | The platform the app runs on | +| pp_xoffset_max | **{dotted-circle}** | integer | Maximum page x offset seen in the last ping period | +| pp_xoffset_min | **{dotted-circle}** | integer | Minimum page x offset seen in the last ping period | +| pp_yoffset_max | **{dotted-circle}** | integer | Maximum page y offset seen in the last ping period | +| pp_yoffset_min | **{dotted-circle}** | integer | Minimum page y offset seen in the last ping period | +| refr_domain_userid | **{dotted-circle}** | string | The Snowplow domain_userid of the referring website | +| refr_dvce_tstamp | **{dotted-circle}** | timestamp | The time of attaching the domain_userid to the inbound link | +| refr_medium | **{dotted-circle}** | string | Type of referer | +| refr_source | **{dotted-circle}** | string | Name of referer if recognised | +| refr_term | **{dotted-circle}** | string | Keywords if source is a search engine | +| refr_urlfragment | **{dotted-circle}** | string | Referer URL fragment | +| refr_urlhost | **{dotted-circle}** | string | Referer host | +| refr_urlpath | **{dotted-circle}** | string | Referer page path | +| refr_urlport | **{dotted-circle}** | integer | Referer port | +| refr_urlquery | **{dotted-circle}** | string | Referer URL querystring | +| refr_urlscheme | **{dotted-circle}** | string | Referer scheme | +| se_action | **{dotted-circle}** | string | The action / event itself | +| se_category | **{dotted-circle}** | string | The category of event | +| se_label | **{dotted-circle}** | string | A label often used to refer to the ‘object’ the action is performed on | +| se_property | **{dotted-circle}** | string | A property associated with either the action or the object | +| se_value | **{dotted-circle}** | decimal | A value associated with the user action | +| ti_category | **{dotted-circle}** | string | Item category | +| ti_currency | **{dotted-circle}** | string | Currency | +| ti_name | **{dotted-circle}** | string | Item name | +| ti_orderid | **{dotted-circle}** | string | Order ID | +| ti_price | **{dotted-circle}** | decimal | Item price | +| ti_price_base | **{dotted-circle}** | decimal | Item price in base currency | +| ti_quantity | **{dotted-circle}** | integer | Item quantity | +| ti_sku | **{dotted-circle}** | string | Item SKU | +| tr_affiliation | **{dotted-circle}** | string | Transaction affiliation (such as channel) | +| tr_city | **{dotted-circle}** | string | Delivery address: city | +| tr_country | **{dotted-circle}** | string | Delivery address: country | +| tr_currency | **{dotted-circle}** | string | Transaction Currency | +| tr_orderid | **{dotted-circle}** | string | Order ID | +| tr_shipping | **{dotted-circle}** | decimal | Delivery cost charged | +| tr_shipping_base | **{dotted-circle}** | decimal | Shipping cost in base currency | +| tr_state | **{dotted-circle}** | string | Delivery address: state | +| tr_tax | **{dotted-circle}** | decimal | Transaction tax value (such as amount of VAT included) | +| tr_tax_base | **{dotted-circle}** | decimal | Tax applied in base currency | +| tr_total | **{dotted-circle}** | decimal | Transaction total value | +| tr_total_base | **{dotted-circle}** | decimal | Total amount of transaction in base currency | +| true_tstamp | **{dotted-circle}** | timestamp | User-set exact timestamp | +| txn_id | **{dotted-circle}** | string | Transaction ID | +| unstruct_event | **{dotted-circle}** | JSON | The properties of the event | +| uploaded_at | **{dotted-circle}** | | | +| user_fingerprint | **{dotted-circle}** | integer | User identifier based on (hopefully unique) browser features | +| user_id | **{dotted-circle}** | string | Unique identifier for user, set by the business using setUserId | +| user_ipaddress | **{dotted-circle}** | string | IP address | +| useragent | **{dotted-circle}** | string | User agent (expressed as a browser string) | +| v_collector | **{dotted-circle}** | string | Collector version | +| v_etl | **{dotted-circle}** | string | ETL version | +| v_tracker | **{dotted-circle}** | string | Identifier for Snowplow tracker | diff --git a/doc/development/stage_group_dashboards.md b/doc/development/stage_group_dashboards.md index 15457eb2e7e..453d71411c3 100644 --- a/doc/development/stage_group_dashboards.md +++ b/doc/development/stage_group_dashboards.md @@ -75,7 +75,7 @@ All the dashboards are powered by [Grafana](https://grafana.com/), a frontend fo  -- [Sentry](https://sentry.gitlab.net/gitlab/gitlabcom) gives us a way to find the exception where we can filter by transaction type and correlation_id from a Kibana's result item. +- [Sentry](https://sentry.gitlab.net/gitlab/gitlabcom/) gives us a way to find the exception where we can filter by transaction type and correlation_id from a Kibana's result item.  diff --git a/doc/development/telemetry/event_dictionary.md b/doc/development/telemetry/event_dictionary.md index bc230a46441..b3b3b0b4fdd 100644 --- a/doc/development/telemetry/event_dictionary.md +++ b/doc/development/telemetry/event_dictionary.md @@ -1,8 +1,8 @@ --- -redirect_to: '../product_analytics/event_dictionary.md' +redirect_to: 'https://about.gitlab.com/handbook/product/product-intelligence-guide/' --- -This document was moved to [another location](../product_analytics/event_dictionary.md). +This document was moved to [another location](https://about.gitlab.com/handbook/product/product-intelligence-guide/). <!-- This redirect file can be deleted after February 1, 2021. --> <!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/#move-or-rename-a-page --> diff --git a/doc/development/telemetry/index.md b/doc/development/telemetry/index.md index 24e83ffc524..b3b3b0b4fdd 100644 --- a/doc/development/telemetry/index.md +++ b/doc/development/telemetry/index.md @@ -1,8 +1,8 @@ --- -redirect_to: '../product_analytics/index.md' +redirect_to: 'https://about.gitlab.com/handbook/product/product-intelligence-guide/' --- -This document was moved to [another location](../product_analytics/index.md). +This document was moved to [another location](https://about.gitlab.com/handbook/product/product-intelligence-guide/). <!-- This redirect file can be deleted after February 1, 2021. --> <!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/#move-or-rename-a-page --> diff --git a/doc/development/telemetry/snowplow.md b/doc/development/telemetry/snowplow.md index 7cd385be681..bb056ffddfe 100644 --- a/doc/development/telemetry/snowplow.md +++ b/doc/development/telemetry/snowplow.md @@ -1,8 +1,8 @@ --- -redirect_to: '../product_analytics/snowplow.md' +redirect_to: '../snowplow.md' --- -This document was moved to [another location](../product_analytics/snowplow.md). +This document was moved to [another location](../snowplow.md). <!-- This redirect file can be deleted after February 1, 2021. --> <!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/#move-or-rename-a-page --> diff --git a/doc/development/telemetry/usage_ping.md b/doc/development/telemetry/usage_ping.md index c890353fe3b..5fbdb508bb1 100644 --- a/doc/development/telemetry/usage_ping.md +++ b/doc/development/telemetry/usage_ping.md @@ -1,8 +1,8 @@ --- -redirect_to: '../product_analytics/usage_ping.md' +redirect_to: '../usage_ping.md' --- -This document was moved to [another location](../product_analytics/usage_ping.md). +This document was moved to [another location](../usage_ping.md). <!-- This redirect file can be deleted after February 1, 2021. --> <!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/#move-or-rename-a-page --> diff --git a/doc/development/usage_ping.md b/doc/development/usage_ping.md new file mode 100644 index 00000000000..cb38fb9802e --- /dev/null +++ b/doc/development/usage_ping.md @@ -0,0 +1,1065 @@ +--- +stage: Growth +group: Product Intelligence +info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments +--- + +# Usage Ping Guide + +> - Introduced in GitLab Enterprise Edition 8.10. +> - More statistics were added in GitLab Enterprise Edition 8.12. +> - Moved to GitLab Core in 9.1. +> - More statistics were added in GitLab Ultimate 11.2. + +This guide describes Usage Ping's purpose and how it's implemented. + +For more information about Product Intelligence, see: + +- [Product Intelligence Guide](https://about.gitlab.com/handbook/product/product-intelligence-guide/) +- [Snowplow Guide](snowplow.md) + +More useful links: + +- [Product Intelligence Direction](https://about.gitlab.com/direction/product-intelligence/) +- [Data Analysis Process](https://about.gitlab.com/handbook/business-ops/data-team/#data-analysis-process/) +- [Data for Product Managers](https://about.gitlab.com/handbook/business-ops/data-team/programs/data-for-product-managers/) +- [Data Infrastructure](https://about.gitlab.com/handbook/business-ops/data-team/platform/infrastructure/) + +## What is Usage Ping? + +- GitLab sends a weekly payload containing usage data to GitLab Inc. Usage Ping provides high-level data to help our product, support, and sales teams. It does not send any project names, usernames, or any other specific data. The information from the usage ping is not anonymous, it is linked to the hostname of the instance. Sending usage ping is optional, and any instance can disable analytics. +- The usage data is primarily composed of row counts for different tables in the instance’s database. By comparing these counts month over month (or week over week), we can get a rough sense for how an instance is using the different features within the product. In addition to counts, other facts + that help us classify and understand GitLab installations are collected. +- Usage ping is important to GitLab as we use it to calculate our Stage Monthly Active Users (SMAU) which helps us measure the success of our stages and features. +- While usage ping is enabled, GitLab gathers data from the other instances and can show usage statistics of your instance to your users. + +### Why should we enable Usage Ping? + +- The main purpose of Usage Ping is to build a better GitLab. Data about how GitLab is used is collected to better understand feature/stage adoption and usage, which helps us understand how GitLab is adding value and helps our team better understand the reasons why people use GitLab and with this knowledge we're able to make better product decisions. +- As a benefit of having the usage ping active, GitLab lets you analyze the users’ activities over time of your GitLab installation. +- As a benefit of having the usage ping active, GitLab provides you with The DevOps Report,which gives you an overview of your entire instance’s adoption of Concurrent DevOps from planning to monitoring. +- You get better, more proactive support. (assuming that our TAMs and support organization used the data to deliver more value) +- You get insight and advice into how to get the most value out of your investment in GitLab. Wouldn't you want to know that a number of features or values are not being adopted in your organization? +- You get a report that illustrates how you compare against other similar organizations (anonymized), with specific advice and recommendations on how to improve your DevOps processes. +- Usage Ping is enabled by default. To disable it, see [Disable Usage Ping](#disable-usage-ping). + +### Limitations + +- Usage Ping does not track frontend events things like page views, link clicks, or user sessions, and only focuses on aggregated backend events. +- Because of these limitations we recommend instrumenting your products with Snowplow for more detailed analytics on GitLab.com and use Usage Ping to track aggregated backend events on self-managed. + +## Usage Ping payload + +You can view the exact JSON payload sent to GitLab Inc. in the administration panel. To view the payload: + +1. Navigate to **Admin Area > Settings > Metrics and profiling**. +1. Expand the **Usage statistics** section. +1. Click the **Preview payload** button. + +For an example payload, see [Example Usage Ping payload](#example-usage-ping-payload). + +## Disable Usage Ping + +To disable Usage Ping in the GitLab UI, go to the **Settings** page of your administration panel and uncheck the **Usage Ping** checkbox. + +To disable Usage Ping and prevent it from being configured in the future through the administration panel, Omnibus installs can set the following in [`gitlab.rb`](https://docs.gitlab.com/omnibus/settings/configuration.html#configuration-options): + +```ruby +gitlab_rails['usage_ping_enabled'] = false +``` + +Source installations can set the following in `gitlab.yml`: + +```yaml +production: &base + # ... + gitlab: + # ... + usage_ping_enabled: false +``` + +## Usage Ping request flow + +The following example shows a basic request/response flow between a GitLab instance, the Versions Application, the License Application, Salesforce, the GitLab S3 Bucket, the GitLab Snowflake Data Warehouse, and Sisense: + +```mermaid +sequenceDiagram + participant GitLab Instance + participant Versions Application + participant Licenses Application + participant Salesforce + participant S3 Bucket + participant Snowflake DW + participant Sisense Dashboards + GitLab Instance->>Versions Application: Send Usage Ping + loop Process usage data + Versions Application->>Versions Application: Parse usage data + Versions Application->>Versions Application: Write to database + Versions Application->>Versions Application: Update license ping time + end + loop Process data for Salesforce + Versions Application-xLicenses Application: Request Zuora subscription id + Licenses Application-xVersions Application: Zuora subscription id + Versions Application-xSalesforce: Request Zuora account id by Zuora subscription id + Salesforce-xVersions Application: Zuora account id + Versions Application-xSalesforce: Usage data for the Zuora account + end + Versions Application->>S3 Bucket: Export Versions database + S3 Bucket->>Snowflake DW: Import data + Snowflake DW->>Snowflake DW: Transform data using dbt + Snowflake DW->>Sisense Dashboards: Data available for querying + Versions Application->>GitLab Instance: DevOps Report (Conversational Development Index) +``` + +## How Usage Ping works + +1. The Usage Ping [cron job](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/workers/gitlab_usage_ping_worker.rb#L30) is set in Sidekiq to run weekly. +1. When the cron job runs, it calls [`Gitlab::UsageData.to_json`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/services/submit_usage_ping_service.rb#L22). +1. `Gitlab::UsageData.to_json` [cascades down](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data.rb#L22) to ~400+ other counter method calls. +1. The response of all methods calls are [merged together](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data.rb#L14) into a single JSON payload in `Gitlab::UsageData.to_json`. +1. The JSON payload is then [posted to the Versions application]( https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/services/submit_usage_ping_service.rb#L20) + If a firewall exception is needed, the required URL depends on several things. If + the hostname is `version.gitlab.com`, the protocol is `TCP`, and the port number is `443`, + the required URL is <https://version.gitlab.com/>. + +## Implementing Usage Ping + +Usage Ping consists of two kinds of data, counters and observations. Counters track how often a certain event +happened over time, such as how many CI pipelines have run. They are monotonic and always trend up. +Observations are facts collected from one or more GitLab instances and can carry arbitrary data. There are no +general guidelines around how to collect those, due to the individual nature of that data. + +There are several types of counters which are all found in `usage_data.rb`: + +- **Ordinary Batch Counters:** Simple count of a given ActiveRecord_Relation +- **Distinct Batch Counters:** Distinct count of a given ActiveRecord_Relation on given column +- **Sum Batch Counters:** Sum the values of a given ActiveRecord_Relation on given column +- **Alternative Counters:** Used for settings and configurations +- **Redis Counters:** Used for in-memory counts. + +NOTE: +Only use the provided counter methods. Each counter method contains a built in fail safe to isolate each counter to avoid breaking the entire Usage Ping. + +### Why batch counting + +For large tables, PostgreSQL can take a long time to count rows due to MVCC [(Multi-version Concurrency Control)](https://en.wikipedia.org/wiki/Multiversion_concurrency_control). Batch counting is a counting method where a single large query is broken into multiple smaller queries. For example, instead of a single query querying 1,000,000 records, with batch counting, you can execute 100 queries of 10,000 records each. Batch counting is useful for avoiding database timeouts as each batch query is significantly shorter than one single long running query. + +For GitLab.com, there are extremely large tables with 15 second query timeouts, so we use batch counting to avoid encountering timeouts. Here are the sizes of some GitLab.com tables: + +| Table | Row counts in millions | +|------------------------------|------------------------| +| `merge_request_diff_commits` | 2280 | +| `ci_build_trace_sections` | 1764 | +| `merge_request_diff_files` | 1082 | +| `events` | 514 | + +There are two batch counting methods provided, `Ordinary Batch Counters` and `Distinct Batch Counters`. Batch counting requires indexes on columns to calculate max, min, and range queries. In some cases, a specialized index may need to be added on the columns involved in a counter. + +### Ordinary Batch Counters + +Handles `ActiveRecord::StatementInvalid` error + +Simple count of a given ActiveRecord_Relation, does a non-distinct batch count, smartly reduces batch_size and handles errors. + +Method: `count(relation, column = nil, batch: true, start: nil, finish: nil)` + +Arguments: + +- `relation` the ActiveRecord_Relation to perform the count +- `column` the column to perform the count on, by default is the primary key +- `batch`: default `true` in order to use batch counting +- `start`: custom start of the batch counting in order to avoid complex min calculations +- `end`: custom end of the batch counting in order to avoid complex min calculations + +Examples: + +```ruby +count(User.active) +count(::Clusters::Cluster.aws_installed.enabled, :cluster_id) +count(::Clusters::Cluster.aws_installed.enabled, :cluster_id, start: ::Clusters::Cluster.minimum(:id), finish: ::Clusters::Cluster.maximum(:id)) +``` + +### Distinct Batch Counters + +Handles `ActiveRecord::StatementInvalid` error + +Distinct count of a given ActiveRecord_Relation on given column, a distinct batch count, smartly reduces batch_size and handles errors. + +Method: `distinct_count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil)` + +Arguments: + +- `relation` the ActiveRecord_Relation to perform the count +- `column` the column to perform the distinct count, by default is the primary key +- `batch`: default `true` in order to use batch counting +- `batch_size`: if none set it uses default value 10000 from `Gitlab::Database::BatchCounter` +- `start`: custom start of the batch counting in order to avoid complex min calculations +- `end`: custom end of the batch counting in order to avoid complex min calculations + +WARNING: +Counting over non-unique columns can lead to performance issues. Take a look at the [iterating tables in batches](iterating_tables_in_batches.md) guide for more details. + +Examples: + +```ruby +distinct_count(::Project, :creator_id) +distinct_count(::Note.with_suggestions.where(time_period), :author_id, start: ::User.minimum(:id), finish: ::User.maximum(:id)) +distinct_count(::Clusters::Applications::CertManager.where(time_period).available.joins(:cluster), 'clusters.user_id') +``` + +### Sum Batch Counters + +Handles `ActiveRecord::StatementInvalid` error + +Sum the values of a given ActiveRecord_Relation on given column and handles errors. + +Method: `sum(relation, column, batch_size: nil, start: nil, finish: nil)` + +Arguments: + +- `relation` the ActiveRecord_Relation to perform the operation +- `column` the column to sum on +- `batch_size`: if none set it uses default value 1000 from `Gitlab::Database::BatchCounter` +- `start`: custom start of the batch counting in order to avoid complex min calculations +- `end`: custom end of the batch counting in order to avoid complex min calculations + +Examples: + +```ruby +sum(JiraImportState.finished, :imported_issues_count) +``` + +### Grouping & Batch Operations + +The `count`, `distinct_count`, and `sum` batch counters can accept an `ActiveRecord::Relation` +object, which groups by a specified column. With a grouped relation, the methods do batch counting, +handle errors, and returns a hash table of key-value pairs. + +Examples: + +```ruby +count(Namespace.group(:type)) +# returns => {nil=>179, "Group"=>54} + +distinct_count(Project.group(:visibility_level), :creator_id) +# returns => {0=>1, 10=>1, 20=>11} + +sum(Issue.group(:state_id), :weight)) +# returns => {1=>3542, 2=>6820} +``` + +### Redis Counters + +Handles `::Redis::CommandError` and `Gitlab::UsageDataCounters::BaseCounter::UnknownEvent` +returns -1 when a block is sent or hash with all values -1 when a `counter(Gitlab::UsageDataCounters)` is sent +different behavior due to 2 different implementations of Redis counter + +Method: `redis_usage_data(counter, &block)` + +Arguments: + +- `counter`: a counter from `Gitlab::UsageDataCounters`, that has `fallback_totals` method implemented +- or a `block`: which is evaluated + +#### Ordinary Redis Counters + +Examples of implementation: + +- Using Redis methods [`INCR`](https://redis.io/commands/incr), [`GET`](https://redis.io/commands/get), and [`Gitlab::UsageDataCounters::WikiPageCounter`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/wiki_page_counter.rb) +- Using Redis methods [`HINCRBY`](https://redis.io/commands/hincrby), [`HGETALL`](https://redis.io/commands/hgetall), and [`Gitlab::UsageCounters::PodLogs`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_counters/pod_logs.rb) + +##### UsageData API Tracking + +<!-- There's nearly identical content in `##### Adding new events`. If you fix errors here, you may need to fix the same errors in the other location. --> + +1. Track event using `UsageData` API + + Increment event count using ordinary Redis counter, for given event name. + + Tracking events using the `UsageData` API requires the `usage_data_api` feature flag to be enabled, which is enabled by default. + + API requests are protected by checking for a valid CSRF token. + + In order to be able to increment the values the related feature `usage_data_<event_name>` should be enabled. + + ```plaintext + POST /usage_data/increment_counter + ``` + + | Attribute | Type | Required | Description | + | :-------- | :--- | :------- | :---------- | + | `event` | string | yes | The event name it should be tracked | + + Response + + - `200` if event was tracked + - `400 Bad request` if event parameter is missing + - `401 Unauthorized` if user is not authenticated + - `403 Forbidden` for invalid CSRF token provided + +1. Track events using JavaScript/Vue API helper which calls the API above + + Note that `usage_data_api` and `usage_data_#{event_name}` should be enabled in order to be able to track events + + ```javascript + import api from '~/api'; + + api.trackRedisCounterEvent('my_already_defined_event_name'), + ``` + +#### Redis HLL Counters + +With `Gitlab::UsageDataCounters::HLLRedisCounter` we have available data structures used to count unique values. + +Implemented using Redis methods [PFADD](https://redis.io/commands/pfadd) and [PFCOUNT](https://redis.io/commands/pfcount). + +##### Adding new events + +1. Define events in [`known_events`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/known_events/). + + Example event: + + ```yaml + - name: i_compliance_credential_inventory + category: compliance + redis_slot: compliance + expiry: 42 # 6 weeks + aggregation: weekly + ``` + + Keys: + + - `name`: unique event name. + + Name format `<prefix>_<redis_slot>_name`. + + Use one of the following prefixes for the event's name: + + - `g_` for group, as an event which is tracked for group. + - `p_` for project, as an event which is tracked for project. + - `i_` for instance, as an event which is tracked for instance. + - `a_` for events encompassing all `g_`, `p_`, `i_`. + - `o_` for other. + + Consider including in the event's name the Redis slot in order to be able to count totals for a specific category. + + Example names: `i_compliance_credential_inventory`, `g_analytics_contribution`. + + - `category`: event category. Used for getting total counts for events in a category, for easier + access to a group of events. + - `redis_slot`: optional Redis slot; default value: event name. Used if needed to calculate totals + for a group of metrics. Ensure keys are in the same slot. For example: + `i_compliance_credential_inventory` with `redis_slot: 'compliance'` builds Redis key + `i_{compliance}_credential_inventory-2020-34`. If `redis_slot` is not defined the Redis key will + be `{i_compliance_credential_inventory}-2020-34`. + - `expiry`: expiry time in days. Default: 29 days for daily aggregation and 6 weeks for weekly + aggregation. + - `aggregation`: may be set to a `:daily` or `:weekly` key. Defines how counting data is stored in Redis. + Aggregation on a `daily` basis does not pull more fine grained data. + - `feature_flag`: optional. For details, see our [GitLab internal Feature flags](feature_flags/) documentation. + +1. Track event in controller using `RedisTracking` module with `track_redis_hll_event(*controller_actions, name:, feature:, feature_default_enabled: false)`. + + Arguments: + + - `controller_actions`: controller actions we want to track. + - `name`: event name. + - `feature`: feature name, all metrics we track should be under feature flag. + - `feature_default_enabled`: feature flag is disabled by default, set to `true` for it to be enabled by default. + + Example usage: + + ```ruby + # controller + class ProjectsController < Projects::ApplicationController + include RedisTracking + + skip_before_action :authenticate_user!, only: :show + track_redis_hll_event :index, :show, name: 'g_compliance_example_feature_visitors', feature: :compliance_example_feature, feature_default_enabled: true + + def index + render html: 'index' + end + + def new + render html: 'new' + end + + def show + render html: 'show' + end + end + ``` + +1. Track event in API using `increment_unique_values(event_name, values)` helper method. + + In order to be able to track the event, Usage Ping must be enabled and the event feature `usage_data_<event_name>` must be enabled. + + Arguments: + + - `event_name`: event name. + - `values`: values counted, one value or array of values. + + Example usage: + + ```ruby + get ':id/registry/repositories' do + repositories = ContainerRepositoriesFinder.new( + user: current_user, subject: user_group + ).execute + + increment_unique_values('i_list_repositories', current_user.id) + + present paginate(repositories), with: Entities::ContainerRegistry::Repository, tags: params[:tags], tags_count: params[:tags_count] + end + ``` + +1. Track event using `track_usage_event(event_name, values) in services and graphql + + Increment unique values count using Redis HLL, for given event name. + + Example: + + [Track usage event for incident created in service](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/services/issues/update_service.rb) + + [Track usage event for incident created in graphql](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/graphql/mutations/alert_management/update_alert_status.rb) + + ```ruby + track_usage_event(:incident_management_incident_created, current_user.id) + ``` + +<!-- There's nearly identical content in `##### UsageData API Tracking`. If you find / fix errors here, you may need to fix errors in that section too. --> + +1. Track event using `UsageData` API + + Increment unique users count using Redis HLL, for given event name. + + Tracking events using the `UsageData` API requires the `usage_data_api` feature flag to be enabled, which is enabled by default. + + API requests are protected by checking for a valid CSRF token. + + In order to increment the values, the related feature `usage_data_<event_name>` should be + set to `default_enabled: true`. For more information, see + [Feature flags in development of GitLab](feature_flags/index.md). + + ```plaintext + POST /usage_data/increment_unique_users + ``` + + | Attribute | Type | Required | Description | + | :-------- | :--- | :------- | :---------- | + | `event` | string | yes | The event name it should be tracked | + + Response + + Return 200 if tracking failed for any reason. + + - `200` if event was tracked or any errors + - `400 Bad request` if event parameter is missing + - `401 Unauthorized` if user is not authenticated + - `403 Forbidden` for invalid CSRF token provided + +1. Track events using JavaScript/Vue API helper which calls the API above + + Example usage for an existing event already defined in [known events](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/known_events/): + + Usage Data API is behind `usage_data_api` feature flag which, as of GitLab 13.7, is + now set to `default_enabled: true`. + + Each event tracked using Usage Data API is behind a feature flag `usage_data_#{event_name}` which should be `default_enabled: true` + + ```javascript + import api from '~/api'; + + api.trackRedisHllUserEvent('my_already_defined_event_name'), + ``` + +1. Track event using base module `Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event_name, values:)`. + + Arguments: + + - `event_name`: event name. + - `values`: One value or array of values we count. For example: user_id, visitor_id, user_ids. + +1. Track event on context level using base module `Gitlab::UsageDataCounters::HLLRedisCounter.track_event_in_context(event_name, values:, context:)`. + + Arguments: + + - `event_name`: event name. + - `values`: values we count. For example: user_id, visitor_id. + - `context`: context value. Allowed values are `default`, `free`, `bronze`, `silver`, `gold`, `starter`, `premium`, `ultimate` + +1. Get event data using `Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names:, start_date:, end_date:, context: '')`. + + Arguments: + + - `event_names`: the list of event names. + - `start_date`: start date of the period for which we want to get event data. + - `end_date`: end date of the period for which we want to get event data. + - `context`: context of the event. Allowed values are `default`, `free`, `bronze`, `silver`, `gold`, `starter`, `premium`, `ultimate`. + +1. Testing tracking and getting unique events + +Trigger events in rails console by using `track_event` method + + ```ruby + Gitlab::UsageDataCounters::HLLRedisCounter.track_event('g_compliance_audit_events', values: 1) + Gitlab::UsageDataCounters::HLLRedisCounter.track_event('g_compliance_audit_events', values: [2, 3]) + ``` + +Next, get the unique events for the current week. + + ```ruby + # Get unique events for metric for current_week + Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: 'g_compliance_audit_events', + start_date: Date.current.beginning_of_week, end_date: Date.current.end_of_week) + ``` + +##### Recommendations + +We have the following recommendations for [Adding new events](#adding-new-events): + +- Event aggregation: weekly. +- Key expiry time: + - Daily: 29 days. + - Weekly: 42 days. +- When adding new metrics, use a [feature flag](../operations/feature_flags.md) to control the impact. +- For feature flags triggered by another service, set `default_enabled: false`, + - Events can be triggered using the `UsageData` API, which helps when there are > 10 events per change + +##### Enable/Disable Redis HLL tracking + +Events are tracked behind [feature flags](feature_flags/index.md) due to concerns for Redis performance and scalability. + +For a full list of events and corresponding feature flags see, [known_events](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/known_events/) files. + +To enable or disable tracking for specific event within <https://gitlab.com> or <https://about.staging.gitlab.com>, run commands such as the following to +[enable or disable the corresponding feature](feature_flags/index.md). + +```shell +/chatops run feature set <feature_name> true +/chatops run feature set <feature_name> false +``` + +##### Known events are added automatically in usage data payload + +All events added in [`known_events/common.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/known_events/common.yml) are automatically added to usage data generation under the `redis_hll_counters` key. This column is stored in [version-app as a JSON](https://gitlab.com/gitlab-services/version-gitlab-com/-/blob/master/db/schema.rb#L209). +For each event we add metrics for the weekly and monthly time frames, and totals for each where applicable: + +- `#{event_name}_weekly`: Data for 7 days for daily [aggregation](#adding-new-events) events and data for the last complete week for weekly [aggregation](#adding-new-events) events. +- `#{event_name}_monthly`: Data for 28 days for daily [aggregation](#adding-new-events) events and data for the last 4 complete weeks for weekly [aggregation](#adding-new-events) events. + +Redis HLL implementation calculates automatic total metrics, if there are more than one metric for the same category, aggregation and Redis slot. + +- `#{category}_total_unique_counts_weekly`: Total unique counts for events in the same category for the last 7 days or the last complete week, if events are in the same Redis slot and we have more than one metric. +- `#{category}_total_unique_counts_monthly`: Total unique counts for events in same category for the last 28 days or the last 4 complete weeks, if events are in the same Redis slot and we have more than one metric. + +Example of `redis_hll_counters` data: + +```ruby +{:redis_hll_counters=> + {"compliance"=> + {"g_compliance_dashboard_weekly"=>0, + "g_compliance_dashboard_monthly"=>0, + "g_compliance_audit_events_weekly"=>0, + "g_compliance_audit_events_monthly"=>0, + "compliance_total_unique_counts_weekly"=>0, + "compliance_total_unique_counts_monthly"=>0}, + "analytics"=> + {"g_analytics_contribution_weekly"=>0, + "g_analytics_contribution_monthly"=>0, + "g_analytics_insights_weekly"=>0, + "g_analytics_insights_monthly"=>0, + "analytics_total_unique_counts_weekly"=>0, + "analytics_total_unique_counts_monthly"=>0}, + "ide_edit"=> + {"g_edit_by_web_ide_weekly"=>0, + "g_edit_by_web_ide_monthly"=>0, + "g_edit_by_sfe_weekly"=>0, + "g_edit_by_sfe_monthly"=>0, + "ide_edit_total_unique_counts_weekly"=>0, + "ide_edit_total_unique_counts_monthly"=>0}, + "search"=> + {"i_search_total_weekly"=>0, "i_search_total_monthly"=>0, "i_search_advanced_weekly"=>0, "i_search_advanced_monthly"=>0, "i_search_paid_weekly"=>0, "i_search_paid_monthly"=>0, "search_total_unique_counts_weekly"=>0, "search_total_unique_counts_monthly"=>0}, + "source_code"=>{"wiki_action_weekly"=>0, "wiki_action_monthly"=>0} + } +``` + +Example usage: + +```ruby +# Redis Counters +redis_usage_data(Gitlab::UsageDataCounters::WikiPageCounter) +redis_usage_data { ::Gitlab::UsageCounters::PodLogs.usage_totals[:total] } + +# Define events in common.yml https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/known_events/common.yml + +# Tracking events +Gitlab::UsageDataCounters::HLLRedisCounter.track_event('expand_vulnerabilities', values: visitor_id) + +# Get unique events for metric +redis_usage_data { Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: 'expand_vulnerabilities', start_date: 28.days.ago, end_date: Date.current) } +``` + +### Alternative Counters + +Handles `StandardError` and fallbacks into -1 this way not all measures fail if we encounter one exception. +Mainly used for settings and configurations. + +Method: `alt_usage_data(value = nil, fallback: -1, &block)` + +Arguments: + +- `value`: a simple static value in which case the value is simply returned. +- or a `block`: which is evaluated +- `fallback: -1`: the common value used for any metrics that are failing. + +Example of usage: + +```ruby +alt_usage_data { Gitlab::VERSION } +alt_usage_data { Gitlab::CurrentSettings.uuid } +alt_usage_data(999) +``` + +### Prometheus Queries + +In those cases where operational metrics should be part of Usage Ping, a database or Redis query is unlikely +to provide useful data. Instead, Prometheus might be more appropriate, since most GitLab architectural +components publish metrics to it that can be queried back, aggregated, and included as usage data. + +NOTE: +Prometheus as a data source for Usage Ping is currently only available for single-node Omnibus installations +that are running the [bundled Prometheus](../administration/monitoring/prometheus/index.md) instance. + +To query Prometheus for metrics, a helper method is available to `yield` a fully configured +`PrometheusClient`, given it is available as per the note above: + +```ruby +with_prometheus_client do |client| + response = client.query('<your query>') + ... +end +``` + +Please refer to [the `PrometheusClient` definition](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/prometheus_client.rb) +for how to use its API to query for data. + +## Developing and testing Usage Ping + +### 1. Naming and placing the metrics + +Add the metric in one of the top level keys + +- `license`: for license related metrics. +- `settings`: for settings related metrics. +- `counts_weekly`: for counters that have data for the most recent 7 days. +- `counts_monthly`: for counters that have data for the most recent 28 days. +- `counts`: for counters that have data for all time. + +### 2. Use your Rails console to manually test counters + +```ruby +# count +Gitlab::UsageData.count(User.active) +Gitlab::UsageData.count(::Clusters::Cluster.aws_installed.enabled, :cluster_id) + +# count distinct +Gitlab::UsageData.distinct_count(::Project, :creator_id) +Gitlab::UsageData.distinct_count(::Note.with_suggestions.where(time_period), :author_id, start: ::User.minimum(:id), finish: ::User.maximum(:id)) +``` + +### 3. Generate the SQL query + +Your Rails console returns the generated SQL queries. + +Example: + +```ruby +pry(main)> Gitlab::UsageData.count(User.active) + (2.6ms) SELECT "features"."key" FROM "features" + (15.3ms) SELECT MIN("users"."id") FROM "users" WHERE ("users"."state" IN ('active')) AND ("users"."user_type" IS NULL OR "users"."user_type" IN (6, 4)) + (2.4ms) SELECT MAX("users"."id") FROM "users" WHERE ("users"."state" IN ('active')) AND ("users"."user_type" IS NULL OR "users"."user_type" IN (6, 4)) + (1.9ms) SELECT COUNT("users"."id") FROM "users" WHERE ("users"."state" IN ('active')) AND ("users"."user_type" IS NULL OR "users"."user_type" IN (6, 4)) AND "users"."id" BETWEEN 1 AND 100000 +``` + +### 4. Optimize queries with #database-lab + +Paste the SQL query into `#database-lab` to see how the query performs at scale. + +- `#database-lab` is a Slack channel which uses a production-sized environment to test your queries. +- GitLab.com’s production database has a 15 second timeout. +- Any single query must stay below [1 second execution time](query_performance.md#timing-guidelines-for-queries) with cold caches. +- Add a specialized index on columns involved to reduce the execution time. + +In order to have an understanding of the query's execution we add in the MR description the following information: + +- For counters that have a `time_period` test we add information for both cases: + - `time_period = {}` for all time periods + - `time_period = { created_at: 28.days.ago..Time.current }` for last 28 days period +- Execution plan and query time before and after optimization +- Query generated for the index and time +- Migration output for up and down execution + +We also use `#database-lab` and [explain.depesz.com](https://explain.depesz.com/). For more details, see the [database review guide](database_review.md#preparation-when-adding-or-modifying-queries). + +#### Optimization recommendations and examples + +- Use specialized indexes [example 1](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26871), [example 2](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26445). +- Use defined `start` and `finish`, and simple queries, because these values can be memoized and reused, [example](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/37155). +- Avoid joins and write the queries as simply as possible, [example](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/36316). +- Set a custom `batch_size` for `distinct_count`, [example](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38000). + +### 5. Add the metric definition + +When adding, changing, or updating metrics, please update the [Event Dictionary's **Usage Ping** table](https://about.gitlab.com/handbook/product/product-intelligence-guide/#event-dictionary). + +### 6. Add new metric to Versions Application + +Check if new metrics need to be added to the Versions Application. See `usage_data` [schema](https://gitlab.com/gitlab-services/version-gitlab-com/-/blob/master/db/schema.rb#L147) and usage data [parameters accepted](https://gitlab.com/gitlab-services/version-gitlab-com/-/blob/master/app/services/usage_ping.rb). Any metrics added under the `counts` key are saved in the `stats` column. + +### 7. Add the feature label + +Add the `feature` label to the Merge Request for new Usage Ping metrics. These are user-facing changes and are part of expanding the Usage Ping feature. + +### 8. Add a changelog file + +Ensure you comply with the [Changelog entries guide](changelog.md). + +### 9. Ask for a Product Intelligence Review + +On GitLab.com, we have DangerBot setup to monitor Product Intelligence related files and DangerBot recommends a Product Intelligence review. Mention `@gitlab-org/growth/product_intelligence/engineers` in your MR for a review. + +### 10. Verify your metric + +On GitLab.com, the Product Intelligence team regularly monitors Usage Ping. They may alert you that your metrics need further optimization to run quicker and with greater success. You may also use the [Usage Ping QA dashboard](https://app.periscopedata.com/app/gitlab/632033/Usage-Ping-QA) to check how well your metric performs. The dashboard allows filtering by GitLab version, by "Self-managed" & "Saas" and shows you how many failures have occurred for each metric. Whenever you notice a high failure rate, you may re-optimize your metric. + +### Optional: Test Prometheus based Usage Ping + +If the data submitted includes metrics [queried from Prometheus](#prometheus-queries) that you would like to inspect and verify, +then you need to ensure that a Prometheus server is running locally, and that furthermore the respective GitLab components +are exporting metrics to it. If you do not need to test data coming from Prometheus, no further action +is necessary, since Usage Ping should degrade gracefully in the absence of a running Prometheus server. + +There are currently three kinds of components that may export data to Prometheus, and which are included in Usage Ping: + +- [`node_exporter`](https://github.com/prometheus/node_exporter) - Exports node metrics from the host machine +- [`gitlab-exporter`](https://gitlab.com/gitlab-org/gitlab-exporter) - Exports process metrics from various GitLab components +- various GitLab services such as Sidekiq and the Rails server that export their own metrics + +#### Test with an Omnibus container + +This is the recommended approach to test Prometheus based Usage Ping. + +The easiest way to verify your changes is to build a new Omnibus image from your code branch via CI, then download the image +and run a local container instance: + +1. From your merge request, click on the `qa` stage, then trigger the `package-and-qa` job. This job triggers an Omnibus +build in a [downstream pipeline of the `omnibus-gitlab-mirror` project](https://gitlab.com/gitlab-org/build/omnibus-gitlab-mirror/-/pipelines). +1. In the downstream pipeline, wait for the `gitlab-docker` job to finish. +1. Open the job logs and locate the full container name including the version. It takes the following form: `registry.gitlab.com/gitlab-org/build/omnibus-gitlab-mirror/gitlab-ee:<VERSION>`. +1. On your local machine, make sure you are logged in to the GitLab Docker registry. You can find the instructions for this in +[Authenticate to the GitLab Container Registry](../user/packages/container_registry/index.md#authenticate-with-the-container-registry). +1. Once logged in, download the new image via `docker pull registry.gitlab.com/gitlab-org/build/omnibus-gitlab-mirror/gitlab-ee:<VERSION>` +1. For more information about working with and running Omnibus GitLab containers in Docker, please refer to [GitLab Docker images](https://docs.gitlab.com/omnibus/docker/README.html) in the Omnibus documentation. + +#### Test with GitLab development toolkits + +This is the less recommended approach, since it comes with a number of difficulties when emulating a real GitLab deployment. + +The [GDK](https://gitlab.com/gitlab-org/gitlab-development-kit) is not currently set up to run a Prometheus server or `node_exporter` alongside other GitLab components. If you would +like to do so, [Monitoring the GDK with Prometheus](https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/master/doc/howto/prometheus/index.md#monitoring-the-gdk-with-prometheus) is a good start. + +The [GCK](https://gitlab.com/gitlab-org/gitlab-compose-kit) has limited support for testing Prometheus based Usage Ping. +By default, it already comes with a fully configured Prometheus service that is set up to scrape a number of components, +but with the following limitations: + +- It does not currently run a `gitlab-exporter` instance, so several `process_*` metrics from services such as Gitaly may be missing. +- While it runs a `node_exporter`, `docker-compose` services emulate hosts, meaning that it would normally report itself to not be associated +with any of the other services that are running. That is not how node metrics are reported in a production setup, where `node_exporter` +always runs as a process alongside other GitLab components on any given node. From Usage Ping's perspective none of the node data would therefore +appear to be associated to any of the services running, since they all appear to be running on different hosts. To alleviate this problem, the `node_exporter` in GCK was arbitrarily "assigned" to the `web` service, meaning only for this service `node_*` metrics appears in Usage Ping. + +## Aggregated metrics + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/45979) in GitLab 13.6. +> - It's [deployed behind a feature flag](../user/feature_flags.md), disabled by default. +> - It's enabled on GitLab.com. + +WARNING: +This feature is intended solely for internal GitLab use. + +In order to add data for aggregated metrics into Usage Ping payload you should add corresponding definition in [`aggregated_metrics`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/usage_data_counters/aggregated_metrics/). Each aggregate definition includes following parts: + +- name: unique name under which aggregate metric is added to Usage Ping payload +- operator: operator that defines how aggregated metric data is counted. Available operators are: + - `OR`: removes duplicates and counts all entries that triggered any of listed events + - `AND`: removes duplicates and counts all elements that were observed triggering all of following events +- events: list of events names (from [`known_events/`](#known-events-are-added-automatically-in-usage-data-payload)) to aggregate into metric. All events in this list must have the same `redis_slot` and `aggregation` attributes. +- feature_flag: name of [development feature flag](feature_flags/development.md#development-type) that is checked before +metrics aggregation is performed. Corresponding feature flag should have `default_enabled` attribute set to `false`. +`feature_flag` attribute is **OPTIONAL** and can be omitted, when `feature_flag` is missing no feature flag is checked. + +Example aggregated metric entries: + +```yaml +- name: product_analytics_test_metrics_union + operator: OR + events: ['i_search_total', 'i_search_advanced', 'i_search_paid'] +- name: product_analytics_test_metrics_intersection_with_feautre_flag + operator: AND + events: ['i_search_total', 'i_search_advanced', 'i_search_paid'] + feature_flag: example_aggregated_metric +``` + +Aggregated metrics are added under `aggregated_metrics` key in both `counts_weekly` and `counts_monthly` top level keys in Usage Ping payload. + +```ruby +{ + :counts_monthly => { + :deployments => 1003, + :successful_deployments => 78, + :failed_deployments => 275, + :packages => 155, + :personal_snippets => 2106, + :project_snippets => 407, + :promoted_issues => 719, + :aggregated_metrics => { + :product_analytics_test_metrics_union => 7, + :product_analytics_test_metrics_intersection_with_feautre_flag => 2 + }, + :snippets => 2513 + } +} +``` + +## Example Usage Ping payload + +The following is example content of the Usage Ping payload. + +```json +{ + "uuid": "0000000-0000-0000-0000-000000000000", + "hostname": "example.com", + "version": "12.10.0-pre", + "installation_type": "omnibus-gitlab", + "active_user_count": 999, + "recorded_at": "2020-04-17T07:43:54.162+00:00", + "edition": "EEU", + "license_md5": "00000000000000000000000000000000", + "license_id": null, + "historical_max_users": 999, + "licensee": { + "Name": "ABC, Inc.", + "Email": "email@example.com", + "Company": "ABC, Inc." + }, + "license_user_count": 999, + "license_starts_at": "2020-01-01", + "license_expires_at": "2021-01-01", + "license_plan": "ultimate", + "license_add_ons": { + }, + "license_trial": false, + "counts": { + "assignee_lists": 999, + "boards": 999, + "ci_builds": 999, + ... + }, + "container_registry_enabled": true, + "dependency_proxy_enabled": false, + "gitlab_shared_runners_enabled": true, + "gravatar_enabled": true, + "influxdb_metrics_enabled": true, + "ldap_enabled": false, + "mattermost_enabled": false, + "omniauth_enabled": true, + "prometheus_enabled": false, + "prometheus_metrics_enabled": false, + "reply_by_email_enabled": "incoming+%{key}@incoming.gitlab.com", + "signup_enabled": true, + "web_ide_clientside_preview_enabled": true, + "ingress_modsecurity_enabled": true, + "projects_with_expiration_policy_disabled": 999, + "projects_with_expiration_policy_enabled": 999, + ... + "elasticsearch_enabled": true, + "license_trial_ends_on": null, + "geo_enabled": false, + "git": { + "version": { + "major": 2, + "minor": 26, + "patch": 1 + } + }, + "gitaly": { + "version": "12.10.0-rc1-93-g40980d40", + "servers": 56, + "clusters": 14, + "filesystems": [ + "EXT_2_3_4" + ] + }, + "gitlab_pages": { + "enabled": true, + "version": "1.17.0" + }, + "container_registry_server": { + "vendor": "gitlab", + "version": "2.9.1-gitlab" + }, + "database": { + "adapter": "postgresql", + "version": "9.6.15", + "pg_system_id": 6842684531675334351 + }, + "analytics_unique_visits": { + "g_analytics_contribution": 999, + ... + }, + "usage_activity_by_stage": { + "configure": { + "project_clusters_enabled": 999, + ... + }, + "create": { + "merge_requests": 999, + ... + }, + "manage": { + "events": 999, + ... + }, + "monitor": { + "clusters": 999, + ... + }, + "package": { + "projects_with_packages": 999 + }, + "plan": { + "issues": 999, + ... + }, + "release": { + "deployments": 999, + ... + }, + "secure": { + "user_container_scanning_jobs": 999, + ... + }, + "verify": { + "ci_builds": 999, + ... + } + }, + "usage_activity_by_stage_monthly": { + "configure": { + "project_clusters_enabled": 999, + ... + }, + "create": { + "merge_requests": 999, + ... + }, + "manage": { + "events": 999, + ... + }, + "monitor": { + "clusters": 999, + ... + }, + "package": { + "projects_with_packages": 999 + }, + "plan": { + "issues": 999, + ... + }, + "release": { + "deployments": 999, + ... + }, + "secure": { + "user_container_scanning_jobs": 999, + ... + }, + "verify": { + "ci_builds": 999, + ... + } + }, + "topology": { + "duration_s": 0.013836685999194742, + "application_requests_per_hour": 4224, + "query_apdex_weekly_average": 0.996, + "failures": [], + "nodes": [ + { + "node_memory_total_bytes": 33269903360, + "node_memory_utilization": 0.35, + "node_cpus": 16, + "node_cpu_utilization": 0.2, + "node_uname_info": { + "machine": "x86_64", + "sysname": "Linux", + "release": "4.19.76-linuxkit" + }, + "node_services": [ + { + "name": "web", + "process_count": 16, + "process_memory_pss": 233349888, + "process_memory_rss": 788220927, + "process_memory_uss": 195295487, + "server": "puma" + }, + { + "name": "sidekiq", + "process_count": 1, + "process_memory_pss": 734080000, + "process_memory_rss": 750051328, + "process_memory_uss": 731533312 + }, + ... + ], + ... + }, + ... + ] + } +} +``` + +## Notable changes + +In GitLab 13.5, `pg_system_id` was added to send the [PostgreSQL system identifier](https://www.2ndquadrant.com/en/blog/support-for-postgresqls-system-identifier-in-barman/). + +## Exporting Usage Ping SQL queries and definitions + +Two Rake tasks exist to export Usage Ping definitions. + +- The Rake tasks export the raw SQL queries for `count`, `distinct_count`, `sum`. +- The Rake tasks export the Redis counter class or the line of the Redis block for `redis_usage_data`. +- The Rake tasks calculate the `alt_usage_data` metrics. + +In the home directory of your local GitLab installation run the following Rake tasks for the YAML and JSON versions respectively: + +```shell +# for YAML export +bin/rake gitlab:usage_data:dump_sql_in_yaml + +# for JSON export +bin/rake gitlab:usage_data:dump_sql_in_json + +# You may pipe the output into a file +bin/rake gitlab:usage_data:dump_sql_in_yaml > ~/Desktop/usage-metrics-2020-09-02.yaml +``` + +## Generating and troubleshooting usage ping + +To get a usage ping, or to troubleshoot caching issues on your GitLab instance, please follow [instructions to generate usage ping](../administration/troubleshooting/gitlab_rails_cheat_sheet.md#generate-usage-ping). diff --git a/doc/integration/github.md b/doc/integration/github.md index c65027e3585..858614a0571 100644 --- a/doc/integration/github.md +++ b/doc/integration/github.md @@ -12,7 +12,7 @@ with your GitHub account. ## Enabling GitHub OAuth -To enable the GitHub OmniAuth provider, you need an OAuth 2 Client ID and Client Secret from GitHub. To get these credentials, sign into GitHub and follow their procedure for [Creating an OAuth App](https://developer.github.com/apps/building-oauth-apps/creating-an-oauth-app/). +To enable the GitHub OmniAuth provider, you need an OAuth 2 Client ID and Client Secret from GitHub. To get these credentials, sign into GitHub and follow their procedure for [Creating an OAuth App](https://docs.github.com/apps/building-oauth-apps/creating-an-oauth-app/). When you create an OAuth 2 app in GitHub, you need the following information: diff --git a/doc/raketasks/README.md b/doc/raketasks/README.md index a42bf2a5d91..e119563fd25 100644 --- a/doc/raketasks/README.md +++ b/doc/raketasks/README.md @@ -42,7 +42,7 @@ The following are available Rake tasks: | [Repository storage](../administration/raketasks/storage.md) | List and migrate existing projects and attachments from legacy storage to hashed storage. | | [Uploads migrate](../administration/raketasks/uploads/migrate.md) | Migrate uploads between storage local and object storage. | | [Uploads sanitize](../administration/raketasks/uploads/sanitize.md) | Remove EXIF data from images uploaded to earlier versions of GitLab. | -| [Usage data](../administration/troubleshooting/gitlab_rails_cheat_sheet.md#generate-usage-ping) | Generate and troubleshoot [Usage Ping](../development/product_analytics/usage_ping.md).| +| [Usage data](../administration/troubleshooting/gitlab_rails_cheat_sheet.md#generate-usage-ping) | Generate and troubleshoot [Usage Ping](../development/usage_ping.md).| | [User management](user_management.md) | Perform user management tasks. | | [Webhooks administration](web_hooks.md) | Maintain project Webhooks. | | [X.509 signatures](x509_signatures.md) | Update X.509 commit signatures, useful if certificate store has changed. | diff --git a/doc/telemetry/index.md b/doc/telemetry/index.md index 57740672fb4..b3b3b0b4fdd 100644 --- a/doc/telemetry/index.md +++ b/doc/telemetry/index.md @@ -1,8 +1,8 @@ --- -redirect_to: '../development/product_analytics/index.md' +redirect_to: 'https://about.gitlab.com/handbook/product/product-intelligence-guide/' --- -This document was moved to [another location](../development/product_analytics/index.md). +This document was moved to [another location](https://about.gitlab.com/handbook/product/product-intelligence-guide/). <!-- This redirect file can be deleted after February 1, 2021. --> <!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/#move-or-rename-a-page --> diff --git a/doc/telemetry/snowplow.md b/doc/telemetry/snowplow.md index ac157a8e639..709c61b9e64 100644 --- a/doc/telemetry/snowplow.md +++ b/doc/telemetry/snowplow.md @@ -1,8 +1,8 @@ --- -redirect_to: '../development/product_analytics/snowplow.md' +redirect_to: '../development/snowplow.md' --- -This document was moved to [another location](../development/product_analytics/snowplow.md). +This document was moved to [another location](../development/snowplow.md). <!-- This redirect file can be deleted after February 1, 2021. --> <!-- Before deletion, see: https://docs.gitlab.com/ee/development/documentation/#move-or-rename-a-page --> diff --git a/doc/user/admin_area/settings/index.md b/doc/user/admin_area/settings/index.md index a7641ec22ca..9a661fa9716 100644 --- a/doc/user/admin_area/settings/index.md +++ b/doc/user/admin_area/settings/index.md @@ -38,7 +38,7 @@ Access the default page for admin area settings by navigating to **Admin Area > | [PlantUML](../../../administration/integration/plantuml.md#gitlab) | Allow rendering of PlantUML diagrams in AsciiDoc and Markdown documents. | | [Slack application](../../../user/project/integrations/gitlab_slack_application.md#configuration) **(FREE ONLY)** | Slack integration allows you to interact with GitLab via slash commands in a chat window. This option is only available on GitLab.com, though it may be [available for self-managed instances in the future](https://gitlab.com/gitlab-org/gitlab/-/issues/28164). | | [Third party offers](third_party_offers.md) | Control the display of third party offers. | -| [Snowplow](../../../development/product_analytics/snowplow.md) | Configure the Snowplow integration. | +| [Snowplow](../../../development/snowplow.md) | Configure the Snowplow integration. | | [Google GKE](../../project/clusters/add_gke_clusters.md) | Google GKE integration allows you to provision GKE clusters from GitLab. | | [Amazon EKS](../../project/clusters/add_eks_clusters.md) | Amazon EKS integration allows you to provision EKS clusters from GitLab. | diff --git a/doc/user/admin_area/settings/usage_statistics.md b/doc/user/admin_area/settings/usage_statistics.md index 7afa64bcce0..06b39d67228 100644 --- a/doc/user/admin_area/settings/usage_statistics.md +++ b/doc/user/admin_area/settings/usage_statistics.md @@ -61,7 +61,7 @@ sequenceDiagram ## Usage Ping **(CORE ONLY)** -See [Usage Ping guide](../../../development/product_analytics/usage_ping.md). +See [Usage Ping guide](../../../development/usage_ping.md). ## Instance-level analytics availability diff --git a/doc/user/application_security/dependency_scanning/index.md b/doc/user/application_security/dependency_scanning/index.md index 70f74492560..e62a45d72cd 100644 --- a/doc/user/application_security/dependency_scanning/index.md +++ b/doc/user/application_security/dependency_scanning/index.md @@ -490,7 +490,7 @@ ensure that it can reach your private repository. Here is an example configurati ### Referencing local dependencies using a path in JavaScript projects The [Retire.js](https://gitlab.com/gitlab-org/security-products/analyzers/retire.js) analyzer -doesn't support dependency references made with [local paths](https://docs.npmjs.com/files/package.json#local-paths) +doesn't support dependency references made with [local paths](https://docs.npmjs.com/cli/v6/configuring-npm/package-json#local-paths) in the `package.json` of JavaScript projects. The dependency scan outputs the following error for such references: diff --git a/doc/user/project/merge_requests/revert_changes.md b/doc/user/project/merge_requests/revert_changes.md index 0d95f16281b..23ad881087e 100644 --- a/doc/user/project/merge_requests/revert_changes.md +++ b/doc/user/project/merge_requests/revert_changes.md @@ -59,7 +59,7 @@ mainline: git revert -m 2 7a39eb0 ``` -From [GitLab 13.7 onwards](https://gitlab.com/gitlab-org/gitlab/-/issues/35824), merge requests +From [GitLab 13.8 onwards](https://gitlab.com/gitlab-org/gitlab/-/issues/35824), merge requests reverted by another merge request through one of the methods described in this document will display a link to the reverted merge request at the top-left corner within the **Merged** badge. diff --git a/lib/api/debian_project_packages.rb b/lib/api/debian_project_packages.rb index bcb4e8c8cbc..f8129c18dff 100644 --- a/lib/api/debian_project_packages.rb +++ b/lib/api/debian_project_packages.rb @@ -21,6 +21,8 @@ module API end namespace 'incoming/:file_name', requirements: FILE_NAME_REQUIREMENTS do + content_type :json, Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE + # PUT {projects|groups}/:id/-/packages/debian/incoming/:file_name params do requires :file, type: ::API::Validations::Types::WorkhorseFile, desc: 'The package file to be published (generated by Multipart middleware)' @@ -42,10 +44,9 @@ module API # PUT {projects|groups}/:id/-/packages/debian/incoming/:file_name/authorize route_setting :authentication, deploy_token_allowed: true, basic_auth_personal_access_token: true, job_token_allowed: :basic_auth - post 'authorize' do + put 'authorize' do authorize_workhorse!( subject: authorized_user_project, - has_length: false, maximum_size: authorized_user_project.actual_limits.debian_max_file_size ) end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 2271915a72b..fe3c2bca551 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -134,7 +134,7 @@ module Gitlab stage.seeds_names.include?(need[:name]) end - "#{name}: needs '#{need[:name]}'" unless result + "'#{name}' job needs '#{need[:name]}' job, but it was not added to the pipeline" unless result end.compact end diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index 993e1dbf84b..2ac408d59e2 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -253,6 +253,11 @@ redis_slot: testing aggregation: weekly feature_flag: usage_data_i_testing_metrics_report_widget_total +- name: i_testing_group_code_coverage_visit_total + category: testing + redis_slot: testing + aggregation: weekly + feature_flag: usage_data_i_testing_group_code_coverage_visit_total # Project Management group - name: g_project_management_issue_title_changed category: issues_edit @@ -101,6 +101,11 @@ module QA autoload :ProjectCluster, 'qa/resource/kubernetes_cluster/project_cluster' end + module Clusters + autoload :Agent, 'qa/resource/clusters/agent.rb' + autoload :AgentToken, 'qa/resource/clusters/agent_token.rb' + end + module Events autoload :Base, 'qa/resource/events/base' autoload :Project, 'qa/resource/events/project' diff --git a/qa/qa/fixtures/kubernetes_agent/agentk-config.yaml.erb b/qa/qa/fixtures/kubernetes_agent/agentk-config.yaml.erb new file mode 100644 index 00000000000..2d29cdecbcc --- /dev/null +++ b/qa/qa/fixtures/kubernetes_agent/agentk-config.yaml.erb @@ -0,0 +1,3 @@ +gitops: + manifest_projects: + - id: <%= project.full_path %>
\ No newline at end of file diff --git a/qa/qa/fixtures/kubernetes_agent/agentk-manifest.yaml.erb b/qa/qa/fixtures/kubernetes_agent/agentk-manifest.yaml.erb new file mode 100644 index 00000000000..a13c92d5c6d --- /dev/null +++ b/qa/qa/fixtures/kubernetes_agent/agentk-manifest.yaml.erb @@ -0,0 +1,95 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: gitlab-agent +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: gitlab-agent +spec: + replicas: 1 + selector: + matchLabels: + app: gitlab-agent + template: + metadata: + labels: + app: gitlab-agent + spec: + serviceAccountName: gitlab-agent + containers: + - name: agent + image: "registry.gitlab.com/gitlab-org/cluster-integration/gitlab-agent/agentk:<%= Runtime::Env.gitlab_agentk_version %>" + args: + - --token-file=/config/token + - --kas-address + - "<%= kas_wss_address %>" # Use this for GitLab chart deployments + # - "<%= kas_grpc_address %>" # Use this for GDK + volumeMounts: + - name: token-volume + mountPath: /config + volumes: + - name: token-volume + secret: + secretName: gitlab-agent-token + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: gitlab-agent-write +rules: + - resources: + - "*" + apiGroups: + - "*" + verbs: + - create + - update + - delete + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: gitlab-agent-write-binding +roleRef: + name: gitlab-agent-write + kind: ClusterRole + apiGroup: rbac.authorization.k8s.io +subjects: + - name: gitlab-agent + kind: ServiceAccount + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: gitlab-agent-read +rules: + - resources: + - "*" + apiGroups: + - "*" + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: gitlab-agent-read-binding +roleRef: + name: gitlab-agent-read + kind: ClusterRole + apiGroup: rbac.authorization.k8s.io +subjects: + - name: gitlab-agent + kind: ServiceAccount + namespace: default diff --git a/qa/qa/fixtures/kubernetes_agent/galatic-empire-manifest.yaml b/qa/qa/fixtures/kubernetes_agent/galatic-empire-manifest.yaml new file mode 100644 index 00000000000..78c42f2b94d --- /dev/null +++ b/qa/qa/fixtures/kubernetes_agent/galatic-empire-manifest.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: "galatic-empire" +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: "imperial-starfleet" + namespace: "galatic-empire" +data: + deathStars: "1" + starDestroyers: "5" + tieFighters: "25" diff --git a/qa/qa/resource/clusters/agent.rb b/qa/qa/resource/clusters/agent.rb new file mode 100644 index 00000000000..cad5a4c6b1d --- /dev/null +++ b/qa/qa/resource/clusters/agent.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module QA + module Resource + module Clusters + class Agent < QA::Resource::Base + attribute :id + attribute :name + attribute :project do + QA::Resource::Project.fabricate_via_api! do |project| + project.name = 'project-with-cluster-agent' + end + end + + def initialize + @name = "my-agent" + end + + def fabricate! + puts 'TODO: FABRICATE VIA UI' + end + # TODO + # + # The UI for this model is not yet implemented. So far it can only be + # created through the GraphQL API + # def fabricate + # + # end + + def api_get_path + "gid://gitlab/Clusters::Agent/#{id}" + end + + def api_post_path + "/graphql" + end + + def api_post_body + <<~GQL + mutation createAgent { + createClusterAgent(input: { projectPath: "#{project.full_path}", name: "#{@name}" }) { + clusterAgent { + id + name + } + errors + } + } + GQL + end + end + end + end +end diff --git a/qa/qa/resource/clusters/agent_token.rb b/qa/qa/resource/clusters/agent_token.rb new file mode 100644 index 00000000000..6a5e861b650 --- /dev/null +++ b/qa/qa/resource/clusters/agent_token.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module QA + module Resource + module Clusters + class AgentToken < QA::Resource::Base + attribute :id + attribute :secret + attribute :agent do + QA::Resource::Clusters::Agent.fabricate_via_api! + end + + def fabricate! + puts 'TODO: FABRICATE VIA UI' + end + # TODO + # + # The UI for this model is not yet implemented. So far it can only be + # created through the GraphQL API + # def fabricate + # + # end + + def api_get_path + "gid://gitlab/Clusters::AgentToken/#{id}" + end + + def api_post_path + "/graphql" + end + + def api_post_body + <<~GQL + mutation createToken { + clusterAgentTokenCreate(input: { clusterAgentId: "gid://gitlab/Clusters::Agent/#{agent.id}" }) { + secret # This is the value you need to use on the next step + token { + createdAt + id + } + errors + } + } + GQL + end + end + end + end +end diff --git a/qa/qa/runtime/env.rb b/qa/qa/runtime/env.rb index 24be4e4f978..1c637db5385 100644 --- a/qa/qa/runtime/env.rb +++ b/qa/qa/runtime/env.rb @@ -407,6 +407,10 @@ module QA QA::Runtime::Scenario.attributes.include?(:geo_secondary_address) end + def gitlab_agentk_version + ENV.fetch('GITLAB_AGENTK_VERSION', 'v13.7.0') + end + private def remote_grid_credentials diff --git a/qa/qa/service/kubernetes_cluster.rb b/qa/qa/service/kubernetes_cluster.rb index 84196556547..ddf97046fb0 100644 --- a/qa/qa/service/kubernetes_cluster.rb +++ b/qa/qa/service/kubernetes_cluster.rb @@ -43,6 +43,14 @@ module QA cluster_name end + def create_secret(secret, secret_name) + shell("kubectl create secret generic #{secret_name} --from-literal=token='#{secret}'") + end + + def apply_manifest(manifest) + shell('kubectl apply -f -', stdin_data: manifest) + end + private def fetch_api_url diff --git a/spec/features/issues/issue_state_spec.rb b/spec/features/issues/issue_state_spec.rb index 0ef6eb56dff..9145089f5fc 100644 --- a/spec/features/issues/issue_state_spec.rb +++ b/spec/features/issues/issue_state_spec.rb @@ -13,10 +13,13 @@ RSpec.describe 'issue state', :js do shared_examples 'issue closed' do |selector| it 'can close an issue' do + wait_for_requests + expect(find('.status-box')).to have_content 'Open' within selector do click_button 'Close issue' + wait_for_requests end expect(find('.status-box')).to have_content 'Closed' @@ -25,10 +28,13 @@ RSpec.describe 'issue state', :js do shared_examples 'issue reopened' do |selector| it 'can reopen an issue' do + wait_for_requests + expect(find('.status-box')).to have_content 'Closed' within selector do click_button 'Reopen issue' + wait_for_requests end expect(find('.status-box')).to have_content 'Open' diff --git a/spec/lib/gitlab/ci/lint_spec.rb b/spec/lib/gitlab/ci/lint_spec.rb index c67f8464123..67324c09d86 100644 --- a/spec/lib/gitlab/ci/lint_spec.rb +++ b/spec/lib/gitlab/ci/lint_spec.rb @@ -247,7 +247,7 @@ RSpec.describe Gitlab::Ci::Lint do include_context 'advanced validations' do it 'runs advanced logical validations' do expect(subject).not_to be_valid - expect(subject.errors).to eq(["test: needs 'build'"]) + expect(subject.errors).to eq(["'test' job needs 'build' job, but it was not added to the pipeline"]) end end diff --git a/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb b/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb index bc10e94c81d..cf020fc343c 100644 --- a/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/seed/build_spec.rb @@ -966,7 +966,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build do it "returns an error" do expect(subject.errors).to contain_exactly( - "rspec: needs 'build'") + "'rspec' job needs 'build' job, but it was not added to the pipeline") end end diff --git a/spec/lib/gitlab/ci/pipeline/seed/pipeline_spec.rb b/spec/lib/gitlab/ci/pipeline/seed/pipeline_spec.rb index 1790388da03..860b07647bd 100644 --- a/spec/lib/gitlab/ci/pipeline/seed/pipeline_spec.rb +++ b/spec/lib/gitlab/ci/pipeline/seed/pipeline_spec.rb @@ -62,7 +62,8 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Pipeline do needs_attributes: [{ name: 'non-existent', artifacts: true }] } - expect(seed.errors).to contain_exactly("invalid_job: needs 'non-existent'") + expect(seed.errors).to contain_exactly( + "'invalid_job' job needs 'non-existent' job, but it was not added to the pipeline") end end end diff --git a/spec/requests/api/debian_project_packages_spec.rb b/spec/requests/api/debian_project_packages_spec.rb index d2f208d0079..663b69b1b76 100644 --- a/spec/requests/api/debian_project_packages_spec.rb +++ b/spec/requests/api/debian_project_packages_spec.rb @@ -42,5 +42,12 @@ RSpec.describe API::DebianProjectPackages do it_behaves_like 'Debian project repository PUT endpoint', :created, nil end + + describe 'PUT projects/:id/-/packages/debian/incoming/:file_name/authorize' do + let(:method) { :put } + let(:url) { "/projects/#{project.id}/-/packages/debian/incoming/#{file_name}/authorize" } + + it_behaves_like 'Debian project repository PUT endpoint', :created, nil, is_authorize: true + end end end diff --git a/spec/services/ci/create_pipeline_service/dry_run_spec.rb b/spec/services/ci/create_pipeline_service/dry_run_spec.rb index 60c56ed0f67..c21a4ef0917 100644 --- a/spec/services/ci/create_pipeline_service/dry_run_spec.rb +++ b/spec/services/ci/create_pipeline_service/dry_run_spec.rb @@ -108,7 +108,7 @@ RSpec.describe Ci::CreatePipelineService do it_behaves_like 'returns a non persisted pipeline' it 'returns a pipeline with errors', :aggregate_failures do - error_message = "test: needs 'build'" + error_message = "'test' job needs 'build' job, but it was not added to the pipeline" expect(subject.error_messages.map(&:content)).to eq([error_message]) expect(subject.errors).not_to be_empty diff --git a/spec/services/ci/create_pipeline_service_spec.rb b/spec/services/ci/create_pipeline_service_spec.rb index 14bec4f5610..e1f1bdc41a1 100644 --- a/spec/services/ci/create_pipeline_service_spec.rb +++ b/spec/services/ci/create_pipeline_service_spec.rb @@ -1713,9 +1713,11 @@ RSpec.describe Ci::CreatePipelineService do shared_examples 'has errors' do it 'contains the expected errors' do expect(pipeline.builds).to be_empty - expect(pipeline.yaml_errors).to eq("test_a: needs 'build_a'") - expect(pipeline.error_messages.map(&:content)).to contain_exactly("test_a: needs 'build_a'") - expect(pipeline.errors[:base]).to contain_exactly("test_a: needs 'build_a'") + + error_message = "'test_a' job needs 'build_a' job, but it was not added to the pipeline" + expect(pipeline.yaml_errors).to eq(error_message) + expect(pipeline.error_messages.map(&:content)).to contain_exactly(error_message) + expect(pipeline.errors[:base]).to contain_exactly(error_message) end end diff --git a/spec/support/shared_examples/requests/api/debian_packages_shared_examples.rb b/spec/support/shared_examples/requests/api/debian_packages_shared_examples.rb index f55043fe64f..83ba72c12aa 100644 --- a/spec/support/shared_examples/requests/api/debian_packages_shared_examples.rb +++ b/spec/support/shared_examples/requests/api/debian_packages_shared_examples.rb @@ -37,9 +37,9 @@ RSpec.shared_context 'Debian repository shared context' do |object_type| let(:params) { workhorse_params } let(:auth_headers) { {} } + let(:workhorse_token) { JWT.encode({ 'iss' => 'gitlab-workhorse' }, Gitlab::Workhorse.secret, 'HS256') } let(:workhorse_headers) do if method == :put - workhorse_token = JWT.encode({ 'iss' => 'gitlab-workhorse' }, Gitlab::Workhorse.secret, 'HS256') { 'GitLab-Workhorse' => '1.0', Gitlab::Workhorse::INTERNAL_API_REQUEST_HEADER => workhorse_token } else {} @@ -117,12 +117,13 @@ RSpec.shared_examples 'Debian project repository PUT request' do |user_role, add and_body = body.nil? ? '' : ' and expected body' if status == :created - it 'creates package files' do + it 'creates package files', :aggregate_failures do pending "Debian package creation not implemented" expect { subject } .to change { project.packages.debian.count }.by(1) expect(response).to have_gitlab_http_status(status) + expect(response.media_type).to eq('text/plain') unless body.nil? expect(response.body).to eq(body) @@ -130,7 +131,59 @@ RSpec.shared_examples 'Debian project repository PUT request' do |user_role, add end it_behaves_like 'a package tracking event', described_class.name, 'push_package' else - it "returns #{status}#{and_body}" do + it "returns #{status}#{and_body}", :aggregate_failures do + subject + + expect(response).to have_gitlab_http_status(status) + + unless body.nil? + expect(response.body).to eq(body) + end + end + end + end +end + +RSpec.shared_examples 'Debian project repository PUT authorize request' do |user_role, add_member, status, body, is_authorize| + context "for user type #{user_role}" do + before do + project.send("add_#{user_role}", user) if add_member && user_role != :anonymous + end + + and_body = body.nil? ? '' : ' and expected body' + + if status == :created + it 'authorizes package file upload', :aggregate_failures do + subject + + expect(response).to have_gitlab_http_status(:ok) + expect(response.media_type).to eq(Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE) + expect(json_response['TempPath']).to eq(Packages::PackageFileUploader.workhorse_local_upload_path) + expect(json_response['RemoteObject']).to be_nil + expect(json_response['MaximumSize']).to be_nil + end + + context 'without a valid token' do + let(:workhorse_token) { 'invalid' } + + it 'rejects request' do + subject + + expect(response).to have_gitlab_http_status(:forbidden) + end + end + + context 'bypassing gitlab-workhorse' do + let(:workhorse_headers) { {} } + + it 'rejects request' do + subject + + expect(response).to have_gitlab_http_status(:forbidden) + end + end + else + it "returns #{status}#{and_body}", :aggregate_failures do subject expect(response).to have_gitlab_http_status(status) @@ -194,7 +247,7 @@ RSpec.shared_examples 'Debian project repository GET endpoint' do |success_statu it_behaves_like 'rejects Debian access with unknown project id' end -RSpec.shared_examples 'Debian project repository PUT endpoint' do |success_status, success_body| +RSpec.shared_examples 'Debian project repository PUT endpoint' do |success_status, success_body, is_authorize = false| context 'with valid project' do using RSpec::Parameterized::TableSyntax @@ -221,7 +274,13 @@ RSpec.shared_examples 'Debian project repository PUT endpoint' do |success_statu with_them do include_context 'Debian repository project access', params[:project_visibility_level], params[:user_role], params[:user_token], :basic do - it_behaves_like 'Debian project repository PUT request', params[:user_role], params[:member], params[:expected_status], params[:expected_body] + desired_behavior = if is_authorize + 'Debian project repository PUT authorize request' + else + 'Debian project repository PUT request' + end + + it_behaves_like desired_behavior, params[:user_role], params[:member], params[:expected_status], params[:expected_body] end end end |