diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-02-07 21:08:39 +0000 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-02-07 21:08:39 +0000 |
| commit | 0c6bc5443aa6c8f3e4becccb89fc0f135b4c64c8 (patch) | |
| tree | 55f13e752e9061c1800cce510a52fc78b13282ca | |
| parent | d7ce7307dca551759ffa972015875f8ebe476927 (diff) | |
| download | gitlab-ce-0c6bc5443aa6c8f3e4becccb89fc0f135b4c64c8.tar.gz | |
Add latest changes from gitlab-org/gitlab@master
56 files changed, 1205 insertions, 603 deletions
diff --git a/app/assets/javascripts/cycle_analytics/components/banner.vue b/app/assets/javascripts/cycle_analytics/components/banner.vue index ae8c430dcd6..0db9d2dbcf9 100644 --- a/app/assets/javascripts/cycle_analytics/components/banner.vue +++ b/app/assets/javascripts/cycle_analytics/components/banner.vue @@ -27,7 +27,7 @@ export default { <template> <div class="landing content-block"> <button - :aria-label="__('Dismiss Cycle Analytics introduction box')" + :aria-label="__('Dismiss Value Stream Analytics introduction box')" class="js-ca-dismiss-button dismiss-button" type="button" @click="dismissOverviewDialog" @@ -36,10 +36,10 @@ export default { </button> <div class="svg-container" v-html="iconCycleAnalyticsSplash"></div> <div class="inner-content"> - <h4>{{ __('Introducing Cycle Analytics') }}</h4> + <h4>{{ __('Introducing Value Stream Analytics') }}</h4> <p> {{ - __(`Cycle Analytics gives an overview + __(`Value Stream Analytics gives an overview of how much time it takes to go from idea to production in your project.`) }} </p> diff --git a/app/assets/javascripts/cycle_analytics/cycle_analytics_bundle.js b/app/assets/javascripts/cycle_analytics/cycle_analytics_bundle.js index 1074ce0e744..43f5e9954ce 100644 --- a/app/assets/javascripts/cycle_analytics/cycle_analytics_bundle.js +++ b/app/assets/javascripts/cycle_analytics/cycle_analytics_bundle.js @@ -71,7 +71,7 @@ export default () => { }, created() { // Conditional check placed here to prevent this method from being called on the - // new Cycle Analytics page (i.e. the new page will be initialized blank and only + // new Value Stream Analytics page (i.e. the new page will be initialized blank and only // after a group is selected the cycle analyitcs data will be fetched). Once the // old (current) page has been removed this entire created method as well as the // variable itself can be completely removed. @@ -81,7 +81,7 @@ export default () => { methods: { handleError() { this.store.setErrorState(true); - return new Flash(__('There was an error while fetching cycle analytics data.')); + return new Flash(__('There was an error while fetching value stream analytics data.')); }, initDropdown() { const $dropdown = $('.js-ca-dropdown'); diff --git a/app/assets/javascripts/diffs/components/compare_versions.vue b/app/assets/javascripts/diffs/components/compare_versions.vue index 63ce43a193d..3a2146147cc 100644 --- a/app/assets/javascripts/diffs/components/compare_versions.vue +++ b/app/assets/javascripts/diffs/components/compare_versions.vue @@ -1,6 +1,6 @@ <script> import { mapActions, mapGetters, mapState } from 'vuex'; -import { GlTooltipDirective, GlLink, GlButton } from '@gitlab/ui'; +import { GlTooltipDirective, GlLink, GlButton, GlSprintf } from '@gitlab/ui'; import { __ } from '~/locale'; import { polyfillSticky } from '~/lib/utils/sticky'; import Icon from '~/vue_shared/components/icon.vue'; @@ -15,6 +15,7 @@ export default { Icon, GlLink, GlButton, + GlSprintf, SettingsDropdown, DiffStats, }, @@ -106,23 +107,29 @@ export default { > <icon name="file-tree" /> </button> - <div v-if="showDropdowns" class="d-flex align-items-center compare-versions-container"> - {{ __('Compare') }} - <compare-versions-dropdown - :other-versions="mergeRequestDiffs" - :merge-request-version="mergeRequestDiff" - :show-commit-count="true" - class="mr-version-dropdown" - /> - {{ __('and') }} - <compare-versions-dropdown - :other-versions="comparableDiffs" - :base-version-path="baseVersionPath" - :start-version="startVersion" - :target-branch="targetBranch" - class="mr-version-compare-dropdown" - /> - </div> + <gl-sprintf + v-if="showDropdowns" + class="d-flex align-items-center compare-versions-container" + :message="s__('MergeRequest|Compare %{source} and %{target}')" + > + <template #source> + <compare-versions-dropdown + :other-versions="mergeRequestDiffs" + :merge-request-version="mergeRequestDiff" + :show-commit-count="true" + class="mr-version-dropdown" + /> + </template> + <template #target> + <compare-versions-dropdown + :other-versions="comparableDiffs" + :base-version-path="baseVersionPath" + :start-version="startVersion" + :target-branch="targetBranch" + class="mr-version-compare-dropdown" + /> + </template> + </gl-sprintf> <div v-else-if="commit"> {{ __('Viewing commit') }} <gl-link :href="commit.commit_url" class="monospace">{{ commit.short_id }}</gl-link> diff --git a/app/assets/javascripts/ide/components/commit_sidebar/actions.vue b/app/assets/javascripts/ide/components/commit_sidebar/actions.vue index 549324831e9..d9cd4f3acf1 100644 --- a/app/assets/javascripts/ide/components/commit_sidebar/actions.vue +++ b/app/assets/javascripts/ide/components/commit_sidebar/actions.vue @@ -1,7 +1,7 @@ <script> import _ from 'underscore'; import { mapState, mapGetters, createNamespacedHelpers } from 'vuex'; -import { sprintf, __ } from '~/locale'; +import { sprintf, s__ } from '~/locale'; import consts from '../../stores/modules/commit/constants'; import RadioGroup from './radio_group.vue'; import NewMergeRequestOption from './new_merge_request_option.vue'; @@ -21,7 +21,7 @@ export default { ...mapGetters(['currentBranch']), commitToCurrentBranchText() { return sprintf( - __('Commit to %{branchName} branch'), + s__('IDE|Commit to %{branchName} branch'), { branchName: `<strong class="monospace">${_.escape(this.currentBranchId)}</strong>` }, false, ); @@ -56,8 +56,8 @@ export default { }, commitToCurrentBranch: consts.COMMIT_TO_CURRENT_BRANCH, commitToNewBranch: consts.COMMIT_TO_NEW_BRANCH, - currentBranchPermissionsTooltip: __( - "This option is disabled as you don't have write permissions for the current branch", + currentBranchPermissionsTooltip: s__( + "IDE|This option is disabled because you don't have write permissions for the current branch.", ), }; </script> diff --git a/app/assets/javascripts/ide/components/commit_sidebar/radio_group.vue b/app/assets/javascripts/ide/components/commit_sidebar/radio_group.vue index 9161eb3d9b1..07073f5f879 100644 --- a/app/assets/javascripts/ide/components/commit_sidebar/radio_group.vue +++ b/app/assets/javascripts/ide/components/commit_sidebar/radio_group.vue @@ -1,10 +1,10 @@ <script> import { mapActions, mapState, mapGetters } from 'vuex'; -import tooltip from '~/vue_shared/directives/tooltip'; +import { GlTooltipDirective } from '@gitlab/ui'; export default { directives: { - tooltip, + GlTooltip: GlTooltipDirective, }, props: { value: { @@ -53,8 +53,7 @@ export default { <template> <fieldset> <label - v-tooltip - :title="tooltipTitle" + v-gl-tooltip="tooltipTitle" :class="{ 'is-disabled': disabled, }" diff --git a/app/assets/javascripts/notes/components/comment_form.vue b/app/assets/javascripts/notes/components/comment_form.vue index 4ca32b9b005..9a809b71a58 100644 --- a/app/assets/javascripts/notes/components/comment_form.vue +++ b/app/assets/javascripts/notes/components/comment_form.vue @@ -1,7 +1,7 @@ <script> import $ from 'jquery'; import { mapActions, mapGetters, mapState } from 'vuex'; -import _ from 'underscore'; +import { isEmpty } from 'lodash'; import Autosize from 'autosize'; import { __, sprintf } from '~/locale'; import TimelineEntryItem from '~/vue_shared/components/notes/timeline_entry_item.vue'; @@ -161,7 +161,7 @@ export default { 'toggleStateButtonLoading', ]), setIsSubmitButtonDisabled(note, isSubmitting) { - if (!_.isEmpty(note) && !isSubmitting) { + if (!isEmpty(note) && !isSubmitting) { this.isSubmitButtonDisabled = false; } else { this.isSubmitButtonDisabled = true; diff --git a/app/assets/javascripts/notes/components/diff_discussion_header.vue b/app/assets/javascripts/notes/components/diff_discussion_header.vue index 4c9075912ee..50d224a2f08 100644 --- a/app/assets/javascripts/notes/components/diff_discussion_header.vue +++ b/app/assets/javascripts/notes/components/diff_discussion_header.vue @@ -1,6 +1,6 @@ <script> import { mapActions } from 'vuex'; -import _ from 'underscore'; +import { escape } from 'lodash'; import { s__, __, sprintf } from '~/locale'; import { truncateSha } from '~/lib/utils/text_utility'; @@ -45,7 +45,7 @@ export default { return this.notes.length > 1 ? this.lastNote.created_at : null; }, headerText() { - const linkStart = `<a href="${_.escape(this.discussion.discussion_path)}">`; + const linkStart = `<a href="${escape(this.discussion.discussion_path)}">`; const linkEnd = '</a>'; const { commit_id: commitId } = this.discussion; diff --git a/app/assets/javascripts/notes/components/noteable_note.vue b/app/assets/javascripts/notes/components/noteable_note.vue index b3dae69d0bc..dea782683f2 100644 --- a/app/assets/javascripts/notes/components/noteable_note.vue +++ b/app/assets/javascripts/notes/components/noteable_note.vue @@ -1,7 +1,7 @@ <script> import $ from 'jquery'; import { mapGetters, mapActions } from 'vuex'; -import { escape } from 'underscore'; +import { escape } from 'lodash'; import draftMixin from 'ee_else_ce/notes/mixins/draft'; import { truncateSha } from '~/lib/utils/text_utility'; import TimelineEntryItem from '~/vue_shared/components/notes/timeline_entry_item.vue'; diff --git a/app/assets/javascripts/notes/components/toggle_replies_widget.vue b/app/assets/javascripts/notes/components/toggle_replies_widget.vue index f1b0b12bdce..dd132d4f608 100644 --- a/app/assets/javascripts/notes/components/toggle_replies_widget.vue +++ b/app/assets/javascripts/notes/components/toggle_replies_widget.vue @@ -1,5 +1,5 @@ <script> -import _ from 'underscore'; +import { uniqBy } from 'lodash'; import Icon from '~/vue_shared/components/icon.vue'; import UserAvatarLink from '~/vue_shared/components/user_avatar/user_avatar_link.vue'; import TimeAgoTooltip from '~/vue_shared/components/time_ago_tooltip.vue'; @@ -27,7 +27,7 @@ export default { uniqueAuthors() { const authors = this.replies.map(reply => reply.author || {}); - return _.uniq(authors, author => author.username); + return uniqBy(authors, author => author.username); }, className() { return this.collapsed ? 'collapsed' : 'expanded'; diff --git a/app/assets/javascripts/notes/stores/getters.js b/app/assets/javascripts/notes/stores/getters.js index 35398999abc..4f8ff8240b2 100644 --- a/app/assets/javascripts/notes/stores/getters.js +++ b/app/assets/javascripts/notes/stores/getters.js @@ -1,4 +1,4 @@ -import _ from 'underscore'; +import { flattenDeep } from 'lodash'; import * as constants from '../constants'; import { collapseSystemNotes } from './collapse_utils'; @@ -50,7 +50,7 @@ const isLastNote = (note, state) => !note.system && state.userData && note.author && note.author.id === state.userData.id; export const getCurrentUserLastNote = state => - _.flatten(reverseNotes(state.discussions).map(note => reverseNotes(note.notes))).find(el => + flattenDeep(reverseNotes(state.discussions).map(note => reverseNotes(note.notes))).find(el => isLastNote(el, state), ); diff --git a/app/assets/stylesheets/framework/variables.scss b/app/assets/stylesheets/framework/variables.scss index 90600ecf615..e4853ca7bf5 100644 --- a/app/assets/stylesheets/framework/variables.scss +++ b/app/assets/stylesheets/framework/variables.scss @@ -579,7 +579,7 @@ $calendar-border-color: rgba(#000, 0.1); $calendar-user-contrib-text: #959494; /* - * Cycle Analytics + * Value Stream Analytics */ $cycle-analytics-box-padding: 30px; $cycle-analytics-box-text-color: #8c8c8c; diff --git a/app/helpers/analytics_navbar_helper.rb b/app/helpers/analytics_navbar_helper.rb index c3416aeeac0..bbb7470d1fd 100644 --- a/app/helpers/analytics_navbar_helper.rb +++ b/app/helpers/analytics_navbar_helper.rb @@ -35,7 +35,7 @@ module AnalyticsNavbarHelper return unless project_nav_tab?(:cycle_analytics) navbar_sub_item( - title: _('Cycle Analytics'), + title: _('Value Stream Analytics'), path: 'cycle_analytics#show', link: project_cycle_analytics_path(project), link_to_options: { class: 'shortcuts-project-cycle-analytics' } diff --git a/app/views/layouts/nav/sidebar/_project.html.haml b/app/views/layouts/nav/sidebar/_project.html.haml index f17981e501c..c587d0ca053 100644 --- a/app/views/layouts/nav/sidebar/_project.html.haml +++ b/app/views/layouts/nav/sidebar/_project.html.haml @@ -42,8 +42,8 @@ - unless should_display_analytics_pages_in_sidebar - if can?(current_user, :read_cycle_analytics, @project) = nav_link(path: 'cycle_analytics#show') do - = link_to project_cycle_analytics_path(@project), title: _('Cycle Analytics'), class: 'shortcuts-project-cycle-analytics' do - %span= _('Cycle Analytics') + = link_to project_cycle_analytics_path(@project), title: _('Value Stream Analytics'), class: 'shortcuts-project-cycle-analytics' do + %span= _('Value Stream Analytics') = render_if_exists 'layouts/nav/project_insights_link' diff --git a/app/views/projects/cycle_analytics/_overview.html.haml b/app/views/projects/cycle_analytics/_overview.html.haml index ea94b637f89..2ca72b141be 100644 --- a/app/views/projects/cycle_analytics/_overview.html.haml +++ b/app/views/projects/cycle_analytics/_overview.html.haml @@ -4,12 +4,12 @@ .col-md-10.offset-md-1 .row.overview-details .col-md-6.overview-text - %h4 Introducing Cycle Analytics + %h4 Introducing Value Stream Analytics %p - Cycle Analytics gives an overview of how much time it takes to go from idea to production in your project. - To set up CA, you must first define a production environment by setting up your CI and then deploy to production. + Value Stream Analytics (VSA) gives an overview of how much time it takes to go from idea to production in your project. + To set up VSA, you must first define a production environment by setting up your CI and then deploy to production. %p - %a.btn{ href: help_page_path('user/analytics/cycle_analytics.md'), target: '_blank' } Read more + %a.btn{ href: help_page_path('user/analytics/value_stream_analytics.md'), target: '_blank' } Read more .col-md-6.overview-image %span.overview-icon = custom_icon ('icon_cycle_analytics_overview') diff --git a/app/views/projects/cycle_analytics/show.html.haml b/app/views/projects/cycle_analytics/show.html.haml index 8bbe4e66c50..70409774a50 100644 --- a/app/views/projects/cycle_analytics/show.html.haml +++ b/app/views/projects/cycle_analytics/show.html.haml @@ -1,9 +1,9 @@ -- page_title "Cycle Analytics" +- page_title "Value Stream Analytics" #cycle-analytics{ "v-cloak" => "true", data: { request_path: project_cycle_analytics_path(@project) } } - if @cycle_analytics_no_data %banner{ "v-if" => "!isOverviewDialogDismissed", - "documentation-link": help_page_path('user/analytics/cycle_analytics.md'), + "documentation-link": help_page_path('user/analytics/value_stream_analytics.md'), "v-on:dismiss-overview-dialog" => "dismissOverviewDialog()" } = icon("spinner spin", "v-show" => "isLoading") .wrapper{ "v-show" => "!isLoading && !hasError" } diff --git a/changelogs/unreleased/196858-cleanup-temporary-index-at-services-table-on-project_id.yml b/changelogs/unreleased/196858-cleanup-temporary-index-at-services-table-on-project_id.yml new file mode 100644 index 00000000000..264291d552c --- /dev/null +++ b/changelogs/unreleased/196858-cleanup-temporary-index-at-services-table-on-project_id.yml @@ -0,0 +1,5 @@ +--- +title: Remove temporary index at services on project_id +merge_request: 24263 +author: +type: removed diff --git a/changelogs/unreleased/38319-rename-interfaces-cycle-analytics-to-value-stream.yml b/changelogs/unreleased/38319-rename-interfaces-cycle-analytics-to-value-stream.yml new file mode 100644 index 00000000000..afb1ba8b062 --- /dev/null +++ b/changelogs/unreleased/38319-rename-interfaces-cycle-analytics-to-value-stream.yml @@ -0,0 +1,5 @@ +--- +title: Rename cycle analytics interfaces to value stream analytics +merge_request: 23427 +author: +type: changed diff --git a/changelogs/unreleased/fe-ide-minor-improvements-for-commit-options.yml b/changelogs/unreleased/fe-ide-minor-improvements-for-commit-options.yml new file mode 100644 index 00000000000..4dbf6850a7a --- /dev/null +++ b/changelogs/unreleased/fe-ide-minor-improvements-for-commit-options.yml @@ -0,0 +1,5 @@ +--- +title: Minor text update to IDE commit to branch disabled tooltip +merge_request: 24521 +author: +type: other diff --git a/config/routes/project.rb b/config/routes/project.rb index 82f998f185c..24e60be21de 100644 --- a/config/routes/project.rb +++ b/config/routes/project.rb @@ -195,9 +195,8 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do end end - resource :cycle_analytics, only: [:show] - - namespace :cycle_analytics do + resource :cycle_analytics, only: :show, path: 'value_stream_analytics' + scope module: :cycle_analytics, as: 'cycle_analytics', path: 'value_stream_analytics' do scope :events, controller: 'events' do get :issue get :plan @@ -208,6 +207,7 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do get :production end end + get '/cycle_analytics', to: redirect('%{namespace_id}/%{project_id}/-/value_stream_analytics') concerns :clusterable diff --git a/db/fixtures/development/17_cycle_analytics.rb b/db/fixtures/development/17_cycle_analytics.rb index b2252d31cac..2bf3c918006 100644 --- a/db/fixtures/development/17_cycle_analytics.rb +++ b/db/fixtures/development/17_cycle_analytics.rb @@ -109,7 +109,7 @@ class Gitlab::Seeder::CycleAnalytics def create_issues Array.new(@issue_count) do issue_params = { - title: "Cycle Analytics: #{FFaker::Lorem.sentence(6)}", + title: "Value Stream Analytics: #{FFaker::Lorem.sentence(6)}", description: FFaker::Lorem.sentence, state: 'opened', assignees: [@project.team.users.sample] @@ -166,7 +166,7 @@ class Gitlab::Seeder::CycleAnalytics Timecop.travel 12.hours.from_now opts = { - title: 'Cycle Analytics merge_request', + title: 'Value Stream Analytics merge_request', description: "Fixes #{issue.to_reference}", source_branch: branch, target_branch: 'master' diff --git a/db/post_migrate/20200203104214_services_remove_temporary_index_on_project_id.rb b/db/post_migrate/20200203104214_services_remove_temporary_index_on_project_id.rb new file mode 100644 index 00000000000..c9566d0256c --- /dev/null +++ b/db/post_migrate/20200203104214_services_remove_temporary_index_on_project_id.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class ServicesRemoveTemporaryIndexOnProjectId < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + INDEX_NAME = 'tmp_index_on_project_id_partial_with_prometheus_services' + PARTIAL_FILTER = "type = 'PrometheusService'" + + disable_ddl_transaction! + + def up + remove_concurrent_index :services, :project_id, where: PARTIAL_FILTER, name: INDEX_NAME + end + + def down + add_concurrent_index :services, :project_id, where: PARTIAL_FILTER, name: INDEX_NAME + end +end diff --git a/db/schema.rb b/db/schema.rb index 8d08f7a9301..c76b08467fe 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -3846,7 +3846,6 @@ ActiveRecord::Schema.define(version: 2020_02_06_111847) do t.boolean "instance", default: false t.index ["instance"], name: "index_services_on_instance" t.index ["project_id"], name: "index_services_on_project_id" - t.index ["project_id"], name: "tmp_index_on_project_id_partial_with_prometheus_services", where: "((type)::text = 'PrometheusService'::text)" t.index ["type"], name: "index_services_on_type" end diff --git a/doc/README.md b/doc/README.md index 201a2b57fbd..59c27d11d99 100644 --- a/doc/README.md +++ b/doc/README.md @@ -88,7 +88,7 @@ The following documentation relates to the DevOps **Manage** stage: | Manage Topics | Description | |:--------------------------------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | [Authentication and<br/>Authorization](administration/auth/README.md) **(CORE ONLY)** | Supported authentication and authorization providers. | -| [GitLab Cycle Analytics](user/project/cycle_analytics.md) | Measure the time it takes to go from an [idea to production](https://about.gitlab.com/blog/2016/08/05/continuous-integration-delivery-and-deployment-with-gitlab/#from-idea-to-production-with-gitlab) for each project you have. | +| [GitLab Value Stream Analytics](user/project/cycle_analytics.md) | Measure the time it takes to go from an [idea to production](https://about.gitlab.com/blog/2016/08/05/continuous-integration-delivery-and-deployment-with-gitlab/#from-idea-to-production-with-gitlab) for each project you have. | | [Instance Statistics](user/instance_statistics/index.md) | Discover statistics on how many GitLab features you use and user activity. | <div align="right"> diff --git a/doc/administration/monitoring/gitlab_self_monitoring_project/index.md b/doc/administration/monitoring/gitlab_self_monitoring_project/index.md index edd263428cb..da4dd09bc67 100644 --- a/doc/administration/monitoring/gitlab_self_monitoring_project/index.md +++ b/doc/administration/monitoring/gitlab_self_monitoring_project/index.md @@ -19,17 +19,26 @@ members to the group in order to give them maintainer access to the project. This project will be used for self monitoring your GitLab instance. -## Activating or deactivating the self monitoring project +## Activating the self monitoring project -1. Navigate to **Admin Area > Settings > Metrics and profiling** and expand the **Self monitoring** section. -1. Toggle on or off the **Create Project** button to create or remove the "GitLab self monitoring" project. -1. Click **Save changes** for the changes to take effect. +1. Navigate to **Admin Area > Settings > Metrics and profiling**, and expand the **Self monitoring** section. +1. Toggle the **Create Project** button on. +1. It can take a few seconds for the project to be created. After the project is +created, GitLab displays a message with a link to the project. The project +will also be linked in the help text above the **Create Project** button. You can also +find the project under **Projects > Your projects**. -If you activated the monitoring project, it should now be visible in **Projects > Your projects**. +## Deactivating the self monitoring project CAUTION: **Warning:** If you deactivate the self monitoring project, it will be permanently deleted. +1. Navigate to **Admin Area > Settings > Metrics and profiling**, and expand the **Self monitoring** section. +1. Toggle the **Create Project** button off. +1. In the confirmation dialog that opens, click **Delete project**. + It can take a few seconds for it to be deleted. +1. After the project is deleted, GitLab displays a message confirming your action. + ## Connection to Prometheus The project will be automatically configured to connect to the diff --git a/doc/ci/environments.md b/doc/ci/environments.md index bd3e4a17ef6..61f09bf6e00 100644 --- a/doc/ci/environments.md +++ b/doc/ci/environments.md @@ -220,7 +220,7 @@ deployment will be recorded as a new environment named `production`. NOTE: **Note:** If your environment's name is `production` (all lowercase), -it will get recorded in [Cycle Analytics](../user/project/cycle_analytics.md). +it will get recorded in [Value Stream Analytics](../user/project/cycle_analytics.md). ### Configuring dynamic environments diff --git a/doc/development/README.md b/doc/development/README.md index ba1714b2746..7511221b246 100644 --- a/doc/development/README.md +++ b/doc/development/README.md @@ -80,7 +80,7 @@ Complementary reads: - [File uploads](uploads.md) - [Auto DevOps development guide](auto_devops.md) - [Mass Inserting Models](mass_insert.md) -- [Cycle Analytics development guide](cycle_analytics.md) +- [Value Stream Analytics development guide](value_stream_analytics.md) - [Issue types vs first-class types](issue_types.md) - [Application limits](application_limits.md) - [Redis guidelines](redis.md) diff --git a/doc/development/cycle_analytics.md b/doc/development/cycle_analytics.md index 90abafb3e03..3947a012bd5 100644 --- a/doc/development/cycle_analytics.md +++ b/doc/development/cycle_analytics.md @@ -1,246 +1,5 @@ -# Cycle Analytics development guide +--- +redirect_to: 'value_stream_analytics.md' +--- -Cycle analytics calculates the time between two arbitrary events recorded on domain objects and provides aggregated statistics about the duration. - -## Stage - -During development, events occur that move issues and merge requests through different stages of progress until they are considered finished. These stages can be expressed with the `Stage` model. - -Example stage: - -- Name: Development -- Start event: Issue created -- End event: Issue first mentioned in commit -- Parent: `Group: gitlab-org` - -### Events - -Events are the smallest building blocks of the cycle analytics feature. A stage consists of two events: - -- Start -- End - -These events play a key role in the duration calculation. - -Formula: `duration = end_event_time - start_event_time` - -To make the duration calculation flexible, each `Event` is implemented as a separate class. They're responsible for defining a timestamp expression that will be used in the calculation query. - -#### Implementing an `Event` class - -There are a few methods that are required to be implemented, the `StageEvent` base class describes them in great detail. The most important ones are: - -- `object_type` -- `timestamp_projection` - -The `object_type` method defines which domain object will be queried for the calculation. Currently two models are allowed: - -- `Issue` -- `MergeRequest` - -For the duration calculation the `timestamp_projection` method will be used. - -```ruby -def timestamp_projection - # your timestamp expression comes here -end - -# event will use the issue creation time in the duration calculation -def timestamp_projection - Issue.arel_table[:created_at] -end -``` - -NOTE: **Note:** -More complex expressions are also possible (e.g. using `COALESCE`). Look at the existing event classes for examples. - -In some cases, defining the `timestamp_projection` method is not enough. The calculation query should know which table contains the timestamp expression. Each `Event` class is responsible for making modifications to the calculation query to make the `timestamp_projection` work. This usually means joining an additional table. - -Example for joining the `issue_metrics` table and using the `first_mentioned_in_commit_at` column as the timestamp expression: - -```ruby -def object_type - Issue -end - -def timestamp_projection - IssueMetrics.arel_table[:first_mentioned_in_commit_at] -end - -def apply_query_customization(query) - # in this case the query attribute will be based on the Issue model: `Issue.where(...)` - query.joins(:metrics) -end -``` - -### Validating start and end events - -Some start/end event pairs are not "compatible" with each other. For example: - -- "Issue created" to "Merge Request created": The event classes are defined on different domain models, the `object_type` method is different. -- "Issue closed" to "Issue created": Issue must be created first before it can be closed. -- "Issue closed" to "Issue closed": Duration is always 0. - -The `StageEvents` module describes the allowed `start_event` and `end_event` pairings (`PAIRING_RULES` constant). If a new event is added, it needs to be registered in this module. -To add a new event: - -1. Add an entry in `ENUM_MAPPING` with a unique number, it'll be used in the `Stage` model as `enum`. -1. Define which events are compatible with the event in the `PAIRING_RULES` hash. - -Supported start/end event pairings: - -```mermaid -graph LR; - IssueCreated --> IssueClosed; - IssueCreated --> IssueFirstAddedToBoard; - IssueCreated --> IssueFirstAssociatedWithMilestone; - IssueCreated --> IssueFirstMentionedInCommit; - IssueCreated --> IssueLastEdited; - IssueCreated --> IssueLabelAdded; - IssueCreated --> IssueLabelRemoved; - MergeRequestCreated --> MergeRequestMerged; - MergeRequestCreated --> MergeRequestClosed; - MergeRequestCreated --> MergeRequestFirstDeployedToProduction; - MergeRequestCreated --> MergeRequestLastBuildStarted; - MergeRequestCreated --> MergeRequestLastBuildFinished; - MergeRequestCreated --> MergeRequestLastEdited; - MergeRequestCreated --> MergeRequestLabelAdded; - MergeRequestCreated --> MergeRequestLabelRemoved; - MergeRequestLastBuildStarted --> MergeRequestLastBuildFinished; - MergeRequestLastBuildStarted --> MergeRequestClosed; - MergeRequestLastBuildStarted --> MergeRequestFirstDeployedToProduction; - MergeRequestLastBuildStarted --> MergeRequestLastEdited; - MergeRequestLastBuildStarted --> MergeRequestMerged; - MergeRequestLastBuildStarted --> MergeRequestLabelAdded; - MergeRequestLastBuildStarted --> MergeRequestLabelRemoved; - MergeRequestMerged --> MergeRequestFirstDeployedToProduction; - MergeRequestMerged --> MergeRequestClosed; - MergeRequestMerged --> MergeRequestFirstDeployedToProduction; - MergeRequestMerged --> MergeRequestLastEdited; - MergeRequestMerged --> MergeRequestLabelAdded; - MergeRequestMerged --> MergeRequestLabelRemoved; - IssueLabelAdded --> IssueLabelAdded; - IssueLabelAdded --> IssueLabelRemoved; - IssueLabelAdded --> IssueClosed; - IssueLabelRemoved --> IssueClosed; - IssueFirstAddedToBoard --> IssueClosed; - IssueFirstAddedToBoard --> IssueFirstAssociatedWithMilestone; - IssueFirstAddedToBoard --> IssueFirstMentionedInCommit; - IssueFirstAddedToBoard --> IssueLastEdited; - IssueFirstAddedToBoard --> IssueLabelAdded; - IssueFirstAddedToBoard --> IssueLabelRemoved; - IssueFirstAssociatedWithMilestone --> IssueClosed; - IssueFirstAssociatedWithMilestone --> IssueFirstAddedToBoard; - IssueFirstAssociatedWithMilestone --> IssueFirstMentionedInCommit; - IssueFirstAssociatedWithMilestone --> IssueLastEdited; - IssueFirstAssociatedWithMilestone --> IssueLabelAdded; - IssueFirstAssociatedWithMilestone --> IssueLabelRemoved; - IssueFirstMentionedInCommit --> IssueClosed; - IssueFirstMentionedInCommit --> IssueFirstAssociatedWithMilestone; - IssueFirstMentionedInCommit --> IssueFirstAddedToBoard; - IssueFirstMentionedInCommit --> IssueLastEdited; - IssueFirstMentionedInCommit --> IssueLabelAdded; - IssueFirstMentionedInCommit --> IssueLabelRemoved; - IssueClosed --> IssueLastEdited; - IssueClosed --> IssueLabelAdded; - IssueClosed --> IssueLabelRemoved; - MergeRequestClosed --> MergeRequestFirstDeployedToProduction; - MergeRequestClosed --> MergeRequestLastEdited; - MergeRequestClosed --> MergeRequestLabelAdded; - MergeRequestClosed --> MergeRequestLabelRemoved; - MergeRequestFirstDeployedToProduction --> MergeRequestLastEdited; - MergeRequestFirstDeployedToProduction --> MergeRequestLabelAdded; - MergeRequestFirstDeployedToProduction --> MergeRequestLabelRemoved; - MergeRequestLastBuildFinished --> MergeRequestClosed; - MergeRequestLastBuildFinished --> MergeRequestFirstDeployedToProduction; - MergeRequestLastBuildFinished --> MergeRequestLastEdited; - MergeRequestLastBuildFinished --> MergeRequestMerged; - MergeRequestLastBuildFinished --> MergeRequestLabelAdded; - MergeRequestLastBuildFinished --> MergeRequestLabelRemoved; - MergeRequestLabelAdded --> MergeRequestLabelAdded; - MergeRequestLabelAdded --> MergeRequestLabelRemoved; - MergeRequestLabelRemoved --> MergeRequestLabelAdded; - MergeRequestLabelRemoved --> MergeRequestLabelRemoved; -``` - -### Parent - -Teams and organizations might define their own way of building software, thus stages can be completely different. For each stage, a parent object needs to be defined. - -Currently supported parents: - -- `Project` -- `Group` - -#### How parent relationship it work - -1. User navigates to the cycle analytics page. -1. User selects a group. -1. Backend loads the defined stages for the selected group. -1. Additions and modifications to the stages will be persisted within the selected group only. - -### Default stages - -The [original implementation](https://gitlab.com/gitlab-org/gitlab/issues/847) of cycle analytics defined 7 stages. These stages are always available for each parent, however altering these stages is not possible. - -To make things efficient and reduce the number of records created, the default stages are expressed as in-memory objects (not persisted). When the user creates a custom stage for the first time, all the stages will be persisted. This behaviour is implemented in the cycle analytics service objects. - -The reason for this was that we'd like to add the abilities to hide and order stages later on. - -## Data Collector - -`DataCollector` is the central point where the data will be queried from the database. The class always operates on a single stage and consists of the following components: - -- `BaseQueryBuilder`: - - Responsible for composing the initial query. - - Deals with `Stage` specific configuration: events and their query customizations. - - Parameters coming from the UI: date ranges. -- `Median`: Calculates the median duration for a stage using the query from `BaseQueryBuilder`. -- `RecordsFetcher`: Loads relevant records for a stage using the query from `BaseQueryBuilder` and specific `Finder` classes to apply visibility rules. -- `DataForDurationChart`: Loads calculated durations with the finish time (end event timestamp) for the scatterplot chart. - -For a new calculation or a query, implement it as a new method call in the `DataCollector` class. - -## Database query - -Structure of the database query: - -```sql -SELECT (customized by: Median or RecordsFetcher or DataForDurationChart) -FROM OBJECT_TYPE (Issue or MergeRequest) -INNER JOIN (several JOIN statements, depending on the events) -WHERE - (Filter by the PARENT model, example: filter Issues from Project A) - (Date range filter based on the OBJECT_TYPE.created_at) - (Check if the START_EVENT is earlier than END_EVENT, preventing negative duration) -``` - -Structure of the `SELECT` statement for `Median`: - -```sql -SELECT (calculate median from START_EVENT_TIME-END_EVENT_TIME) -``` - -Structure of the `SELECT` statement for `DataForDurationChart`: - -```sql -SELECT (START_EVENT_TIME-END_EVENT_TIME) as duration, END_EVENT.timestamp -``` - -## High-level overview - -- Rails Controller (`Analytics::CycleAnalytics` module): Cycle analytics exposes its data via JSON endpoints, implemented within the `analytics` workspace. Configuring the stages are also implements JSON endpoints (CRUD). -- Services (`Analytics::CycleAnalytics` module): All `Stage` related actions will be delegated to respective service objects. -- Models (`Analytics::CycleAnalytics` module): Models are used to persist the `Stage` objects `ProjectStage` and `GroupStage`. -- Feature classes (`Gitlab::Analytics::CycleAnalytics` module): - - Responsible for composing queries and define feature specific busines logic. - - `DataCollector`, `Event`, `StageEvents`, etc. - -## Testing - -Since we have a lots of events and possible pairings, testing each pairing is not possible. The rule is to have at least one test case using an `Event` class. - -Writing a test case for a stage using a new `Event` can be challenging since data must be created for both events. To make this a bit simpler, each test case must be implemented in the `data_collector_spec.rb` where the stage is tested through the `DataCollector`. Each test case will be turned into multiple tests, covering the following cases: - -- Different parents: `Group` or `Project` -- Different calculations: `Median`, `RecordsFetcher` or `DataForDurationChart` +This document was moved to [another location](value_stream_analytics.md) diff --git a/doc/development/i18n/externalization.md b/doc/development/i18n/externalization.md index b3a08eed717..2595ee5b6d3 100644 --- a/doc/development/i18n/externalization.md +++ b/doc/development/i18n/externalization.md @@ -559,5 +559,5 @@ Let's suppose you want to add translations for a new language, let's say French. ```shell git add locale/fr/ app/assets/javascripts/locale/fr/ - git commit -m "Add French translations for Cycle Analytics page" + git commit -m "Add French translations for Value Stream Analytics page" ``` diff --git a/doc/development/testing_guide/best_practices.md b/doc/development/testing_guide/best_practices.md index 54666d67b8d..3285aaed095 100644 --- a/doc/development/testing_guide/best_practices.md +++ b/doc/development/testing_guide/best_practices.md @@ -572,6 +572,27 @@ Example: expect(response).to have_gitlab_http_status(:ok) ``` +### Testing query performance + +Testing query performance allows us to: + +- Assert that N+1 problems do not exist within a block of code. +- Ensure that the number of queries within a block of code does not increase unnoticed. + +#### QueryRecorder + +`QueryRecorder` allows profiling and testing of the number of database queries +performed within a given block of code. + +See the [`QueryRecorder`](../query_recorder.md) section for more details. + +#### GitalyClient + +`Gitlab::GitalyClient.get_request_count` allows tests of the number of Gitaly queries +made by a given block of code: + +See the [`Gitaly Request Counts`](../gitaly.md#request-counts) section for more details. + ### Shared contexts Shared contexts only used in one spec file can be declared inline. diff --git a/doc/development/value_stream_analytics.md b/doc/development/value_stream_analytics.md new file mode 100644 index 00000000000..5c6d3f18d9a --- /dev/null +++ b/doc/development/value_stream_analytics.md @@ -0,0 +1,248 @@ +# Value Stream Analytics development guide + +Value stream analytics calculates the time between two arbitrary events recorded on domain objects and provides aggregated statistics about the duration. + +For information on how to configure Value Stream Analytics in GitLab, see our [analytics documentation](../user/analytics/value_stream_analytics.md). + +## Stage + +During development, events occur that move issues and merge requests through different stages of progress until they are considered finished. These stages can be expressed with the `Stage` model. + +Example stage: + +- Name: Development +- Start event: Issue created +- End event: Issue first mentioned in commit +- Parent: `Group: gitlab-org` + +### Events + +Events are the smallest building blocks of the value stream analytics feature. A stage consists of two events: + +- Start +- End + +These events play a key role in the duration calculation. + +Formula: `duration = end_event_time - start_event_time` + +To make the duration calculation flexible, each `Event` is implemented as a separate class. They're responsible for defining a timestamp expression that will be used in the calculation query. + +#### Implementing an `Event` class + +There are a few methods that are required to be implemented, the `StageEvent` base class describes them in great detail. The most important ones are: + +- `object_type` +- `timestamp_projection` + +The `object_type` method defines which domain object will be queried for the calculation. Currently two models are allowed: + +- `Issue` +- `MergeRequest` + +For the duration calculation the `timestamp_projection` method will be used. + +```ruby +def timestamp_projection + # your timestamp expression comes here +end + +# event will use the issue creation time in the duration calculation +def timestamp_projection + Issue.arel_table[:created_at] +end +``` + +NOTE: **Note:** +More complex expressions are also possible (e.g. using `COALESCE`). Look at the existing event classes for examples. + +In some cases, defining the `timestamp_projection` method is not enough. The calculation query should know which table contains the timestamp expression. Each `Event` class is responsible for making modifications to the calculation query to make the `timestamp_projection` work. This usually means joining an additional table. + +Example for joining the `issue_metrics` table and using the `first_mentioned_in_commit_at` column as the timestamp expression: + +```ruby +def object_type + Issue +end + +def timestamp_projection + IssueMetrics.arel_table[:first_mentioned_in_commit_at] +end + +def apply_query_customization(query) + # in this case the query attribute will be based on the Issue model: `Issue.where(...)` + query.joins(:metrics) +end +``` + +### Validating start and end events + +Some start/end event pairs are not "compatible" with each other. For example: + +- "Issue created" to "Merge Request created": The event classes are defined on different domain models, the `object_type` method is different. +- "Issue closed" to "Issue created": Issue must be created first before it can be closed. +- "Issue closed" to "Issue closed": Duration is always 0. + +The `StageEvents` module describes the allowed `start_event` and `end_event` pairings (`PAIRING_RULES` constant). If a new event is added, it needs to be registered in this module. +To add a new event: + +1. Add an entry in `ENUM_MAPPING` with a unique number, it'll be used in the `Stage` model as `enum`. +1. Define which events are compatible with the event in the `PAIRING_RULES` hash. + +Supported start/end event pairings: + +```mermaid +graph LR; + IssueCreated --> IssueClosed; + IssueCreated --> IssueFirstAddedToBoard; + IssueCreated --> IssueFirstAssociatedWithMilestone; + IssueCreated --> IssueFirstMentionedInCommit; + IssueCreated --> IssueLastEdited; + IssueCreated --> IssueLabelAdded; + IssueCreated --> IssueLabelRemoved; + MergeRequestCreated --> MergeRequestMerged; + MergeRequestCreated --> MergeRequestClosed; + MergeRequestCreated --> MergeRequestFirstDeployedToProduction; + MergeRequestCreated --> MergeRequestLastBuildStarted; + MergeRequestCreated --> MergeRequestLastBuildFinished; + MergeRequestCreated --> MergeRequestLastEdited; + MergeRequestCreated --> MergeRequestLabelAdded; + MergeRequestCreated --> MergeRequestLabelRemoved; + MergeRequestLastBuildStarted --> MergeRequestLastBuildFinished; + MergeRequestLastBuildStarted --> MergeRequestClosed; + MergeRequestLastBuildStarted --> MergeRequestFirstDeployedToProduction; + MergeRequestLastBuildStarted --> MergeRequestLastEdited; + MergeRequestLastBuildStarted --> MergeRequestMerged; + MergeRequestLastBuildStarted --> MergeRequestLabelAdded; + MergeRequestLastBuildStarted --> MergeRequestLabelRemoved; + MergeRequestMerged --> MergeRequestFirstDeployedToProduction; + MergeRequestMerged --> MergeRequestClosed; + MergeRequestMerged --> MergeRequestFirstDeployedToProduction; + MergeRequestMerged --> MergeRequestLastEdited; + MergeRequestMerged --> MergeRequestLabelAdded; + MergeRequestMerged --> MergeRequestLabelRemoved; + IssueLabelAdded --> IssueLabelAdded; + IssueLabelAdded --> IssueLabelRemoved; + IssueLabelAdded --> IssueClosed; + IssueLabelRemoved --> IssueClosed; + IssueFirstAddedToBoard --> IssueClosed; + IssueFirstAddedToBoard --> IssueFirstAssociatedWithMilestone; + IssueFirstAddedToBoard --> IssueFirstMentionedInCommit; + IssueFirstAddedToBoard --> IssueLastEdited; + IssueFirstAddedToBoard --> IssueLabelAdded; + IssueFirstAddedToBoard --> IssueLabelRemoved; + IssueFirstAssociatedWithMilestone --> IssueClosed; + IssueFirstAssociatedWithMilestone --> IssueFirstAddedToBoard; + IssueFirstAssociatedWithMilestone --> IssueFirstMentionedInCommit; + IssueFirstAssociatedWithMilestone --> IssueLastEdited; + IssueFirstAssociatedWithMilestone --> IssueLabelAdded; + IssueFirstAssociatedWithMilestone --> IssueLabelRemoved; + IssueFirstMentionedInCommit --> IssueClosed; + IssueFirstMentionedInCommit --> IssueFirstAssociatedWithMilestone; + IssueFirstMentionedInCommit --> IssueFirstAddedToBoard; + IssueFirstMentionedInCommit --> IssueLastEdited; + IssueFirstMentionedInCommit --> IssueLabelAdded; + IssueFirstMentionedInCommit --> IssueLabelRemoved; + IssueClosed --> IssueLastEdited; + IssueClosed --> IssueLabelAdded; + IssueClosed --> IssueLabelRemoved; + MergeRequestClosed --> MergeRequestFirstDeployedToProduction; + MergeRequestClosed --> MergeRequestLastEdited; + MergeRequestClosed --> MergeRequestLabelAdded; + MergeRequestClosed --> MergeRequestLabelRemoved; + MergeRequestFirstDeployedToProduction --> MergeRequestLastEdited; + MergeRequestFirstDeployedToProduction --> MergeRequestLabelAdded; + MergeRequestFirstDeployedToProduction --> MergeRequestLabelRemoved; + MergeRequestLastBuildFinished --> MergeRequestClosed; + MergeRequestLastBuildFinished --> MergeRequestFirstDeployedToProduction; + MergeRequestLastBuildFinished --> MergeRequestLastEdited; + MergeRequestLastBuildFinished --> MergeRequestMerged; + MergeRequestLastBuildFinished --> MergeRequestLabelAdded; + MergeRequestLastBuildFinished --> MergeRequestLabelRemoved; + MergeRequestLabelAdded --> MergeRequestLabelAdded; + MergeRequestLabelAdded --> MergeRequestLabelRemoved; + MergeRequestLabelRemoved --> MergeRequestLabelAdded; + MergeRequestLabelRemoved --> MergeRequestLabelRemoved; +``` + +### Parent + +Teams and organizations might define their own way of building software, thus stages can be completely different. For each stage, a parent object needs to be defined. + +Currently supported parents: + +- `Project` +- `Group` + +#### How parent relationship it work + +1. User navigates to the value stream analytics page. +1. User selects a group. +1. Backend loads the defined stages for the selected group. +1. Additions and modifications to the stages will be persisted within the selected group only. + +### Default stages + +The [original implementation](https://gitlab.com/gitlab-org/gitlab/issues/847) of value stream analytics defined 7 stages. These stages are always available for each parent, however altering these stages is not possible. + +To make things efficient and reduce the number of records created, the default stages are expressed as in-memory objects (not persisted). When the user creates a custom stage for the first time, all the stages will be persisted. This behaviour is implemented in the value stream analytics service objects. + +The reason for this was that we'd like to add the abilities to hide and order stages later on. + +## Data Collector + +`DataCollector` is the central point where the data will be queried from the database. The class always operates on a single stage and consists of the following components: + +- `BaseQueryBuilder`: + - Responsible for composing the initial query. + - Deals with `Stage` specific configuration: events and their query customizations. + - Parameters coming from the UI: date ranges. +- `Median`: Calculates the median duration for a stage using the query from `BaseQueryBuilder`. +- `RecordsFetcher`: Loads relevant records for a stage using the query from `BaseQueryBuilder` and specific `Finder` classes to apply visibility rules. +- `DataForDurationChart`: Loads calculated durations with the finish time (end event timestamp) for the scatterplot chart. + +For a new calculation or a query, implement it as a new method call in the `DataCollector` class. + +## Database query + +Structure of the database query: + +```sql +SELECT (customized by: Median or RecordsFetcher or DataForDurationChart) +FROM OBJECT_TYPE (Issue or MergeRequest) +INNER JOIN (several JOIN statements, depending on the events) +WHERE + (Filter by the PARENT model, example: filter Issues from Project A) + (Date range filter based on the OBJECT_TYPE.created_at) + (Check if the START_EVENT is earlier than END_EVENT, preventing negative duration) +``` + +Structure of the `SELECT` statement for `Median`: + +```sql +SELECT (calculate median from START_EVENT_TIME-END_EVENT_TIME) +``` + +Structure of the `SELECT` statement for `DataForDurationChart`: + +```sql +SELECT (START_EVENT_TIME-END_EVENT_TIME) as duration, END_EVENT.timestamp +``` + +## High-level overview + +- Rails Controller (`Analytics::CycleAnalytics` module): Value stream analytics exposes its data via JSON endpoints, implemented within the `analytics` workspace. Configuring the stages are also implements JSON endpoints (CRUD). +- Services (`Analytics::CycleAnalytics` module): All `Stage` related actions will be delegated to respective service objects. +- Models (`Analytics::CycleAnalytics` module): Models are used to persist the `Stage` objects `ProjectStage` and `GroupStage`. +- Feature classes (`Gitlab::Analytics::CycleAnalytics` module): + - Responsible for composing queries and define feature specific busines logic. + - `DataCollector`, `Event`, `StageEvents`, etc. + +## Testing + +Since we have a lots of events and possible pairings, testing each pairing is not possible. The rule is to have at least one test case using an `Event` class. + +Writing a test case for a stage using a new `Event` can be challenging since data must be created for both events. To make this a bit simpler, each test case must be implemented in the `data_collector_spec.rb` where the stage is tested through the `DataCollector`. Each test case will be turned into multiple tests, covering the following cases: + +- Different parents: `Group` or `Project` +- Different calculations: `Median`, `RecordsFetcher` or `DataForDurationChart` diff --git a/doc/topics/web_application_firewall/img/guide_waf_ingress_installation.png b/doc/topics/web_application_firewall/img/guide_waf_ingress_installation.png Binary files differnew file mode 100644 index 00000000000..a150fa4e46f --- /dev/null +++ b/doc/topics/web_application_firewall/img/guide_waf_ingress_installation.png diff --git a/doc/topics/web_application_firewall/index.md b/doc/topics/web_application_firewall/index.md new file mode 100644 index 00000000000..db1265e08ac --- /dev/null +++ b/doc/topics/web_application_firewall/index.md @@ -0,0 +1,95 @@ +# Web Application Firewall - ModSecurity + +A web application firewall (or WAF) filters, monitors, and blocks HTTP traffic to +and from a web application. By inspecting HTTP traffic, it can prevent attacks +stemming from web application security flaws. It can be used to detect SQL injection, +Cross-Site Scripting (XSS), Remote File Inclusion, Security Misconfigurations, and +much more. + +## Overview + +GitLab provides a WAF out of the box after Ingress is deployed. +All you need to do is deploy your application along with a service +and Ingress resource. + +In GitLab's [Ingress](../../user/clusters/applications.md#ingress) deployment, the [ModSecurity](https://modsecurity.org/) module is loaded +into Ingress-NGINX by default and monitors the traffic going to the +applications which have an Ingress. + +The ModSecurity module runs with the [OWASP Core Rule Set (CRS)](https://coreruleset.org/) by default. The OWASP CRS will detect and log a wide range of common attacks. + +NOTE: **Note** +The WAF is deployed in "Detection-only mode" by default and will only log attack +attempts. + +## Requirements + +The Web Application Firewall requires: + +- **Kubernetes** + + To enable the WAF, you need: + + - Kubernetes 1.12+. + - A load balancer. You can use NGINX-Ingress by deploying it to your + Kubernetes cluster by either: + - Using the [`nginx-ingress` Helm chart](https://github.com/helm/charts/tree/master/stable/nginx-ingress). + - Installing the [Ingress GitLab Managed App](../../user/clusters/applications.md#ingress) with WAF enabled. + +- **Configured Kubernetes objects** + + To use the WAF on an application, you need to deploy the following Kubernetes resources: + + - [Deployment](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/) + - [Service](https://kubernetes.io/docs/concepts/services-networking/service/) + - [Ingress Resource](https://kubernetes.io/docs/concepts/services-networking/ingress/) + +## Quick start + +If you are using GitLab.com, see the [quick start guide](quick_start_guide.md) for +how to use the WAF with GitLab.com and a Kubernetes cluster on Google Kubernetes Engine (GKE). + +If you are using a self-hosted instance of GitLab, you need to configure the +[Google OAuth2 OmniAuth Provider](../../integration/google.md) before +you can configure a cluster on GKE. Once this is set up, you can follow the steps on the [quick start guide](quick_start_guide.md) to get started. + +NOTE: **Note** +This guide shows how the WAF can be deployed using Auto DevOps. The WAF +is avaliable by default to all applications no matter how they are deployed, +as long as they are using Ingress. + +## Network firewall vs. Web Application Firewall + +A network firewall or packet filter looks at traffic at the Network (L3) and Transport (L4) layers +of the [OSI Model](https://en.wikipedia.org/wiki/OSI_model), and denies packets from entry based on +a set of rules regarding the network in general. + +A Web Application Firewall operates at the Application (L7) layer of the OSI Model and can +examine all the packets traveling to and from a specific application. A WAF can set +more advanced rules around threat detection. + +## Features + +ModSecurity is enabled with the [OWASP Core Rule Set (CRS)](https://modsecurity.org/crs/) by +default. The OWASP CRS logs attempts to the following attacks: + +- [SQL Injection](https://www.owasp.org/index.php/OWASP_Periodic_Table_of_Vulnerabilities_-_SQL_Injection) +- [Cross-Site Scripting](https://www.owasp.org/index.php/OWASP_Periodic_Table_of_Vulnerabilities_-_Cross-Site_Scripting_(XSS)) +- [Local File Inclusion](https://www.owasp.org/index.php/Testing_for_Local_File_Inclusion) +- [Remote File Inclusion](https://www.owasp.org/index.php/OWASP_Periodic_Table_of_Vulnerabilities_-_Remote_File_Inclusion) +- [Code Injection](https://www.owasp.org/index.php/Code_Injection) +- [Session Fixation](https://www.owasp.org/index.php/Session_fixation) +- [Scanner Detection](https://www.owasp.org/index.php/Category:Vulnerability_Scanning_Tools) +- [Metadata/Error Leakages](https://www.owasp.org/index.php/Improper_Error_Handling) + +It is good to have a basic knowledge of the following: + +- [Kubernetes](https://kubernetes.io/docs/home/) +- [Ingress](https://kubernetes.github.io/ingress-nginx/) +- [ModSecurity](https://www.modsecurity.org/) +- [OWASP Core Rule Set](https://modsecurity.org/crs/) + +## Roadmap + +More information on the direction of the WAF can be +found in [Product Vision - Defend](https://about.gitlab.com/direction/defend/#waf) diff --git a/doc/topics/web_application_firewall/quick_start_guide.md b/doc/topics/web_application_firewall/quick_start_guide.md new file mode 100644 index 00000000000..35003785753 --- /dev/null +++ b/doc/topics/web_application_firewall/quick_start_guide.md @@ -0,0 +1,266 @@ +# Getting started with the Web Application Firewall + +This is a step-by-step guide that will help you use GitLab's [Web Application Firewall](index.md) after +deploying a project hosted on GitLab.com to Google Kubernetes Engine using [Auto DevOps](../autodevops/index.md). + +We will use GitLab's native Kubernetes integration, so you will not need +to create a Kubernetes cluster manually using the Google Cloud Platform console. +We will create and deploy a simple application that we create from a GitLab template. + +These instructions will also work for a self-hosted GitLab instance. However, you will +need to ensure your own [Runners are configured](../../ci/runners/README.md) and +[Google OAuth is enabled](../../integration/google.md). + +**Note**: GitLab's Web Application Firewall is deployed with [Ingress](../../user/clusters/applications.md#Ingress), +so it will be avaliable to your applications no matter how you deploy them to Kubernetes. + +## Enable or disable ModSecurity + +ModSecurity is enabled by default on GitLab.com. You can toggle the feature flag to false by running the following command in the Rails console: + +```ruby +Feature.disable(:ingress_modsecurity) +``` + +Once disabled, you must uninstall and reinstall your Ingress application for the changes to take effect. See the [Feature Flag](../../user/project/operations/feature_flags.md) documentation for more information. + +## Configuring your Google account + +Before creating and connecting your Kubernetes cluster to your GitLab project, +you need a Google Cloud Platform account. If you do not already have one, +sign up at <https://console.cloud.google.com>. You will need to either sign in with an existing +Google account (for example, one that you use to access Gmail, Drive, etc.) or create a new one. + +1. To enable the required APIs and related services, follow the steps in the ["Before you begin" section of the Kubernetes Engine docs](https://cloud.google.com/kubernetes-engine/docs/quickstart#before-you-begin). +1. Make sure you have created a [billing account](https://cloud.google.com/billing/docs/how-to/manage-billing-account). + +TIP: **Tip:** +Every new Google Cloud Platform (GCP) account receives [$300 in credit](https://console.cloud.google.com/freetrial), +and in partnership with Google, GitLab is able to offer an additional $200 for new GCP accounts to get started with GitLab's +Google Kubernetes Engine integration. All you have to do is [follow this link](https://cloud.google.com/partners/partnercredit/?PCN=a0n60000006Vpz4AAC) and apply for credit. + +## Creating a new project from a template + +We will use one of GitLab's project templates to get started. As the name suggests, +those projects provide a barebones application built on some well-known frameworks. + +1. In GitLab, click the plus icon (**+**) at the top of the navigation bar and select + **New project**. +1. Go to the **Create from template** tab where you can choose for example a Ruby on + Rails, Spring, or NodeJS Express project. + We will use the Ruby on Rails template. + +  + +1. Give your project a name, optionally a description, and make it public so that + you can take advantage of the features available in the + [GitLab Gold plan](https://about.gitlab.com/pricing/#gitlab-com). + +  + +1. Click **Create project**. + +Now that the project is created, the next step is to create the Kubernetes cluster +under which this application will be deployed. + +## Creating a Kubernetes cluster from within GitLab + +1. On the project's landing page, click **Add Kubernetes cluster** + (note that this option is also available when you navigate to **Operations > Kubernetes**). + +  + +1. On the **Create new cluster on GKE** tab, click **Sign in with Google**. + +  + +1. Connect with your Google account and click **Allow** when asked (this + appears only the first time you connect GitLab with your Google account). + +  + +1. The last step is to provide the cluster details. + 1. Give it a name, leave the environment scope as is, and choose the GCP project under which the cluster + will be created (per the instructions to [configure your Google account](#configuring-your-google-account), a project should have already been created for you). + 1. Choose the [region/zone](https://cloud.google.com/compute/docs/regions-zones/) under which the cluster will be created. + 1. Enter the number of nodes you want it to have. + 1. Choose the [machine type](https://cloud.google.com/compute/docs/machine-types). + +  + +1. Click **Create Kubernetes cluster**. + +After a couple of minutes, the cluster is created. You can also see its +status on your [GCP dashboard](https://console.cloud.google.com/kubernetes). + +The next step is to install some applications on your cluster that are needed +to take full advantage of Auto DevOps. + +## Installing Helm and Ingress + +GitLab's Kubernetes integration comes with some +[pre-defined applications](../../user/project/clusters/index.md#installing-applications) +for you to install. + + + +The first one to install is Helm Tiller, a package manager for Kubernetes, which +is needed in order to install the rest of the applications. Go ahead and click +its **Install** button. +Once it is installed, the other applications that rely on it will each have their +**Install** buttons enabled. + +For this guide, we need to install Ingress. Ingress provides load balancing, +SSL termination, and name-based virtual hosting, using NGINX behind +the scenes. Make sure that the **Enable Web Application Firewall** button is checked +before installing. + + + +After Ingress is installed, wait a few seconds and copy the IP address that +is displayed in order to add in your base **Domain** at the top of the page. For +the purpose of this guide, we will use the one suggested by GitLab. Once you have +filled in the domain, click **Save changes**. + + + +Prometheus should also be installed. It is an open-source monitoring and +alerting system that we will use to supervise the deployed application. +We will not install GitLab Runners as we will use the shared Runners that +GitLab.com provides. + +## Enabling Auto DevOps (optional) + +Starting with GitLab 11.3, Auto DevOps is enabled by default. However, it is possible to disable +Auto DevOps at both the instance-level (for self-managed instances) and the group-level. +Follow these steps if Auto DevOps has been manually disabled: + +1. Navigate to **Settings > CI/CD > Auto DevOps**. +1. Select **Default to Auto DevOps pipeline**. +1. Select the [continuous deployment strategy](../autodevops/index.md#deployment-strategy) + which automatically deploys the application to production once the pipeline + successfully runs on the `master` branch. +1. Click **Save changes**. + +  + +Once you complete all the above and save your changes, a new pipeline is +automatically created. To view the pipeline, go to **CI/CD > Pipelines**. + + + +The next section explains what each pipeline job does. + +## Deploying the application + +By now you should see the pipeline running, but what is it running exactly? + +To navigate inside the pipeline, click its status badge (its status should be "Running"). +The pipeline is split into a few stages, each running a couple of jobs. + + + +In the **build** stage, the application is built into a Docker image and then +uploaded to your project's [Container Registry](../../user/packages/container_registry/index.md) ([Auto Build](../autodevops/index.md#auto-build)). + +In the **test** stage, GitLab runs various checks on the application. + +The **production** stage is run after the tests and checks finish, and it automatically +deploys the application in Kubernetes ([Auto Deploy](../autodevops/index.md#auto-deploy)). + +The **production** stage creates Kubernetes objects +like a Deployment, Service, and Ingress resource. The +application will be monitored by the WAF automatically. + +## Validating Ingress is running ModSecurity + +Now we can make sure that Ingress is running properly with ModSecurity and send +a request to ensure our application is responding correctly. You must connect to +your cluster either using [Cloud Shell](https://cloud.google.com/shell/) or the [Google Cloud SDK](https://cloud.google.com/sdk/install). + +1. After connecting to your cluster, check if the Ingress-NGINX controller is running and ModSecurity is enabled. + + This is done by running the following commands: + + ```bash + $ kubectl get pods -n gitlab-managed-apps | grep 'ingress-controller' + ingress-nginx-ingress-controller-55f9cf6584-dxljn 2/2 Running + + $ kubectl -n gitlab-managed-apps exec -it $(kubectl get pods -n gitlab-managed-apps | grep 'ingress-controller' | awk '{print $1}') -- cat /etc/nginx/nginx.conf | grep 'modsecurity on;' + modsecurity on; + ``` + +1. Verify the Rails application has been installed properly. + + ```bash + $ kubectl get ns + auto-devv-2-16730183-production Active + + $ kubectl get pods -n auto-devv-2-16730183-production + NAME READY STATUS RESTARTS + production-5778cfcfcd-nqjcm 1/1 Running 0 + production-postgres-6449f8cc98-r7xgg 1/1 Running 0 + ``` + +1. To make sure the Rails application is responding, send a request to it by running: + + ```bash + $ kubectl get ing -n auto-devv-2-16730183-production + NAME HOSTS PORTS + production-auto-deploy fjdiaz-auto-devv-2.34.68.60.207.nip.io,le-16730183.34.68.60.207.nip.io 80, 443 + + $ curl --location --insecure fjdiaz-auto-devv-2.34.68.60.207.nip.io | grep 'Rails!' --after 2 --before 2 + <body> + <p>You're on Rails!</p> + </body> + ``` + +Now that we have confirmed our system is properly setup, we can go ahead and test +the WAF with OWASP CRS! + +## Testing out the OWASP Core Rule Set + +Now let's send a potentially malicious request, as if we were a scanner, +checking for vulnerabilities within our application and examine the modsecurity logs: + +```bash +$ curl --location --insecure fjdiaz-auto-devv-2.34.68.60.207.nip.io --header "User-Agent: absinthe" | grep 'Rails!' --after 2 --before 2 +<body> + <p>You're on Rails!</p> +</body> + +$ kubectl -n gitlab-managed-apps exec -it $(kubectl get pods -n gitlab-managed-apps | grep 'ingress-controller' | awk '{print $1}') -- cat /var/log/modsec/audit.log | grep 'absinthe' +{ + "message": "Found User-Agent associated with security scanner", + "details": { + "match": "Matched \"Operator `PmFromFile' with parameter `scanners-user-agents.data' against variable `REQUEST_HEADERS:user-agent' (Value: `absinthe' )", + "reference": "o0,8v84,8t:lowercase", + "ruleId": "913100", + "file": "/etc/nginx/owasp-modsecurity-crs/rules/REQUEST-913-SCANNER-DETECTION.conf", + "lineNumber": "33", + "data": "Matched Data: absinthe found within REQUEST_HEADERS:user-agent: absinthe", + "severity": "2", + "ver": "OWASP_CRS/3.2.0", + "rev": "", + "tags": ["application-multi", "language-multi", "platform-multi", "attack-reputation-scanner", "OWASP_CRS", "OWASP_CRS/AUTOMATION/SECURITY_SCANNER", "WASCTC/WASC-21", "OWASP_TOP_10/A7", "PCI/6.5.10"], + "maturity": "0", + "accuracy": "0" + } +} +``` + +You can see that ModSecurity logs the suspicous behavior. By sending a request +with the `User Agent: absinthe` header, which [absinthe](https://github.com/cameronhotchkies/Absinthe), a tool for testing for SQL injections uses, we can detect that someone was +searching for vulnerabilities on our system. Detecting scanners is useful, because we +can learn if someone is trying to exploit our system. + +## Conclusion + +You can now see the benefits of a using a Web Application Firewall. +ModSecurity and the OWASP Core Rule Set, offer many more benefits. +You can explore them in more detail: + +- [GitLab Defend Vision](https://about.gitlab.com/direction/defend/#waf) +- [ModSecurity](https://www.modsecurity.org/) +- [OWASP Core Rule Set](https://modsecurity.org/crs/) +- [AutoDevOps](../autodevops/index.md) diff --git a/doc/user/analytics/code_review_analytics.md b/doc/user/analytics/code_review_analytics.md index 5861afd92a2..bb74e673b56 100644 --- a/doc/user/analytics/code_review_analytics.md +++ b/doc/user/analytics/code_review_analytics.md @@ -36,7 +36,7 @@ identify improvements that might substantially accelerate your development cycle Code Review Analytics can be used when: - Your team agrees that code review is moving too slow. -- The [Cycle Analytics feature](cycle_analytics.md) shows that reviews are your team's most time-consuming step. +- The [Value Stream Analytics feature](value_stream_analytics.md) shows that reviews are your team's most time-consuming step. You can use Code Review Analytics to see the types of work that are currently moving the slowest, and analyze the patterns and trends between them. For example: diff --git a/doc/user/analytics/cycle_analytics.md b/doc/user/analytics/cycle_analytics.md index c6de0138478..9d1cc508f63 100644 --- a/doc/user/analytics/cycle_analytics.md +++ b/doc/user/analytics/cycle_analytics.md @@ -1,221 +1,5 @@ -# Cycle Analytics +--- +redirect_to: '../analytics/value_stream_analytics.md' +--- -> - Introduced prior to GitLab 12.3 at the project level. -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/12077) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.3 at the group level. - -Cycle Analytics measures the time spent to go from an -[idea to production](https://about.gitlab.com/blog/2016/08/05/continuous-integration-delivery-and-deployment-with-gitlab/#from-idea-to-production-with-gitlab) -(also known as cycle time) for each of your projects. Cycle Analytics displays the median time -spent in each stage defined in the process. - -NOTE: **Note:** -Use the `cycle_analytics` feature flag to enable at the group level. - -Cycle Analytics is useful in order to quickly determine the velocity of a given -project. It points to bottlenecks in the development process, enabling management -to uncover, triage, and identify the root cause of slowdowns in the software development life cycle. - -Cycle Analytics is tightly coupled with the [GitLab flow](../../topics/gitlab_flow.md) and -calculates a separate median for each stage. - -## Overview - -Cycle Analytics is available: - -- From GitLab 12.3, at the group level in the analytics workspace (top navigation bar) at - **Analytics > Cycle Analytics**. **(PREMIUM)** - - In the future, multiple groups will be selectable which will effectively make this an - instance-level feature. - -- At the project level via **Project > Cycle Analytics**. - -There are seven stages that are tracked as part of the Cycle Analytics calculations. - -- **Issue** (Tracker) - - Time to schedule an issue (by milestone or by adding it to an issue board) -- **Plan** (Board) - - Time to first commit -- **Code** (IDE) - - Time to create a merge request -- **Test** (CI) - - Time it takes GitLab CI/CD to test your code -- **Review** (Merge Request/MR) - - Time spent on code review -- **Staging** (Continuous Deployment) - - Time between merging and deploying to production -- **Total** (Total) - - Total lifecycle time. That is, the velocity of the project or team. [Previously known](https://gitlab.com/gitlab-org/gitlab/issues/38317) as **Production**. - -## Date ranges - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/13216) in GitLab 12.4. - -GitLab provides the ability to filter analytics based on a date range. To filter results: - -1. Select a group. -1. Optionally select a project. -1. Select a date range using the available date pickers. - -## How the data is measured - -Cycle Analytics records cycle time and data based on the project issues with the -exception of the staging and total stages, where only data deployed to -production are measured. - -Specifically, if your CI is not set up and you have not defined a `production` -or `production/*` [environment](../../ci/yaml/README.md#environment), then you will not have any -data for this stage. - -Each stage of Cycle Analytics is further described in the table below. - -| **Stage** | **Description** | -| --------- | --------------- | -| Issue | Measures the median time between creating an issue and taking action to solve it, by either labeling it or adding it to a milestone, whatever comes first. The label will be tracked only if it already has an [Issue Board list](../project/issue_board.md#creating-a-new-list) created for it. | -| Plan | Measures the median time between the action you took for the previous stage, and pushing the first commit to the branch. The very first commit of the branch is the one that triggers the separation between **Plan** and **Code**, and at least one of the commits in the branch needs to contain the related issue number (e.g., `#42`). If none of the commits in the branch mention the related issue number, it is not considered to the measurement time of the stage. | -| Code | Measures the median time between pushing a first commit (previous stage) and creating a merge request (MR) related to that commit. The key to keep the process tracked is to include the [issue closing pattern](../project/issues/managing_issues.md#closing-issues-automatically) to the description of the merge request (for example, `Closes #xxx`, where `xxx` is the number of the issue related to this merge request). If the issue closing pattern is not present in the merge request description, the MR is not considered to the measurement time of the stage. | -| Test | Measures the median time to run the entire pipeline for that project. It's related to the time GitLab CI takes to run every job for the commits pushed to that merge request defined in the previous stage. It is basically the start->finish time for all pipelines. | -| Review | Measures the median time taken to review the merge request that has closing issue pattern, between its creation and until it's merged. | -| Staging | Measures the median time between merging the merge request with closing issue pattern until the very first deployment to production. It's tracked by the environment set to `production` or matching `production/*` (case-sensitive, `Production` won't work) in your GitLab CI configuration. If there isn't a production environment, this is not tracked. | -| Total | The sum of all time (medians) taken to run the entire process, from issue creation to deploying the code to production. [Previously known](https://gitlab.com/gitlab-org/gitlab/issues/38317) as **Production**. | - -How this works, behind the scenes: - -1. Issues and merge requests are grouped together in pairs, such that for each - `<issue, merge request>` pair, the merge request has the [issue closing pattern](../project/issues/managing_issues.md#closing-issues-automatically) - for the corresponding issue. All other issues and merge requests are **not** - considered. -1. Then the `<issue, merge request>` pairs are filtered out by last XX days (specified - by the UI - default is 90 days). So it prohibits these pairs from being considered. -1. For the remaining `<issue, merge request>` pairs, we check the information that - we need for the stages, like issue creation date, merge request merge time, - etc. - -To sum up, anything that doesn't follow [GitLab flow](../../workflow/gitlab_flow.md) will not be tracked and the -Cycle Analytics dashboard will not present any data for: - -- Merge requests that do not close an issue. -- Issues not labeled with a label present in the Issue Board or for issues not assigned a milestone. -- Staging and production stages, if the project has no `production` or `production/*` - environment. - -## Example workflow - -Below is a simple fictional workflow of a single cycle that happens in a -single day passing through all seven stages. Note that if a stage does not have -a start and a stop mark, it is not measured and hence not calculated in the median -time. It is assumed that milestones are created and CI for testing and setting -environments is configured. - -1. Issue is created at 09:00 (start of **Issue** stage). -1. Issue is added to a milestone at 11:00 (stop of **Issue** stage / start of - **Plan** stage). -1. Start working on the issue, create a branch locally and make one commit at - 12:00. -1. Make a second commit to the branch which mentions the issue number at 12.30 - (stop of **Plan** stage / start of **Code** stage). -1. Push branch and create a merge request that contains the [issue closing pattern](../project/issues/managing_issues.md#closing-issues-automatically) - in its description at 14:00 (stop of **Code** stage / start of **Test** and - **Review** stages). -1. The CI starts running your scripts defined in [`.gitlab-ci.yml`](../../ci/yaml/README.md) and - takes 5min (stop of **Test** stage). -1. Review merge request, ensure that everything is OK and merge the merge - request at 19:00. (stop of **Review** stage / start of **Staging** stage). -1. Now that the merge request is merged, a deployment to the `production` - environment starts and finishes at 19:30 (stop of **Staging** stage). -1. The cycle completes and the sum of the median times of the previous stages - is recorded to the **Total** stage. That is the time between creating an - issue and deploying its relevant merge request to production. - -From the above example you can conclude the time it took each stage to complete -as long as their total time: - -- **Issue**: 2h (11:00 - 09:00) -- **Plan**: 1h (12:00 - 11:00) -- **Code**: 2h (14:00 - 12:00) -- **Test**: 5min -- **Review**: 5h (19:00 - 14:00) -- **Staging**: 30min (19:30 - 19:00) -- **Total**: Since this stage measures the sum of median time of all - previous stages, we cannot calculate it if we don't know the status of the - stages before. In case this is the very first cycle that is run in the project, - then the **Total** time is 10h 30min (19:30 - 09:00) - -A few notes: - -- In the above example we demonstrated that it doesn't matter if your first - commit doesn't mention the issue number, you can do this later in any commit - of the branch you are working on. -- You can see that the **Test** stage is not calculated to the overall time of - the cycle since it is included in the **Review** process (every MR should be - tested). -- The example above was just **one cycle** of the seven stages. Add multiple - cycles, calculate their median time and the result is what the dashboard of - Cycle Analytics is showing. - -## Days to completion chart - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/21631) in GitLab 12.6. - -This chart visually depicts the total number of days it takes for cycles to be completed. - -This chart uses the global page filters for displaying data based on the selected -group, projects, and timeframe. In addition, specific stages can be selected -from within the chart itself. - -### Chart median line - -> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/36675) in GitLab 12.7. - -The median line on the chart displays data that is offset by the number of days selected. -For example, if 30 days worth of data has been selected (for example, 2019-12-16 to 2020-01-15) the -median line will represent the previous 30 days worth of data (2019-11-16 to 2019-12-16) -as a metric to compare against. - -### Disabling chart - -This chart is enabled by default. If you have a self-managed instance, an -administrator can open a Rails console and disable it with the following command: - -```ruby -Feature.disable(:cycle_analytics_scatterplot_enabled) -``` - -### Disabling chart median line - -This chart's median line is enabled by default. If you have a self-managed instance, an -administrator can open a Rails console and disable it with the following command: - -```ruby -Feature.disable(:cycle_analytics_scatterplot_median_enabled) -``` - -## Permissions - -The current permissions on the Project Cycle Analytics dashboard are: - -- Public projects - anyone can access. -- Internal projects - any authenticated user can access. -- Private projects - any member Guest and above can access. - -You can [read more about permissions](../../ci/yaml/README.md) in general. - -NOTE: **Note:** -As of GitLab 12.3, the project-level page is deprecated. You should access -project-level Cycle Analytics from **Analytics > Cycle Analytics** in the top -navigation bar. We will ensure that the same project-level functionality is available -to CE users in the new analytics space. - -For Cycle Analytics functionality introduced in GitLab 12.3 and later: - -- Users must have Reporter access or above. -- Features are available only on - [Premium or Silver tiers](https://about.gitlab.com/pricing/) and above. - -## More resources - -Learn more about Cycle Analytics in the following resources: - -- [Cycle Analytics feature page](https://about.gitlab.com/product/cycle-analytics/). -- [Cycle Analytics feature preview](https://about.gitlab.com/blog/2016/09/16/feature-preview-introducing-cycle-analytics/). -- [Cycle Analytics feature highlight](https://about.gitlab.com/blog/2016/09/21/cycle-analytics-feature-highlight/). +This document was moved to [another location](../analytics/value_stream_analytics.md) diff --git a/doc/user/analytics/index.md b/doc/user/analytics/index.md index 3117a5dfbca..2aa9d31d8bf 100644 --- a/doc/user/analytics/index.md +++ b/doc/user/analytics/index.md @@ -16,13 +16,13 @@ Once enabled, click on **Analytics** from the top navigation bar. From the centralized analytics workspace, the following analytics are available: - [Code Review Analytics](code_review_analytics.md). **(STARTER)** -- [Cycle Analytics](cycle_analytics.md), enabled with the `cycle_analytics` +- [Value Stream Analytics](value_stream_analytics.md), enabled with the `cycle_analytics` [feature flag](../../development/feature_flags/development.md#enabling-a-feature-flag-in-development). **(PREMIUM)** - [Productivity Analytics](productivity_analytics.md), enabled with the `productivity_analytics` [feature flag](../../development/feature_flags/development.md#enabling-a-feature-flag-in-development). **(PREMIUM)** NOTE: **Note:** -Project-level Cycle Analytics are still available at a project's **Project > Cycle Analytics**. +Project-level Value Stream Analytics are still available at a project's **Project > Value Stream Analytics**. ## Other analytics tools diff --git a/doc/user/analytics/productivity_analytics.md b/doc/user/analytics/productivity_analytics.md index 798b12fc2bd..572265b5b09 100644 --- a/doc/user/analytics/productivity_analytics.md +++ b/doc/user/analytics/productivity_analytics.md @@ -7,7 +7,7 @@ Track development velocity with Productivity Analytics. For many companies, the development cycle is a blackbox and getting an estimate of how long, on average, it takes to deliver features is an enormous endeavor. -While [Cycle Analytics](../project/cycle_analytics.md) focuses on the entire +While [Value Stream Analytics](../project/cycle_analytics.md) focuses on the entire Software Development Life Cycle (SDLC) process, Productivity Analytics provides a way for Engineering Management to drill down in a systematic way to uncover patterns and causes for success or failure at an individual, project or group level. Productivity can slow down for many reasons ranging from degrading code base to quickly growing teams. In order to investigate, department or team leaders can start by visualizing the time it takes for merge requests to be merged. diff --git a/doc/user/analytics/value_stream_analytics.md b/doc/user/analytics/value_stream_analytics.md new file mode 100644 index 00000000000..718367dc69d --- /dev/null +++ b/doc/user/analytics/value_stream_analytics.md @@ -0,0 +1,224 @@ +# Value Stream Analytics + +> - Introduced as Cycle Analytics prior to GitLab 12.3 at the project level. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/12077) in [GitLab Premium](https://about.gitlab.com/pricing/) 12.3 at the group level. +> - [Renamed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/23427) from Cycle Analytics to Value Stream Analytics in GitLab 12.8. + +Value Stream Analytics measures the time spent to go from an +[idea to production](https://about.gitlab.com/blog/2016/08/05/continuous-integration-delivery-and-deployment-with-gitlab/#from-idea-to-production-with-gitlab) +(also known as cycle time) for each of your projects. Value Stream Analytics displays the median time +spent in each stage defined in the process. + +For information on how to contribute to the development of Value Stream Analytics, see our [contributor documentation](../../development/value_stream_analytics.md). + +NOTE: **Note:** +Use the `cycle_analytics` feature flag to enable at the group level. + +Value Stream Analytics is useful in order to quickly determine the velocity of a given +project. It points to bottlenecks in the development process, enabling management +to uncover, triage, and identify the root cause of slowdowns in the software development life cycle. + +Value Stream Analytics is tightly coupled with the [GitLab flow](../../topics/gitlab_flow.md) and +calculates a separate median for each stage. + +## Overview + +Value Stream Analytics is available: + +- From GitLab 12.3, at the group level in the analytics workspace (top navigation bar) at + **Analytics > Value Stream Analytics**. **(PREMIUM)** + + In the future, multiple groups will be selectable which will effectively make this an + instance-level feature. + +- At the project level via **Project > Value Stream Analytics**. + +There are seven stages that are tracked as part of the Value Stream Analytics calculations. + +- **Issue** (Tracker) + - Time to schedule an issue (by milestone or by adding it to an issue board) +- **Plan** (Board) + - Time to first commit +- **Code** (IDE) + - Time to create a merge request +- **Test** (CI) + - Time it takes GitLab CI/CD to test your code +- **Review** (Merge Request/MR) + - Time spent on code review +- **Staging** (Continuous Deployment) + - Time between merging and deploying to production +- **Total** (Total) + - Total lifecycle time. That is, the velocity of the project or team. [Previously known](https://gitlab.com/gitlab-org/gitlab/issues/38317) as **Production**. + +## Date ranges + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/13216) in GitLab 12.4. + +GitLab provides the ability to filter analytics based on a date range. To filter results: + +1. Select a group. +1. Optionally select a project. +1. Select a date range using the available date pickers. + +## How the data is measured + +Value Stream Analytics records cycle time and data based on the project issues with the +exception of the staging and total stages, where only data deployed to +production are measured. + +Specifically, if your CI is not set up and you have not defined a `production` +or `production/*` [environment](../../ci/yaml/README.md#environment), then you will not have any +data for this stage. + +Each stage of Value Stream Analytics is further described in the table below. + +| **Stage** | **Description** | +| --------- | --------------- | +| Issue | Measures the median time between creating an issue and taking action to solve it, by either labeling it or adding it to a milestone, whatever comes first. The label will be tracked only if it already has an [Issue Board list](../project/issue_board.md#creating-a-new-list) created for it. | +| Plan | Measures the median time between the action you took for the previous stage, and pushing the first commit to the branch. The very first commit of the branch is the one that triggers the separation between **Plan** and **Code**, and at least one of the commits in the branch needs to contain the related issue number (e.g., `#42`). If none of the commits in the branch mention the related issue number, it is not considered to the measurement time of the stage. | +| Code | Measures the median time between pushing a first commit (previous stage) and creating a merge request (MR) related to that commit. The key to keep the process tracked is to include the [issue closing pattern](../project/issues/managing_issues.md#closing-issues-automatically) to the description of the merge request (for example, `Closes #xxx`, where `xxx` is the number of the issue related to this merge request). If the issue closing pattern is not present in the merge request description, the MR is not considered to the measurement time of the stage. | +| Test | Measures the median time to run the entire pipeline for that project. It's related to the time GitLab CI takes to run every job for the commits pushed to that merge request defined in the previous stage. It is basically the start->finish time for all pipelines. | +| Review | Measures the median time taken to review the merge request that has closing issue pattern, between its creation and until it's merged. | +| Staging | Measures the median time between merging the merge request with closing issue pattern until the very first deployment to production. It's tracked by the environment set to `production` or matching `production/*` (case-sensitive, `Production` won't work) in your GitLab CI configuration. If there isn't a production environment, this is not tracked. | +| Total | The sum of all time (medians) taken to run the entire process, from issue creation to deploying the code to production. [Previously known](https://gitlab.com/gitlab-org/gitlab/issues/38317) as **Production**. | + +How this works, behind the scenes: + +1. Issues and merge requests are grouped together in pairs, such that for each + `<issue, merge request>` pair, the merge request has the [issue closing pattern](../project/issues/managing_issues.md#closing-issues-automatically) + for the corresponding issue. All other issues and merge requests are **not** + considered. +1. Then the `<issue, merge request>` pairs are filtered out by last XX days (specified + by the UI - default is 90 days). So it prohibits these pairs from being considered. +1. For the remaining `<issue, merge request>` pairs, we check the information that + we need for the stages, like issue creation date, merge request merge time, + etc. + +To sum up, anything that doesn't follow [GitLab flow](../../workflow/gitlab_flow.md) will not be tracked and the +Value Stream Analytics dashboard will not present any data for: + +- Merge requests that do not close an issue. +- Issues not labeled with a label present in the Issue Board or for issues not assigned a milestone. +- Staging and production stages, if the project has no `production` or `production/*` + environment. + +## Example workflow + +Below is a simple fictional workflow of a single cycle that happens in a +single day passing through all seven stages. Note that if a stage does not have +a start and a stop mark, it is not measured and hence not calculated in the median +time. It is assumed that milestones are created and CI for testing and setting +environments is configured. + +1. Issue is created at 09:00 (start of **Issue** stage). +1. Issue is added to a milestone at 11:00 (stop of **Issue** stage / start of + **Plan** stage). +1. Start working on the issue, create a branch locally and make one commit at + 12:00. +1. Make a second commit to the branch which mentions the issue number at 12.30 + (stop of **Plan** stage / start of **Code** stage). +1. Push branch and create a merge request that contains the [issue closing pattern](../project/issues/managing_issues.md#closing-issues-automatically) + in its description at 14:00 (stop of **Code** stage / start of **Test** and + **Review** stages). +1. The CI starts running your scripts defined in [`.gitlab-ci.yml`](../../ci/yaml/README.md) and + takes 5min (stop of **Test** stage). +1. Review merge request, ensure that everything is OK and merge the merge + request at 19:00. (stop of **Review** stage / start of **Staging** stage). +1. Now that the merge request is merged, a deployment to the `production` + environment starts and finishes at 19:30 (stop of **Staging** stage). +1. The cycle completes and the sum of the median times of the previous stages + is recorded to the **Total** stage. That is the time between creating an + issue and deploying its relevant merge request to production. + +From the above example you can conclude the time it took each stage to complete +as long as their total time: + +- **Issue**: 2h (11:00 - 09:00) +- **Plan**: 1h (12:00 - 11:00) +- **Code**: 2h (14:00 - 12:00) +- **Test**: 5min +- **Review**: 5h (19:00 - 14:00) +- **Staging**: 30min (19:30 - 19:00) +- **Total**: Since this stage measures the sum of median time of all + previous stages, we cannot calculate it if we don't know the status of the + stages before. In case this is the very first cycle that is run in the project, + then the **Total** time is 10h 30min (19:30 - 09:00) + +A few notes: + +- In the above example we demonstrated that it doesn't matter if your first + commit doesn't mention the issue number, you can do this later in any commit + of the branch you are working on. +- You can see that the **Test** stage is not calculated to the overall time of + the cycle since it is included in the **Review** process (every MR should be + tested). +- The example above was just **one cycle** of the seven stages. Add multiple + cycles, calculate their median time and the result is what the dashboard of + Value Stream Analytics is showing. + +## Days to completion chart + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/21631) in GitLab 12.6. + +This chart visually depicts the total number of days it takes for cycles to be completed. + +This chart uses the global page filters for displaying data based on the selected +group, projects, and timeframe. In addition, specific stages can be selected +from within the chart itself. + +### Chart median line + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/issues/36675) in GitLab 12.7. + +The median line on the chart displays data that is offset by the number of days selected. +For example, if 30 days worth of data has been selected (for example, 2019-12-16 to 2020-01-15) the +median line will represent the previous 30 days worth of data (2019-11-16 to 2019-12-16) +as a metric to compare against. + +### Disabling chart + +This chart is enabled by default. If you have a self-managed instance, an +administrator can open a Rails console and disable it with the following command: + +```ruby +Feature.disable(:cycle_analytics_scatterplot_enabled) +``` + +### Disabling chart median line + +This chart's median line is enabled by default. If you have a self-managed instance, an +administrator can open a Rails console and disable it with the following command: + +```ruby +Feature.disable(:cycle_analytics_scatterplot_median_enabled) +``` + +## Permissions + +The current permissions on the Project Value Stream Analytics dashboard are: + +- Public projects - anyone can access. +- Internal projects - any authenticated user can access. +- Private projects - any member Guest and above can access. + +You can [read more about permissions](../../ci/yaml/README.md) in general. + +NOTE: **Note:** +As of GitLab 12.3, the project-level page is deprecated. You should access +project-level Value Stream Analytics from **Analytics > Value Stream Analytics** in the top +navigation bar. We will ensure that the same project-level functionality is available +to CE users in the new analytics space. + +For Value Stream Analytics functionality introduced in GitLab 12.3 and later: + +- Users must have Reporter access or above. +- Features are available only on + [Premium or Silver tiers](https://about.gitlab.com/pricing/) and above. + +## More resources + +Learn more about Value Stream Analytics in the following resources: + +- [Value Stream Analytics feature page](https://about.gitlab.com/product/cycle-analytics/). +- [Value Stream Analytics feature preview](https://about.gitlab.com/blog/2016/09/16/feature-preview-introducing-cycle-analytics/). +- [Value Stream Analytics feature highlight](https://about.gitlab.com/blog/2016/09/21/cycle-analytics-feature-highlight/). diff --git a/doc/user/clusters/applications.md b/doc/user/clusters/applications.md index 20aa6648c65..7e8ae151e47 100644 --- a/doc/user/clusters/applications.md +++ b/doc/user/clusters/applications.md @@ -139,10 +139,12 @@ file. Customizing installation by modifying this file is not supported. > - Introduced in GitLab 10.2 for project-level clusters. > - Introduced in GitLab 11.6 for group-level clusters. -[Ingress](https://kubernetes.github.io/ingress-nginx/) can provide load -balancing, SSL termination, and name-based virtual hosting. It acts as a -web proxy for your applications and is useful if you want to use [Auto -DevOps](../../topics/autodevops/index.md) or deploy your own web apps. +[Ingress](https://kubernetes.io/docs/concepts/services-networking/ingress/) provides load balancing, SSL termination, and name-based virtual hosting +out of the box. It acts as a web proxy for your applications and is useful +if you want to use [Auto DevOps](../../topics/autodevops/index.md) or deploy your own web apps. + +The Ingress Controller installed is [Ingress-NGINX](https://kubernetes.io/docs/concepts/services-networking/ingress/), +which is supported by the Kubernetes community. NOTE: **Note:** With the following procedure, a load balancer must be installed in your cluster @@ -255,12 +257,20 @@ use an A record. If your external endpoint is a hostname, use a CNAME record. > [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/21966) in GitLab 12.7. -Out of the box, GitLab provides you real-time security monitoring with -[ModSecurity](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#modsecurity). +A Web Application Firewall (WAF) is able to examine traffic being sent/received +and can block malicious traffic before it reaches your application. The benefits +of a WAF are: + +- Real-time security monitoring for your application +- Logging of all your HTTP traffic to the application +- Access control for your application +- Highly configurable logging and blocking rules + +Out of the box, GitLab provides you with a WAF known as [`ModSecurity`](https://www.modsecurity.org/) -Modsecurity is a toolkit for real-time web application monitoring, logging, -and access control. With GitLab's offering, the [OWASP's Core Rule Set](https://www.modsecurity.org/CRS/Documentation/), which provides generic attack detection capabilities, -is automatically applied. +ModSecurity is a toolkit for real-time web application monitoring, logging, +and access control. With GitLab's offering, the [OWASP's Core Rule Set](https://www.modsecurity.org/CRS/Documentation/), +which provides generic attack detection capabilities, is automatically applied. This feature: @@ -275,6 +285,12 @@ This feature: To enable ModSecurity, check the **Enable Web Application Firewall** checkbox when installing your [Ingress application](#ingress). +If this is your first time using GitLab's WAF, we recommend you follow the +[quick start guide](../../topics/web_application_firewall/quick_start_guide.md). + +There is a small performance overhead by enabling ModSecurity. However, +if this is considered significant for your application, you can disable it. + There is a small performance overhead by enabling ModSecurity. If this is considered significant for your application, you can disable ModSecurity's rule engine for your deployed application by setting diff --git a/doc/user/index.md b/doc/user/index.md index ab953b6d8bf..ec8a53b842d 100644 --- a/doc/user/index.md +++ b/doc/user/index.md @@ -45,7 +45,7 @@ GitLab is a Git-based platform that integrates a great number of essential tools - Building, testing, and deploying with built-in [Continuous Integration](../ci/README.md). - Deploying personal and professional static websites with [GitLab Pages](project/pages/index.md). - Integrating with Docker by using [GitLab Container Registry](packages/container_registry/index.md). -- Tracking the development lifecycle by using [GitLab Cycle Analytics](project/cycle_analytics.md). +- Tracking the development lifecycle by using [GitLab Value Stream Analytics](project/cycle_analytics.md). With GitLab Enterprise Edition, you can also: diff --git a/doc/user/permissions.md b/doc/user/permissions.md index 41040258e01..4f42afe4e79 100644 --- a/doc/user/permissions.md +++ b/doc/user/permissions.md @@ -139,10 +139,10 @@ The following table depicts the various user permission levels in a project. | Force push to protected branches (*4*) | | | | | | | Remove protected branches (*4*) | | | | | | -\* Owner permission is only available at the group or personal namespace level (and for instance admins) and is inherited by its projects. -(*1*): Guest users are able to perform this action on public and internal projects, but not private projects. -(*2*): Guest users can only view the confidential issues they created themselves. -(*3*): If **Public pipelines** is enabled in **Project Settings > CI/CD**. +\* Owner permission is only available at the group or personal namespace level (and for instance admins) and is inherited by its projects. +(*1*): Guest users are able to perform this action on public and internal projects, but not private projects. +(*2*): Guest users can only view the confidential issues they created themselves. +(*3*): If **Public pipelines** is enabled in **Project Settings > CI/CD**. (*4*): Not allowed for Guest, Reporter, Developer, Maintainer, or Owner. See [Protected Branches](./project/protected_branches.md). (*5*): If the [branch is protected](./project/protected_branches.md#using-the-allowed-to-merge-and-allowed-to-push-settings), this depends on the access Developers and Maintainers are given. @@ -166,10 +166,10 @@ Maintainers and Developers from pushing to a protected branch. Read through the [Allowed to Merge and Allowed to Push settings](project/protected_branches.md#using-the-allowed-to-merge-and-allowed-to-push-settings) to learn more. -### Cycle Analytics permissions +### Value Stream Analytics permissions -Find the current permissions on the Cycle Analytics dashboard on -the [documentation on Cycle Analytics permissions](analytics/cycle_analytics.md#permissions). +Find the current permissions on the Value Stream Analytics dashboard, as described in +[related documentation](analytics/value_stream_analytics.md#permissions). ### Issue Board permissions diff --git a/doc/user/project/cycle_analytics.md b/doc/user/project/cycle_analytics.md index 87577c9ec88..9d1cc508f63 100644 --- a/doc/user/project/cycle_analytics.md +++ b/doc/user/project/cycle_analytics.md @@ -1,5 +1,5 @@ --- -redirect_to: '../analytics/cycle_analytics.md' +redirect_to: '../analytics/value_stream_analytics.md' --- -This document was moved to [another location](../analytics/cycle_analytics.md) +This document was moved to [another location](../analytics/value_stream_analytics.md) diff --git a/doc/user/project/index.md b/doc/user/project/index.md index fb72445538a..87837d50bbe 100644 --- a/doc/user/project/index.md +++ b/doc/user/project/index.md @@ -87,7 +87,7 @@ When you create a project in GitLab, you'll have access to a large number of - [Wiki](wiki/index.md): document your GitLab project in an integrated Wiki. - [Snippets](../snippets.md): store, share and collaborate on code snippets. -- [Cycle Analytics](cycle_analytics.md): review your development lifecycle. +- [Value Stream Analytics](cycle_analytics.md): review your development lifecycle. - [Insights](insights/index.md): configure the Insights that matter for your projects. **(ULTIMATE)** - [Security Dashboard](security_dashboard.md): Security Dashboard. **(ULTIMATE)** - [Syntax highlighting](highlighting.md): an alternative to customize diff --git a/lib/gitlab/analytics/cycle_analytics/default_stages.rb b/lib/gitlab/analytics/cycle_analytics/default_stages.rb index 8e70236ce75..79e60e28fc7 100644 --- a/lib/gitlab/analytics/cycle_analytics/default_stages.rb +++ b/lib/gitlab/analytics/cycle_analytics/default_stages.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -# This module represents the default Cycle Analytics stages that are currently provided by CE +# This module represents the default Value Stream Analytics stages that are currently provided by CE # Each method returns a hash that can be used to build a new stage object. # # Example: diff --git a/lib/gitlab/cycle_analytics/base_stage.rb b/lib/gitlab/cycle_analytics/base_stage.rb index 1cd54238bb4..06f0cbed147 100644 --- a/lib/gitlab/cycle_analytics/base_stage.rb +++ b/lib/gitlab/cycle_analytics/base_stage.rb @@ -50,7 +50,7 @@ module Gitlab # Build a `SELECT` query. We find the first of the `end_time_attrs` that isn't `NULL` (call this end_time). # Next, we find the first of the start_time_attrs that isn't `NULL` (call this start_time). # We compute the (end_time - start_time) interval, and give it an alias based on the current - # cycle analytics stage. + # value stream analytics stage. median_datetime(cte_table, interval_query(project_ids), name) end diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 10df4ed72d9..5a6cc0159c6 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -13,6 +13,11 @@ module Gitlab # use load_all_data!. MAX_DATA_DISPLAY_SIZE = 10.megabytes + # The number of blobs loaded in a single Gitaly call + # When a large number of blobs requested, we'd want to fetch them in + # multiple Gitaly calls + BATCH_SIZE = 250 + # These limits are used as a heuristic to ignore files which can't be LFS # pointers. The format of these is described in # https://github.com/git-lfs/git-lfs/blob/master/docs/spec.md#the-pointer @@ -67,7 +72,13 @@ module Gitlab # to the caller to limit the number of blobs and blob_size_limit. # def batch(repository, blob_references, blob_size_limit: MAX_DATA_DISPLAY_SIZE) - repository.gitaly_blob_client.get_blobs(blob_references, blob_size_limit).to_a + if Feature.enabled?(:blobs_fetch_in_batches, default_enabled: true) + blob_references.each_slice(BATCH_SIZE).flat_map do |refs| + repository.gitaly_blob_client.get_blobs(refs, blob_size_limit).to_a + end + else + repository.gitaly_blob_client.get_blobs(blob_references, blob_size_limit).to_a + end end # Returns an array of Blob instances just with the metadata, that means diff --git a/locale/gitlab.pot b/locale/gitlab.pot index cea8e92013f..2a62c6eaa05 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -5802,15 +5802,6 @@ msgstr "" msgid "Customize your pipeline configuration." msgstr "" -msgid "Cycle Analytics" -msgstr "" - -msgid "Cycle Analytics can help you determine your team’s velocity" -msgstr "" - -msgid "Cycle Analytics gives an overview of how much time it takes to go from idea to production in your project." -msgstr "" - msgid "CycleAnalyticsEvent|Issue closed" msgstr "" @@ -6714,15 +6705,15 @@ msgstr "" msgid "Dismiss" msgstr "" -msgid "Dismiss Cycle Analytics introduction box" -msgstr "" - msgid "Dismiss DevOps Score introduction" msgstr "" msgid "Dismiss Merge Request promotion" msgstr "" +msgid "Dismiss Value Stream Analytics introduction box" +msgstr "" + msgid "Dismiss trial promotion" msgstr "" @@ -10064,6 +10055,9 @@ msgstr "" msgid "IDE|Commit" msgstr "" +msgid "IDE|Commit to %{branchName} branch" +msgstr "" + msgid "IDE|Edit" msgstr "" @@ -10091,6 +10085,9 @@ msgstr "" msgid "IDE|Successful commit" msgstr "" +msgid "IDE|This option is disabled because you don't have write permissions for the current branch." +msgstr "" + msgid "IP Address" msgstr "" @@ -10459,7 +10456,7 @@ msgstr "" msgid "Interval Pattern" msgstr "" -msgid "Introducing Cycle Analytics" +msgid "Introducing Value Stream Analytics" msgstr "" msgid "Introducing Your DevOps Score" @@ -11944,6 +11941,9 @@ msgstr "" msgid "MergeRequests|started a thread on commit %{linkStart}%{commitDisplay}%{linkEnd}" msgstr "" +msgid "MergeRequest|Compare %{source} and %{target}" +msgstr "" + msgid "MergeRequest|Error dismissing suggestion popover. Please try again." msgstr "" @@ -13042,7 +13042,7 @@ msgstr "" msgid "One or more of your personal access tokens will expire in %{days_to_expire} days or less." msgstr "" -msgid "Only 'Reporter' roles and above on tiers Premium / Silver and above can see Cycle Analytics." +msgid "Only 'Reporter' roles and above on tiers Premium / Silver and above can see Value Stream Analytics." msgstr "" msgid "Only Project Members" @@ -19182,9 +19182,6 @@ msgstr "" msgid "There was an error fetching configuration for charts" msgstr "" -msgid "There was an error fetching cycle analytics stages." -msgstr "" - msgid "There was an error fetching data for the selected stage" msgstr "" @@ -19200,6 +19197,9 @@ msgstr "" msgid "There was an error fetching the Designs" msgstr "" +msgid "There was an error fetching value stream analytics stages." +msgstr "" + msgid "There was an error gathering the chart data" msgstr "" @@ -19242,16 +19242,16 @@ msgstr "" msgid "There was an error when unsubscribing from this label." msgstr "" -msgid "There was an error while fetching cycle analytics data." +msgid "There was an error while fetching value stream analytics data." msgstr "" -msgid "There was an error while fetching cycle analytics duration data." +msgid "There was an error while fetching value stream analytics duration data." msgstr "" -msgid "There was an error while fetching cycle analytics duration median data." +msgid "There was an error while fetching value stream analytics duration median data." msgstr "" -msgid "There was an error while fetching cycle analytics summary data." +msgid "There was an error while fetching value stream analytics summary data." msgstr "" msgid "There was an error with the reCAPTCHA. Please solve the reCAPTCHA again." @@ -19524,9 +19524,6 @@ msgstr "" msgid "This namespace has already been taken! Please choose another one." msgstr "" -msgid "This option is disabled as you don't have write permissions for the current branch" -msgstr "" - msgid "This option is only available on GitLab.com" msgstr "" @@ -21100,6 +21097,15 @@ msgstr "" msgid "Value" msgstr "" +msgid "Value Stream Analytics" +msgstr "" + +msgid "Value Stream Analytics can help you determine your team’s velocity" +msgstr "" + +msgid "Value Stream Analytics gives an overview of how much time it takes to go from idea to production in your project." +msgstr "" + msgid "Variables" msgstr "" @@ -21899,10 +21905,10 @@ msgstr "" msgid "You don't have sufficient permission to perform this action." msgstr "" -msgid "You don’t have access to Cycle Analytics for this group" +msgid "You don’t have access to Productivity Analytics in this group" msgstr "" -msgid "You don’t have access to Productivity Analytics in this group" +msgid "You don’t have access to Value Stream Analytics for this group" msgstr "" msgid "You have been granted %{access_level} access to the %{source_link} %{source_type}." @@ -22261,9 +22267,6 @@ msgstr "" msgid "among other things" msgstr "" -msgid "and" -msgstr "" - msgid "any-approver for the merge request already exists" msgstr "" diff --git a/spec/controllers/projects/pipelines_controller_spec.rb b/spec/controllers/projects/pipelines_controller_spec.rb index 74d7e056737..d464827d63a 100644 --- a/spec/controllers/projects/pipelines_controller_spec.rb +++ b/spec/controllers/projects/pipelines_controller_spec.rb @@ -608,7 +608,7 @@ describe Projects::PipelinesController do describe 'GET test_report.json' do subject(:get_test_report_json) do - post :test_report, params: { + get :test_report, params: { namespace_id: project.namespace, project_id: project, id: pipeline.id diff --git a/spec/features/cycle_analytics_spec.rb b/spec/features/cycle_analytics_spec.rb index 0cafdb4e982..666a45485b9 100644 --- a/spec/features/cycle_analytics_spec.rb +++ b/spec/features/cycle_analytics_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -describe 'Cycle Analytics', :js do +describe 'Value Stream Analytics', :js do let(:user) { create(:user) } let(:guest) { create(:user) } let(:project) { create(:project, :repository) } @@ -23,7 +23,7 @@ describe 'Cycle Analytics', :js do end it 'shows introductory message' do - expect(page).to have_content('Introducing Cycle Analytics') + expect(page).to have_content('Introducing Value Stream Analytics') end it 'shows pipeline summary' do @@ -38,7 +38,7 @@ describe 'Cycle Analytics', :js do end end - context "when there's cycle analytics data" do + context "when there's value stream analytics data" do before do allow_next_instance_of(Gitlab::ReferenceExtractor) do |instance| allow(instance).to receive(:issues).and_return([issue]) diff --git a/spec/features/projects/navbar_spec.rb b/spec/features/projects/navbar_spec.rb index 4917a0b0a59..bcb05e1c718 100644 --- a/spec/features/projects/navbar_spec.rb +++ b/spec/features/projects/navbar_spec.rb @@ -67,8 +67,8 @@ describe 'Project navbar' do nav_sub_items: [ _('CI / CD Analytics'), (_('Code Review') if Gitlab.ee?), - _('Cycle Analytics'), - _('Repository Analytics') + _('Repository Analytics'), + _('Value Stream Analytics') ] }, { diff --git a/spec/fixtures/valid.po b/spec/fixtures/valid.po index 28826f05595..bb2dfa419bb 100644 --- a/spec/fixtures/valid.po +++ b/spec/fixtures/valid.po @@ -267,11 +267,11 @@ msgstr "Eventos de notificaciones personalizadas" msgid "Custom notification levels are the same as participating levels. With custom notification levels you will also receive notifications for select events. To find out more, check out %{notification_link}." msgstr "Los niveles de notificación personalizados son los mismos que los niveles participantes. Con los niveles de notificación personalizados, también recibirá notificaciones para eventos seleccionados. Para obtener más información, consulte %{notification_link}." -msgid "Cycle Analytics" -msgstr "Cycle Analytics" +msgid "Value Stream Analytics" +msgstr "Value Stream Analytics" -msgid "Cycle Analytics gives an overview of how much time it takes to go from idea to production in your project." -msgstr "Cycle Analytics ofrece una visión general de cuánto tiempo tarda en pasar de idea a producción en su proyecto." +msgid "Value Stream Analytics gives an overview of how much time it takes to go from idea to production in your project." +msgstr "Value Stream Analytics ofrece una visión general de cuánto tiempo tarda en pasar de idea a producción en su proyecto." msgid "CycleAnalyticsStage|Code" msgstr "Código" @@ -412,8 +412,8 @@ msgstr "Importar repositorio" msgid "Interval Pattern" msgstr "Patrón de intervalo" -msgid "Introducing Cycle Analytics" -msgstr "Introducción a Cycle Analytics" +msgid "Introducing Value Stream Analytics" +msgstr "Introducción a Value Stream Analytics" msgid "Jobs for last month" msgstr "Trabajos del mes pasado" diff --git a/spec/javascripts/cycle_analytics/banner_spec.js b/spec/javascripts/cycle_analytics/banner_spec.js index 86408c18dda..06fbaa68ffc 100644 --- a/spec/javascripts/cycle_analytics/banner_spec.js +++ b/spec/javascripts/cycle_analytics/banner_spec.js @@ -16,8 +16,10 @@ describe('Cycle analytics banner', () => { vm.$destroy(); }); - it('should render cycle analytics information', () => { - expect(vm.$el.querySelector('h4').textContent.trim()).toEqual('Introducing Cycle Analytics'); + it('should render value stream analytics information', () => { + expect(vm.$el.querySelector('h4').textContent.trim()).toEqual( + 'Introducing Value Stream Analytics', + ); expect( vm.$el @@ -25,7 +27,7 @@ describe('Cycle analytics banner', () => { .textContent.trim() .replace(/[\r\n]+/g, ' '), ).toContain( - 'Cycle Analytics gives an overview of how much time it takes to go from idea to production in your project.', + 'Value Stream Analytics gives an overview of how much time it takes to go from idea to production in your project.', ); expect(vm.$el.querySelector('a').textContent.trim()).toEqual('Read more'); diff --git a/spec/lib/gitlab/git/blob_spec.rb b/spec/lib/gitlab/git/blob_spec.rb index a659af3d22e..2495bc46c07 100644 --- a/spec/lib/gitlab/git/blob_spec.rb +++ b/spec/lib/gitlab/git/blob_spec.rb @@ -244,6 +244,61 @@ describe Gitlab::Git::Blob, :seed_helper do end end end + + context 'when large number of blobs requested' do + let(:first_batch) do + [ + [SeedRepo::Commit::ID, 'files/ruby/popen.rb'], + [SeedRepo::Commit::ID, 'six'] + ] + end + + let(:second_batch) do + [ + [SeedRepo::Commit::ID, 'some'], + [SeedRepo::Commit::ID, 'other'] + ] + end + + let(:third_batch) do + [ + [SeedRepo::Commit::ID, 'files'] + ] + end + + let(:blob_references) do + first_batch + second_batch + third_batch + end + + let(:client) { repository.gitaly_blob_client } + let(:limit) { 10.megabytes } + + before do + stub_const('Gitlab::Git::Blob::BATCH_SIZE', 2) + end + + context 'blobs_fetch_in_batches is enabled' do + it 'fetches the blobs in batches' do + expect(client).to receive(:get_blobs).with(first_batch, limit).ordered + expect(client).to receive(:get_blobs).with(second_batch, limit).ordered + expect(client).to receive(:get_blobs).with(third_batch, limit).ordered + + subject + end + end + + context 'blobs_fetch_in_batches is disabled' do + before do + stub_feature_flags(blobs_fetch_in_batches: false) + end + + it 'fetches the blobs in a single batch' do + expect(client).to receive(:get_blobs).with(blob_references, limit) + + subject + end + end + end end describe '.batch_metadata' do diff --git a/spec/migrations/services_remove_temporary_index_on_project_id_spec.rb b/spec/migrations/services_remove_temporary_index_on_project_id_spec.rb new file mode 100644 index 00000000000..f730d7aecfd --- /dev/null +++ b/spec/migrations/services_remove_temporary_index_on_project_id_spec.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +require 'spec_helper' +require Rails.root.join('db', 'post_migrate', '20200203104214_services_remove_temporary_index_on_project_id.rb') + +describe ServicesRemoveTemporaryIndexOnProjectId, :migration do + let(:migration_instance) { described_class.new } + + it 'adds and removes temporary partial index in up and down methods' do + reversible_migration do |migration| + migration.before -> { + expect(migration_instance.index_exists?(:services, :project_id, name: described_class::INDEX_NAME)).to be true + } + + migration.after -> { + expect(migration_instance.index_exists?(:services, :project_id, name: described_class::INDEX_NAME)).to be false + } + end + end + + describe '#up' do + context 'index does not exist' do + it 'skips removal action' do + migrate! + + expect { migrate! }.not_to change { migration_instance.index_exists?(:services, :project_id, name: described_class::INDEX_NAME) } + end + end + end + + describe '#down' do + context 'index already exists' do + it 'skips creation of duplicated temporary partial index on project_id' do + schema_migrate_down! + + expect { schema_migrate_down! }.not_to change { migration_instance.index_exists?(:services, :project_id, name: described_class::INDEX_NAME) } + end + end + end +end diff --git a/spec/requests/projects/cycle_analytics_events_spec.rb b/spec/requests/projects/cycle_analytics_events_spec.rb index 93a1aafde23..773f243e733 100644 --- a/spec/requests/projects/cycle_analytics_events_spec.rb +++ b/spec/requests/projects/cycle_analytics_events_spec.rb @@ -2,12 +2,12 @@ require 'spec_helper' -describe 'cycle analytics events' do +describe 'value stream analytics events' do let(:user) { create(:user) } let(:project) { create(:project, :repository, public_builds: false) } let(:issue) { create(:issue, project: project, created_at: 2.days.ago) } - describe 'GET /:namespace/:project/cycle_analytics/events/issues' do + describe 'GET /:namespace/:project/value_stream_analytics/events/issues' do before do project.add_developer(user) diff --git a/spec/views/layouts/nav/sidebar/_project.html.haml_spec.rb b/spec/views/layouts/nav/sidebar/_project.html.haml_spec.rb index 7decfa58153..6ca8fa2bc5c 100644 --- a/spec/views/layouts/nav/sidebar/_project.html.haml_spec.rb +++ b/spec/views/layouts/nav/sidebar/_project.html.haml_spec.rb @@ -160,4 +160,31 @@ describe 'layouts/nav/sidebar/_project' do end end end + + describe 'value stream analytics entry' do + let(:read_cycle_analytics) { true } + + before do + allow(view).to receive(:can?).with(nil, :read_cycle_analytics, project).and_return(read_cycle_analytics) + stub_feature_flags(analytics_pages_under_project_analytics_sidebar: { enabled: false, thing: project }) + end + + describe 'when value stream analytics is enabled' do + it 'shows the value stream analytics entry' do + render + + expect(rendered).to have_link('Value Stream Analytics', href: project_cycle_analytics_path(project)) + end + end + + describe 'when value stream analytics is disabled' do + let(:read_cycle_analytics) { false } + + it 'does not show the value stream analytics entry' do + render + + expect(rendered).not_to have_link('Value Stream Analytics', href: project_cycle_analytics_path(project)) + end + end + end end |
