diff options
48 files changed, 946 insertions, 186 deletions
diff --git a/.haml-lint_todo.yml b/.haml-lint_todo.yml index dd94d0b494d..1417f37f76d 100644 --- a/.haml-lint_todo.yml +++ b/.haml-lint_todo.yml @@ -409,7 +409,6 @@ linters: - 'ee/app/views/projects/push_rules/_index.html.haml' - 'ee/app/views/projects/services/gitlab_slack_application/_help.html.haml' - 'ee/app/views/projects/services/gitlab_slack_application/_slack_integration_form.html.haml' - - 'ee/app/views/projects/services/prometheus/_metrics.html.haml' - 'ee/app/views/projects/settings/slacks/edit.html.haml' - 'ee/app/views/shared/_additional_email_text.html.haml' - 'ee/app/views/shared/_mirror_update_button.html.haml' diff --git a/app/assets/javascripts/jira_import/components/jira_import_app.vue b/app/assets/javascripts/jira_import/components/jira_import_app.vue index 6efac949979..437239ce0be 100644 --- a/app/assets/javascripts/jira_import/components/jira_import_app.vue +++ b/app/assets/javascripts/jira_import/components/jira_import_app.vue @@ -1,10 +1,12 @@ <script> import getJiraProjects from '../queries/getJiraProjects.query.graphql'; +import JiraImportForm from './jira_import_form.vue'; import JiraImportSetup from './jira_import_setup.vue'; export default { name: 'JiraImportApp', components: { + JiraImportForm, JiraImportSetup, }, props: { @@ -41,6 +43,6 @@ export default { <template> <div> <jira-import-setup v-if="!isJiraConfigured" :illustration="setupIllustration" /> - <div v-else></div> + <jira-import-form v-else /> </div> </template> diff --git a/app/assets/javascripts/jira_import/components/jira_import_form.vue b/app/assets/javascripts/jira_import/components/jira_import_form.vue new file mode 100644 index 00000000000..4de04efe1b0 --- /dev/null +++ b/app/assets/javascripts/jira_import/components/jira_import_form.vue @@ -0,0 +1,94 @@ +<script> +import { GlAvatar, GlNewButton, GlFormGroup, GlFormSelect, GlLabel } from '@gitlab/ui'; + +export default { + name: 'JiraImportForm', + components: { + GlAvatar, + GlNewButton, + GlFormGroup, + GlFormSelect, + GlLabel, + }, + currentUserAvatarUrl: gon.current_user_avatar_url, + currentUsername: gon.current_username, +}; +</script> + +<template> + <div> + <h3 class="page-title">{{ __('New Jira import') }}</h3> + <hr /> + <form> + <gl-form-group + class="row align-items-center" + :label="__('Import from')" + label-cols-sm="2" + label-for="jira-project-select" + > + <gl-form-select id="jira-project-select" class="mb-2" /> + </gl-form-group> + + <gl-form-group + class="row align-items-center" + :label="__('Issue label')" + label-cols-sm="2" + label-for="jira-project-label" + > + <gl-label + id="jira-project-label" + class="mb-2" + background-color="#428BCA" + title="jira-import::KEY-1" + scoped + /> + </gl-form-group> + + <hr /> + + <p class="offset-md-1"> + {{ + __( + "For each Jira issue successfully imported, we'll create a new GitLab issue with the following data:", + ) + }} + </p> + + <gl-form-group + class="row align-items-center mb-1" + :label="__('Title')" + label-cols-sm="2" + label-for="jira-project-title" + > + <p id="jira-project-title" class="mb-2">{{ __('jira.issue.summary') }}</p> + </gl-form-group> + <gl-form-group + class="row align-items-center mb-1" + :label="__('Reporter')" + label-cols-sm="2" + label-for="jira-project-reporter" + > + <gl-avatar + id="jira-project-reporter" + class="mb-2" + :src="$options.currentUserAvatarUrl" + :size="24" + :aria-label="$options.currentUsername" + /> + </gl-form-group> + <gl-form-group + class="row align-items-center mb-1" + :label="__('Description')" + label-cols-sm="2" + label-for="jira-project-description" + > + <p id="jira-project-description" class="mb-2">{{ __('jira.issue.description.content') }}</p> + </gl-form-group> + + <div class="footer-block row-content-block d-flex justify-content-between"> + <gl-new-button category="primary" variant="success">{{ __('Next') }}</gl-new-button> + <gl-new-button>{{ __('Cancel') }}</gl-new-button> + </div> + </form> + </div> +</template> diff --git a/app/assets/javascripts/pages/projects/services/edit/index.js b/app/assets/javascripts/pages/projects/services/edit/index.js index 49862b64c27..5249709a2a3 100644 --- a/app/assets/javascripts/pages/projects/services/edit/index.js +++ b/app/assets/javascripts/pages/projects/services/edit/index.js @@ -1,5 +1,5 @@ import IntegrationSettingsForm from '~/integrations/integration_settings_form'; -import PrometheusMetrics from 'ee_else_ce/prometheus_metrics/prometheus_metrics'; +import PrometheusMetrics from '~/prometheus_metrics/custom_metrics'; import PrometheusAlerts from '~/prometheus_alerts'; import initAlertsSettings from '~/alerts_service_settings'; diff --git a/app/assets/javascripts/prometheus_metrics/constants.js b/app/assets/javascripts/prometheus_metrics/constants.js index 50f1248456e..0ffd977f04c 100644 --- a/app/assets/javascripts/prometheus_metrics/constants.js +++ b/app/assets/javascripts/prometheus_metrics/constants.js @@ -2,4 +2,5 @@ export default { EMPTY: 'empty', LOADING: 'loading', LIST: 'list', + NO_INTEGRATION: 'no-integration', }; diff --git a/app/assets/javascripts/prometheus_metrics/custom_metrics.js b/app/assets/javascripts/prometheus_metrics/custom_metrics.js new file mode 100644 index 00000000000..36df7837785 --- /dev/null +++ b/app/assets/javascripts/prometheus_metrics/custom_metrics.js @@ -0,0 +1,163 @@ +import $ from 'jquery'; +import { escape, sortBy } from 'lodash'; +import PrometheusMetrics from './prometheus_metrics'; +import PANEL_STATE from './constants'; +import axios from '~/lib/utils/axios_utils'; +import { s__ } from '~/locale'; +import { capitalizeFirstCharacter } from '~/lib/utils/text_utility'; + +export default class CustomMetrics extends PrometheusMetrics { + constructor(wrapperSelector) { + super(wrapperSelector); + this.customMetrics = []; + this.environmentsData = []; + this.$els = []; + + this.$wrapperCustomMetrics = $(wrapperSelector); + + this.$monitoredCustomMetricsPanel = this.$wrapperCustomMetrics.find( + '.js-panel-custom-monitored-metrics', + ); + this.$monitoredCustomMetricsCount = this.$monitoredCustomMetricsPanel.find( + '.js-custom-monitored-count', + ); + this.$monitoredCustomMetricsLoading = this.$monitoredCustomMetricsPanel.find( + '.js-loading-custom-metrics', + ); + this.$monitoredCustomMetricsEmpty = this.$monitoredCustomMetricsPanel.find( + '.js-empty-custom-metrics', + ); + this.$monitoredCustomMetricsList = this.$monitoredCustomMetricsPanel.find( + '.js-custom-metrics-list', + ); + this.$monitoredCustomMetricsNoIntegrationText = this.$monitoredCustomMetricsPanel.find( + '.js-no-active-integration-text', + ); + this.$newCustomMetricButton = this.$monitoredCustomMetricsPanel.find('.js-new-metric-button'); + this.$newCustomMetricText = this.$monitoredCustomMetricsPanel.find('.js-new-metric-text'); + this.$flashCustomMetricsContainer = this.$wrapperCustomMetrics.find('.flash-container'); + + this.$els = [ + this.$monitoredCustomMetricsLoading, + this.$monitoredCustomMetricsList, + this.$newCustomMetricButton, + this.$newCustomMetricText, + this.$monitoredCustomMetricsNoIntegrationText, + this.$monitoredCustomMetricsEmpty, + ]; + + this.activeCustomMetricsEndpoint = this.$monitoredCustomMetricsPanel.data( + 'active-custom-metrics', + ); + this.environmentsDataEndpoint = this.$monitoredCustomMetricsPanel.data( + 'environments-data-endpoint', + ); + this.isServiceActive = this.$monitoredCustomMetricsPanel.data('service-active'); + } + + init() { + if (this.isServiceActive) { + this.loadActiveCustomMetrics(); + } else { + this.setNoIntegrationActiveState(); + } + } + + // eslint-disable-next-line class-methods-use-this + setHidden(els) { + els.forEach(el => el.addClass('hidden')); + } + + setVisible(...els) { + this.setHidden(this.$els.filter(el => !els.includes(el))); + els.forEach(el => el.removeClass('hidden')); + } + + showMonitoringCustomMetricsPanelState(stateName) { + switch (stateName) { + case PANEL_STATE.LOADING: + this.setVisible(this.$monitoredCustomMetricsLoading); + break; + case PANEL_STATE.LIST: + this.setVisible(this.$newCustomMetricButton, this.$monitoredCustomMetricsList); + break; + case PANEL_STATE.NO_INTEGRATION: + this.setVisible( + this.$monitoredCustomMetricsNoIntegrationText, + this.$monitoredCustomMetricsEmpty, + ); + break; + default: + this.setVisible( + this.$monitoredCustomMetricsEmpty, + this.$newCustomMetricButton, + this.$newCustomMetricText, + ); + break; + } + } + + populateCustomMetrics() { + const capitalizeGroup = metric => ({ + ...metric, + group: capitalizeFirstCharacter(metric.group), + }); + + const sortedMetrics = sortBy(this.customMetrics.map(capitalizeGroup), ['group', 'title']); + + sortedMetrics.forEach(metric => { + this.$monitoredCustomMetricsList.append(CustomMetrics.customMetricTemplate(metric)); + }); + + this.$monitoredCustomMetricsCount.text(this.customMetrics.length); + this.showMonitoringCustomMetricsPanelState(PANEL_STATE.LIST); + if (!this.environmentsData) { + this.showFlashMessage( + s__( + 'PrometheusService|These metrics will only be monitored after your first deployment to an environment', + ), + ); + } + } + + showFlashMessage(message) { + this.$flashCustomMetricsContainer.removeClass('hidden'); + this.$flashCustomMetricsContainer.find('.flash-text').text(message); + } + + setNoIntegrationActiveState() { + this.showMonitoringCustomMetricsPanelState(PANEL_STATE.NO_INTEGRATION); + this.showMonitoringMetricsPanelState(PANEL_STATE.EMPTY); + } + + loadActiveCustomMetrics() { + super.loadActiveMetrics(); + Promise.all([ + axios.get(this.activeCustomMetricsEndpoint), + axios.get(this.environmentsDataEndpoint), + ]) + .then(([customMetrics, environmentsData]) => { + this.environmentsData = environmentsData.data.environments; + if (!customMetrics.data || !customMetrics.data.metrics) { + this.showMonitoringCustomMetricsPanelState(PANEL_STATE.EMPTY); + } else { + this.customMetrics = customMetrics.data.metrics; + this.populateCustomMetrics(customMetrics.data.metrics); + } + }) + .catch(customMetricError => { + this.showFlashMessage(customMetricError); + this.showMonitoringCustomMetricsPanelState(PANEL_STATE.EMPTY); + }); + } + + static customMetricTemplate(metric) { + return ` + <li class="custom-metric"> + <a href="${escape(metric.edit_path)}" class="custom-metric-link-bold"> + ${escape(metric.group)} / ${escape(metric.title)} (${escape(metric.unit)}) + </a> + </li> + `; + } +} diff --git a/app/helpers/clusters_helper.rb b/app/helpers/clusters_helper.rb index 21bd2ff9e32..a97216f8a22 100644 --- a/app/helpers/clusters_helper.rb +++ b/app/helpers/clusters_helper.rb @@ -41,6 +41,23 @@ module ClustersHelper end end + def cluster_type_label(cluster_type) + case cluster_type + when 'project_type' + s_('ClusterIntegration|Project cluster') + when 'group_type' + s_('ClusterIntegration|Group cluster') + when 'instance_type' + s_('ClusterIntegration|Instance cluster') + else + Gitlab::ErrorTracking.track_and_raise_for_dev_exception( + ArgumentError.new('Cluster Type Missing'), + cluster_error: { error: 'Cluster Type Missing', cluster_type: cluster_type } + ) + _('Cluster') + end + end + def has_rbac_enabled?(cluster) return cluster.platform_kubernetes_rbac? if cluster.platform_kubernetes diff --git a/app/models/performance_monitoring/prometheus_panel.rb b/app/models/performance_monitoring/prometheus_panel.rb index c03218b4219..0381c5cff10 100644 --- a/app/models/performance_monitoring/prometheus_panel.rb +++ b/app/models/performance_monitoring/prometheus_panel.rb @@ -4,7 +4,7 @@ module PerformanceMonitoring class PrometheusPanel include ActiveModel::Model - attr_accessor :type, :title, :y_label, :weight, :metrics + attr_accessor :type, :title, :y_label, :weight, :metrics, :y_axis validates :title, presence: true validates :metrics, presence: true @@ -20,5 +20,9 @@ module PerformanceMonitoring panel.tap(&:validate!) end + + def id(group_title) + Digest::SHA2.hexdigest([group_title, type, title].join) + end end end diff --git a/app/services/metrics/dashboard/base_service.rb b/app/services/metrics/dashboard/base_service.rb index a19f3f78b3d..219b26defb1 100644 --- a/app/services/metrics/dashboard/base_service.rb +++ b/app/services/metrics/dashboard/base_service.rb @@ -11,6 +11,7 @@ module Metrics SEQUENCE = [ STAGES::CommonMetricsInserter, STAGES::EndpointInserter, + STAGES::PanelIdsInserter, STAGES::Sorter ].freeze diff --git a/app/services/metrics/dashboard/gitlab_alert_embed_service.rb b/app/services/metrics/dashboard/gitlab_alert_embed_service.rb index 5515b84f112..38e89d392ad 100644 --- a/app/services/metrics/dashboard/gitlab_alert_embed_service.rb +++ b/app/services/metrics/dashboard/gitlab_alert_embed_service.rb @@ -10,7 +10,10 @@ module Metrics class GitlabAlertEmbedService < ::Metrics::Dashboard::BaseEmbedService include Gitlab::Utils::StrongMemoize - SEQUENCE = [STAGES::EndpointInserter].freeze + SEQUENCE = [ + STAGES::EndpointInserter, + STAGES::PanelIdsInserter + ].freeze class << self # Determines whether the provided params are sufficient diff --git a/app/services/metrics/dashboard/grafana_metric_embed_service.rb b/app/services/metrics/dashboard/grafana_metric_embed_service.rb index 274057b8262..d58b80162f5 100644 --- a/app/services/metrics/dashboard/grafana_metric_embed_service.rb +++ b/app/services/metrics/dashboard/grafana_metric_embed_service.rb @@ -10,7 +10,8 @@ module Metrics include ReactiveCaching SEQUENCE = [ - ::Gitlab::Metrics::Dashboard::Stages::GrafanaFormatter + ::Gitlab::Metrics::Dashboard::Stages::GrafanaFormatter, + ::Gitlab::Metrics::Dashboard::Stages::PanelIdsInserter ].freeze self.reactive_cache_key = ->(service) { service.cache_key } diff --git a/app/services/metrics/dashboard/predefined_dashboard_service.rb b/app/services/metrics/dashboard/predefined_dashboard_service.rb index 297f00b1be9..f454df63773 100644 --- a/app/services/metrics/dashboard/predefined_dashboard_service.rb +++ b/app/services/metrics/dashboard/predefined_dashboard_service.rb @@ -11,6 +11,7 @@ module Metrics SEQUENCE = [ STAGES::EndpointInserter, + STAGES::PanelIdsInserter, STAGES::Sorter ].freeze diff --git a/app/services/metrics/dashboard/self_monitoring_dashboard_service.rb b/app/services/metrics/dashboard/self_monitoring_dashboard_service.rb index bcdc8a8844f..d97668d1c7c 100644 --- a/app/services/metrics/dashboard/self_monitoring_dashboard_service.rb +++ b/app/services/metrics/dashboard/self_monitoring_dashboard_service.rb @@ -11,6 +11,7 @@ module Metrics SEQUENCE = [ STAGES::CustomMetricsInserter, STAGES::EndpointInserter, + STAGES::PanelIdsInserter, STAGES::Sorter ].freeze diff --git a/app/services/metrics/dashboard/system_dashboard_service.rb b/app/services/metrics/dashboard/system_dashboard_service.rb index 1c9d7f4968a..c28b7b875df 100644 --- a/app/services/metrics/dashboard/system_dashboard_service.rb +++ b/app/services/metrics/dashboard/system_dashboard_service.rb @@ -13,6 +13,7 @@ module Metrics STAGES::CustomMetricsInserter, STAGES::CustomMetricsDetailsInserter, STAGES::EndpointInserter, + STAGES::PanelIdsInserter, STAGES::Sorter ].freeze diff --git a/app/services/tags/create_service.rb b/app/services/tags/create_service.rb index 4de6b2d2774..8a7b98ab944 100644 --- a/app/services/tags/create_service.rb +++ b/app/services/tags/create_service.rb @@ -4,7 +4,7 @@ module Tags class CreateService < BaseService def execute(tag_name, target, message) valid_tag = Gitlab::GitRefValidator.validate(tag_name) - return error('Tag name invalid') unless valid_tag + return error('Tag name invalid', 400) unless valid_tag repository = project.repository message = message&.strip @@ -14,7 +14,7 @@ module Tags begin new_tag = repository.add_tag(current_user, tag_name, target, message) rescue Gitlab::Git::Repository::TagExistsError - return error("Tag #{tag_name} already exists") + return error("Tag #{tag_name} already exists", 409) rescue Gitlab::Git::PreReceiveError => ex return error(ex.message) end @@ -24,7 +24,7 @@ module Tags success.merge(tag: new_tag) else - error("Target #{target} is invalid") + error("Target #{target} is invalid", 400) end end end diff --git a/app/views/clusters/clusters/show.html.haml b/app/views/clusters/clusters/show.html.haml index 8797178f920..7fc76880480 100644 --- a/app/views/clusters/clusters/show.html.haml +++ b/app/views/clusters/clusters/show.html.haml @@ -41,7 +41,12 @@ .js-serverless-survey-banner{ data: { user_name: current_user.name, user_email: current_user.email } } - %h4= @cluster.name + .d-flex.my-3 + %p.badge.badge-light.p-2.mr-2 + = cluster_type_label(@cluster.cluster_type) + %h4.m-0 + = @cluster.name + = render 'banner' - if cluster_created?(@cluster) @@ -56,7 +61,3 @@ .tab-content.py-3 .tab-pane.active{ role: 'tabpanel' } = render_cluster_info_tab_content(params[:tab], expanded_by_default?) - - - - diff --git a/app/views/projects/services/prometheus/_custom_metrics.html.haml b/app/views/projects/services/prometheus/_custom_metrics.html.haml new file mode 100644 index 00000000000..21f9d1125e0 --- /dev/null +++ b/app/views/projects/services/prometheus/_custom_metrics.html.haml @@ -0,0 +1,30 @@ +- project = local_assigns.fetch(:project) + +.col-lg-3 + %p + = s_('PrometheusService|Custom metrics require Prometheus installed on a cluster with environment scope "*" OR a manually configured Prometheus to be available.') + = link_to s_('PrometheusService|More information'), help_page_path('user/project/integrations/prometheus', anchor: 'adding-custom-metrics'), target: '_blank', rel: "noopener noreferrer" + +.col-lg-9 + .card.custom-monitored-metrics.js-panel-custom-monitored-metrics{ data: { active_custom_metrics: project_prometheus_metrics_path(project), environments_data: environments_list_data, service_active: "#{@service.active}" } } + .card-header + %strong + = s_('PrometheusService|Custom metrics') + -# haml-lint:disable NoPlainNodes + %span.badge.badge-pill.js-custom-monitored-count 0 + -# haml-lint:enable NoPlainNodes + = link_to s_('PrometheusService|New metric'), new_project_prometheus_metric_path(project), class: 'btn btn-success js-new-metric-button hidden' + .card-body + .flash-container.hidden + .flash-warning + .flash-text + .loading-metrics.js-loading-custom-metrics + %p.m-3 + = icon('spinner spin', class: 'metrics-load-spinner') + = s_('PrometheusService|Finding custom metrics...') + .empty-metrics.hidden.js-empty-custom-metrics + %p.text-tertiary.m-3.js-no-active-integration-text.hidden + = s_('PrometheusService|Enable Prometheus to define custom metrics, using either option above') + %p.text-tertiary.m-3.js-new-metric-text.hidden + = s_('PrometheusService|No custom metrics have been created. Create one using the button above') + %ul.list-unstyled.metrics-list.hidden.js-custom-metrics-list diff --git a/app/views/projects/services/prometheus/_metrics.html.haml b/app/views/projects/services/prometheus/_metrics.html.haml index 7685dee08fc..3bd5f69f67e 100644 --- a/app/views/projects/services/prometheus/_metrics.html.haml +++ b/app/views/projects/services/prometheus/_metrics.html.haml @@ -1,5 +1,7 @@ - project = local_assigns.fetch(:project) += render 'projects/services/prometheus/custom_metrics', project: project + .col-lg-3 %p = s_('PrometheusService|Common metrics are automatically monitored based on a library of metrics from popular exporters.') diff --git a/changelogs/unreleased/fix-release-api-internal-server-error.yml b/changelogs/unreleased/fix-release-api-internal-server-error.yml new file mode 100644 index 00000000000..e0816a6deb0 --- /dev/null +++ b/changelogs/unreleased/fix-release-api-internal-server-error.yml @@ -0,0 +1,5 @@ +--- +title: Fix 500 error on create release API when providing an invalid tag_name +merge_request: 28969 +author: Sashi Kumar +type: fixed diff --git a/changelogs/unreleased/mwaw-212617-generate-panel-ids-based-on-panel-title-and-group-title-in-th.yml b/changelogs/unreleased/mwaw-212617-generate-panel-ids-based-on-panel-title-and-group-title-in-th.yml new file mode 100644 index 00000000000..9b8a3d8af8a --- /dev/null +++ b/changelogs/unreleased/mwaw-212617-generate-panel-ids-based-on-panel-title-and-group-title-in-th.yml @@ -0,0 +1,5 @@ +--- +title: Automatically assign id to each panel within dashboard to support panel scoped annotations +merge_request: 28341 +author: +type: added diff --git a/changelogs/unreleased/show-cluster-type.yml b/changelogs/unreleased/show-cluster-type.yml new file mode 100644 index 00000000000..672f77ccabf --- /dev/null +++ b/changelogs/unreleased/show-cluster-type.yml @@ -0,0 +1,5 @@ +--- +title: Display cluster type in cluster info page +merge_request: 27366 +author: +type: added diff --git a/doc/administration/availability/index.md b/doc/administration/availability/index.md new file mode 100644 index 00000000000..90113985ad5 --- /dev/null +++ b/doc/administration/availability/index.md @@ -0,0 +1,131 @@ +--- +type: reference, concepts +--- + +# Availability + +GitLab offers high availability options for organizations that require +the fault tolerance and redundancy necessary to maintain high-uptime operations. + +Please consult our [scaling documentation](../scaling) if you want to resolve +performance bottlenecks you encounter in individual GitLab components without +incurring the additional complexity costs associated with maintaining a +highly-available architecture. + +On this page, we present examples of self-managed instances which demonstrate +how GitLab can be scaled out and made highly available. These examples progress +from simple to complex as scaling or highly-available components are added. + +For larger setups serving 2,000 or more users, we provide +[reference architectures](../scaling/index.md#reference-architectures) based on GitLab's +experience with GitLab.com and internal scale testing that aim to achieve the +right balance of scalability and availability. + +For detailed insight into how GitLab scales and configures GitLab.com, you can +watch [this 1 hour Q&A](https://www.youtube.com/watch?v=uCU8jdYzpac) +with [John Northrup](https://gitlab.com/northrup), and live questions coming +in from some of our customers. + +## High availability + +### Omnibus installation with automatic database failover + +By adding automatic failover for database systems, we can enable higher uptime with an additional layer of complexity. + +- For PostgreSQL, we provide repmgr for server cluster management and failover + and a combination of [PgBouncer](../high_availability/pgbouncer.md) and [Consul](../high_availability/consul.md) for + database client cutover. +- For Redis, we use [Redis Sentinel](../high_availability/redis.md) for server failover and client cutover. + +You can also optionally run [additional Sidekiq processes on dedicated hardware](../high_availability/sidekiq.md) +and configure individual Sidekiq processes to +[process specific background job queues](../operations/extra_sidekiq_processes.md) +if you need to scale out background job processing. + +### GitLab Geo + +GitLab Geo allows you to replicate your GitLab instance to other geographical locations as a read-only fully operational instance that can also be promoted in case of disaster. + +This configuration is supported in [GitLab Premium and Ultimate](https://about.gitlab.com/pricing/). + +References: + +- [Geo Documentation](../geo/replication/index.md) +- [GitLab Geo with a highly available configuration](../geo/replication/high_availability.md) + +## GitLab components and configuration instructions + +The GitLab application depends on the following [components](../../development/architecture.md#component-diagram). +It can also depend on several third party services depending on +your environment setup. Here we'll detail both in the order in which +you would typically configure them along with our recommendations for +their use and configuration. + +### Third party services + +Here's some details of several third party services a typical environment +will depend on. The services can be provided by numerous applications +or providers and further advice can be given on how best to select. +These should be configured first, before the [GitLab components](#gitlab-components). + +| Component | Description | Configuration instructions | +|--------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| +| [Load Balancer(s)](../high_availability/load_balancer.md)[^6] | Handles load balancing for the GitLab nodes where required | [Load balancer HA configuration](../high_availability/load_balancer.md) | +| [Cloud Object Storage service](../high_availability/object_storage.md)[^4] | Recommended store for shared data objects | [Cloud Object Storage configuration](../high_availability/object_storage.md) | +| [NFS](../high_availability/nfs.md)[^5] [^7] | Shared disk storage service. Can be used as an alternative for Gitaly or Object Storage. Required for GitLab Pages | [NFS configuration](../high_availability/nfs.md) | + +### GitLab components + +Next are all of the components provided directly by GitLab. As mentioned +earlier, they are presented in the typical order you would configure +them. + +| Component | Description | Configuration instructions | +|---------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------|---------------------------------------------------------------| +| [Consul](../../development/architecture.md#consul)[^3] | Service discovery and health checks/failover | [Consul HA configuration](../high_availability/consul.md) **(PREMIUM ONLY)** | +| [PostgreSQL](../../development/architecture.md#postgresql) | Database | [Database HA configuration](../high_availability/database.md) | +| [PgBouncer](../../development/architecture.md#pgbouncer) | Database Pool Manager | [PgBouncer HA configuration](../high_availability/pgbouncer.md) **(PREMIUM ONLY)** | +| [Redis](../../development/architecture.md#redis)[^3] with Redis Sentinel | Key/Value store for shared data with HA watcher service | [Redis HA configuration](../high_availability/redis.md) | +| [Gitaly](../../development/architecture.md#gitaly)[^2] [^5] [^7] | Recommended high-level storage for Git repository data | [Gitaly HA configuration](../high_availability/gitaly.md) | +| [Sidekiq](../../development/architecture.md#sidekiq) | Asynchronous/Background jobs | [Sidekiq configuration](../high_availability/sidekiq.md) | +| [GitLab application nodes](../../development/architecture.md#unicorn)[^1] | (Unicorn / Puma, Workhorse) - Web-requests (UI, API, Git over HTTP) | [GitLab app HA/scaling configuration](../high_availability/gitlab.md) | +| [Prometheus](../../development/architecture.md#prometheus) and [Grafana](../../development/architecture.md#grafana) | GitLab environment monitoring | [Monitoring node for scaling/HA](../high_availability/monitoring_node.md) | + +In some cases, components can be combined on the same nodes to reduce complexity as well. + +[^1]: In our architectures we run each GitLab Rails node using the Puma webserver + and have its number of workers set to 90% of available CPUs along with 4 threads. + +[^2]: Gitaly node requirements are dependent on customer data, specifically the number of + projects and their sizes. We recommend 2 nodes as an absolute minimum for HA environments + and at least 4 nodes should be used when supporting 50,000 or more users. + We also recommend that each Gitaly node should store no more than 5TB of data + and have the number of [`gitaly-ruby` workers](../gitaly/index.md#gitaly-ruby) + set to 20% of available CPUs. Additional nodes should be considered in conjunction + with a review of expected data size and spread based on the recommendations above. + +[^3]: Recommended Redis setup differs depending on the size of the architecture. + For smaller architectures (up to 5,000 users) we suggest one Redis cluster for all + classes and that Redis Sentinel is hosted alongside Consul. + For larger architectures (10,000 users or more) we suggest running a separate + [Redis Cluster](../high_availability/redis.md#running-multiple-redis-clusters) for the Cache class + and another for the Queues and Shared State classes respectively. We also recommend + that you run the Redis Sentinel clusters separately as well for each Redis Cluster. + +[^4]: For data objects such as LFS, Uploads, Artifacts, etc. We recommend a [Cloud Object Storage service](../object_storage.md) + over NFS where possible, due to better performance and availability. + +[^5]: NFS can be used as an alternative for both repository data (replacing Gitaly) and + object storage but this isn't typically recommended for performance reasons. Note however it is required for + [GitLab Pages](https://gitlab.com/gitlab-org/gitlab-pages/issues/196). + +[^6]: Our architectures have been tested and validated with [HAProxy](https://www.haproxy.org/) + as the load balancer. However other reputable load balancers with similar feature sets + should also work instead but be aware these aren't validated. + +[^7]: We strongly recommend that any Gitaly and / or NFS nodes are set up with SSD disks over + HDD with a throughput of at least 8,000 IOPS for read operations and 2,000 IOPS for write + as these components have heavy I/O. These IOPS values are recommended only as a starter + as with time they may be adjusted higher or lower depending on the scale of your + environment's workload. If you're running the environment on a Cloud provider + you may need to refer to their documentation on how configure IOPS correctly. diff --git a/doc/administration/file_hooks.md b/doc/administration/file_hooks.md index 60bfca74b6b..7d935af8e37 100644 --- a/doc/administration/file_hooks.md +++ b/doc/administration/file_hooks.md @@ -113,4 +113,4 @@ Validating file hooks from /plugins directory [system hooks]: ../system_hooks/system_hooks.md [webhooks]: ../user/project/integrations/webhooks.md -[highly available]: ./high_availability/README.md +[highly available]: ./availability/index.md diff --git a/doc/administration/geo/replication/external_database.md b/doc/administration/geo/replication/external_database.md index f0fff3c6712..fd14c100ffb 100644 --- a/doc/administration/geo/replication/external_database.md +++ b/doc/administration/geo/replication/external_database.md @@ -129,7 +129,7 @@ To configure the connection to the external read-replica database and enable Log database to keep track of replication status and automatically recover from potential replication issues. Omnibus automatically configures a tracking database when `roles ['geo_secondary_role']` is set. For high availability, -refer to [Geo High Availability](../../high_availability/README.md). +refer to [Geo High Availability](../../availability/index.md). If you want to run this database external to Omnibus, please follow the instructions below. The tracking database requires an [FDW](https://www.postgresql.org/docs/9.6/postgres-fdw.html) diff --git a/doc/administration/geo/replication/high_availability.md b/doc/administration/geo/replication/high_availability.md index d64262e0399..5099e73d5e8 100644 --- a/doc/administration/geo/replication/high_availability.md +++ b/doc/administration/geo/replication/high_availability.md @@ -47,12 +47,12 @@ It is possible to use cloud hosted services for PostgreSQL and Redis, but this i ## Prerequisites: Two working GitLab HA clusters One cluster will serve as the **primary** node. Use the -[GitLab HA documentation](../../high_availability/README.md) to set this up. If +[GitLab HA documentation](../../availability/index.md) to set this up. If you already have a working GitLab instance that is in-use, it can be used as a **primary**. The second cluster will serve as the **secondary** node. Again, use the -[GitLab HA documentation](../../high_availability/README.md) to set this up. +[GitLab HA documentation](../../availability/index.md) to set this up. It's a good idea to log in and test it, however, note that its data will be wiped out as part of the process of replicating from the **primary**. @@ -371,7 +371,7 @@ more information. The minimal reference architecture diagram above shows all application services running together on the same machines. However, for high availability we -[strongly recommend running all services separately](../../high_availability/README.md). +[strongly recommend running all services separately](../../availability/index.md). For example, a Sidekiq server could be configured similarly to the frontend application servers above, with some changes to run only the `sidekiq` service: diff --git a/doc/administration/geo/replication/index.md b/doc/administration/geo/replication/index.md index 74f9f8c05f4..7c661abef9a 100644 --- a/doc/administration/geo/replication/index.md +++ b/doc/administration/geo/replication/index.md @@ -2,7 +2,7 @@ > - Introduced in GitLab Enterprise Edition 8.9. > - Using Geo in combination with -> [High Availability](../../high_availability/README.md) +> [High Availability](../../availability/index.md) > is considered **Generally Available** (GA) in > [GitLab Premium](https://about.gitlab.com/pricing/) 10.4. diff --git a/doc/administration/high_availability/README.md b/doc/administration/high_availability/README.md index 585c4903692..55ec3b8d6c4 100644 --- a/doc/administration/high_availability/README.md +++ b/doc/administration/high_availability/README.md @@ -4,137 +4,4 @@ type: reference, concepts # High Availability -GitLab offers high availability options for organizations that require -the fault tolerance and redundancy necessary to maintain high-uptime operations. - -Please consult our [scaling documentation](../scaling) if you want to resolve -performance bottlenecks you encounter in individual GitLab components without -incurring the additional complexity costs associated with maintaining a -highly-available architecture. - -On this page, we present examples of self-managed instances which demonstrate -how GitLab can be scaled out and made highly available. These examples progress -from simple to complex as scaling or highly-available components are added. - -For larger setups serving 2,000 or more users, we provide -[reference architectures](../scaling/index.md#reference-architectures) based on GitLab's -experience with GitLab.com and internal scale testing that aim to achieve the -right balance of scalability and availability. - -For detailed insight into how GitLab scales and configures GitLab.com, you can -watch [this 1 hour Q&A](https://www.youtube.com/watch?v=uCU8jdYzpac) -with [John Northrup](https://gitlab.com/northrup), and live questions coming -in from some of our customers. - -## Examples - -### Omnibus installation with automatic database failover - -By adding automatic failover for database systems, we can enable higher uptime with an additional layer of complexity. - -- For PostgreSQL, we provide repmgr for server cluster management and failover - and a combination of [PgBouncer](pgbouncer.md) and [Consul](consul.md) for - database client cutover. -- For Redis, we use [Redis Sentinel](redis.md) for server failover and client cutover. - -You can also optionally run [additional Sidekiq processes on dedicated hardware](sidekiq.md) -and configure individual Sidekiq processes to -[process specific background job queues](../operations/extra_sidekiq_processes.md) -if you need to scale out background job processing. - -### GitLab Geo - -GitLab Geo allows you to replicate your GitLab instance to other geographical locations as a read-only fully operational instance that can also be promoted in case of disaster. - -This configuration is supported in [GitLab Premium and Ultimate](https://about.gitlab.com/pricing/). - -References: - -- [Geo Documentation](../geo/replication/index.md) -- [GitLab Geo with a highly available configuration](../geo/replication/high_availability.md) - -## GitLab components and configuration instructions - -The GitLab application depends on the following [components](../../development/architecture.md#component-diagram). -It can also depend on several third party services depending on -your environment setup. Here we'll detail both in the order in which -you would typically configure them along with our recommendations for -their use and configuration. - -### Third party services - -Here's some details of several third party services a typical environment -will depend on. The services can be provided by numerous applications -or providers and further advice can be given on how best to select. -These should be configured first, before the [GitLab components](#gitlab-components). - -| Component | Description | Configuration instructions | -|--------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| -| [Load Balancer(s)](load_balancer.md)[^6] | Handles load balancing for the GitLab nodes where required | [Load balancer HA configuration](load_balancer.md) | -| [Cloud Object Storage service](object_storage.md)[^4] | Recommended store for shared data objects | [Cloud Object Storage configuration](object_storage.md) | -| [NFS](nfs.md)[^5] [^7] | Shared disk storage service. Can be used as an alternative for Gitaly or Object Storage. Required for GitLab Pages | [NFS configuration](nfs.md) | - -### GitLab components - -Next are all of the components provided directly by GitLab. As mentioned -earlier, they are presented in the typical order you would configure -them. - -| Component | Description | Configuration instructions | -|---------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------|---------------------------------------------------------------| -| [Consul](../../development/architecture.md#consul)[^3] | Service discovery and health checks/failover | [Consul HA configuration](consul.md) **(PREMIUM ONLY)** | -| [PostgreSQL](../../development/architecture.md#postgresql) | Database | [Database HA configuration](database.md) | -| [PgBouncer](../../development/architecture.md#pgbouncer) | Database Pool Manager | [PgBouncer HA configuration](pgbouncer.md) **(PREMIUM ONLY)** | -| [Redis](../../development/architecture.md#redis)[^3] with Redis Sentinel | Key/Value store for shared data with HA watcher service | [Redis HA configuration](redis.md) | -| [Gitaly](../../development/architecture.md#gitaly)[^2] [^5] [^7] | Recommended high-level storage for Git repository data | [Gitaly HA configuration](gitaly.md) | -| [Sidekiq](../../development/architecture.md#sidekiq) | Asynchronous/Background jobs | [Sidekiq configuration](sidekiq.md) | -| [GitLab application nodes](../../development/architecture.md#unicorn)[^1] | (Unicorn / Puma, Workhorse) - Web-requests (UI, API, Git over HTTP) | [GitLab app HA/scaling configuration](gitlab.md) | -| [Prometheus](../../development/architecture.md#prometheus) and [Grafana](../../development/architecture.md#grafana) | GitLab environment monitoring | [Monitoring node for scaling/HA](monitoring_node.md) | - -In some cases, components can be combined on the same nodes to reduce complexity as well. - -[^1]: In our architectures we run each GitLab Rails node using the Puma webserver - and have its number of workers set to 90% of available CPUs along with 4 threads. - -[^2]: Gitaly node requirements are dependent on customer data, specifically the number of - projects and their sizes. We recommend 2 nodes as an absolute minimum for HA environments - and at least 4 nodes should be used when supporting 50,000 or more users. - We also recommend that each Gitaly node should store no more than 5TB of data - and have the number of [`gitaly-ruby` workers](../gitaly/index.md#gitaly-ruby) - set to 20% of available CPUs. Additional nodes should be considered in conjunction - with a review of expected data size and spread based on the recommendations above. - -[^3]: Recommended Redis setup differs depending on the size of the architecture. - For smaller architectures (up to 5,000 users) we suggest one Redis cluster for all - classes and that Redis Sentinel is hosted alongside Consul. - For larger architectures (10,000 users or more) we suggest running a separate - [Redis Cluster](redis.md#running-multiple-redis-clusters) for the Cache class - and another for the Queues and Shared State classes respectively. We also recommend - that you run the Redis Sentinel clusters separately as well for each Redis Cluster. - -[^4]: For data objects such as LFS, Uploads, Artifacts, etc. We recommend a [Cloud Object Storage service](object_storage.md) - over NFS where possible, due to better performance and availability. - -[^5]: NFS can be used as an alternative for both repository data (replacing Gitaly) and - object storage but this isn't typically recommended for performance reasons. Note however it is required for - [GitLab Pages](https://gitlab.com/gitlab-org/gitlab-pages/issues/196). - -[^6]: Our architectures have been tested and validated with [HAProxy](https://www.haproxy.org/) - as the load balancer. However other reputable load balancers with similar feature sets - should also work instead but be aware these aren't validated. - -[^7]: We strongly recommend that any Gitaly and / or NFS nodes are set up with SSD disks over - HDD with a throughput of at least 8,000 IOPS for read operations and 2,000 IOPS for write - as these components have heavy I/O. These IOPS values are recommended only as a starter - as with time they may be adjusted higher or lower depending on the scale of your - environment's workload. If you're running the environment on a Cloud provider - you may need to refer to their documentation on how configure IOPS correctly. - -[^8]: The architectures were built and tested with the [Intel Xeon E5 v3 (Haswell)](https://cloud.google.com/compute/docs/cpu-platforms) - CPU platform on GCP. On different hardware you may find that adjustments, either lower - or higher, are required for your CPU or Node counts accordingly. For more information, a - [Sysbench](https://github.com/akopytov/sysbench) benchmark of the CPU can be found - [here](https://gitlab.com/gitlab-org/quality/performance/-/wikis/Reference-Architectures/GCP-CPU-Benchmarks). - -[^9]: AWS-equivalent configurations are rough suggestions and may change in the - future. They have not yet been tested and validated. +This content has been moved to the [availability page](../availability/index.md). diff --git a/doc/administration/high_availability/database.md b/doc/administration/high_availability/database.md index 01af971d664..f06870be93c 100644 --- a/doc/administration/high_availability/database.md +++ b/doc/administration/high_availability/database.md @@ -24,7 +24,7 @@ If you use a cloud-managed service, or provide your own PostgreSQL: ## PostgreSQL in a Scaled and Highly Available Environment -This section is relevant for [Scalable and Highly Available Setups](README.md). +This section is relevant for [Scalable and Highly Available Setups](../scaling/index.md). ### Provide your own PostgreSQL instance **(CORE ONLY)** diff --git a/doc/administration/high_availability/gitaly.md b/doc/administration/high_availability/gitaly.md index bb40747b24c..5d66d3c5c94 100644 --- a/doc/administration/high_availability/gitaly.md +++ b/doc/administration/high_availability/gitaly.md @@ -11,7 +11,7 @@ should consider using Gitaly on a separate node. See the [Gitaly HA Epic](https://gitlab.com/groups/gitlab-org/-/epics/289) to track plans and progress toward high availability support. -This document is relevant for [Scalable and Highly Available Setups](README.md). +This document is relevant for [Scalable and Highly Available Setups](../scaling/index.md). ## Running Gitaly on its own server @@ -19,7 +19,7 @@ See [Running Gitaly on its own server](../gitaly/index.md#running-gitaly-on-its- in Gitaly documentation. Continue configuration of other components by going back to the -[Scaling and High Availability](README.md#gitlab-components-and-configuration-instructions) page. +[High Availability](../availability/index.md#gitlab-components-and-configuration-instructions) page. ## Enable Monitoring diff --git a/doc/administration/high_availability/monitoring_node.md b/doc/administration/high_availability/monitoring_node.md index a12b865cb91..79e583c5fcb 100644 --- a/doc/administration/high_availability/monitoring_node.md +++ b/doc/administration/high_availability/monitoring_node.md @@ -11,7 +11,7 @@ You can configure a Prometheus node to monitor GitLab. ## Standalone Monitoring node using GitLab Omnibus The GitLab Omnibus package can be used to configure a standalone Monitoring node running [Prometheus](../monitoring/prometheus/index.md) and [Grafana](../monitoring/performance/grafana_configuration.md). -The monitoring node is not highly available. See [Scaling and High Availability](README.md) +The monitoring node is not highly available. See [Scaling and High Availability](../scaling/index.md) for an overview of GitLab scaling and high availability options. The steps below are the minimum necessary to configure a Monitoring node running Prometheus and Grafana with diff --git a/doc/administration/high_availability/redis.md b/doc/administration/high_availability/redis.md index d7cdf614c0a..a1e5482e5dc 100644 --- a/doc/administration/high_availability/redis.md +++ b/doc/administration/high_availability/redis.md @@ -22,7 +22,7 @@ These will be necessary when configuring the GitLab application servers later. ## Redis in a Scaled and Highly Available Environment -This section is relevant for [Scalable and Highly Available Setups](README.md). +This section is relevant for [Scalable and Highly Available Setups](../scaling/index.md). ### Provide your own Redis instance **(CORE ONLY)** @@ -38,7 +38,7 @@ In this configuration Redis is not highly available, and represents a single point of failure. However, in a scaled environment the objective is to allow the environment to handle more users or to increase throughput. Redis itself is generally stable and can handle many requests so it is an acceptable -trade off to have only a single instance. See [Scaling and High Availability](README.md) +trade off to have only a single instance. See [High Availability](../availability/index.md) for an overview of GitLab scaling and high availability options. The steps below are the minimum necessary to configure a Redis server with @@ -84,7 +84,7 @@ Advanced configuration options are supported and can be added if needed. Continue configuration of other components by going back to the -[Scaling and High Availability](README.md#gitlab-components-and-configuration-instructions) page. +[High Availability](../availability/index.md#gitlab-components-and-configuration-instructions) page. ### High Availability with GitLab Omnibus **(PREMIUM ONLY)** diff --git a/doc/administration/index.md b/doc/administration/index.md index 9c5a970b65f..1f4e23fce8a 100644 --- a/doc/administration/index.md +++ b/doc/administration/index.md @@ -34,7 +34,7 @@ Learn how to install, configure, update, and maintain your GitLab instance. - [Install](../install/README.md): Requirements, directory structures, and installation methods. - [Database load balancing](database_load_balancing.md): Distribute database queries among multiple database servers. **(STARTER ONLY)** - [Omnibus support for log forwarding](https://docs.gitlab.com/omnibus/settings/logs.html#udp-log-shipping-gitlab-enterprise-edition-only) **(STARTER ONLY)** -- [High Availability](high_availability/README.md): Configure multiple servers for scaling or high availability. +- [High Availability](availability/index.md): Configure multiple servers for scaling or high availability. - [Installing GitLab HA on Amazon Web Services (AWS)](../install/aws/index.md): Set up GitLab High Availability on Amazon AWS. - [Geo](geo/replication/index.md): Replicate your GitLab instance to other geographic locations as a read-only fully operational version. **(PREMIUM ONLY)** - [Disaster Recovery](geo/disaster_recovery/index.md): Quickly fail-over to a different site with minimal effort in a disaster situation. **(PREMIUM ONLY)** diff --git a/doc/administration/object_storage.md b/doc/administration/object_storage.md index 80305c89e81..1b0c7f8a907 100644 --- a/doc/administration/object_storage.md +++ b/doc/administration/object_storage.md @@ -77,9 +77,8 @@ with the Fog library that GitLab uses. Symptoms include: ### GitLab Pages requires NFS -If you're working to add more GitLab servers for [scaling](scaling/index.md) or -[fault tolerance](high_availability/README.md) and one of your requirements -is [GitLab Pages](../user/project/pages/index.md) this currently requires +If you're working to add more GitLab servers for [scaling or fault tolerance](scaling/index.md) +and one of your requirements is [GitLab Pages](../user/project/pages/index.md) this currently requires NFS. There is [work in progress](https://gitlab.com/gitlab-org/gitlab-pages/issues/196) to remove this dependency. In the future, GitLab Pages may use [object storage](https://gitlab.com/gitlab-org/gitlab/-/issues/208135). diff --git a/doc/administration/scaling/index.md b/doc/administration/scaling/index.md index 23a96e11805..c91fe395b3d 100644 --- a/doc/administration/scaling/index.md +++ b/doc/administration/scaling/index.md @@ -8,7 +8,7 @@ GitLab supports a number of scaling options to ensure that your self-managed instance is able to scale out to meet your organization's needs when scaling up a single-box GitLab installation is no longer practical or feasible. -Please consult our [high availability documentation](../high_availability/README.md) +Please consult our [high availability documentation](../availability/index.md) if your organization requires fault tolerance and redundancy features, such as automatic database system failover. diff --git a/doc/install/README.md b/doc/install/README.md index 06066bc0c7d..83910b0a281 100644 --- a/doc/install/README.md +++ b/doc/install/README.md @@ -22,7 +22,7 @@ There are many ways you can install GitLab depending on your platform: TIP: **If in doubt, choose Omnibus:** The Omnibus GitLab packages are mature, scalable, support -[high availability](../administration/high_availability/README.md) and are used +[high availability](../administration/availability/index.md) and are used today on GitLab.com. The Helm charts are recommended for those who are familiar with Kubernetes. @@ -36,7 +36,7 @@ The Omnibus GitLab package uses our official deb/rpm repositories. This is recommended for most users. If you need additional flexibility and resilience, we recommend deploying -GitLab as described in our [High Availability documentation](../administration/high_availability/README.md). +GitLab as described in our [High Availability documentation](../administration/availability/index.md). [**> Install GitLab using the Omnibus GitLab package.**](https://about.gitlab.com/install/) diff --git a/doc/install/aws/index.md b/doc/install/aws/index.md index 6708e40abb4..109cd635dc0 100644 --- a/doc/install/aws/index.md +++ b/doc/install/aws/index.md @@ -719,7 +719,7 @@ Have a read through these other resources and feel free to [open an issue](https://gitlab.com/gitlab-org/gitlab/issues/new) to request additional material: -- [GitLab High Availability](../../administration/high_availability/README.md): +- [Scaling GitLab](../../administration/scaling/index.md): GitLab supports several different types of clustering and high-availability. - [Geo replication](../../administration/geo/replication/index.md): Geo is the solution for widely distributed development teams. diff --git a/lib/gitlab/metrics/dashboard/stages/base_stage.rb b/lib/gitlab/metrics/dashboard/stages/base_stage.rb index f9e4ae4b4b5..622d5aa8cdb 100644 --- a/lib/gitlab/metrics/dashboard/stages/base_stage.rb +++ b/lib/gitlab/metrics/dashboard/stages/base_stage.rb @@ -37,10 +37,8 @@ module Gitlab def for_metrics missing_panel_groups! unless dashboard[:panel_groups].is_a?(Array) - dashboard[:panel_groups].each do |panel_group| - missing_panels! unless panel_group[:panels].is_a?(Array) - - panel_group[:panels].each do |panel| + for_panel_groups do |panel_group| + for_panels_in(panel_group) do |panel| missing_metrics! unless panel[:metrics].is_a?(Array) panel[:metrics].each do |metric| @@ -49,6 +47,20 @@ module Gitlab end end end + + def for_panel_groups + dashboard[:panel_groups].each do |panel_group| + yield panel_group + end + end + + def for_panels_in(panel_group) + missing_panels! unless panel_group[:panels].is_a?(Array) + + panel_group[:panels].each do |panel| + yield panel + end + end end end end diff --git a/lib/gitlab/metrics/dashboard/stages/panel_ids_inserter.rb b/lib/gitlab/metrics/dashboard/stages/panel_ids_inserter.rb new file mode 100644 index 00000000000..301c54b9f23 --- /dev/null +++ b/lib/gitlab/metrics/dashboard/stages/panel_ids_inserter.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + module Stages + class PanelIdsInserter < BaseStage + # For each panel within given dashboard inserts panel_id unique in scope of the dashboard + def transform! + missing_panel_groups! unless dashboard[:panel_groups] + + for_panels_group_with_panels do |panel_group, panel| + id = generate_panel_id(panel_group, panel) + remove_panel_ids! && break if duplicated_panel_id?(id) + + insert_panel_id(id, panel) + end + end + + private + + def generate_panel_id(group, panel) + ::PerformanceMonitoring::PrometheusPanel.new(panel.with_indifferent_access).id(group[:group]) + end + + def insert_panel_id(id, panel) + track_inserted_panel_ids(id, panel) + panel[:id] = id + end + + def track_inserted_panel_ids(id, panel) + panel_ids[id] = panel + end + + def duplicated_panel_id?(id) + panel_ids.key?(id) + end + + def remove_panel_ids! + panel_ids.each_value { |panel| panel.delete(:id) } + end + + def panel_ids + @_panel_ids ||= {} + end + + def for_panels_group_with_panels + for_panel_groups do |panel_group| + for_panels_in(panel_group) do |panel| + yield panel_group, panel + end + end + end + end + end + end + end +end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 78d1ba110b9..52446d172f5 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -4135,6 +4135,9 @@ msgstr "" msgid "Closes this %{quick_action_target}." msgstr "" +msgid "Cluster" +msgstr "" + msgid "Cluster Health" msgstr "" @@ -9168,6 +9171,9 @@ msgstr "" msgid "Footer message" msgstr "" +msgid "For each Jira issue successfully imported, we'll create a new GitLab issue with the following data:" +msgstr "" + msgid "For internal projects, any logged in user can view pipelines and access job details (output logs and artifacts)" msgstr "" @@ -10897,6 +10903,9 @@ msgstr "" msgid "Import an exported GitLab project" msgstr "" +msgid "Import from" +msgstr "" + msgid "Import from Jira" msgstr "" @@ -11304,6 +11313,9 @@ msgstr "" msgid "Issue first depoloyed to production" msgstr "" +msgid "Issue label" +msgstr "" + msgid "Issue or Merge Request ID is required" msgstr "" @@ -13311,6 +13323,9 @@ msgid_plural "New Issues" msgstr[0] "" msgstr[1] "" +msgid "New Jira import" +msgstr "" + msgid "New Label" msgstr "" @@ -17004,6 +17019,9 @@ msgstr "" msgid "Reported %{timeAgo} by %{reportedBy}" msgstr "" +msgid "Reporter" +msgstr "" + msgid "Reporting" msgstr "" @@ -24491,6 +24509,12 @@ msgstr "" msgid "jigsaw is not defined" msgstr "" +msgid "jira.issue.description.content" +msgstr "" + +msgid "jira.issue.summary" +msgstr "" + msgid "latest" msgstr "" diff --git a/spec/features/clusters/cluster_detail_page_spec.rb b/spec/features/clusters/cluster_detail_page_spec.rb index 28d6c10f04f..4b478163952 100644 --- a/spec/features/clusters/cluster_detail_page_spec.rb +++ b/spec/features/clusters/cluster_detail_page_spec.rb @@ -14,6 +14,12 @@ describe 'Clusterable > Show page' do end shared_examples 'show page' do + it 'displays cluster type label' do + visit cluster_path + + expect(page).to have_content(cluster_type_label) + end + it 'allow the user to set domain' do visit cluster_path @@ -125,7 +131,9 @@ describe 'Clusterable > Show page' do clusterable.add_maintainer(current_user) end - it_behaves_like 'show page' + it_behaves_like 'show page' do + let(:cluster_type_label) { 'Project cluster' } + end it_behaves_like 'editing a GCP cluster' @@ -143,7 +151,9 @@ describe 'Clusterable > Show page' do clusterable.add_maintainer(current_user) end - it_behaves_like 'show page' + it_behaves_like 'show page' do + let(:cluster_type_label) { 'Group cluster' } + end it_behaves_like 'editing a GCP cluster' @@ -157,7 +167,9 @@ describe 'Clusterable > Show page' do let(:cluster_path) { admin_cluster_path(cluster) } let(:cluster) { create(:cluster, :provided_by_gcp, :instance) } - it_behaves_like 'show page' + it_behaves_like 'show page' do + let(:cluster_type_label) { 'Instance cluster' } + end it_behaves_like 'editing a GCP cluster' diff --git a/spec/fixtures/lib/gitlab/metrics/dashboard/schemas/panels.json b/spec/fixtures/lib/gitlab/metrics/dashboard/schemas/panels.json index 9f39e9c77cb..fe2da16c9b7 100644 --- a/spec/fixtures/lib/gitlab/metrics/dashboard/schemas/panels.json +++ b/spec/fixtures/lib/gitlab/metrics/dashboard/schemas/panels.json @@ -7,6 +7,7 @@ ], "properties": { "title": { "type": "string" }, + "id": { "type": "string" }, "type": { "type": "string" }, "y_label": { "type": "string" }, "y_axis": { "$ref": "axis.json" }, diff --git a/spec/frontend/jira_import/components/jira_import_form_spec.js b/spec/frontend/jira_import/components/jira_import_form_spec.js new file mode 100644 index 00000000000..315ccccd991 --- /dev/null +++ b/spec/frontend/jira_import/components/jira_import_form_spec.js @@ -0,0 +1,62 @@ +import { GlAvatar, GlNewButton, GlFormSelect, GlLabel } from '@gitlab/ui'; +import { shallowMount } from '@vue/test-utils'; +import JiraImportForm from '~/jira_import/components/jira_import_form.vue'; + +describe('JiraImportForm', () => { + let wrapper; + + beforeEach(() => { + wrapper = shallowMount(JiraImportForm); + }); + + afterEach(() => { + wrapper.destroy(); + wrapper = null; + }); + + it('shows a dropdown to choose the Jira project to import from', () => { + expect(wrapper.find(GlFormSelect).exists()).toBe(true); + }); + + it('shows a label which will be applied to imported Jira projects', () => { + expect(wrapper.find(GlLabel).attributes('title')).toBe('jira-import::KEY-1'); + }); + + it('shows information to the user', () => { + expect(wrapper.find('p').text()).toBe( + "For each Jira issue successfully imported, we'll create a new GitLab issue with the following data:", + ); + }); + + it('shows jira.issue.summary for the Title', () => { + expect(wrapper.find('[id="jira-project-title"]').text()).toBe('jira.issue.summary'); + }); + + it('shows an avatar for the Reporter', () => { + expect(wrapper.find(GlAvatar).exists()).toBe(true); + }); + + it('shows jira.issue.description.content for the Description', () => { + expect(wrapper.find('[id="jira-project-description"]').text()).toBe( + 'jira.issue.description.content', + ); + }); + + it('shows a Next button', () => { + const nextButton = wrapper + .findAll(GlNewButton) + .at(0) + .text(); + + expect(nextButton).toBe('Next'); + }); + + it('shows a Cancel button', () => { + const cancelButton = wrapper + .findAll(GlNewButton) + .at(1) + .text(); + + expect(cancelButton).toBe('Cancel'); + }); +}); diff --git a/spec/frontend/prometheus_metrics/custom_metrics_spec.js b/spec/frontend/prometheus_metrics/custom_metrics_spec.js new file mode 100644 index 00000000000..3396b3694c0 --- /dev/null +++ b/spec/frontend/prometheus_metrics/custom_metrics_spec.js @@ -0,0 +1,122 @@ +import MockAdapter from 'axios-mock-adapter'; +import PrometheusMetrics from '~/prometheus_metrics/custom_metrics'; +import axios from '~/lib/utils/axios_utils'; +import PANEL_STATE from '~/prometheus_metrics/constants'; +import metrics from './mock_data'; + +describe('PrometheusMetrics', () => { + const FIXTURE = 'services/prometheus/prometheus_service.html'; + const customMetricsEndpoint = + 'http://test.host/frontend-fixtures/services-project/prometheus/metrics'; + let mock; + preloadFixtures(FIXTURE); + + beforeEach(() => { + mock = new MockAdapter(axios); + mock.onGet(customMetricsEndpoint).reply(200, { + metrics, + }); + loadFixtures(FIXTURE); + }); + + afterEach(() => { + mock.restore(); + }); + + describe('Custom Metrics', () => { + let prometheusMetrics; + + beforeEach(() => { + prometheusMetrics = new PrometheusMetrics('.js-prometheus-metrics-monitoring'); + }); + + it('should initialize wrapper element refs on the class object', () => { + expect(prometheusMetrics.$wrapperCustomMetrics).not.toBeNull(); + expect(prometheusMetrics.$monitoredCustomMetricsPanel).not.toBeNull(); + expect(prometheusMetrics.$monitoredCustomMetricsCount).not.toBeNull(); + expect(prometheusMetrics.$monitoredCustomMetricsLoading).not.toBeNull(); + expect(prometheusMetrics.$monitoredCustomMetricsEmpty).not.toBeNull(); + expect(prometheusMetrics.$monitoredCustomMetricsList).not.toBeNull(); + expect(prometheusMetrics.$newCustomMetricButton).not.toBeNull(); + expect(prometheusMetrics.$flashCustomMetricsContainer).not.toBeNull(); + }); + + it('should contain api endpoints', () => { + expect(prometheusMetrics.activeCustomMetricsEndpoint).toEqual(customMetricsEndpoint); + }); + + it('should show loading state when called with `loading`', () => { + prometheusMetrics.showMonitoringCustomMetricsPanelState(PANEL_STATE.LOADING); + + expect(prometheusMetrics.$monitoredCustomMetricsLoading.hasClass('hidden')).toEqual(false); + expect(prometheusMetrics.$monitoredCustomMetricsEmpty.hasClass('hidden')).toBeTruthy(); + expect(prometheusMetrics.$monitoredCustomMetricsList.hasClass('hidden')).toBeTruthy(); + expect( + prometheusMetrics.$monitoredCustomMetricsNoIntegrationText.hasClass('hidden'), + ).toBeTruthy(); + + expect(prometheusMetrics.$newCustomMetricButton.hasClass('hidden')).toBeTruthy(); + expect(prometheusMetrics.$newCustomMetricText.hasClass('hidden')).toBeTruthy(); + }); + + it('should show metrics list when called with `list`', () => { + prometheusMetrics.showMonitoringCustomMetricsPanelState(PANEL_STATE.LIST); + + expect(prometheusMetrics.$monitoredCustomMetricsLoading.hasClass('hidden')).toBeTruthy(); + expect(prometheusMetrics.$monitoredCustomMetricsEmpty.hasClass('hidden')).toBeTruthy(); + expect(prometheusMetrics.$monitoredCustomMetricsList.hasClass('hidden')).toEqual(false); + expect( + prometheusMetrics.$monitoredCustomMetricsNoIntegrationText.hasClass('hidden'), + ).toBeTruthy(); + + expect(prometheusMetrics.$newCustomMetricButton.hasClass('hidden')).toEqual(false); + expect(prometheusMetrics.$newCustomMetricText.hasClass('hidden')).toBeTruthy(); + }); + + it('should show empty state when called with `empty`', () => { + prometheusMetrics.showMonitoringCustomMetricsPanelState(PANEL_STATE.EMPTY); + + expect(prometheusMetrics.$monitoredCustomMetricsLoading.hasClass('hidden')).toBeTruthy(); + expect(prometheusMetrics.$monitoredCustomMetricsEmpty.hasClass('hidden')).toEqual(false); + expect(prometheusMetrics.$monitoredCustomMetricsList.hasClass('hidden')).toBeTruthy(); + expect( + prometheusMetrics.$monitoredCustomMetricsNoIntegrationText.hasClass('hidden'), + ).toBeTruthy(); + + expect(prometheusMetrics.$newCustomMetricButton.hasClass('hidden')).toEqual(false); + expect(prometheusMetrics.$newCustomMetricText.hasClass('hidden')).toEqual(false); + }); + + it('should show monitored metrics list', () => { + prometheusMetrics.customMetrics = metrics; + prometheusMetrics.populateCustomMetrics(); + + const $metricsListLi = prometheusMetrics.$monitoredCustomMetricsList.find('li'); + + expect(prometheusMetrics.$monitoredCustomMetricsLoading.hasClass('hidden')).toBeTruthy(); + expect(prometheusMetrics.$monitoredCustomMetricsList.hasClass('hidden')).toEqual(false); + expect( + prometheusMetrics.$monitoredCustomMetricsNoIntegrationText.hasClass('hidden'), + ).toBeTruthy(); + + expect(prometheusMetrics.$newCustomMetricButton.hasClass('hidden')).toEqual(false); + expect(prometheusMetrics.$newCustomMetricText.hasClass('hidden')).toBeTruthy(); + + expect($metricsListLi.length).toEqual(metrics.length); + }); + + it('should show the NO-INTEGRATION empty state', () => { + prometheusMetrics.setNoIntegrationActiveState(); + + expect(prometheusMetrics.$monitoredCustomMetricsEmpty.hasClass('hidden')).toEqual(false); + expect(prometheusMetrics.$monitoredCustomMetricsNoIntegrationText.hasClass('hidden')).toEqual( + false, + ); + + expect(prometheusMetrics.$monitoredCustomMetricsLoading.hasClass('hidden')).toBeTruthy(); + expect(prometheusMetrics.$monitoredCustomMetricsList.hasClass('hidden')).toBeTruthy(); + expect(prometheusMetrics.$newCustomMetricButton.hasClass('hidden')).toBeTruthy(); + expect(prometheusMetrics.$newCustomMetricText.hasClass('hidden')).toBeTruthy(); + }); + }); +}); diff --git a/spec/frontend/prometheus_metrics/mock_data.js b/spec/frontend/prometheus_metrics/mock_data.js new file mode 100644 index 00000000000..d5532537302 --- /dev/null +++ b/spec/frontend/prometheus_metrics/mock_data.js @@ -0,0 +1,22 @@ +const metrics = [ + { + edit_path: '/root/prometheus-test/prometheus/metrics/3/edit', + id: 3, + title: 'Requests', + group: 'Business', + }, + { + edit_path: '/root/prometheus-test/prometheus/metrics/2/edit', + id: 2, + title: 'Sales by the hour', + group: 'Business', + }, + { + edit_path: '/root/prometheus-test/prometheus/metrics/1/edit', + id: 1, + title: 'Requests', + group: 'Business', + }, +]; + +export default metrics; diff --git a/spec/helpers/clusters_helper_spec.rb b/spec/helpers/clusters_helper_spec.rb index 5651b899ed0..eec62bbb990 100644 --- a/spec/helpers/clusters_helper_spec.rb +++ b/spec/helpers/clusters_helper_spec.rb @@ -58,4 +58,39 @@ describe ClustersHelper do it { is_expected.to eq('Create new cluster') } end end + + describe '#cluster_type_label' do + subject { helper.cluster_type_label(cluster_type) } + + context 'project cluster' do + let(:cluster_type) { 'project_type' } + + it { is_expected.to eq('Project cluster') } + end + + context 'group cluster' do + let(:cluster_type) { 'group_type' } + + it { is_expected.to eq('Group cluster') } + end + + context 'instance cluster' do + let(:cluster_type) { 'instance_type' } + + it { is_expected.to eq('Instance cluster') } + end + + context 'other values' do + let(:cluster_type) { 'not_supported' } + + it 'Diplays generic cluster and reports error' do + expect(Gitlab::ErrorTracking).to receive(:track_and_raise_for_dev_exception).with( + an_instance_of(ArgumentError), + cluster_error: { error: 'Cluster Type Missing', cluster_type: 'not_supported' } + ) + + is_expected.to eq('Cluster') + end + end + end end diff --git a/spec/lib/gitlab/metrics/dashboard/stages/panel_ids_inserter_spec.rb b/spec/lib/gitlab/metrics/dashboard/stages/panel_ids_inserter_spec.rb new file mode 100644 index 00000000000..426a54bea78 --- /dev/null +++ b/spec/lib/gitlab/metrics/dashboard/stages/panel_ids_inserter_spec.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::Metrics::Dashboard::Stages::PanelIdsInserter do + let(:project) { build_stubbed(:project) } + + def fetch_panel_ids(dashboard_hash) + dashboard_hash[:panel_groups].flat_map { |group| group[:panels].flat_map { |panel| panel[:id] } } + end + + describe '#transform!' do + subject(:transform!) { described_class.new(project, dashboard, nil).transform! } + + let(:dashboard) { YAML.safe_load(fixture_file('lib/gitlab/metrics/dashboard/sample_dashboard.yml')).deep_symbolize_keys } + + context 'when dashboard panels are present' do + it 'assigns unique ids to each panel using PerformanceMonitoring::PrometheusPanel', :aggregate_failures do + dashboard.fetch(:panel_groups).each do |group| + group.fetch(:panels).each do |panel| + panel_double = instance_double(::PerformanceMonitoring::PrometheusPanel) + + expect(::PerformanceMonitoring::PrometheusPanel).to receive(:new).with(panel).and_return(panel_double) + expect(panel_double).to receive(:id).with(group[:group]).and_return(FFaker::Lorem.unique.characters(125)) + end + end + + transform! + + expect(fetch_panel_ids(dashboard)).not_to include nil + end + end + + context 'when dashboard panels has duplicated ids' do + it 'no panel has assigned id' do + panel_double = instance_double(::PerformanceMonitoring::PrometheusPanel) + allow(::PerformanceMonitoring::PrometheusPanel).to receive(:new).and_return(panel_double) + allow(panel_double).to receive(:id).and_return('duplicated id') + + transform! + + expect(fetch_panel_ids(dashboard)).to all be_nil + expect(fetch_panel_ids(dashboard)).not_to include 'duplicated id' + end + end + + context 'when there are no panels in the dashboard' do + it 'raises a processing error' do + dashboard[:panel_groups][0].delete(:panels) + + expect { transform! }.to( + raise_error(::Gitlab::Metrics::Dashboard::Errors::DashboardProcessingError) + ) + end + end + + context 'when there are no panel_groups in the dashboard' do + it 'raises a processing error' do + dashboard.delete(:panel_groups) + + expect { transform! }.to( + raise_error(::Gitlab::Metrics::Dashboard::Errors::DashboardProcessingError) + ) + end + end + end +end diff --git a/spec/requests/api/releases_spec.rb b/spec/requests/api/releases_spec.rb index 0589554bf44..237782a681c 100644 --- a/spec/requests/api/releases_spec.rb +++ b/spec/requests/api/releases_spec.rb @@ -702,10 +702,10 @@ describe API::Releases do context 'when tag name is HEAD' do let(:tag_name) { 'HEAD' } - it 'returns an error as failure on tag creation' do + it 'returns a 400 error as failure on tag creation' do post api("/projects/#{project.id}/releases", maintainer), params: params - expect(response).to have_gitlab_http_status(:internal_server_error) + expect(response).to have_gitlab_http_status(:bad_request) expect(json_response['message']).to eq('Tag name invalid') end end @@ -713,10 +713,10 @@ describe API::Releases do context 'when tag name is empty' do let(:tag_name) { '' } - it 'returns an error as failure on tag creation' do + it 'returns a 400 error as failure on tag creation' do post api("/projects/#{project.id}/releases", maintainer), params: params - expect(response).to have_gitlab_http_status(:internal_server_error) + expect(response).to have_gitlab_http_status(:bad_request) expect(json_response['message']).to eq('Tag name invalid') end end diff --git a/spec/services/tags/create_service_spec.rb b/spec/services/tags/create_service_spec.rb index ee558f90d6f..e505960d3c7 100644 --- a/spec/services/tags/create_service_spec.rb +++ b/spec/services/tags/create_service_spec.rb @@ -21,8 +21,9 @@ describe Tags::CreateService do it 'returns an error' do response = service.execute('v1.1.0', 'foo', 'Foo') - expect(response).to eq(status: :error, - message: 'Target foo is invalid') + expect(response[:status]).to eq(:error) + expect(response[:http_status]).to eq(400) + expect(response[:message]).to eq('Target foo is invalid') end end @@ -34,8 +35,19 @@ describe Tags::CreateService do response = service.execute('v1.1.0', 'master', 'Foo') - expect(response).to eq(status: :error, - message: 'Tag v1.1.0 already exists') + expect(response[:status]).to eq(:error) + expect(response[:http_status]).to eq(409) + expect(response[:message]).to eq('Tag v1.1.0 already exists') + end + end + + context 'when tag name is invalid' do + it 'returns an error' do + response = service.execute('HEAD', 'master', 'Foo') + + expect(response[:status]).to eq(:error) + expect(response[:http_status]).to eq(400) + expect(response[:message]).to eq('Tag name invalid') end end @@ -47,8 +59,8 @@ describe Tags::CreateService do response = service.execute('v1.1.0', 'master', 'Foo') - expect(response).to eq(status: :error, - message: 'something went wrong') + expect(response[:status]).to eq(:error) + expect(response[:message]).to eq('something went wrong') end end end |