diff options
52 files changed, 997 insertions, 310 deletions
diff --git a/app/assets/javascripts/jira_import/components/jira_import_app.vue b/app/assets/javascripts/jira_import/components/jira_import_app.vue new file mode 100644 index 00000000000..4b19c4d1b17 --- /dev/null +++ b/app/assets/javascripts/jira_import/components/jira_import_app.vue @@ -0,0 +1,28 @@ +<script> +import getJiraProjects from '../queries/getJiraProjects.query.graphql'; + +export default { + name: 'JiraImportApp', + props: { + projectPath: { + type: String, + required: true, + }, + }, + apollo: { + getJiraImports: { + query: getJiraProjects, + variables() { + return { + fullPath: this.projectPath, + }; + }, + update: data => data.project.jiraImports, + }, + }, +}; +</script> + +<template> + <div></div> +</template> diff --git a/app/assets/javascripts/jira_import/index.js b/app/assets/javascripts/jira_import/index.js new file mode 100644 index 00000000000..a17313fd774 --- /dev/null +++ b/app/assets/javascripts/jira_import/index.js @@ -0,0 +1,31 @@ +import Vue from 'vue'; +import VueApollo from 'vue-apollo'; +import createDefaultClient from '~/lib/graphql'; +import App from './components/jira_import_app.vue'; + +Vue.use(VueApollo); + +const defaultClient = createDefaultClient(); + +const apolloProvider = new VueApollo({ + defaultClient, +}); + +export default function mountJiraImportApp() { + const el = document.querySelector('.js-jira-import-root'); + if (!el) { + return false; + } + + return new Vue({ + el, + apolloProvider, + render(createComponent) { + return createComponent(App, { + props: { + projectPath: el.dataset.projectPath, + }, + }); + }, + }); +} diff --git a/app/assets/javascripts/jira_import/queries/getJiraProjects.query.graphql b/app/assets/javascripts/jira_import/queries/getJiraProjects.query.graphql new file mode 100644 index 00000000000..13100eac221 --- /dev/null +++ b/app/assets/javascripts/jira_import/queries/getJiraProjects.query.graphql @@ -0,0 +1,14 @@ +query getJiraProjects($fullPath: ID!) { + project(fullPath: $fullPath) { + jiraImportStatus + jiraImports { + nodes { + jiraProjectKey + scheduledAt + scheduledBy { + username + } + } + } + } +} diff --git a/app/assets/javascripts/notebook/cells/code.vue b/app/assets/javascripts/notebook/cells/code.vue index 1782e5bfe5a..f5a6f3a9817 100644 --- a/app/assets/javascripts/notebook/cells/code.vue +++ b/app/assets/javascripts/notebook/cells/code.vue @@ -21,11 +21,11 @@ export default { }, computed: { rawInputCode() { - if (this.cell.source) { + if (this.cell.source && Array.isArray(this.cell.source)) { return this.cell.source.join(''); } - return ''; + return this.cell.source || ''; }, hasOutput() { return this.cell.outputs.length; diff --git a/app/assets/javascripts/pages/projects/import/jira/index.js b/app/assets/javascripts/pages/projects/import/jira/index.js new file mode 100644 index 00000000000..cb7a7bde55d --- /dev/null +++ b/app/assets/javascripts/pages/projects/import/jira/index.js @@ -0,0 +1,3 @@ +import mountJiraImportApp from '~/jira_import'; + +document.addEventListener('DOMContentLoaded', mountJiraImportApp); diff --git a/app/assets/javascripts/pages/static_site_editor/index.js b/app/assets/javascripts/pages/projects/static_site_editor/show/index.js index 8f808dae56c..8f808dae56c 100644 --- a/app/assets/javascripts/pages/static_site_editor/index.js +++ b/app/assets/javascripts/pages/projects/static_site_editor/show/index.js diff --git a/app/assets/javascripts/projects/default_project_templates.js b/app/assets/javascripts/projects/default_project_templates.js new file mode 100644 index 00000000000..37bad1efaaf --- /dev/null +++ b/app/assets/javascripts/projects/default_project_templates.js @@ -0,0 +1,84 @@ +import { s__ } from '~/locale'; + +export default { + rails: { + text: s__('ProjectTemplates|Ruby on Rails'), + icon: '.template-option .icon-rails', + }, + express: { + text: s__('ProjectTemplates|NodeJS Express'), + icon: '.template-option .icon-express', + }, + spring: { + text: s__('ProjectTemplates|Spring'), + icon: '.template-option .icon-spring', + }, + iosswift: { + text: s__('ProjectTemplates|iOS (Swift)'), + icon: '.template-option .icon-iosswift', + }, + dotnetcore: { + text: s__('ProjectTemplates|.NET Core'), + icon: '.template-option .icon-dotnetcore', + }, + android: { + text: s__('ProjectTemplates|Android'), + icon: '.template-option .icon-android', + }, + gomicro: { + text: s__('ProjectTemplates|Go Micro'), + icon: '.template-option .icon-gomicro', + }, + gatsby: { + text: s__('ProjectTemplates|Pages/Gatsby'), + icon: '.template-option .icon-gatsby', + }, + hugo: { + text: s__('ProjectTemplates|Pages/Hugo'), + icon: '.template-option .icon-hugo', + }, + jekyll: { + text: s__('ProjectTemplates|Pages/Jekyll'), + icon: '.template-option .icon-jekyll', + }, + plainhtml: { + text: s__('ProjectTemplates|Pages/Plain HTML'), + icon: '.template-option .icon-plainhtml', + }, + gitbook: { + text: s__('ProjectTemplates|Pages/GitBook'), + icon: '.template-option .icon-gitbook', + }, + hexo: { + text: s__('ProjectTemplates|Pages/Hexo'), + icon: '.template-option .icon-hexo', + }, + nfhugo: { + text: s__('ProjectTemplates|Netlify/Hugo'), + icon: '.template-option .icon-nfhugo', + }, + nfjekyll: { + text: s__('ProjectTemplates|Netlify/Jekyll'), + icon: '.template-option .icon-nfjekyll', + }, + nfplainhtml: { + text: s__('ProjectTemplates|Netlify/Plain HTML'), + icon: '.template-option .icon-nfplainhtml', + }, + nfgitbook: { + text: s__('ProjectTemplates|Netlify/GitBook'), + icon: '.template-option .icon-nfgitbook', + }, + nfhexo: { + text: s__('ProjectTemplates|Netlify/Hexo'), + icon: '.template-option .icon-nfhexo', + }, + salesforcedx: { + text: s__('ProjectTemplates|SalesforceDX'), + icon: '.template-option .icon-salesforcedx', + }, + serverless_framework: { + text: s__('ProjectTemplates|Serverless Framework/JS'), + icon: '.template-option .icon-serverless_framework', + }, +}; diff --git a/app/assets/javascripts/projects/project_new.js b/app/assets/javascripts/projects/project_new.js index 9cbda324aff..ebf745fd046 100644 --- a/app/assets/javascripts/projects/project_new.js +++ b/app/assets/javascripts/projects/project_new.js @@ -1,7 +1,7 @@ import $ from 'jquery'; import { addSelectOnFocusBehaviour } from '../lib/utils/common_utils'; import { convertToTitleCase, humanize, slugify } from '../lib/utils/text_utility'; -import { s__ } from '~/locale'; +import DEFAULT_PROJECT_TEMPLATES from 'ee_else_ce/projects/default_project_templates'; let hasUserDefinedProjectPath = false; let hasUserDefinedProjectName = false; @@ -140,90 +140,8 @@ const bindEvents = () => { $projectFieldsForm.addClass('selected'); $selectedIcon.empty(); const value = $(this).val(); - const templates = { - rails: { - text: s__('ProjectTemplates|Ruby on Rails'), - icon: '.template-option .icon-rails', - }, - express: { - text: s__('ProjectTemplates|NodeJS Express'), - icon: '.template-option .icon-express', - }, - spring: { - text: s__('ProjectTemplates|Spring'), - icon: '.template-option .icon-spring', - }, - iosswift: { - text: s__('ProjectTemplates|iOS (Swift)'), - icon: '.template-option .icon-iosswift', - }, - dotnetcore: { - text: s__('ProjectTemplates|.NET Core'), - icon: '.template-option .icon-dotnetcore', - }, - android: { - text: s__('ProjectTemplates|Android'), - icon: '.template-option .icon-android', - }, - gomicro: { - text: s__('ProjectTemplates|Go Micro'), - icon: '.template-option .icon-gomicro', - }, - gatsby: { - text: s__('ProjectTemplates|Pages/Gatsby'), - icon: '.template-option .icon-gatsby', - }, - hugo: { - text: s__('ProjectTemplates|Pages/Hugo'), - icon: '.template-option .icon-hugo', - }, - jekyll: { - text: s__('ProjectTemplates|Pages/Jekyll'), - icon: '.template-option .icon-jekyll', - }, - plainhtml: { - text: s__('ProjectTemplates|Pages/Plain HTML'), - icon: '.template-option .icon-plainhtml', - }, - gitbook: { - text: s__('ProjectTemplates|Pages/GitBook'), - icon: '.template-option .icon-gitbook', - }, - hexo: { - text: s__('ProjectTemplates|Pages/Hexo'), - icon: '.template-option .icon-hexo', - }, - nfhugo: { - text: s__('ProjectTemplates|Netlify/Hugo'), - icon: '.template-option .icon-nfhugo', - }, - nfjekyll: { - text: s__('ProjectTemplates|Netlify/Jekyll'), - icon: '.template-option .icon-nfjekyll', - }, - nfplainhtml: { - text: s__('ProjectTemplates|Netlify/Plain HTML'), - icon: '.template-option .icon-nfplainhtml', - }, - nfgitbook: { - text: s__('ProjectTemplates|Netlify/GitBook'), - icon: '.template-option .icon-nfgitbook', - }, - nfhexo: { - text: s__('ProjectTemplates|Netlify/Hexo'), - icon: '.template-option .icon-nfhexo', - }, - salesforcedx: { - text: s__('ProjectTemplates|SalesforceDX'), - icon: '.template-option .icon-salesforcedx', - }, - serverless_framework: { - text: s__('ProjectTemplates|Serverless Framework/JS'), - icon: '.template-option .icon-serverless_framework', - }, - }; - const selectedTemplate = templates[value]; + const selectedTemplate = DEFAULT_PROJECT_TEMPLATES[value]; $selectedTemplateText.text(selectedTemplate.text); $(selectedTemplate.icon) .clone() diff --git a/app/controllers/projects/import/jira_controller.rb b/app/controllers/projects/import/jira_controller.rb index 6af630a9528..c8f53cef5b2 100644 --- a/app/controllers/projects/import/jira_controller.rb +++ b/app/controllers/projects/import/jira_controller.rb @@ -7,6 +7,8 @@ module Projects before_action :jira_integration_configured? def show + return if Feature.enabled?(:jira_issue_import_vue, @project) + unless @project.import_state&.in_progress? jira_client = @project.jira_service.client jira_projects = jira_client.Project.all diff --git a/app/controllers/projects/prometheus/metrics_controller.rb b/app/controllers/projects/prometheus/metrics_controller.rb index c9c7ba1253f..0340cb5beb0 100644 --- a/app/controllers/projects/prometheus/metrics_controller.rb +++ b/app/controllers/projects/prometheus/metrics_controller.rb @@ -20,6 +20,85 @@ module Projects end end + def validate_query + respond_to do |format| + format.json do + result = prometheus_adapter.query(:validate, params[:query]) + + if result + render json: result + else + head :accepted + end + end + end + end + + def new + @metric = project.prometheus_metrics.new + end + + def index + respond_to do |format| + format.json do + metrics = ::PrometheusMetricsFinder.new( + project: project, + ordered: true + ).execute.to_a + + response = {} + if metrics.any? + response[:metrics] = ::PrometheusMetricSerializer + .new(project: project) + .represent(metrics) + end + + render json: response + end + end + end + + def create + @metric = project.prometheus_metrics.create( + metrics_params.to_h.symbolize_keys + ) + + if @metric.persisted? + redirect_to edit_project_service_path(project, ::PrometheusService), + notice: _('Metric was successfully added.') + else + render 'new' + end + end + + def update + @metric = update_metrics_service(prometheus_metric).execute + + if @metric.persisted? + redirect_to edit_project_service_path(project, ::PrometheusService), + notice: _('Metric was successfully updated.') + else + render 'edit' + end + end + + def edit + @metric = prometheus_metric + end + + def destroy + destroy_metrics_service(prometheus_metric).execute + + respond_to do |format| + format.html do + redirect_to edit_project_service_path(project, ::PrometheusService), status: :see_other + end + format.json do + head :ok + end + end + end + private def prometheus_adapter @@ -29,8 +108,22 @@ module Projects def require_prometheus_metrics! render_404 unless prometheus_adapter&.can_query? end + + def prometheus_metric + @prometheus_metric ||= ::PrometheusMetricsFinder.new(id: params[:id]).execute.first + end + + def update_metrics_service(metric) + ::Projects::Prometheus::Metrics::UpdateService.new(metric, metrics_params) + end + + def destroy_metrics_service(metric) + ::Projects::Prometheus::Metrics::DestroyService.new(metric) + end + + def metrics_params + params.require(:prometheus_metric).permit(:title, :query, :y_label, :unit, :legend, :group) + end end end end - -Projects::Prometheus::MetricsController.prepend_if_ee('EE::Projects::Prometheus::MetricsController') diff --git a/app/controllers/projects/registry/repositories_controller.rb b/app/controllers/projects/registry/repositories_controller.rb index 8852ae04d5e..2418ea97409 100644 --- a/app/controllers/projects/registry/repositories_controller.rb +++ b/app/controllers/projects/registry/repositories_controller.rb @@ -28,6 +28,7 @@ module Projects end def destroy + image.delete_scheduled! DeleteContainerRepositoryWorker.perform_async(current_user.id, image.id) # rubocop:disable CodeReuse/Worker track_event(:delete_repository) diff --git a/app/helpers/custom_metrics_helper.rb b/app/helpers/custom_metrics_helper.rb new file mode 100644 index 00000000000..fbea6d2050f --- /dev/null +++ b/app/helpers/custom_metrics_helper.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module CustomMetricsHelper + def custom_metrics_data(project, metric) + custom_metrics_path = project.namespace.becomes(::Namespace) + + { + 'custom-metrics-path' => url_for([custom_metrics_path, project, metric]), + 'metric-persisted' => metric.persisted?.to_s, + 'edit-project-service-path' => edit_project_service_path(project, PrometheusService), + 'validate-query-path' => validate_query_project_prometheus_metrics_path(project), + 'title' => metric.title.to_s, + 'query' => metric.query.to_s, + 'y-label' => metric.y_label.to_s, + 'unit' => metric.unit.to_s, + 'group' => metric.group.to_s, + 'legend' => metric.legend.to_s + } + end +end diff --git a/app/helpers/environments_helper.rb b/app/helpers/environments_helper.rb index fcf99644e66..5b640ea6538 100644 --- a/app/helpers/environments_helper.rb +++ b/app/helpers/environments_helper.rb @@ -18,6 +18,10 @@ module EnvironmentsHelper } end + def custom_metrics_available?(project) + can?(current_user, :admin_project, project) + end + def metrics_data(project, environment) { "settings-path" => edit_project_service_path(project, 'prometheus'), @@ -39,7 +43,10 @@ module EnvironmentsHelper "has-metrics" => "#{environment.has_metrics?}", "prometheus-status" => "#{environment.prometheus_status}", "external-dashboard-url" => project.metrics_setting_external_dashboard_url, - "environment-state" => "#{environment.state}" + "environment-state" => "#{environment.state}", + "custom-metrics-path" => project_prometheus_metrics_path(project), + "validate-query-path" => validate_query_project_prometheus_metrics_path(project), + "custom-metrics-available" => "#{custom_metrics_available?(project)}" } end diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index b74c044b687..3bff7cb06c1 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -8,6 +8,8 @@ class ContainerRepository < ApplicationRecord validates :name, length: { minimum: 0, allow_nil: false } validates :name, uniqueness: { scope: :project_id } + enum status: { delete_scheduled: 0, delete_failed: 1 } + delegate :client, to: :registry scope :ordered, -> { order(:name) } diff --git a/app/models/project_services/prometheus_service.rb b/app/models/project_services/prometheus_service.rb index fd4ee069041..17bc29ebdcf 100644 --- a/app/models/project_services/prometheus_service.rb +++ b/app/models/project_services/prometheus_service.rb @@ -13,9 +13,9 @@ class PrometheusService < MonitoringService # to allow localhost URLs when the following conditions are true: # 1. project is the self-monitoring project. # 2. api_url is the internal Prometheus URL. - with_options presence: true, if: :manual_configuration? do - validates :api_url, public_url: true, unless: proc { |object| object.allow_local_api_url? } - validates :api_url, url: true, if: proc { |object| object.allow_local_api_url? } + with_options presence: true do + validates :api_url, public_url: true, if: ->(object) { object.manual_configuration? && !object.allow_local_api_url? } + validates :api_url, url: true, if: ->(object) { object.manual_configuration? && object.allow_local_api_url? } end before_save :synchronize_service_state diff --git a/app/serializers/container_repository_entity.rb b/app/serializers/container_repository_entity.rb index db9cf1c7835..46aa0adc5a0 100644 --- a/app/serializers/container_repository_entity.rb +++ b/app/serializers/container_repository_entity.rb @@ -3,7 +3,7 @@ class ContainerRepositoryEntity < Grape::Entity include RequestAwareEntity - expose :id, :name, :path, :location, :created_at + expose :id, :name, :path, :location, :created_at, :status expose :tags_path do |repository| project_registry_repository_tags_path(project, repository, format: :json) diff --git a/app/serializers/prometheus_metric_entity.rb b/app/serializers/prometheus_metric_entity.rb new file mode 100644 index 00000000000..ab63f3f401e --- /dev/null +++ b/app/serializers/prometheus_metric_entity.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class PrometheusMetricEntity < Grape::Entity + include RequestAwareEntity + + expose :id + expose :title + + expose :group + expose :group_title + expose :unit + + expose :edit_path do |prometheus_metric| + edit_project_prometheus_metric_path(prometheus_metric.project, prometheus_metric) + end +end diff --git a/app/serializers/prometheus_metric_serializer.rb b/app/serializers/prometheus_metric_serializer.rb new file mode 100644 index 00000000000..886a6ba852f --- /dev/null +++ b/app/serializers/prometheus_metric_serializer.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +class PrometheusMetricSerializer < BaseSerializer + entity PrometheusMetricEntity +end diff --git a/app/services/projects/container_repository/destroy_service.rb b/app/services/projects/container_repository/destroy_service.rb index 1f5af7970d6..83bb8624bba 100644 --- a/app/services/projects/container_repository/destroy_service.rb +++ b/app/services/projects/container_repository/destroy_service.rb @@ -8,7 +8,7 @@ module Projects # Delete tags outside of the transaction to avoid hitting an idle-in-transaction timeout container_repository.delete_tags! - container_repository.destroy + container_repository.delete_failed! unless container_repository.destroy end end end diff --git a/app/views/projects/import/jira/show.html.haml b/app/views/projects/import/jira/show.html.haml index c734297d7f3..cfc4baa1c25 100644 --- a/app/views/projects/import/jira/show.html.haml +++ b/app/views/projects/import/jira/show.html.haml @@ -1,24 +1,27 @@ -- title = _('Jira Issue Import') -- page_title title -- breadcrumb_title title -- header_title _("Projects"), root_path +- if Feature.enabled?(:jira_issue_import_vue, @project) + .js-jira-import-root{ data: { project_path: @project.full_path } } +- else + - title = _('Jira Issue Import') + - page_title title + - breadcrumb_title title + - header_title _("Projects"), root_path -= render 'import/shared/errors' + = render 'import/shared/errors' -- if @project.import_state&.in_progress? - %h3.page-title.d-flex.align-items-center - = sprite_icon('issues', size: 16, css_class: 'mr-1') - = _('Import in progress') -- elsif @jira_projects.present? - %h3.page-title.d-flex.align-items-center - = sprite_icon('issues', size: 16, css_class: 'mr-1') - = _('Import issues from Jira') + - if @project.import_state&.in_progress? + %h3.page-title.d-flex.align-items-center + = sprite_icon('issues', size: 16, css_class: 'mr-1') + = _('Import in progress') + - elsif @jira_projects.present? + %h3.page-title.d-flex.align-items-center + = sprite_icon('issues', size: 16, css_class: 'mr-1') + = _('Import issues from Jira') - = form_tag import_project_import_jira_path(@project), method: :post do - .form-group.row - = label_tag :jira_project_key, _('From project'), class: 'col-form-label col-md-2' - .col-md-4 - = select_tag :jira_project_key, options_for_select(@jira_projects, ''), { class: 'select2' } - .form-actions - = submit_tag _('Import issues'), class: 'btn btn-success' - = link_to _('Cancel'), project_issues_path(@project), class: 'btn btn-cancel' + = form_tag import_project_import_jira_path(@project), method: :post do + .form-group.row + = label_tag :jira_project_key, _('From project'), class: 'col-form-label col-md-2' + .col-md-4 + = select_tag :jira_project_key, options_for_select(@jira_projects, ''), { class: 'select2' } + .form-actions + = submit_tag _('Import issues'), class: 'btn btn-success' + = link_to _('Cancel'), project_issues_path(@project), class: 'btn btn-cancel' diff --git a/app/views/projects/merge_requests/_merge_request.html.haml b/app/views/projects/merge_requests/_merge_request.html.haml index 1bde1a41975..f7f5388a54a 100644 --- a/app/views/projects/merge_requests/_merge_request.html.haml +++ b/app/views/projects/merge_requests/_merge_request.html.haml @@ -47,9 +47,7 @@ %li.issuable-status.d-none.d-sm-inline-block = icon('ban') = _('CLOSED') - - if can?(current_user, :read_pipeline, merge_request.head_pipeline) - %li.issuable-pipeline-status.d-none.d-sm-flex - = render 'ci/status/icon', status: merge_request.head_pipeline.detailed_status(current_user), option_css_classes: 'd-flex' + = render 'shared/merge_request_pipeline_status', merge_request: merge_request - if merge_request.open? && merge_request.broken? %li.issuable-pipeline-broken.d-none.d-sm-flex = link_to merge_request_path(merge_request), class: "has-tooltip", title: _('Cannot be merged automatically') do diff --git a/app/views/projects/prometheus/metrics/_form.html.haml b/app/views/projects/prometheus/metrics/_form.html.haml new file mode 100644 index 00000000000..a87d81e6325 --- /dev/null +++ b/app/views/projects/prometheus/metrics/_form.html.haml @@ -0,0 +1,4 @@ +- project = local_assigns.fetch(:project) +- metric = local_assigns.fetch(:metric) + +#js-custom-metrics{ data: custom_metrics_data(project, metric) } diff --git a/app/views/projects/prometheus/metrics/edit.html.haml b/app/views/projects/prometheus/metrics/edit.html.haml new file mode 100644 index 00000000000..15a9c922ca6 --- /dev/null +++ b/app/views/projects/prometheus/metrics/edit.html.haml @@ -0,0 +1,6 @@ +- add_to_breadcrumbs _("Settings"), edit_project_path(@project) +- add_to_breadcrumbs _("Integrations"), project_settings_integrations_path(@project) +- add_to_breadcrumbs "Prometheus", edit_project_service_path(@project, PrometheusService) +- breadcrumb_title s_('Metrics|Edit metric') +- page_title @metric.title, s_('Metrics|Edit metric') += render 'form', project: @project, metric: @metric diff --git a/app/views/projects/prometheus/metrics/new.html.haml b/app/views/projects/prometheus/metrics/new.html.haml new file mode 100644 index 00000000000..fa925d090cb --- /dev/null +++ b/app/views/projects/prometheus/metrics/new.html.haml @@ -0,0 +1,6 @@ +- add_to_breadcrumbs _("Settings"), edit_project_path(@project) +- add_to_breadcrumbs _("Integrations"), project_settings_integrations_path(@project) +- add_to_breadcrumbs "Prometheus", edit_project_service_path(@project, PrometheusService) +- breadcrumb_title s_('Metrics|New metric') +- page_title s_('Metrics|New metric') += render 'form', project: @project, metric: @metric diff --git a/app/views/shared/_merge_request_pipeline_status.html.haml b/app/views/shared/_merge_request_pipeline_status.html.haml new file mode 100644 index 00000000000..1c15b82367e --- /dev/null +++ b/app/views/shared/_merge_request_pipeline_status.html.haml @@ -0,0 +1,3 @@ +- if can?(current_user, :read_pipeline, merge_request.head_pipeline) + %li.issuable-pipeline-status.d-none.d-sm-flex + = render 'ci/status/icon', status: merge_request.head_pipeline.detailed_status(current_user), option_css_classes: 'd-flex' diff --git a/changelogs/unreleased/208429-add-delete-status.yml b/changelogs/unreleased/208429-add-delete-status.yml new file mode 100644 index 00000000000..e4b27049b6b --- /dev/null +++ b/changelogs/unreleased/208429-add-delete-status.yml @@ -0,0 +1,5 @@ +--- +title: Add status column to container_registry +merge_request: 28682 +author: +type: changed diff --git a/changelogs/unreleased/27374-jupyter-notebook-cellsource.yml b/changelogs/unreleased/27374-jupyter-notebook-cellsource.yml new file mode 100644 index 00000000000..0897800477c --- /dev/null +++ b/changelogs/unreleased/27374-jupyter-notebook-cellsource.yml @@ -0,0 +1,5 @@ +--- +title: Fix display of PyCharm generated Jupyter notebooks +merge_request: 28810 +author: Jan Beckmann +type: fixed diff --git a/config/routes/project.rb b/config/routes/project.rb index 51084cb0e7e..943ff24ffee 100644 --- a/config/routes/project.rb +++ b/config/routes/project.rb @@ -356,6 +356,7 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do resources :metrics, constraints: { id: %r{[^\/]+} }, only: [:index, :new, :create, :edit, :update, :destroy] do get :active_common, on: :collection + post :validate_query, on: :collection end end diff --git a/db/migrate/20200402135250_add_delete_status_to_container_repository.rb b/db/migrate/20200402135250_add_delete_status_to_container_repository.rb new file mode 100644 index 00000000000..a0dbb307bfc --- /dev/null +++ b/db/migrate/20200402135250_add_delete_status_to_container_repository.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class AddDeleteStatusToContainerRepository < ActiveRecord::Migration[6.0] + DOWNTIME = false + + def up + add_column(:container_repositories, :status, :integer, limit: 2) + end + + def down + remove_column(:container_repositories, :status) + end +end diff --git a/db/structure.sql b/db/structure.sql index ef3319df364..2697dbbe2b1 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -1850,7 +1850,8 @@ CREATE TABLE public.container_repositories ( project_id integer NOT NULL, name character varying NOT NULL, created_at timestamp without time zone NOT NULL, - updated_at timestamp without time zone NOT NULL + updated_at timestamp without time zone NOT NULL, + status smallint ); CREATE SEQUENCE public.container_repositories_id_seq @@ -12935,6 +12936,7 @@ COPY "schema_migrations" (version) FROM STDIN; 20200330123739 20200330132913 20200331220930 +20200402135250 20200403184110 20200403185127 20200403185422 diff --git a/doc/administration/audit_events.md b/doc/administration/audit_events.md index aa70890d3cd..26b4434de77 100644 --- a/doc/administration/audit_events.md +++ b/doc/administration/audit_events.md @@ -41,6 +41,7 @@ From there, you can see the following actions: - Group created or deleted - Group changed visibility - User was added to group and with which [permissions] +- User sign-in via [Group SAML](../user/group/saml_sso/index.md) - Permissions changes of a user assigned to a group - Removed user from group - Project added to group and with which visibility level diff --git a/doc/administration/high_availability/README.md b/doc/administration/high_availability/README.md index b5258a66e59..4734df324e0 100644 --- a/doc/administration/high_availability/README.md +++ b/doc/administration/high_availability/README.md @@ -2,18 +2,15 @@ type: reference, concepts --- -# Scaling and High Availability +# High Availability -GitLab supports a number of scaling options to ensure that your self-managed -instance is able to scale out to meet your organization's needs when scaling up -is no longer practical or feasible. - -GitLab also offers high availability options for organizations that require +GitLab offers high availability options for organizations that require the fault tolerance and redundancy necessary to maintain high-uptime operations. -Scaling and high availability can be tackled separately as GitLab comprises -modular components which can be individually scaled or made highly available -depending on your organization's needs and resources. +Please consult our [scaling documentation](../scaling) if you want to resolve +performance bottlenecks you encounter in individual GitLab components without +incurring the additional complexity costs associated with maintaining a +highly-available architecture. On this page, we present examples of self-managed instances which demonstrate how GitLab can be scaled out and made highly available. These examples progress @@ -29,39 +26,7 @@ watch [this 1 hour Q&A](https://www.youtube.com/watch?v=uCU8jdYzpac) with [John Northrup](https://gitlab.com/northrup), and live questions coming in from some of our customers. -## Scaling examples - -### Single-node Omnibus installation - -This solution is appropriate for many teams that have a single server at their disposal. With automatic backup of the GitLab repositories, configuration, and the database, this can be an optimal solution if you don't have strict availability requirements. - -You can also optionally configure GitLab to use an [external PostgreSQL service](../external_database.md) -or an [external object storage service](object_storage.md) for added -performance and reliability at a relatively low complexity cost. - -References: - -- [Installation Docs](../../install/README.md) -- [Backup/Restore Docs](https://docs.gitlab.com/omnibus/settings/backups.html#backup-and-restore-omnibus-gitlab-configuration) - -### Omnibus installation with multiple application servers - -This solution is appropriate for teams that are starting to scale out when -scaling up is no longer meeting their needs. In this configuration, additional application nodes will handle frontend traffic, with a load balancer in front to distribute traffic across those nodes. Meanwhile, each application node connects to a shared file server and PostgreSQL and Redis services on the back end. - -The additional application servers adds limited fault tolerance to your GitLab -instance. As long as one application node is online and capable of handling the -instance's usage load, your team's productivity will not be interrupted. Having -multiple application nodes also enables [zero-downtime updates](https://docs.gitlab.com/omnibus/update/#zero-downtime-updates). - -References: - -- [Configure your load balancer for GitLab](load_balancer.md) -- [Configure your NFS server to work with GitLab](nfs.md) -- [Configure packaged PostgreSQL server to listen on TCP/IP](https://docs.gitlab.com/omnibus/settings/database.html#configure-packaged-postgresql-server-to-listen-on-tcpip) -- [Setting up a Redis-only server](https://docs.gitlab.com/omnibus/settings/redis.html#setting-up-a-redis-only-server) - -## High-availability examples +## Examples ### Omnibus installation with automatic database failover diff --git a/doc/administration/scaling/index.md b/doc/administration/scaling/index.md new file mode 100644 index 00000000000..99e8ca9a65f --- /dev/null +++ b/doc/administration/scaling/index.md @@ -0,0 +1,73 @@ +--- +type: reference, concepts +--- + +# Scaling + +GitLab supports a number of scaling options to ensure that your self-managed +instance is able to scale out to meet your organization's needs when scaling up +a single-box GitLab installation is no longer practical or feasible. + +Please consult our [high availability documentation](../high_availability/README.md) +if your organization requires fault tolerance and redundancy features, such as +automatic database system failover. + +## GitLab components and scaling instructions + +Here's a list of components directly provided by Omnibus GitLab or installed as +part of a source installation and their configuration instructions for scaling. + +| Component | Description | Configuration instructions | +|-----------|-------------|----------------------------| +| [PostgreSQL](../../development/architecture.md#postgresql) | Database | [PostgreSQL configuration](https://docs.gitlab.com/omnibus/settings/database.html) | +| [Redis](../../development/architecture.md#redis) | Key/value store for fast data lookup and caching | [Redis configuration](../high_availability/redis.md) | +| [GitLab application services](../../development/architecture.md#unicorn) | Unicorn/Puma, Workhorse, GitLab Shell - serves front-end requests requests (UI, API, Git over HTTP/SSH) | [GitLab app scaling configuration](../high_availability/gitlab.md) | +| [PgBouncer](../../development/architecture.md#pgbouncer) | Database connection pooler | [PgBouncer configuration](../high_availability/pgbouncer.md#running-pgbouncer-as-part-of-a-non-ha-gitlab-installation) **(PREMIUM ONLY)** | +| [Sidekiq](../../development/architecture.md#sidekiq) | Asynchronous/background jobs | [Sidekiq configuration](../high_availability/sidekiq.md) | +| [Gitaly](../../development/architecture.md#gitaly) | Provides access to Git repositories | [Gitaly configuration](../gitaly/index.md#running-gitaly-on-its-own-server) | +| [Prometheus](../../development/architecture.md#prometheus) and [Grafana](../../development/architecture.md#grafana) | GitLab environment monitoring | [Monitoring node for scaling](../high_availability/monitoring_node.md) | + +## Third-party services used for scaling + +Here's a list of third-party services you may require as part of scaling GitLab. +The services can be provided by numerous applications or vendors and further +advice is given on how best to select the right choice for your organization's +needs. + +| Component | Description | Configuration instructions | +|-----------|-------------|----------------------------| +| Load balancer(s) | Handles load balancing, typically when you have multiple GitLab application services nodes | [Load balancer configuration](../high_availability/load_balancer.md) | +| Object storage service | Recommended store for shared data objects | [Cloud Object Storage configuration](../high_availability/object_storage.md) | +| NFS | Shared disk storage service. Can be used as an alternative for Gitaly or Object Storage. Required for GitLab Pages | [NFS configuration](../high_availability/nfs.md) | + +## Examples + +### Single-node Omnibus installation + +This solution is appropriate for many teams that have a single server at their disposal. With automatic backup of the GitLab repositories, configuration, and the database, this can be an optimal solution if you don't have strict availability requirements. + +You can also optionally configure GitLab to use an [external PostgreSQL service](../external_database.md) +or an [external object storage service](../high_availability/object_storage.md) for added +performance and reliability at a relatively low complexity cost. + +References: + +- [Installation Docs](../../install/README.md) +- [Backup/Restore Docs](https://docs.gitlab.com/omnibus/settings/backups.html#backup-and-restore-omnibus-gitlab-configuration) + +### Omnibus installation with multiple application servers + +This solution is appropriate for teams that are starting to scale out when +scaling up is no longer meeting their needs. In this configuration, additional application nodes will handle frontend traffic, with a load balancer in front to distribute traffic across those nodes. Meanwhile, each application node connects to a shared file server and PostgreSQL and Redis services on the back end. + +The additional application servers adds limited fault tolerance to your GitLab +instance. As long as one application node is online and capable of handling the +instance's usage load, your team's productivity will not be interrupted. Having +multiple application nodes also enables [zero-downtime updates](https://docs.gitlab.com/omnibus/update/#zero-downtime-updates). + +References: + +- [Configure your load balancer for GitLab](../high_availability/load_balancer.md) +- [Configure your NFS server to work with GitLab](../high_availability/nfs.md) +- [Configure packaged PostgreSQL server to listen on TCP/IP](https://docs.gitlab.com/omnibus/settings/database.html#configure-packaged-postgresql-server-to-listen-on-tcpip) +- [Setting up a Redis-only server](https://docs.gitlab.com/omnibus/settings/redis.html#setting-up-a-redis-only-server) diff --git a/doc/administration/troubleshooting/ssl.md b/doc/administration/troubleshooting/ssl.md index 475b7d44eac..b66b6e8c90a 100644 --- a/doc/administration/troubleshooting/ssl.md +++ b/doc/administration/troubleshooting/ssl.md @@ -137,3 +137,36 @@ To fix this problem: ```shell git config --global http.sslVerify false ``` + +## SSL_connect wrong version number + +A misconfiguration may result in: + +- `gitlab-rails/exceptions_json.log` entries containing: + + ```plaintext + "exception.class":"Excon::Error::Socket","exception.message":"SSL_connect returned=1 errno=0 state=error: wrong version number (OpenSSL::SSL::SSLError)", + "exception.class":"Excon::Error::Socket","exception.message":"SSL_connect returned=1 errno=0 state=error: wrong version number (OpenSSL::SSL::SSLError)", + ``` + +- `gitlab-workhorse/current` containing: + + ```plaintext + http: server gave HTTP response to HTTPS client + http: server gave HTTP response to HTTPS client + ``` + +- `gitlab-rails/sidekiq.log` or `sidekiq/current` containing: + + ```plaintext + message: SSL_connect returned=1 errno=0 state=error: wrong version number (OpenSSL::SSL::SSLError) + message: SSL_connect returned=1 errno=0 state=error: wrong version number (OpenSSL::SSL::SSLError) + ``` + +Some of these errors come from the Excon Ruby gem, and could be generated in circumstances +where GitLab is configured to initiate an HTTPS session to a remote server +that is serving just HTTP. + +One scenario is that you're using [object storage](../high_availability/object_storage.md) +which is not served under HTTPS. GitLab is misconfigured and attempts a TLS handshake, +but the object storage will respond with plain HTTP. diff --git a/doc/development/documentation/styleguide.md b/doc/development/documentation/styleguide.md index 370effc940c..596ba01f5e0 100644 --- a/doc/development/documentation/styleguide.md +++ b/doc/development/documentation/styleguide.md @@ -1233,6 +1233,14 @@ a helpful link back to how the feature was developed. > - Enabled by default in GitLab 11.4. ``` +- If a feature is moved to another tier: + + ```md + > - [Introduced](<link-to-issue>) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.5. + > - [Moved](<link-to-issue>) to [GitLab Starter](https://about.gitlab.com/pricing/) in 11.8. + > - [Moved](<link-to-issue>) to GitLab Core in 12.0. + ``` + NOTE: **Note:** Version text must be on its own line and surounded by blank lines to render correctly. diff --git a/doc/install/aws/index.md b/doc/install/aws/index.md index c2b1198940b..128d0746df2 100644 --- a/doc/install/aws/index.md +++ b/doc/install/aws/index.md @@ -381,7 +381,103 @@ EC2 instances running Linux use private key files for SSH authentication. You'll Storing private key files on your bastion host is a bad idea. To get around this, use SSH agent forwarding on your client. See [Securely Connect to Linux Instances Running in a Private Amazon VPC](https://aws.amazon.com/blogs/security/securely-connect-to-linux-instances-running-in-a-private-amazon-vpc/) for a step-by-step guide on how to use SSH agent forwarding. -## Setting up Gitaly +## Install GitLab and create custom AMI + +We will need a preconfigured, custom GitLab AMI to use in our launch configuration later. As a starting point, we will use the official GitLab AMI to create a GitLab instance. Then, we'll add our custom configuration for PostgreSQL, Redis, and Gitaly. If you prefer, instead of using the official GitLab AMI, you can also spin up an EC2 instance of your choosing and [manually install GitLab](https://about.gitlab.com/install/). + +### Install GitLab + +From the EC2 dashboard: + +1. Click **Launch Instance** and select **Community AMIs** from the left menu. +1. In the search bar, search for `GitLab EE <version>` where `<version>` is the latest version as seen on the [releases page](https://about.gitlab.com/releases/). Select the latest patch release, for example `GitLab EE 12.9.2`. +1. Select an instance type based on your workload. Consult the [hardware requirements](../../install/requirements.md#hardware-requirements) to choose one that fits your needs (at least `c5.xlarge`, which is sufficient to accommodate 100 users). +1. Click **Configure Instance Details**: + 1. In the **Network** dropdown, select `gitlab-vpc`, the VPC we created earlier. + 1. In the **Subnet** dropdown, `select gitlab-private-10.0.1.0` from the list of subnets we created earlier. + 1. Double check that **Auto-assign Public IP** is set to `Use subnet setting (Disable)`. + 1. Click **Add Storage**. + 1. The root volume is 8GiB by default and should be enough given that we won’t store any data there. +1. Click **Add Tags** and add any tags you may need. In our case, we'll only set `Key: Name` and `Value: GitLab`. +1. Click **Configure Security Group**. Check **Select an existing security group** and select the `gitlab-loadbalancer-sec-group` we created earlier. +1. Click **Review and launch** followed by **Launch** if you’re happy with your settings. +1. Finally, acknowledge that you have access to the selected private key file or create a new one. Click **Launch Instances**. + +### Add custom configuration + +Connect to your GitLab instance via **Bastion Host A** using [SSH Agent Forwarding](#use-ssh-agent-forwarding). Once connected, add the following custom configuration: + +#### Install the `pg_trgm` extension for PostgreSQL + +From your GitLab instance, connect to the RDS instance to verify access and to install the required `pg_trgm` extension. + +To find the host or endpoint, navigate to **Amazon RDS > Databases** and click on the database you created earlier. Look for the endpoint under the **Connectivity & security** tab. + +Do not to include the colon and port number: + +```shell +sudo /opt/gitlab/embedded/bin/psql -U gitlab -h <rds-endpoint> -d gitlabhq_production +``` + +At the `psql` prompt create the extension and then quit the session: + +```shell +psql (10.9) +Type "help" for help. + +gitlab=# CREATE EXTENSION pg_trgm; +gitlab=# \q +``` + +#### Configure GitLab to connect to PostgreSQL and Redis + +1. Edit `/etc/gitlab/gitlab.rb`, find the `external_url 'http://<domain>'` option + and change it to the `https` domain you will be using. + +1. Look for the GitLab database settings and uncomment as necessary. In + our current case we'll specify the database adapter, encoding, host, name, + username, and password: + + ```ruby + # Disable the built-in Postgres + postgresql['enable'] = false + + # Fill in the connection details + gitlab_rails['db_adapter'] = "postgresql" + gitlab_rails['db_encoding'] = "unicode" + gitlab_rails['db_database'] = "gitlabhq_production" + gitlab_rails['db_username'] = "gitlab" + gitlab_rails['db_password'] = "mypassword" + gitlab_rails['db_host'] = "<rds-endpoint>" + ``` + +1. Next, we need to configure the Redis section by adding the host and + uncommenting the port: + + ```ruby + # Disable the built-in Redis + redis['enable'] = false + + # Fill in the connection details + gitlab_rails['redis_host'] = "<redis-endpoint>" + gitlab_rails['redis_port'] = 6379 + ``` + +1. Finally, reconfigure GitLab for the changes to take effect: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +1. You might also find it useful to run a check and a service status to make sure + everything has been setup correctly: + + ```shell + sudo gitlab-rake gitlab:check + sudo gitlab-ctl status + ``` + +#### Set up Gitaly CAUTION: **Caution:** In this architecture, having a single Gitaly server creates a single point of failure. This limitation will be removed once [Gitaly HA](https://gitlab.com/groups/gitlab-org/-/epics/842) is released. @@ -410,7 +506,79 @@ Let's create an EC2 instance where we'll install Gitaly: > **Optional:** Instead of storing configuration _and_ repository data on the root volume, you can also choose to add an additional EBS volume for repository storage. Follow the same guidance as above. See the [Amazon EBS pricing](https://aws.amazon.com/ebs/pricing/). -Now that we have our EC2 instance ready, follow the [documentation to install GitLab and set up Gitaly on its own server](../../administration/gitaly/index.md#running-gitaly-on-its-own-server). +Now that we have our EC2 instance ready, follow the [documentation to install GitLab and set up Gitaly on its own server](../../administration/gitaly/index.md#running-gitaly-on-its-own-server). Perform the client setup steps from that document on the [GitLab instance we created](#install-gitlab) above. + +#### Add Support for Proxied SSL + +As we are terminating SSL at our [load balancer](#load-balancer), follow the steps at [Supporting proxied SSL](https://docs.gitlab.com/omnibus/settings/nginx.html#supporting-proxied-ssl) to configure this in `/etc/gitlab/gitlab.rb`. + +Remember to run `sudo gitlab-ctl reconfigure` after saving the changes to the `gitlab.rb` file. + +#### Disable Let's Encrypt + +Since we're adding our SSL certificate at the load balancer, we do not need GitLab's built-in support for Let's Encrypt. Let's Encrypt [is enabled by default](https://docs.gitlab.com/omnibus/settings/ssl.html#lets-encrypt-integration) when using an `https` domain since GitLab 10.7, so we need to explicitly disable it: + +1. Open `/etc/gitlab/gitlab.rb` and disable it: + + ```ruby + letsencrypt['enable'] = false + ``` + +1. Save the file and reconfigure for the changes to take effect: + + ```shell + sudo gitlab-ctl reconfigure + ``` + +#### Configure host keys + +Ordinarily we would manually copy the contents (primary and public keys) of `/etc/ssh/` on the primary application server to `/etc/ssh` on all secondary servers. This prevents false man-in-the-middle-attack alerts when accessing servers in your High Availability cluster behind a load balancer. + +We'll automate this by creating static host keys as part of our custom AMI. As these host keys are also rotated every time an EC2 instance boots up, "hard coding" them into our custom AMI serves as a handy workaround. + +On your GitLab instance run the following: + +```shell +mkdir /etc/ssh_static +cp -R /etc/ssh/* /etc/ssh_static +``` + +In `/etc/ssh/sshd_config` update the following: + +```bash + # HostKeys for protocol version 2 + HostKey /etc/ssh_static/ssh_host_rsa_key + HostKey /etc/ssh_static/ssh_host_dsa_key + HostKey /etc/ssh_static/ssh_host_ecdsa_key + HosstKey /etc/ssh_static/ssh_host_ed25519_key +``` + +#### Amazon S3 object storage + +Since we're not using NFS for shared storage, we will use [Amazon S3](https://aws.amazon.com/s3/) buckets to store backups, artifacts, LFS objects, uploads, merge request diffs, container registry images, and more. For instructions on how to configure each of these, please see [Cloud Object Storage](../../administration/high_availability/object_storage.md). + +Remember to run `sudo gitlab-ctl reconfigure` after saving the changes to the `gitlab.rb` file. + +NOTE: **Note:** +One current feature of GitLab that still requires a shared directory (NFS) is +[GitLab Pages](../../user/project/pages/index.md). +There is [work in progress](https://gitlab.com/gitlab-org/gitlab-pages/issues/196) +to eliminate the need for NFS to support GitLab Pages. + +--- + +That concludes the configuration changes for our GitLab instance. Next, we'll create a custom AMI based on this instance to use for our launch configuration and auto scaling group. + +### Create custom AMI + +On the EC2 dashboard: + +1. Select the `GitLab` instance we [created earlier](#install-gitLab). +1. Click on **Actions**, scroll down to **Image** and click **Create Image**. +1. Give your image a name and description (we'll use `GitLab-Source` for both). +1. Leave everything else as default and click **Create Image** + +Now we have a custom AMI that we'll use to create our launch configuration the the next step. ## Deploying GitLab inside an auto scaling group @@ -497,129 +665,6 @@ You'll notice that after we save the configuration, AWS starts launching our two instances in different AZs and without a public IP which is exactly what we intended. -## After deployment - -After a few minutes, the instances should be up and accessible via the internet. -Let's connect to the primary and configure some things before logging in. - -### Installing the `pg_trgm` extension for PostgreSQL - -Connect to the RDS instance to verify access and to install the required `pg_trgm` extension. - -To find the host or endpoint, naviagate to **Amazon RDS > Databases** and click on the database you created earlier. Look for the endpoint under the **Connectivity & security** tab. - -Do not to include the colon and port number: - -```shell -sudo /opt/gitlab/embedded/bin/psql -U gitlab -h <rds-endpoint> -d gitlabhq_production -``` - -At the psql prompt create the extension and then quit the session: - -```shell -psql (10.9) -Type "help" for help. - -gitlab=# CREATE EXTENSION pg_trgm; -gitlab=# \q -``` - ---- - -### Configuring GitLab to connect with PostgreSQL and Redis - -Edit the `gitlab.rb` file at `/etc/gitlab/gitlab.rb` -find the `external_url 'http://gitlab.example.com'` option and change it -to the domain you will be using or the public IP address of the current -instance to test the configuration. - -For a more detailed description about configuring GitLab, see [Configuring GitLab for HA](../../administration/high_availability/gitlab.md) - -Now look for the GitLab database settings and uncomment as necessary. In -our current case we'll specify the database adapter, encoding, host, name, -username, and password: - -```ruby -# Disable the built-in Postgres -postgresql['enable'] = false - -# Fill in the connection details -gitlab_rails['db_adapter'] = "postgresql" -gitlab_rails['db_encoding'] = "unicode" -gitlab_rails['db_database'] = "gitlabhq_production" -gitlab_rails['db_username'] = "gitlab" -gitlab_rails['db_password'] = "mypassword" -gitlab_rails['db_host'] = "<rds-endpoint>" -``` - -Next, we need to configure the Redis section by adding the host and -uncommenting the port: - -```ruby -# Disable the built-in Redis -redis['enable'] = false - -# Fill in the connection details -gitlab_rails['redis_host'] = "<redis-endpoint>" -gitlab_rails['redis_port'] = 6379 -``` - -Finally, reconfigure GitLab for the change to take effect: - -```shell -sudo gitlab-ctl reconfigure -``` - -You might also find it useful to run a check and a service status to make sure -everything has been setup correctly: - -```shell -sudo gitlab-rake gitlab:check -sudo gitlab-ctl status -``` - -If everything looks good, you should be able to reach GitLab in your browser. - -### Using Amazon S3 object storage - -GitLab stores many objects outside the Git repository, many of which can be -uploaded to S3. That way, you can offload the root disk volume of these objects -which would otherwise take much space. - -In particular, you can store in S3: - -- [The Git LFS objects](../../administration/lfs/lfs_administration.md#s3-for-omnibus-installations) ((Omnibus GitLab installations)) -- [The Container Registry images](../../administration/packages/container_registry.md#container-registry-storage-driver) (Omnibus GitLab installations) -- [The GitLab CI/CD job artifacts](../../administration/job_artifacts.md#using-object-storage) (Omnibus GitLab installations) - -### Setting up a domain name - -After you SSH into the instance, configure the domain name: - -1. Open `/etc/gitlab/gitlab.rb` with your preferred editor. -1. Edit the `external_url` value: - - ```ruby - external_url 'http://example.com' - ``` - -1. Reconfigure GitLab: - - ```shell - sudo gitlab-ctl reconfigure - ``` - -You should now be able to reach GitLab at the URL you defined. To use HTTPS -(recommended), see the [HTTPS documentation](https://docs.gitlab.com/omnibus/settings/nginx.html#enable-https). - -### Logging in for the first time - -If you followed the previous section, you should be now able to visit GitLab -in your browser. The very first time, you will be asked to set up a password -for the `root` user which has admin privileges on the GitLab instance. - -After you set it up, login with username `root` and the newly created password. - ## Health check and monitoring with Prometheus Apart from Amazon's Cloudwatch which you can enable on various services, diff --git a/doc/user/project/integrations/prometheus.md b/doc/user/project/integrations/prometheus.md index eb0013ab6e5..6d848f73cb6 100644 --- a/doc/user/project/integrations/prometheus.md +++ b/doc/user/project/integrations/prometheus.md @@ -13,7 +13,7 @@ There are two ways to set up Prometheus integration, depending on where your app - For deployments on Kubernetes, GitLab can automatically [deploy and manage Prometheus](#managed-prometheus-on-kubernetes). - For other deployment targets, simply [specify the Prometheus server](#manual-configuration-of-prometheus). -Once enabled, GitLab will automatically detect metrics from known services in the [metric library](#monitoring-cicd-environments). You are also able to [add your own metrics](#adding-additional-metrics-premium) as well. +Once enabled, GitLab will automatically detect metrics from known services in the [metric library](#monitoring-cicd-environments). You can also [add your own metrics](#adding-custom-metrics). ## Enabling Prometheus Integration @@ -132,9 +132,10 @@ GitLab will automatically scan the Prometheus server for metrics from known serv You can view the performance dashboard for an environment by [clicking on the monitoring button](../../../ci/environments.md#monitoring-environments). -### Adding additional metrics **(PREMIUM)** +### Adding custom metrics -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/3799) in [GitLab Premium](https://about.gitlab.com/pricing/) 10.6. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/3799) in [GitLab Premium](https://about.gitlab.com/pricing/) 10.6. +> - [Moved](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/28527) to [GitLab Core](https://about.gitlab.com/pricing/) 12.10. Custom metrics can be monitored by adding them on the monitoring dashboard page. Once saved, they will be displayed on the environment performance dashboard provided that either: @@ -191,8 +192,8 @@ You may create a new file from scratch or duplicate a GitLab-defined Prometheus dashboard. NOTE: **Note:** -The custom metrics as defined below do not support alerts, unlike -[additional metrics](#adding-additional-metrics-premium). +The metrics as defined below do not support alerts, unlike +[custom metrics](#adding-custom-metrics). #### Adding a new dashboard to your project @@ -654,9 +655,9 @@ Data from Prometheus charts on the metrics dashboard can be downloaded as CSV. #### Managed Prometheus instances -> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/6590) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.2 for [custom metrics](#adding-additional-metrics-premium), and 11.3 for [library metrics](prometheus_library/metrics.md). +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/6590) in [GitLab Ultimate](https://about.gitlab.com/pricing/) 11.2 for [custom metrics](#adding-custom-metrics), and 11.3 for [library metrics](prometheus_library/metrics.md). -For managed Prometheus instances using auto configuration, alerts for metrics [can be configured](#adding-additional-metrics-premium) directly in the performance dashboard. +For managed Prometheus instances using auto configuration, alerts for metrics [can be configured](#adding-custom-metrics) directly in the performance dashboard. To set an alert: diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb index 9ed6a23632c..f9a226be3cf 100644 --- a/lib/gitlab/project_template.rb +++ b/lib/gitlab/project_template.rb @@ -74,3 +74,5 @@ module Gitlab end end end + +Gitlab::ProjectTemplate.prepend_if_ee('EE::Gitlab::ProjectTemplate') diff --git a/lib/gitlab/prometheus/queries/validate_query.rb b/lib/gitlab/prometheus/queries/validate_query.rb new file mode 100644 index 00000000000..1f55f3e9768 --- /dev/null +++ b/lib/gitlab/prometheus/queries/validate_query.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Prometheus + module Queries + class ValidateQuery < BaseQuery + def query(query) + client_query(query) + { valid: true } + rescue Gitlab::PrometheusClient::QueryError, Gitlab::HTTP::BlockedUrlError => ex + { valid: false, error: ex.message } + end + + def self.transform_reactive_result(result) + result[:query] = result.delete :data + result + end + end + end + end +end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index c1bda6a223a..7349ca34ae5 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -862,6 +862,9 @@ msgstr "" msgid "A project boilerplate for Salesforce App development with Salesforce Developer tools." msgstr "" +msgid "A project containing issues for each audit inquiry in the HIPAA Audit Protocol published by the U.S. Department of Health & Human Services" +msgstr "" + msgid "A project is where you house your files (repository), plan your work (issues), and publish your documentation (wiki), %{among_other_things_link}." msgstr "" @@ -15802,6 +15805,9 @@ msgstr "" msgid "ProjectTemplates|Go Micro" msgstr "" +msgid "ProjectTemplates|HIPAA Audit Protocol" +msgstr "" + msgid "ProjectTemplates|Netlify/GitBook" msgstr "" diff --git a/spec/controllers/projects/import/jira_controller_spec.rb b/spec/controllers/projects/import/jira_controller_spec.rb index f692f976bc0..57e0aa098c0 100644 --- a/spec/controllers/projects/import/jira_controller_spec.rb +++ b/spec/controllers/projects/import/jira_controller_spec.rb @@ -59,6 +59,7 @@ describe Projects::Import::JiraController do context 'when feature flag enabled' do before do stub_feature_flags(jira_issue_import: true) + stub_feature_flags(jira_issue_import_vue: false) end context 'when jira service is enabled for the project' do diff --git a/spec/controllers/projects/prometheus/metrics_controller_spec.rb b/spec/controllers/projects/prometheus/metrics_controller_spec.rb index 314214ceefb..36f694cda29 100644 --- a/spec/controllers/projects/prometheus/metrics_controller_spec.rb +++ b/spec/controllers/projects/prometheus/metrics_controller_spec.rb @@ -3,8 +3,8 @@ require 'spec_helper' describe Projects::Prometheus::MetricsController do - let(:user) { create(:user) } - let(:project) { create(:project) } + let_it_be(:user) { create(:user) } + let_it_be(:project) { create(:prometheus_project) } let(:prometheus_adapter) { double('prometheus_adapter', can_query?: true) } @@ -71,6 +71,8 @@ describe Projects::Prometheus::MetricsController do end context 'when prometheus_adapter is disabled' do + let(:project) { create(:project) } + it 'renders 404' do get :active_common, params: project_params(format: :json) @@ -79,6 +81,106 @@ describe Projects::Prometheus::MetricsController do end end + describe 'POST #validate_query' do + before do + allow(controller).to receive(:prometheus_adapter).and_return(prometheus_adapter) + allow(prometheus_adapter).to receive(:query).with(:validate, query) { validation_result } + end + + let(:query) { 'avg(metric)' } + + context 'validation information is ready' do + let(:validation_result) { { valid: true } } + + it 'validation data is returned' do + post :validate_query, params: project_params(format: :json, query: query) + + expect(json_response).to eq('valid' => true) + end + end + + context 'validation information is not ready' do + let(:validation_result) { nil } + + it 'validation data is returned' do + post :validate_query, params: project_params(format: :json, query: query) + + expect(response).to have_gitlab_http_status(:accepted) + end + end + end + + describe 'GET #index' do + context 'with custom metric present' do + let!(:prometheus_metric) { create(:prometheus_metric, project: project) } + + it 'returns a list of metrics' do + get :index, params: project_params(format: :json) + + expect(response).to have_gitlab_http_status(:ok) + expect(response).to match_response_schema('prometheus/metrics') + end + end + + context 'without custom metrics ' do + it 'returns an empty json' do + get :index, params: project_params(format: :json) + + expect(response).to have_gitlab_http_status(:ok) + expect(json_response).to eq({}) + end + end + end + + describe 'POST #create' do + context 'metric is valid' do + let(:valid_metric) { { prometheus_metric: { title: 'title', query: 'query', group: 'business', y_label: 'label', unit: 'u', legend: 'legend' } } } + + it 'shows a success flash message' do + post :create, params: project_params(valid_metric) + + expect(flash[:notice]).to include('Metric was successfully added.') + + expect(response).to redirect_to(edit_project_service_path(project, PrometheusService)) + end + end + + context 'metric is invalid' do + let(:invalid_metric) { { prometheus_metric: { title: 'title' } } } + + it 'renders new metric page' do + post :create, params: project_params(invalid_metric) + + expect(response).to have_gitlab_http_status(:ok) + expect(response).to render_template('new') + end + end + end + + describe 'DELETE #destroy' do + context 'format html' do + let!(:metric) { create(:prometheus_metric, project: project) } + + it 'destroys the metric' do + delete :destroy, params: project_params(id: metric.id) + + expect(response).to redirect_to(edit_project_service_path(project, PrometheusService)) + expect(PrometheusMetric.find_by(id: metric.id)).to be_nil + end + end + + context 'format json' do + let!(:metric) { create(:prometheus_metric, project: project) } + + it 'destroys the metric' do + delete :destroy, params: project_params(id: metric.id, format: :json) + + expect(response).to have_gitlab_http_status(:ok) + expect(PrometheusMetric.find_by(id: metric.id)).to be_nil + end + end + end + describe '#prometheus_adapter' do before do allow(controller).to receive(:project).and_return(project) diff --git a/spec/controllers/projects/registry/repositories_controller_spec.rb b/spec/controllers/projects/registry/repositories_controller_spec.rb index e68959b6a96..c641a45a216 100644 --- a/spec/controllers/projects/registry/repositories_controller_spec.rb +++ b/spec/controllers/projects/registry/repositories_controller_spec.rb @@ -110,6 +110,7 @@ describe Projects::Registry::RepositoriesController do delete_repository(repository) + expect(repository.reload).to be_delete_scheduled expect(response).to have_gitlab_http_status(:no_content) end diff --git a/spec/fixtures/api/schemas/prometheus/metrics.json b/spec/fixtures/api/schemas/prometheus/metrics.json new file mode 100644 index 00000000000..0a13f5efc77 --- /dev/null +++ b/spec/fixtures/api/schemas/prometheus/metrics.json @@ -0,0 +1,28 @@ +{ + "type": "object", + "properties": { + "metrics": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "title": { + "type": "string" + }, + "group": { + "type": "string" + }, + "group_title": { + "type": "string" + }, + "edit_path": { + "type": "string" + } + } + } + } + } +} diff --git a/spec/fixtures/api/schemas/registry/repository.json b/spec/fixtures/api/schemas/registry/repository.json index d0a068b65a7..f7469aad235 100644 --- a/spec/fixtures/api/schemas/registry/repository.json +++ b/spec/fixtures/api/schemas/registry/repository.json @@ -32,6 +32,12 @@ "destroy_path": { "type": "string" }, + "status": { + "oneOf": [ + { "type": "null" }, + { "type": "string", "enum": ["delete_scheduled", "delete_failed"] } + ] + }, "tags": { "$ref": "tags.json" } }, "additionalProperties": false diff --git a/spec/helpers/environments_helper_spec.rb b/spec/helpers/environments_helper_spec.rb index aaa90a7d2af..152e9c84ec5 100644 --- a/spec/helpers/environments_helper_spec.rb +++ b/spec/helpers/environments_helper_spec.rb @@ -34,7 +34,10 @@ describe EnvironmentsHelper do 'has-metrics' => "#{environment.has_metrics?}", 'prometheus-status' => "#{environment.prometheus_status}", 'external-dashboard-url' => nil, - 'environment-state' => environment.state + 'environment-state' => environment.state, + 'custom-metrics-path' => project_prometheus_metrics_path(project), + 'validate-query-path' => validate_query_project_prometheus_metrics_path(project), + 'custom-metrics-available' => 'true' ) end @@ -58,4 +61,22 @@ describe EnvironmentsHelper do it { is_expected.to include('environment-state' => 'stopped') } end end + + describe '#custom_metrics_available?' do + subject { helper.custom_metrics_available?(project) } + + before do + project.add_maintainer(user) + + allow(helper).to receive(:current_user).and_return(user) + + allow(helper).to receive(:can?) + .with(user, :admin_project, project) + .and_return(true) + end + + it 'returns true' do + expect(subject).to eq(true) + end + end end diff --git a/spec/javascripts/notebook/cells/code_spec.js b/spec/javascripts/notebook/cells/code_spec.js index 4659b83d1b6..f3f97145ad3 100644 --- a/spec/javascripts/notebook/cells/code_spec.js +++ b/spec/javascripts/notebook/cells/code_spec.js @@ -11,14 +11,19 @@ describe('Code component', () => { json = getJSONFixture('blob/notebook/basic.json'); }); + const setupComponent = cell => { + const comp = new Component({ + propsData: { + cell, + }, + }); + comp.$mount(); + return comp; + }; + describe('without output', () => { beforeEach(done => { - vm = new Component({ - propsData: { - cell: json.cells[0], - }, - }); - vm.$mount(); + vm = setupComponent(json.cells[0]); setTimeout(() => { done(); @@ -32,12 +37,7 @@ describe('Code component', () => { describe('with output', () => { beforeEach(done => { - vm = new Component({ - propsData: { - cell: json.cells[2], - }, - }); - vm.$mount(); + vm = setupComponent(json.cells[2]); setTimeout(() => { done(); @@ -52,4 +52,23 @@ describe('Code component', () => { expect(vm.$el.querySelector('.output')).toBeDefined(); }); }); + + describe('with string for cell.source', () => { + beforeEach(done => { + const cell = json.cells[0]; + cell.source = cell.source.join(''); + + vm = setupComponent(cell); + + setTimeout(() => { + done(); + }); + }); + + it('renders the same input as when cell.source is an array', () => { + const expected = "console.log('test')"; + + expect(vm.$el.querySelector('.input').innerText).toContain(expected); + }); + }); }); diff --git a/spec/lib/gitlab/prometheus/queries/validate_query_spec.rb b/spec/lib/gitlab/prometheus/queries/validate_query_spec.rb new file mode 100644 index 00000000000..9c0170718f4 --- /dev/null +++ b/spec/lib/gitlab/prometheus/queries/validate_query_spec.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::Prometheus::Queries::ValidateQuery do + include PrometheusHelpers + + let(:api_url) { 'https://prometheus.example.com' } + let(:client) { Gitlab::PrometheusClient.new(api_url) } + let(:query) { 'avg(metric)' } + + subject { described_class.new(client) } + + context 'valid query' do + before do + allow(client).to receive(:query).with(query) + end + + it 'passess query to prometheus' do + expect(subject.query(query)).to eq(valid: true) + + expect(client).to have_received(:query).with(query) + end + end + + context 'invalid query' do + let(:query) { 'invalid query' } + let(:error_message) { "invalid parameter 'query': 1:9: parse error: unexpected identifier \"query\"" } + + it 'returns invalid' do + Timecop.freeze do + stub_prometheus_query_error( + prometheus_query_with_time_url(query, Time.now), + error_message + ) + + expect(subject.query(query)).to eq(valid: false, error: error_message) + end + end + end + + context 'when exceptions occur' do + context 'Gitlab::HTTP::BlockedUrlError' do + let(:api_url) { 'http://192.168.1.1' } + + let(:message) do + "URL 'http://192.168.1.1/api/v1/query?query=avg%28metric%29&time=#{Time.now.to_f}'" \ + " is blocked: Requests to the local network are not allowed" + end + + before do + stub_application_setting(allow_local_requests_from_web_hooks_and_services: false) + end + + it 'catches exception and returns invalid' do + Timecop.freeze do + expect(subject.query(query)).to eq(valid: false, error: message) + end + end + end + end +end diff --git a/spec/models/project_services/prometheus_service_spec.rb b/spec/models/project_services/prometheus_service_spec.rb index 297411f7980..f663f0ab7cb 100644 --- a/spec/models/project_services/prometheus_service_spec.rb +++ b/spec/models/project_services/prometheus_service_spec.rb @@ -48,6 +48,18 @@ describe PrometheusService, :use_clean_rails_memory_store_caching do it 'does not validate presence of api_url' do expect(service).not_to validate_presence_of(:api_url) + expect(service.valid?).to eq(true) + end + + context 'local connections allowed' do + before do + stub_application_setting(allow_local_requests_from_web_hooks_and_services: true) + end + + it 'does not validate presence of api_url' do + expect(service).not_to validate_presence_of(:api_url) + expect(service.valid?).to eq(true) + end end end diff --git a/spec/services/projects/container_repository/destroy_service_spec.rb b/spec/services/projects/container_repository/destroy_service_spec.rb index cc8fd2716e1..753b7540d7f 100644 --- a/spec/services/projects/container_repository/destroy_service_spec.rb +++ b/spec/services/projects/container_repository/destroy_service_spec.rb @@ -36,6 +36,16 @@ describe Projects::ContainerRepository::DestroyService do expect(repository).to receive(:delete_tags!).and_call_original expect { described_class.new(project, user).execute(repository) }.to change { ContainerRepository.all.count }.by(-1) end + + context 'when destroy fails' do + it 'set delete_status' do + allow(repository).to receive(:destroy).and_return(false) + + subject.execute(repository) + + expect(repository).to be_delete_failed + end + end end end end diff --git a/spec/support/helpers/api_helpers.rb b/spec/support/helpers/api_helpers.rb index dc263d64bcc..b1e6078c4f2 100644 --- a/spec/support/helpers/api_helpers.rb +++ b/spec/support/helpers/api_helpers.rb @@ -51,11 +51,11 @@ module ApiHelpers expect(json_response).to be_an Array end - def expect_paginated_array_response(items) + def expect_paginated_array_response(*items) expect(response).to have_gitlab_http_status(:ok) expect(response).to include_pagination_headers expect(json_response).to be_an Array - expect(json_response.map { |item| item['id'] }).to eq(Array(items)) + expect(json_response.map { |item| item['id'] }).to eq(items.flatten) end def expect_response_contain_exactly(*items) diff --git a/vendor/project_templates/hipaa_audit_protocol.tar.gz b/vendor/project_templates/hipaa_audit_protocol.tar.gz Binary files differnew file mode 100644 index 00000000000..7ca94675d35 --- /dev/null +++ b/vendor/project_templates/hipaa_audit_protocol.tar.gz |