diff options
34 files changed, 696 insertions, 524 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8b489f1a07c..5556bf5bc0b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,8 +35,14 @@ variables: before_script: - bundle --version + - date - source scripts/utils.sh + - date - source scripts/prepare_build.sh + - date + +after_script: + - date stages: - build @@ -88,6 +94,26 @@ stages: - /(^docs[\/-].*|.*-docs$)/ - /(^qa[\/-].*|.*-qa$)/ +# Jobs that only need to pull cache +.dedicated-no-docs-pull-cache-job: &dedicated-no-docs-pull-cache-job + <<: *dedicated-runner + <<: *except-docs-and-qa + <<: *pull-cache + dependencies: + - setup-test-env + stage: test + +# Jobs that do not need a DB +.dedicated-no-docs-no-db-pull-cache-job: &dedicated-no-docs-no-db-pull-cache-job + <<: *dedicated-no-docs-pull-cache-job + variables: + SETUP_DB: "false" + +.rake-exec: &rake-exec + <<: *dedicated-no-docs-no-db-pull-cache-job + script: + - bundle exec rake $CI_JOB_NAME + .rspec-metadata: &rspec-metadata <<: *dedicated-runner <<: *except-docs-and-qa @@ -164,21 +190,23 @@ stages: - master@gitlab/gitlabhq - master@gitlab/gitlab-ee -## -# Trigger a package build in omnibus-gitlab repository -# -package-qa: - <<: *dedicated-runner - image: ruby:2.4-alpine - before_script: [] - stage: build - cache: {} - when: manual +.gitlab-setup: &gitlab-setup + <<: *dedicated-no-docs-pull-cache-job + <<: *use-pg + variables: + CREATE_DB_USER: "true" script: - - scripts/trigger-build-omnibus - only: - - //@gitlab-org/gitlab-ce - - //@gitlab-org/gitlab-ee + # Manually clone gitlab-test and only seed this project in + # db/fixtures/development/04_project.rb thanks to SIZE=1 below + - git clone https://gitlab.com/gitlab-org/gitlab-test.git + /home/git/repositories/gitlab-org/gitlab-test.git + - scripts/gitaly-test-spawn + - force=yes SIZE=1 FIXTURE_PATH="db/fixtures/development" bundle exec rake gitlab:setup + artifacts: + when: on_failure + expire_in: 1d + paths: + - log/development.log # Review docs base .review-docs: &review-docs @@ -201,6 +229,47 @@ package-qa: only: - branches +# DB migration, rollback, and seed jobs +.db-migrate-reset: &db-migrate-reset + <<: *dedicated-no-docs-pull-cache-job + script: + - bundle exec rake db:migrate:reset + +.migration-paths: &migration-paths + <<: *dedicated-no-docs-pull-cache-job + variables: + CREATE_DB_USER: "true" + script: + - git fetch https://gitlab.com/gitlab-org/gitlab-ce.git v9.3.0 + - git checkout -f FETCH_HEAD + - bundle install $BUNDLE_INSTALL_FLAGS + - date + - cp config/gitlab.yml.example config/gitlab.yml + - bundle exec rake db:drop db:create db:schema:load db:seed_fu + - date + - git checkout $CI_COMMIT_SHA + - bundle install $BUNDLE_INSTALL_FLAGS + - date + - . scripts/prepare_build.sh + - date + - bundle exec rake db:migrate + +## +# Trigger a package build in omnibus-gitlab repository +# +package-qa: + <<: *dedicated-runner + image: ruby:2.4-alpine + before_script: [] + stage: build + cache: {} + when: manual + script: + - scripts/trigger-build-omnibus + only: + - //@gitlab-org/gitlab-ce + - //@gitlab-org/gitlab-ee + # Trigger a docs build in gitlab-docs # Useful to preview the docs changes live review-docs-deploy: @@ -265,7 +334,7 @@ update-tests-metadata: flaky-examples-check: <<: *dedicated-runner - image: ruby:2.3-alpine + image: ruby:2.4-alpine services: [] before_script: [] variables: @@ -299,7 +368,9 @@ compile-assets: <<: *default-cache script: - node --version + - date - yarn install --frozen-lockfile --cache-folder .yarn-cache + - date - bundle exec rake gitlab:assets:compile artifacts: expire_in: 7d @@ -387,26 +458,11 @@ spinach-pg 1 2: *spinach-metadata-pg spinach-mysql 0 2: *spinach-metadata-mysql spinach-mysql 1 2: *spinach-metadata-mysql -# Static analysis jobs -.ruby-static-analysis: &ruby-static-analysis - variables: - SIMPLECOV: "false" - SETUP_DB: "false" - -.rake-exec: &rake-exec - <<: *dedicated-runner - <<: *except-docs-and-qa - <<: *pull-cache - <<: *ruby-static-analysis - stage: test - script: - - bundle exec rake $CI_JOB_NAME - static-analysis: - <<: *dedicated-runner - <<: *except-docs - <<: *ruby-static-analysis - stage: test + <<: *dedicated-no-docs-no-db-pull-cache-job + dependencies: + - compile-assets + - setup-test-env script: - scripts/static-analysis cache: @@ -463,15 +519,6 @@ ee_compat_check: paths: - ee_compat_check/patches/*.patch -# DB migration, rollback, and seed jobs -.db-migrate-reset: &db-migrate-reset - <<: *dedicated-runner - <<: *except-docs-and-qa - <<: *pull-cache - stage: test - script: - - bundle exec rake db:migrate:reset - db:migrate:reset-pg: <<: *db-migrate-reset <<: *use-pg @@ -486,25 +533,6 @@ db:check-schema-pg: script: - source scripts/schema_changed.sh -.migration-paths: &migration-paths - <<: *dedicated-runner - <<: *except-docs-and-qa - <<: *pull-cache - stage: test - variables: - SETUP_DB: "false" - CREATE_DB_USER: "true" - script: - - git fetch https://gitlab.com/gitlab-org/gitlab-ce.git v9.3.0 - - git checkout -f FETCH_HEAD - - bundle install $BUNDLE_INSTALL_FLAGS - - cp config/gitlab.yml.example config/gitlab.yml - - bundle exec rake db:drop db:create db:schema:load db:seed_fu - - git checkout $CI_COMMIT_SHA - - bundle install $BUNDLE_INSTALL_FLAGS - - . scripts/prepare_build.sh - - bundle exec rake db:migrate - migration:path-pg: <<: *migration-paths <<: *use-pg @@ -514,10 +542,7 @@ migration:path-mysql: <<: *use-mysql .db-rollback: &db-rollback - <<: *dedicated-runner - <<: *except-docs-and-qa - <<: *pull-cache - stage: test + <<: *dedicated-no-docs-pull-cache-job script: - bundle exec rake db:rollback STEP=119 - bundle exec rake db:migrate @@ -530,27 +555,6 @@ db:rollback-mysql: <<: *db-rollback <<: *use-mysql -.gitlab-setup: &gitlab-setup - <<: *dedicated-runner - <<: *except-docs-and-qa - <<: *pull-cache - stage: test - variables: - SIZE: "1" - SETUP_DB: "false" - CREATE_DB_USER: "true" - FIXTURE_PATH: db/fixtures/development - script: - - git clone https://gitlab.com/gitlab-org/gitlab-test.git - /home/git/repositories/gitlab-org/gitlab-test.git - - scripts/gitaly-test-spawn - - force=yes bundle exec rake gitlab:setup - artifacts: - when: on_failure - expire_in: 1d - paths: - - log/development.log - gitlab:setup-pg: <<: *gitlab-setup <<: *use-pg @@ -561,10 +565,7 @@ gitlab:setup-mysql: # Frontend-related jobs gitlab:assets:compile: - <<: *dedicated-runner - <<: *except-docs-and-qa - <<: *pull-cache - stage: test + <<: *dedicated-no-docs-no-db-pull-cache-job dependencies: [] variables: NODE_ENV: "production" @@ -574,7 +575,9 @@ gitlab:assets:compile: WEBPACK_REPORT: "true" NO_COMPRESSION: "true" script: + - date - yarn install --frozen-lockfile --production --cache-folder .yarn-cache + - date - bundle exec rake gitlab:assets:compile artifacts: name: webpack-report @@ -583,17 +586,16 @@ gitlab:assets:compile: - webpack-report/ karma: - <<: *dedicated-runner - <<: *except-docs-and-qa - <<: *pull-cache + <<: *dedicated-no-docs-pull-cache-job <<: *use-pg - stage: test - variables: - BABEL_ENV: "coverage" - CHROME_LOG_FILE: "chrome_debug.log" + dependencies: + - compile-assets + - setup-test-env script: + - export BABEL_ENV=coverage CHROME_LOG_FILE=chrome_debug.log + - date - scripts/gitaly-test-spawn - - bundle exec rake gettext:po_to_json + - date - bundle exec rake karma coverage: '/^Statements *: (\d+\.\d+%)/' artifacts: @@ -601,13 +603,11 @@ karma: expire_in: 31d when: always paths: - - chrome_debug.log - - coverage-javascript/ + - chrome_debug.log + - coverage-javascript/ codequality: - <<: *except-docs - <<: *pull-cache - stage: test + <<: *dedicated-no-docs-no-db-pull-cache-job image: docker:latest before_script: [] services: @@ -639,11 +639,7 @@ sast: paths: [gl-sast-report.json] qa:internal: - <<: *dedicated-runner - <<: *except-docs - stage: test - variables: - SETUP_DB: "false" + <<: *dedicated-no-docs-no-db-pull-cache-job services: [] script: - cd qa/ @@ -651,11 +647,7 @@ qa:internal: - bundle exec rspec qa:selectors: - <<: *dedicated-runner - <<: *except-docs - stage: test - variables: - SETUP_DB: "false" + <<: *dedicated-no-docs-no-db-pull-cache-job services: [] script: - cd qa/ @@ -663,14 +655,8 @@ qa:selectors: - bundle exec bin/qa Test::Sanity::Selectors coverage: - <<: *dedicated-runner - <<: *except-docs-and-qa - <<: *pull-cache + <<: *dedicated-no-docs-no-db-pull-cache-job stage: post-test - services: [] - variables: - SETUP_DB: "false" - USE_BUNDLE_INSTALL: "true" script: - bundle exec scripts/merge-simplecov coverage: '/LOC \((\d+\.\d+%)\) covered.$/' @@ -682,26 +668,25 @@ coverage: - coverage/assets/ lint:javascript:report: - <<: *dedicated-runner - <<: *except-docs-and-qa - <<: *pull-cache + <<: *dedicated-no-docs-no-db-pull-cache-job stage: post-test dependencies: - compile-assets - setup-test-env before_script: [] script: + - date - find app/ spec/ -name '*.js' -exec sed --in-place 's|/\* eslint-disable .*\*/||' {} \; # run report over all files + - date - yarn run eslint-report || true # ignore exit code artifacts: name: eslint-report expire_in: 31d paths: - - eslint-report.html + - eslint-report.html pages: - <<: *dedicated-runner - <<: *pull-cache + <<: *dedicated-no-docs-no-db-pull-cache-job before_script: [] stage: pages dependencies: @@ -726,10 +711,7 @@ pages: # Insurance in case a gem needed by one of our releases gets yanked from # rubygems.org in the future. cache gems: - <<: *dedicated-runner - <<: *pull-cache - variables: - SETUP_DB: "false" + <<: *dedicated-no-docs-no-db-pull-cache-job script: - bundle package --all --all-platforms artifacts: diff --git a/app/assets/javascripts/monitoring/utils/multiple_time_series.js b/app/assets/javascripts/monitoring/utils/multiple_time_series.js index 4ce3dad440c..b5b8e3c255d 100644 --- a/app/assets/javascripts/monitoring/utils/multiple_time_series.js +++ b/app/assets/javascripts/monitoring/utils/multiple_time_series.js @@ -76,7 +76,7 @@ function queryTimeSeries(query, graphWidth, graphHeight, graphHeightOffset, xDom metricTag = seriesCustomizationData.value || timeSeriesMetricLabel; [lineColor, areaColor] = pickColor(seriesCustomizationData.color); } else { - metricTag = timeSeriesMetricLabel || `series ${timeSeriesNumber + 1}`; + metricTag = timeSeriesMetricLabel || query.label || `series ${timeSeriesNumber + 1}`; [lineColor, areaColor] = pickColor(); } diff --git a/app/controllers/projects/deployments_controller.rb b/app/controllers/projects/deployments_controller.rb index 1a418d0f15a..b68cdc39cb8 100644 --- a/app/controllers/projects/deployments_controller.rb +++ b/app/controllers/projects/deployments_controller.rb @@ -24,7 +24,7 @@ class Projects::DeploymentsController < Projects::ApplicationController end def additional_metrics - return render_404 unless deployment.has_additional_metrics? + return render_404 unless deployment.has_metrics? respond_to do |format| format.json do diff --git a/app/controllers/projects/prometheus/metrics_controller.rb b/app/controllers/projects/prometheus/metrics_controller.rb index b739d0f0f90..1dd886409a5 100644 --- a/app/controllers/projects/prometheus/metrics_controller.rb +++ b/app/controllers/projects/prometheus/metrics_controller.rb @@ -2,11 +2,12 @@ module Projects module Prometheus class MetricsController < Projects::ApplicationController before_action :authorize_admin_project! + before_action :require_prometheus_metrics! def active_common respond_to do |format| format.json do - matched_metrics = prometheus_service.matched_metrics || {} + matched_metrics = prometheus_adapter.query(:matched_metrics) || {} if matched_metrics.any? render json: matched_metrics @@ -19,8 +20,12 @@ module Projects private - def prometheus_service - @prometheus_service ||= project.find_or_initialize_service('prometheus') + def prometheus_adapter + @prometheus_adapter ||= ::Prometheus::AdapterService.new(project).prometheus_adapter + end + + def require_prometheus_metrics! + render_404 unless prometheus_adapter.can_query? end end end diff --git a/app/models/clusters/applications/prometheus.rb b/app/models/clusters/applications/prometheus.rb index 89ebd63e605..7b25d8c4089 100644 --- a/app/models/clusters/applications/prometheus.rb +++ b/app/models/clusters/applications/prometheus.rb @@ -1,6 +1,8 @@ module Clusters module Applications class Prometheus < ActiveRecord::Base + include PrometheusAdapter + VERSION = "2.0.0".freeze self.table_name = 'clusters_applications_prometheus' @@ -39,7 +41,7 @@ module Clusters ) end - def proxy_client + def prometheus_client return unless kube_client proxy_url = kube_client.proxy_url('service', service_name, service_port, Gitlab::Kubernetes::Helm::NAMESPACE) diff --git a/app/models/clusters/cluster.rb b/app/models/clusters/cluster.rb index 1c0046107d7..49eb069016a 100644 --- a/app/models/clusters/cluster.rb +++ b/app/models/clusters/cluster.rb @@ -51,9 +51,6 @@ module Clusters scope :enabled, -> { where(enabled: true) } scope :disabled, -> { where(enabled: false) } - scope :for_environment, -> (env) { where(environment_scope: ['*', '', env.slug]) } - scope :for_all_environments, -> { where(environment_scope: ['*', '']) } - def status_name if provider provider.status_name diff --git a/app/models/concerns/prometheus_adapter.rb b/app/models/concerns/prometheus_adapter.rb new file mode 100644 index 00000000000..18cbbd871a1 --- /dev/null +++ b/app/models/concerns/prometheus_adapter.rb @@ -0,0 +1,48 @@ +module PrometheusAdapter + extend ActiveSupport::Concern + + included do + include ReactiveCaching + + self.reactive_cache_key = ->(adapter) { [adapter.class.model_name.singular, adapter.id] } + self.reactive_cache_lease_timeout = 30.seconds + self.reactive_cache_refresh_interval = 30.seconds + self.reactive_cache_lifetime = 1.minute + + def prometheus_client + raise NotImplementedError + end + + def prometheus_client_wrapper + Gitlab::PrometheusClient.new(prometheus_client) + end + + def can_query? + prometheus_client.present? + end + + def query(query_name, *args) + return unless can_query? + + query_class = Gitlab::Prometheus::Queries.const_get("#{query_name.to_s.classify}Query") + + args.map!(&:id) + + with_reactive_cache(query_class.name, *args, &query_class.method(:transform_reactive_result)) + end + + # Cache metrics for specific environment + def calculate_reactive_cache(query_class_name, *args) + return unless prometheus_client + + data = Kernel.const_get(query_class_name).new(prometheus_client_wrapper).query(*args) + { + success: true, + data: data, + last_update: Time.now.utc + } + rescue Gitlab::PrometheusClient::Error => err + { success: false, result: err.message } + end + end +end diff --git a/app/models/deployment.rb b/app/models/deployment.rb index b6cf168d60e..66e61c06765 100644 --- a/app/models/deployment.rb +++ b/app/models/deployment.rb @@ -98,28 +98,29 @@ class Deployment < ActiveRecord::Base end def has_metrics? - project.monitoring_service.present? + prometheus_adapter&.can_query? end def metrics return {} unless has_metrics? - project.monitoring_service.deployment_metrics(self) - end - - def has_additional_metrics? - project.prometheus_service.present? + metrics = prometheus_adapter.query(:deployment, self) + metrics&.merge(deployment_time: created_at.to_i) || {} end def additional_metrics - return {} unless project.prometheus_service.present? + return {} unless has_metrics? - metrics = project.prometheus_service.additional_deployment_metrics(self) + metrics = prometheus_adapter.query(:additional_metrics_deployment, self) metrics&.merge(deployment_time: created_at.to_i) || {} end private + def prometheus_adapter + environment.prometheus_adapter + end + def ref_path File.join(environment.ref_path, 'deployments', iid.to_s) end diff --git a/app/models/environment.rb b/app/models/environment.rb index f78c21aebe5..24d4f1d8761 100644 --- a/app/models/environment.rb +++ b/app/models/environment.rb @@ -146,21 +146,19 @@ class Environment < ActiveRecord::Base end def has_metrics? - project.monitoring_service.present? && available? && last_deployment.present? + prometheus_adapter&.can_query? && available? && last_deployment.present? end def metrics - project.monitoring_service.environment_metrics(self) if has_metrics? + prometheus_adapter.query(:environment, self) if has_metrics? end - def has_additional_metrics? - project.prometheus_service.present? && available? && last_deployment.present? + def additional_metrics + prometheus_adapter.query(:additional_metrics_environment, self) if has_metrics? end - def additional_metrics - if has_additional_metrics? - project.prometheus_service.additional_environment_metrics(self) - end + def prometheus_adapter + @prometheus_adapter ||= Prometheus::AdapterService.new(project, deployment_platform).prometheus_adapter end def slug @@ -226,6 +224,10 @@ class Environment < ActiveRecord::Base self.environment_type || self.name end + def deployment_platform + project.deployment_platform + end + private # Slugifying a name may remove the uniqueness guarantee afforded by it being diff --git a/app/models/project_services/monitoring_service.rb b/app/models/project_services/monitoring_service.rb index ee9cd78327a..9af68b4e821 100644 --- a/app/models/project_services/monitoring_service.rb +++ b/app/models/project_services/monitoring_service.rb @@ -9,11 +9,11 @@ class MonitoringService < Service %w() end - def environment_metrics(environment) + def can_query? raise NotImplementedError end - def deployment_metrics(deployment) + def query(_, *_) raise NotImplementedError end end diff --git a/app/models/project_services/prometheus_service.rb b/app/models/project_services/prometheus_service.rb index 58731451429..dcaeb65dc32 100644 --- a/app/models/project_services/prometheus_service.rb +++ b/app/models/project_services/prometheus_service.rb @@ -1,9 +1,5 @@ class PrometheusService < MonitoringService - include ReactiveService - - self.reactive_cache_lease_timeout = 30.seconds - self.reactive_cache_refresh_interval = 30.seconds - self.reactive_cache_lifetime = 1.minute + include PrometheusAdapter # Access to prometheus is directly through the API prop_accessor :api_url @@ -13,7 +9,7 @@ class PrometheusService < MonitoringService validates :api_url, url: true end - before_save :synchronize_service_state! + before_save :synchronize_service_state after_save :clear_reactive_cache! @@ -66,63 +62,15 @@ class PrometheusService < MonitoringService # Check we can connect to the Prometheus API def test(*args) - client.ping + Gitlab::PrometheusClient.new(prometheus_client).ping { success: true, result: 'Checked API endpoint' } rescue Gitlab::PrometheusClient::Error => err { success: false, result: err } end - def environment_metrics(environment) - with_reactive_cache(Gitlab::Prometheus::Queries::EnvironmentQuery.name, environment.id, &rename_field(:data, :metrics)) - end - - def deployment_metrics(deployment) - metrics = with_reactive_cache(Gitlab::Prometheus::Queries::DeploymentQuery.name, deployment.environment.id, deployment.id, &rename_field(:data, :metrics)) - metrics&.merge(deployment_time: deployment.created_at.to_i) || {} - end - - def additional_environment_metrics(environment) - with_reactive_cache(Gitlab::Prometheus::Queries::AdditionalMetricsEnvironmentQuery.name, environment.id, &:itself) - end - - def additional_deployment_metrics(deployment) - with_reactive_cache(Gitlab::Prometheus::Queries::AdditionalMetricsDeploymentQuery.name, deployment.environment.id, deployment.id, &:itself) - end - - def matched_metrics - with_reactive_cache(Gitlab::Prometheus::Queries::MatchedMetricsQuery.name, &:itself) - end - - # Cache metrics for specific environment - def calculate_reactive_cache(query_class_name, *args) - return unless active? && project && !project.pending_delete? - - environment_id = args.first - client = client(environment_id) - - data = Kernel.const_get(query_class_name).new(client).query(*args) - { - success: true, - data: data, - last_update: Time.now.utc - } - rescue Gitlab::PrometheusClient::Error => err - { success: false, result: err.message } - end - - def client(environment_id = nil) - if manual_configuration? - Gitlab::PrometheusClient.new(RestClient::Resource.new(api_url)) - else - cluster = cluster_with_prometheus(environment_id) - raise Gitlab::PrometheusClient::Error, "couldn't find cluster with Prometheus installed" unless cluster - - rest_client = client_from_cluster(cluster) - raise Gitlab::PrometheusClient::Error, "couldn't create proxy Prometheus client" unless rest_client - - Gitlab::PrometheusClient.new(rest_client) - end + def prometheus_client + RestClient::Resource.new(api_url) if api_url && manual_configuration? && active? end def prometheus_installed? @@ -134,32 +82,7 @@ class PrometheusService < MonitoringService private - def cluster_with_prometheus(environment_id = nil) - clusters = if environment_id - ::Environment.find_by(id: environment_id).try do |env| - # sort results by descending order based on environment_scope being longer - # thus more closely matching environment slug - project.clusters.enabled.for_environment(env).sort_by { |c| c.environment_scope&.length }.reverse! - end - else - project.clusters.enabled.for_all_environments - end - - clusters&.detect { |cluster| cluster.application_prometheus&.installed? } - end - - def client_from_cluster(cluster) - cluster.application_prometheus.proxy_client - end - - def rename_field(old_field, new_field) - -> (metrics) do - metrics[new_field] = metrics.delete(old_field) - metrics - end - end - - def synchronize_service_state! + def synchronize_service_state self.active = prometheus_installed? || manual_configuration? true diff --git a/app/services/prometheus/adapter_service.rb b/app/services/prometheus/adapter_service.rb new file mode 100644 index 00000000000..4504d2ccfe6 --- /dev/null +++ b/app/services/prometheus/adapter_service.rb @@ -0,0 +1,36 @@ +module Prometheus + class AdapterService + def initialize(project, deployment_platform = nil) + @project = project + + @deployment_platform = if deployment_platform + deployment_platform + else + project.deployment_platform + end + end + + attr_reader :deployment_platform, :project + + def prometheus_adapter + @prometheus_adapter ||= if service_prometheus_adapter.can_query? + service_prometheus_adapter + else + cluster_prometheus_adapter + end + end + + def service_prometheus_adapter + project.find_or_initialize_service('prometheus') + end + + def cluster_prometheus_adapter + return unless deployment_platform.respond_to?(:cluster) + + cluster = deployment_platform.cluster + return unless cluster.application_prometheus&.installed? + + cluster.application_prometheus + end + end +end diff --git a/doc/development/automatic_ce_ee_merge.md b/doc/development/automatic_ce_ee_merge.md index cf6314f9521..a85e5b1b1cc 100644 --- a/doc/development/automatic_ce_ee_merge.md +++ b/doc/development/automatic_ce_ee_merge.md @@ -103,6 +103,99 @@ Notes: - You can use [`git rerere`](https://git-scm.com/blog/2010/03/08/rerere.html) to avoid resolving the same conflicts multiple times. +### Cherry-picking from CE to EE + +For avoiding merge conflicts, we use a method of creating equivalent branches +for CE and EE. If the `ee-compat-check` job fails, this process is required. + +This method only requires that you have cloned both CE and EE into your computer. +If you don't have them yet, please go ahead and clone them: + +- Clone CE repo: `git clone git@gitlab.com:gitlab-org/gitlab-ce.git` +- Clone EE repo: `git clone git@gitlab.com:gitlab-org/gitlab-ee.git` + +And the only additional setup we need is to add CE as remote of EE and vice-versa: + +- Open two terminal windows, one in CE, and another one in EE: + - In EE: `git remote add ce git@gitlab.com:gitlab-org/gitlab-ce.git` + - In CE: `git remote add ee git@gitlab.com:gitlab-org/gitlab-ee.git` + +That's all setup we need, so that we can cherry-pick a commit from CE to EE, and +from EE to CE. + +Now, every time you create an MR for CE and EE: + +1. Open two terminal windows, one in CE, and another one in EE +1. In the CE terminal: + 1. Create the CE branch, e.g., `branch-example` + 1. Make your changes and push a commit (commit A) + 1. Create the CE merge request in GitLab +1. In the EE terminal: + 1. Create the EE-equivalent branch ending with `-ee`, e.g., + `git checkout -b branch-example-ee` + 1. Fetch the CE branch: `git fetch ce branch-example` + 1. Cherry-pick the commit A: `git cherry-pick commit-A-SHA` + 1. If Git prompts you to fix the conflicts, do a `git status` + to check which files contain conflicts, fix them, save the files + 1. Add the changes with `git add .` but **DO NOT commit** them + 1. Continue cherry-picking: `git cherry-pick --continue` + 1. Push to EE: `git push origin branch-example-ee` +1. Create the EE-equivalent MR and link to the CE MR from the +description "Ports [CE-MR-LINK] to EE" +1. Once all the jobs are passing in both CE and EE, you've addressed the +feedback from your own team, and got them approved, the merge requests can be merged. +1. When both MRs are ready, the EE merge request will be merged first, and the +CE-equivalent will be merged next. + +**Important notes:** + +- The commit SHA can be easily found from the GitLab UI. From a merge request, +open the tab **Commits** and click the copy icon to copy the commit SHA. +- To cherry-pick a **commit range**, such as [A > B > C > D] use: + + ```shell + git cherry-pick "oldest-commit-SHA^..newest-commit-SHA" + ``` + + For example, suppose the commit A is the oldest, and its SHA is `4f5e4018c09ed797fdf446b3752f82e46f5af502`, + and the commit D is the newest, and its SHA is `80e1c9e56783bd57bd7129828ec20b252ebc0538`. + The cherry-pick command will be: + + ```shell + git cherry-pick "4f5e4018c09ed797fdf446b3752f82e46f5af502^..80e1c9e56783bd57bd7129828ec20b252ebc0538" + ``` + +- To cherry-pick a **merge commit**, use the flag `-m 1`. For example, suppose that the +merge commit SHA is `138f5e2f20289bb376caffa0303adb0cac859ce1`: + + ```shell + git cherry-pick -m 1 138f5e2f20289bb376caffa0303adb0cac859ce1 + ``` +- To cherry-pick multiple commits, such as B and D in a range [A > B > C > D], use: + + ```shell + git cherry-pick commmit-B-SHA commit-D-SHA + ``` + + For example, suppose commit B SHA = `4f5e4018c09ed797fdf446b3752f82e46f5af502`, + and the commit D SHA = `80e1c9e56783bd57bd7129828ec20b252ebc0538`. + The cherry-pick command will be: + + ```shell + git cherry-pick 4f5e4018c09ed797fdf446b3752f82e46f5af502 80e1c9e56783bd57bd7129828ec20b252ebc0538 + ``` + + This case is particularly useful when you have a merge commit in a sequence of + commits and you want to cherry-pick all but the merge commit. + +- If you push more commits to the CE branch, you can safely repeat the procedure +to cherry-pick them to the EE-equivalent branch. You can do that as many times as +necessary, using the same CE and EE branches. +- If you submitted the merge request to the CE repo and the `ee-compat-check` job passed, +you are not required to submit the EE-equivalent MR, but it's still recommended. If the +job failed, you are required to submit the EE MR so that you can fix the conflicts in EE +before merging your changes into CE. + --- [Return to Development documentation](README.md) diff --git a/doc/development/writing_documentation.md b/doc/development/writing_documentation.md index 403c9d08752..40c21e5355c 100644 --- a/doc/development/writing_documentation.md +++ b/doc/development/writing_documentation.md @@ -19,7 +19,7 @@ The one responsible for writing the first piece of documentation is the develope wrote the code. It's the job of the Product Manager to ensure all features are shipped with its docs, whether is a small or big change. At the pace GitLab evolves, this is the only way to keep the docs up-to-date. If you have any questions about it, -please ask a Technical Writer. Otherwise, when your content is ready, assign one of +ask a Technical Writer. Otherwise, when your content is ready, assign one of them to review it for you. We use the [monthly release blog post](https://about.gitlab.com/handbook/marketing/blog/release-posts/#monthly-releases) as a changelog checklist to ensure everything @@ -27,6 +27,8 @@ is documented. Whenever you submit a merge request for the documentation, use the documentation MR description template. +Please check the [documentation workflow](https://about.gitlab.com/handbook/product/technical-writing/workflow/) before getting started. + ### Documentation directory structure The documentation is structured based on the GitLab UI structure itself, @@ -40,7 +42,7 @@ all docs should be linked. Every new document should be cross-linked to its rela The directories `/workflow/`, `/gitlab-basics/`, `/university/`, and `/articles/` have been deprecated and the majority their docs have been moved to their correct location -in small iterations. Please don't create new docs in these folders. +in small iterations. Don't create new docs in these folders. To move a document from its location to another directory, read the section [changing document location](doc_styleguide.md#changing-document-location) of the doc style guide. @@ -116,6 +118,49 @@ choices: If your branch name matches any of the above, it will run only the docs tests. If it doesn't, the whole test suite will run (including docs). +### Merge requests for GitLab documentation + +Before getting started, make sure you read the introductory section +"[contributing to docs](#contributing-to-docs)" above and the +[tech writing workflow](https://about.gitlab.com/handbook/product/technical-writing/workflow/) +for GitLab Team members. + +- Use the current [merge request description template](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/.gitlab/merge_request_templates/Documentation.md) +- Use the correct [branch name](#branch-naming) +- Label the MR `Documentation` +- Assign the correct milestone (see note below) + + +NOTE: **Note:** +If the release version you want to add the documentation to has already been +frozen or released, use the label `Pick into X.Y` to get it merged into +the correct release. Avoid picking into a past release as much as you can, as +it increases the work of the release managers. + +#### Cherry-picking from CE to EE + +As we have the `master` branch of CE merged into EE once a day, it's common to +run into merge conflicts. To avoid them, we [test for merge conflicts against EE](#testing) +with the `ee-compat-check` job, and use the following method of creating equivalent +branches for CE and EE. + +Follow this [method for cherry-picking from CE to EE](automatic_ce_ee_merge.md#cherry-picking-from-ce-to-ee), with a few adjustments: + +- Create the [CE branch](#branch-naming) starting with `docs-`, + e.g.: `git checkout -b docs-example` +- Create the EE-equivalent branch ending with `-ee`, e.g., + `git checkout -b docs-example-ee` +- Once all the jobs are passing in CE and EE, and you've addressed the +feedback from your own team, assign the CE MR to a technical writer for review +- When both MRs are ready, the EE merge request will be merged first, and the +CE-equivalent will be merged next. +- Note that the review will occur only in the CE MR, as the EE MR +contains the same commits as the CE MR. +- If you have a few more changes that apply to the EE-version only, you can submit +a couple more commits to the EE branch, but ask the reviewer to review the EE merge request +additionally to the CE MR. If there are many EE-only changes though, start a new MR +to EE only. + ### Previewing the changes live If you want to preview the doc changes of your merge request live, you can use diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 8c02972b421..ead1bb7957b 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -6,6 +6,32 @@ module API helpers ::Gitlab::IssuableMetadata + # EE::API::MergeRequests would override the following helpers + helpers do + params :optional_params_ee do + end + + params :merge_params_ee do + end + + def update_merge_request_ee(merge_request) + end + end + + def self.update_params_at_least_one_of + %i[ + assignee_id + description + labels + milestone_id + remove_source_branch + state_event + target_branch + title + discussion_locked + ] + end + helpers do def find_merge_requests(args = {}) args = declared_params.merge(args) @@ -31,6 +57,12 @@ module API mr.all_pipelines end + def check_sha_param!(params, merge_request) + if params[:sha] && merge_request.diff_head_sha != params[:sha] + render_api_error!("SHA does not match HEAD of source branch: #{merge_request.diff_head_sha}", 409) + end + end + params :merge_requests_params do optional :state, type: String, values: %w[opened closed merged all], default: 'all', desc: 'Return opened, closed, merged, or all merge requests' @@ -106,16 +138,14 @@ module API render_api_error!(errors, 400) end - params :optional_params_ce do + params :optional_params do optional :description, type: String, desc: 'The description of the merge request' optional :assignee_id, type: Integer, desc: 'The ID of a user to assign the merge request' optional :milestone_id, type: Integer, desc: 'The ID of a milestone to assign the merge request' optional :labels, type: String, desc: 'Comma-separated list of label names' optional :remove_source_branch, type: Boolean, desc: 'Remove source branch when merging' - end - params :optional_params do - use :optional_params_ce + use :optional_params_ee end end @@ -240,18 +270,6 @@ module API success Entities::MergeRequest end params do - # CE - at_least_one_of_ce = [ - :assignee_id, - :description, - :labels, - :milestone_id, - :remove_source_branch, - :state_event, - :target_branch, - :title, - :discussion_locked - ] optional :title, type: String, allow_blank: false, desc: 'The title of the merge request' optional :target_branch, type: String, allow_blank: false, desc: 'The target branch' optional :state_event, type: String, values: %w[close reopen], @@ -259,7 +277,7 @@ module API optional :discussion_locked, type: Boolean, desc: 'Whether the MR discussion is locked' use :optional_params - at_least_one_of(*at_least_one_of_ce) + at_least_one_of(*::API::MergeRequests.update_params_at_least_one_of) end put ':id/merge_requests/:merge_request_iid' do Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42318') @@ -282,13 +300,14 @@ module API success Entities::MergeRequest end params do - # CE optional :merge_commit_message, type: String, desc: 'Custom merge commit message' optional :should_remove_source_branch, type: Boolean, desc: 'When true, the source branch will be deleted if possible' optional :merge_when_pipeline_succeeds, type: Boolean, desc: 'When true, this merge request will be merged when the pipeline succeeds' optional :sha, type: String, desc: 'When present, must have the HEAD SHA of the source branch' + + use :merge_params_ee end put ':id/merge_requests/:merge_request_iid/merge' do Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab-ce/issues/42317') @@ -304,9 +323,9 @@ module API render_api_error!('Branch cannot be merged', 406) unless merge_request.mergeable?(skip_ci_check: merge_when_pipeline_succeeds) - if params[:sha] && merge_request.diff_head_sha != params[:sha] - render_api_error!("SHA does not match HEAD of source branch: #{merge_request.diff_head_sha}", 409) - end + check_sha_param!(params, merge_request) + + update_merge_request_ee(merge_request) merge_params = { commit_message: params[:merge_commit_message], diff --git a/lib/gitlab/prometheus/additional_metrics_parser.rb b/lib/gitlab/prometheus/additional_metrics_parser.rb index cb95daf2260..bb1172f82a1 100644 --- a/lib/gitlab/prometheus/additional_metrics_parser.rb +++ b/lib/gitlab/prometheus/additional_metrics_parser.rb @@ -1,10 +1,12 @@ module Gitlab module Prometheus module AdditionalMetricsParser + CONFIG_ROOT = 'config/prometheus'.freeze + MUTEX = Mutex.new extend self - def load_groups_from_yaml - additional_metrics_raw.map(&method(:group_from_entry)) + def load_groups_from_yaml(file_name = 'additional_metrics.yml') + yaml_metrics_raw(file_name).map(&method(:group_from_entry)) end private @@ -22,13 +24,20 @@ module Gitlab MetricGroup.new(entry).tap(&method(:validate!)) end - def additional_metrics_raw - load_yaml_file&.map(&:deep_symbolize_keys).freeze + def yaml_metrics_raw(file_name) + load_yaml_file(file_name)&.map(&:deep_symbolize_keys).freeze end - def load_yaml_file - @loaded_yaml_file ||= YAML.load_file(Rails.root.join('config/prometheus/additional_metrics.yml')) + # rubocop:disable Gitlab/ModuleWithInstanceVariables + def load_yaml_file(file_name) + return YAML.load_file(Rails.root.join(CONFIG_ROOT, file_name)) if Rails.env.development? + + MUTEX.synchronize do + @loaded_yaml_cache ||= {} + @loaded_yaml_cache[file_name] ||= YAML.load_file(Rails.root.join(CONFIG_ROOT, file_name)) + end end + # rubocop:enable Gitlab/ModuleWithInstanceVariables end end end diff --git a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb index 972ab75d1d5..e677ec84cd4 100644 --- a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb +++ b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb @@ -4,7 +4,7 @@ module Gitlab class AdditionalMetricsDeploymentQuery < BaseQuery include QueryAdditionalMetrics - def query(environment_id, deployment_id) + def query(deployment_id) Deployment.find_by(id: deployment_id).try do |deployment| query_metrics( deployment.project, diff --git a/lib/gitlab/prometheus/queries/base_query.rb b/lib/gitlab/prometheus/queries/base_query.rb index c60828165bd..29cab6e9c15 100644 --- a/lib/gitlab/prometheus/queries/base_query.rb +++ b/lib/gitlab/prometheus/queries/base_query.rb @@ -20,6 +20,10 @@ module Gitlab def query(*args) raise NotImplementedError end + + def self.transform_reactive_result(result) + result + end end end end diff --git a/lib/gitlab/prometheus/queries/deployment_query.rb b/lib/gitlab/prometheus/queries/deployment_query.rb index 6e6da593178..c2626581897 100644 --- a/lib/gitlab/prometheus/queries/deployment_query.rb +++ b/lib/gitlab/prometheus/queries/deployment_query.rb @@ -2,7 +2,7 @@ module Gitlab module Prometheus module Queries class DeploymentQuery < BaseQuery - def query(environment_id, deployment_id) + def query(deployment_id) Deployment.find_by(id: deployment_id).try do |deployment| environment_slug = deployment.environment.slug @@ -25,6 +25,11 @@ module Gitlab } end end + + def self.transform_reactive_result(result) + result[:metrics] = result.delete :data + result + end end end end diff --git a/lib/gitlab/prometheus/queries/environment_query.rb b/lib/gitlab/prometheus/queries/environment_query.rb index 1d17d3cfd56..b62910c8de6 100644 --- a/lib/gitlab/prometheus/queries/environment_query.rb +++ b/lib/gitlab/prometheus/queries/environment_query.rb @@ -19,6 +19,11 @@ module Gitlab } end end + + def self.transform_reactive_result(result) + result[:metrics] = result.delete :data + result + end end end end diff --git a/lib/gitlab/prometheus/queries/matched_metrics_query.rb b/lib/gitlab/prometheus/queries/matched_metric_query.rb index 5710ad47c1a..d920e9a749f 100644 --- a/lib/gitlab/prometheus/queries/matched_metrics_query.rb +++ b/lib/gitlab/prometheus/queries/matched_metric_query.rb @@ -1,7 +1,7 @@ module Gitlab module Prometheus module Queries - class MatchedMetricsQuery < BaseQuery + class MatchedMetricQuery < BaseQuery MAX_QUERY_ITEMS = 40.freeze def query diff --git a/lib/gitlab/prometheus/queries/query_additional_metrics.rb b/lib/gitlab/prometheus/queries/query_additional_metrics.rb index 0c280dc9a3c..aad76e335af 100644 --- a/lib/gitlab/prometheus/queries/query_additional_metrics.rb +++ b/lib/gitlab/prometheus/queries/query_additional_metrics.rb @@ -3,9 +3,16 @@ module Gitlab module Queries module QueryAdditionalMetrics def query_metrics(project, query_context) + matched_metrics(project).map(&query_group(query_context)) + .select(&method(:group_with_any_metrics)) + end + + protected + + def query_group(query_context) query_processor = method(:process_query).curry[query_context] - groups = matched_metrics(project).map do |group| + lambda do |group| metrics = group.metrics.map do |metric| { title: metric.title, @@ -21,8 +28,6 @@ module Gitlab metrics: metrics.select(&method(:metric_with_any_queries)) } end - - groups.select(&method(:group_with_any_metrics)) end private @@ -72,12 +77,17 @@ module Gitlab end def common_query_context(environment, timeframe_start:, timeframe_end:) - { - timeframe_start: timeframe_start, - timeframe_end: timeframe_end, + base_query_context(timeframe_start, timeframe_end).merge({ ci_environment_slug: environment.slug, kube_namespace: environment.project.deployment_platform&.actual_namespace || '', environment_filter: %{container_name!="POD",environment="#{environment.slug}"} + }) + end + + def base_query_context(timeframe_start, timeframe_end) + { + timeframe_start: timeframe_start, + timeframe_end: timeframe_end } end end diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb index 659021c9ac9..b66253a10e0 100644 --- a/lib/gitlab/prometheus_client.rb +++ b/lib/gitlab/prometheus_client.rb @@ -57,7 +57,11 @@ module Gitlab rescue OpenSSL::SSL::SSLError raise PrometheusClient::Error, "#{rest_client.url} contains invalid SSL data" rescue RestClient::ExceptionWithResponse => ex - handle_exception_response(ex.response) + if ex.response + handle_exception_response(ex.response) + else + raise PrometheusClient::Error, "Network connection error" + end rescue RestClient::Exception raise PrometheusClient::Error, "Network connection error" end diff --git a/spec/controllers/projects/deployments_controller_spec.rb b/spec/controllers/projects/deployments_controller_spec.rb index 73e7921fab7..6c67dfde63a 100644 --- a/spec/controllers/projects/deployments_controller_spec.rb +++ b/spec/controllers/projects/deployments_controller_spec.rb @@ -129,10 +129,10 @@ describe Projects::DeploymentsController do end context 'when metrics are enabled' do - let(:prometheus_service) { double('prometheus_service') } + let(:prometheus_adapter) { double('prometheus_adapter', can_query?: true) } before do - allow(deployment.project).to receive(:prometheus_service).and_return(prometheus_service) + allow(deployment).to receive(:prometheus_adapter).and_return(prometheus_adapter) end context 'when environment has no metrics' do diff --git a/spec/controllers/projects/prometheus/metrics_controller_spec.rb b/spec/controllers/projects/prometheus/metrics_controller_spec.rb index f17f819feee..b2b245dba90 100644 --- a/spec/controllers/projects/prometheus/metrics_controller_spec.rb +++ b/spec/controllers/projects/prometheus/metrics_controller_spec.rb @@ -4,21 +4,22 @@ describe Projects::Prometheus::MetricsController do let(:user) { create(:user) } let(:project) { create(:project) } - let(:prometheus_service) { double('prometheus_service') } + let(:prometheus_adapter) { double('prometheus_adapter', can_query?: true) } before do - allow(controller).to receive(:project).and_return(project) - allow(project).to receive(:find_or_initialize_service).with('prometheus').and_return(prometheus_service) - project.add_master(user) sign_in(user) end describe 'GET #active_common' do + before do + allow(controller).to receive(:prometheus_adapter).and_return(prometheus_adapter) + end + context 'when prometheus metrics are enabled' do context 'when data is not present' do before do - allow(prometheus_service).to receive(:matched_metrics).and_return({}) + allow(prometheus_adapter).to receive(:query).with(:matched_metrics).and_return({}) end it 'returns no content response' do @@ -32,7 +33,7 @@ describe Projects::Prometheus::MetricsController do let(:sample_response) { { some_data: 1 } } before do - allow(prometheus_service).to receive(:matched_metrics).and_return(sample_response) + allow(prometheus_adapter).to receive(:query).with(:matched_metrics).and_return(sample_response) end it 'returns no content response' do @@ -53,6 +54,18 @@ describe Projects::Prometheus::MetricsController do end end + describe '#prometheus_adapter' do + before do + allow(controller).to receive(:project).and_return(project) + end + + it 'calls prometheus adapter service' do + expect_any_instance_of(::Prometheus::AdapterService).to receive(:prometheus_adapter) + + subject.__send__(:prometheus_adapter) + end + end + def project_params(opts = {}) opts.reverse_merge(namespace_id: project.namespace, project_id: project) end diff --git a/spec/lib/gitlab/prometheus/queries/additional_metrics_deployment_query_spec.rb b/spec/lib/gitlab/prometheus/queries/additional_metrics_deployment_query_spec.rb index 0697cb2def6..c7169717fc1 100644 --- a/spec/lib/gitlab/prometheus/queries/additional_metrics_deployment_query_spec.rb +++ b/spec/lib/gitlab/prometheus/queries/additional_metrics_deployment_query_spec.rb @@ -7,7 +7,7 @@ describe Gitlab::Prometheus::Queries::AdditionalMetricsDeploymentQuery do include_examples 'additional metrics query' do let(:deployment) { create(:deployment, environment: environment) } - let(:query_params) { [environment.id, deployment.id] } + let(:query_params) { [deployment.id] } it 'queries using specific time' do expect(client).to receive(:query_range).with(anything, diff --git a/spec/lib/gitlab/prometheus/queries/deployment_query_spec.rb b/spec/lib/gitlab/prometheus/queries/deployment_query_spec.rb index 84dc31d9732..ffe3ad85baa 100644 --- a/spec/lib/gitlab/prometheus/queries/deployment_query_spec.rb +++ b/spec/lib/gitlab/prometheus/queries/deployment_query_spec.rb @@ -31,7 +31,7 @@ describe Gitlab::Prometheus::Queries::DeploymentQuery do expect(client).to receive(:query).with('avg(rate(container_cpu_usage_seconds_total{container_name!="POD",environment="environment-slug"}[30m])) * 100', time: stop_time) - expect(subject.query(environment.id, deployment.id)).to eq(memory_values: nil, memory_before: nil, memory_after: nil, - cpu_values: nil, cpu_before: nil, cpu_after: nil) + expect(subject.query(deployment.id)).to eq(memory_values: nil, memory_before: nil, memory_after: nil, + cpu_values: nil, cpu_before: nil, cpu_after: nil) end end diff --git a/spec/lib/gitlab/prometheus/queries/matched_metrics_query_spec.rb b/spec/lib/gitlab/prometheus/queries/matched_metric_query_spec.rb index c95719eff1d..420218a695a 100644 --- a/spec/lib/gitlab/prometheus/queries/matched_metrics_query_spec.rb +++ b/spec/lib/gitlab/prometheus/queries/matched_metric_query_spec.rb @@ -1,6 +1,6 @@ require 'spec_helper' -describe Gitlab::Prometheus::Queries::MatchedMetricsQuery do +describe Gitlab::Prometheus::Queries::MatchedMetricQuery do include Prometheus::MetricBuilders let(:metric_group_class) { Gitlab::Prometheus::MetricGroup } diff --git a/spec/models/clusters/applications/prometheus_spec.rb b/spec/models/clusters/applications/prometheus_spec.rb index df8a508e021..2905b58066b 100644 --- a/spec/models/clusters/applications/prometheus_spec.rb +++ b/spec/models/clusters/applications/prometheus_spec.rb @@ -22,11 +22,11 @@ describe Clusters::Applications::Prometheus do end end - describe '#proxy_client' do + describe '#prometheus_client' do context 'cluster is nil' do it 'returns nil' do expect(subject.cluster).to be_nil - expect(subject.proxy_client).to be_nil + expect(subject.prometheus_client).to be_nil end end @@ -35,7 +35,7 @@ describe Clusters::Applications::Prometheus do subject { create(:clusters_applications_prometheus, cluster: cluster) } it 'returns nil' do - expect(subject.proxy_client).to be_nil + expect(subject.prometheus_client).to be_nil end end @@ -63,15 +63,15 @@ describe Clusters::Applications::Prometheus do end it 'creates proxy prometheus rest client' do - expect(subject.proxy_client).to be_instance_of(RestClient::Resource) + expect(subject.prometheus_client).to be_instance_of(RestClient::Resource) end it 'creates proper url' do - expect(subject.proxy_client.url).to eq('http://example.com/api/v1/proxy/namespaces/gitlab-managed-apps/service/prometheus-prometheus-server:80') + expect(subject.prometheus_client.url).to eq('http://example.com/api/v1/proxy/namespaces/gitlab-managed-apps/service/prometheus-prometheus-server:80') end it 'copies options and headers from kube client to proxy client' do - expect(subject.proxy_client.options).to eq(kube_client.rest_client.options.merge(headers: kube_client.headers)) + expect(subject.prometheus_client.options).to eq(kube_client.rest_client.options.merge(headers: kube_client.headers)) end end end diff --git a/spec/models/concerns/prometheus_adapter_spec.rb b/spec/models/concerns/prometheus_adapter_spec.rb new file mode 100644 index 00000000000..f4b9c57e71a --- /dev/null +++ b/spec/models/concerns/prometheus_adapter_spec.rb @@ -0,0 +1,121 @@ +require 'spec_helper' + +describe PrometheusAdapter, :use_clean_rails_memory_store_caching do + include PrometheusHelpers + include ReactiveCachingHelpers + + class TestClass + include PrometheusAdapter + end + + let(:project) { create(:prometheus_project) } + let(:service) { project.prometheus_service } + + let(:described_class) { TestClass } + let(:environment_query) { Gitlab::Prometheus::Queries::EnvironmentQuery } + + describe '#query' do + describe 'environment' do + let(:environment) { build_stubbed(:environment, slug: 'env-slug') } + + around do |example| + Timecop.freeze { example.run } + end + + context 'with valid data' do + subject { service.query(:environment, environment) } + + before do + stub_reactive_cache(service, prometheus_data, environment_query, environment.id) + end + + it 'returns reactive data' do + is_expected.to eq(prometheus_metrics_data) + end + end + end + + describe 'matched_metrics' do + let(:matched_metrics_query) { Gitlab::Prometheus::Queries::MatchedMetricQuery } + let(:prometheus_client_wrapper) { double(:prometheus_client_wrapper, label_values: nil) } + + context 'with valid data' do + subject { service.query(:matched_metrics) } + + before do + allow(service).to receive(:prometheus_client_wrapper).and_return(prometheus_client_wrapper) + synchronous_reactive_cache(service) + end + + it 'returns reactive data' do + expect(subject[:success]).to be_truthy + expect(subject[:data]).to eq([]) + end + end + end + + describe 'deployment' do + let(:deployment) { build_stubbed(:deployment) } + let(:deployment_query) { Gitlab::Prometheus::Queries::DeploymentQuery } + + around do |example| + Timecop.freeze { example.run } + end + + context 'with valid data' do + subject { service.query(:deployment, deployment) } + + before do + stub_reactive_cache(service, prometheus_data, deployment_query, deployment.id) + end + + it 'returns reactive data' do + expect(subject).to eq(prometheus_metrics_data) + end + end + end + end + + describe '#calculate_reactive_cache' do + let(:environment) { create(:environment, slug: 'env-slug') } + before do + service.manual_configuration = true + service.active = true + end + + subject do + service.calculate_reactive_cache(environment_query.name, environment.id) + end + + around do |example| + Timecop.freeze { example.run } + end + + context 'when service is inactive' do + before do + service.active = false + end + + it { is_expected.to be_nil } + end + + context 'when Prometheus responds with valid data' do + before do + stub_all_prometheus_requests(environment.slug) + end + + it { expect(subject.to_json).to eq(prometheus_data.to_json) } + it { expect(subject.to_json).to eq(prometheus_data.to_json) } + end + + [404, 500].each do |status| + context "when Prometheus responds with #{status}" do + before do + stub_all_prometheus_requests(environment.slug, status: status, body: "QUERY FAILED!") + end + + it { is_expected.to eq(success: false, result: %(#{status} - "QUERY FAILED!")) } + end + end + end +end diff --git a/spec/models/deployment_spec.rb b/spec/models/deployment_spec.rb index ba8aa13d5ad..ac30cd27e0c 100644 --- a/spec/models/deployment_spec.rb +++ b/spec/models/deployment_spec.rb @@ -64,6 +64,7 @@ describe Deployment do describe '#metrics' do let(:deployment) { create(:deployment) } + let(:prometheus_adapter) { double('prometheus_adapter', can_query?: true) } subject { deployment.metrics } @@ -76,17 +77,16 @@ describe Deployment do { success: true, metrics: {}, - last_update: 42, - deployment_time: 1494408956 + last_update: 42 } end before do - allow(deployment.project).to receive_message_chain(:monitoring_service, :deployment_metrics) - .with(any_args).and_return(simple_metrics) + allow(deployment).to receive(:prometheus_adapter).and_return(prometheus_adapter) + allow(prometheus_adapter).to receive(:query).with(:deployment, deployment).and_return(simple_metrics) end - it { is_expected.to eq(simple_metrics) } + it { is_expected.to eq(simple_metrics.merge({ deployment_time: deployment.created_at.to_i })) } end end @@ -109,11 +109,11 @@ describe Deployment do } end - let(:prometheus_service) { double('prometheus_service') } + let(:prometheus_adapter) { double('prometheus_adapter', can_query?: true) } before do - allow(project).to receive(:prometheus_service).and_return(prometheus_service) - allow(prometheus_service).to receive(:additional_deployment_metrics).and_return(simple_metrics) + allow(deployment).to receive(:prometheus_adapter).and_return(prometheus_adapter) + allow(prometheus_adapter).to receive(:query).with(:additional_metrics_deployment, deployment).and_return(simple_metrics) end it { is_expected.to eq(simple_metrics.merge({ deployment_time: deployment.created_at.to_i })) } diff --git a/spec/models/environment_spec.rb b/spec/models/environment_spec.rb index 6f24a039998..ceb570ac777 100644 --- a/spec/models/environment_spec.rb +++ b/spec/models/environment_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe Environment do - set(:project) { create(:project) } + let(:project) { create(:project) } subject(:environment) { create(:environment, project: project) } it { is_expected.to belong_to(:project) } @@ -452,8 +452,8 @@ describe Environment do end it 'returns the metrics from the deployment service' do - expect(project.monitoring_service) - .to receive(:environment_metrics).with(environment) + expect(environment.prometheus_adapter) + .to receive(:query).with(:environment, environment) .and_return(:fake_metrics) is_expected.to eq(:fake_metrics) @@ -508,12 +508,12 @@ describe Environment do context 'when the environment has additional metrics' do before do - allow(environment).to receive(:has_additional_metrics?).and_return(true) + allow(environment).to receive(:has_metrics?).and_return(true) end it 'returns the additional metrics from the deployment service' do - expect(project.prometheus_service).to receive(:additional_environment_metrics) - .with(environment) + expect(environment.prometheus_adapter).to receive(:query) + .with(:additional_metrics_environment, environment) .and_return(:fake_metrics) is_expected.to eq(:fake_metrics) @@ -522,46 +522,13 @@ describe Environment do context 'when the environment does not have metrics' do before do - allow(environment).to receive(:has_additional_metrics?).and_return(false) + allow(environment).to receive(:has_metrics?).and_return(false) end it { is_expected.to be_nil } end end - describe '#has_additional_metrics??' do - subject { environment.has_additional_metrics? } - - context 'when the enviroment is available' do - context 'with a deployment service' do - let(:project) { create(:prometheus_project) } - - context 'and a deployment' do - let!(:deployment) { create(:deployment, environment: environment) } - it { is_expected.to be_truthy } - end - - context 'but no deployments' do - it { is_expected.to be_falsy } - end - end - - context 'without a monitoring service' do - it { is_expected.to be_falsy } - end - end - - context 'when the environment is unavailable' do - let(:project) { create(:prometheus_project) } - - before do - environment.stop - end - - it { is_expected.to be_falsy } - end - end - describe '#slug' do it "is automatically generated" do expect(environment.slug).not_to be_nil @@ -654,4 +621,12 @@ describe Environment do end end end + + describe '#prometheus_adapter' do + it 'calls prometheus adapter service' do + expect_any_instance_of(Prometheus::AdapterService).to receive(:prometheus_adapter) + + subject.prometheus_adapter + end + end end diff --git a/spec/models/project_services/prometheus_service_spec.rb b/spec/models/project_services/prometheus_service_spec.rb index 6693e5783a5..7afb1b4a8e3 100644 --- a/spec/models/project_services/prometheus_service_spec.rb +++ b/spec/models/project_services/prometheus_service_spec.rb @@ -6,7 +6,6 @@ describe PrometheusService, :use_clean_rails_memory_store_caching do let(:project) { create(:prometheus_project) } let(:service) { project.prometheus_service } - let(:environment_query) { Gitlab::Prometheus::Queries::EnvironmentQuery } describe "Associations" do it { is_expected.to belong_to :project } @@ -55,197 +54,31 @@ describe PrometheusService, :use_clean_rails_memory_store_caching do end end - describe '#environment_metrics' do - let(:environment) { build_stubbed(:environment, slug: 'env-slug') } - - around do |example| - Timecop.freeze { example.run } - end - - context 'with valid data' do - subject { service.environment_metrics(environment) } - - before do - stub_reactive_cache(service, prometheus_data, environment_query, environment.id) - end - - it 'returns reactive data' do - is_expected.to eq(prometheus_metrics_data) - end - end - end - - describe '#matched_metrics' do - let(:matched_metrics_query) { Gitlab::Prometheus::Queries::MatchedMetricsQuery } - let(:client) { double(:client, label_values: nil) } - - context 'with valid data' do - subject { service.matched_metrics } - - before do - allow(service).to receive(:client).and_return(client) - synchronous_reactive_cache(service) - end - - it 'returns reactive data' do - expect(subject[:success]).to be_truthy - expect(subject[:data]).to eq([]) - end - end - end - - describe '#deployment_metrics' do - let(:deployment) { build_stubbed(:deployment) } - let(:deployment_query) { Gitlab::Prometheus::Queries::DeploymentQuery } - - around do |example| - Timecop.freeze { example.run } - end - - context 'with valid data' do - subject { service.deployment_metrics(deployment) } - let(:fake_deployment_time) { 10 } - - before do - stub_reactive_cache(service, prometheus_data, deployment_query, deployment.environment.id, deployment.id) - end - - it 'returns reactive data' do - expect(deployment).to receive(:created_at).and_return(fake_deployment_time) - - expect(subject).to eq(prometheus_metrics_data.merge(deployment_time: fake_deployment_time)) - end - end - end - - describe '#calculate_reactive_cache' do - let(:environment) { create(:environment, slug: 'env-slug') } - before do - service.manual_configuration = true - service.active = true - end - - subject do - service.calculate_reactive_cache(environment_query.name, environment.id) - end - - around do |example| - Timecop.freeze { example.run } - end - - context 'when service is inactive' do - before do - service.active = false - end - - it { is_expected.to be_nil } - end - - context 'when Prometheus responds with valid data' do - before do - stub_all_prometheus_requests(environment.slug) - end - - it { expect(subject.to_json).to eq(prometheus_data.to_json) } - it { expect(subject.to_json).to eq(prometheus_data.to_json) } - end - - [404, 500].each do |status| - context "when Prometheus responds with #{status}" do - before do - stub_all_prometheus_requests(environment.slug, status: status, body: "QUERY FAILED!") - end - - it { is_expected.to eq(success: false, result: %(#{status} - "QUERY FAILED!")) } - end - end - end - - describe '#client' do + describe '#prometheus_client' do context 'manual configuration is enabled' do let(:api_url) { 'http://some_url' } + before do + subject.active = true subject.manual_configuration = true subject.api_url = api_url end - it 'returns simple rest client from api_url' do - expect(subject.client).to be_instance_of(Gitlab::PrometheusClient) - expect(subject.client.rest_client.url).to eq(api_url) + it 'returns rest client from api_url' do + expect(subject.prometheus_client.url).to eq(api_url) end end context 'manual configuration is disabled' do - let!(:cluster_for_all) { create(:cluster, environment_scope: '*', projects: [project]) } - let!(:cluster_for_dev) { create(:cluster, environment_scope: 'dev', projects: [project]) } - - let!(:prometheus_for_dev) { create(:clusters_applications_prometheus, :installed, cluster: cluster_for_dev) } - let(:proxy_client) { double('proxy_client') } + let(:api_url) { 'http://some_url' } before do - service.manual_configuration = false - end - - context 'with cluster for all environments with prometheus installed' do - let!(:prometheus_for_all) { create(:clusters_applications_prometheus, :installed, cluster: cluster_for_all) } - - context 'without environment supplied' do - it 'returns client handling all environments' do - expect(service).to receive(:client_from_cluster).with(cluster_for_all).and_return(proxy_client).twice - - expect(service.client).to be_instance_of(Gitlab::PrometheusClient) - expect(service.client.rest_client).to eq(proxy_client) - end - end - - context 'with dev environment supplied' do - let!(:environment) { create(:environment, project: project, name: 'dev') } - - it 'returns dev cluster client' do - expect(service).to receive(:client_from_cluster).with(cluster_for_dev).and_return(proxy_client).twice - - expect(service.client(environment.id)).to be_instance_of(Gitlab::PrometheusClient) - expect(service.client(environment.id).rest_client).to eq(proxy_client) - end - end - - context 'with prod environment supplied' do - let!(:environment) { create(:environment, project: project, name: 'prod') } - - it 'returns dev cluster client' do - expect(service).to receive(:client_from_cluster).with(cluster_for_all).and_return(proxy_client).twice - - expect(service.client(environment.id)).to be_instance_of(Gitlab::PrometheusClient) - expect(service.client(environment.id).rest_client).to eq(proxy_client) - end - end + subject.manual_configuration = false + subject.api_url = api_url end - context 'with cluster for all environments without prometheus installed' do - context 'without environment supplied' do - it 'raises PrometheusClient::Error because cluster was not found' do - expect { service.client }.to raise_error(Gitlab::PrometheusClient::Error, /couldn't find cluster with Prometheus installed/) - end - end - - context 'with dev environment supplied' do - let!(:environment) { create(:environment, project: project, name: 'dev') } - - it 'returns dev cluster client' do - expect(service).to receive(:client_from_cluster).with(cluster_for_dev).and_return(proxy_client).twice - - expect(service.client(environment.id)).to be_instance_of(Gitlab::PrometheusClient) - expect(service.client(environment.id).rest_client).to eq(proxy_client) - end - end - - context 'with prod environment supplied' do - let!(:environment) { create(:environment, project: project, name: 'prod') } - - it 'raises PrometheusClient::Error because cluster was not found' do - expect { service.client }.to raise_error(Gitlab::PrometheusClient::Error, /couldn't find cluster with Prometheus installed/) - end - end + it 'no client provided' do + expect(subject.prometheus_client).to be_nil end end end @@ -284,7 +117,7 @@ describe PrometheusService, :use_clean_rails_memory_store_caching do end end - describe '#synchronize_service_state! before_save callback' do + describe '#synchronize_service_state before_save callback' do context 'no clusters with prometheus are installed' do context 'when service is inactive' do before do diff --git a/spec/services/prometheus/adapter_service_spec.rb b/spec/services/prometheus/adapter_service_spec.rb new file mode 100644 index 00000000000..335fc5844aa --- /dev/null +++ b/spec/services/prometheus/adapter_service_spec.rb @@ -0,0 +1,40 @@ +require 'spec_helper' + +describe Prometheus::AdapterService do + let(:project) { create(:project) } + subject { described_class.new(project) } + + describe '#prometheus_adapter' do + let(:cluster) { create(:cluster, :provided_by_user, environment_scope: '*', projects: [project]) } + + context 'prometheus service can execute queries' do + let(:prometheus_service) { double(:prometheus_service, can_query?: true) } + + before do + allow(project).to receive(:find_or_initialize_service).with('prometheus').and_return prometheus_service + end + + it 'return prometheus service as prometheus adapter' do + expect(subject.prometheus_adapter).to eq(prometheus_service) + end + end + + context "prometheus service can't execute queries" do + let(:prometheus_service) { double(:prometheus_service, can_query?: false) } + + context 'with cluster with prometheus installed' do + let!(:prometheus) { create(:clusters_applications_prometheus, :installed, cluster: cluster) } + + it 'returns application handling all environments' do + expect(subject.prometheus_adapter).to eq(prometheus) + end + end + + context 'with cluster without prometheus installed' do + it 'returns nil' do + expect(subject.prometheus_adapter).to be_nil + end + end + end + end +end |