Merge branch 'pawel/connect_to_prometheus_through_proxy-30480' into 'master'

Deploy prometheus through kubernetes and autoconnect to cluster

Closes #30480 and #28916

See merge request gitlab-org/gitlab-ce!16182

47 files changed, 612 insertions, 118 deletions

diff --git a/app/assets/javascripts/clusters/clusters_bundle.js b/app/assets/javascripts/clusters/clusters_bundle.js
index 3d6ec37e6dd..b070a59cf15 100644
--- a/app/assets/javascripts/clusters/clusters_bundle.js
+++ b/app/assets/javascripts/clusters/clusters_bundle.js
@@ -32,6 +32,7 @@ export default class Clusters {
       installIngressPath,
       installRunnerPath,
       installPrometheusPath,
+      managePrometheusPath,
       clusterStatus,
       clusterStatusReason,
       helpPath,
@@ -40,6 +41,7 @@ export default class Clusters {
 
     this.store = new ClustersStore();
     this.store.setHelpPaths(helpPath, ingressHelpPath);
+    this.store.setManagePrometheusPath(managePrometheusPath);
     this.store.updateStatus(clusterStatus);
     this.store.updateStatusReason(clusterStatusReason);
     this.service = new ClustersService({
@@ -95,6 +97,7 @@ export default class Clusters {
             applications: this.state.applications,
             helpPath: this.state.helpPath,
             ingressHelpPath: this.state.ingressHelpPath,
+            managePrometheusPath: this.state.managePrometheusPath,
           },
         });
       },
diff --git a/app/assets/javascripts/clusters/components/application_row.vue b/app/assets/javascripts/clusters/components/application_row.vue
index c13bbcee863..50e35bbbba5 100644
--- a/app/assets/javascripts/clusters/components/application_row.vue
+++ b/app/assets/javascripts/clusters/components/application_row.vue
@@ -32,6 +32,10 @@
         type: String,
         required: false,
       },
+      manageLink: {
+        type: String,
+        required: false,
+      },
       description: {
         type: String,
         required: true,
@@ -89,6 +93,12 @@
 
         return label;
       },
+      showManageButton() {
+        return this.manageLink && this.status === APPLICATION_INSTALLED;
+      },
+      manageButtonLabel() {
+        return s__('ClusterIntegration|Manage');
+      },
       hasError() {
         return this.status === APPLICATION_ERROR ||
         this.requestStatus === REQUEST_FAILURE;
@@ -141,9 +151,21 @@
         <div v-html="description"></div>
       </div>
       <div
-        class="table-section table-button-footer section-15 section-align-top"
+        class="table-section table-button-footer section-align-top"
+        :class="{ 'section-20': showManageButton, 'section-15': !showManageButton }"
         role="gridcell"
       >
+        <div
+          v-if="showManageButton"
+          class="btn-group table-action-buttons"
+        >
+          <a
+            class="btn"
+            :href="manageLink"
+          >
+            {{ manageButtonLabel }}
+          </a>
+        </div>
         <div class="btn-group table-action-buttons">
           <loading-button
             class="js-cluster-application-install-button"
diff --git a/app/assets/javascripts/clusters/components/applications.vue b/app/assets/javascripts/clusters/components/applications.vue
index f4259700370..978881a4831 100644
--- a/app/assets/javascripts/clusters/components/applications.vue
+++ b/app/assets/javascripts/clusters/components/applications.vue
@@ -23,13 +23,19 @@
         required: false,
         default: '',
       },
+      managePrometheusPath: {
+        type: String,
+        required: false,
+        default: '',
+      },
     },
     computed: {
       generalApplicationDescription() {
         return sprintf(
-          _.escape(s__(`ClusterIntegration|Install applications on your Kubernetes cluster.
-  Read more about %{helpLink}`)),
-          {
+          _.escape(s__(
+            `ClusterIntegration|Install applications on your Kubernetes cluster.
+            Read more about %{helpLink}`,
+          )), {
             helpLink: `<a href="${this.helpPath}">
               ${_.escape(s__('ClusterIntegration|installing applications'))}
             </a>`,
@@ -96,11 +102,12 @@
       },
       prometheusDescription() {
         return sprintf(
-          _.escape(s__(`ClusterIntegration|Prometheus is an open-source monitoring system
-  with %{gitlabIntegrationLink} to monitor deployed applications.`)),
-          {
+          _.escape(s__(
+            `ClusterIntegration|Prometheus is an open-source monitoring system
+            with %{gitlabIntegrationLink} to monitor deployed applications.`,
+          )), {
             gitlabIntegrationLink: `<a href="https://docs.gitlab.com/ce/user/project/integrations/prometheus.html"
-target="_blank" rel="noopener noreferrer">
+              target="_blank" rel="noopener noreferrer">
               ${_.escape(s__('ClusterIntegration|GitLab Integration'))}</a>`,
           },
           false,
@@ -149,6 +156,7 @@ target="_blank" rel="noopener noreferrer">
           id="prometheus"
           :title="applications.prometheus.title"
           title-link="https://prometheus.io/docs/introduction/overview/"
+          :manage-link="managePrometheusPath"
           :description="prometheusDescription"
           :status="applications.prometheus.status"
           :status-reason="applications.prometheus.statusReason"
diff --git a/app/assets/javascripts/clusters/stores/clusters_store.js b/app/assets/javascripts/clusters/stores/clusters_store.js
index 49c3d184ef9..904ee5fd475 100644
--- a/app/assets/javascripts/clusters/stores/clusters_store.js
+++ b/app/assets/javascripts/clusters/stores/clusters_store.js
@@ -45,6 +45,10 @@ export default class ClusterStore {
     this.state.ingressHelpPath = ingressHelpPath;
   }
 
+  setManagePrometheusPath(managePrometheusPath) {
+    this.state.managePrometheusPath = managePrometheusPath;
+  }
+
   updateStatus(status) {
     this.state.status = status;
   }
diff --git a/app/assets/javascripts/monitoring/components/dashboard.vue b/app/assets/javascripts/monitoring/components/dashboard.vue
index 5afae93724b..031badc7026 100644
--- a/app/assets/javascripts/monitoring/components/dashboard.vue
+++ b/app/assets/javascripts/monitoring/components/dashboard.vue
@@ -27,6 +27,7 @@
         hasMetrics: convertPermissionToBoolean(metricsData.hasMetrics),
         documentationPath: metricsData.documentationPath,
         settingsPath: metricsData.settingsPath,
+        clustersPath: metricsData.clustersPath,
         tagsPath: metricsData.tagsPath,
         projectPath: metricsData.projectPath,
         metricsEndpoint: metricsData.additionalMetrics,
@@ -132,6 +133,7 @@
     :selected-state="state"
     :documentation-path="documentationPath"
     :settings-path="settingsPath"
+    :clusters-path="clustersPath"
     :empty-getting-started-svg-path="emptyGettingStartedSvgPath"
     :empty-loading-svg-path="emptyLoadingSvgPath"
     :empty-unable-to-connect-svg-path="emptyUnableToConnectSvgPath"
diff --git a/app/assets/javascripts/monitoring/components/empty_state.vue b/app/assets/javascripts/monitoring/components/empty_state.vue
index 56cd60c583b..9517b8ccb67 100644
--- a/app/assets/javascripts/monitoring/components/empty_state.vue
+++ b/app/assets/javascripts/monitoring/components/empty_state.vue
@@ -10,6 +10,11 @@
         required: false,
         default: '',
       },
+      clustersPath: {
+        type: String,
+        required: false,
+        default: '',
+      },
       selectedState: {
         type: String,
         required: true,
@@ -35,7 +40,10 @@
             title: 'Get started with performance monitoring',
             description: `Stay updated about the performance and health
               of your environment by configuring Prometheus to monitor your deployments.`,
-            buttonText: 'Configure Prometheus',
+            buttonText: 'Install Prometheus on clusters',
+            buttonPath: this.clustersPath,
+            secondaryButtonText: 'Configure existing Prometheus',
+            secondaryButtonPath: this.settingsPath,
           },
           loading: {
             svgUrl: this.emptyLoadingSvgPath,
@@ -43,6 +51,7 @@
             description: `Creating graphs uses the data from the Prometheus server.
               If this takes a long time, ensure that data is available.`,
             buttonText: 'View documentation',
+            buttonPath: this.documentationPath,
           },
           noData: {
             svgUrl: this.emptyUnableToConnectSvgPath,
@@ -50,12 +59,14 @@
             description: `You are connected to the Prometheus server, but there is currently
               no data to display.`,
             buttonText: 'Configure Prometheus',
+            buttonPath: this.settingsPath,
           },
           unableToConnect: {
             svgUrl: this.emptyUnableToConnectSvgPath,
             title: 'Unable to connect to Prometheus server',
             description: 'Ensure connectivity is available from the GitLab server to the ',
             buttonText: 'View documentation',
+            buttonPath: this.documentationPath,
           },
         },
       };
@@ -65,13 +76,6 @@
         return this.states[this.selectedState];
       },
 
-      buttonPath() {
-        if (this.selectedState === 'gettingStarted') {
-          return this.settingsPath;
-        }
-        return this.documentationPath;
-      },
-
       showButtonDescription() {
         if (this.selectedState === 'unableToConnect') return true;
         return false;
@@ -99,11 +103,21 @@
     </p>
     <div class="state-button">
       <a
+        v-if="currentState.buttonPath"
         class="btn btn-success"
-        :href="buttonPath"
+        :href="currentState.buttonPath"
       >
         {{ currentState.buttonText }}
       </a>
     </div>
+    <div class="state-button">
+      <a
+        v-if="currentState.secondaryButtonPath"
+        class="btn"
+        :href="currentState.secondaryButtonPath"
+      >
+        {{ currentState.secondaryButtonText }}
+      </a>
+    </div>
   </div>
 </template>
diff --git a/app/assets/stylesheets/pages/environments.scss b/app/assets/stylesheets/pages/environments.scss
index 794bc668562..4eba05a492d 100644
--- a/app/assets/stylesheets/pages/environments.scss
+++ b/app/assets/stylesheets/pages/environments.scss
@@ -205,7 +205,7 @@
 }
 
 .prometheus-state {
-  max-width: 430px;
+  max-width: 460px;
   margin: 10px auto;
   text-align: center;
 
@@ -213,6 +213,10 @@
     max-width: 80vw;
     margin: 0 auto;
   }
+
+  .state-button {
+    padding: $gl-padding / 2;
+  }
 }
 
 .environments-actions {
diff --git a/app/assets/stylesheets/pages/settings.scss b/app/assets/stylesheets/pages/settings.scss
index 6353482ede7..47672783d5a 100644
--- a/app/assets/stylesheets/pages/settings.scss
+++ b/app/assets/stylesheets/pages/settings.scss
@@ -135,6 +135,17 @@
   padding-top: 0;
 }
 
+.integration-settings-form {
+  .well {
+    padding: $gl-padding / 2;
+    box-shadow: none;
+  }
+
+  .svg-container {
+    max-width: 150px;
+  }
+}
+
 .token-token-container {
   #impersonation-token-token {
     width: 80%;
diff --git a/app/controllers/concerns/service_params.rb b/app/controllers/concerns/service_params.rb
index 3d61458c064..c1acb50b76c 100644
--- a/app/controllers/concerns/service_params.rb
+++ b/app/controllers/concerns/service_params.rb
@@ -32,6 +32,7 @@ module ServiceParams
     :issues_events,
     :issues_url,
     :jira_issue_transition_id,
+    :manual_configuration,
     :merge_requests_events,
     :mock_service_url,
     :namespace,
diff --git a/app/models/clusters/applications/prometheus.rb b/app/models/clusters/applications/prometheus.rb
index 9b0787ee6ca..aa22e9d5d58 100644
--- a/app/models/clusters/applications/prometheus.rb
+++ b/app/models/clusters/applications/prometheus.rb
@@ -10,10 +10,26 @@ module Clusters
 
       default_value_for :version, VERSION
 
+      state_machine :status do
+        after_transition any => [:installed] do |application|
+          application.cluster.projects.each do |project|
+            project.find_or_initialize_service('prometheus').update(active: true)
+          end
+        end
+      end
+
       def chart
         'stable/prometheus'
       end
 
+      def service_name
+        'prometheus-prometheus-server'
+      end
+
+      def service_port
+        80
+      end
+
       def chart_values_file
         "#{Rails.root}/vendor/#{name}/values.yaml"
       end
@@ -21,6 +37,22 @@ module Clusters
       def install_command
         Gitlab::Kubernetes::Helm::InstallCommand.new(name, chart: chart, chart_values_file: chart_values_file)
       end
+
+      def proxy_client
+        return unless kube_client
+
+        proxy_url = kube_client.proxy_url('service', service_name, service_port, Gitlab::Kubernetes::Helm::NAMESPACE)
+
+        # ensures headers containing auth data are appended to original k8s client options
+        options = kube_client.rest_client.options.merge(headers: kube_client.headers)
+        RestClient::Resource.new(proxy_url, options)
+      end
+
+      private
+
+      def kube_client
+        cluster&.kubeclient
+      end
     end
   end
 end
diff --git a/app/models/clusters/cluster.rb b/app/models/clusters/cluster.rb
index 5ecbd4cbceb..8678f70f78c 100644
--- a/app/models/clusters/cluster.rb
+++ b/app/models/clusters/cluster.rb
@@ -49,6 +49,9 @@ module Clusters
     scope :enabled, -> { where(enabled: true) }
     scope :disabled, -> { where(enabled: false) }
 
+    scope :for_environment, -> (env) { where(environment_scope: ['*', '', env.slug]) }
+    scope :for_all_environments, -> { where(environment_scope: ['*', '']) }
+
     def status_name
       if provider
         provider.status_name
diff --git a/app/models/project_services/prometheus_service.rb b/app/models/project_services/prometheus_service.rb
index fa7b3f2bcaf..1bb576ff971 100644
--- a/app/models/project_services/prometheus_service.rb
+++ b/app/models/project_services/prometheus_service.rb
@@ -7,11 +7,14 @@ class PrometheusService < MonitoringService
 
   #  Access to prometheus is directly through the API
   prop_accessor :api_url
+  boolean_accessor :manual_configuration
 
-  with_options presence: true, if: :activated? do
+  with_options presence: true, if: :manual_configuration? do
     validates :api_url, url: true
   end
 
+  before_save :synchronize_service_state!
+
   after_save :clear_reactive_cache!
 
   def initialize_properties
@@ -20,12 +23,20 @@ class PrometheusService < MonitoringService
     end
   end
 
+  def show_active_box?
+    false
+  end
+
+  def editable?
+    manual_configuration? || !prometheus_installed?
+  end
+
   def title
     'Prometheus'
   end
 
   def description
-    s_('PrometheusService|Prometheus monitoring')
+    s_('PrometheusService|Time-series monitoring service')
   end
 
   def self.to_param
@@ -33,8 +44,16 @@ class PrometheusService < MonitoringService
   end
 
   def fields
+    return [] unless editable?
+
     [
       {
+        type: 'checkbox',
+        name: 'manual_configuration',
+        title: s_('PrometheusService|Active'),
+        required: true
+      },
+      {
         type: 'text',
         name: 'api_url',
         title: 'API URL',
@@ -59,7 +78,7 @@ class PrometheusService < MonitoringService
   end
 
   def deployment_metrics(deployment)
-    metrics = with_reactive_cache(Gitlab::Prometheus::Queries::DeploymentQuery.name, deployment.id, &method(:rename_data_to_metrics))
+    metrics = with_reactive_cache(Gitlab::Prometheus::Queries::DeploymentQuery.name, deployment.environment.id, deployment.id, &method(:rename_data_to_metrics))
     metrics&.merge(deployment_time: deployment.created_at.to_i) || {}
   end
 
@@ -68,7 +87,7 @@ class PrometheusService < MonitoringService
   end
 
   def additional_deployment_metrics(deployment)
-    with_reactive_cache(Gitlab::Prometheus::Queries::AdditionalMetricsDeploymentQuery.name, deployment.id, &:itself)
+    with_reactive_cache(Gitlab::Prometheus::Queries::AdditionalMetricsDeploymentQuery.name, deployment.environment.id, deployment.id, &:itself)
   end
 
   def matched_metrics
@@ -79,6 +98,9 @@ class PrometheusService < MonitoringService
   def calculate_reactive_cache(query_class_name, *args)
     return unless active? && project && !project.pending_delete?
 
+    environment_id = args.first
+    client = client(environment_id)
+
     data = Kernel.const_get(query_class_name).new(client).query(*args)
     {
       success: true,
@@ -89,14 +111,55 @@ class PrometheusService < MonitoringService
     { success: false, result: err.message }
   end
 
-  def client
-    @prometheus ||= Gitlab::PrometheusClient.new(api_url: api_url)
+  def client(environment_id = nil)
+    if manual_configuration?
+      Gitlab::PrometheusClient.new(RestClient::Resource.new(api_url))
+    else
+      cluster = cluster_with_prometheus(environment_id)
+      raise Gitlab::PrometheusError, "couldn't find cluster with Prometheus installed" unless cluster
+
+      rest_client = client_from_cluster(cluster)
+      raise Gitlab::PrometheusError, "couldn't create proxy Prometheus client" unless rest_client
+
+      Gitlab::PrometheusClient.new(rest_client)
+    end
+  end
+
+  def prometheus_installed?
+    return false if template?
+    return false unless project
+
+    project.clusters.enabled.any? { |cluster| cluster.application_prometheus&.installed? }
   end
 
   private
 
+  def cluster_with_prometheus(environment_id = nil)
+    clusters = if environment_id
+                 ::Environment.find_by(id: environment_id).try do |env|
+                   # sort results by descending order based on environment_scope being longer
+                   # thus more closely matching environment slug
+                   project.clusters.enabled.for_environment(env).sort_by { |c| c.environment_scope&.length }.reverse!
+                 end
+               else
+                 project.clusters.enabled.for_all_environments
+               end
+
+    clusters&.detect { |cluster| cluster.application_prometheus&.installed? }
+  end
+
+  def client_from_cluster(cluster)
+    cluster.application_prometheus.proxy_client
+  end
+
   def rename_data_to_metrics(metrics)
     metrics[:metrics] = metrics.delete :data
     metrics
   end
+
+  def synchronize_service_state!
+    self.active = prometheus_installed? || manual_configuration?
+
+    true
+  end
 end
diff --git a/app/views/projects/clusters/show.html.haml b/app/views/projects/clusters/show.html.haml
index a60afde06d2..2b1b23ba198 100644
--- a/app/views/projects/clusters/show.html.haml
+++ b/app/views/projects/clusters/show.html.haml
@@ -14,7 +14,8 @@
   cluster_status: @cluster.status_name,
   cluster_status_reason: @cluster.status_reason,
   help_path: help_page_path('user/project/clusters/index.md', anchor: 'installing-applications'),
-  ingress_help_path: help_page_path('user/project/clusters/index.md', anchor: 'getting-the-external-ip-address') } }
+  ingress_help_path: help_page_path('user/project/clusters/index.md', anchor: 'getting-the-external-ip-address'),
+  manage_prometheus_path: edit_project_service_path(@cluster.project, 'prometheus') } }
 
   .js-cluster-application-notice
     .flash-container
diff --git a/app/views/projects/environments/metrics.html.haml b/app/views/projects/environments/metrics.html.haml
index 5257b42548e..10812f67cbe 100644
--- a/app/views/projects/environments/metrics.html.haml
+++ b/app/views/projects/environments/metrics.html.haml
@@ -13,6 +13,7 @@
           = link_to @environment.name, environment_path(@environment)
 
   #prometheus-graphs{ data: { "settings-path": edit_project_service_path(@project, 'prometheus'),
+    "clusters-path": project_clusters_path(@project),
     "documentation-path": help_page_path('administration/monitoring/prometheus/index.md'),
     "empty-getting-started-svg-path": image_path('illustrations/monitoring/getting_started.svg'),
     "empty-loading-svg-path": image_path('illustrations/monitoring/loading.svg'),
diff --git a/app/views/projects/services/_form.html.haml b/app/views/projects/services/_form.html.haml
index 21acd857ce7..0808b28a9df 100644
--- a/app/views/projects/services/_form.html.haml
+++ b/app/views/projects/services/_form.html.haml
@@ -9,7 +9,7 @@
 
     %p= @service.description
   .col-lg-9
-    = form_for(@service, as: :service, url: project_service_path(@project, @service.to_param), method: :put, html: { class: 'gl-show-field-errors form-horizontal js-integration-settings-form', data: { 'can-test' => @service.can_test?, 'test-url' => test_project_service_path(@project, @service) } }) do |form|
+    = form_for(@service, as: :service, url: project_service_path(@project, @service.to_param), method: :put, html: { class: 'gl-show-field-errors form-horizontal integration-settings-form js-integration-settings-form', data: { 'can-test' => @service.can_test?, 'test-url' => test_project_service_path(@project, @service) } }) do |form|
       = render 'shared/service_settings', form: form, subject: @service
       - if @service.editable?
         .footer-block.row-content-block
diff --git a/app/views/projects/services/prometheus/_help.html.haml b/app/views/projects/services/prometheus/_help.html.haml
new file mode 100644
index 00000000000..5e320a252d8
--- /dev/null
+++ b/app/views/projects/services/prometheus/_help.html.haml
@@ -0,0 +1,33 @@
+%h4
+  = s_('PrometheusService|Auto configuration')
+
+- if @service.manual_configuration?
+  .well
+    = s_('PrometheusService|To enable the installation of Prometheus on your clusters, deactivate the manual configuration below')
+- else
+  .container-fluid
+    .row
+      - if @service.prometheus_installed?
+        .col-sm-2
+          .svg-container
+            = image_tag 'illustrations/monitoring/getting_started.svg'
+        .col-sm-10
+          %p.text-success.prepend-top-default
+            = s_('PrometheusService|Prometheus is being automatically managed on your clusters')
+          = link_to s_('PrometheusService|Manage clusters'), project_clusters_path(@project), class: 'btn'
+      - else
+        .col-sm-2
+          = image_tag 'illustrations/monitoring/loading.svg'
+        .col-sm-10
+          %p.prepend-top-default
+            = s_('PrometheusService|Automatically deploy and configure Prometheus on your clusters to monitor your project’s environments')
+          = link_to s_('PrometheusService|Install Prometheus on clusters'), project_clusters_path(@project), class: 'btn btn-success'
+
+%hr
+
+%h4.append-bottom-default
+  = s_('PrometheusService|Manual configuration')
+
+- unless @service.editable?
+  .well
+    = s_('PrometheusService|To enable manual configuration, uninstall Prometheus from your clusters')
diff --git a/changelogs/unreleased/pawel-connect_to_prometheus_through_proxy-30480.yml b/changelogs/unreleased/pawel-connect_to_prometheus_through_proxy-30480.yml
new file mode 100644
index 00000000000..b2bb173912a
--- /dev/null
+++ b/changelogs/unreleased/pawel-connect_to_prometheus_through_proxy-30480.yml
@@ -0,0 +1,6 @@
+---
+title: Implement multi server support and use kube proxy to connect to Prometheus
+  servers inside K8S cluster
+merge_request: 16182
+author:
+type: added
diff --git a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb
index 69d055c901c..294a6ae34ca 100644
--- a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb
+++ b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb
@@ -4,7 +4,7 @@ module Gitlab
       class AdditionalMetricsDeploymentQuery < BaseQuery
         include QueryAdditionalMetrics
 
-        def query(deployment_id)
+        def query(environment_id, deployment_id)
           Deployment.find_by(id: deployment_id).try do |deployment|
             query_metrics(
               common_query_context(
diff --git a/lib/gitlab/prometheus/queries/deployment_query.rb b/lib/gitlab/prometheus/queries/deployment_query.rb
index 170f483540e..6e6da593178 100644
--- a/lib/gitlab/prometheus/queries/deployment_query.rb
+++ b/lib/gitlab/prometheus/queries/deployment_query.rb
@@ -2,7 +2,7 @@ module Gitlab
   module Prometheus
     module Queries
       class DeploymentQuery < BaseQuery
-        def query(deployment_id)
+        def query(environment_id, deployment_id)
           Deployment.find_by(id: deployment_id).try do |deployment|
             environment_slug = deployment.environment.slug
 
diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb
index aa94614bf18..10527972663 100644
--- a/lib/gitlab/prometheus_client.rb
+++ b/lib/gitlab/prometheus_client.rb
@@ -3,10 +3,10 @@ module Gitlab
 
   # Helper methods to interact with Prometheus network services & resources
   class PrometheusClient
-    attr_reader :api_url
+    attr_reader :rest_client, :headers
 
-    def initialize(api_url:)
-      @api_url = api_url
+    def initialize(rest_client)
+      @rest_client = rest_client
     end
 
     def ping
@@ -40,37 +40,40 @@ module Gitlab
     private
 
     def json_api_get(type, args = {})
-      get(join_api_url(type, args))
+      path = ['api', 'v1', type].join('/')
+      get(path, args)
+    rescue JSON::ParserError
+      raise PrometheusError, 'Parsing response failed'
     rescue Errno::ECONNREFUSED
       raise PrometheusError, 'Connection refused'
     end
 
-    def join_api_url(type, args = {})
-      url = URI.parse(api_url)
-    rescue URI::Error
-      raise PrometheusError, "Invalid API URL: #{api_url}"
-    else
-      url.path = [url.path.sub(%r{/+\z}, ''), 'api', 'v1', type].join('/')
-      url.query = args.to_query
-
-      url.to_s
-    end
-
-    def get(url)
-      handle_response(HTTParty.get(url))
+    def get(path, args)
+      response = rest_client[path].get(params: args)
+      handle_response(response)
     rescue SocketError
-      raise PrometheusError, "Can't connect to #{url}"
+      raise PrometheusError, "Can't connect to #{rest_client.url}"
     rescue OpenSSL::SSL::SSLError
-      raise PrometheusError, "#{url} contains invalid SSL data"
-    rescue HTTParty::Error
+      raise PrometheusError, "#{rest_client.url} contains invalid SSL data"
+    rescue RestClient::ExceptionWithResponse => ex
+      handle_exception_response(ex.response)
+    rescue RestClient::Exception
       raise PrometheusError, "Network connection error"
     end
 
     def handle_response(response)
-      if response.code == 200 && response['status'] == 'success'
-        response['data'] || {}
-      elsif response.code == 400
-        raise PrometheusError, response['error'] || 'Bad data received'
+      json_data = JSON.parse(response.body)
+      if response.code == 200 && json_data['status'] == 'success'
+        json_data['data'] || {}
+      else
+        raise PrometheusError, "#{response.code} - #{response.body}"
+      end
+    end
+
+    def handle_exception_response(response)
+      if response.code == 400
+        json_data = JSON.parse(response.body)
+        raise PrometheusError, json_data['error'] || 'Bad data received'
       else
         raise PrometheusError, "#{response.code} - #{response.body}"
       end
diff --git a/locale/bg/gitlab.po b/locale/bg/gitlab.po
index 8432914d6a7..81ce8ec506e 100644
--- a/locale/bg/gitlab.po
+++ b/locale/bg/gitlab.po
@@ -2018,7 +2018,7 @@ msgstr ""
 msgid "PrometheusService|Prometheus API Base URL, like http://prometheus.example.com/"
 msgstr ""
 
-msgid "PrometheusService|Prometheus monitoring"
+msgid "PrometheusService|Time-series monitoring service"
 msgstr ""
 
 msgid "PrometheusService|View environments"
diff --git a/locale/de/gitlab.po b/locale/de/gitlab.po
index db7f41c5476..5b33ed0a628 100644
--- a/locale/de/gitlab.po
+++ b/locale/de/gitlab.po
@@ -2018,7 +2018,7 @@ msgstr ""
 msgid "PrometheusService|Prometheus API Base URL, like http://prometheus.example.com/"
 msgstr ""
 
-msgid "PrometheusService|Prometheus monitoring"
+msgid "PrometheusService|Time-series monitoring service"
 msgstr ""
 
 msgid "PrometheusService|View environments"
diff --git a/locale/eo/gitlab.po b/locale/eo/gitlab.po
index be7cfa6e4b5..54906417e75 100644
--- a/locale/eo/gitlab.po
+++ b/locale/eo/gitlab.po
@@ -2018,7 +2018,7 @@ msgstr ""
 msgid "PrometheusService|Prometheus API Base URL, like http://prometheus.example.com/"
 msgstr ""
 
-msgid "PrometheusService|Prometheus monitoring"
+msgid "PrometheusService|Time-series monitoring service"
 msgstr ""
 
 msgid "PrometheusService|View environments"
diff --git a/locale/es/gitlab.po b/locale/es/gitlab.po
index 44ad3d4633a..750dde61a03 100644
--- a/locale/es/gitlab.po
+++ b/locale/es/gitlab.po
@@ -2018,7 +2018,7 @@ msgstr ""
 msgid "PrometheusService|Prometheus API Base URL, like http://prometheus.example.com/"
 msgstr ""
 
-msgid "PrometheusService|Prometheus monitoring"
+msgid "PrometheusService|Time-series monitoring service"
 msgstr ""
 
 msgid "PrometheusService|View environments"
diff --git a/locale/fr/gitlab.po b/locale/fr/gitlab.po
index ace6a5d2f66..2a713917684 100644
--- a/locale/fr/gitlab.po
+++ b/locale/fr/gitlab.po
@@ -2018,7 +2018,7 @@ msgstr ""
 msgid "PrometheusService|Prometheus API Base URL, like http://prometheus.example.com/"
 msgstr ""
 
-msgid "PrometheusService|Prometheus monitoring"
+msgid "PrometheusService|Time-series monitoring service"
 msgstr ""
 
 msgid "PrometheusService|View environments"
diff --git a/locale/gitlab.pot b/locale/gitlab.pot
index fbb3f949430..501bcef93de 100644
--- a/locale/gitlab.pot
+++ b/locale/gitlab.pot
@@ -2141,7 +2141,7 @@ msgstr ""
 msgid "PrometheusService|Prometheus API Base URL, like http://prometheus.example.com/"
 msgstr ""
 
-msgid "PrometheusService|Prometheus monitoring"
+msgid "PrometheusService|Time-series monitoring service"
 msgstr ""
 
 msgid "PrometheusService|View environments"
diff --git a/locale/it/gitlab.po b/locale/it/gitlab.po
index 52bbc28ac10..8b237dfe450 100644
--- a/locale/it/gitlab.po
+++ b/locale/it/gitlab.po
@@ -2018,7 +2018,7 @@ msgstr "Nessuna metrica è stata monitorata. Per iniziare a monitorare, rilascia
 msgid "PrometheusService|Prometheus API Base URL, like http://prometheus.example.com/"
 msgstr ""
 
-msgid "PrometheusService|Prometheus monitoring"
+msgid "PrometheusService|Time-series monitoring service"
 msgstr ""
 
 msgid "PrometheusService|View environments"
diff --git a/locale/ja/gitlab.po b/locale/ja/gitlab.po
index 1314bad87fe..700d70cad3f 100644
--- a/locale/ja/gitlab.po
+++ b/locale/ja/gitlab.po
@@ -2004,7 +2004,7 @@ msgstr ""
 msgid "PrometheusService|Prometheus API Base URL, like http://prometheus.example.com/"
 msgstr ""
 
-msgid "PrometheusService|Prometheus monitoring"
+msgid "PrometheusService|Time-series monitoring service"
 msgstr ""
 
 msgid "PrometheusService|View environments"
diff --git a/locale/ko/gitlab.po b/locale/ko/gitlab.po
index 9ec3d395c15..aab1650a0bc 100644
--- a/locale/ko/gitlab.po
+++ b/locale/ko/gitlab.po
@@ -2004,7 +2004,7 @@ msgstr ""
 msgid "PrometheusService|Prometheus API Base URL, like http://prometheus.example.com/"
 msgstr ""
 
-msgid "PrometheusService|Prometheus monitoring"
+msgid "PrometheusService|Time-series monitoring service"
 msgstr ""
 
 msgid "PrometheusService|View environments"
diff --git a/locale/nl_NL/gitlab.po b/locale/nl_NL/gitlab.po
index 0abb727037c..eca1923fd2c 100644
--- a/locale/nl_NL/gitlab.po
+++ b/locale/nl_NL/gitlab.po
@@ -2018,7 +2018,7 @@ msgstr ""
 msgid "PrometheusService|Prometheus API Base URL, like http://prometheus.example.com/"
 msgstr ""
 
-msgid "PrometheusService|Prometheus monitoring"
+msgid "PrometheusService|Time-series monitoring service"
 msgstr ""
 
 msgid "PrometheusService|View environments"
diff --git a/locale/pl_PL/gitlab.po b/locale/pl_PL/gitlab.po
index 5b65c42097e..a9059b0a73b 100644
--- a/locale/pl_PL/gitlab.po
+++ b/locale/pl_PL/gitlab.po
@@ -2032,7 +2032,7 @@ msgstr ""
 msgid "PrometheusService|Prometheus API Base URL, like http://prometheus.example.com/"
 msgstr ""
 
-msgid "PrometheusService|Prometheus monitoring"
+msgid "PrometheusService|Time-series monitoring service"
 msgstr ""
 
 msgid "PrometheusService|View environments"
diff --git a/locale/pt_BR/gitlab.po b/locale/pt_BR/gitlab.po
index 9fe1cc3c11a..24753f2b140 100644
--- a/locale/pt_BR/gitlab.po
+++ b/locale/pt_BR/gitlab.po
@@ -2018,7 +2018,7 @@ msgstr "Nenhuma métrica está sendo monitorada. Para inicar o monitoramento, fa
 msgid "PrometheusService|Prometheus API Base URL, like http://prometheus.example.com/"
 msgstr "URL da API base do Prometheus. como http://prometheus.example.com/"
 
-msgid "PrometheusService|Prometheus monitoring"
+msgid "PrometheusService|Time-series monitoring service"
 msgstr "Monitoramento com Prometheus"
 
 msgid "PrometheusService|View environments"
diff --git a/locale/ru/gitlab.po b/locale/ru/gitlab.po
index 898d55e7d4e..1b3b65325ac 100644
--- a/locale/ru/gitlab.po
+++ b/locale/ru/gitlab.po
@@ -2032,7 +2032,7 @@ msgstr ""
 msgid "PrometheusService|Prometheus API Base URL, like http://prometheus.example.com/"
 msgstr ""
 
-msgid "PrometheusService|Prometheus monitoring"
+msgid "PrometheusService|Time-series monitoring service"
 msgstr ""
 
 msgid "PrometheusService|View environments"
diff --git a/locale/uk/gitlab.po b/locale/uk/gitlab.po
index fc62776a7a4..0f20f0c9ceb 100644
--- a/locale/uk/gitlab.po
+++ b/locale/uk/gitlab.po
@@ -2032,7 +2032,7 @@ msgstr "Жодні метрики не відслідковуються. Для
 msgid "PrometheusService|Prometheus API Base URL, like http://prometheus.example.com/"
 msgstr "Базова адреса Prometheus API, наприклад http://prometheus.example.com/"
 
-msgid "PrometheusService|Prometheus monitoring"
+msgid "PrometheusService|Time-series monitoring service"
 msgstr "Моніторинг Prometheus"
 
 msgid "PrometheusService|View environments"
diff --git a/locale/zh_CN/gitlab.po b/locale/zh_CN/gitlab.po
index f0a5453f224..9c0d8dd5ddc 100644
--- a/locale/zh_CN/gitlab.po
+++ b/locale/zh_CN/gitlab.po
@@ -2004,7 +2004,7 @@ msgstr "没有监测指标。要开始监测，请部署到环境中。"
 msgid "PrometheusService|Prometheus API Base URL, like http://prometheus.example.com/"
 msgstr "Prometheus API 地址，例如 http://prometheus.example.com/"
 
-msgid "PrometheusService|Prometheus monitoring"
+msgid "PrometheusService|Time-series monitoring service"
 msgstr "Prometheus 监测"
 
 msgid "PrometheusService|View environments"
diff --git a/locale/zh_HK/gitlab.po b/locale/zh_HK/gitlab.po
index b368487ac71..99024ee527c 100644
--- a/locale/zh_HK/gitlab.po
+++ b/locale/zh_HK/gitlab.po
@@ -2004,7 +2004,7 @@ msgstr ""
 msgid "PrometheusService|Prometheus API Base URL, like http://prometheus.example.com/"
 msgstr ""
 
-msgid "PrometheusService|Prometheus monitoring"
+msgid "PrometheusService|Time-series monitoring service"
 msgstr ""
 
 msgid "PrometheusService|View environments"
diff --git a/locale/zh_TW/gitlab.po b/locale/zh_TW/gitlab.po
index 76c1e598433..14bc24c0e08 100644
--- a/locale/zh_TW/gitlab.po
+++ b/locale/zh_TW/gitlab.po
@@ -2004,7 +2004,7 @@ msgstr ""
 msgid "PrometheusService|Prometheus API Base URL, like http://prometheus.example.com/"
 msgstr ""
 
-msgid "PrometheusService|Prometheus monitoring"
+msgid "PrometheusService|Time-series monitoring service"
 msgstr ""
 
 msgid "PrometheusService|View environments"
diff --git a/spec/factories/projects.rb b/spec/factories/projects.rb
index 20976977f21..f92b307fee4 100644
--- a/spec/factories/projects.rb
+++ b/spec/factories/projects.rb
@@ -249,7 +249,8 @@ FactoryBot.define do
       project.create_prometheus_service(
         active: true,
         properties: {
-          api_url: 'https://prometheus.example.com'
+          api_url: 'https://prometheus.example.com/',
+          manual_configuration: true
         }
       )
     end
diff --git a/spec/factories/services.rb b/spec/factories/services.rb
index 110ef33c6f7..0d4fd49bf3a 100644
--- a/spec/factories/services.rb
+++ b/spec/factories/services.rb
@@ -30,7 +30,8 @@ FactoryBot.define do
     project
     active true
     properties({
-      api_url: 'https://prometheus.example.com/'
+      api_url: 'https://prometheus.example.com/',
+      manual_configuration: true
     })
   end
 
diff --git a/spec/javascripts/monitoring/dashboard_state_spec.js b/spec/javascripts/monitoring/dashboard_state_spec.js
index 3319eeb3f31..df3198dd3e2 100644
--- a/spec/javascripts/monitoring/dashboard_state_spec.js
+++ b/spec/javascripts/monitoring/dashboard_state_spec.js
@@ -29,34 +29,6 @@ describe('EmptyState', () => {
       expect(component.currentState).toBe(component.states.gettingStarted);
     });
 
-    it('buttonPath returns settings path for the state "gettingStarted"', () => {
-      const component = createComponent({
-        selectedState: 'gettingStarted',
-        settingsPath: statePaths.settingsPath,
-        documentationPath: statePaths.documentationPath,
-        emptyGettingStartedSvgPath: 'foo',
-        emptyLoadingSvgPath: 'foo',
-        emptyUnableToConnectSvgPath: 'foo',
-      });
-
-      expect(component.buttonPath).toEqual(statePaths.settingsPath);
-      expect(component.buttonPath).not.toEqual(statePaths.documentationPath);
-    });
-
-    it('buttonPath returns documentation path for any of the other states', () => {
-      const component = createComponent({
-        selectedState: 'loading',
-        settingsPath: statePaths.settingsPath,
-        documentationPath: statePaths.documentationPath,
-        emptyGettingStartedSvgPath: 'foo',
-        emptyLoadingSvgPath: 'foo',
-        emptyUnableToConnectSvgPath: 'foo',
-      });
-
-      expect(component.buttonPath).toEqual(statePaths.documentationPath);
-      expect(component.buttonPath).not.toEqual(statePaths.settingsPath);
-    });
-
     it('showButtonDescription returns a description with a link for the unableToConnect state', () => {
       const component = createComponent({
         selectedState: 'unableToConnect',
@@ -88,6 +60,7 @@ describe('EmptyState', () => {
     const component = createComponent({
       selectedState: 'gettingStarted',
       settingsPath: statePaths.settingsPath,
+      clustersPath: statePaths.clustersPath,
       documentationPath: statePaths.documentationPath,
       emptyGettingStartedSvgPath: 'foo',
       emptyLoadingSvgPath: 'foo',
diff --git a/spec/javascripts/monitoring/mock_data.js b/spec/javascripts/monitoring/mock_data.js
index 2bbe963e393..f30208b27b6 100644
--- a/spec/javascripts/monitoring/mock_data.js
+++ b/spec/javascripts/monitoring/mock_data.js
@@ -2471,6 +2471,7 @@ export const deploymentData = [
 
 export const statePaths = {
   settingsPath: '/root/hello-prometheus/services/prometheus/edit',
+  clustersPath: '/root/hello-prometheus/clusters',
   documentationPath: '/help/administration/monitoring/prometheus/index.md',
 };
 
diff --git a/spec/lib/gitlab/prometheus/queries/additional_metrics_deployment_query_spec.rb b/spec/lib/gitlab/prometheus/queries/additional_metrics_deployment_query_spec.rb
index c7169717fc1..0697cb2def6 100644
--- a/spec/lib/gitlab/prometheus/queries/additional_metrics_deployment_query_spec.rb
+++ b/spec/lib/gitlab/prometheus/queries/additional_metrics_deployment_query_spec.rb
@@ -7,7 +7,7 @@ describe Gitlab::Prometheus::Queries::AdditionalMetricsDeploymentQuery do
 
   include_examples 'additional metrics query' do
     let(:deployment) { create(:deployment, environment: environment) }
-    let(:query_params) { [deployment.id] }
+    let(:query_params) { [environment.id, deployment.id] }
 
     it 'queries using specific time' do
       expect(client).to receive(:query_range).with(anything,
diff --git a/spec/lib/gitlab/prometheus/queries/deployment_query_spec.rb b/spec/lib/gitlab/prometheus/queries/deployment_query_spec.rb
index ffe3ad85baa..84dc31d9732 100644
--- a/spec/lib/gitlab/prometheus/queries/deployment_query_spec.rb
+++ b/spec/lib/gitlab/prometheus/queries/deployment_query_spec.rb
@@ -31,7 +31,7 @@ describe Gitlab::Prometheus::Queries::DeploymentQuery do
     expect(client).to receive(:query).with('avg(rate(container_cpu_usage_seconds_total{container_name!="POD",environment="environment-slug"}[30m])) * 100',
                                            time: stop_time)
 
-    expect(subject.query(deployment.id)).to eq(memory_values: nil, memory_before: nil, memory_after: nil,
-                                               cpu_values: nil, cpu_before: nil, cpu_after: nil)
+    expect(subject.query(environment.id, deployment.id)).to eq(memory_values: nil, memory_before: nil, memory_after: nil,
+                                                               cpu_values: nil, cpu_before: nil, cpu_after: nil)
   end
 end
diff --git a/spec/lib/gitlab/prometheus_client_spec.rb b/spec/lib/gitlab/prometheus_client_spec.rb
index de625324092..5d86007f71f 100644
--- a/spec/lib/gitlab/prometheus_client_spec.rb
+++ b/spec/lib/gitlab/prometheus_client_spec.rb
@@ -3,7 +3,7 @@ require 'spec_helper'
 describe Gitlab::PrometheusClient do
   include PrometheusHelpers
 
-  subject { described_class.new(api_url: 'https://prometheus.example.com') }
+  subject { described_class.new(RestClient::Resource.new('https://prometheus.example.com')) }
 
   describe '#ping' do
     it 'issues a "query" request to the API endpoint' do
@@ -47,16 +47,28 @@ describe Gitlab::PrometheusClient do
         expect(req_stub).to have_been_requested
       end
     end
+
+    context 'when request returns non json data' do
+      it 'raises a Gitlab::PrometheusError error' do
+        req_stub = stub_prometheus_request(query_url, status: 200, body: 'not json')
+
+        expect { execute_query }
+          .to raise_error(Gitlab::PrometheusError, 'Parsing response failed')
+        expect(req_stub).to have_been_requested
+      end
+    end
   end
 
   describe 'failure to reach a provided prometheus url' do
     let(:prometheus_url) {"https://prometheus.invalid.example.com"}
 
+    subject { described_class.new(RestClient::Resource.new(prometheus_url)) }
+
     context 'exceptions are raised' do
       it 'raises a Gitlab::PrometheusError error when a SocketError is rescued' do
         req_stub = stub_prometheus_request_with_exception(prometheus_url, SocketError)
 
-        expect { subject.send(:get, prometheus_url) }
+        expect { subject.send(:get, '/', {}) }
           .to raise_error(Gitlab::PrometheusError, "Can't connect to #{prometheus_url}")
         expect(req_stub).to have_been_requested
       end
@@ -64,15 +76,15 @@ describe Gitlab::PrometheusClient do
       it 'raises a Gitlab::PrometheusError error when a SSLError is rescued' do
         req_stub = stub_prometheus_request_with_exception(prometheus_url, OpenSSL::SSL::SSLError)
 
-        expect { subject.send(:get, prometheus_url) }
+        expect { subject.send(:get, '/', {}) }
           .to raise_error(Gitlab::PrometheusError, "#{prometheus_url} contains invalid SSL data")
         expect(req_stub).to have_been_requested
       end
 
-      it 'raises a Gitlab::PrometheusError error when a HTTParty::Error is rescued' do
-        req_stub = stub_prometheus_request_with_exception(prometheus_url, HTTParty::Error)
+      it 'raises a Gitlab::PrometheusError error when a RestClient::Exception is rescued' do
+        req_stub = stub_prometheus_request_with_exception(prometheus_url, RestClient::Exception)
 
-        expect { subject.send(:get, prometheus_url) }
+        expect { subject.send(:get, '/', {}) }
           .to raise_error(Gitlab::PrometheusError, "Network connection error")
         expect(req_stub).to have_been_requested
       end
diff --git a/spec/models/clusters/applications/prometheus_spec.rb b/spec/models/clusters/applications/prometheus_spec.rb
index 696099f7cf7..01037919530 100644
--- a/spec/models/clusters/applications/prometheus_spec.rb
+++ b/spec/models/clusters/applications/prometheus_spec.rb
@@ -6,6 +6,24 @@ describe Clusters::Applications::Prometheus do
 
   include_examples 'cluster application specs', described_class
 
+  describe 'transition to installed' do
+    let(:project) { create(:project) }
+    let(:cluster) { create(:cluster, projects: [project]) }
+    let(:prometheus_service) { double('prometheus_service') }
+
+    subject { create(:clusters_applications_prometheus, :installing, cluster: cluster) }
+
+    before do
+      allow(project).to receive(:find_or_initialize_service).with('prometheus').and_return prometheus_service
+    end
+
+    it 'ensures Prometheus service is activated' do
+      expect(prometheus_service).to receive(:update).with(active: true)
+
+      subject.make_installed
+    end
+  end
+
   describe "#chart_values_file" do
     subject { create(:clusters_applications_prometheus).chart_values_file }
 
@@ -13,4 +31,58 @@ describe Clusters::Applications::Prometheus do
       expect(subject).to eq("#{Rails.root}/vendor/prometheus/values.yaml")
     end
   end
+
+  describe '#proxy_client' do
+    context 'cluster is nil' do
+      it 'returns nil' do
+        expect(subject.cluster).to be_nil
+        expect(subject.proxy_client).to be_nil
+      end
+    end
+
+    context "cluster doesn't have kubeclient" do
+      let(:cluster) { create(:cluster) }
+      subject { create(:clusters_applications_prometheus, cluster: cluster) }
+
+      it 'returns nil' do
+        expect(subject.proxy_client).to be_nil
+      end
+    end
+
+    context 'cluster has kubeclient' do
+      let(:kubernetes_url) { 'http://example.com' }
+      let(:k8s_discover_response) do
+        {
+          resources: [
+            {
+              name: 'service',
+              kind: 'Service'
+            }
+          ]
+        }
+      end
+
+      let(:kube_client) { Kubeclient::Client.new(kubernetes_url) }
+
+      let(:cluster) { create(:cluster) }
+      subject { create(:clusters_applications_prometheus, cluster: cluster) }
+
+      before do
+        allow(kube_client.rest_client).to receive(:get).and_return(k8s_discover_response.to_json)
+        allow(subject.cluster).to receive(:kubeclient).and_return(kube_client)
+      end
+
+      it 'creates proxy prometheus rest client' do
+        expect(subject.proxy_client).to be_instance_of(RestClient::Resource)
+      end
+
+      it 'creates proper url' do
+        expect(subject.proxy_client.url).to eq('http://example.com/api/v1/proxy/namespaces/gitlab-managed-apps/service/prometheus-prometheus-server:80')
+      end
+
+      it 'copies options and headers from kube client to proxy client' do
+        expect(subject.proxy_client.options).to eq(kube_client.rest_client.options.merge(headers: kube_client.headers))
+      end
+    end
+  end
 end
diff --git a/spec/models/project_services/prometheus_service_spec.rb b/spec/models/project_services/prometheus_service_spec.rb
index bf39e8d7a39..ed17e019d42 100644
--- a/spec/models/project_services/prometheus_service_spec.rb
+++ b/spec/models/project_services/prometheus_service_spec.rb
@@ -13,17 +13,17 @@ describe PrometheusService, :use_clean_rails_memory_store_caching do
   end
 
   describe 'Validations' do
-    context 'when service is active' do
+    context 'when manual_configuration is enabled' do
       before do
-        subject.active = true
+        subject.manual_configuration = true
       end
 
       it { is_expected.to validate_presence_of(:api_url) }
     end
 
-    context 'when service is inactive' do
+    context 'when manual configuration is disabled' do
       before do
-        subject.active = false
+        subject.manual_configuration = false
       end
 
       it { is_expected.not_to validate_presence_of(:api_url) }
@@ -31,12 +31,17 @@ describe PrometheusService, :use_clean_rails_memory_store_caching do
   end
 
   describe '#test' do
+    before do
+      service.manual_configuration = true
+    end
+
     let!(:req_stub) { stub_prometheus_request(prometheus_query_url('1'), body: prometheus_value_body('vector')) }
 
     context 'success' do
       it 'reads the discovery endpoint' do
+        expect(service.test[:result]).to eq('Checked API endpoint')
         expect(service.test[:success]).to be_truthy
-        expect(req_stub).to have_been_requested
+        expect(req_stub).to have_been_requested.twice
       end
     end
 
@@ -70,6 +75,25 @@ describe PrometheusService, :use_clean_rails_memory_store_caching do
     end
   end
 
+  describe '#matched_metrics' do
+    let(:matched_metrics_query) { Gitlab::Prometheus::Queries::MatchedMetricsQuery }
+    let(:client) { double(:client, label_values: nil) }
+
+    context 'with valid data' do
+      subject { service.matched_metrics }
+
+      before do
+        allow(service).to receive(:client).and_return(client)
+        synchronous_reactive_cache(service)
+      end
+
+      it 'returns reactive data' do
+        expect(subject[:success]).to be_truthy
+        expect(subject[:data]).to eq([])
+      end
+    end
+  end
+
   describe '#deployment_metrics' do
     let(:deployment) { build_stubbed(:deployment) }
     let(:deployment_query) { Gitlab::Prometheus::Queries::DeploymentQuery }
@@ -83,7 +107,7 @@ describe PrometheusService, :use_clean_rails_memory_store_caching do
       let(:fake_deployment_time) { 10 }
 
       before do
-        stub_reactive_cache(service, prometheus_data, deployment_query, deployment.id)
+        stub_reactive_cache(service, prometheus_data, deployment_query, deployment.environment.id, deployment.id)
       end
 
       it 'returns reactive data' do
@@ -96,13 +120,17 @@ describe PrometheusService, :use_clean_rails_memory_store_caching do
 
   describe '#calculate_reactive_cache' do
     let(:environment) { create(:environment, slug: 'env-slug') }
-
-    around do |example|
-      Timecop.freeze { example.run }
+    before do
+      service.manual_configuration = true
+      service.active = true
     end
 
     subject do
-      service.calculate_reactive_cache(environment_query.to_s, environment.id)
+      service.calculate_reactive_cache(environment_query.name, environment.id)
+    end
+
+    around do |example|
+      Timecop.freeze { example.run }
     end
 
     context 'when service is inactive' do
@@ -132,4 +160,193 @@ describe PrometheusService, :use_clean_rails_memory_store_caching do
       end
     end
   end
+
+  describe '#client' do
+    context 'manual configuration is enabled' do
+      let(:api_url) { 'http://some_url' }
+      before do
+        subject.manual_configuration = true
+        subject.api_url = api_url
+      end
+
+      it 'returns simple rest client from api_url' do
+        expect(subject.client).to be_instance_of(Gitlab::PrometheusClient)
+        expect(subject.client.rest_client.url).to eq(api_url)
+      end
+    end
+
+    context 'manual configuration is disabled' do
+      let!(:cluster_for_all) { create(:cluster, environment_scope: '*', projects: [project]) }
+      let!(:cluster_for_dev) { create(:cluster, environment_scope: 'dev', projects: [project]) }
+
+      let!(:prometheus_for_dev) { create(:clusters_applications_prometheus, :installed, cluster: cluster_for_dev) }
+      let(:proxy_client) { double('proxy_client') }
+
+      before do
+        service.manual_configuration = false
+      end
+
+      context 'with cluster for all environments with prometheus installed' do
+        let!(:prometheus_for_all) { create(:clusters_applications_prometheus, :installed, cluster: cluster_for_all) }
+
+        context 'without environment supplied' do
+          it 'returns client handling all environments' do
+            expect(service).to receive(:client_from_cluster).with(cluster_for_all).and_return(proxy_client).twice
+
+            expect(service.client).to be_instance_of(Gitlab::PrometheusClient)
+            expect(service.client.rest_client).to eq(proxy_client)
+          end
+        end
+
+        context 'with dev environment supplied' do
+          let!(:environment) { create(:environment, project: project, name: 'dev') }
+
+          it 'returns dev cluster client' do
+            expect(service).to receive(:client_from_cluster).with(cluster_for_dev).and_return(proxy_client).twice
+
+            expect(service.client(environment.id)).to be_instance_of(Gitlab::PrometheusClient)
+            expect(service.client(environment.id).rest_client).to eq(proxy_client)
+          end
+        end
+
+        context 'with prod environment supplied' do
+          let!(:environment) { create(:environment, project: project, name: 'prod') }
+
+          it 'returns dev cluster client' do
+            expect(service).to receive(:client_from_cluster).with(cluster_for_all).and_return(proxy_client).twice
+
+            expect(service.client(environment.id)).to be_instance_of(Gitlab::PrometheusClient)
+            expect(service.client(environment.id).rest_client).to eq(proxy_client)
+          end
+        end
+      end
+
+      context 'with cluster for all environments without prometheus installed' do
+        context 'without environment supplied' do
+          it 'raises PrometheusError because cluster was not found' do
+            expect { service.client }.to raise_error(Gitlab::PrometheusError, /couldn't find cluster with Prometheus installed/)
+          end
+        end
+
+        context 'with dev environment supplied' do
+          let!(:environment) { create(:environment, project: project, name: 'dev') }
+
+          it 'returns dev cluster client' do
+            expect(service).to receive(:client_from_cluster).with(cluster_for_dev).and_return(proxy_client).twice
+
+            expect(service.client(environment.id)).to be_instance_of(Gitlab::PrometheusClient)
+            expect(service.client(environment.id).rest_client).to eq(proxy_client)
+          end
+        end
+
+        context 'with prod environment supplied' do
+          let!(:environment) { create(:environment, project: project, name: 'prod') }
+
+          it 'raises PrometheusError because cluster was not found' do
+            expect { service.client }.to raise_error(Gitlab::PrometheusError, /couldn't find cluster with Prometheus installed/)
+          end
+        end
+      end
+    end
+  end
+
+  describe '#prometheus_installed?' do
+    context 'clusters with installed prometheus' do
+      let!(:cluster) { create(:cluster, projects: [project]) }
+      let!(:prometheus) { create(:clusters_applications_prometheus, :installed, cluster: cluster) }
+
+      it 'returns true' do
+        expect(service.prometheus_installed?).to be(true)
+      end
+    end
+
+    context 'clusters without prometheus installed' do
+      let(:cluster) { create(:cluster, projects: [project]) }
+      let!(:prometheus) { create(:clusters_applications_prometheus, cluster: cluster) }
+
+      it 'returns false' do
+        expect(service.prometheus_installed?).to be(false)
+      end
+    end
+
+    context 'clusters without prometheus' do
+      let(:cluster) { create(:cluster, projects: [project]) }
+
+      it 'returns false' do
+        expect(service.prometheus_installed?).to be(false)
+      end
+    end
+
+    context 'no clusters' do
+      it 'returns false' do
+        expect(service.prometheus_installed?).to be(false)
+      end
+    end
+  end
+
+  describe '#synchronize_service_state! before_save callback' do
+    context 'no clusters with prometheus are installed' do
+      context 'when service is inactive' do
+        before do
+          service.active = false
+        end
+
+        it 'activates service when manual_configuration is enabled' do
+          expect { service.update!(manual_configuration: true) }.to change { service.active }.from(false).to(true)
+        end
+
+        it 'keeps service inactive when manual_configuration is disabled' do
+          expect { service.update!(manual_configuration: false) }.not_to change { service.active }.from(false)
+        end
+      end
+
+      context 'when service is active' do
+        before do
+          service.active = true
+        end
+
+        it 'keeps the service active when manual_configuration is enabled' do
+          expect { service.update!(manual_configuration: true) }.not_to change { service.active }.from(true)
+        end
+
+        it 'inactivates the service when manual_configuration is disabled' do
+          expect { service.update!(manual_configuration: false) }.to change { service.active }.from(true).to(false)
+        end
+      end
+    end
+
+    context 'with prometheus installed in the cluster' do
+      before do
+        allow(service).to receive(:prometheus_installed?).and_return(true)
+      end
+
+      context 'when service is inactive' do
+        before do
+          service.active = false
+        end
+
+        it 'activates service when manual_configuration is enabled' do
+          expect { service.update!(manual_configuration: true) }.to change { service.active }.from(false).to(true)
+        end
+
+        it 'activates service when manual_configuration is disabled' do
+          expect { service.update!(manual_configuration: false) }.to change { service.active }.from(false).to(true)
+        end
+      end
+
+      context 'when service is active' do
+        before do
+          service.active = true
+        end
+
+        it 'keeps service active when manual_configuration is enabled' do
+          expect { service.update!(manual_configuration: true) }.not_to change { service.active }.from(true)
+        end
+
+        it 'keeps service active when manual_configuration is disabled' do
+          expect { service.update!(manual_configuration: false) }.not_to change { service.active }.from(true)
+        end
+      end
+    end
+  end
 end
diff --git a/spec/support/reactive_caching_helpers.rb b/spec/support/reactive_caching_helpers.rb
index 34124f02133..e22dd974c6a 100644
--- a/spec/support/reactive_caching_helpers.rb
+++ b/spec/support/reactive_caching_helpers.rb
@@ -13,6 +13,12 @@ module ReactiveCachingHelpers
     write_reactive_cache(subject, data, *qualifiers) if data
   end
 
+  def synchronous_reactive_cache(subject)
+    allow(service).to receive(:with_reactive_cache) do |*args, &block|
+      block.call(service.calculate_reactive_cache(*args))
+    end
+  end
+
   def read_reactive_cache(subject, *qualifiers)
     Rails.cache.read(reactive_cache_key(subject, *qualifiers))
   end