Update metrics dashboard API to load yml from repo

Updates the EnvironmentController#metrics_dashboard endpoint
to support a "dashboard" param, which can be used to specify
the filepath of a dashboard configuration from a project
repository. Dashboard configurations are expected to be
stored in .gitlab/dashboards/.

Updates dashboard post-processing steps to exclude custom
metrics, which should only display on the system dashboard.

9 files changed, 231 insertions, 47 deletions

diff --git a/lib/gitlab/file_detector.rb b/lib/gitlab/file_detector.rb
index 2770469ca9f..9fc2217ad43 100644
--- a/lib/gitlab/file_detector.rb
+++ b/lib/gitlab/file_detector.rb
@@ -16,6 +16,7 @@ module Gitlab
       avatar: /\Alogo\.(png|jpg|gif)\z/,
       issue_template: %r{\A\.gitlab/issue_templates/[^/]+\.md\z},
       merge_request_template: %r{\A\.gitlab/merge_request_templates/[^/]+\.md\z},
+      metrics_dashboard: %r{\A\.gitlab/dashboards/[^/]+\.yml\z},
       xcode_config: %r{\A[^/]*\.(xcodeproj|xcworkspace)(/.+)?\z},
 
       # Configuration files
diff --git a/lib/gitlab/metrics/dashboard/base_service.rb b/lib/gitlab/metrics/dashboard/base_service.rb
new file mode 100644
index 00000000000..94aabd0466c
--- /dev/null
+++ b/lib/gitlab/metrics/dashboard/base_service.rb
@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+# Searches a projects repository for a metrics dashboard and formats the output.
+# Expects any custom dashboards will be located in `.gitlab/dashboards`
+module Gitlab
+  module Metrics
+    module Dashboard
+      class BaseService < ::BaseService
+        DASHBOARD_LAYOUT_ERROR = Gitlab::Metrics::Dashboard::Stages::BaseStage::DashboardLayoutError
+
+        def get_dashboard
+          return error("#{dashboard_path} could not be found.", :not_found) unless path_available?
+
+          success(dashboard: process_dashboard)
+        rescue DASHBOARD_LAYOUT_ERROR => e
+          error(e.message, :unprocessable_entity)
+        end
+
+        # Summary of all known dashboards for the service.
+        # @return [Array<Hash>] ex) [{ path: String, default: Boolean }]
+        def all_dashboard_paths(_project)
+          raise NotImplementedError
+        end
+
+        private
+
+        # Returns a new dashboard Hash, supplemented with DB info
+        def process_dashboard
+          Gitlab::Metrics::Dashboard::Processor
+            .new(project, params[:environment], raw_dashboard)
+            .process(insert_project_metrics: insert_project_metrics?)
+        end
+
+        # @return [String] Relative filepath of the dashboard yml
+        def dashboard_path
+          params[:dashboard_path]
+        end
+
+        # Returns an un-processed dashboard from the cache.
+        def raw_dashboard
+          Rails.cache.fetch(cache_key) { get_raw_dashboard }
+        end
+
+        # @return [Hash] an unmodified dashboard
+        def get_raw_dashboard
+          raise NotImplementedError
+        end
+
+        # @return [String]
+        def cache_key
+          raise NotImplementedError
+        end
+
+        # Determines whether custom metrics should be included
+        # in the processed output.
+        def insert_project_metrics?
+          false
+        end
+
+        # Checks if dashboard path exists or should be rejected
+        # as a result of file-changes to the project repository.
+        # @return [Boolean]
+        def path_available?
+          available_paths = Gitlab::Metrics::Dashboard::Finder.find_all_paths(project)
+
+          available_paths.any? do |path_params|
+            path_params[:path] == dashboard_path
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/dashboard/finder.rb b/lib/gitlab/metrics/dashboard/finder.rb
new file mode 100644
index 00000000000..4a41590f000
--- /dev/null
+++ b/lib/gitlab/metrics/dashboard/finder.rb
@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+# Returns DB-supplmented dashboard info for determining
+# the layout of UI. Intended entry-point for the Metrics::Dashboard
+# module.
+module Gitlab
+  module Metrics
+    module Dashboard
+      class Finder
+        class << self
+          # Returns a formatted dashboard packed with DB info.
+          # @return [Hash]
+          def find(project, user, environment, dashboard_path = nil)
+            service = system_dashboard?(dashboard_path) ? system_service : project_service
+
+            service
+              .new(project, user, environment: environment, dashboard_path: dashboard_path)
+              .get_dashboard
+          end
+
+          # Summary of all known dashboards.
+          # @return [Array<Hash>] ex) [{ path: String, default: Boolean }]
+          def find_all_paths(project)
+            project.repository.metrics_dashboard_paths
+          end
+
+          # Summary of all known dashboards. Used to populate repo cache.
+          # Prefer #find_all_paths.
+          def find_all_paths_from_source(project)
+            system_service.all_dashboard_paths(project)
+            .+ project_service.all_dashboard_paths(project)
+          end
+
+          private
+
+          def system_service
+            Gitlab::Metrics::Dashboard::SystemDashboardService
+          end
+
+          def project_service
+            Gitlab::Metrics::Dashboard::ProjectDashboardService
+          end
+
+          def system_dashboard?(filepath)
+            !filepath || system_service.system_dashboard?(filepath)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/dashboard/processor.rb b/lib/gitlab/metrics/dashboard/processor.rb
index cc34ac53051..dd986020693 100644
--- a/lib/gitlab/metrics/dashboard/processor.rb
+++ b/lib/gitlab/metrics/dashboard/processor.rb
@@ -8,12 +8,17 @@ module Gitlab
       # the UI. These includes shared metric info, custom metrics
       # info, and alerts (only in EE).
       class Processor
-        SEQUENCE = [
+        SYSTEM_SEQUENCE = [
           Stages::CommonMetricsInserter,
           Stages::ProjectMetricsInserter,
           Stages::Sorter
         ].freeze
 
+        PROJECT_SEQUENCE = [
+          Stages::CommonMetricsInserter,
+          Stages::Sorter
+        ].freeze
+
         def initialize(project, environment, dashboard)
           @project = project
           @environment = environment
@@ -22,9 +27,9 @@ module Gitlab
 
         # Returns a new dashboard hash with the results of
         # running transforms on the dashboard.
-        def process
+        def process(insert_project_metrics:)
           @dashboard.deep_symbolize_keys.tap do |dashboard|
-            sequence.each do |stage|
+            sequence(insert_project_metrics).each do |stage|
               stage.new(@project, @environment, dashboard).transform!
             end
           end
@@ -32,8 +37,8 @@ module Gitlab
 
         private
 
-        def sequence
-          SEQUENCE
+        def sequence(insert_project_metrics)
+          insert_project_metrics ? SYSTEM_SEQUENCE : PROJECT_SEQUENCE
         end
       end
     end
diff --git a/lib/gitlab/metrics/dashboard/project_dashboard_service.rb b/lib/gitlab/metrics/dashboard/project_dashboard_service.rb
new file mode 100644
index 00000000000..fdffd067c93
--- /dev/null
+++ b/lib/gitlab/metrics/dashboard/project_dashboard_service.rb
@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+# Searches a projects repository for a metrics dashboard and formats the output.
+# Expects any custom dashboards will be located in `.gitlab/dashboards`
+# Use Gitlab::Metrics::Dashboard::Finder to retrive dashboards.
+module Gitlab
+  module Metrics
+    module Dashboard
+      class ProjectDashboardService < Gitlab::Metrics::Dashboard::BaseService
+        DASHBOARD_ROOT = ".gitlab/dashboards"
+
+        class << self
+          def all_dashboard_paths(project)
+            file_finder(project)
+              .list_files_for(DASHBOARD_ROOT)
+              .map do |filepath|
+                Rails.cache.delete(cache_key(project.id, filepath))
+
+                { path: filepath, default: false }
+              end
+          end
+
+          def file_finder(project)
+            Gitlab::Template::Finders::RepoTemplateFinder.new(project, DASHBOARD_ROOT, '.yml')
+          end
+
+          def cache_key(id, dashboard_path)
+            "project_#{id}_metrics_dashboard_#{dashboard_path}"
+          end
+        end
+
+        private
+
+        # Searches the project repo for a custom-defined dashboard.
+        def get_raw_dashboard
+          yml = self.class.file_finder(project).read(dashboard_path)
+
+          YAML.safe_load(yml)
+        end
+
+        def cache_key
+          self.class.cache_key(project.id, dashboard_path)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/dashboard/service.rb b/lib/gitlab/metrics/dashboard/service.rb
deleted file mode 100644
index 79d563cce4f..00000000000
--- a/lib/gitlab/metrics/dashboard/service.rb
+++ /dev/null
@@ -1,40 +0,0 @@
-# frozen_string_literal: true
-
-# Fetches the metrics dashboard layout and supplemented the output with DB info.
-module Gitlab
-  module Metrics
-    module Dashboard
-      class Service < ::BaseService
-        SYSTEM_DASHBOARD_NAME = 'common_metrics'
-        SYSTEM_DASHBOARD_PATH = Rails.root.join('config', 'prometheus', "#{SYSTEM_DASHBOARD_NAME}.yml")
-
-        # Returns a DB-supplemented json representation of a dashboard config file.
-        def get_dashboard
-          dashboard_string = Rails.cache.fetch(cache_key) { system_dashboard }
-
-          dashboard = process_dashboard(dashboard_string)
-
-          success(dashboard: dashboard)
-        rescue Gitlab::Metrics::Dashboard::Stages::BaseStage::DashboardLayoutError => e
-          error(e.message, :unprocessable_entity)
-        end
-
-        private
-
-        # Returns the base metrics shipped with every GitLab service.
-        def system_dashboard
-          YAML.safe_load(File.read(SYSTEM_DASHBOARD_PATH))
-        end
-
-        def cache_key
-          "metrics_dashboard_#{SYSTEM_DASHBOARD_NAME}"
-        end
-
-        # Returns a new dashboard Hash, supplemented with DB info
-        def process_dashboard(dashboard)
-          Gitlab::Metrics::Dashboard::Processor.new(project, params[:environment], dashboard).process
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/metrics/dashboard/stages/base_stage.rb b/lib/gitlab/metrics/dashboard/stages/base_stage.rb
index dd4aae6c115..a6d1f974556 100644
--- a/lib/gitlab/metrics/dashboard/stages/base_stage.rb
+++ b/lib/gitlab/metrics/dashboard/stages/base_stage.rb
@@ -36,7 +36,7 @@ module Gitlab
             raise DashboardLayoutError.new('Each "panel" must define an array :metrics')
           end
 
-          def for_metrics(dashboard)
+          def for_metrics
             missing_panel_groups! unless dashboard[:panel_groups].is_a?(Array)
 
             dashboard[:panel_groups].each do |panel_group|
diff --git a/lib/gitlab/metrics/dashboard/stages/common_metrics_inserter.rb b/lib/gitlab/metrics/dashboard/stages/common_metrics_inserter.rb
index 3406021bbea..188912bedb4 100644
--- a/lib/gitlab/metrics/dashboard/stages/common_metrics_inserter.rb
+++ b/lib/gitlab/metrics/dashboard/stages/common_metrics_inserter.rb
@@ -11,7 +11,7 @@ module Gitlab
           def transform!
             common_metrics = ::PrometheusMetric.common
 
-            for_metrics(dashboard) do |metric|
+            for_metrics do |metric|
               metric_record = common_metrics.find { |m| m.identifier == metric[:id] }
               metric[:metric_id] = metric_record.id if metric_record
             end
diff --git a/lib/gitlab/metrics/dashboard/system_dashboard_service.rb b/lib/gitlab/metrics/dashboard/system_dashboard_service.rb
new file mode 100644
index 00000000000..67509ed4230
--- /dev/null
+++ b/lib/gitlab/metrics/dashboard/system_dashboard_service.rb
@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+# Fetches the system metrics dashboard and formats the output.
+# Use Gitlab::Metrics::Dashboard::Finder to retrive dashboards.
+module Gitlab
+  module Metrics
+    module Dashboard
+      class SystemDashboardService < Gitlab::Metrics::Dashboard::BaseService
+        SYSTEM_DASHBOARD_PATH = 'config/prometheus/common_metrics.yml'
+
+        class << self
+          def all_dashboard_paths(_project)
+            [{
+              path: SYSTEM_DASHBOARD_PATH,
+              default: true
+            }]
+          end
+
+          def system_dashboard?(filepath)
+            filepath == SYSTEM_DASHBOARD_PATH
+          end
+        end
+
+        private
+
+        def dashboard_path
+          SYSTEM_DASHBOARD_PATH
+        end
+
+        # Returns the base metrics shipped with every GitLab service.
+        def get_raw_dashboard
+          yml = File.read(Rails.root.join(dashboard_path))
+
+          YAML.safe_load(yml)
+        end
+
+        def cache_key
+          "metrics_dashboard_#{dashboard_path}"
+        end
+
+        def insert_project_metrics?
+          true
+        end
+      end
+    end
+  end
+end