Merge branch 'instrument-infra' into 'master'

Add Prometheus metrics endpoint and basic infrastructure to meter code

See merge request !11553

6 files changed, 265 insertions, 154 deletions

diff --git a/lib/api/settings.rb b/lib/api/settings.rb
index 82f513c984e..25027c3b114 100644
--- a/lib/api/settings.rb
+++ b/lib/api/settings.rb
@@ -110,6 +110,7 @@ module API
       optional :default_artifacts_expire_in, type: String, desc: "Set the default expiration time for each job's artifacts"
       optional :max_pages_size, type: Integer, desc: 'Maximum size of pages in MB'
       optional :container_registry_token_expire_delay, type: Integer, desc: 'Authorization token duration (minutes)'
+      optional :prometheus_metrics_enabled, type: Boolean, desc: 'Enable Prometheus metrics'
       optional :metrics_enabled, type: Boolean, desc: 'Enable the InfluxDB metrics'
       given metrics_enabled: ->(val) { val } do
         requires :metrics_host, type: String, desc: 'The InfluxDB host'
diff --git a/lib/gitlab/health_checks/prometheus_text_format.rb b/lib/gitlab/health_checks/prometheus_text_format.rb
new file mode 100644
index 00000000000..462c8e736a0
--- /dev/null
+++ b/lib/gitlab/health_checks/prometheus_text_format.rb
@@ -0,0 +1,40 @@
+module Gitlab
+  module HealthChecks
+    class PrometheusTextFormat
+      def marshal(metrics)
+        "#{metrics_with_type_declarations(metrics).join("\n")}\n"
+      end
+
+      private
+
+      def metrics_with_type_declarations(metrics)
+        type_declaration_added = {}
+
+        metrics.flat_map do |metric|
+          metric_lines = []
+
+          unless type_declaration_added.has_key?(metric.name)
+            type_declaration_added[metric.name] = true
+            metric_lines << metric_type_declaration(metric)
+          end
+
+          metric_lines << metric_text(metric)
+        end
+      end
+
+      def metric_type_declaration(metric)
+        "# TYPE #{metric.name} gauge"
+      end
+
+      def metric_text(metric)
+        labels = metric.labels&.map { |key, value| "#{key}=\"#{value}\"" }&.join(',') || ''
+
+        if labels.empty?
+          "#{metric.name} #{metric.value}"
+        else
+          "#{metric.name}{#{labels}} #{metric.value}"
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb
index cb8db2f1e9f..4779755bb22 100644
--- a/lib/gitlab/metrics.rb
+++ b/lib/gitlab/metrics.rb
@@ -1,161 +1,10 @@
 module Gitlab
   module Metrics
-    extend Gitlab::CurrentSettings
-
-    RAILS_ROOT   = Rails.root.to_s
-    METRICS_ROOT = Rails.root.join('lib', 'gitlab', 'metrics').to_s
-    PATH_REGEX   = /^#{RAILS_ROOT}\/?/
-
-    def self.settings
-      @settings ||= {
-        enabled:               current_application_settings[:metrics_enabled],
-        pool_size:             current_application_settings[:metrics_pool_size],
-        timeout:               current_application_settings[:metrics_timeout],
-        method_call_threshold: current_application_settings[:metrics_method_call_threshold],
-        host:                  current_application_settings[:metrics_host],
-        port:                  current_application_settings[:metrics_port],
-        sample_interval:       current_application_settings[:metrics_sample_interval] || 15,
-        packet_size:           current_application_settings[:metrics_packet_size] || 1
-      }
-    end
+    extend Gitlab::Metrics::InfluxDb
+    extend Gitlab::Metrics::Prometheus
 
     def self.enabled?
-      settings[:enabled] || false
-    end
-
-    def self.mri?
-      RUBY_ENGINE == 'ruby'
-    end
-
-    def self.method_call_threshold
-      # This is memoized since this method is called for every instrumented
-      # method. Loading data from an external cache on every method call slows
-      # things down too much.
-      @method_call_threshold ||= settings[:method_call_threshold]
-    end
-
-    def self.pool
-      @pool
-    end
-
-    def self.submit_metrics(metrics)
-      prepared = prepare_metrics(metrics)
-
-      pool.with do |connection|
-        prepared.each_slice(settings[:packet_size]) do |slice|
-          begin
-            connection.write_points(slice)
-          rescue StandardError
-          end
-        end
-      end
-    rescue Errno::EADDRNOTAVAIL, SocketError => ex
-      Gitlab::EnvironmentLogger.error('Cannot resolve InfluxDB address. GitLab Performance Monitoring will not work.')
-      Gitlab::EnvironmentLogger.error(ex)
-    end
-
-    def self.prepare_metrics(metrics)
-      metrics.map do |hash|
-        new_hash = hash.symbolize_keys
-
-        new_hash[:tags].each do |key, value|
-          if value.blank?
-            new_hash[:tags].delete(key)
-          else
-            new_hash[:tags][key] = escape_value(value)
-          end
-        end
-
-        new_hash
-      end
-    end
-
-    def self.escape_value(value)
-      value.to_s.gsub('=', '\\=')
-    end
-
-    # Measures the execution time of a block.
-    #
-    # Example:
-    #
-    #     Gitlab::Metrics.measure(:find_by_username_duration) do
-    #       User.find_by_username(some_username)
-    #     end
-    #
-    # name - The name of the field to store the execution time in.
-    #
-    # Returns the value yielded by the supplied block.
-    def self.measure(name)
-      trans = current_transaction
-
-      return yield unless trans
-
-      real_start = Time.now.to_f
-      cpu_start = System.cpu_time
-
-      retval = yield
-
-      cpu_stop = System.cpu_time
-      real_stop = Time.now.to_f
-
-      real_time = (real_stop - real_start) * 1000.0
-      cpu_time = cpu_stop - cpu_start
-
-      trans.increment("#{name}_real_time", real_time)
-      trans.increment("#{name}_cpu_time", cpu_time)
-      trans.increment("#{name}_call_count", 1)
-
-      retval
-    end
-
-    # Adds a tag to the current transaction (if any)
-    #
-    # name - The name of the tag to add.
-    # value - The value of the tag.
-    def self.tag_transaction(name, value)
-      trans = current_transaction
-
-      trans&.add_tag(name, value)
-    end
-
-    # Sets the action of the current transaction (if any)
-    #
-    # action - The name of the action.
-    def self.action=(action)
-      trans = current_transaction
-
-      trans&.action = action
-    end
-
-    # Tracks an event.
-    #
-    # See `Gitlab::Metrics::Transaction#add_event` for more details.
-    def self.add_event(*args)
-      trans = current_transaction
-
-      trans&.add_event(*args)
-    end
-
-    # Returns the prefix to use for the name of a series.
-    def self.series_prefix
-      @series_prefix ||= Sidekiq.server? ? 'sidekiq_' : 'rails_'
-    end
-
-    # Allow access from other metrics related middlewares
-    def self.current_transaction
-      Transaction.current
-    end
-
-    # When enabled this should be set before being used as the usual pattern
-    # "@foo ||= bar" is _not_ thread-safe.
-    if enabled?
-      @pool = ConnectionPool.new(size: settings[:pool_size], timeout: settings[:timeout]) do
-        host = settings[:host]
-        port = settings[:port]
-
-        InfluxDB::Client.
-          new(udp: { host: host, port: port })
-      end
+      influx_metrics_enabled? || prometheus_metrics_enabled?
     end
   end
 end
diff --git a/lib/gitlab/metrics/influx_db.rb b/lib/gitlab/metrics/influx_db.rb
new file mode 100644
index 00000000000..3a39791edbf
--- /dev/null
+++ b/lib/gitlab/metrics/influx_db.rb
@@ -0,0 +1,170 @@
+module Gitlab
+  module Metrics
+    module InfluxDb
+      extend Gitlab::CurrentSettings
+      extend self
+
+      MUTEX = Mutex.new
+      private_constant :MUTEX
+
+      def influx_metrics_enabled?
+        settings[:enabled] || false
+      end
+
+      RAILS_ROOT = Rails.root.to_s
+      METRICS_ROOT = Rails.root.join('lib', 'gitlab', 'metrics').to_s
+      PATH_REGEX = /^#{RAILS_ROOT}\/?/
+
+      def settings
+        @settings ||= {
+          enabled: current_application_settings[:metrics_enabled],
+          pool_size: current_application_settings[:metrics_pool_size],
+          timeout: current_application_settings[:metrics_timeout],
+          method_call_threshold: current_application_settings[:metrics_method_call_threshold],
+          host: current_application_settings[:metrics_host],
+          port: current_application_settings[:metrics_port],
+          sample_interval: current_application_settings[:metrics_sample_interval] || 15,
+          packet_size: current_application_settings[:metrics_packet_size] || 1
+        }
+      end
+
+      def mri?
+        RUBY_ENGINE == 'ruby'
+      end
+
+      def method_call_threshold
+        # This is memoized since this method is called for every instrumented
+        # method. Loading data from an external cache on every method call slows
+        # things down too much.
+        @method_call_threshold ||= settings[:method_call_threshold]
+      end
+
+      def submit_metrics(metrics)
+        prepared = prepare_metrics(metrics)
+
+        pool&.with do |connection|
+          prepared.each_slice(settings[:packet_size]) do |slice|
+            begin
+              connection.write_points(slice)
+            rescue StandardError
+            end
+          end
+        end
+      rescue Errno::EADDRNOTAVAIL, SocketError => ex
+        Gitlab::EnvironmentLogger.error('Cannot resolve InfluxDB address. GitLab Performance Monitoring will not work.')
+        Gitlab::EnvironmentLogger.error(ex)
+      end
+
+      def prepare_metrics(metrics)
+        metrics.map do |hash|
+          new_hash = hash.symbolize_keys
+
+          new_hash[:tags].each do |key, value|
+            if value.blank?
+              new_hash[:tags].delete(key)
+            else
+              new_hash[:tags][key] = escape_value(value)
+            end
+          end
+
+          new_hash
+        end
+      end
+
+      def escape_value(value)
+        value.to_s.gsub('=', '\\=')
+      end
+
+      # Measures the execution time of a block.
+      #
+      # Example:
+      #
+      #     Gitlab::Metrics.measure(:find_by_username_duration) do
+      #       User.find_by_username(some_username)
+      #     end
+      #
+      # name - The name of the field to store the execution time in.
+      #
+      # Returns the value yielded by the supplied block.
+      def measure(name)
+        trans = current_transaction
+
+        return yield unless trans
+
+        real_start = Time.now.to_f
+        cpu_start = System.cpu_time
+
+        retval = yield
+
+        cpu_stop = System.cpu_time
+        real_stop = Time.now.to_f
+
+        real_time = (real_stop - real_start) * 1000.0
+        cpu_time = cpu_stop - cpu_start
+
+        trans.increment("#{name}_real_time", real_time)
+        trans.increment("#{name}_cpu_time", cpu_time)
+        trans.increment("#{name}_call_count", 1)
+
+        retval
+      end
+
+      # Adds a tag to the current transaction (if any)
+      #
+      # name - The name of the tag to add.
+      # value - The value of the tag.
+      def tag_transaction(name, value)
+        trans = current_transaction
+
+        trans&.add_tag(name, value)
+      end
+
+      # Sets the action of the current transaction (if any)
+      #
+      # action - The name of the action.
+      def action=(action)
+        trans = current_transaction
+
+        trans&.action = action
+      end
+
+      # Tracks an event.
+      #
+      # See `Gitlab::Metrics::Transaction#add_event` for more details.
+      def add_event(*args)
+        trans = current_transaction
+
+        trans&.add_event(*args)
+      end
+
+      # Returns the prefix to use for the name of a series.
+      def series_prefix
+        @series_prefix ||= Sidekiq.server? ? 'sidekiq_' : 'rails_'
+      end
+
+      # Allow access from other metrics related middlewares
+      def current_transaction
+        Transaction.current
+      end
+
+      # When enabled this should be set before being used as the usual pattern
+      # "@foo ||= bar" is _not_ thread-safe.
+      def pool
+        if influx_metrics_enabled?
+          if @pool.nil?
+            MUTEX.synchronize do
+              @pool ||= ConnectionPool.new(size: settings[:pool_size], timeout: settings[:timeout]) do
+                host = settings[:host]
+                port = settings[:port]
+
+                InfluxDB::Client.
+                  new(udp: { host: host, port: port })
+              end
+            end
+          end
+          @pool
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/null_metric.rb b/lib/gitlab/metrics/null_metric.rb
new file mode 100644
index 00000000000..3b5a2907195
--- /dev/null
+++ b/lib/gitlab/metrics/null_metric.rb
@@ -0,0 +1,10 @@
+module Gitlab
+  module Metrics
+    # Mocks ::Prometheus::Client::Metric and all derived metrics
+    class NullMetric
+      def method_missing(name, *args, &block)
+        nil
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/metrics/prometheus.rb b/lib/gitlab/metrics/prometheus.rb
new file mode 100644
index 00000000000..60686509332
--- /dev/null
+++ b/lib/gitlab/metrics/prometheus.rb
@@ -0,0 +1,41 @@
+require 'prometheus/client'
+
+module Gitlab
+  module Metrics
+    module Prometheus
+      include Gitlab::CurrentSettings
+
+      def prometheus_metrics_enabled?
+        @prometheus_metrics_enabled ||= current_application_settings[:prometheus_metrics_enabled] || false
+      end
+
+      def registry
+        @registry ||= ::Prometheus::Client.registry
+      end
+
+      def counter(name, docstring, base_labels = {})
+        provide_metric(name) || registry.counter(name, docstring, base_labels)
+      end
+
+      def summary(name, docstring, base_labels = {})
+        provide_metric(name) || registry.summary(name, docstring, base_labels)
+      end
+
+      def gauge(name, docstring, base_labels = {})
+        provide_metric(name) || registry.gauge(name, docstring, base_labels)
+      end
+
+      def histogram(name, docstring, base_labels = {}, buckets = ::Prometheus::Client::Histogram::DEFAULT_BUCKETS)
+        provide_metric(name) || registry.histogram(name, docstring, base_labels, buckets)
+      end
+
+      def provide_metric(name)
+        if prometheus_metrics_enabled?
+          registry.get(name)
+        else
+          NullMetric.new
+        end
+      end
+    end
+  end
+end