diff options
author | Yorick Peterse <yorickpeterse@gmail.com> | 2015-12-29 13:40:42 +0100 |
---|---|---|
committer | Yorick Peterse <yorickpeterse@gmail.com> | 2015-12-29 14:53:45 +0100 |
commit | 620e7bb3d60c3685b494b26e256b793a47621da4 (patch) | |
tree | 68291922508c3ea49ffa16f0f8c3bd92d2489ae3 /lib | |
parent | 03478e6d5b98a723fbb349dac2c8495f75909a08 (diff) | |
download | gitlab-ce-620e7bb3d60c3685b494b26e256b793a47621da4.tar.gz |
Write to InfluxDB directly via UDP
This removes the need for Sidekiq and any overhead/problems introduced
by TCP. There are a few things to take into account:
1. When writing data to InfluxDB you may still get an error if the
server becomes unavailable during the write. Because of this we're
catching all exceptions and just ignore them (for now).
2. Writing via UDP apparently requires the timestamp to be in
nanoseconds. Without this data either isn't written properly.
3. Due to the restrictions on UDP buffer sizes we're writing metrics one
by one, instead of writing all of them at once.
Diffstat (limited to 'lib')
-rw-r--r-- | lib/gitlab/metrics.rb | 38 | ||||
-rw-r--r-- | lib/gitlab/metrics/metric.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/metrics/obfuscated_sql.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/metrics/sampler.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/metrics/sidekiq_middleware.rb | 7 | ||||
-rw-r--r-- | lib/gitlab/metrics/transaction.rb | 2 |
6 files changed, 40 insertions, 13 deletions
diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index 9470633b065..0869d007ca5 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -66,6 +66,39 @@ module Gitlab end end + def self.submit_metrics(metrics) + prepared = prepare_metrics(metrics) + + pool.with do |connection| + prepared.each do |metric| + begin + connection.write_points([metric]) + rescue StandardError + end + end + end + end + + def self.prepare_metrics(metrics) + metrics.map do |hash| + new_hash = hash.symbolize_keys + + new_hash[:tags].each do |key, value| + if value.blank? + new_hash[:tags].delete(key) + else + new_hash[:tags][key] = escape_value(value) + end + end + + new_hash + end + end + + def self.escape_value(value) + value.to_s.gsub('=', '\\=') + end + @hostname = Socket.gethostname # When enabled this should be set before being used as the usual pattern @@ -73,11 +106,12 @@ module Gitlab if enabled? @pool = ConnectionPool.new(size: pool_size, timeout: timeout) do host = settings[:metrics_host] - db = settings[:metrics_database] user = settings[:metrics_username] pw = settings[:metrics_password] + port = settings[:metrics_port] - InfluxDB::Client.new(db, host: host, username: user, password: pw) + InfluxDB::Client. + new(udp: { host: host, port: port }, username: user, password: pw) end end end diff --git a/lib/gitlab/metrics/metric.rb b/lib/gitlab/metrics/metric.rb index f592f4e571f..79241f56874 100644 --- a/lib/gitlab/metrics/metric.rb +++ b/lib/gitlab/metrics/metric.rb @@ -26,7 +26,7 @@ module Gitlab process_type: Sidekiq.server? ? 'sidekiq' : 'rails' ), values: @values, - timestamp: @created_at.to_i + timestamp: @created_at.to_i * 1_000_000_000 } end end diff --git a/lib/gitlab/metrics/obfuscated_sql.rb b/lib/gitlab/metrics/obfuscated_sql.rb index 2e932fb3049..fe97d7a0534 100644 --- a/lib/gitlab/metrics/obfuscated_sql.rb +++ b/lib/gitlab/metrics/obfuscated_sql.rb @@ -40,7 +40,7 @@ module Gitlab sql = sql.delete('"') end - sql.gsub("\n", ' ') + sql.tr("\n", ' ') end end end diff --git a/lib/gitlab/metrics/sampler.rb b/lib/gitlab/metrics/sampler.rb index 828ee1f8c62..998578e1c0a 100644 --- a/lib/gitlab/metrics/sampler.rb +++ b/lib/gitlab/metrics/sampler.rb @@ -46,7 +46,7 @@ module Gitlab end def flush - MetricsWorker.perform_async(@metrics.map(&:to_hash)) + Metrics.submit_metrics(@metrics.map(&:to_hash)) end def sample_memory_usage diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb index ec10707d1fb..ad441decfa2 100644 --- a/lib/gitlab/metrics/sidekiq_middleware.rb +++ b/lib/gitlab/metrics/sidekiq_middleware.rb @@ -5,13 +5,6 @@ module Gitlab # This middleware is intended to be used as a server-side middleware. class SidekiqMiddleware def call(worker, message, queue) - # We don't want to track the MetricsWorker itself as otherwise we'll end - # up in an infinite loop. - if worker.class == MetricsWorker - yield - return - end - trans = Transaction.new begin diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index 568f9d6ae0c..a61dbd989e7 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -59,7 +59,7 @@ module Gitlab end def submit - MetricsWorker.perform_async(@metrics.map(&:to_hash)) + Metrics.submit_metrics(@metrics.map(&:to_hash)) end end end |