diff options
author | Igor Drozdov <idrozdov@gitlab.com> | 2021-07-21 11:24:00 +0300 |
---|---|---|
committer | Igor Drozdov <idrozdov@gitlab.com> | 2021-07-21 13:00:48 +0300 |
commit | 368a64546c97abdc7019f98cbc97eecdbda2b6a5 (patch) | |
tree | 6ba91a9dfb68e955575c6d851bddf0adfa1f5797 | |
parent | 50da611814d256c77e689977265ec7e07633a4dc (diff) | |
download | gitlab-shell-368a64546c97abdc7019f98cbc97eecdbda2b6a5.tar.gz |
Prometheus metrics for HTTP requests
A RoundTripper for tracking the duration of an http request
is introduced
-rw-r--r-- | internal/config/config.go | 10 | ||||
-rw-r--r-- | internal/config/config_test.go | 26 | ||||
-rw-r--r-- | internal/metrics/metrics.go | 63 | ||||
-rw-r--r-- | internal/sshd/connection.go | 42 |
4 files changed, 101 insertions, 40 deletions
diff --git a/internal/config/config.go b/internal/config/config.go index 4c75ad5..ba21219 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -10,9 +10,11 @@ import ( "sync" "time" + "github.com/prometheus/client_golang/prometheus/promhttp" yaml "gopkg.in/yaml.v2" "gitlab.com/gitlab-org/gitlab-shell/client" + "gitlab.com/gitlab-org/gitlab-shell/internal/metrics" ) const ( @@ -95,7 +97,7 @@ func (c *Config) ApplyGlobalState() { func (c *Config) HttpClient() *client.HttpClient { c.httpClientOnce.Do(func() { - c.httpClient = client.NewHTTPClient( + client := client.NewHTTPClient( c.GitlabUrl, c.GitlabRelativeURLRoot, c.HttpSettings.CaFile, @@ -103,6 +105,12 @@ func (c *Config) HttpClient() *client.HttpClient { c.HttpSettings.SelfSignedCert, c.HttpSettings.ReadTimeoutSeconds, ) + + tr := client.Transport + client.Transport = promhttp.InstrumentRoundTripperDuration(metrics.HttpRequestDuration, tr) + + + c.httpClient = client }) return c.httpClient diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 1a79320..521dd35 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -5,8 +5,10 @@ import ( "testing" "github.com/stretchr/testify/require" + "github.com/prometheus/client_golang/prometheus" "gitlab.com/gitlab-org/gitlab-shell/internal/testhelper" + "gitlab.com/gitlab-org/gitlab-shell/client/testserver" ) func TestConfigApplyGlobalState(t *testing.T) { @@ -22,3 +24,27 @@ func TestConfigApplyGlobalState(t *testing.T) { require.Equal(t, "foo", os.Getenv("SSL_CERT_DIR")) } + +func TestHttpClient(t *testing.T) { + url := testserver.StartHttpServer(t, []testserver.TestRequestHandler{}) + + config := &Config{GitlabUrl: url} + client := config.HttpClient() + + _, err := client.Get("http://host.com/path") + require.NoError(t, err) + + ms, err := prometheus.DefaultGatherer.Gather() + require.NoError(t, err) + + lastMetric := ms[0] + require.Equal(t, lastMetric.GetName(), "gitlab_shell_http_request_seconds") + + labels := lastMetric.GetMetric()[0].Label + + require.Equal(t, "code", labels[0].GetName()) + require.Equal(t, "404", labels[0].GetValue()) + + require.Equal(t, "method", labels[1].GetName()) + require.Equal(t, "get", labels[1].GetValue()) +} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go new file mode 100644 index 0000000..6174bca --- /dev/null +++ b/internal/metrics/metrics.go @@ -0,0 +1,63 @@ +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +const ( + namespace = "gitlab_shell" + sshdSubsystem = "sshd" + httpSubsystem = "http" +) + +var ( + SshdConnectionDuration = promauto.NewHistogram( + prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: sshdSubsystem, + Name: "connection_duration_seconds", + Help: "A histogram of latencies for connections to gitlab-shell sshd.", + Buckets: []float64{ + 0.005, /* 5ms */ + 0.025, /* 25ms */ + 0.1, /* 100ms */ + 0.5, /* 500ms */ + 1.0, /* 1s */ + 10.0, /* 10s */ + 30.0, /* 30s */ + 60.0, /* 1m */ + 300.0, /* 5m */ + }, + }, + ) + + SshdHitMaxSessions = promauto.NewCounter( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: sshdSubsystem, + Name: "concurrent_limited_sessions_total", + Help: "The number of times the concurrent sessions limit was hit in gitlab-shell sshd.", + }, + ) + + HttpRequestDuration = promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: httpSubsystem, + Name: "request_seconds", + Help: "A histogram of latencies for gitlab-shell http requests", + Buckets: []float64{ + 0.01, /* 10ms */ + 0.05, /* 50ms */ + 0.1, /* 100ms */ + 0.25, /* 250ms */ + 0.5, /* 500ms */ + 1.0, /* 1s */ + 5.0, /* 5s */ + 10.0, /* 10s */ + }, + }, + []string{"code", "method"}, + ) +) diff --git a/internal/sshd/connection.go b/internal/sshd/connection.go index a9b9e97..5ff055c 100644 --- a/internal/sshd/connection.go +++ b/internal/sshd/connection.go @@ -4,47 +4,11 @@ import ( "context" "time" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" log "github.com/sirupsen/logrus" "golang.org/x/crypto/ssh" "golang.org/x/sync/semaphore" -) - -const ( - namespace = "gitlab_shell" - sshdSubsystem = "sshd" -) - -var ( - sshdConnectionDuration = promauto.NewHistogram( - prometheus.HistogramOpts{ - Namespace: namespace, - Subsystem: sshdSubsystem, - Name: "connection_duration_seconds", - Help: "A histogram of latencies for connections to gitlab-shell sshd.", - Buckets: []float64{ - 0.005, /* 5ms */ - 0.025, /* 25ms */ - 0.1, /* 100ms */ - 0.5, /* 500ms */ - 1.0, /* 1s */ - 10.0, /* 10s */ - 30.0, /* 30s */ - 60.0, /* 1m */ - 300.0, /* 5m */ - }, - }, - ) - sshdHitMaxSessions = promauto.NewCounter( - prometheus.CounterOpts{ - Namespace: namespace, - Subsystem: sshdSubsystem, - Name: "concurrent_limited_sessions_total", - Help: "The number of times the concurrent sessions limit was hit in gitlab-shell sshd.", - }, - ) + "gitlab.com/gitlab-org/gitlab-shell/internal/metrics" ) type connection struct { @@ -64,7 +28,7 @@ func newConnection(maxSessions int64, remoteAddr string) *connection { } func (c *connection) handle(ctx context.Context, chans <-chan ssh.NewChannel, handler channelHandler) { - defer sshdConnectionDuration.Observe(time.Since(c.begin).Seconds()) + defer metrics.SshdConnectionDuration.Observe(time.Since(c.begin).Seconds()) for newChannel := range chans { if newChannel.ChannelType() != "session" { @@ -73,7 +37,7 @@ func (c *connection) handle(ctx context.Context, chans <-chan ssh.NewChannel, ha } if !c.concurrentSessions.TryAcquire(1) { newChannel.Reject(ssh.ResourceShortage, "too many concurrent sessions") - sshdHitMaxSessions.Inc() + metrics.SshdHitMaxSessions.Inc() continue } channel, requests, err := newChannel.Accept() |