Merge branch 'id-prometheus-metrics-for-http' into 'main'

Prometheus metrics for HTTP requests

See merge request gitlab-org/gitlab-shell!496

4 files changed, 101 insertions, 40 deletions

diff --git a/internal/config/config.go b/internal/config/config.go
index 4c75ad5..ba21219 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -10,9 +10,11 @@ import (
 	"sync"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	yaml "gopkg.in/yaml.v2"
 
 	"gitlab.com/gitlab-org/gitlab-shell/client"
+	"gitlab.com/gitlab-org/gitlab-shell/internal/metrics"
 )
 
 const (
@@ -95,7 +97,7 @@ func (c *Config) ApplyGlobalState() {
 
 func (c *Config) HttpClient() *client.HttpClient {
 	c.httpClientOnce.Do(func() {
-		c.httpClient = client.NewHTTPClient(
+		client := client.NewHTTPClient(
 			c.GitlabUrl,
 			c.GitlabRelativeURLRoot,
 			c.HttpSettings.CaFile,
@@ -103,6 +105,12 @@ func (c *Config) HttpClient() *client.HttpClient {
 			c.HttpSettings.SelfSignedCert,
 			c.HttpSettings.ReadTimeoutSeconds,
 		)
+
+		tr := client.Transport
+		client.Transport = promhttp.InstrumentRoundTripperDuration(metrics.HttpRequestDuration, tr)
+
+
+		c.httpClient = client
 	})
 
 	return c.httpClient
diff --git a/internal/config/config_test.go b/internal/config/config_test.go
index 1a79320..521dd35 100644
--- a/internal/config/config_test.go
+++ b/internal/config/config_test.go
@@ -5,8 +5,10 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/require"
+	"github.com/prometheus/client_golang/prometheus"
 
 	"gitlab.com/gitlab-org/gitlab-shell/internal/testhelper"
+	"gitlab.com/gitlab-org/gitlab-shell/client/testserver"
 )
 
 func TestConfigApplyGlobalState(t *testing.T) {
@@ -22,3 +24,27 @@ func TestConfigApplyGlobalState(t *testing.T) {
 
 	require.Equal(t, "foo", os.Getenv("SSL_CERT_DIR"))
 }
+
+func TestHttpClient(t *testing.T) {
+	url := testserver.StartHttpServer(t, []testserver.TestRequestHandler{})
+
+	config := &Config{GitlabUrl: url}
+	client := config.HttpClient()
+
+	_, err := client.Get("http://host.com/path")
+	require.NoError(t, err)
+
+	ms, err := prometheus.DefaultGatherer.Gather()
+	require.NoError(t, err)
+
+	lastMetric := ms[0]
+	require.Equal(t, lastMetric.GetName(), "gitlab_shell_http_request_seconds")
+
+	labels := lastMetric.GetMetric()[0].Label
+
+	require.Equal(t, "code", labels[0].GetName())
+	require.Equal(t, "404", labels[0].GetValue())
+
+	require.Equal(t, "method", labels[1].GetName())
+	require.Equal(t, "get", labels[1].GetValue())
+}
diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go
new file mode 100644
index 0000000..6174bca
--- /dev/null
+++ b/internal/metrics/metrics.go
@@ -0,0 +1,63 @@
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+const (
+	namespace     = "gitlab_shell"
+	sshdSubsystem = "sshd"
+	httpSubsystem = "http"
+)
+
+var (
+	SshdConnectionDuration = promauto.NewHistogram(
+		prometheus.HistogramOpts{
+			Namespace: namespace,
+			Subsystem: sshdSubsystem,
+			Name:      "connection_duration_seconds",
+			Help:      "A histogram of latencies for connections to gitlab-shell sshd.",
+			Buckets: []float64{
+				0.005, /* 5ms */
+				0.025, /* 25ms */
+				0.1,   /* 100ms */
+				0.5,   /* 500ms */
+				1.0,   /* 1s */
+				10.0,  /* 10s */
+				30.0,  /* 30s */
+				60.0,  /* 1m */
+				300.0, /* 5m */
+			},
+		},
+	)
+
+	SshdHitMaxSessions = promauto.NewCounter(
+		prometheus.CounterOpts{
+			Namespace: namespace,
+			Subsystem: sshdSubsystem,
+			Name:      "concurrent_limited_sessions_total",
+			Help:      "The number of times the concurrent sessions limit was hit in gitlab-shell sshd.",
+		},
+	)
+
+	HttpRequestDuration = promauto.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Namespace: namespace,
+			Subsystem: httpSubsystem,
+			Name:      "request_seconds",
+			Help:      "A histogram of latencies for gitlab-shell http requests",
+			Buckets: []float64{
+				0.01, /* 10ms */
+				0.05, /* 50ms */
+				0.1,  /* 100ms */
+				0.25, /* 250ms */
+				0.5,  /* 500ms */
+				1.0,  /* 1s */
+				5.0,  /* 5s */
+				10.0, /* 10s */
+			},
+		},
+		[]string{"code", "method"},
+	)
+)
diff --git a/internal/sshd/connection.go b/internal/sshd/connection.go
index a9b9e97..5ff055c 100644
--- a/internal/sshd/connection.go
+++ b/internal/sshd/connection.go
@@ -4,47 +4,11 @@ import (
 	"context"
 	"time"
 
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto"
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/crypto/ssh"
 	"golang.org/x/sync/semaphore"
-)
-
-const (
-	namespace     = "gitlab_shell"
-	sshdSubsystem = "sshd"
-)
-
-var (
-	sshdConnectionDuration = promauto.NewHistogram(
-		prometheus.HistogramOpts{
-			Namespace: namespace,
-			Subsystem: sshdSubsystem,
-			Name:      "connection_duration_seconds",
-			Help:      "A histogram of latencies for connections to gitlab-shell sshd.",
-			Buckets: []float64{
-				0.005, /* 5ms */
-				0.025, /* 25ms */
-				0.1,   /* 100ms */
-				0.5,   /* 500ms */
-				1.0,   /* 1s */
-				10.0,  /* 10s */
-				30.0,  /* 30s */
-				60.0,  /* 1m */
-				300.0, /* 5m */
-			},
-		},
-	)
 
-	sshdHitMaxSessions = promauto.NewCounter(
-		prometheus.CounterOpts{
-			Namespace: namespace,
-			Subsystem: sshdSubsystem,
-			Name:      "concurrent_limited_sessions_total",
-			Help:      "The number of times the concurrent sessions limit was hit in gitlab-shell sshd.",
-		},
-	)
+	"gitlab.com/gitlab-org/gitlab-shell/internal/metrics"
 )
 
 type connection struct {
@@ -64,7 +28,7 @@ func newConnection(maxSessions int64, remoteAddr string) *connection {
 }
 
 func (c *connection) handle(ctx context.Context, chans <-chan ssh.NewChannel, handler channelHandler) {
-	defer sshdConnectionDuration.Observe(time.Since(c.begin).Seconds())
+	defer metrics.SshdConnectionDuration.Observe(time.Since(c.begin).Seconds())
 
 	for newChannel := range chans {
 		if newChannel.ChannelType() != "session" {
@@ -73,7 +37,7 @@ func (c *connection) handle(ctx context.Context, chans <-chan ssh.NewChannel, ha
 		}
 		if !c.concurrentSessions.TryAcquire(1) {
 			newChannel.Reject(ssh.ResourceShortage, "too many concurrent sessions")
-			sshdHitMaxSessions.Inc()
+			metrics.SshdHitMaxSessions.Inc()
 			continue
 		}
 		channel, requests, err := newChannel.Accept()