1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
|
package sshd
import (
"context"
"errors"
"time"
"golang.org/x/crypto/ssh"
"golang.org/x/sync/semaphore"
grpccodes "google.golang.org/grpc/codes"
grpcstatus "google.golang.org/grpc/status"
"gitlab.com/gitlab-org/gitlab-shell/client"
"gitlab.com/gitlab-org/gitlab-shell/internal/config"
"gitlab.com/gitlab-org/gitlab-shell/internal/metrics"
"gitlab.com/gitlab-org/labkit/log"
)
const KeepAliveMsg = "keepalive@openssh.com"
var EOFTimeout = 10 * time.Second
type connection struct {
cfg *config.Config
concurrentSessions *semaphore.Weighted
remoteAddr string
sconn *ssh.ServerConn
maxSessions int64
}
type channelHandler func(context.Context, ssh.Channel, <-chan *ssh.Request) error
func newConnection(cfg *config.Config, remoteAddr string, sconn *ssh.ServerConn) *connection {
maxSessions := cfg.Server.ConcurrentSessionsLimit
return &connection{
cfg: cfg,
maxSessions: maxSessions,
concurrentSessions: semaphore.NewWeighted(maxSessions),
remoteAddr: remoteAddr,
sconn: sconn,
}
}
func (c *connection) handle(ctx context.Context, chans <-chan ssh.NewChannel, handler channelHandler) {
ctxlog := log.WithContextFields(ctx, log.Fields{"remote_addr": c.remoteAddr})
if c.cfg.Server.ClientAliveIntervalSeconds > 0 {
ticker := time.NewTicker(c.cfg.Server.ClientAliveInterval())
defer ticker.Stop()
go c.sendKeepAliveMsg(ctx, ticker)
}
for newChannel := range chans {
ctxlog.WithField("channel_type", newChannel.ChannelType()).Info("connection: handle: new channel requested")
if newChannel.ChannelType() != "session" {
ctxlog.Info("connection: handle: unknown channel type")
newChannel.Reject(ssh.UnknownChannelType, "unknown channel type")
continue
}
if !c.concurrentSessions.TryAcquire(1) {
ctxlog.Info("connection: handle: too many concurrent sessions")
newChannel.Reject(ssh.ResourceShortage, "too many concurrent sessions")
metrics.SshdHitMaxSessions.Inc()
continue
}
channel, requests, err := newChannel.Accept()
if err != nil {
ctxlog.WithError(err).Error("connection: handle: accepting channel failed")
c.concurrentSessions.Release(1)
continue
}
go func() {
defer func(started time.Time) {
metrics.SshdSessionDuration.Observe(time.Since(started).Seconds())
}(time.Now())
defer c.concurrentSessions.Release(1)
// Prevent a panic in a single session from taking out the whole server
defer func() {
if err := recover(); err != nil {
ctxlog.WithField("recovered_error", err).Warn("panic handling session")
}
}()
metrics.SliSshdSessionsTotal.Inc()
err := handler(ctx, channel, requests)
if err != nil {
if grpcstatus.Convert(err).Code() == grpccodes.Canceled {
metrics.SshdCanceledSessions.Inc()
} else {
var apiError *client.ApiError
if !errors.As(err, &apiError) {
metrics.SliSshdSessionsErrorsTotal.Inc()
}
}
}
ctxlog.Info("connection: handle: done")
}()
}
// When a connection has been prematurely closed we block execution until all concurrent sessions are released
// in order to allow Gitaly complete the operations and close all the channels gracefully.
// If it didn't happen within timeout, we unblock the execution
// Related issue: https://gitlab.com/gitlab-org/gitlab-shell/-/issues/563
ctx, cancel := context.WithTimeout(ctx, EOFTimeout)
defer cancel()
c.concurrentSessions.Acquire(ctx, c.maxSessions)
}
func (c *connection) sendKeepAliveMsg(ctx context.Context, ticker *time.Ticker) {
ctxlog := log.WithContextFields(ctx, log.Fields{"remote_addr": c.remoteAddr})
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
ctxlog.Debug("session: handleShell: send keepalive message to a client")
c.sconn.SendRequest(KeepAliveMsg, true, nil)
}
}
}
|
