cgroup2: use shim V2

* Requires containerd binaries from containerd/containerd#3799 . Metrics are unimplemented yet.
* Works with crun v0.10.4, but `--security-opt seccomp=unconfined` is needed unless using master version of libseccomp
  ( containers/crun#156, seccomp/libseccomp#177 )
* Doesn't work with master runc yet
* Resource limitations are unimplemented

Signed-off-by: Akihiro Suda <akihiro.suda.cz@hco.ntt.co.jp>

10 files changed, 93 insertions, 26 deletions

diff --git a/daemon/daemon.go b/daemon/daemon.go
index 8a11fa5faa..f139fc6d83 100644
--- a/daemon/daemon.go
+++ b/daemon/daemon.go
@@ -794,6 +794,7 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
 		PluginStore: pluginStore,
 		startupDone: make(chan struct{}),
 	}
+
 	// Ensure the daemon is properly shutdown if there is a failure during
 	// initialization
 	defer func() {
@@ -914,7 +915,7 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
 			}
 		}
 
-		return pluginexec.New(ctx, getPluginExecRoot(config.Root), pluginCli, config.ContainerdPluginNamespace, m)
+		return pluginexec.New(ctx, getPluginExecRoot(config.Root), pluginCli, config.ContainerdPluginNamespace, m, d.useShimV2())
 	}
 
 	// Plugin system initialization should happen before restore. Do not change order.
@@ -1063,7 +1064,7 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
 
 	go d.execCommandGC()
 
-	d.containerd, err = libcontainerd.NewClient(ctx, d.containerdCli, filepath.Join(config.ExecRoot, "containerd"), config.ContainerdNamespace, d)
+	d.containerd, err = libcontainerd.NewClient(ctx, d.containerdCli, filepath.Join(config.ExecRoot, "containerd"), config.ContainerdNamespace, d, d.useShimV2())
 	if err != nil {
 		return nil, err
 	}
diff --git a/daemon/daemon_unix.go b/daemon/daemon_unix.go
index 37bac78603..92d6a5440f 100644
--- a/daemon/daemon_unix.go
+++ b/daemon/daemon_unix.go
@@ -1639,3 +1639,7 @@ func (daemon *Daemon) setupSeccompProfile() error {
 	}
 	return nil
 }
+
+func (daemon *Daemon) useShimV2() bool {
+	return cgroups.IsCgroup2UnifiedMode()
+}
diff --git a/daemon/daemon_windows.go b/daemon/daemon_windows.go
index 8a912f493e..021b7b8f0a 100644
--- a/daemon/daemon_windows.go
+++ b/daemon/daemon_windows.go
@@ -653,3 +653,7 @@ func (daemon *Daemon) initRuntimes(_ map[string]types.Runtime) error {
 
 func setupResolvConf(config *config.Config) {
 }
+
+func (daemon *Daemon) useShimV2() bool {
+	return true
+}
diff --git a/daemon/start_unix.go b/daemon/start_unix.go
index e680b95f42..73963b9cf6 100644
--- a/daemon/start_unix.go
+++ b/daemon/start_unix.go
@@ -8,6 +8,7 @@ import (
 	"path/filepath"
 
 	"github.com/containerd/containerd/runtime/linux/runctypes"
+	v2runcoptions "github.com/containerd/containerd/runtime/v2/runc/options"
 	"github.com/docker/docker/container"
 	"github.com/docker/docker/errdefs"
 	"github.com/pkg/errors"
@@ -43,6 +44,20 @@ func (daemon *Daemon) getLibcontainerdCreateOptions(container *container.Contain
 	if err != nil {
 		return nil, err
 	}
+	if daemon.useShimV2() {
+		opts := &v2runcoptions.Options{
+			BinaryName: path,
+			Root: filepath.Join(daemon.configStore.ExecRoot,
+				fmt.Sprintf("runtime-%s", container.HostConfig.Runtime)),
+		}
+
+		if UsingSystemd(daemon.configStore) {
+			opts.SystemdCgroup = true
+		}
+
+		return opts, nil
+
+	}
 	opts := &runctypes.RuncOptions{
 		Runtime: path,
 		RuntimeRoot: filepath.Join(daemon.configStore.ExecRoot,
diff --git a/libcontainerd/libcontainerd_linux.go b/libcontainerd/libcontainerd_linux.go
index ec195a7905..3b008fe256 100644
--- a/libcontainerd/libcontainerd_linux.go
+++ b/libcontainerd/libcontainerd_linux.go
@@ -9,6 +9,6 @@ import (
 )
 
 // NewClient creates a new libcontainerd client from a containerd client
-func NewClient(ctx context.Context, cli *containerd.Client, stateDir, ns string, b libcontainerdtypes.Backend) (libcontainerdtypes.Client, error) {
-	return remote.NewClient(ctx, cli, stateDir, ns, b)
+func NewClient(ctx context.Context, cli *containerd.Client, stateDir, ns string, b libcontainerdtypes.Backend, useShimV2 bool) (libcontainerdtypes.Client, error) {
+	return remote.NewClient(ctx, cli, stateDir, ns, b, useShimV2)
 }
diff --git a/libcontainerd/libcontainerd_windows.go b/libcontainerd/libcontainerd_windows.go
index 61f19ba087..5a64180be4 100644
--- a/libcontainerd/libcontainerd_windows.go
+++ b/libcontainerd/libcontainerd_windows.go
@@ -11,9 +11,10 @@ import (
 )
 
 // NewClient creates a new libcontainerd client from a containerd client
-func NewClient(ctx context.Context, cli *containerd.Client, stateDir, ns string, b libcontainerdtypes.Backend) (libcontainerdtypes.Client, error) {
+func NewClient(ctx context.Context, cli *containerd.Client, stateDir, ns string, b libcontainerdtypes.Backend, useShimV2 bool) (libcontainerdtypes.Client, error) {
 	if !system.ContainerdRuntimeSupported() {
+		// useShimV2 is ignored for windows
 		return local.NewClient(ctx, cli, stateDir, ns, b)
 	}
-	return remote.NewClient(ctx, cli, stateDir, ns, b)
+	return remote.NewClient(ctx, cli, stateDir, ns, b, useShimV2)
 }
diff --git a/libcontainerd/remote/client.go b/libcontainerd/remote/client.go
index 93cc4e73b9..9bc4ffde02 100644
--- a/libcontainerd/remote/client.go
+++ b/libcontainerd/remote/client.go
@@ -23,6 +23,7 @@ import (
 	"github.com/containerd/containerd/events"
 	"github.com/containerd/containerd/images"
 	"github.com/containerd/containerd/runtime/linux/runctypes"
+	v2runcoptions "github.com/containerd/containerd/runtime/v2/runc/options"
 	"github.com/containerd/typeurl"
 	"github.com/docker/docker/errdefs"
 	"github.com/docker/docker/libcontainerd/queue"
@@ -45,21 +46,27 @@ type client struct {
 	logger   *logrus.Entry
 	ns       string
 
-	backend libcontainerdtypes.Backend
-	eventQ  queue.Queue
-	oomMu   sync.Mutex
-	oom     map[string]bool
+	backend         libcontainerdtypes.Backend
+	eventQ          queue.Queue
+	oomMu           sync.Mutex
+	oom             map[string]bool
+	useShimV2       bool
+	v2runcoptionsMu sync.Mutex
+	// v2runcoptions is used for copying options specified on Create() to Start()
+	v2runcoptions map[string]v2runcoptions.Options
 }
 
 // NewClient creates a new libcontainerd client from a containerd client
-func NewClient(ctx context.Context, cli *containerd.Client, stateDir, ns string, b libcontainerdtypes.Backend) (libcontainerdtypes.Client, error) {
+func NewClient(ctx context.Context, cli *containerd.Client, stateDir, ns string, b libcontainerdtypes.Backend, useShimV2 bool) (libcontainerdtypes.Client, error) {
 	c := &client{
-		client:   cli,
-		stateDir: stateDir,
-		logger:   logrus.WithField("module", "libcontainerd").WithField("namespace", ns),
-		ns:       ns,
-		backend:  b,
-		oom:      make(map[string]bool),
+		client:        cli,
+		stateDir:      stateDir,
+		logger:        logrus.WithField("module", "libcontainerd").WithField("namespace", ns),
+		ns:            ns,
+		backend:       b,
+		oom:           make(map[string]bool),
+		useShimV2:     useShimV2,
+		v2runcoptions: make(map[string]v2runcoptions.Options),
 	}
 
 	go c.processEventStream(ctx, ns)
@@ -126,9 +133,13 @@ func (c *client) Create(ctx context.Context, id string, ociSpec *specs.Spec, run
 	bdir := c.bundleDir(id)
 	c.logger.WithField("bundle", bdir).WithField("root", ociSpec.Root.Path).Debug("bundle dir created")
 
+	rt := runtimeName
+	if c.useShimV2 {
+		rt = shimV2RuntimeName
+	}
 	newOpts := []containerd.NewContainerOpts{
 		containerd.WithSpec(ociSpec),
-		containerd.WithRuntime(runtimeName, runtimeOptions),
+		containerd.WithRuntime(rt, runtimeOptions),
 		WithBundle(bdir, ociSpec),
 	}
 	opts = append(opts, newOpts...)
@@ -140,6 +151,13 @@ func (c *client) Create(ctx context.Context, id string, ociSpec *specs.Spec, run
 		}
 		return wrapError(err)
 	}
+	if c.useShimV2 {
+		if x, ok := runtimeOptions.(*v2runcoptions.Options); ok {
+			c.v2runcoptionsMu.Lock()
+			c.v2runcoptions[id] = *x
+			c.v2runcoptionsMu.Unlock()
+		}
+	}
 	return nil
 }
 
@@ -200,11 +218,26 @@ func (c *client) Start(ctx context.Context, id, checkpointDir string, withStdin
 
 	if runtime.GOOS != "windows" {
 		taskOpts = append(taskOpts, func(_ context.Context, _ *containerd.Client, info *containerd.TaskInfo) error {
-			info.Options = &runctypes.CreateOptions{
-				IoUid:       uint32(uid),
-				IoGid:       uint32(gid),
-				NoPivotRoot: os.Getenv("DOCKER_RAMDISK") != "",
+			if c.useShimV2 {
+				// For v2, we need to inherit options specified on Create
+				c.v2runcoptionsMu.Lock()
+				opts, ok := c.v2runcoptions[id]
+				c.v2runcoptionsMu.Unlock()
+				if !ok {
+					opts = v2runcoptions.Options{}
+				}
+				opts.IoUid = uint32(uid)
+				opts.IoGid = uint32(gid)
+				opts.NoPivotRoot = os.Getenv("DOCKER_RAMDISK") != ""
+				info.Options = &opts
+			} else {
+				info.Options = &runctypes.CreateOptions{
+					IoUid:       uint32(uid),
+					IoGid:       uint32(gid),
+					NoPivotRoot: os.Getenv("DOCKER_RAMDISK") != "",
+				}
 			}
+
 			return nil
 		})
 	} else {
@@ -466,6 +499,9 @@ func (c *client) Delete(ctx context.Context, containerID string) error {
 	c.oomMu.Lock()
 	delete(c.oom, containerID)
 	c.oomMu.Unlock()
+	c.v2runcoptionsMu.Lock()
+	delete(c.v2runcoptions, containerID)
+	c.v2runcoptionsMu.Unlock()
 	if os.Getenv("LIBCONTAINERD_NOCLEAN") != "1" {
 		if err := os.RemoveAll(bundle); err != nil {
 			c.logger.WithError(err).WithFields(logrus.Fields{
diff --git a/libcontainerd/remote/client_linux.go b/libcontainerd/remote/client_linux.go
index 486c8538e0..637ac94d82 100644
--- a/libcontainerd/remote/client_linux.go
+++ b/libcontainerd/remote/client_linux.go
@@ -16,7 +16,10 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-const runtimeName = "io.containerd.runtime.v1.linux"
+const (
+	runtimeName       = "io.containerd.runtime.v1.linux"
+	shimV2RuntimeName = "io.containerd.runc.v2"
+)
 
 func summaryFromInterface(i interface{}) (*libcontainerdtypes.Summary, error) {
 	return &libcontainerdtypes.Summary{}, nil
diff --git a/libcontainerd/remote/client_windows.go b/libcontainerd/remote/client_windows.go
index a086d7b3ba..c371b9a8b4 100644
--- a/libcontainerd/remote/client_windows.go
+++ b/libcontainerd/remote/client_windows.go
@@ -16,7 +16,10 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-const runtimeName = "io.containerd.runhcs.v1"
+const (
+	runtimeName       = "io.containerd.runhcs.v1"
+	shimV2RuntimeName = runtimeName
+)
 
 func summaryFromInterface(i interface{}) (*libcontainerdtypes.Summary, error) {
 	switch pd := i.(type) {
diff --git a/plugin/executor/containerd/containerd.go b/plugin/executor/containerd/containerd.go
index 91bae6c6b9..aeeb2184ec 100644
--- a/plugin/executor/containerd/containerd.go
+++ b/plugin/executor/containerd/containerd.go
@@ -26,13 +26,13 @@ type ExitHandler interface {
 }
 
 // New creates a new containerd plugin executor
-func New(ctx context.Context, rootDir string, cli *containerd.Client, ns string, exitHandler ExitHandler) (*Executor, error) {
+func New(ctx context.Context, rootDir string, cli *containerd.Client, ns string, exitHandler ExitHandler, useShimV2 bool) (*Executor, error) {
 	e := &Executor{
 		rootDir:     rootDir,
 		exitHandler: exitHandler,
 	}
 
-	client, err := libcontainerd.NewClient(ctx, cli, rootDir, ns, e)
+	client, err := libcontainerd.NewClient(ctx, cli, rootDir, ns, e, useShimV2)
 	if err != nil {
 		return nil, errors.Wrap(err, "error creating containerd exec client")
 	}