add rlimit_nofile functionality

Signed-off-by: Shishir Mahajan <smahajan@roblox.com>

.drone.yml

@@ -0,0 +1,31 @@
+---
+kind: pipeline
+type: docker
+name: default
+
+steps:
+ - name: build
+   image: golang
+   environment:
+     CGO_ENABLED: 0
+   commands:
+    - go vet
+    - go build
+
+ - name: upload artifact
+   image: git.burble.dn42/burble.dn42/drone-gitea-pkg-plugin:latest
+   settings:
+     token:
+       from_secret: TOKEN
+     version: .version
+     artifact: nomad-driver-containerd
+     package: nomad-driver-containerd
+     owner: burble
+
+---
+kind: secret
+name: TOKEN
+get:
+  path: burble.dn42/kv/data/drone/git.burble.dn42
+  name: artifact-token
+

.gitignore

@@ -1,2 +1,52 @@
 .vagrant/
-/containerd-driver
+/nomad-containerd-driver
+
+# ---> Emacs
+# -*- mode: gitignore; -*-
+*~
+\#*\#
+/.emacs.desktop
+/.emacs.desktop.lock
+*.elc
+auto-save-list
+tramp
+.\#*
+
+# Org-mode
+.org-id-locations
+*_archive
+
+# flymake-mode
+*_flymake.*
+
+# eshell files
+/eshell/history
+/eshell/lastdir
+
+# elpa packages
+/elpa/
+
+# reftex files
+*.rel
+
+# AUCTeX auto folder
+/auto/
+
+# cask packages
+.cask/
+dist/
+
+# Flycheck
+flycheck_*.el
+
+# server auth directory
+/server/
+
+# projectiles files
+.projectile
+
+# directory configuration
+.dir-locals.el
+
+# network security
+/network-security.data

.version

@@ -0,0 +1 @@
+1.0.0

README.md

@@ -87,18 +87,26 @@ More detailed instructions are in the [`example README.md`](https://github.com/R
 
 To interact with `images` and `containers` directly, you can use [`nerdctl`](https://github.com/containerd/nerdctl) which is a docker compatible CLI for `containerd`. `nerdctl` is already installed in the vagrant VM at `/usr/local/bin`.
 
-## Supported options
+## Supported Options
 
 **Driver Config**
 
 | Option | Type | Required | Default | Description |
 | :---: | :---: | :---: | :---: | :--- |
 | **enabled** | bool | no | true | Enable/Disable task driver. |
-| **containerd_runtime** | string | yes | N/A | Runtime for containerd e.g. `io.containerd.runc.v1` or `io.containerd.runc.v2`. |
+| **containerd_runtime** | string | no | `io.containerd.runc.v2` | Runtime for containerd. |
 | **stats_interval** | string | no | 1s | Interval for collecting `TaskStats`. |
 | **allow_privileged** | bool | no | true | If set to `false`, driver will deny running privileged jobs. |
 | **auth** | block | no | N/A | Provide authentication for a private registry. See [Authentication](#authentication-private-registry) for more details. |
 
+## Supported Runtimes
+
+Valid options for `containerd_runtime` (**Driver Config**).
+
+- `io.containerd.runc.v1`: `runc` runtime that supports a single container.
+- `io.containerd.runc.v2` (Default): `runc` runtime that supports multiple containers per shim.
+- `io.containerd.runsc.v1`: `gVisor` is an OCI compliant container runtime which provides better security than `runc`. They achieve this by implementing a user space kernel written in go, which implements a substantial portion of the Linux system call interface. For more details, please check their [`official documentation`](https://gvisor.dev/docs/)
+
 **Task Config**
 
 | Option | Type | Required | Description |
@@ -119,10 +127,13 @@ To interact with `images` and `containers` directly, you can use [`nerdctl`](htt
 | **shm_size** | string | no | Size of /dev/shm e.g. "128M" if you want 128 MB of /dev/shm. |
 | **sysctl** | map[string]string | no | A key-value map of sysctl configurations to set to the containers on start. |
 | **readonly_rootfs** | bool | no | Container root filesystem will be read-only. |
+| **runtime** | string | no | A string representing a configured runtime to pass to containerd. This is equivalent to the `--runtime` argument in the docker CLI. |
 | **host_network** | bool | no | Enable host network. This is equivalent to `--net=host` in docker. |
 | **extra_hosts** | []string | no | A list of hosts, given as host:IP, to be added to /etc/hosts. |
 | **cap_add** | []string | no | Add individual capabilities. |
 | **cap_drop** | []string | no | Drop invidual capabilities. |
+| **cpuset_cpus** | string | no | CPUs in which to allow execution (0-3, 0,1). |
+| **cpuset_mems** | string | no | MEMs in which to allow execution (0-3, 0,1). |
 | **devices** | []string | no | A list of devices to be exposed to the container. |
 | **auth** | block | no | Provide authentication for a private registry. See [Authentication](#authentication-private-registry) for more details. |
 | **mounts** | []block | no | A list of mounts to be mounted in the container. Volume, bind and tmpfs type mounts are supported. fstab style [`mount options`](https://github.com/containerd/containerd/blob/master/mount/mount_linux.go#L211-L235) are supported. |

Vagrantfile

@@ -5,7 +5,7 @@ VAGRANTFILE_API_VERSION = "2"
 # Create box
 Vagrant.configure("2") do |config|
   config.vm.define "containerd-linux"
-  config.vm.box = "hashicorp/bionic64"
+  config.vm.box = "generic/ubuntu2204"
   config.vm.provider "libvirt" do |v, override|
     override.vm.box = "generic/debian10"
     override.vm.synced_folder ".", "/home/vagrant/go/src/github.com/Roblox/nomad-driver-containerd", type: "nfs", nfs_version: 4, nfs_udp: false
@@ -20,7 +20,7 @@ Vagrant.configure("2") do |config|
   end
   config.vm.provision "shell", inline: <<-SHELL
     apt-get update
-    apt-get install -y unzip gcc runc jq
+    apt-get install -y unzip gcc runc jq make
     echo "export GOPATH=/home/vagrant/go" >> /home/vagrant/.bashrc
     echo "export PATH=$PATH:/usr/local/go/bin" >> /home/vagrant/.bashrc
     echo "export CONTAINERD_NAMESPACE=nomad" >> /home/vagrant/.bashrc

containerd/containerd.go

@@ -20,12 +20,14 @@ package containerd
 import (
 	"context"
 	"fmt"
+	"strconv"
 	"strings"
 	"time"
 
 	etchosts "github.com/Roblox/nomad-driver-containerd/etchosts"
 	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/cio"
+	"github.com/containerd/containerd/containers"
 	"github.com/containerd/containerd/contrib/seccomp"
 	"github.com/containerd/containerd/oci"
 	refdocker "github.com/containerd/containerd/reference/docker"
@@ -85,6 +87,21 @@ func (d *Driver) parshAuth(auth *RegistryAuth) CredentialsOpt {
 	}
 }
 
+// the containerd driver inexplicably appears to be missing options
+// to set RLIMITS NOFILE so have to roll our own
+
+func withRLimitNoFile(hard, soft uint64) oci.SpecOpts {
+	return func(_ context.Context, _ oci.Client, _ *containers.Container, s *specs.Spec) error {
+		for i, _ := range s.Process.Rlimits {
+			if s.Process.Rlimits[i].Type == "RLIMIT_NOFILE" {
+				s.Process.Rlimits[i].Hard = hard
+				s.Process.Rlimits[i].Soft = soft
+			}
+		}
+		return nil
+	}
+}
+
 func withResolver(creds CredentialsOpt) containerd.RemoteOpt {
 	resolver := remotesdocker.NewResolver(remotesdocker.ResolverOptions{
 		Hosts: remotesdocker.ConfigureDefaultRegistries(remotesdocker.WithAuthorizer(
@@ -217,6 +234,41 @@ func (d *Driver) createContainer(containerConfig *ContainerConfig, config *TaskC
 		opts = append(opts, oci.WithDroppedCapabilities(config.CapDrop))
 	}
 
+	// This translates to docker create/run --cpuset-cpus option.
+	// --cpuset-cpus limit the specific CPUs or cores a container can use.
+	if config.CPUSetCPUs != "" {
+		opts = append(opts, oci.WithCPUs(config.CPUSetCPUs))
+	}
+
+	// --cpuset-mems is the list of memory nodes on which processes
+	// in this cpuset are allowed to allocate memory.
+	if config.CPUSetMEMs != "" {
+		opts = append(opts, oci.WithCPUsMems(config.CPUSetMEMs))
+	}
+
+	// allow setting of RLIMIT_NOFILE
+	if config.RLimitNoFile != "" {
+		s := strings.Split(config.RLimitNoFile, ":")
+		var hard, soft uint64
+
+		if tmp, err := strconv.ParseUint(s[0], 10, 64); err != nil {
+			return nil, fmt.Errorf("rlimit_nofile, failed to convert string to uint64: %s (%s)", s[0], err.Error())
+		} else {
+			hard = tmp
+		}
+
+		if len(s) > 1 {
+			if tmp, err := strconv.ParseUint(s[1], 10, 64); err != nil {
+				return nil, fmt.Errorf("rlimit_nofile, failed to convert string to uint64: %s (%s)", s[1], err.Error())
+			} else {
+				soft = tmp
+			}
+		} else {
+			soft = hard
+		}
+		opts = append(opts, withRLimitNoFile(hard, soft))
+	}
+
 	// Set current working directory (cwd).
 	if config.Cwd != "" {
 		opts = append(opts, oci.WithProcessCwd(config.Cwd))
@@ -339,7 +391,7 @@ func (d *Driver) createContainer(containerConfig *ContainerConfig, config *TaskC
 	return d.client.NewContainer(
 		ctxWithTimeout,
 		containerConfig.ContainerName,
-		containerd.WithRuntime(d.config.ContainerdRuntime, nil),
+		buildRuntime(d.config.ContainerdRuntime, config.Runtime),
 		containerd.WithNewSnapshot(containerConfig.ContainerSnapshotName, containerConfig.Image),
 		containerd.WithNewSpec(opts...),
 	)

containerd/driver.go

@@ -79,7 +79,7 @@ var (
 			hclspec.NewAttr("enabled", "bool", false),
 			hclspec.NewLiteral("true"),
 		),
-		"containerd_runtime": hclspec.NewAttr("containerd_runtime", "string", true),
+		"containerd_runtime": hclspec.NewAttr("containerd_runtime", "string", false),
 		"stats_interval":     hclspec.NewAttr("stats_interval", "string", false),
 		"allow_privileged": hclspec.NewDefault(
 			hclspec.NewAttr("allow_privileged", "bool", false),
@@ -101,6 +101,9 @@ var (
 		"args":          hclspec.NewAttr("args", "list(string)", false),
 		"cap_add":       hclspec.NewAttr("cap_add", "list(string)", false),
 		"cap_drop":      hclspec.NewAttr("cap_drop", "list(string)", false),
+		"cpuset_cpus":   hclspec.NewAttr("cpuset_cpus", "string", false),
+		"cpuset_mems":   hclspec.NewAttr("cpuset_mems", "string", false),
+		"rlimit_nofile": hclspec.NewAttr("rlimit_nofile", "string", false),
 		"cwd":           hclspec.NewAttr("cwd", "string", false),
 		"devices":       hclspec.NewAttr("devices", "list(string)", false),
 		"privileged":    hclspec.NewAttr("privileged", "bool", false),
@@ -122,6 +125,7 @@ var (
 		"shm_size":        hclspec.NewAttr("shm_size", "string", false),
 		"sysctl":          hclspec.NewAttr("sysctl", "list(map(string))", false),
 		"readonly_rootfs": hclspec.NewAttr("readonly_rootfs", "bool", false),
+		"runtime":         hclspec.NewAttr("runtime", "string", false),
 		"host_network":    hclspec.NewAttr("host_network", "bool", false),
 		"auth": hclspec.NewBlock("auth", false, hclspec.NewObject(map[string]*hclspec.Spec{
 			"username": hclspec.NewAttr("username", "string", true),
@@ -181,6 +185,9 @@ type TaskConfig struct {
 	Args             []string           `codec:"args"`
 	CapAdd           []string           `codec:"cap_add"`
 	CapDrop          []string           `codec:"cap_drop"`
+	CPUSetCPUs       string             `codec:"cpuset_cpus"`
+	CPUSetMEMs       string             `codec:"cpuset_mems"`
+	RLimitNoFile     string             `codec:"rlimit_nofile"`
 	Cwd              string             `codec:"cwd"`
 	Devices          []string           `codec:"devices"`
 	Seccomp          bool               `codec:"seccomp"`
@@ -195,6 +202,7 @@ type TaskConfig struct {
 	ImagePullTimeout string             `codec:"image_pull_timeout"`
 	ExtraHosts       []string           `codec:"extra_hosts"`
 	Entrypoint       []string           `codec:"entrypoint"`
+	Runtime          string             `codec:"runtime"`
 	ReadOnlyRootfs   bool               `codec:"readonly_rootfs"`
 	HostNetwork      bool               `codec:"host_network"`
 	Auth             RegistryAuth       `codec:"auth"`

containerd/utils.go

@@ -20,10 +20,14 @@ package containerd
 import (
 	"context"
 	"os"
+	"strings"
 	"syscall"
 
+	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/containers"
 	"github.com/containerd/containerd/oci"
+	"github.com/containerd/containerd/plugin"
+	runcoptions "github.com/containerd/containerd/runtime/v2/runc/options"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 )
 
@@ -99,3 +103,30 @@ func WithMemoryLimits(soft, hard int64) oci.SpecOpts {
 		return nil
 	}
 }
+
+// buildRuntime sets the container runtime e.g. runc or runsc (gVisor).
+func buildRuntime(pluginRuntime, jobRuntime string) containerd.NewContainerOpts {
+	var (
+		runcOpts    runcoptions.Options
+		runtimeOpts interface{} = &runcOpts
+	)
+
+	// plugin.RuntimeRuncV2 = io.containerd.runc.v2
+	runtime := plugin.RuntimeRuncV2
+
+	if jobRuntime != "" {
+		if strings.HasPrefix(jobRuntime, "io.containerd.runc.") {
+			runtime = jobRuntime
+		} else {
+			runcOpts.BinaryName = jobRuntime
+		}
+	} else if pluginRuntime != "" {
+		if strings.HasPrefix(pluginRuntime, "io.containerd.runc.") {
+			runtime = pluginRuntime
+		} else {
+			runcOpts.BinaryName = pluginRuntime
+		}
+	}
+
+	return containerd.WithRuntime(runtime, runtimeOpts)
+}

example/agent.hcl

@@ -4,7 +4,6 @@ data_dir = "/tmp/nomad"
 plugin "containerd-driver" {
   config {
     enabled = true
-    containerd_runtime = "io.containerd.runc.v2"
     stats_interval = "5s"
   }
 }

example/redis.nomad

@@ -10,6 +10,8 @@ job "redis" {
         hostname    = "foobar"
         seccomp     = true
         cwd         = "/home/redis"
+        cpuset_cpus = "0-1"
+        cpuset_mems = "0"
       }
 
       resources {

go.mod

@@ -1,6 +1,6 @@
 module github.com/Roblox/nomad-driver-containerd
 
-go 1.17
+go 1.21
 
 require (
 	github.com/containerd/cgroups v1.0.3