.drone.yml

@@ -1,31 +0,0 @@
----
-kind: pipeline
-type: docker
-name: default
-
-steps:
- - name: build
-   image: golang
-   environment:
-     CGO_ENABLED: 0
-   commands:
-    - go vet
-    - go build
-
- - name: upload artifact
-   image: git.burble.dn42/burble.dn42/drone-gitea-pkg-plugin:latest
-   settings:
-     token:
-       from_secret: TOKEN
-     version: .version
-     artifact: nomad-driver-containerd
-     package: nomad-driver-containerd
-     owner: burble
-
----
-kind: secret
-name: TOKEN
-get:
-  path: burble.dn42/kv/data/drone/git.burble.dn42
-  name: artifact-token
-

.gitignore

@@ -1,52 +1,2 @@
 .vagrant/
-/nomad-containerd-driver
+/containerd-driver
-
-# ---> Emacs
-# -*- mode: gitignore; -*-
-*~
-\#*\#
-/.emacs.desktop
-/.emacs.desktop.lock
-*.elc
-auto-save-list
-tramp
-.\#*
-
-# Org-mode
-.org-id-locations
-*_archive
-
-# flymake-mode
-*_flymake.*
-
-# eshell files
-/eshell/history
-/eshell/lastdir
-
-# elpa packages
-/elpa/
-
-# reftex files
-*.rel
-
-# AUCTeX auto folder
-/auto/
-
-# cask packages
-.cask/
-dist/
-
-# Flycheck
-flycheck_*.el
-
-# server auth directory
-/server/
-
-# projectiles files
-.projectile
-
-# directory configuration
-.dir-locals.el
-
-# network security
-/network-security.data

.version

@@ -1 +0,0 @@
-1.0.0

README.md

@@ -87,26 +87,18 @@ More detailed instructions are in the [`example README.md`](https://github.com/R
 
 To interact with `images` and `containers` directly, you can use [`nerdctl`](https://github.com/containerd/nerdctl) which is a docker compatible CLI for `containerd`. `nerdctl` is already installed in the vagrant VM at `/usr/local/bin`.
 
-## Supported Options
+## Supported options
 
 **Driver Config**
 
 | Option | Type | Required | Default | Description |
 | :---: | :---: | :---: | :---: | :--- |
 | **enabled** | bool | no | true | Enable/Disable task driver. |
-| **containerd_runtime** | string | no | `io.containerd.runc.v2` | Runtime for containerd. |
+| **containerd_runtime** | string | yes | N/A | Runtime for containerd e.g. `io.containerd.runc.v1` or `io.containerd.runc.v2`. |
 | **stats_interval** | string | no | 1s | Interval for collecting `TaskStats`. |
 | **allow_privileged** | bool | no | true | If set to `false`, driver will deny running privileged jobs. |
 | **auth** | block | no | N/A | Provide authentication for a private registry. See [Authentication](#authentication-private-registry) for more details. |
 
-## Supported Runtimes
-
-Valid options for `containerd_runtime` (**Driver Config**).
-
-- `io.containerd.runc.v1`: `runc` runtime that supports a single container.
-- `io.containerd.runc.v2` (Default): `runc` runtime that supports multiple containers per shim.
-- `io.containerd.runsc.v1`: `gVisor` is an OCI compliant container runtime which provides better security than `runc`. They achieve this by implementing a user space kernel written in go, which implements a substantial portion of the Linux system call interface. For more details, please check their [`official documentation`](https://gvisor.dev/docs/)
-
 **Task Config**
 
 | Option | Type | Required | Description |
@@ -127,13 +119,10 @@ Valid options for `containerd_runtime` (**Driver Config**).
 | **shm_size** | string | no | Size of /dev/shm e.g. "128M" if you want 128 MB of /dev/shm. |
 | **sysctl** | map[string]string | no | A key-value map of sysctl configurations to set to the containers on start. |
 | **readonly_rootfs** | bool | no | Container root filesystem will be read-only. |
-| **runtime** | string | no | A string representing a configured runtime to pass to containerd. This is equivalent to the `--runtime` argument in the docker CLI. |
 | **host_network** | bool | no | Enable host network. This is equivalent to `--net=host` in docker. |
 | **extra_hosts** | []string | no | A list of hosts, given as host:IP, to be added to /etc/hosts. |
 | **cap_add** | []string | no | Add individual capabilities. |
 | **cap_drop** | []string | no | Drop invidual capabilities. |
-| **cpuset_cpus** | string | no | CPUs in which to allow execution (0-3, 0,1). |
-| **cpuset_mems** | string | no | MEMs in which to allow execution (0-3, 0,1). |
 | **devices** | []string | no | A list of devices to be exposed to the container. |
 | **auth** | block | no | Provide authentication for a private registry. See [Authentication](#authentication-private-registry) for more details. |
 | **mounts** | []block | no | A list of mounts to be mounted in the container. Volume, bind and tmpfs type mounts are supported. fstab style [`mount options`](https://github.com/containerd/containerd/blob/master/mount/mount_linux.go#L211-L235) are supported. |

Vagrantfile

@@ -5,7 +5,7 @@ VAGRANTFILE_API_VERSION = "2"
 # Create box
 Vagrant.configure("2") do |config|
   config.vm.define "containerd-linux"
-  config.vm.box = "generic/ubuntu2204"
+  config.vm.box = "hashicorp/bionic64"
   config.vm.provider "libvirt" do |v, override|
     override.vm.box = "generic/debian10"
     override.vm.synced_folder ".", "/home/vagrant/go/src/github.com/Roblox/nomad-driver-containerd", type: "nfs", nfs_version: 4, nfs_udp: false
@@ -20,7 +20,7 @@ Vagrant.configure("2") do |config|
   end
   config.vm.provision "shell", inline: <<-SHELL
     apt-get update
-    apt-get install -y unzip gcc runc jq make
+    apt-get install -y unzip gcc runc jq
     echo "export GOPATH=/home/vagrant/go" >> /home/vagrant/.bashrc
     echo "export PATH=$PATH:/usr/local/go/bin" >> /home/vagrant/.bashrc
     echo "export CONTAINERD_NAMESPACE=nomad" >> /home/vagrant/.bashrc

containerd/containerd.go

@@ -20,14 +20,12 @@ package containerd
 import (
 	"context"
 	"fmt"
-	"strconv"
 	"strings"
 	"time"
 
 	etchosts "github.com/Roblox/nomad-driver-containerd/etchosts"
 	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/cio"
-	"github.com/containerd/containerd/containers"
 	"github.com/containerd/containerd/contrib/seccomp"
 	"github.com/containerd/containerd/oci"
 	refdocker "github.com/containerd/containerd/reference/docker"
@@ -87,21 +85,6 @@ func (d *Driver) parshAuth(auth *RegistryAuth) CredentialsOpt {
 	}
 }
 
-// the containerd driver inexplicably appears to be missing options
-// to set RLIMITS NOFILE so have to roll our own
-
-func withRLimitNoFile(hard, soft uint64) oci.SpecOpts {
-	return func(_ context.Context, _ oci.Client, _ *containers.Container, s *specs.Spec) error {
-		for i, _ := range s.Process.Rlimits {
-			if s.Process.Rlimits[i].Type == "RLIMIT_NOFILE" {
-				s.Process.Rlimits[i].Hard = hard
-				s.Process.Rlimits[i].Soft = soft
-			}
-		}
-		return nil
-	}
-}
-
 func withResolver(creds CredentialsOpt) containerd.RemoteOpt {
 	resolver := remotesdocker.NewResolver(remotesdocker.ResolverOptions{
 		Hosts: remotesdocker.ConfigureDefaultRegistries(remotesdocker.WithAuthorizer(
@@ -234,41 +217,6 @@ func (d *Driver) createContainer(containerConfig *ContainerConfig, config *TaskC
 		opts = append(opts, oci.WithDroppedCapabilities(config.CapDrop))
 	}
 
-	// This translates to docker create/run --cpuset-cpus option.
-	// --cpuset-cpus limit the specific CPUs or cores a container can use.
-	if config.CPUSetCPUs != "" {
-		opts = append(opts, oci.WithCPUs(config.CPUSetCPUs))
-	}
-
-	// --cpuset-mems is the list of memory nodes on which processes
-	// in this cpuset are allowed to allocate memory.
-	if config.CPUSetMEMs != "" {
-		opts = append(opts, oci.WithCPUsMems(config.CPUSetMEMs))
-	}
-
-	// allow setting of RLIMIT_NOFILE
-	if config.RLimitNoFile != "" {
-		s := strings.Split(config.RLimitNoFile, ":")
-		var hard, soft uint64
-
-		if tmp, err := strconv.ParseUint(s[0], 10, 64); err != nil {
-			return nil, fmt.Errorf("rlimit_nofile, failed to convert string to uint64: %s (%s)", s[0], err.Error())
-		} else {
-			hard = tmp
-		}
-
-		if len(s) > 1 {
-			if tmp, err := strconv.ParseUint(s[1], 10, 64); err != nil {
-				return nil, fmt.Errorf("rlimit_nofile, failed to convert string to uint64: %s (%s)", s[1], err.Error())
-			} else {
-				soft = tmp
-			}
-		} else {
-			soft = hard
-		}
-		opts = append(opts, withRLimitNoFile(hard, soft))
-	}
-
 	// Set current working directory (cwd).
 	if config.Cwd != "" {
 		opts = append(opts, oci.WithProcessCwd(config.Cwd))
@@ -391,7 +339,7 @@ func (d *Driver) createContainer(containerConfig *ContainerConfig, config *TaskC
 	return d.client.NewContainer(
 		ctxWithTimeout,
 		containerConfig.ContainerName,
-		buildRuntime(d.config.ContainerdRuntime, config.Runtime),
+		containerd.WithRuntime(d.config.ContainerdRuntime, nil),
 		containerd.WithNewSnapshot(containerConfig.ContainerSnapshotName, containerConfig.Image),
 		containerd.WithNewSpec(opts...),
 	)

containerd/driver.go

@@ -79,7 +79,7 @@ var (
 			hclspec.NewAttr("enabled", "bool", false),
 			hclspec.NewLiteral("true"),
 		),
-		"containerd_runtime": hclspec.NewAttr("containerd_runtime", "string", false),
+		"containerd_runtime": hclspec.NewAttr("containerd_runtime", "string", true),
 		"stats_interval":     hclspec.NewAttr("stats_interval", "string", false),
 		"allow_privileged": hclspec.NewDefault(
 			hclspec.NewAttr("allow_privileged", "bool", false),
@@ -96,20 +96,17 @@ var (
 	// this is used to validate the configuration specified for the plugin
 	// when a job is submitted.
 	taskConfigSpec = hclspec.NewObject(map[string]*hclspec.Spec{
-		"image":         hclspec.NewAttr("image", "string", true),
+		"image":      hclspec.NewAttr("image", "string", true),
-		"command":       hclspec.NewAttr("command", "string", false),
+		"command":    hclspec.NewAttr("command", "string", false),
-		"args":          hclspec.NewAttr("args", "list(string)", false),
+		"args":       hclspec.NewAttr("args", "list(string)", false),
-		"cap_add":       hclspec.NewAttr("cap_add", "list(string)", false),
+		"cap_add":    hclspec.NewAttr("cap_add", "list(string)", false),
-		"cap_drop":      hclspec.NewAttr("cap_drop", "list(string)", false),
+		"cap_drop":   hclspec.NewAttr("cap_drop", "list(string)", false),
-		"cpuset_cpus":   hclspec.NewAttr("cpuset_cpus", "string", false),
+		"cwd":        hclspec.NewAttr("cwd", "string", false),
-		"cpuset_mems":   hclspec.NewAttr("cpuset_mems", "string", false),
+		"devices":    hclspec.NewAttr("devices", "list(string)", false),
-		"rlimit_nofile": hclspec.NewAttr("rlimit_nofile", "string", false),
+		"privileged": hclspec.NewAttr("privileged", "bool", false),
-		"cwd":           hclspec.NewAttr("cwd", "string", false),
+		"pids_limit": hclspec.NewAttr("pids_limit", "number", false),
-		"devices":       hclspec.NewAttr("devices", "list(string)", false),
+		"pid_mode":   hclspec.NewAttr("pid_mode", "string", false),
-		"privileged":    hclspec.NewAttr("privileged", "bool", false),
+		"hostname":   hclspec.NewAttr("hostname", "string", false),
-		"pids_limit":    hclspec.NewAttr("pids_limit", "number", false),
-		"pid_mode":      hclspec.NewAttr("pid_mode", "string", false),
-		"hostname":      hclspec.NewAttr("hostname", "string", false),
 		"host_dns": hclspec.NewDefault(
 			hclspec.NewAttr("host_dns", "bool", false),
 			hclspec.NewLiteral("true"),
@@ -125,7 +122,6 @@ var (
 		"shm_size":        hclspec.NewAttr("shm_size", "string", false),
 		"sysctl":          hclspec.NewAttr("sysctl", "list(map(string))", false),
 		"readonly_rootfs": hclspec.NewAttr("readonly_rootfs", "bool", false),
-		"runtime":         hclspec.NewAttr("runtime", "string", false),
 		"host_network":    hclspec.NewAttr("host_network", "bool", false),
 		"auth": hclspec.NewBlock("auth", false, hclspec.NewObject(map[string]*hclspec.Spec{
 			"username": hclspec.NewAttr("username", "string", true),
@@ -185,9 +181,6 @@ type TaskConfig struct {
 	Args             []string           `codec:"args"`
 	CapAdd           []string           `codec:"cap_add"`
 	CapDrop          []string           `codec:"cap_drop"`
-	CPUSetCPUs       string             `codec:"cpuset_cpus"`
-	CPUSetMEMs       string             `codec:"cpuset_mems"`
-	RLimitNoFile     string             `codec:"rlimit_nofile"`
 	Cwd              string             `codec:"cwd"`
 	Devices          []string           `codec:"devices"`
 	Seccomp          bool               `codec:"seccomp"`
@@ -202,7 +195,6 @@ type TaskConfig struct {
 	ImagePullTimeout string             `codec:"image_pull_timeout"`
 	ExtraHosts       []string           `codec:"extra_hosts"`
 	Entrypoint       []string           `codec:"entrypoint"`
-	Runtime          string             `codec:"runtime"`
 	ReadOnlyRootfs   bool               `codec:"readonly_rootfs"`
 	HostNetwork      bool               `codec:"host_network"`
 	Auth             RegistryAuth       `codec:"auth"`

containerd/utils.go

@@ -20,14 +20,10 @@ package containerd
 import (
 	"context"
 	"os"
-	"strings"
 	"syscall"
 
-	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/containers"
 	"github.com/containerd/containerd/oci"
-	"github.com/containerd/containerd/plugin"
-	runcoptions "github.com/containerd/containerd/runtime/v2/runc/options"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 )
 
@@ -103,30 +99,3 @@ func WithMemoryLimits(soft, hard int64) oci.SpecOpts {
 		return nil
 	}
 }
-
-// buildRuntime sets the container runtime e.g. runc or runsc (gVisor).
-func buildRuntime(pluginRuntime, jobRuntime string) containerd.NewContainerOpts {
-	var (
-		runcOpts    runcoptions.Options
-		runtimeOpts interface{} = &runcOpts
-	)
-
-	// plugin.RuntimeRuncV2 = io.containerd.runc.v2
-	runtime := plugin.RuntimeRuncV2
-
-	if jobRuntime != "" {
-		if strings.HasPrefix(jobRuntime, "io.containerd.runc.") {
-			runtime = jobRuntime
-		} else {
-			runcOpts.BinaryName = jobRuntime
-		}
-	} else if pluginRuntime != "" {
-		if strings.HasPrefix(pluginRuntime, "io.containerd.runc.") {
-			runtime = pluginRuntime
-		} else {
-			runcOpts.BinaryName = pluginRuntime
-		}
-	}
-
-	return containerd.WithRuntime(runtime, runtimeOpts)
-}

example/agent.hcl

@@ -4,6 +4,7 @@ data_dir = "/tmp/nomad"
 plugin "containerd-driver" {
   config {
     enabled = true
+    containerd_runtime = "io.containerd.runc.v2"
     stats_interval = "5s"
   }
 }

example/redis.nomad

@@ -6,12 +6,10 @@ job "redis" {
       driver = "containerd-driver"
 
       config {
-        image       = "redis:alpine"
+        image    = "redis:alpine"
-        hostname    = "foobar"
+        hostname = "foobar"
-        seccomp     = true
+        seccomp  = true
-        cwd         = "/home/redis"
+        cwd      = "/home/redis"
-        cpuset_cpus = "0-1"
-        cpuset_mems = "0"
       }
 
       resources {

go.mod

@@ -1,6 +1,6 @@
 module github.com/Roblox/nomad-driver-containerd
 
-go 1.21
+go 1.17
 
 require (
 	github.com/containerd/cgroups v1.0.3