Merge pull request #90 from notnoop/f-memory-oversubscription
Support Memory Oversubscription
This commit is contained in:
Shishir
GitHub
commit
5945e435f1
8
Vagrantfile
vendored
8
Vagrantfile
vendored
@ -35,12 +35,12 @@ Vagrant.configure("2") do |config|
|
||||
rm -f go1.14.3.linux-amd64.tar.gz
|
||||
fi
|
||||
|
||||
# Install nomad-1.0.2
|
||||
# Install nomad-1.1.0
|
||||
if [ ! -f "/usr/bin/nomad" ]; then
|
||||
wget --quiet https://releases.hashicorp.com/nomad/1.0.2/nomad_1.0.2_linux_amd64.zip
|
||||
unzip nomad_1.0.2_linux_amd64.zip -d /usr/bin
|
||||
wget --quiet https://releases.hashicorp.com/nomad/1.1.0/nomad_1.1.0_linux_amd64.zip
|
||||
unzip nomad_1.1.0_linux_amd64.zip -d /usr/bin
|
||||
chmod +x /usr/bin/nomad
|
||||
rm -f nomad_1.0.2_linux_amd64.zip
|
||||
rm -f nomad_1.1.0_linux_amd64.zip
|
||||
fi
|
||||
|
||||
# Install containerd-1.3.4
|
||||
|
||||
@ -45,6 +45,7 @@ type ContainerConfig struct {
|
||||
AllocDirDest string
|
||||
Env []string
|
||||
MemoryLimit int64
|
||||
MemoryHardLimit int64
|
||||
CPUShares int64
|
||||
}
|
||||
|
||||
@ -196,7 +197,7 @@ func (d *Driver) createContainer(containerConfig *ContainerConfig, config *TaskC
|
||||
opts = append(opts, oci.WithEnv(containerConfig.Env))
|
||||
|
||||
// Set cgroups memory limit.
|
||||
opts = append(opts, oci.WithMemoryLimit(uint64(containerConfig.MemoryLimit)))
|
||||
opts = append(opts, WithMemoryLimits(containerConfig.MemoryLimit, containerConfig.MemoryHardLimit))
|
||||
|
||||
// Set CPU Shares.
|
||||
opts = append(opts, oci.WithCPUShares(uint64(containerConfig.CPUShares)))
|
||||
|
||||
@ -454,7 +454,8 @@ func (d *Driver) StartTask(cfg *drivers.TaskConfig) (*drivers.TaskHandle, *drive
|
||||
|
||||
// memory and cpu are coming from the resources stanza of the nomad job.
|
||||
// https://www.nomadproject.io/docs/job-specification/resources
|
||||
containerConfig.MemoryLimit = cfg.Resources.LinuxResources.MemoryLimitBytes
|
||||
containerConfig.MemoryLimit = cfg.Resources.NomadResources.Memory.MemoryMB * 1024 * 1024
|
||||
containerConfig.MemoryHardLimit = cfg.Resources.NomadResources.Memory.MemoryMaxMB * 1024 * 1024
|
||||
containerConfig.CPUShares = cfg.Resources.LinuxResources.CPUShares
|
||||
|
||||
container, err := d.createContainer(&containerConfig, &driverConfig)
|
||||
|
||||
@ -60,3 +60,28 @@ func WithSysctls(sysctls map[string]string) oci.SpecOpts {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// WithMemoryLimits accepts soft (`memory`) and hard (`memory_max`) limits as parameters and set the desired
|
||||
// limits. With `Nomad<1.1.0` releases, soft (`memory`) will act as a hard limit, and if the container process exceeds
|
||||
// that limit, it will be OOM'ed. With `Nomad>=1.1.0` releases, users can over-provision using `soft` and `hard`
|
||||
// limits. The container process will only get OOM'ed if the hard limit is exceeded.
|
||||
func WithMemoryLimits(soft, hard int64) oci.SpecOpts {
|
||||
return func(_ context.Context, _ oci.Client, _ *containers.Container, s *oci.Spec) error {
|
||||
if s.Linux != nil {
|
||||
if s.Linux.Resources == nil {
|
||||
s.Linux.Resources = &specs.LinuxResources{}
|
||||
}
|
||||
if s.Linux.Resources.Memory == nil {
|
||||
s.Linux.Resources.Memory = &specs.LinuxMemory{}
|
||||
}
|
||||
|
||||
if hard > 0 {
|
||||
s.Linux.Resources.Memory.Limit = &hard
|
||||
s.Linux.Resources.Memory.Reservation = &soft
|
||||
} else {
|
||||
s.Linux.Resources.Memory.Limit = &soft
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
28
go.mod
28
go.mod
@ -4,44 +4,25 @@ module github.com/Roblox/nomad-driver-containerd
|
||||
go 1.12
|
||||
|
||||
require (
|
||||
github.com/LK4D4/joincontext v0.0.0-20171026170139-1724345da6d5 // indirect
|
||||
github.com/NVIDIA/gpu-monitoring-tools v0.0.0-20191126014920-0d8df858cca4 // indirect
|
||||
github.com/Sirupsen/logrus v0.0.0-00010101000000-000000000000 // indirect
|
||||
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
|
||||
github.com/appc/spec v0.8.11 // indirect
|
||||
github.com/checkpoint-restore/go-criu v0.0.0-20191125063657-fcdcd07065c5 // indirect
|
||||
github.com/containerd/cgroups v0.0.0-20200609174450-80c669f4bad0
|
||||
github.com/containerd/containerd v1.4.3
|
||||
github.com/containerd/go-cni v0.0.0-20191121212822-60d125212faf // indirect
|
||||
github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd
|
||||
github.com/containernetworking/plugins v0.8.3 // indirect
|
||||
github.com/coreos/go-iptables v0.4.3 // indirect
|
||||
github.com/coreos/go-semver v0.3.0 // indirect
|
||||
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect
|
||||
github.com/docker/docker v17.12.0-ce-rc1.0.20200330121334-7f8b4b621b5d+incompatible
|
||||
github.com/docker/docker-credential-helpers v0.6.3 // indirect
|
||||
github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
|
||||
github.com/docker/go-metrics v0.0.1 // indirect
|
||||
github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect
|
||||
github.com/gogo/googleapis v1.4.0 // indirect
|
||||
github.com/google/uuid v1.1.2 // indirect
|
||||
github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75 // indirect
|
||||
github.com/hashicorp/consul-template v0.25.1
|
||||
github.com/hashicorp/go-envparse v0.0.0-20190703193109-150b3a2a4611 // indirect
|
||||
github.com/hashicorp/go-hclog v0.12.0
|
||||
github.com/hashicorp/go-plugin v1.0.2-0.20191004171845-809113480b55
|
||||
github.com/hashicorp/go-hclog v0.14.1
|
||||
github.com/hashicorp/go-uuid v1.0.1
|
||||
github.com/hashicorp/hcl2 v0.0.0-20191002203319-fb75b3253c80 // indirect
|
||||
github.com/hashicorp/nomad v1.0.2
|
||||
github.com/mitchellh/go-ps v0.0.0-20190716172923-621e5597135b // indirect
|
||||
github.com/opencontainers/runtime-spec v1.0.3-0.20200728170252-4d89ac9fbff6
|
||||
github.com/hashicorp/nomad v1.1.0
|
||||
github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d
|
||||
github.com/spf13/cobra v1.1.1
|
||||
github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 // indirect
|
||||
github.com/ugorji/go v1.1.7 // indirect
|
||||
github.com/vbatts/tar-split v0.11.1 // indirect
|
||||
go4.org v0.0.0-20191010144846-132d2879e1e9 // indirect
|
||||
google.golang.org/grpc v1.32.0 // indirect
|
||||
istio.io/gogo-genproto v0.0.0-20190124151557-6d926a6e6feb // indirect
|
||||
)
|
||||
|
||||
// use lower-case sirupsen
|
||||
@ -59,3 +40,6 @@ replace github.com/docker/distribution v2.7.1+incompatible => github.com/docker/
|
||||
|
||||
// fix the version of hashicorp/go-msgpack to 96ddbed8d05b
|
||||
replace github.com/hashicorp/go-msgpack => github.com/hashicorp/go-msgpack v0.0.0-20191101193846-96ddbed8d05b
|
||||
|
||||
// Workaround Nomad using an old version
|
||||
replace google.golang.org/api => google.golang.org/api v0.46.0
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
|
||||
set -eo pipefail
|
||||
|
||||
export NOMAD_VERSION=1.0.2
|
||||
export NOMAD_VERSION=1.1.0
|
||||
export CONTAINERD_VERSION=1.3.4
|
||||
export PATH=$PATH:/usr/local/go/bin
|
||||
export PATH=$PATH:/usr/local/bin
|
||||
@ -144,7 +144,7 @@ EOF
|
||||
sudo chmod +x /usr/local/go
|
||||
rm -f go${GO_VERSION}.linux-amd64.tar.gz
|
||||
|
||||
# Install nomad 1.0.2
|
||||
# Install nomad 1.1.0
|
||||
curl -L -o nomad_${NOMAD_VERSION}_linux_amd64.zip https://releases.hashicorp.com/nomad/${NOMAD_VERSION}/nomad_${NOMAD_VERSION}_linux_amd64.zip
|
||||
sudo unzip -d /usr/local/bin nomad_${NOMAD_VERSION}_linux_amd64.zip
|
||||
sudo chmod +x /usr/local/bin/nomad
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user