burble/nomad-driver-containerd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #3 from Roblox/exec

Browse Source 
ExecTask API
This commit is contained in:
Shishir 2020-06-09 16:09:43 -07:00 committed by GitHub
commit 80585b3a0a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 201 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
containerd/driver.go
View File

@ -515,6 +515,16 @@ func (d *Driver) SignalTask(taskID string, signal string) error {
return handle.signal(d.ctxContainerd, sig)
}
// ExecTaskStreaming returns the result of executing the given command inside a task.
func (d *Driver) ExecTaskStreaming(ctx context.Context, taskID string, opts *drivers.ExecOptions) (*drivers.ExitResult, error) {
handle, ok := d.tasks.Get(taskID)
if !ok {
return nil, drivers.ErrTaskNotFound
}
return handle.exec(ctx, d.ctxContainerd, taskID, opts)
}
// ExecTask returns the result of executing the given command inside a task.
// This is an optional capability.
func (d *Driver) ExecTask(taskID string, cmd []string, timeout time.Duration) (*drivers.ExecTaskResult, error) {

64
containerd/handle.go
View File

@ -9,7 +9,9 @@ import (
"time"
"github.com/containerd/containerd"
"github.com/containerd/containerd/cio"
"github.com/hashicorp/go-hclog"
uuid "github.com/hashicorp/go-uuid"
"github.com/hashicorp/nomad/plugins/drivers"
)
@ -80,6 +82,68 @@ func (h *taskHandle) run(ctxContainerd context.Context) {
h.task.Start(ctxContainerd)
}
// exec launches a new process in a running container.
func (h *taskHandle) exec(ctx, ctxContainerd context.Context, taskID string, opts *drivers.ExecOptions) (*drivers.ExitResult, error) {
defer opts.Stdout.Close()
defer opts.Stderr.Close()
spec, err := h.container.Spec(ctxContainerd)
if err != nil {
return nil, err
}
pspec := spec.Process
pspec.Terminal = opts.Tty
pspec.Args = opts.Command
execID, err := uuid.GenerateUUID()
if err != nil {
return nil, err
}
cioOpts := []cio.Opt{cio.WithStreams(opts.Stdin, opts.Stdout, opts.Stderr)}
if opts.Tty {
cioOpts = append(cioOpts, cio.WithTerminal)
}
ioCreator := cio.NewCreator(cioOpts...)
process, err := h.task.Exec(ctxContainerd, execID[:8], pspec, ioCreator)
if err != nil {
return nil, err
}
defer process.Delete(ctxContainerd)
statusC, err := process.Wait(ctxContainerd)
if err != nil {
return nil, err
}
if err := process.Start(ctxContainerd); err != nil {
return nil, err
}
var code uint32
status := <-statusC
code, _, err = status.Result()
if err != nil {
return nil, err
}
go func() {
for {
select {
case <-ctx.Done():
return
}
}
}()
return &drivers.ExitResult{
ExitCode: int(code),
}, nil
}
func (h *taskHandle) shutdown(ctxContainerd context.Context, timeout time.Duration, signal syscall.Signal) error {
if err := h.task.Kill(ctxContainerd, signal); err != nil {
return err

1
go.mod
View File

@ -30,6 +30,7 @@ require (
github.com/hashicorp/go-getter v1.4.0 // indirect
github.com/hashicorp/go-hclog v0.10.0
github.com/hashicorp/go-plugin v1.0.1
github.com/hashicorp/go-uuid v1.0.1
github.com/hashicorp/hcl2 v0.0.0-20191002203319-fb75b3253c80 // indirect
github.com/hashicorp/nomad v0.10.1
github.com/hashicorp/nomad/api v0.0.0-20191203164002-b31573ae7206 // indirect

148
tests/run_tests.sh
View File

@ -3,7 +3,9 @@
set -euo pipefail
export NOMAD_VERSION=0.11.2
export CONTAINERD_VERSION=1.3.4
export PATH=$PATH:/usr/local/go/bin
export PATH=$PATH:/usr/local/bin
export GOPATH=/home/circleci/go
export GO_VERSION=1.14.3
@ -14,49 +16,58 @@ main() {
echo "Starting setup."
setup
echo "Setup finished successfully."
echo "Checking if nomad-driver-containerd is up and running, and nomad is ready to accept jobs."
while true
do
set +e
status=$(curl -s http://127.0.0.1:4646/v1/nodes|jq '.[0] ."Drivers" ."containerd-driver" ."Healthy"')
rc=$?
set -e
if [[ $rc -eq 0 && $status = "true" ]]; then
echo "nomad is up and running"
break
fi
echo "nomad is down, wait for 5 seconds."
sleep 5s
done
is_containerd_driver_active
cd ~/go/src/github.com/Roblox/nomad-driver-containerd/example
echo "Starting nomad redis job using nomad-driver-containerd."
nomad job run redis.nomad
echo "Starting nomad signal handler job using nomad-driver-containerd."
nomad job run signal.nomad
echo "Checking status of redis job."
redis_status=$(nomad job status -short redis|grep Status|awk '{split($0,a,"="); print a[2]}'|tr -d ' ')
if [ $redis_status != "running" ];then
echo "Error in getting redis job status."
exit 1
fi
# Even though $(nomad job status) reports redis job status as "running"
# The actual container process might not be running yet.
# We need to wait for actual container to start running before trying exec.
echo "Wait for redis container to get into RUNNING state, before trying exec."
is_redis_container_active
echo "Checking status of signal handler job."
signal_status=$(nomad job status -short signal|grep Status|awk '{split($0,a,"="); print a[2]}'|tr -d ' ')
if [ $signal_status != "running" ];then
echo "Error in getting signal handler job status."
exit 1
fi
echo "Inspecting redis job."
redis_status=$(nomad job inspect redis|jq -r '.Job .Status')
if [ $redis_status != "running" ];then
echo "Error in inspecting redis job."
exit 1
fi
echo "Inspecting signal handler job."
signal_status=$(nomad job inspect signal|jq -r '.Job .Status')
if [ $signal_status != "running" ]; then
echo "Error in inspecting signal handler job."
exit 1
fi
echo "Exec redis job."
exec_output=$(nomad alloc exec -job redis echo hello_exec)
if [ $exec_output != "hello_exec" ]; then
echo "Error in exec'ing redis job."
exit 1
fi
echo "Stopping nomad redis job."
nomad job stop redis
redis_status=$(nomad job status -short redis|grep Status|awk '{split($0,a,"="); print a[2]}'|tr -d ' ')
@ -64,6 +75,7 @@ main() {
echo "Error in stopping redis job."
exit 1
fi
echo "Stopping nomad signal handler job."
nomad job stop signal
signal_status=$(nomad job status -short signal|grep Status|awk '{split($0,a,"="); print a[2]}'|tr -d ' ')
@ -81,18 +93,55 @@ setup() {
sudo pkill --signal SIGKILL -P $(ps faux | grep 'daily' | awk '{print $2}')
set -e
sudo apt-get install -y apt-utils curl unzip make build-essential
# Stop docker daemon. We only want containerd daemon running.
# Remove docker daemon and containerd.
sudo systemctl stop docker
sudo systemctl stop containerd
sudo apt-get purge -y docker-ce docker-ce-cli containerd.io
sudo apt-get install -y apt-utils curl runc unzip make build-essential
# Change $(pwd) to /tmp
cd /tmp
# Install containerd 1.3.4
curl -L -o containerd-${CONTAINERD_VERSION}.linux-amd64.tar.gz https://github.com/containerd/containerd/releases/download/v${CONTAINERD_VERSION}/containerd-${CONTAINERD_VERSION}.linux-amd64.tar.gz
sudo tar -C /usr/local -xzf containerd-${CONTAINERD_VERSION}.linux-amd64.tar.gz
rm -f containerd-${CONTAINERD_VERSION}.linux-amd64.tar.gz
# Drop containerd systemd unit file into /lib/systemd/system.
cat << EOF > containerd.service
# /lib/systemd/system/nomad.service
[Unit]
Description=containerd container runtime
Documentation=https://containerd.io
After=network.target
[Service]
ExecStartPre=-/sbin/modprobe overlay
ExecStart=/usr/local/bin/containerd
KillMode=process
Delegate=yes
LimitNOFILE=1048576
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNPROC=infinity
LimitCORE=infinity
TasksMax=infinity
[Install]
WantedBy=multi-user.target
EOF
sudo mv containerd.service /lib/systemd/system/containerd.service
sudo systemctl daemon-reload
echo "Starting containerd daemon."
sudo systemctl start containerd
is_systemd_service_active "containerd.service"
# Remove default golang (1.7.3) and install a custom version (1.14.3) of golang.
# This is required for supporting go mod, and to be able to compile nomad-driver-containerd.
sudo rm -rf /usr/local/go
# Change $(pwd) to /tmp
cd /tmp
# Install golang 1.14.3
curl -L -o go${GO_VERSION}.linux-amd64.tar.gz https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz
sudo tar -C /usr/local -xzf go${GO_VERSION}.linux-amd64.tar.gz
@ -136,13 +185,68 @@ EOF
sudo systemctl daemon-reload
echo "Starting nomad server and nomad-driver-containerd."
sudo systemctl start nomad
while true
is_systemd_service_active "nomad.service"
}
is_containerd_driver_active() {
i="0"
while test $i -lt 5
do
if (systemctl -q is-active "nomad.service"); then
echo "systemd nomad.service is up and running"
set +e
status=$(curl -s http://127.0.0.1:4646/v1/nodes|jq '.[0] ."Drivers" ."containerd-driver" ."Healthy"')
rc=$?
set -e
if [[ $rc -eq 0 && $status = "true" ]]; then
echo "containerd driver is up and running."
break
fi
echo "containerd driver is down, sleep for 3 seconds."
sleep 3s
i=$[$i+1]
done
if [ $i -ge 5 ]; then
echo "containerd driver didn't come up. exit 1."
exit 1
fi
}
is_redis_container_active() {
set +e
i="0"
while test $i -lt 5
do
sudo CONTAINERD_NAMESPACE=nomad ctr task ls|grep -q RUNNING
if [ $? -eq 0 ]; then
echo "redis container is up and running"
break
fi
echo "redis container is down, sleep for 3 seconds."
sleep 3s
i=$[$i+1]
done
set -e
if [ $i -ge 5 ]; then
echo "redis container didn't come up. exit 1."
exit 1
fi
}
is_systemd_service_active() {
local service_name=$1
i="0"
while test $i -lt 5 && !(systemctl -q is-active "$service_name"); do
printf "%s is down, sleep for 3 seconds.\n" $service_name
sleep 3s
i=$[$i+1]
done
if [ $i -ge 5 ]; then
printf "%s didn't come up. exit 1.\n" $service_name
exit 1
fi
printf "%s is up and running\n" $service_name
}
main "$@"