burble/nomad-driver-containerd
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update versions, run nomad in non-dev mode and fix tests.

Browse Source 
Signed-off-by: Shishir Mahajan <smahajan@roblox.com>
This commit is contained in:
Shishir Mahajan 2021-09-02 12:41:12 -07:00
parent d40a7e36f5
commit 42fc67cc80
15 changed files with 67 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
Vagrantfile vendored
View File

@ -28,34 +28,34 @@ Vagrant.configure("2") do |config|
# without keeping HOME env, 'sudo make test' will try to find files under /root/go/
echo "Defaults env_keep += HOME" | sudo tee /etc/sudoers.d/keep_home
# Install golang-1.14.3
# Install golang-1.17
if [ ! -f "/usr/local/go/bin/go" ]; then
curl -s -L -o go1.14.3.linux-amd64.tar.gz https://dl.google.com/go/go1.14.3.linux-amd64.tar.gz
sudo tar -C /usr/local -xzf go1.14.3.linux-amd64.tar.gz
curl -s -L -o go1.17.linux-amd64.tar.gz https://dl.google.com/go/go1.17.linux-amd64.tar.gz
sudo tar -C /usr/local -xzf go1.17.linux-amd64.tar.gz
sudo chmod +x /usr/local/go
rm -f go1.14.3.linux-amd64.tar.gz
rm -f go1.17.linux-amd64.tar.gz
fi
# Install nomad-1.1.0
# Install nomad-1.1.4
if [ ! -f "/usr/bin/nomad" ]; then
wget --quiet https://releases.hashicorp.com/nomad/1.1.0/nomad_1.1.0_linux_amd64.zip
unzip nomad_1.1.0_linux_amd64.zip -d /usr/bin
wget --quiet https://releases.hashicorp.com/nomad/1.1.4/nomad_1.1.4_linux_amd64.zip
unzip nomad_1.1.4_linux_amd64.zip -d /usr/bin
chmod +x /usr/bin/nomad
rm -f nomad_1.1.0_linux_amd64.zip
rm -f nomad_1.1.4_linux_amd64.zip
fi
# Install containerd-1.3.4
# Install containerd-1.5.5
if [ ! -f "/usr/local/bin/containerd" ]; then
curl -L --silent -o containerd-1.3.4.linux-amd64.tar.gz https://github.com/containerd/containerd/releases/download/v1.3.4/containerd-1.3.4.linux-amd64.tar.gz
tar -C /usr/local -xzf containerd-1.3.4.linux-amd64.tar.gz
rm -f containerd-1.3.4.linux-amd64.tar.gz
curl -L --silent -o containerd-1.5.5-linux-amd64.tar.gz https://github.com/containerd/containerd/releases/download/v1.5.5/containerd-1.5.5-linux-amd64.tar.gz
tar -C /usr/local -xzf containerd-1.5.5-linux-amd64.tar.gz
rm -f containerd-1.5.5-linux-amd64.tar.gz
fi
# Install nerdctl 0.10.0
# Install nerdctl 0.11.1
if [ ! -f "/usr/local/bin/nerdctl" ]; then
curl -L --silent -o nerdctl-0.10.0-linux-amd64.tar.gz https://github.com/containerd/nerdctl/releases/download/v0.10.0/nerdctl-0.10.0-linux-amd64.tar.gz
tar -C /usr/local/bin -xzf nerdctl-0.10.0-linux-amd64.tar.gz
rm -f nerdctl-0.10.0-linux-amd64.tar.gz
curl -L --silent -o nerdctl-0.11.1-linux-amd64.tar.gz https://github.com/containerd/nerdctl/releases/download/v0.11.1/nerdctl-0.11.1-linux-amd64.tar.gz
tar -C /usr/local/bin -xzf nerdctl-0.11.1-linux-amd64.tar.gz
rm -f nerdctl-0.11.1-linux-amd64.tar.gz
fi
# Create source directory for privileged.nomad example job.

4
example/agent.hcl
View File

@ -1,4 +1,5 @@
log_level = "INFO"
data_dir = "/tmp/nomad"
plugin "containerd-driver" {
config {
@ -9,6 +10,8 @@ plugin "containerd-driver" {
}
server {
enabled = true
bootstrap_expect = 1
default_scheduler_config {
scheduler_algorithm = "spread"
memory_oversubscription_enabled = true
@ -22,6 +25,7 @@ server {
}
client {
enabled = true
host_volume "s1" {
path = "/tmp/host_volume/s1"
read_only = false

6
example/entrypoint.nomad
View File

@ -6,9 +6,9 @@ job "entrypoint" {
driver = "containerd-driver"
config {
image = "ubuntu:16.04"
entrypoint = ["/bin/echo"]
args = ["container1", "container2"]
image = "ubuntu:16.04"
entrypoint = ["/bin/bash"]
args = ["-c", "for i in {1..100}; do echo container1 container2; sleep 1s; done"]
}
resources {

6
tests/001-test-redis.sh
View File

@ -6,7 +6,7 @@ test_redis_nomad_job() {
pushd ~/go/src/github.com/Roblox/nomad-driver-containerd/example
echo "INFO: Starting nomad redis job using nomad-driver-containerd."
nomad job run redis.nomad
nomad job run -detach redis.nomad
redis_status=$(nomad job status -short redis|grep Status|awk '{split($0,a,"="); print a[2]}'|tr -d ' ')
if [ $redis_status != "running" ];then
@ -63,7 +63,7 @@ test_redis_nomad_job() {
fi
echo "INFO: Stopping nomad redis job."
nomad job stop redis
nomad job stop -detach redis
redis_status=$(nomad job status -short redis|grep Status|awk '{split($0,a,"="); print a[2]}'|tr -d ' ')
if [ $redis_status != "dead(stopped)" ];then
echo "ERROR: Error in stopping redis job."
@ -71,7 +71,7 @@ test_redis_nomad_job() {
fi
echo "INFO: purge nomad redis job."
nomad job stop -purge redis
nomad job stop -detach -purge redis
popd
}

6
tests/002-test-signal-handler.sh
View File

@ -6,7 +6,7 @@ test_signal_handler_nomad_job() {
pushd ~/go/src/github.com/Roblox/nomad-driver-containerd/example
echo "INFO: Starting nomad signal handler job using nomad-driver-containerd."
nomad job run signal.nomad
nomad job run -detach signal.nomad
echo "INFO: Checking status of signal handler job."
signal_status=$(nomad job status -short signal|grep Status|awk '{split($0,a,"="); print a[2]}'|tr -d ' ')
@ -40,7 +40,7 @@ test_signal_handler_nomad_job() {
cleanup "$outfile"
echo "INFO: Stopping nomad signal handler job."
nomad job stop signal
nomad job stop -detach signal
signal_status=$(nomad job status -short signal|grep Status|awk '{split($0,a,"="); print a[2]}'|tr -d ' ')
if [ $signal_status != "dead(stopped)" ];then
echo "ERROR: Error in stopping signal handler job."
@ -48,7 +48,7 @@ test_signal_handler_nomad_job() {
fi
echo "INFO: purge nomad signal handler job."
nomad job stop -purge signal
nomad job stop -detach -purge signal
popd
}

6
tests/003-test-capabilities.sh
View File

@ -7,7 +7,7 @@ test_capabilities_nomad_job() {
pushd ~/go/src/github.com/Roblox/nomad-driver-containerd/example
echo "INFO: Starting nomad capabilities job using nomad-driver-containerd."
nomad job run capabilities.nomad
nomad job run -detach capabilities.nomad
echo "INFO: Checking status of capabilities job."
cap_status=$(nomad job status -short capabilities|grep Status|awk '{split($0,a,"="); print a[2]}'|tr -d ' ')
@ -59,7 +59,7 @@ test_capabilities_nomad_job() {
cleanup "$outfile"
echo "INFO: Stopping nomad capabilities job."
nomad job stop capabilities
nomad job stop -detach capabilities
cap_status=$(nomad job status -short capabilities|grep Status|awk '{split($0,a,"="); print a[2]}'|tr -d ' ')
if [ $cap_status != "dead(stopped)" ];then
echo "ERROR: Error in stopping capabilities job."
@ -67,7 +67,7 @@ test_capabilities_nomad_job() {
fi
echo "INFO: purge nomad capabilities job."
nomad job stop -purge capabilities
nomad job stop -detach -purge capabilities
popd
}

6
tests/004-test-privileged.sh
View File

@ -9,7 +9,7 @@ test_privileged_nomad_job() {
setup_bind_source
echo "INFO: Starting nomad privileged job using nomad-driver-containerd."
nomad job run privileged.nomad
nomad job run -detach privileged.nomad
echo "INFO: Checking status of privileged job."
job_status=$(nomad job status -short privileged|grep Status|awk '{split($0,a,"="); print a[2]}'|tr -d ' ')
@ -68,7 +68,7 @@ test_privileged_nomad_job() {
fi
echo "INFO: Stopping nomad privileged job."
nomad job stop privileged
nomad job stop -detach privileged
job_status=$(nomad job status -short privileged|grep Status|awk '{split($0,a,"="); print a[2]}'|tr -d ' ')
if [ $job_status != "dead(stopped)" ];then
echo "ERROR: Error in stopping privileged job."
@ -76,7 +76,7 @@ test_privileged_nomad_job() {
fi
echo "INFO: purge nomad privileged job."
nomad job stop -purge privileged
nomad job stop -detach -purge privileged
popd
}

6
tests/005-test-volume-mount.sh
View File

@ -12,7 +12,7 @@ test_volume_mount_nomad_job() {
setup_bind_source
echo "INFO: Starting nomad $job_name job using nomad-driver-containerd."
nomad job run $job_name.nomad
nomad job run -detach $job_name.nomad
# Even though $(nomad job status) reports job status as "running"
# The actual container process might not be running yet.
@ -54,7 +54,7 @@ test_volume_mount_nomad_job() {
fi
echo "INFO: Stopping nomad ${job_name} job."
nomad job stop ${job_name}
nomad job stop -detach ${job_name}
job_status=$(nomad job status -short ${job_name}|grep Status|awk '{split($0,a,"="); print a[2]}'|tr -d ' ')
if [ $job_status != "dead(stopped)" ];then
echo "ERROR: Error in stopping ${job_name} job."
@ -62,7 +62,7 @@ test_volume_mount_nomad_job() {
fi
echo "INFO: purge nomad ${job_name} job."
nomad job stop -purge ${job_name}
nomad job stop -detach -purge ${job_name}
popd
}

6
tests/006-test-dns.sh
View File

@ -8,7 +8,7 @@ test_dns_nomad_job() {
pushd ~/go/src/github.com/Roblox/nomad-driver-containerd/example
echo "INFO: Starting nomad $job_name job using nomad-driver-containerd."
nomad job run $job_name.nomad
nomad job run -detach $job_name.nomad
# Even though $(nomad job status) reports job status as "running"
# The actual container process might not be running yet.
@ -62,7 +62,7 @@ test_dns_nomad_job() {
fi
echo "INFO: Stopping nomad ${job_name} job."
nomad job stop ${job_name}
nomad job stop -detach ${job_name}
job_status=$(nomad job status -short ${job_name}|grep Status|awk '{split($0,a,"="); print a[2]}'|tr -d ' ')
if [ $job_status != "dead(stopped)" ];then
echo "ERROR: Error in stopping ${job_name} job."
@ -70,7 +70,7 @@ test_dns_nomad_job() {
fi
echo "INFO: purge nomad ${job_name} job."
nomad job stop -purge ${job_name}
nomad job stop -detach -purge ${job_name}
popd
}

6
tests/007-test-extra-hosts.sh
View File

@ -8,7 +8,7 @@ test_extra_hosts_nomad_job() {
pushd ~/go/src/github.com/Roblox/nomad-driver-containerd/example
echo "INFO: Starting nomad $job_name job using nomad-driver-containerd."
nomad job run $job_name.nomad
nomad job run -detach $job_name.nomad
# Even though $(nomad job status) reports job status as "running"
# The actual container process might not be running yet.
@ -34,7 +34,7 @@ test_extra_hosts_nomad_job() {
done
echo "INFO: Stopping nomad ${job_name} job."
nomad job stop ${job_name}
nomad job stop -detach ${job_name}
job_status=$(nomad job status -short ${job_name}|grep Status|awk '{split($0,a,"="); print a[2]}'|tr -d ' ')
if [ $job_status != "dead(stopped)" ];then
echo "ERROR: Error in stopping ${job_name} job."
@ -42,7 +42,7 @@ test_extra_hosts_nomad_job() {
fi
echo "INFO: purge nomad ${job_name} job."
nomad job stop -purge ${job_name}
nomad job stop -detach -purge ${job_name}
popd
}

4
tests/008-test-entrypoint.sh
View File

@ -8,7 +8,7 @@ test_entrypoint_nomad_job() {
pushd ~/go/src/github.com/Roblox/nomad-driver-containerd/example
echo "INFO: Starting nomad $job_name job using nomad-driver-containerd."
nomad job run $job_name.nomad
nomad job run -detach $job_name.nomad
# Even though $(nomad job status) reports job status as "running"
# The actual container process might not be running yet.
@ -33,7 +33,7 @@ test_entrypoint_nomad_job() {
done
echo "INFO: purge nomad ${job_name} job."
nomad job stop -purge ${job_name}
nomad job stop -detach -purge ${job_name}
popd
}

4
tests/009-test-auth.sh
View File

@ -9,7 +9,7 @@ test_auth_nomad_job() {
pushd ~/go/src/github.com/Roblox/nomad-driver-containerd/example
echo "INFO: Starting nomad $job_name job using nomad-driver-containerd."
nomad job run $job_name.nomad
nomad job run -detach $job_name.nomad
wait_nomad_job_status $job_name failed
@ -23,7 +23,7 @@ test_auth_nomad_job() {
fi
echo "INFO: purge nomad ${job_name} job."
nomad job stop -purge ${job_name}
nomad job stop -detach -purge ${job_name}
popd
}

6
tests/010-test-allow-privileged.sh
View File

@ -9,12 +9,12 @@ test_allow_privileged() {
cp agent.hcl agent.hcl.bkp
sed -i '8 i \ allow_privileged = false' agent.hcl
sed -i '9 i \ allow_privileged = false' agent.hcl
sudo systemctl restart nomad
is_systemd_service_active "nomad.service" true
echo "INFO: Starting nomad ${job_name} job using nomad-driver-containerd."
nomad job run privileged_not_allowed.nomad
nomad job run -detach privileged_not_allowed.nomad
# Sleep for 5 seconds, to allow ${alloc_id} to get populated.
sleep 5s
@ -28,7 +28,7 @@ test_allow_privileged() {
fi
echo "INFO: purge nomad ${job_name} job."
nomad job stop -purge ${job_name}
nomad job stop -detach -purge ${job_name}
mv agent.hcl.bkp agent.hcl
popd

26
tests/run_tests.sh
View File

@ -2,8 +2,8 @@
set -eo pipefail
export NOMAD_VERSION=1.1.0
export CONTAINERD_VERSION=1.3.4
export NOMAD_VERSION=1.1.4
export CONTAINERD_VERSION=1.5.5
export PATH=$PATH:/usr/local/go/bin
export PATH=$PATH:/usr/local/bin
if [ -e /home/runner ]; then
@ -11,7 +11,7 @@ if [ -e /home/runner ]; then
else
export GOPATH=$HOME/go
fi
export GO_VERSION=1.14.3
export GO_VERSION=1.17
export SRCDIR=`dirname $0`
source $SRCDIR/utils.sh
@ -83,10 +83,10 @@ setup() {
# Change $(pwd) to /tmp
pushd /tmp
# Install containerd 1.3.4
curl -L -o containerd-${CONTAINERD_VERSION}.linux-amd64.tar.gz https://github.com/containerd/containerd/releases/download/v${CONTAINERD_VERSION}/containerd-${CONTAINERD_VERSION}.linux-amd64.tar.gz
sudo tar -C /usr/local -xzf containerd-${CONTAINERD_VERSION}.linux-amd64.tar.gz
rm -f containerd-${CONTAINERD_VERSION}.linux-amd64.tar.gz
# Install containerd 1.5.5
curl -L -o containerd-${CONTAINERD_VERSION}-linux-amd64.tar.gz https://github.com/containerd/containerd/releases/download/v${CONTAINERD_VERSION}/containerd-${CONTAINERD_VERSION}-linux-amd64.tar.gz
sudo tar -C /usr/local -xzf containerd-${CONTAINERD_VERSION}-linux-amd64.tar.gz
rm -f containerd-${CONTAINERD_VERSION}-linux-amd64.tar.gz
# Drop containerd systemd unit file into /lib/systemd/system.
cat << EOF > containerd.service
@ -120,17 +120,17 @@ EOF
sudo systemctl start containerd
is_systemd_service_active "containerd.service" false
# Remove default golang (1.7.3) and install a custom version (1.14.3) of golang.
# Remove default golang (1.7.3) and install a custom version (1.17) of golang.
# This is required for supporting go mod, and to be able to compile nomad-driver-containerd.
sudo rm -rf /usr/local/go
# Install golang 1.14.3
# Install golang 1.17
curl -L -o go${GO_VERSION}.linux-amd64.tar.gz https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz
sudo tar -C /usr/local -xzf go${GO_VERSION}.linux-amd64.tar.gz
sudo chmod +x /usr/local/go
rm -f go${GO_VERSION}.linux-amd64.tar.gz
# Install nomad 1.1.0
# Install nomad 1.1.4
curl -L -o nomad_${NOMAD_VERSION}_linux_amd64.zip https://releases.hashicorp.com/nomad/${NOMAD_VERSION}/nomad_${NOMAD_VERSION}_linux_amd64.zip
sudo unzip -d /usr/local/bin nomad_${NOMAD_VERSION}_linux_amd64.zip
sudo chmod +x /usr/local/bin/nomad
@ -143,16 +143,16 @@ EOF
mkdir -p /tmp/nomad-driver-containerd
mv containerd-driver /tmp/nomad-driver-containerd
# Drop nomad server (dev) + nomad-driver-containerd systemd unit file into /lib/systemd/system.
# Drop nomad server + nomad-driver-containerd systemd unit file into /lib/systemd/system.
cat << EOF > nomad.service
# /lib/systemd/system/nomad.service
[Unit]
Description=nomad server (dev) + nomad-driver-containerd
Description=nomad server + nomad-driver-containerd
Documentation=https://nomadproject.io
After=network.target
[Service]
ExecStart=/usr/local/bin/nomad agent -dev -config=$GOPATH/src/github.com/Roblox/nomad-driver-containerd/example/agent.hcl -plugin-dir=/tmp/nomad-driver-containerd
ExecStart=/usr/local/bin/nomad agent -config=$GOPATH/src/github.com/Roblox/nomad-driver-containerd/example/agent.hcl -plugin-dir=/tmp/nomad-driver-containerd
KillMode=process
Delegate=yes
LimitNOFILE=1048576

6
vagrant/setup.sh
View File

@ -45,16 +45,16 @@ main() {
drop_nomad_unit_file() {
local nomad=$(which nomad)
# Drop nomad server (dev) + nomad-driver-containerd systemd unit file into /lib/systemd/system.
# Drop nomad server + nomad-driver-containerd systemd unit file into /lib/systemd/system.
cat << EOF > nomad.service
# /lib/systemd/system/nomad.service
[Unit]
Description=nomad server (dev) + nomad-driver-containerd
Description=nomad server + nomad-driver-containerd
Documentation=https://nomadproject.io
After=network.target containerd.service
[Service]
ExecStart=$nomad agent -dev -bind=0.0.0.0 -config=$1/example/agent.hcl -plugin-dir=/tmp/nomad-driver-containerd
ExecStart=$nomad agent -bind=0.0.0.0 -config=$1/example/agent.hcl -plugin-dir=/tmp/nomad-driver-containerd
KillMode=process
Delegate=yes
LimitNOFILE=1048576