From 9ffe5d4fa8e616e38fe592dca970c9670e163667 Mon Sep 17 00:00:00 2001 From: lisongmin Date: Wed, 6 Jan 2021 10:38:09 +0800 Subject: [PATCH] add test for volume_mount --- example/agent_tests.hcl | 16 ++++++ example/volume_mount.nomad | 42 +++++++++++++++ tests/005-test-volume_mount.sh | 95 ++++++++++++++++++++++++++++++++++ tests/run_tests.sh | 10 +++- 4 files changed, 162 insertions(+), 1 deletion(-) create mode 100644 example/agent_tests.hcl create mode 100644 example/volume_mount.nomad create mode 100755 tests/005-test-volume_mount.sh diff --git a/example/agent_tests.hcl b/example/agent_tests.hcl new file mode 100644 index 0000000..02fa770 --- /dev/null +++ b/example/agent_tests.hcl @@ -0,0 +1,16 @@ +log_level = "INFO" + +plugin "containerd-driver" { + config { + enabled = true + containerd_runtime = "io.containerd.runc.v2" + stats_interval = "5s" + } +} + +client { + host_volume "s1" { + path = "/tmp/host_volume/s1" + read_only = false + } +} diff --git a/example/volume_mount.nomad b/example/volume_mount.nomad new file mode 100644 index 0000000..89eb6f2 --- /dev/null +++ b/example/volume_mount.nomad @@ -0,0 +1,42 @@ +job "volume_mount" { + datacenters = ["dc1"] + + group "volume_mount-group" { + + volume "data" { + type = "host" + source = "s1" + read_only = false + } + + volume "read_only_data" { + type = "host" + source = "s1" + read_only = true + } + + task "volume_mount-task" { + driver = "containerd-driver" + config { + image = "docker.io/library/ubuntu:16.04" + command = "sleep" + args = ["600s"] + } + + volume_mount { + destination = "/tmp/t1" + volume = "data" + } + + volume_mount { + destination = "/tmp/read_only_target" + volume = "read_only_data" + } + + resources { + cpu = 500 + memory = 256 + } + } + } +} diff --git a/tests/005-test-volume_mount.sh b/tests/005-test-volume_mount.sh new file mode 100755 index 0000000..f2e64d6 --- /dev/null +++ b/tests/005-test-volume_mount.sh @@ -0,0 +1,95 @@ +#!/bin/bash + +job_name=volume_mount +host_volume_path=/tmp/host_volume/s1 + +# test volume_mount +test_volume_mount_nomad_job() { + pushd ~/go/src/github.com/Roblox/nomad-driver-containerd/example + + setup_bind_source + + echo "INFO: Starting nomad $job_name job using nomad-driver-containerd." + nomad job run $job_name.nomad + + # Even though $(nomad job status) reports job status as "running" + # The actual container process might not be running yet. + # We need to wait for actual container to start running before trying exec. + echo "INFO: Wait for ${job_name} container to get into RUNNING state, before trying exec." + is_${job_name}_container_active + + echo "INFO: Checking status of $job_name job." + job_status=$(nomad job status -short $job_name|grep Status|awk '{split($0,a,"="); print a[2]}'|tr -d ' ') + if [ "$job_status" != "running" ];then + echo "ERROR: Error in getting ${job_name} job status." + exit 1 + fi + + # Check if bind mount exists. + echo "INFO: Checking if bind mount exists." + for mountpoint in t1 read_only_target ; do + output=$(nomad alloc exec -job ${job_name} cat /tmp/${mountpoint}/bind.txt) + if [ "$output" != "hello" ]; then + echo "ERROR: bind mount /tmp/${mountpoint} does not exist in container rootfs." + exit 1 + fi + done + + # Check read only mount can not write. + echo "INFO: Checking read only mount is not writable." + nomad alloc exec -job ${job_name} touch /tmp/read_only_target/writable_test.txt &>/dev/null + if [ -e ${host_volume_path}/writable_test.txt ];then + echo "ERROR: Read only bind mount in /tmp/read_only_target should not be writable." + exit 1 + fi + + # Check writable mount can write. + echo "INFO: Checking non read_only mount is writable." + nomad alloc exec -job ${job_name} touch /tmp/t1/writable_test.txt + if [ ! -e ${host_volume_path}/writable_test.txt ];then + echo "ERROR: bind mount in /tmp/t1 should be writable." + exit 1 + fi + + echo "INFO: Stopping nomad ${job_name} job." + nomad job stop ${job_name} + job_status=$(nomad job status -short ${job_name}|grep Status|awk '{split($0,a,"="); print a[2]}'|tr -d ' ') + if [ $job_status != "dead(stopped)" ];then + echo "ERROR: Error in stopping ${job_name} job." + exit 1 + fi + + echo "INFO: purge nomad ${job_name} job." + nomad job stop -purge ${job_name} + popd +} + +setup_bind_source() { + rm -f ${host_volume_path}/bind.txt + rm -f ${host_volume_path}/writable_test.txt + + echo hello > ${host_volume_path}/bind.txt +} + +is_volume_mount_container_active() { + i="0" + while test $i -lt 5 + do + sudo CONTAINERD_NAMESPACE=nomad ctr task ls|grep -q RUNNING + if [ $? -eq 0 ]; then + echo "INFO: ${job_name} container is up and running" + sleep 5s + break + fi + echo "INFO: ${job_name} container is down, sleep for 4 seconds." + sleep 4s + i=$[$i+1] + done + + if [ $i -ge 5 ]; then + echo "ERROR: ${job_name} container didn't come up. exit 1." + exit 1 + fi +} + +test_volume_mount_nomad_job diff --git a/tests/run_tests.sh b/tests/run_tests.sh index 539e258..8118db5 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -152,7 +152,7 @@ Documentation=https://nomadproject.io After=network.target [Service] -ExecStart=/usr/local/bin/nomad agent -dev -config=/home/circleci/go/src/github.com/Roblox/nomad-driver-containerd/example/agent.hcl -plugin-dir=/tmp/nomad-driver-containerd +ExecStart=/usr/local/bin/nomad agent -dev -config=/home/circleci/go/src/github.com/Roblox/nomad-driver-containerd/example/agent_tests.hcl -plugin-dir=/tmp/nomad-driver-containerd KillMode=process Delegate=yes LimitNOFILE=1048576 @@ -165,12 +165,20 @@ WantedBy=multi-user.target EOF sudo mv nomad.service /lib/systemd/system/nomad.service sudo systemctl daemon-reload + + prepare_nomad_host_volume + echo "INFO: Starting nomad server and nomad-driver-containerd." sudo systemctl start nomad is_systemd_service_active "nomad.service" popd } +prepare_nomad_host_volume() { + echo "INFO: Prepare nomad host volume." + mkdir -p /tmp/host_volume/s1 +} + is_containerd_driver_active() { i="0" while test $i -lt 5