Merge pull request #1 from Roblox/tests

Integration tests + CI
2020-05-27 15:29:08 -07:00 · 2020-05-27 15:29:08 -07:00 · c711ead981
commit c711ead981
parent 6ca77fda0c 2bf9f0a0c0
5 changed files with 169 additions and 41 deletions
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@ -0,0 +1,8 @@
+version: 2
+jobs:
+  build:
+    machine: true
+    working_directory: ~/go/src/github.com/Roblox/nomad-driver-containerd
+    steps:
+      - checkout
+      - run: make test
--- a/14
+++ b/14
@ -1,12 +1,18 @@
-PLUGIN_BINARY=containerd-driver
+BINARY ?= containerd-driver
+GOLANG ?= /usr/local/go/bin/go
+
 export GO111MODULE=on

 default: build

 .PHONY: clean
-clean: ## Remove build artifacts
-	rm -rf ${PLUGIN_BINARY}
+clean:
+	rm -f $(BINARY)

 .PHONY: build
 build:
-	go build -o ${PLUGIN_BINARY} .
+	$(GOLANG) build -o $(BINARY) .
+
+.PHONY: test
+test:
+	./tests/run_tests.sh
--- a/build/Dockerfile
+++ b/build/Dockerfile
@ -1,34 +0,0 @@
-# Docker image for nomad-driver-containerd
-# This image will contain nomad (server + client), golang and containerd.
-FROM ubuntu:18.04
-
-ENV NOMAD_VERSION 0.11.2
-ENV GO_VERSION 1.14.3
-ENV CONTAINERD_VERSION 1.3.3
-
-RUN apt-get update \
-  && apt-get install -y apt-utils curl unzip
-
-# Install golang
-RUN set -x \
-  && cd /tmp \
-  && curl -L -o go${GO_VERSION}.linux-amd64.tar.gz https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz \
-  && tar -C /usr/local -xzf go${GO_VERSION}.linux-amd64.tar.gz \
-  && chmod +x /usr/local/go \
-  && echo "export PATH=$PATH:/usr/local/go/bin" >> $HOME/.bashrc \
-  && rm -f go${GO_VERSION}.linux-amd64.tar.gz
-
-# Install containerd
-RUN set -x \
-  && cd /tmp \
-  && curl -L -o containerd-${CONTAINERD_VERSION}.linux-amd64.tar.gz https://github.com/containerd/containerd/releases/download/v${CONTAINERD_VERSION}/containerd-${CONTAINERD_VERSION}.linux-amd64.tar.gz \
-  && tar -C /usr/local -xzf containerd-${CONTAINERD_VERSION}.linux-amd64.tar.gz \
-  && rm -f containerd-${CONTAINERD_VERSION}.linux-amd64.tar.gz
-
-# Install nomad
-RUN set -x \
-  && cd /tmp \
-  && curl -L -o nomad_${NOMAD_VERSION}_linux_amd64.zip https://releases.hashicorp.com/nomad/${NOMAD_VERSION}/nomad_${NOMAD_VERSION}_linux_amd64.zip \
-  && unzip -d /usr/local/bin nomad_${NOMAD_VERSION}_linux_amd64.zip \
-  && chmod +x /usr/local/bin/nomad \
-  && rm -f nomad_${NOMAD_VERSION}_linux_amd64.zip
--- a/example/example.nomad
+++ b/example/example.nomad
@ -1,8 +1,8 @@
-job "example" {
+job "redis" {
  datacenters = ["dc1"]

-  group "cache" {
-    task "redis" {
+  group "redis-group" {
+    task "redis-task" {
      driver = "containerd-driver"

      config {
--- a/tests/run_tests.sh
+++ b/tests/run_tests.sh
@ -0,0 +1,148 @@
+#!/bin/bash
+
+set -euo pipefail
+
+export NOMAD_VERSION=0.11.2
+export PATH=$PATH:/usr/local/go/bin
+export GOPATH=/home/circleci/go
+export GO_VERSION=1.14.3
+
+# These tests are designed to be run as part of continous integration (CI) and not on local host.
+# Please don't run these tests (./run_tests.sh) on your local host, as these are meant to be
+# destructive and can modify (or destroy) software on your host system.
+main() {
+	echo "Starting setup."
+	setup
+	echo "Setup finished successfully."
+	echo "Checking if nomad-driver-containerd is up and running, and nomad is ready to accept jobs."
+	while true
+	do
+		set +e
+		status=$(curl -s http://127.0.0.1:4646/v1/nodes|jq '.[0] ."Drivers" ."containerd-driver" ."Healthy"')
+		rc=$?
+		set -e
+		if [[ $rc -eq 0 && $status = "true" ]]; then
+			echo "nomad is up and running"
+			break
+		fi
+		echo "nomad is down, wait for 5 seconds."
+		sleep 5s
+	done
+	cd ~/go/src/github.com/Roblox/nomad-driver-containerd/example
+	echo "Starting nomad redis job using nomad-driver-containerd."
+	nomad job run redis.nomad
+	echo "Starting nomad signal handler job using nomad-driver-containerd."
+	nomad job run signal.nomad
+	echo "Checking status of redis job."
+	redis_status=$(nomad job status -short redis|grep Status|awk '{split($0,a,"="); print a[2]}'|tr -d ' ')
+	if [ $redis_status != "running" ];then
+		echo "Error in getting redis job status."
+		exit 1
+	fi
+	echo "Checking status of signal handler job."
+	signal_status=$(nomad job status -short signal|grep Status|awk '{split($0,a,"="); print a[2]}'|tr -d ' ')
+        if [ $signal_status != "running" ];then
+		echo "Error in getting signal handler job status."
+		exit 1
+	fi
+	echo "Inspecting redis job."
+	redis_status=$(nomad job inspect redis|jq -r '.Job .Status')
+	if [ $redis_status != "running" ];then
+		echo "Error in inspecting redis job."
+		exit 1
+	fi
+	echo "Inspecting signal handler job."
+	signal_status=$(nomad job inspect signal|jq -r '.Job .Status')
+	if [ $signal_status != "running" ]; then
+		echo "Error in inspecting signal handler job."
+		exit 1
+	fi
+	echo "Stopping nomad redis job."
+	nomad job stop redis
+	redis_status=$(nomad job status -short redis|grep Status|awk '{split($0,a,"="); print a[2]}'|tr -d ' ')
+	if [ $redis_status != "dead(stopped)" ];then
+		echo "Error in stopping redis job."
+		exit 1
+	fi
+	echo "Stopping nomad signal handler job."
+	nomad job stop signal
+	signal_status=$(nomad job status -short signal|grep Status|awk '{split($0,a,"="); print a[2]}'|tr -d ' ')
+        if [ $signal_status != "dead(stopped)" ];then
+                echo "Error in stopping signal handler job."
+                exit 1
+        fi
+	echo "Tests finished successfully."
+}
+
+setup() {
+	sudo systemctl stop apt-daily-upgrade apt-daily >/dev/null 2>&1
+
+	set +e
+	sudo pkill --signal SIGKILL -P $(ps faux | grep 'daily' | awk '{print $2}')
+	set -e
+
+	sudo apt-get install -y apt-utils curl unzip make build-essential
+
+	# Stop docker daemon. We only want containerd daemon running.
+	sudo systemctl stop docker
+
+	# Remove default golang (1.7.3) and install a custom version (1.14.3) of golang.
+	# This is required for supporting go mod, and to be able to compile nomad-driver-containerd.
+	sudo rm -rf /usr/local/go
+
+	# Change $(pwd) to /tmp
+	cd /tmp
+
+	# Install golang 1.14.3
+	curl -L -o go${GO_VERSION}.linux-amd64.tar.gz https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz
+	sudo tar -C /usr/local -xzf go${GO_VERSION}.linux-amd64.tar.gz
+	sudo chmod +x /usr/local/go
+	rm -f go${GO_VERSION}.linux-amd64.tar.gz
+
+	# Install nomad 0.11.2
+	curl -L -o nomad_${NOMAD_VERSION}_linux_amd64.zip https://releases.hashicorp.com/nomad/${NOMAD_VERSION}/nomad_${NOMAD_VERSION}_linux_amd64.zip
+	sudo unzip -d /usr/local/bin nomad_${NOMAD_VERSION}_linux_amd64.zip
+	sudo chmod +x /usr/local/bin/nomad
+	rm -f nomad_${NOMAD_VERSION}_linux_amd64.zip
+
+	echo "Building nomad-driver-containerd."
+	cd ~/go/src/github.com/Roblox/nomad-driver-containerd
+	make build
+	echo "move containerd-driver to /tmp/nomad-driver-containerd."
+	mkdir -p /tmp/nomad-driver-containerd
+	mv containerd-driver /tmp/nomad-driver-containerd
+
+	# Drop nomad server (dev) + nomad-driver-containerd systemd unit file into /lib/systemd/system.
+	cat << EOF > nomad.service
+# /lib/systemd/system/nomad.service
+[Unit]
+Description=nomad server (dev) + nomad-driver-containerd
+Documentation=https://nomadproject.io
+After=network.target
+
+[Service]
+ExecStart=/usr/local/bin/nomad agent -dev -config=/home/circleci/go/src/github.com/Roblox/nomad-driver-containerd/example/agent.hcl -plugin-dir=/tmp/nomad-driver-containerd
+KillMode=process
+Delegate=yes
+LimitNOFILE=1048576
+LimitNPROC=infinity
+LimitCORE=infinity
+TasksMax=infinity
+
+[Install]
+WantedBy=multi-user.target
+EOF
+	sudo mv nomad.service /lib/systemd/system/nomad.service
+	sudo systemctl daemon-reload
+	echo "Starting nomad server and nomad-driver-containerd."
+	sudo systemctl start nomad
+	while true
+	do
+		if (systemctl -q is-active "nomad.service"); then
+			echo "systemd nomad.service is up and running"
+			break
+		fi
+	done
+}
+
+main "$@"