Add monitoring for commit status

dn42promsrv.go

@@ -88,9 +88,10 @@ func collectMetrics(stop chan bool,
 
 func initMetrics() []Metric {
 
-	metrics := make([]Metric, 1)
+	metrics := make([]Metric, 2)
 
 	metrics[0] = &DNSMetrics{}
+	metrics[1] = &DNSCommitMetrics{}
 
 	return metrics
 }

dns.go

@@ -7,19 +7,24 @@ package main
 //////////////////////////////////////////////////////////////////////////
 
 import (
+	"encoding/json"
+	"errors"
 	//	"fmt"
 	dns "github.com/miekg/dns"
 	"github.com/prometheus/client_golang/prometheus"
 	log "github.com/sirupsen/logrus"
+	"io/ioutil"
 	"math"
+	"net/http"
 	"strconv"
+	"strings"
 	"time"
 )
 
 //////////////////////////////////////////////////////////////////////////
 // data structures
 
-// for holding the metrics
+// for holding the DNS metrics
 
 type DNSMetrics struct {
 	soa       *prometheus.GaugeVec
@@ -65,6 +70,28 @@ var dns_servers = []*DNSServer{
 	&DNSServer{"burble.dn42", "burble", "lt-vil1", 6, "[fd42:4242:2601:3d::ac:53]:53", 0},
 }
 
+// data structures for querying the current commit metric
+
+type DNSCommitMetrics struct {
+	match        *prometheus.GaugeVec
+	lastUpdate   time.Time
+	mismatch     bool
+	since        time.Time
+	labelInvalid prometheus.Labels
+	labelUpdate  prometheus.Labels
+}
+
+type DNSExplorerCommit struct {
+	Commit string
+}
+
+type DNSMasterCommit struct {
+	Type   string `json:"type"`
+	URI    string `json:"uri"`
+	Branch string `json:"branch"`
+	Commit string `json:"commit"`
+}
+
 //////////////////////////////////////////////////////////////////////////
 // initialisation function to register metrics
 
@@ -190,5 +217,150 @@ func (s *DNSServer) Query() (float64, float64) {
 	return 0, 0
 }
 
+//////////////////////////////////////////////////////////////////////////
+
+func (m *DNSCommitMetrics) Register() {
+	m.match = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "dn42_dns_commit",
+		Help: "Time since commits last matched (or -1 for error)",
+	}, []string{"metric"})
+	prometheus.MustRegister(m.match)
+
+	m.labelInvalid = prometheus.Labels{
+		"metric": "invalid",
+	}
+
+	m.labelUpdate = prometheus.Labels{
+		"metric": "update",
+	}
+
+}
+
+func (m *DNSCommitMetrics) Collect() {
+
+	now := time.Now()
+	interval := now.Sub(m.lastUpdate)
+
+	// only check if it's more than 60 mins since the last successful match
+	if (interval.Hours() >= 1.0) || m.mismatch {
+
+		// fetch the current commit from the explorer
+		ec, err := m.FetchExplorerCommit()
+		if err != nil {
+			// couldn't fetch the explorer commit value, set the metric to bad
+			m.match.With(m.labelUpdate).Set(-1.0)
+			m.match.With(m.labelInvalid).Set(-1.0)
+			m.mismatch = true
+			return
+		}
+
+		mc, err := m.FetchMasterCommit()
+		if err != nil {
+			// couldn't fetch master commit value, set the metric to bad
+			m.match.With(m.labelUpdate).Set(-2.0)
+			m.match.With(m.labelInvalid).Set(-1.0)
+			m.mismatch = true
+			return
+		}
+
+		m.lastUpdate = now
+		m.match.With(m.labelUpdate).Set(0.0)
+
+		if ec == mc {
+
+			// if the commits match, null the metric
+			m.match.With(m.labelInvalid).Set(0.0)
+			m.mismatch = false
+
+		} else {
+
+			// was this the first mismatch ?
+			if !m.mismatch {
+				m.since = now
+				m.mismatch = true
+			}
+
+			// set the metric to be the interval since last good match
+			m.match.With(m.labelInvalid).Set(float64(now.Sub(m.since).Seconds()))
+		}
+
+	} else {
+
+		// update time since last check
+		m.match.With(m.labelUpdate).Set(float64(interval.Seconds()))
+	}
+
+}
+
+//////////////////////////////////////////////////////////////////////////
+
+func (m *DNSCommitMetrics) FetchExplorerCommit() (string, error) {
+
+	response, err := http.Get("http://collector.dn42:8043/api/registry/.meta")
+	if err != nil {
+		log.WithFields(log.Fields{
+			"error": err,
+		}).Warn("Unable to query registry explorer")
+		return "", err
+	}
+
+	data, err := ioutil.ReadAll(response.Body)
+	if err != nil {
+		log.WithFields(log.Fields{
+			"error": err,
+		}).Warn("Unable read explorer response")
+		return "", err
+	}
+
+	var ec DNSExplorerCommit
+	if err := json.Unmarshal(data, &ec); err != nil {
+		log.WithFields(log.Fields{
+			"error": err,
+		}).Warn("Unable to parse explorer JSON")
+		return "", err
+	}
+
+	return strings.ToLower(ec.Commit), nil
+
+}
+
+func (m *DNSCommitMetrics) FetchMasterCommit() (string, error) {
+
+	response, err := http.Get("http://[2a0c:3800:1:1011:5054:ff:fe87:ea39]:8080/api/git-db-state")
+	if err != nil {
+		log.WithFields(log.Fields{
+			"error": err,
+		}).Warn("Unable to query registry master")
+		return "", err
+	}
+
+	data, err := ioutil.ReadAll(response.Body)
+	if err != nil {
+		log.WithFields(log.Fields{
+			"error": err,
+		}).Warn("Unable read master response")
+		return "", err
+	}
+
+	var mc []DNSMasterCommit
+	if err := json.Unmarshal(data, &mc); err != nil {
+		log.WithFields(log.Fields{
+			"error": err,
+		}).Warn("Unable to parse master JSON")
+		return "", err
+	}
+
+	for _, c := range mc {
+		if c.URI == "https://git.dn42.us/dn42/registry.git" {
+			return strings.ToLower(c.Commit), nil
+		}
+	}
+
+	log.WithFields(log.Fields{
+		"MasterCommit": mc,
+	}).Warn("Unable to find registry commit from master")
+	return "", errors.New("Unable to find registry commit from master")
+}
+
 //////////////////////////////////////////////////////////////////////////
 // end of code