From 3a7108f6207770df447ca2472416a306a76e7bf1 Mon Sep 17 00:00:00 2001 From: Simon Marsh Date: Mon, 27 May 2019 12:37:44 +0100 Subject: [PATCH] Add monitoring for commit status --- dn42promsrv.go | 3 +- dns.go | 174 ++++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 175 insertions(+), 2 deletions(-) diff --git a/dn42promsrv.go b/dn42promsrv.go index 43a03b2..29d0f62 100644 --- a/dn42promsrv.go +++ b/dn42promsrv.go @@ -88,9 +88,10 @@ func collectMetrics(stop chan bool, func initMetrics() []Metric { - metrics := make([]Metric, 1) + metrics := make([]Metric, 2) metrics[0] = &DNSMetrics{} + metrics[1] = &DNSCommitMetrics{} return metrics } diff --git a/dns.go b/dns.go index b6e46e9..8db104f 100644 --- a/dns.go +++ b/dns.go @@ -7,19 +7,24 @@ package main ////////////////////////////////////////////////////////////////////////// import ( + "encoding/json" + "errors" // "fmt" dns "github.com/miekg/dns" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" + "io/ioutil" "math" + "net/http" "strconv" + "strings" "time" ) ////////////////////////////////////////////////////////////////////////// // data structures -// for holding the metrics +// for holding the DNS metrics type DNSMetrics struct { soa *prometheus.GaugeVec @@ -65,6 +70,28 @@ var dns_servers = []*DNSServer{ &DNSServer{"burble.dn42", "burble", "lt-vil1", 6, "[fd42:4242:2601:3d::ac:53]:53", 0}, } +// data structures for querying the current commit metric + +type DNSCommitMetrics struct { + match *prometheus.GaugeVec + lastUpdate time.Time + mismatch bool + since time.Time + labelInvalid prometheus.Labels + labelUpdate prometheus.Labels +} + +type DNSExplorerCommit struct { + Commit string +} + +type DNSMasterCommit struct { + Type string `json:"type"` + URI string `json:"uri"` + Branch string `json:"branch"` + Commit string `json:"commit"` +} + ////////////////////////////////////////////////////////////////////////// // initialisation function to register metrics @@ -190,5 +217,150 @@ func (s *DNSServer) Query() (float64, float64) { return 0, 0 } +////////////////////////////////////////////////////////////////////////// + +func (m *DNSCommitMetrics) Register() { + m.match = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "dn42_dns_commit", + Help: "Time since commits last matched (or -1 for error)", + }, []string{"metric"}) + prometheus.MustRegister(m.match) + + m.labelInvalid = prometheus.Labels{ + "metric": "invalid", + } + + m.labelUpdate = prometheus.Labels{ + "metric": "update", + } + +} + +func (m *DNSCommitMetrics) Collect() { + + now := time.Now() + interval := now.Sub(m.lastUpdate) + + // only check if it's more than 60 mins since the last successful match + if (interval.Hours() >= 1.0) || m.mismatch { + + // fetch the current commit from the explorer + ec, err := m.FetchExplorerCommit() + if err != nil { + // couldn't fetch the explorer commit value, set the metric to bad + m.match.With(m.labelUpdate).Set(-1.0) + m.match.With(m.labelInvalid).Set(-1.0) + m.mismatch = true + return + } + + mc, err := m.FetchMasterCommit() + if err != nil { + // couldn't fetch master commit value, set the metric to bad + m.match.With(m.labelUpdate).Set(-2.0) + m.match.With(m.labelInvalid).Set(-1.0) + m.mismatch = true + return + } + + m.lastUpdate = now + m.match.With(m.labelUpdate).Set(0.0) + + if ec == mc { + + // if the commits match, null the metric + m.match.With(m.labelInvalid).Set(0.0) + m.mismatch = false + + } else { + + // was this the first mismatch ? + if !m.mismatch { + m.since = now + m.mismatch = true + } + + // set the metric to be the interval since last good match + m.match.With(m.labelInvalid).Set(float64(now.Sub(m.since).Seconds())) + } + + } else { + + // update time since last check + m.match.With(m.labelUpdate).Set(float64(interval.Seconds())) + } + +} + +////////////////////////////////////////////////////////////////////////// + +func (m *DNSCommitMetrics) FetchExplorerCommit() (string, error) { + + response, err := http.Get("http://collector.dn42:8043/api/registry/.meta") + if err != nil { + log.WithFields(log.Fields{ + "error": err, + }).Warn("Unable to query registry explorer") + return "", err + } + + data, err := ioutil.ReadAll(response.Body) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + }).Warn("Unable read explorer response") + return "", err + } + + var ec DNSExplorerCommit + if err := json.Unmarshal(data, &ec); err != nil { + log.WithFields(log.Fields{ + "error": err, + }).Warn("Unable to parse explorer JSON") + return "", err + } + + return strings.ToLower(ec.Commit), nil + +} + +func (m *DNSCommitMetrics) FetchMasterCommit() (string, error) { + + response, err := http.Get("http://[2a0c:3800:1:1011:5054:ff:fe87:ea39]:8080/api/git-db-state") + if err != nil { + log.WithFields(log.Fields{ + "error": err, + }).Warn("Unable to query registry master") + return "", err + } + + data, err := ioutil.ReadAll(response.Body) + if err != nil { + log.WithFields(log.Fields{ + "error": err, + }).Warn("Unable read master response") + return "", err + } + + var mc []DNSMasterCommit + if err := json.Unmarshal(data, &mc); err != nil { + log.WithFields(log.Fields{ + "error": err, + }).Warn("Unable to parse master JSON") + return "", err + } + + for _, c := range mc { + if c.URI == "https://git.dn42.us/dn42/registry.git" { + return strings.ToLower(c.Commit), nil + } + } + + log.WithFields(log.Fields{ + "MasterCommit": mc, + }).Warn("Unable to find registry commit from master") + return "", errors.New("Unable to find registry commit from master") +} + ////////////////////////////////////////////////////////////////////////// // end of code