skydns: prometheus working

Completely reworked it, added tests.

Author: miekg

.travis.yml

@@ -9,6 +9,7 @@ install:
   - go get github.com/coreos/go-systemd/activation
   - go get github.com/miekg/dns
   - go get github.com/rcrowley/go-metrics
+  - go get github.com/prometheus/client_golang/prometheus
   - go get github.com/rcrowley/go-metrics/stathat
 
 before_script:

main.go

@@ -156,8 +156,8 @@ func main() {
 		}()
 	}
 
-	stats.Collect() // Graphite
-	stats.Metrics()	// Prometheus
+	stats.Collect()  // Graphite
+	server.Metrics() // Prometheus
 
 	if err := s.Run(); err != nil {
 		log.Fatalf("skydns: %s", err)

server/dnssec.go

@@ -48,7 +48,7 @@ func (s *server) Sign(m *dns.Msg, bufsize uint16) {
 	expir := uint32(now.Add(7 * 24 * time.Hour).Unix()) // sign for a week
 
 	defer func() {
-		PromSCacheSize.Set(float64(s.scache.Size()))
+		promSCacheSize.Set(float64(s.scache.Size()))
 	}()
 
 	for _, r := range rrSets(m.Answer) {
@@ -108,7 +108,7 @@ func (s *server) signSet(r []dns.RR, now time.Time, incep, expir uint32) (*dns.R
 	log.Printf("skydns: scache miss for %s type %d", r[0].Header().Name, r[0].Header().Rrtype)
 
 	StatsDnssecCacheMiss.Inc(1)
-	PromSCacheMiss.Inc()
+	promSCacheMiss.Inc()
 
 	sig, err, shared := inflight.Do(key, func() (*dns.RRSIG, error) {
 		sig1 := s.NewRRSIG(incep, expir)

server/forwarding.go

@@ -15,7 +15,7 @@ import (
 // ServeDNSForward forwards a request to a nameservers and returns the response.
 func (s *server) ServeDNSForward(w dns.ResponseWriter, req *dns.Msg) {
 	StatsForwardCount.Inc(1)
-	PromForwardCount.Inc()
+	promForwardCount.Inc()
 
 	if s.config.NoRec {
 		m := new(dns.Msg)
@@ -105,7 +105,7 @@ func (s *server) ServeDNSReverse(w dns.ResponseWriter, req *dns.Msg) {
 // in the server's config. If none defined it returns an error
 func (s *server) Lookup(n string, t, bufsize uint16, dnssec bool) (*dns.Msg, error) {
 	StatsLookupCount.Inc(1)
-	PromLookupCount.Inc()
+	promLookupCount.Inc()
 
 	if len(s.config.Nameservers) == 0 {
 		return nil, fmt.Errorf("no nameservers configured can not lookup name")

server/server.go

@@ -202,20 +202,21 @@ func (s *server) ServeDNS(w dns.ResponseWriter, req *dns.Msg) {
 	}
 
 	if tcp {
-		PromRequestCountTCP.Inc()
+		promRequestCount.WithLabelValues("tcp").Inc()
 	} else {
-		PromRequestCountUDP.Inc()
+		promRequestCount.WithLabelValues("udp").Inc()
 	}
-	PromRequestCountTotal.Inc()
+	promRequestCount.WithLabelValues("total").Inc()
+
 	StatsRequestCount.Inc(1)
 
 	if dnssec {
 		StatsDnssecOkCount.Inc(1)
-		PromDnssecOkCount.Inc()
+		promDnssecOkCount.Inc()
 	}
 
 	defer func() {
-		PromRCacheSize.Set(float64(s.rcache.Size()))
+		promRCacheSize.Set(float64(s.rcache.Size()))
 	}()
 
 	// Check cache first.
@@ -252,7 +253,7 @@ func (s *server) ServeDNS(w dns.ResponseWriter, req *dns.Msg) {
 		s.rcache.Remove(key)
 	}
 
-	PromRCacheMiss.Inc()
+	promRCacheMiss.Inc()
 
 	q := req.Question[0]
 	name := strings.ToLower(q.Name)
@@ -780,7 +781,7 @@ func (s *server) NameError(m, req *dns.Msg) {
 	m.Ns[0].Header().Ttl = s.config.MinTtl
 
 	StatsNameErrorCount.Inc(1)
-	PromNameErrorCount.Inc()
+	promNameErrorCount.Inc()
 }
 
 func (s *server) NoDataError(m, req *dns.Msg) {
@@ -789,7 +790,7 @@ func (s *server) NoDataError(m, req *dns.Msg) {
 	m.Ns[0].Header().Ttl = s.config.MinTtl
 
 	StatsNoDataCount.Inc(1)
-	PromNoDataCount.Inc()
+	promNoDataCount.Inc()
 }
 
 func (s *server) logNoConnection(e error) {

server/server_test.go

@@ -8,8 +8,6 @@ package server
 
 import (
 	"encoding/json"
-	"io/ioutil"
-	"net/http"
 	"sort"
 	"strconv"
 	"strings"
@@ -19,7 +17,6 @@ import (
 
 	"github.com/coreos/go-etcd/etcd"
 	"github.com/miekg/dns"
-	"github.com/miekg/skydns/stats"
 	backendetcd "github.com/skynetservices/skydns/backends/etcd"
 	"github.com/skynetservices/skydns/cache"
 	"github.com/skynetservices/skydns/msg"
@@ -1284,39 +1281,3 @@ func BenchmarkDNSSECSingleNoCache(b *testing.B) {
 		c.Exchange(m, "127.0.0.1:"+StrPort)
 	}
 }
-
-// Metric testing
-
-func newMetricServer(t *testing.T) *server {
-	s := newTestServer(t, false)
-
-	stats.PrometheusPort = "12300"
-	stats.PrometheusNamespace = "test"
-	stats.Metrics()
-	return s
-}
-
-func query(n string, t uint16) {
-	// qeury the local server to get some metrics
-	m := new(dns.Msg)
-	m.SetQuestion(n, t)
-	dns.Exchange(m, "127.0.0.1:"+StrPort)
-}
-
-func scrape(t *testing.T, key string) {
-	resp, err := http.Get("http://localhost:12300/metrics")
-	if err != nil {
-		t.Fatal("could not get metrics")
-	}
-
-	body, _ := ioutil.ReadAll(resp.Body)
-	println(body)
-}
-
-func TestMetricRequests(t *testing.T) {
-	s := newMetricServer(t)
-	defer s.Stop()
-
-	query("miek.nl.", dns.TypeMX)
-	scrape(t, "dns_request_count_udp")
-}

server/stats.go

@@ -5,6 +5,11 @@
 package server
 
 import (
+	"log"
+	"net/http"
+	"os"
+	"strconv"
+
 	"github.com/prometheus/client_golang/prometheus"
 )
 
@@ -21,7 +26,7 @@ func (nopCounter) Inc(_ int64) {}
 // used by graphite.
 var (
 	// Pondering deletion in favor of the better and more
-	// maintained (by me) Prometheus reporting.
+	// maintained (by me) prometheus reporting.
 
 	StatsForwardCount     Counter = nopCounter{}
 	StatsStubForwardCount Counter = nopCounter{}
@@ -34,21 +39,139 @@ var (
 	StatsDnssecCacheMiss Counter = nopCounter{}
 )
 
-// Prometheus counters and gauges
 var (
-	PromForwardCount     prometheus.Counter
-	PromStubForwardCount prometheus.Counter
-	PromLookupCount      prometheus.Counter
-
-	PromRequestCountTotal prometheus.Counter
-	PromRequestCountTCP   prometheus.Counter
-	PromRequestCountUDP   prometheus.Counter
-
-	PromDnssecOkCount  prometheus.Counter
-	PromNameErrorCount prometheus.Counter
-	PromNoDataCount    prometheus.Counter
-	PromRCacheSize     prometheus.Gauge
-	PromSCacheSize     prometheus.Gauge
-	PromRCacheMiss     prometheus.Counter
-	PromSCacheMiss     prometheus.Counter
+	prometheusPort      = os.Getenv("PROMETHEUS_PORT")
+	prometheusPath      = os.Getenv("PROMETHEUS_PATH")
+	prometheusNamespace = os.Getenv("PROMETHEUS_NAMESPACE")
+	prometheusSubsystem = os.Getenv("PROMETHEUS_SUBSYSTEM")
+)
+
+var (
+	promForwardCount     prometheus.Counter
+	promStubForwardCount prometheus.Counter
+	promLookupCount      prometheus.Counter
+	promRequestCount     *prometheus.CounterVec
+	promDnssecOkCount    prometheus.Counter
+	promNameErrorCount   prometheus.Counter
+	promNoDataCount      prometheus.Counter
+	promRCacheSize       prometheus.Gauge
+	promSCacheSize       prometheus.Gauge
+	promRCacheMiss       prometheus.Counter
+	promSCacheMiss       prometheus.Counter
 )
+
+func Metrics() {
+	if prometheusPort == "" {
+		return
+	}
+
+	if prometheusPath == "" {
+		prometheusPath = "/metrics"
+	}
+	if prometheusNamespace == "" {
+		prometheusNamespace = "skydns"
+	}
+
+	promForwardCount = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: prometheusNamespace,
+		Subsystem: prometheusSubsystem,
+		Name:      "dns_forward_count",
+		Help:      "Counter of DNS requests forwarded.",
+	})
+
+	promStubForwardCount = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: prometheusNamespace,
+		Subsystem: prometheusSubsystem,
+		Name:      "dns_stub_forward_count",
+		Help:      "Counter of DNS requests forwarded to stubs.",
+	})
+
+	// convert to VEC and use labels
+	promLookupCount = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: prometheusNamespace,
+		Subsystem: prometheusSubsystem,
+		Name:      "dns_lookup_count",
+		Help:      "Counter of DNS lookups performed.",
+	})
+
+	promRequestCount = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace: prometheusNamespace,
+		Subsystem: prometheusSubsystem,
+		Name:      "dns_request_count",
+		Help:      "Counter of total DNS requests made.",
+	}, []string{"type"}, // total, udp, tcp
+	)
+	prometheus.MustRegister(promRequestCount)
+
+	promDnssecOkCount = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: prometheusNamespace,
+		Subsystem: prometheusSubsystem,
+		Name:      "dns_dnssec_ok_count",
+		Help:      "Counter of DNSSEC requests.",
+	})
+
+	promNameErrorCount = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: prometheusNamespace,
+		Subsystem: prometheusSubsystem,
+		Name:      "dns_name_error_count",
+		Help:      "Counter of DNS requests resulting in a name error.",
+	})
+
+	promNoDataCount = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: prometheusNamespace,
+		Subsystem: prometheusSubsystem,
+		Name:      "dns_no_data_count",
+		Help:      "Counter of DNS requests that contained no data.",
+	})
+
+	// Caches
+	promRCacheSize = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: prometheusNamespace,
+		Subsystem: prometheusSubsystem,
+		Name:      "rcache_total_size",
+		Help:      "The total size of all DNS messages in the rcache.",
+	})
+
+	promSCacheSize = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace: prometheusNamespace,
+		Subsystem: prometheusSubsystem,
+		Name:      "scache_total_size",
+		Help:      "The total size of all RRSIGs in the scache.",
+	})
+
+	promRCacheMiss = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: prometheusNamespace,
+		Subsystem: prometheusSubsystem,
+		Name:      "dns_rcache_miss_count",
+		Help:      "Counter of DNS requests that result in cache miss.",
+	})
+
+	promSCacheMiss = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: prometheusNamespace,
+		Subsystem: prometheusSubsystem,
+		Name:      "dns_scache_miss_count",
+		Help:      "Counter of signature requests that result in cache miss.",
+	})
+
+	prometheus.MustRegister(promForwardCount)
+	prometheus.MustRegister(promStubForwardCount)
+	prometheus.MustRegister(promLookupCount)
+
+	prometheus.MustRegister(promDnssecOkCount)
+	prometheus.MustRegister(promNameErrorCount)
+	prometheus.MustRegister(promNoDataCount)
+	prometheus.MustRegister(promRCacheSize)
+	prometheus.MustRegister(promSCacheSize)
+	prometheus.MustRegister(promRCacheMiss)
+	prometheus.MustRegister(promSCacheMiss)
+
+	_, err := strconv.Atoi(prometheusPort)
+	if err != nil {
+		return
+	}
+
+	http.Handle(prometheusPath, prometheus.Handler())
+	go func() {
+		log.Fatalf("skydns: %s", http.ListenAndServe(":"+prometheusPort, nil))
+	}()
+}

server/stub.go

@@ -70,7 +70,7 @@ func (s *server) UpdateStubZones() {
 // ServeDNSStubForward forwards a request to a nameservers and returns the response.
 func (s *server) ServeDNSStubForward(w dns.ResponseWriter, req *dns.Msg, ns []string) {
 	StatsStubForwardCount.Inc(1)
-	PromStubForwardCount.Inc()
+	promStubForwardCount.Inc()
 
 	// Check EDNS0 Stub option, if set drop the packet.
 	option := req.IsEdns0()