Merge pull request #162 from fastly/extend-metric-blocklist

Fix metric-blocklist for datacenter_info, service_info, and last_successful_response

README.md

@@ -104,7 +104,7 @@ fastly-exporter [common flags] -service-shard 3/3
 
 By default, all metrics provided by the Fastly real-time stats API are exported
 as Prometheus metrics. You can export only those metrics whose name matches a
-regex by using the `-metric-allowlist 'bytes_total$'` flag, or elide any metric
+regex by using the `-metric-allowlist 'bytes_total$'` flag, or exclude any metric
 whose name matches a regex by using the `-metric-blocklist imgopto` flag.
 
 ### Filter semantics

cmd/fastly-exporter/main.go

@@ -269,7 +269,8 @@ func main() {
 
 	var datacenterCache *api.DatacenterCache
 	{
-		datacenterCache = api.NewDatacenterCache(apiClient, token)
+		enabled := !metricNameFilter.Blocked(prometheus.BuildFQName(namespace, deprecatedSubsystem, "datacenter_info"))
+		datacenterCache = api.NewDatacenterCache(apiClient, token, enabled)
 	}
 
 	var productCache *api.ProductCache
@@ -285,12 +286,14 @@ func main() {
 			}
 			return nil
 		})
-		g.Go(func() error {
-			if err := datacenterCache.Refresh(context.Background()); err != nil {
-				level.Warn(logger).Log("during", "initial fetch of datacenters", "err", err, "msg", "datacenter labels unavailable, will retry")
-			}
-			return nil
-		})
+		if datacenterCache.Enabled() {
+			g.Go(func() error {
+				if err := datacenterCache.Refresh(context.Background()); err != nil {
+					level.Warn(logger).Log("during", "initial fetch of datacenters", "err", err, "msg", "datacenter labels unavailable, will retry")
+				}
+				return nil
+			})
+		}
 		g.Go(func() error {
 			if err := productCache.Refresh(context.Background()); err != nil {
 				level.Warn(logger).Log("during", "initial fetch of products", "err", err, "msg", "products API unavailable, will retry")
@@ -302,7 +305,7 @@ func main() {
 	}
 
 	var defaultGatherers prometheus.Gatherers
-	{
+	if datacenterCache.Enabled() {
 		dcs, err := datacenterCache.Gatherer(namespace, deprecatedSubsystem)
 		if err != nil {
 			level.Error(apiLogger).Log("during", "create datacenter gatherer", "err", err)
@@ -311,6 +314,20 @@ func main() {
 		defaultGatherers = append(defaultGatherers, dcs)
 	}
 
+	if !metricNameFilter.Blocked(prometheus.BuildFQName(namespace, deprecatedSubsystem, "token_expiration")) {
+		tokenRecorder := api.NewTokenRecorder(apiClient, token)
+		tg, err := tokenRecorder.Gatherer(namespace, deprecatedSubsystem)
+		if err != nil {
+			level.Error(apiLogger).Log("during", "create token gatherer", "err", err)
+		} else {
+			err = tokenRecorder.Set(context.Background())
+			if err != nil {
+				level.Error(apiLogger).Log("during", "set token gauge metric", "err", err)
+			}
+			defaultGatherers = append(defaultGatherers, tg)
+		}
+	}
+
 	var registry *prom.Registry
 	{
 		registry = prom.NewRegistry(programVersion, namespace, deprecatedSubsystem, metricNameFilter, defaultGatherers)
@@ -331,7 +348,9 @@ func main() {
 	}
 
 	var g run.Group
-	{
+	// only setup the ticker if the datacenterCache is enabled.
+	if datacenterCache.Enabled() {
+
 		// Every datacenterRefresh, ask the api.DatacenterCache to refresh
 		// metadata from the api.fastly.com/datacenters endpoint.
 		var (

go.mod

@@ -17,6 +17,7 @@ require (
 require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/go-logfmt/logfmt v0.5.1 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect

pkg/api/datacenter_cache.go

@@ -29,24 +29,29 @@ type Coördinates struct {
 // DatacenterCache polls api.fastly.com/datacenters and maintains a local cache
 // of the returned metadata. That information is exposed as Prometheus metrics.
 type DatacenterCache struct {
-	client HTTPClient
-	token  string
+	client  HTTPClient
+	token   string
+	enabled bool
 
 	mtx sync.Mutex
 	dcs []Datacenter
 }
 
 // NewDatacenterCache returns an empty cache of datacenter metadata. Use the
 // Refresh method to update the cache.
-func NewDatacenterCache(client HTTPClient, token string) *DatacenterCache {
+func NewDatacenterCache(client HTTPClient, token string, enabled bool) *DatacenterCache {
 	return &DatacenterCache{
-		client: client,
-		token:  token,
+		client:  client,
+		token:   token,
+		enabled: enabled,
 	}
 }
 
 // Refresh the cache with metadata retreived from the Fastly API.
 func (c *DatacenterCache) Refresh(ctx context.Context) error {
+	if !c.enabled {
+		return nil
+	}
 	req, err := http.NewRequestWithContext(ctx, "GET", "https://api.fastly.com/datacenters", nil)
 	if err != nil {
 		return fmt.Errorf("error constructing API datacenters request: %w", err)
@@ -109,6 +114,11 @@ func (c *DatacenterCache) Gatherer(namespace, subsystem string) (prometheus.Gath
 	return registry, nil
 }
 
+// Enabled returns true if the DatacenterCache is enabled
+func (c *DatacenterCache) Enabled() bool {
+	return c.enabled
+}
+
 type datacenterCollector struct {
 	desc  *prometheus.Desc
 	cache *DatacenterCache

pkg/api/datacenter_cache_test.go

@@ -38,7 +38,7 @@ func TestDatacenterCache(t *testing.T) {
 			var (
 				ctx    = context.Background()
 				client = testcase.client
-				cache  = api.NewDatacenterCache(client, "irrelevant token")
+				cache  = api.NewDatacenterCache(client, "irrelevant token", true)
 			)
 
 			if want, have := testcase.wantErr, cache.Refresh(ctx); !cmp.Equal(want, have) {

pkg/api/token.go

@@ -0,0 +1,88 @@
+package api
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// TokenRecorder requests api.fastly.com/tokens/self once and sets a gauge metric
+type TokenRecorder struct {
+	client HTTPClient
+	token  string
+	metric *prometheus.GaugeVec
+}
+
+// NewTokenRecorder returns an empty token recorder. Use the
+// Set method to get token data and set the gauge metric.
+func NewTokenRecorder(client HTTPClient, token string) *TokenRecorder {
+	return &TokenRecorder{
+		client: client,
+		token:  token,
+	}
+}
+
+// Gatherer returns a Prometheus gatherer which will yield current
+// token expiration as a gauge metric.
+func (t *TokenRecorder) Gatherer(namespace, subsystem string) (prometheus.Gatherer, error) {
+	registry := prometheus.NewRegistry()
+	tokenExpiration := prometheus.NewGaugeVec(prometheus.GaugeOpts{Namespace: namespace, Subsystem: subsystem, Name: "token_expiration", Help: "Unix timestamp of the expiration time of the Fastly API Token"}, []string{"token_id", "user_id"})
+	err := registry.Register(tokenExpiration)
+	if err != nil {
+		return nil, fmt.Errorf("registering token collector: %w", err)
+	}
+	t.metric = tokenExpiration
+	return registry, nil
+}
+
+// Set retreives token metadata from the Fastly API and sets the gauge metric
+func (t *TokenRecorder) Set(ctx context.Context) error {
+	token, err := t.getToken(ctx)
+	if err != nil {
+		return err
+	}
+
+	if !token.Expiration.IsZero() {
+		t.metric.WithLabelValues(token.ID, token.UserID).Set(float64(token.Expiration.Unix()))
+	}
+	return nil
+}
+
+type token struct {
+	ID         string    `json:"id"`
+	UserID     string    `json:"user_id"`
+	Expiration time.Time `json:"expires_at,omitempty"`
+}
+
+func (t *TokenRecorder) getToken(ctx context.Context) (*token, error) {
+	uri := "https://api.fastly.com/tokens/self"
+
+	req, err := http.NewRequestWithContext(ctx, "GET", uri, nil)
+	if err != nil {
+		return nil, fmt.Errorf("error constructing API tokens request: %w", err)
+	}
+
+	req.Header.Set("Fastly-Key", t.token)
+	req.Header.Set("Accept", "application/json")
+	resp, err := t.client.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("error executing API tokens request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, NewError(resp)
+	}
+
+	var response token
+
+	if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
+		return nil, fmt.Errorf("error decoding API Tokens response: %w", err)
+	}
+
+	return &response, nil
+}

pkg/api/token_test.go

@@ -0,0 +1,72 @@
+package api_test
+
+import (
+	"context"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/fastly/fastly-exporter/pkg/api"
+	"github.com/prometheus/client_golang/prometheus/testutil"
+)
+
+func TestTokenMetric(t *testing.T) {
+	var (
+		namespace = "fastly"
+		subsystem = "rt"
+	)
+	client := api.NewTokenRecorder(fixedResponseClient{code: http.StatusOK, response: tokenReponseExpiresAt}, "")
+	gatherer, _ := client.Gatherer(namespace, subsystem)
+	client.Set(context.Background())
+
+	expected := `
+# HELP fastly_rt_token_expiration Unix timestamp of the expiration time of the Fastly API Token
+# TYPE fastly_rt_token_expiration gauge
+fastly_rt_token_expiration{token_id="id1234",user_id="user456"} 1.7764704e+09
+`
+	err := testutil.GatherAndCompare(gatherer, strings.NewReader(expected), "fastly_rt_token_expiration")
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func TestTokenMetricWithoutExpiration(t *testing.T) {
+	var (
+		namespace = "fastly"
+		subsystem = "rt"
+	)
+	client := api.NewTokenRecorder(fixedResponseClient{code: http.StatusOK, response: tokenReponseNoExpiry}, "")
+	gatherer, _ := client.Gatherer(namespace, subsystem)
+	client.Set(context.Background())
+
+	expected := `
+# HELP fastly_rt_token_expiration Unix timestamp of the expiration time of the Fastly API Token
+# TYPE fastly_rt_token_expiration gauge
+`
+	err := testutil.GatherAndCompare(gatherer, strings.NewReader(expected), "fastly_rt_token_expiration")
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+const tokenReponseExpiresAt = `
+{
+  "id": "id1234",
+  "user_id": "user456",
+  "customer_id": "customer987",
+  "name": "Fastly API Token",
+  "last_used_at": "2024-04-18T13:37:06Z",
+  "created_at": "2016-10-11T18:36:35Z",
+  "expires_at": "2026-04-18T00:00:00Z"
+}`
+
+const tokenReponseNoExpiry = `
+{
+  "id": "id1234",
+  "user_id": "user456",
+  "customer_id": "customer987",
+  "name": "Fastly API Token",
+  "last_used_at": "2024-04-18T13:37:06Z",
+  "created_at": "2016-10-11T18:36:35Z",
+  "expires_at": null
+}`

pkg/filter/filter.go

@@ -41,6 +41,12 @@ func (f *Filter) Permit(s string) (allowed bool) {
 	return f.passAllowlist(s) && f.passBlocklist(s)
 }
 
+// Blocked checks if the provided string is blocked, according to the current
+// blocklist expressions.
+func (f *Filter) Blocked(s string) (blocked bool) {
+	return !f.passBlocklist(s)
+}
+
 func (f *Filter) passAllowlist(s string) bool {
 	if len(f.allowlist) <= 0 {
 		return true // default pass

pkg/prom/metrics.go

@@ -1,6 +1,8 @@
 package prom
 
 import (
+	"regexp"
+
 	"github.com/fastly/fastly-exporter/pkg/domain"
 	"github.com/fastly/fastly-exporter/pkg/filter"
 	"github.com/fastly/fastly-exporter/pkg/origin"
@@ -25,7 +27,13 @@ func NewMetrics(namespace, rtSubsystemWillBeDeprecated string, nameFilter filter
 		serviceInfo            = prometheus.NewGaugeVec(prometheus.GaugeOpts{Namespace: namespace, Subsystem: rtSubsystemWillBeDeprecated, Name: "service_info", Help: "Static gauge with service ID, name, and version information."}, []string{"service_id", "service_name", "service_version"})
 		lastSuccessfulResponse = prometheus.NewGaugeVec(prometheus.GaugeOpts{Namespace: namespace, Subsystem: rtSubsystemWillBeDeprecated, Name: "last_successful_response", Help: "Unix timestamp of the last successful response received from the real-time stats API."}, []string{"service_id", "service_name"})
 	)
-	r.MustRegister(serviceInfo, lastSuccessfulResponse)
+
+	if name := getName(serviceInfo); !nameFilter.Blocked(name) {
+		r.MustRegister(serviceInfo)
+	}
+	if name := getName(lastSuccessfulResponse); !nameFilter.Blocked(name) {
+		r.MustRegister(lastSuccessfulResponse)
+	}
 
 	return &Metrics{
 		ServiceInfo:            serviceInfo,
@@ -35,3 +43,16 @@ func NewMetrics(namespace, rtSubsystemWillBeDeprecated string, nameFilter filter
 		Domain:                 domain.NewMetrics(namespace, "domain", nameFilter, r),
 	}
 }
+
+var descNameRegex = regexp.MustCompile("fqName: \"([^\"]+)\"")
+
+func getName(c prometheus.Collector) string {
+	d := make(chan *prometheus.Desc, 1)
+	c.Describe(d)
+	desc := (<-d).String()
+	matches := descNameRegex.FindAllStringSubmatch(desc, -1)
+	if len(matches) == 1 && len(matches[0]) == 2 {
+		return matches[0][1]
+	}
+	return ""
+}

pkg/prom/registry_test.go

@@ -17,11 +17,14 @@ import (
 func TestRegistryEndpoints(t *testing.T) {
 	t.Parallel()
 
+	var f filter.Filter
+	f.Block("fastly_rt_service_info")
+
 	var (
 		version          = "dev"
 		namespace        = "fastly"
 		subsystem        = "rt"
-		metricNameFilter = filter.Filter{}
+		metricNameFilter = f
 		registry         = prom.NewRegistry(version, namespace, subsystem, metricNameFilter)
 	)
 
@@ -33,6 +36,9 @@ func TestRegistryEndpoints(t *testing.T) {
 		"service_id": "BBB", "service_name": "Service Two", "datacenter": "NYC",
 	}).Add(2)
 
+	registry.MetricsFor("AAA").ServiceInfo.WithLabelValues("AAA", "Service One", "1").Set(1)
+	registry.MetricsFor("BBB").ServiceInfo.WithLabelValues("BBB", "Service Two", "1").Set(1)
+
 	server := httptest.NewServer(registry)
 	defer server.Close()
 
@@ -123,7 +129,10 @@ func TestRegistryEndpoints(t *testing.T) {
 		want, dont := []string{
 			`fastly_rt_requests_total{datacenter="NYC",service_id="AAA",service_name="Service One"} 1`,
 			`fastly_rt_requests_total{datacenter="NYC",service_id="BBB",service_name="Service Two"} 2`,
-		}, []string{}
+		}, []string{
+			`fastly_rt_service_info{service_id="AAA",service_name="Service One",service_version="1"} 1`,
+			`fastly_rt_service_info{service_id="BBB",service_name="Service Two",service_version="1"} 1`,
+		}
 		checkMetrics(body, want, dont)
 	})