Skip to content

Commit

Permalink
Merge pull request #14 from hashicorp/metrics
Browse files Browse the repository at this point in the history
Add metrics package with go-metrics shim
  • Loading branch information
banks authored Dec 13, 2022
2 parents 2d1be46 + 70da782 commit 3cecb00
Show file tree
Hide file tree
Showing 15 changed files with 492 additions and 157 deletions.
6 changes: 5 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ go 1.18

require (
github.com/HdrHistogram/hdrhistogram-go v1.1.0
github.com/armon/go-metrics v0.3.10
github.com/benbjohnson/immutable v0.4.0
github.com/benmathews/bench v0.0.0-20210120214102-f7c75b9ef6e7
github.com/benmathews/hdrhistogram-writer v0.0.0-20210120211942-3cb1c7c33f95
Expand All @@ -13,13 +14,13 @@ require (
github.com/hashicorp/raft v1.3.10
github.com/hashicorp/raft-boltdb v0.0.0-20220329195025-15018e9b97e0
github.com/hashicorp/raft-boltdb/v2 v2.2.2
github.com/ryboe/q v1.0.18
github.com/segmentio/fasthash v1.0.3
github.com/stretchr/testify v1.8.0
go.etcd.io/bbolt v1.3.6
)

require (
github.com/armon/go-metrics v0.3.10 // indirect
github.com/boltdb/bolt v1.3.1 // indirect
github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect
github.com/coreos/pkg v0.0.0-20220810130054-c7d1c02cb6cf // indirect
Expand All @@ -28,9 +29,12 @@ require (
github.com/hashicorp/go-immutable-radix v1.3.0 // indirect
github.com/hashicorp/go-msgpack v1.1.5 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/mattn/go-colorable v0.1.6 // indirect
github.com/mattn/go-isatty v0.0.12 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rogpeppe/go-internal v1.9.0 // indirect
golang.org/x/exp v0.0.0-20220827204233-334a2380cb91 // indirect
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect
golang.org/x/time v0.1.0 // indirect
Expand Down
7 changes: 7 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,8 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
Expand Down Expand Up @@ -158,6 +160,7 @@ github.com/nsqio/go-nsq v1.0.8/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQT
github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/pierrec/lz4 v2.4.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
Expand All @@ -178,6 +181,10 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT
github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
github.com/ryboe/q v1.0.18 h1:uTonPt1eZjy7GSpB0XpYpsCvX+Yf9f+M4CUKuH2r+vg=
github.com/ryboe/q v1.0.18/go.mod h1:elqvVf/GBuZHvZ9gvHv4MKM6NZAMz2rFajnTgQZ46wU=
github.com/segmentio/fasthash v1.0.3 h1:EI9+KE1EwvMLBWwjpRDc+fEM+prwxDYbslddQGtrmhM=
github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
Expand Down
161 changes: 67 additions & 94 deletions metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,102 +4,75 @@
package wal

import (
"sync/atomic"
"github.com/hashicorp/raft-wal/metrics"
)

var (
// All metrics are held in an array of uint64s accessed atomically. The names
// are defined in the list below. The order doesn't really matter for
// backwards compatibility since it is not persisted state although it might
// impact the order we print things. Why not a struct? This makes it easier to
// iterate them without reflection and stops us hard coding the same list in
// several places.
metrics = []string{
// COUNTERS

// LogEntryBytesWritten counts the bytes of log entry after encoding with
// Codec. Actual bytes written to disk might be slightly higher as it includes
// headers and index entries.
"log_entry_bytes_written",

// log_entries_written counts the number of entries written.
"log_entries_written",

// log_appends counts the number of calls to StoreLog(s) i.e. number of
// batches of entries appended.
"log_appends",

// log_entry_bytes_read counts the bytes of log entry read from segments before
// decoding. actual bytes read from disk might be higher as it includes
// headers and index entries and possible secondary reads for large entries
// that don't fit in buffers.
"log_entry_bytes_read",

// log_entries_read counts the number of calls to get_log.
"log_entries_read",

// segment_rotations counts how many times we move to a new segment file.
"segment_rotations",

// head_truncations counts how many log entries have been truncated from the
// head - i.e. the oldest entries. by graphing the rate of change over time
// you can see individual truncate calls as spikes.
"head_truncations",

// tail_truncations counts how many log entries have been truncated from the
// head - i.e. the newest entries. by graphing the rate of change over time
// you can see individual truncate calls as spikes.
"tail_truncations",

"stable_gets",
"stable_sets",

// gauges

// last_segment_age_seconds is a gauge that is set each time we rotate a segment
// and describes the number of seconds between when that segment file was
// first created and when it was sealed. this gives a rough estimate how
// quickly writes are filling the disk.
"last_segment_age_seconds",
// MetricDefinitions describe the metrics emitted by this library via the
// provided metrics.Collector implementation. It's public so that these can be
// registered during init with metrics clients that support pre-defining
// metrics.
MetricDefinitions = metrics.Definitions{
Counters: []metrics.Descriptor{
{
Name: "log_entry_bytes_written",
Desc: "log_entry_bytes_written counts the bytes of log entry after encoding" +
" with Codec. Actual bytes written to disk might be slightly higher as it" +
" includes headers and index entries.",
},
{
Name: "log_entries_written",
Desc: "log_entries_written counts the number of entries written.",
},
{
Name: "log_appends",
Desc: "log_appends counts the number of calls to StoreLog(s) i.e." +
" number of batches of entries appended.",
},
{
Name: "log_entry_bytes_read",
Desc: "log_entry_bytes_read counts the bytes of log entry read from" +
" segments before decoding. actual bytes read from disk might be higher" +
" as it includes headers and index entries and possible secondary reads" +
" for large entries that don't fit in buffers.",
},
{
Name: "log_entries_read",
Desc: "log_entries_read counts the number of calls to get_log.",
},
{
Name: "segment_rotations",
Desc: "segment_rotations counts how many times we move to a new segment file.",
},
{
Name: "head_truncations",
Desc: "head_truncations counts how many log entries have been truncated" +
" from the head - i.e. the oldest entries. by graphing the rate of" +
" change over time you can see individual truncate calls as spikes.",
},
{
Name: "tail_truncations",
Desc: "tail_truncations counts how many log entries have been truncated" +
" from the head - i.e. the newest entries. by graphing the rate of" +
" change over time you can see individual truncate calls as spikes.",
},
{
Name: "stable_gets",
Desc: "stable_gets counts how many calls to StableStore.Get or GetUint64.",
},
{
Name: "stable_sets",
Desc: "stable_sets counts how many calls to StableStore.Set or SetUint64.",
},
},
Gauges: []metrics.Descriptor{
{
Name: "last_segment_age_seconds",
Desc: "last_segment_age_seconds is a gauge that is set each time we" +
" rotate a segment and describes the number of seconds between when" +
" that segment file was first created and when it was sealed. this" +
" gives a rough estimate how quickly writes are filling the disk.",
},
},
}

metricIDs map[string]int
numMetrics int
)

func init() {
numMetrics = len(metrics)
metricIDs = make(map[string]int, numMetrics)
for i, n := range metrics {
metricIDs[n] = i
}
}

func (w *WAL) incr(name string, delta uint64) uint64 {
id, ok := metricIDs[name]
if !ok {
panic("invalid metric name: " + name)
}
return atomic.AddUint64(&w.metrics[id], delta)
}

func (w *WAL) setGauge(name string, val uint64) {
id, ok := metricIDs[name]
if !ok {
panic("invalid metric name: " + name)
}
atomic.StoreUint64(&w.metrics[id], val)
}

// Metrics returns a summary of the performance counters for the WAL since
// startup. This is intentionally agnostic to any metrics collector library.
// Package users should be able to poll this and report counter, gauge or timing
// information in their native format from this.
func (w *WAL) Metrics() map[string]uint64 {
// Copy the fields out with atomic reads
m := make(map[string]uint64)
for i, n := range metrics {
m[n] = atomic.LoadUint64(&w.metrics[i])
}
return m
}
86 changes: 86 additions & 0 deletions metrics/atomic_collector.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
package metrics

import "sync/atomic"

var (
_ Collector = &AtomicCollector{}
)

// AtomicCollector is a simple Collector that atomically stores
// counters and gauges in memory.
type AtomicCollector struct {
counters []uint64
gauges []uint64

counterIndex, gaugeIndex map[string]int
}

// NewAtomicCollector creates a collector for the given set of Definitions.
func NewAtomicCollector(defs Definitions) *AtomicCollector {
c := &AtomicCollector{
counters: make([]uint64, len(defs.Counters)),
gauges: make([]uint64, len(defs.Gauges)),
counterIndex: make(map[string]int),
gaugeIndex: make(map[string]int),
}
for i, d := range defs.Counters {
if _, ok := c.counterIndex[d.Name]; ok {
panic("duplicate metrics named " + d.Name)
}
c.counterIndex[d.Name] = i
}
for i, d := range defs.Gauges {
if _, ok := c.counterIndex[d.Name]; ok {
panic("duplicate metrics named " + d.Name)
}
if _, ok := c.gaugeIndex[d.Name]; ok {
panic("duplicate metrics named " + d.Name)
}
c.gaugeIndex[d.Name] = i
}
return c
}

// IncrementCounter record val occurrences of the named event. Names will
// follow prometheus conventions with lower_case_and_underscores. We don't
// need any additional labels currently.
func (c *AtomicCollector) IncrementCounter(name string, delta uint64) {
id, ok := c.counterIndex[name]
if !ok {
panic("invalid metric name: " + name)
}
atomic.AddUint64(&c.counters[id], delta)
}

// SetGauge sets the value of the named gauge overriding any previous value.
func (c *AtomicCollector) SetGauge(name string, val uint64) {
id, ok := c.gaugeIndex[name]
if !ok {
panic("invalid metric name: " + name)
}
atomic.StoreUint64(&c.gauges[id], val)
}

// Summary returns a summary of the metrics since startup. Each value is
// atomically loaded but the set is not atomic overall and may represent an
// inconsistent snapshot e.g. with some metrics reflecting the most recent
// operation while others don't.
func (c *AtomicCollector) Summary() Summary {
s := Summary{
Counters: make(map[string]uint64, len(c.counters)),
Gauges: make(map[string]uint64, len(c.gauges)),
}
for name, id := range c.counterIndex {
s.Counters[name] = atomic.LoadUint64(&c.counters[id])
}
for name, id := range c.gaugeIndex {
s.Gauges[name] = atomic.LoadUint64(&c.gauges[id])
}
return s
}

// Summary is a copy of the values recorded so far for each metric.
type Summary struct {
Counters map[string]uint64
Gauges map[string]uint64
}
58 changes: 58 additions & 0 deletions metrics/atomic_collector_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
package metrics

import (
"sync"
"testing"

"github.com/stretchr/testify/require"
)

func TestAtomicCollector(t *testing.T) {
defs := Definitions{
Counters: []Descriptor{
{
Name: "c1",
Desc: "counter one.",
},
{
Name: "c2",
Desc: "counter two.",
},
},
Gauges: []Descriptor{
{
Name: "g1",
Desc: "gauge one.",
},
{
Name: "g2",
Desc: "gauge two.",
},
},
}

c := NewAtomicCollector(defs)

var wg sync.WaitGroup

for i := 0; i < 10; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for j := 0; j < 10; j++ {
c.IncrementCounter("c1", 1)
c.IncrementCounter("c2", 2)
c.SetGauge("g1", uint64(j))
c.SetGauge("g2", uint64(j*2))
}
}()
}

wg.Wait()

s := c.Summary()
require.Equal(t, 100, int(s.Counters["c1"]))
require.Equal(t, 200, int(s.Counters["c2"]))
require.Equal(t, 9, int(s.Gauges["g1"]))
require.Equal(t, 18, int(s.Gauges["g2"]))
}
Loading

0 comments on commit 3cecb00

Please sign in to comment.