fix(backend): reduce allocations in fingerprint code

Author: prymitive

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## v0.115
+
+### Fixed
+
+- Improved memory usage.
+
 ## v0.114
 
 ### Fixed

cmd/karma/api_test.go

@@ -56,7 +56,7 @@ var groupTests = []groupTest{
 				Receiver: "by-name",
 			},
 		},
-		id: "099c5ca6d1c92f615b13056b935d0c8dee70f18c",
+		id: "990fb0cdc86aae89",
 		stateCount: map[string]int{
 			models.AlertStateActive:      1,
 			models.AlertStateSuppressed:  0,
@@ -92,7 +92,7 @@ var groupTests = []groupTest{
 				Receiver: "by-cluster-service",
 			},
 		},
-		id: "0b1963665aac588dc4b18e17c7a4f70466c622ea",
+		id: "6b15d34b0ed69d02",
 		stateCount: map[string]int{
 			models.AlertStateActive:      1,
 			models.AlertStateSuppressed:  0,
@@ -159,7 +159,7 @@ var groupTests = []groupTest{
 				Receiver: "by-cluster-service",
 			},
 		},
-		id: "2d3f39413b41c873cb72e0b8065aa7b8631e983e",
+		id: "f08998b6581752f4",
 		stateCount: map[string]int{
 			models.AlertStateActive:      3,
 			models.AlertStateSuppressed:  0,
@@ -228,7 +228,7 @@ var groupTests = []groupTest{
 				Receiver: "by-cluster-service",
 			},
 		},
-		id: "3c09c4156e6784dcf6d5b2e1629253798f82909b",
+		id: "97dba9e211f41cf6",
 		stateCount: map[string]int{
 			models.AlertStateActive:      0,
 			models.AlertStateSuppressed:  3,
@@ -396,7 +396,7 @@ var groupTests = []groupTest{
 				Receiver: "by-name",
 			},
 		},
-		id: "58c6a3467cebc53abe68ecbe8643ce478c5a1573",
+		id: "db6e3af075b36419",
 		stateCount: map[string]int{
 			models.AlertStateActive:      5,
 			models.AlertStateSuppressed:  3,
@@ -433,7 +433,7 @@ var groupTests = []groupTest{
 				Receiver: "by-cluster-service",
 			},
 		},
-		id: "8ca8151d9e30baba2334507dca53e16b7be93c5e",
+		id: "c8a1ec76d51e9d96",
 		stateCount: map[string]int{
 			models.AlertStateActive:      1,
 			models.AlertStateSuppressed:  0,
@@ -485,7 +485,7 @@ var groupTests = []groupTest{
 				Receiver: "by-cluster-service",
 			},
 		},
-		id: "98c1a53d0f71af9c734c9180697383f3b8aff80f",
+		id: "922d04650baba3fe",
 		stateCount: map[string]int{
 			models.AlertStateActive:      2,
 			models.AlertStateSuppressed:  0,
@@ -535,7 +535,7 @@ var groupTests = []groupTest{
 				Receiver: "by-name",
 			},
 		},
-		id: "bc4845fec77585cdfebe946234279d785ca93891",
+		id: "69b99170489a6c64",
 		stateCount: map[string]int{
 			models.AlertStateActive:      1,
 			models.AlertStateSuppressed:  1,
@@ -572,7 +572,7 @@ var groupTests = []groupTest{
 				Receiver: "by-name",
 			},
 		},
-		id: "bf78806d2a80b1c8150c1391669813722428e858",
+		id: "37f9b50559e97fd0",
 		stateCount: map[string]int{
 			models.AlertStateActive:      1,
 			models.AlertStateSuppressed:  0,
@@ -624,7 +624,7 @@ var groupTests = []groupTest{
 				Receiver: "by-cluster-service",
 			},
 		},
-		id: "ecefc3705b1ab4e4c3283c879540be348d2d9dce",
+		id: "ca10a29d2e729cff",
 		stateCount: map[string]int{
 			models.AlertStateActive:      1,
 			models.AlertStateSuppressed:  1,
@@ -905,15 +905,15 @@ var sortTests = []sortTest{
 		sortLabel:      "",
 		sortReverse:    false,
 		expectedLabel:  "cluster",
-		expectedValues: []string{"dev", "prod", "staging", "dev", "staging", "prod"},
+		expectedValues: []string{"staging", "dev", "staging", "dev", "prod", "prod"},
 	},
 	{
 		filter:         []string{"@receiver=by-cluster-service"},
 		sortOrder:      "disabled",
 		sortLabel:      "",
 		sortReverse:    true,
 		expectedLabel:  "cluster",
-		expectedValues: []string{"prod", "staging", "dev", "staging", "prod", "dev"},
+		expectedValues: []string{"prod", "prod", "dev", "staging", "dev", "staging"},
 	},
 	{
 		filter:         []string{"@receiver=by-cluster-service"},

go.mod

@@ -5,6 +5,7 @@ go 1.20
 require (
 	github.com/Masterminds/semver/v3 v3.2.1
 	github.com/beme/abide v0.0.0-20190723115211-635a09831760
+	github.com/cespare/xxhash/v2 v2.2.0
 	github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08
 	github.com/fvbommel/sortorder v1.1.0
 	github.com/go-chi/chi/v5 v5.0.10
@@ -35,7 +36,6 @@ require (
 require (
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/go-logr/logr v1.2.4 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect

internal/models/alert.go

@@ -2,10 +2,11 @@ package models
 
 import (
 	"fmt"
+	"strconv"
 	"strings"
 	"time"
 
-	"github.com/cnf/structhash"
+	"github.com/cespare/xxhash/v2"
 	"github.com/fvbommel/sortorder"
 
 	"github.com/prymitive/karma/internal/config"
@@ -130,23 +131,77 @@ type Alert struct {
 	// those are not exposed in JSON, Alertmanager specific value will be in kept
 	// in the Alertmanager slice
 	// skip those when generating alert fingerprint too
-	Fingerprint  string   `json:"-" hash:"-"`
-	GeneratorURL string   `json:"-" hash:"-"`
-	SilencedBy   []string `json:"-" hash:"-"`
-	InhibitedBy  []string `json:"-" hash:"-"`
+	Fingerprint  string   `json:"-"`
+	GeneratorURL string   `json:"-"`
+	SilencedBy   []string `json:"-"`
+	InhibitedBy  []string `json:"-"`
 	// karma fields
 	Alertmanager []AlertmanagerInstance `json:"alertmanager"`
 	Receiver     string                 `json:"receiver"`
 	// fingerprints are precomputed for speed
-	LabelsFP  string `json:"id" hash:"-"`
-	contentFP string `hash:"-"`
+	LabelsFP  string `json:"id"`
+	contentFP string
 }
 
+var seps = []byte{'\xff'}
+
 // UpdateFingerprints will generate a new set of fingerprints for this alert
-// it should be called after modifying any field that isn't tagged with hash:"-"
 func (a *Alert) UpdateFingerprints() {
-	a.LabelsFP = fmt.Sprintf("%x", structhash.Sha1(a.Labels.Map(), 1))
-	a.contentFP = fmt.Sprintf("%x", structhash.Sha1(a, 1))
+	h := xxhash.New()
+	for _, l := range a.Labels {
+		_, _ = h.WriteString(l.Name)
+		_, _ = h.Write(seps)
+		_, _ = h.WriteString(l.Value)
+		_, _ = h.Write(seps)
+	}
+	a.LabelsFP = fmt.Sprintf("%x", h.Sum64())
+
+	h.Reset()
+	for _, a := range a.Annotations {
+		_, _ = h.WriteString(a.Name)
+		_, _ = h.Write(seps)
+		_, _ = h.WriteString(a.Value)
+		_, _ = h.Write(seps)
+		_, _ = h.WriteString(strconv.FormatBool(a.IsAction))
+		_, _ = h.Write(seps)
+		_, _ = h.WriteString(strconv.FormatBool(a.IsLink))
+		_, _ = h.Write(seps)
+		_, _ = h.WriteString(strconv.FormatBool(a.Visible))
+		_, _ = h.Write(seps)
+
+	}
+	for _, l := range a.Labels {
+		_, _ = h.WriteString(l.Name)
+		_, _ = h.Write(seps)
+		_, _ = h.WriteString(l.Value)
+		_, _ = h.Write(seps)
+	}
+	_, _ = h.WriteString(a.StartsAt.Format(time.RFC3339))
+	_, _ = h.WriteString(a.State)
+	for _, am := range a.Alertmanager {
+		_, _ = h.WriteString(am.Fingerprint)
+		_, _ = h.Write(seps)
+		_, _ = h.WriteString(am.Name)
+		_, _ = h.Write(seps)
+		_, _ = h.WriteString(am.Cluster)
+		_, _ = h.Write(seps)
+		_, _ = h.WriteString(am.State)
+		_, _ = h.Write(seps)
+		_, _ = h.WriteString(am.StartsAt.Format(time.RFC3339))
+		_, _ = h.Write(seps)
+		_, _ = h.WriteString(am.Source)
+		_, _ = h.Write(seps)
+		for _, s := range am.SilencedBy {
+			_, _ = h.WriteString(s)
+			_, _ = h.Write(seps)
+		}
+		for _, s := range am.InhibitedBy {
+			_, _ = h.WriteString(s)
+			_, _ = h.Write(seps)
+		}
+	}
+	_, _ = h.WriteString(a.Receiver)
+	a.contentFP = fmt.Sprintf("%x", h.Sum64())
 }
 
 // LabelsFingerprint is a checksum computed only from labels which should be
@@ -156,7 +211,6 @@ func (a *Alert) LabelsFingerprint() string {
 }
 
 // ContentFingerprint is a checksum computed from entire alert object
-// except some blacklisted fields tagged with hash:"-"
 func (a *Alert) ContentFingerprint() string {
 	return a.contentFP
 }

internal/models/alertgroup.go

@@ -1,12 +1,11 @@
 package models
 
 import (
-	"crypto/sha1"
 	"fmt"
 	"io"
 	"time"
 
-	"github.com/cnf/structhash"
+	"github.com/cespare/xxhash/v2"
 )
 
 // AlertList is flat list of karmaAlert objects
@@ -48,19 +47,25 @@ type AlertGroup struct {
 // LabelsFingerprint is a checksum of this AlertGroup labels and the receiver
 // it should be unique for each AlertGroup
 func (ag AlertGroup) LabelsFingerprint() string {
-	agIDHasher := sha1.New()
-	_, _ = io.WriteString(agIDHasher, ag.Receiver)
-	_, _ = io.WriteString(agIDHasher, fmt.Sprintf("%x", structhash.Sha1(ag.Labels.Map(), 1)))
-	return fmt.Sprintf("%x", agIDHasher.Sum(nil))
+	h := xxhash.New()
+	_, _ = h.WriteString(ag.Receiver)
+	_, _ = h.Write(seps)
+	for _, l := range ag.Labels {
+		_, _ = h.WriteString(l.Name)
+		_, _ = h.Write(seps)
+		_, _ = h.WriteString(l.Value)
+		_, _ = h.Write(seps)
+	}
+	return fmt.Sprintf("%x", h.Sum64())
 }
 
 // ContentFingerprint is a checksum of all alerts in the group
 func (ag AlertGroup) ContentFingerprint() string {
-	h := sha1.New()
+	h := xxhash.New()
 	for _, alert := range ag.Alerts {
 		_, _ = io.WriteString(h, alert.ContentFingerprint())
 	}
-	return fmt.Sprintf("%x", h.Sum(nil))
+	return fmt.Sprintf("%x", h.Sum64())
 }
 
 func (ag AlertGroup) FindLatestStartsAt() time.Time {