Update Prometheus

Signed-off-by: 🌲 Harry 🌊 John 🏔 <johrry@amazon.com>

engine/existing_test.go

@@ -32,6 +32,21 @@ func TestRangeQuery(t *testing.T) {
 			Load: `load 30s
 			    bar 0 1 10 100 1000`,
 			Query: `sum_over_time(bar[30s])`,
+			Result: promql.Matrix{
+				promql.Series{
+					Floats: []promql.FPoint{{F: 0, T: 0}, {F: 10, T: 60000}, {F: 1000, T: 120000}},
+					Metric: labels.Labels{},
+				},
+			},
+			Start:    time.Unix(0, 0),
+			End:      time.Unix(120, 0),
+			Interval: 60 * time.Second,
+		},
+		{
+			Name: "sum_over_time with all values",
+			Load: `load 30s
+			    bar 0 1 10 100 1000`,
+			Query: `sum_over_time(bar[45s])`,
 			Result: promql.Matrix{
 				promql.Series{
 					Floats: []promql.FPoint{{F: 0, T: 0}, {F: 11, T: 60000}, {F: 1100, T: 120000}},
@@ -46,7 +61,7 @@ func TestRangeQuery(t *testing.T) {
 			Name: "sum_over_time with trailing values",
 			Load: `load 30s
 			    bar 0 1 10 100 1000 0 0 0 0`,
-			Query: `sum_over_time(bar[30s])`,
+			Query: `sum_over_time(bar[45s])`,
 			Result: promql.Matrix{
 				promql.Series{
 					Floats: []promql.FPoint{{F: 0, T: 0}, {F: 11, T: 60000}, {F: 1100, T: 120000}},
@@ -61,7 +76,7 @@ func TestRangeQuery(t *testing.T) {
 			Name: "sum_over_time with all values long",
 			Load: `load 30s
 			    bar 0 1 10 100 1000 10000 100000 1000000 10000000`,
-			Query: `sum_over_time(bar[30s])`,
+			Query: `sum_over_time(bar[45s])`,
 			Result: promql.Matrix{
 				promql.Series{
 					Floats: []promql.FPoint{{F: 0, T: 0}, {F: 11, T: 60000}, {F: 1100, T: 120000}, {F: 110000, T: 180000}, {F: 11000000, T: 240000}},
@@ -76,7 +91,7 @@ func TestRangeQuery(t *testing.T) {
 			Name: "sum_over_time with all values random",
 			Load: `load 30s
 			    bar 5 17 42 2 7 905 51`,
-			Query: `sum_over_time(bar[30s])`,
+			Query: `sum_over_time(bar[45s])`,
 			Result: promql.Matrix{
 				promql.Series{
 					Floats: []promql.FPoint{{F: 5, T: 0}, {F: 59, T: 60000}, {F: 9, T: 120000}, {F: 956, T: 180000}},
@@ -123,7 +138,6 @@ func TestRangeQuery(t *testing.T) {
 		Timeout: 1 * time.Hour,
 	}
 	ng := engine.New(engine.Opts{EngineOpts: opts})
-
 	for _, c := range cases {
 		c := c
 		t.Run(c.Name, func(t *testing.T) {

engine/sort.go

@@ -123,7 +123,12 @@ func (s sortByLabelFuncResult) comparer(samples *promql.Vector) func(i, j int) b
 				return s.sortOrder == sortOrderDesc
 			}
 		}
-		return valueCompare(s.sortOrder, (*samples)[i].F, (*samples)[j].F)
+		// If all labels provided as arguments were equal, sort by the full label set. This ensures a consistent ordering.
+		if lblsCmp := labels.Compare(iLb.Labels(), jLb.Labels()); lblsCmp < 0 {
+			return s.sortOrder == sortOrderAsc
+		} else {
+			return s.sortOrder == sortOrderDesc
+		}
 	}
 }
 

execution/aggregate/accumulator.go

@@ -287,9 +287,12 @@ func (c *countAcc) Reset(_ float64) {
 }
 
 type avgAcc struct {
-	avg      float64
-	count    int64
-	hasValue bool
+	kahanSum    float64
+	kahanC      float64
+	avg         float64
+	incremental bool
+	count       int64
+	hasValue    bool
 
 	histSum        *histogram.FloatHistogram
 	histScratch    *histogram.FloatHistogram
@@ -326,30 +329,55 @@ func (a *avgAcc) Add(v float64, h *histogram.FloatHistogram) error {
 	a.count++
 	if !a.hasValue {
 		a.hasValue = true
-		a.avg = v
+		a.kahanSum = v
 		return nil
 	}
 
 	a.hasValue = true
+
+	if !a.incremental {
+		newSum, newC := kahanSumInc(v, a.kahanSum, a.kahanC)
+
+		if !math.IsInf(newSum, 0) {
+			// The sum doesn't overflow, so we propagate it to the
+			// group struct and continue with the regular
+			// calculation of the mean value.
+			a.kahanSum, a.kahanC = newSum, newC
+			return nil
+		}
+
+		// If we are here, we know that the sum _would_ overflow. So
+		// instead of continue to sum up, we revert to incremental
+		// calculation of the mean value from here on.
+		a.incremental = true
+		a.avg = a.kahanSum / float64(a.count-1)
+		a.kahanC /= float64(a.count) - 1
+	}
+
 	if math.IsInf(a.avg, 0) {
 		if math.IsInf(v, 0) && (a.avg > 0) == (v > 0) {
-			// The `avg` and `v` values are `Inf` of the same sign.  They
-			// can't be subtracted, but the value of `avg` is correct
+			// The `floatMean` and `s.F` values are `Inf` of the same sign.  They
+			// can't be subtracted, but the value of `floatMean` is correct
 			// already.
 			return nil
 		}
 		if !math.IsInf(v, 0) && !math.IsNaN(v) {
-			// At this stage, the avg is an infinite. If the added
-			// value is neither an Inf or a Nan, we can keep that avg
+			// At this stage, the mean is an infinite. If the added
+			// value is neither an Inf or a Nan, we can keep that mean
 			// value.
 			// This is required because our calculation below removes
-			// the avg value, which would look like Inf += x - Inf and
+			// the mean value, which would look like Inf += x - Inf and
 			// end up as a NaN.
 			return nil
 		}
 	}
-
-	a.avg += v/float64(a.count) - a.avg/float64(a.count)
+	currentMean := a.avg + a.kahanC
+	a.avg, a.kahanC = kahanSumInc(
+		// Divide each side of the `-` by `group.groupCount` to avoid float64 overflows.
+		v/float64(a.count)-currentMean/float64(a.count),
+		a.avg,
+		a.kahanC,
+	)
 	return nil
 }
 
@@ -368,7 +396,10 @@ func (a *avgAcc) AddVector(vs []float64, hs []*histogram.FloatHistogram) error {
 }
 
 func (a *avgAcc) Value() (float64, *histogram.FloatHistogram) {
-	return a.avg, a.histSum
+	if a.incremental {
+		return a.avg + a.kahanC, a.histSum
+	}
+	return (a.kahanSum + a.kahanC) / float64(a.count), a.histSum
 }
 
 func (a *avgAcc) ValueType() ValueType {
@@ -562,3 +593,18 @@ func SumCompensated(s []float64) float64 {
 	}
 	return sum + c
 }
+
+func kahanSumInc(inc, sum, c float64) (newSum, newC float64) {
+	t := sum + inc
+	switch {
+	case math.IsInf(t, 0):
+		c = 0
+
+	// Using Neumaier improvement, swap if next term larger than sum.
+	case math.Abs(sum) >= math.Abs(inc):
+		c += (sum - t) + inc
+	default:
+		c += (inc - t) + sum
+	}
+	return t, c
+}

execution/execution.go

@@ -407,9 +407,9 @@ func getTimeRangesForVectorSelector(n *logicalplan.VectorSelector, opts *query.O
 		end = *n.Timestamp
 	}
 	if evalRange == 0 {
-		start -= opts.LookbackDelta.Milliseconds()
+		start -= opts.LookbackDelta.Milliseconds() - 1
 	} else {
-		start -= evalRange
+		start -= evalRange - 1
 	}
 	offset := n.OriginalOffset.Milliseconds()
 	return start - offset, end - offset

execution/scan/subquery.go

@@ -127,7 +127,7 @@ func (o *subqueryOperator) Next(ctx context.Context) ([]model.StepVector, error)
 
 	res := o.pool.GetVectorBatch()
 	for i := 0; o.currentStep <= o.maxt && i < o.stepsBatch; i++ {
-		mint := o.currentStep - o.subQuery.Range.Milliseconds() - o.subQuery.OriginalOffset.Milliseconds()
+		mint := o.currentStep - o.subQuery.Range.Milliseconds() - o.subQuery.OriginalOffset.Milliseconds() + 1
 		maxt := o.currentStep - o.subQuery.OriginalOffset.Milliseconds()
 		for _, b := range o.buffers {
 			b.Reset(mint, maxt+o.subQuery.Offset.Milliseconds())

go.mod

@@ -11,67 +11,80 @@ require (
 	github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb
 	github.com/go-kit/log v0.2.1
 	github.com/google/go-cmp v0.6.0
-	github.com/prometheus/client_golang v1.19.1
-	github.com/prometheus/common v0.54.0
-	github.com/prometheus/prometheus v0.53.1-0.20240710110745-b1c106a9ce09
+	github.com/prometheus/client_golang v1.20.5
+	github.com/prometheus/common v0.60.1
+	github.com/prometheus/prometheus v0.300.0-beta.0.0.20241007135006-65f610353919
 	github.com/stretchr/testify v1.9.0
 	github.com/zhangyunhao116/umap v0.0.0-20221211160557-cb7705fafa39
 	go.uber.org/goleak v1.3.0
 	golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8
-	golang.org/x/tools v0.22.0
+	golang.org/x/tools v0.26.0
 	gonum.org/v1/gonum v0.15.0
 )
 
 require (
-	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.12.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 // indirect
-	github.com/Azure/azure-sdk-for-go/sdk/internal v1.9.0 // indirect
+	cloud.google.com/go/auth v0.10.0 // indirect
+	cloud.google.com/go/auth/oauth2adapt v0.2.5 // indirect
+	cloud.google.com/go/compute/metadata v0.5.2 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
-	github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 // indirect
-	github.com/aws/aws-sdk-go v1.54.7 // indirect
+	github.com/alecthomas/units v0.0.0-20240626203959-61d1e3462e30 // indirect
+	github.com/aws/aws-sdk-go v1.55.5 // indirect
 	github.com/bboreham/go-loser v0.0.0-20230920113527-fcc2c21820a3 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/dennwc/varint v1.0.0 // indirect
-	github.com/edsrzf/mmap-go v1.1.0 // indirect
+	github.com/edsrzf/mmap-go v1.2.0 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/go-logfmt/logfmt v0.6.0 // indirect
 	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/snappy v0.0.4 // indirect
+	github.com/google/s2a-go v0.1.8 // indirect
 	github.com/google/uuid v1.6.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect
 	github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/jpillora/backoff v1.0.0 // indirect
-	github.com/klauspost/compress v1.17.9 // indirect
+	github.com/klauspost/compress v1.17.11 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
 	github.com/oklog/ulid v1.3.1 // indirect
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/client_model v0.6.1 // indirect
 	github.com/prometheus/common/sigv4 v0.1.0 // indirect
 	github.com/prometheus/procfs v0.15.1 // indirect
-	go.opentelemetry.io/otel v1.27.0 // indirect
-	go.opentelemetry.io/otel/metric v1.27.0 // indirect
-	go.opentelemetry.io/otel/trace v1.27.0 // indirect
+	go.opencensus.io v0.24.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.56.0 // indirect
+	go.opentelemetry.io/otel v1.31.0 // indirect
+	go.opentelemetry.io/otel/metric v1.31.0 // indirect
+	go.opentelemetry.io/otel/trace v1.31.0 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
-	golang.org/x/crypto v0.24.0 // indirect
-	golang.org/x/mod v0.18.0 // indirect
-	golang.org/x/net v0.26.0 // indirect
-	golang.org/x/oauth2 v0.21.0 // indirect
-	golang.org/x/sync v0.7.0 // indirect
-	golang.org/x/sys v0.21.0 // indirect
-	golang.org/x/text v0.16.0 // indirect
-	golang.org/x/time v0.5.0 // indirect
-	google.golang.org/protobuf v1.34.2 // indirect
+	golang.org/x/crypto v0.28.0 // indirect
+	golang.org/x/mod v0.21.0 // indirect
+	golang.org/x/net v0.30.0 // indirect
+	golang.org/x/oauth2 v0.23.0 // indirect
+	golang.org/x/sync v0.8.0 // indirect
+	golang.org/x/sys v0.26.0 // indirect
+	golang.org/x/text v0.19.0 // indirect
+	golang.org/x/time v0.7.0 // indirect
+	google.golang.org/api v0.204.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20241021214115-324edc3d5d38 // indirect
+	google.golang.org/grpc v1.67.1 // indirect
+	google.golang.org/protobuf v1.35.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/apimachinery v0.30.2 // indirect
-	k8s.io/client-go v0.30.2 // indirect
+	k8s.io/apimachinery v0.31.1 // indirect
+	k8s.io/client-go v0.31.1 // indirect
 	k8s.io/klog/v2 v2.130.1 // indirect
-	k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 // indirect
+	k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect
 )
 
 exclude (