WIP: add percentile debug and mae statistics

pkg/prediction/percentile/debug.go

@@ -0,0 +1,74 @@
+package percentile
+
+import (
+	"fmt"
+	"time"
+
+	"k8s.io/klog/v2"
+
+	"github.com/gocrane/crane/pkg/common"
+	"github.com/gocrane/crane/pkg/metricnaming"
+	"github.com/gocrane/crane/pkg/prediction"
+	"github.com/gocrane/crane/pkg/prediction/config"
+)
+
+func Debug(predictor prediction.Interface, namer metricnaming.MetricNamer, config *config.Config) ([]*common.TimeSeries, []*common.TimeSeries, error) {
+	cfg, err := makeInternalConfig(config.Percentile, config.InitMode)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	p := predictor.(*percentilePrediction)
+	historyTimeSeriesList, err := queryHistoryTimeSeries(p, namer, cfg)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	queryExpr := namer.BuildUniqueKey()
+	klog.V(4).Infof("process analyzing metric namer: %v, config: %+v", namer.BuildUniqueKey(), *cfg)
+
+	signals := map[string]*aggregateSignal{}
+	if cfg.aggregated {
+		signal := newAggregateSignal(cfg)
+		for _, ts := range historyTimeSeriesList {
+			for _, s := range ts.Samples {
+				t := time.Unix(s.Timestamp, 0)
+				signal.addSample(t, s.Value)
+			}
+		}
+		signals[keyAll] = signal
+	} else {
+		for _, ts := range historyTimeSeriesList {
+			if len(ts.Samples) < 1 {
+				continue
+			}
+			key := prediction.AggregateSignalKey(ts.Labels)
+			signal := newAggregateSignal(cfg)
+			for _, s := range ts.Samples {
+				t := time.Unix(s.Timestamp, 0)
+				signal.addSample(t, s.Value)
+			}
+			signal.labels = ts.Labels
+			signals[key] = signal
+		}
+	}
+
+	return historyTimeSeriesList, p.getPredictedValuesFromSignals(queryExpr, signals, cfg), nil
+
+}
+
+func queryHistoryTimeSeries(predictor *percentilePrediction, namer metricnaming.MetricNamer, config *internalConfig) ([]*common.TimeSeries, error) {
+	p := predictor.GetHistoryProvider()
+	if p == nil {
+		return nil, fmt.Errorf("history provider not provisioned")
+	}
+
+	end := time.Now().Truncate(time.Minute)
+	start := end.Add(-config.historyLength)
+	historyTimeSeries, err := predictor.GetHistoryProvider().QueryTimeSeries(namer, start, end, config.sampleInterval)
+	if err != nil {
+		klog.ErrorS(err, "Failed to query history time series.")
+		return nil, err
+	}
+	return historyTimeSeries, nil
+}

pkg/server/handler/prediction/prediction.go

@@ -3,7 +3,9 @@ package prediction
 import (
 	"context"
 	"fmt"
+	"math"
 	"net/http"
+	"time"
 
 	"github.com/gin-gonic/gin"
 	"github.com/go-echarts/go-echarts/v2/charts"
@@ -18,15 +20,21 @@ import (
 
 	craneclientset "github.com/gocrane/api/pkg/generated/clientset/versioned"
 	"github.com/gocrane/api/prediction/v1alpha1"
-
 	"github.com/gocrane/crane/pkg/controller/timeseriesprediction"
 	"github.com/gocrane/crane/pkg/prediction/dsp"
+	"github.com/gocrane/crane/pkg/prediction/percentile"
 	predictormgr "github.com/gocrane/crane/pkg/predictor"
 	"github.com/gocrane/crane/pkg/server/config"
 	"github.com/gocrane/crane/pkg/server/ginwrapper"
 	"github.com/gocrane/crane/pkg/utils/target"
 )
 
+const (
+	day       = time.Hour * 24
+	threeDays = time.Hour * 24 * 3
+	week      = time.Hour * 24 * 7
+)
+
 type DebugHandler struct {
 	craneClient      *craneclientset.Clientset
 	predictorManager predictormgr.Manager
@@ -88,15 +96,94 @@ func (dh *DebugHandler) Display(c *gin.Context) {
 				return
 			}
 
+			dspMAEDay := 0.0
+			daySampleLength := int(estimate.SampleRate * day.Seconds())
+			if len(estimate.Samples) >= daySampleLength {
+				dspMAEDay = mae(test.Samples[:daySampleLength-1], estimate.Samples[:daySampleLength-1])
+			}
+			dspMAEThreeDays := 0.0
+			threeDaySampleLength := int(estimate.SampleRate * threeDays.Seconds())
+			if len(estimate.Samples) >= threeDaySampleLength {
+				dspMAEThreeDays = mae(test.Samples[:threeDaySampleLength-1], estimate.Samples[:threeDaySampleLength-1])
+			}
+			dspMAEWeekly := 0.0
+			weeklyDaySampleLength := int(estimate.SampleRate * week.Seconds())
+			if len(estimate.Samples) >= weeklyDaySampleLength {
+				dspMAEWeekly = mae(test.Samples[:weeklyDaySampleLength-1], estimate.Samples[:weeklyDaySampleLength-1])
+			}
+			klog.Infof("dspMAE1d: %f,dspMAE3d: %f, dspMAE7d: %f", dspMAEDay, dspMAEThreeDays, dspMAEWeekly)
 			page := components.NewPage()
 			page.AddCharts(plot(history, "history", "green", charts.WithTitleOpts(opts.Title{Title: "history"})))
 			page.AddCharts(plots([]*dsp.Signal{test, estimate}, []string{"actual", "forecasted"},
 				charts.WithTitleOpts(opts.Title{Title: "actual/forecasted"})))
+
+			page.AddCharts(bar(dspMAEDay, dspMAEThreeDays, dspMAEWeekly))
 			err = page.Render(c.Writer)
 			if err != nil {
 				klog.ErrorS(err, "Failed to display debug time series")
 			}
 
+			return
+		} else if tsp.Spec.PredictionMetrics[0].Algorithm.AlgorithmType == v1alpha1.AlgorithmTypePercentile && tsp.Spec.PredictionMetrics[0].Algorithm.Percentile != nil {
+			mc, err := timeseriesprediction.NewMetricContext(dh.selectorFetcher, tsp, dh.predictorManager)
+			if err != nil {
+				ginwrapper.WriteResponse(c, err, nil)
+				return
+			}
+
+			internalConf := mc.ConvertApiMetric2InternalConfig(&tsp.Spec.PredictionMetrics[0])
+			internalConf.Percentile.HistoryLength = "7d"
+			namer := mc.GetMetricNamer(&tsp.Spec.PredictionMetrics[0])
+			predictor := dh.predictorManager.GetPredictor(v1alpha1.AlgorithmTypePercentile)
+			history, estimate, err := percentile.Debug(predictor, namer, internalConf)
+			if err != nil {
+				ginwrapper.WriteResponse(c, err, nil)
+				return
+			}
+			if len(history) <= 0 || len(estimate) <= 0 {
+				c.Writer.WriteHeader(http.StatusBadRequest)
+				return
+			}
+
+			samples := history[0].Samples
+			estimateSamples := estimate[0].Samples
+			historySignal := &dsp.Signal{SampleRate: 1 / 60.0}
+			estimateSignal := &dsp.Signal{SampleRate: 1 / 60.0}
+			for _, v := range samples {
+				historySignal.Samples = append(historySignal.Samples, v.Value)
+				estimateSignal.Samples = append(estimateSignal.Samples, estimateSamples[0].Value)
+			}
+
+			predictMAEDay := 0.0
+			daySampleLength := int(estimateSignal.SampleRate * day.Seconds())
+			if len(estimateSignal.Samples) >= daySampleLength {
+				m := maxValue(historySignal.Samples[:daySampleLength-1])
+				predictMAEDay = mae([]float64{m}, []float64{estimateSamples[0].Value})
+			}
+			predictMAEThreeDays := 0.0
+			threeDaySampleLength := int(estimateSignal.SampleRate * threeDays.Seconds())
+			if len(estimateSignal.Samples) >= threeDaySampleLength {
+				m := maxValue(historySignal.Samples[:threeDaySampleLength-1])
+				predictMAEThreeDays = mae([]float64{m}, []float64{estimateSamples[0].Value})
+			}
+			predictMAEWeekly := 0.0
+			weeklySampleLength := int(estimateSignal.SampleRate * week.Seconds())
+			if len(estimateSignal.Samples) >= weeklySampleLength {
+				m := maxValue(historySignal.Samples[:weeklySampleLength-1])
+				predictMAEWeekly = mae([]float64{m}, []float64{estimateSamples[0].Value})
+			}
+
+			page := components.NewPage()
+			page.AddCharts(plots([]*dsp.Signal{historySignal, estimateSignal}, []string{"actual", "recommended"},
+				charts.WithTitleOpts(opts.Title{Title: "actual/recommended"})))
+
+			page.AddCharts(bar(predictMAEDay, predictMAEThreeDays, predictMAEWeekly))
+			err = page.Render(c.Writer)
+			if err != nil {
+				klog.ErrorS(err, "Failed to display debug time series")
+			}
+
+			c.Writer.WriteHeader(http.StatusBadRequest)
 			return
 		}
 	}
@@ -105,6 +192,51 @@ func (dh *DebugHandler) Display(c *gin.Context) {
 	return
 }
 
+func maxValue(testData []float64) float64 {
+	m := 0.0
+	for i := 0; i < len(testData); i++ {
+		if testData[i] > m {
+			m = testData[i]
+		}
+	}
+	return m
+}
+
+func mae(testData, estimateData []float64) float64 {
+	mae := 0.0
+	for i := 0; i < len(estimateData); i++ {
+		mae += math.Abs(testData[i] - estimateData[i])
+	}
+	return mae / float64(len(estimateData))
+}
+
+func bar(dspMAE1d, dspMAE3d, dspMAE7d float64) *charts.Bar {
+	bar := charts.NewBar()
+	bar.SetGlobalOptions(
+		charts.WithInitializationOpts(opts.Initialization{Width: "3000px", Theme: types.ThemeRoma}),
+		charts.WithTitleOpts(opts.Title{
+			Title: "MAE(Mean Absolute Error)",
+		}), charts.WithTooltipOpts(opts.Tooltip{
+			Show:      true,
+			Trigger:   "axis",
+			TriggerOn: "mousemove",
+		}),
+	)
+	bar.SetXAxis([]string{"1d", "3d", "7d"}).
+		AddSeries("mae", []opts.BarData{
+			{
+				Value: dspMAE1d,
+			},
+			{
+				Value: dspMAE3d,
+			},
+			{
+				Value: dspMAE7d,
+			},
+		})
+	return bar
+}
+
 func plot(s *dsp.Signal, name string, color string, o ...charts.GlobalOpts) *charts.Line {
 	x := make([]string, 0)
 	y := make([]opts.LineData, 0)
@@ -155,7 +287,7 @@ func plots(signals []*dsp.Signal, names []string, o ...charts.GlobalOpts) *chart
 
 	line := charts.NewLine()
 	line.SetGlobalOptions(
-		charts.WithInitializationOpts(opts.Initialization{Width: "3000px", Theme: types.ThemeShine}),
+		charts.WithInitializationOpts(opts.Initialization{Width: "3000px", Theme: types.ThemeRoma}),
 		charts.WithLegendOpts(
 			opts.Legend{
 				Show: true,