diff --git a/README.md b/README.md index a3f5d20665..cf0ad9cb4a 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,28 @@ Each of these failures contribute to the number of retests that occur in CI and ![sig-network-retests](https://kubevirt.io/ci-health/output/kubevirt/kubevirt/sig-network-retests.svg) ![sig-operator-retests](https://kubevirt.io/ci-health/output/kubevirt/kubevirt/sig-operator-retests.svg) +Top failed lanes: + +![failedjob1](https://kubevirt.io/ci-health/output/kubevirt/kubevirt/failedjob1.svg) + +![failedjob2](https://kubevirt.io/ci-health/output/kubevirt/kubevirt/failedjob2.svg) + +![failedjob3](https://kubevirt.io/ci-health/output/kubevirt/kubevirt/failedjob3.svg) + +![failedjob4](https://kubevirt.io/ci-health/output/kubevirt/kubevirt/failedjob4.svg) + +![failedjob5](https://kubevirt.io/ci-health/output/kubevirt/kubevirt/failedjob5.svg) + +![failedjob6](https://kubevirt.io/ci-health/output/kubevirt/kubevirt/failedjob6.svg) + +![failedjob7](https://kubevirt.io/ci-health/output/kubevirt/kubevirt/failedjob7.svg) + +![failedjob8](https://kubevirt.io/ci-health/output/kubevirt/kubevirt/failedjob8.svg) + +![failedjob9](https://kubevirt.io/ci-health/output/kubevirt/kubevirt/failedjob9.svg) + +![failedjob10](https://kubevirt.io/ci-health/output/kubevirt/kubevirt/failedjob10.svg) + ## Historical data evolution These plots will be updated every week. diff --git a/pkg/output/output.go b/pkg/output/output.go index b3021b406e..1926ca1b4c 100644 --- a/pkg/output/output.go +++ b/pkg/output/output.go @@ -6,6 +6,7 @@ import ( "io/ioutil" "os" "path/filepath" + "strconv" "github.com/narqo/go-badge" log "github.com/sirupsen/logrus" @@ -158,6 +159,11 @@ func (b *Handler) writeBadges(results *types.Results) error { b.options.SIGRetestsLevels, ) + err = b.writeJobFailureBadges( + results.Data[constants.SIGRetests], + b.options.SIGRetestsLevels, + ) + return err } @@ -210,6 +216,32 @@ func (b *Handler) writeSIGRetestBadge(name, filePath string, data types.RunningA return badge.Render(name, badgeString, color, f) } +func (b *Handler) writeJobFailureBadges(data types.RunningAverageDataItem, levels *Levels) error { + failedJobLeaders := data.FailedJobLeaderBoard + basePath, err := b.initializeSourcePath() + if err != nil { + return err + } + for i, job := range failedJobLeaders { + if i < 10 { + filePath := filepath.Join(basePath, fmt.Sprintf("failedjob%s.svg", strconv.Itoa(i+1))) + color := BadgeColor(float64(job.FailureCount), levels) + f, err := os.Create(filePath) + if err != nil { + return err + } + defer f.Close() + + badgeString := fmt.Sprintf("%.0f", float64(job.FailureCount)) + + err = badge.Render(job.JobName, badgeString, color, f) + } else { + break + } + } + return err +} + func (b *Handler) initializeSourcePath() (string, error) { basePath := filepath.Join(b.options.Path, b.options.Source) err := os.MkdirAll(basePath, 0755) diff --git a/pkg/sigretests/main.go b/pkg/sigretests/main.go index ab19bb4faf..4cce97266e 100644 --- a/pkg/sigretests/main.go +++ b/pkg/sigretests/main.go @@ -19,10 +19,11 @@ type failedJob struct { } type SigRetests struct { - SigCompute int - SigNetwork int - SigStorage int - SigOperator int + SigCompute int + SigNetwork int + SigStorage int + SigOperator int + FailedJobNames []string } var redJobs []failedJob @@ -134,15 +135,19 @@ func FilterJobsPerSigs(jobs []failedJob) (prSigRetests SigRetests) { switch { case strings.Contains(job.jobName, "sig-compute") || strings.Contains(job.jobName, "vgpu"): prSigRetests.SigCompute += 1 + prSigRetests.FailedJobNames = append(prSigRetests.FailedJobNames, job.jobName) case strings.Contains(job.jobName, "sig-network") || strings.Contains(job.jobName, "sriov"): prSigRetests.SigNetwork += 1 + prSigRetests.FailedJobNames = append(prSigRetests.FailedJobNames, job.jobName) case strings.Contains(job.jobName, "sig-storage"): prSigRetests.SigStorage += 1 + prSigRetests.FailedJobNames = append(prSigRetests.FailedJobNames, job.jobName) case strings.Contains(job.jobName, "sig-operator"): prSigRetests.SigOperator += 1 + prSigRetests.FailedJobNames = append(prSigRetests.FailedJobNames, job.jobName) } } return prSigRetests diff --git a/pkg/stats/stats.go b/pkg/stats/stats.go index 47f5aadf68..61b35048d5 100644 --- a/pkg/stats/stats.go +++ b/pkg/stats/stats.go @@ -3,6 +3,7 @@ package stats import ( "fmt" "math" + "slices" "strconv" "time" @@ -254,6 +255,7 @@ func (h *Handler) mergedPRsProcessor(results *types.Results) (*types.Results, er func (h *Handler) sigRetestsProcessor(results *types.Results) (*types.Results, error) { currentTime, err := time.Parse(constants.DateFormat, results.EndDate) + var failedJobNames []string if err != nil { return results, err } @@ -278,10 +280,12 @@ func (h *Handler) sigRetestsProcessor(results *types.Results) (*types.Results, e dataItem.SIGOperatorRetest = dataItem.SIGOperatorRetest + float64(failuresPerSIG.SigOperator) dataItem.DataPoints = append(dataItem.DataPoints, types.DataPoint{ - Value: float64(failuresPerSIG.SigCompute + failuresPerSIG.SigOperator + failuresPerSIG.SigStorage + failuresPerSIG.SigNetwork), + Value: float64(len(failuresPerSIG.FailedJobNames)), PRs: []types.PR{mergedPR}, }) + failedJobNames = slices.Concat(failedJobNames, failuresPerSIG.FailedJobNames) } + dataItem.FailedJobLeaderBoard = types.SortByMostFailed(countFailedJobs(failedJobNames)) results.Data[constants.SIGRetests] = dataItem @@ -316,6 +320,14 @@ func Std(xs []float64) float64 { return round(result) } +func countFailedJobs(jobNames []string) map[string]int { + countFailedJobs := make(map[string]int) + for _, name := range jobNames { + countFailedJobs[name]++ + } + return countFailedJobs +} + func round(value float64) float64 { return math.Round(value*100) / 100 } diff --git a/pkg/types/main.go b/pkg/types/main.go index 63da327d3a..27f8c3dac6 100644 --- a/pkg/types/main.go +++ b/pkg/types/main.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "image/color" + "sort" "time" "github.com/kubevirt/ci-health/pkg/constants" @@ -210,18 +211,41 @@ type DataPoint struct { PRs []PR `json:",omitempty"` } +type FailedJob struct { + JobName string + FailureCount int +} + +type FailedJobs []FailedJob + +func (f FailedJobs) Len() int { return len(f) } +func (f FailedJobs) Less(i, j int) bool { return f[i].FailureCount < f[j].FailureCount } +func (f FailedJobs) Swap(i, j int) { f[i], f[j] = f[j], f[i] } + +func SortByMostFailed(countFailedJobs map[string]int) FailedJobs { + fl := make(FailedJobs, len(countFailedJobs)) + i := 0 + for j, f := range countFailedJobs { + fl[i] = FailedJob{j, f} + i++ + } + sort.Sort(sort.Reverse(fl)) + return fl +} + // RunningAverageDataItem contains data information in the form of a running average. // It contains the actual average value and the data points used to obtain it. type RunningAverageDataItem struct { - Avg float64 - Std float64 - Number float64 - NoRetest float64 - SIGComputeRetest float64 - SIGStorageRetest float64 - SIGNetworkRetest float64 - SIGOperatorRetest float64 - DataPoints []DataPoint + Avg float64 + Std float64 + Number float64 + NoRetest float64 + SIGComputeRetest float64 + SIGStorageRetest float64 + SIGNetworkRetest float64 + SIGOperatorRetest float64 + FailedJobLeaderBoard FailedJobs + DataPoints []DataPoint } func (d *RunningAverageDataItem) String() string {