release-test: add export to JUnit format

...so that a github action can pick it up in the CI, once we add it.
we might want to avoid the conversion and directly store the test
results in a junit compatible format, but I did not want to commit to
refactoring the code until it was clear that this worked as intended

Signed-off-by: Emanuele Di Pascale <emanuele@githedgehog.com>

cmd/hhfab/main.go

@@ -59,6 +59,7 @@ const (
 	FlagNameCollectShowTech       = "collect-show-tech"
 	FlagRegExes                   = "regexes"
 	FlagInvertRegex               = "invert-regex"
+	FlagResultsFile               = "results-file"
 )
 
 func main() {
@@ -954,12 +955,17 @@ func Run(ctx context.Context) error {
 								Name:  FlagInvertRegex,
 								Usage: "invert regex match",
 							},
+							&cli.StringFlag{
+								Name:  FlagResultsFile,
+								Usage: "path to a file to export test results to in JUnit XML format",
+							},
 						),
 						Before: before(false),
 						Action: func(c *cli.Context) error {
 							opts := hhfab.ReleaseTestOpts{
 								Regexes:     c.StringSlice(FlagRegExes),
 								InvertRegex: c.Bool(FlagInvertRegex),
+								ResultsFile: c.String(FlagResultsFile),
 							}
 							if err := hhfab.DoVLABReleaseTest(ctx, workDir, cacheDir, opts); err != nil {
 								return fmt.Errorf("release-test: %w", err)

pkg/hhfab/release.go

@@ -5,9 +5,11 @@ package hhfab
 
 import (
 	"context"
+	"encoding/xml"
 	"fmt"
 	"log"
 	"log/slog"
+	"os"
 	"os/exec"
 	"regexp"
 	"strings"
@@ -23,6 +25,9 @@ import (
 )
 
 var errTestRun = errors.New("test run error")
+var errNoExternals = errors.New("no external peers found")
+var errNoMclags = errors.New("no MCLAG connections found")
+var errNoEslags = errors.New("no ESLAG connections found")
 
 type VPCPeeringTestCtx struct {
 	workDir          string
@@ -311,7 +316,7 @@ func (testCtx *VPCPeeringTestCtx) vpcPeeringsOnlyExternalsTest(ctx context.Conte
 	if len(externalPeerings) == 0 {
 		slog.Info("No external peerings found, skipping test")
 
-		return true, nil
+		return true, errNoExternals
 	}
 	if err := DoSetupPeerings(ctx, testCtx.kube, vpcPeerings, externalPeerings, true); err != nil {
 		return false, errors.Wrap(err, "DoSetupPeerings")
@@ -371,7 +376,7 @@ func (testCtx *VPCPeeringTestCtx) mclagTest(ctx context.Context) (bool, error) {
 	if len(conns.Items) == 0 {
 		slog.Info("No MCLAG connections found, skipping test")
 
-		return true, nil
+		return true, errNoMclags
 	}
 	for _, conn := range conns.Items {
 		slog.Debug("Testing MCLAG connection", "connection", conn.Name)
@@ -428,7 +433,7 @@ func (testCtx *VPCPeeringTestCtx) eslagTest(ctx context.Context) (bool, error) {
 	if len(conns.Items) == 0 {
 		slog.Info("No ESLAG connections found, skipping test")
 
-		return true, nil
+		return true, errNoEslags
 	}
 	for _, conn := range conns.Items {
 		slog.Debug("Testing ESLAG connection", "connection", conn.Name)
@@ -941,6 +946,7 @@ type TestCase struct {
 	F        func(context.Context) (bool, error)
 	Err      error
 	Skipped  bool
+	SkipMsg  string
 	Duration time.Duration
 }
 
@@ -955,7 +961,7 @@ func printSuiteResults(ts *TestSuiteResult) {
 	slog.Info(fmt.Sprintf("Results for %s:", ts.Name))
 	for _, test := range ts.TestCases {
 		if test.Skipped {
-			slog.Info(fmt.Sprintf("SKIP %s", test.Name))
+			slog.Info(fmt.Sprintf("SKIP %s", test.Name), "skip-message", test.SkipMsg)
 		} else if test.Err != nil {
 			slog.Warn(fmt.Sprintf("FAIL %s", test.Name), "error", test.Err)
 			numFailed++
@@ -973,6 +979,7 @@ func doRunTests(ctx context.Context, testCtx *VPCPeeringTestCtx, ts *TestSuiteRe
 	for i, test := range ts.TestCases {
 		if test.Skipped {
 			slog.Info(fmt.Sprintf("Skipping test %s due to regex selection", test.Name))
+			ts.TestCases[i].SkipMsg = "Due to regex selection"
 
 			continue
 		}
@@ -983,9 +990,11 @@ func doRunTests(ctx context.Context, testCtx *VPCPeeringTestCtx, ts *TestSuiteRe
 			continue
 		}
 		skip, err := test.F(ctx)
-		ts.TestCases[i].Err = err
 		if skip {
 			ts.TestCases[i].Skipped = true
+			ts.TestCases[i].SkipMsg = err.Error()
+		} else {
+			ts.TestCases[i].Err = err
 		}
 	}
 
@@ -1153,5 +1162,107 @@ func RunReleaseTestSuites(ctx context.Context, workDir, cacheDir string, rtOtps
 	printSuiteResults(multiVpcResults)
 	printSuiteResults(basicResults)
 
+	if rtOtps.ResultsFile != "" {
+		results := []TestSuiteResult{*singleVpcResults, *multiVpcResults, *basicResults}
+		if err := exportResultsToJUnitXML(rtOtps.ResultsFile, results); err != nil {
+			return errors.Wrap(err, "exportResultsToJUnitXML")
+		}
+	}
+
+	return nil
+}
+
+// Reporting to JUnit format
+
+type JUnitTestSuite struct {
+	XMLName   xml.Name        `xml:"testsuite"`
+	Name      string          `xml:"name,attr"`
+	Tests     int             `xml:"tests,attr"`
+	Failures  int             `xml:"failures,attr"`
+	Time      string          `xml:"time,attr"`
+	TestCases []JUnitTestCase `xml:"testcase"`
+}
+
+type JUnitTestCase struct {
+	XMLName   xml.Name `xml:"testcase"`
+	ClassName string   `xml:"classname,attr"`
+	Name      string   `xml:"name,attr"`
+	Time      string   `xml:"time,attr"`
+	Failure   *Failure `xml:"failure,omitempty"`
+	Skipped   *Skipped `xml:"skipped,omitempty"`
+}
+
+type Failure struct {
+	XMLName xml.Name `xml:"failure"`
+	Message string   `xml:"message,attr"`
+	Type    string   `xml:"type,attr"`
+}
+
+type Skipped struct {
+	XMLName xml.Name `xml:"skipped"`
+	Message string   `xml:"message,attr,omitempty"`
+}
+
+func exportResultsToJUnitXML(filename string, results []TestSuiteResult) error {
+	testSuites := []JUnitTestSuite{}
+	for _, result := range results {
+		testSuite := JUnitTestSuite{
+			Name:      result.Name,
+			Tests:     len(result.TestCases),
+			Failures:  countFailures(result.TestCases),
+			Time:      formatDuration(result.Duration),
+			TestCases: convertTestCases(result),
+		}
+		testSuites = append(testSuites, testSuite)
+	}
+
+	output, err := xml.MarshalIndent(testSuites, "", "  ")
+	if err != nil {
+		return errors.Wrapf(err, "xml.MarshalIndent")
+	}
+	if err := os.WriteFile(filename, output, 0600); err != nil {
+		return errors.Wrapf(err, "os.WriteFile")
+	}
+
 	return nil
 }
+
+func countFailures(tests []TestCase) int {
+	failures := 0
+	for _, test := range tests {
+		if test.Err != nil {
+			failures++
+		}
+	}
+
+	return failures
+}
+
+func formatDuration(d time.Duration) string {
+	return d.String()
+}
+
+func convertTestCases(suite TestSuiteResult) []JUnitTestCase {
+	testCases := []JUnitTestCase{}
+	for _, test := range suite.TestCases {
+		testCase := JUnitTestCase{
+			ClassName: suite.Name,
+			Name:      test.Name,
+			Time:      formatDuration(test.Duration),
+		}
+		if test.Err != nil {
+			testCase.Failure = &Failure{
+				Message: test.Err.Error(),
+				Type:    "failure",
+			}
+		}
+		if test.Skipped {
+			testCase.Skipped = &Skipped{
+				Message: test.SkipMsg,
+			}
+		}
+		testCases = append(testCases, testCase)
+	}
+
+	return testCases
+}

pkg/hhfab/testing.go

@@ -1577,6 +1577,7 @@ func (c *Config) Inspect(ctx context.Context, vlab *VLAB, opts InspectOpts) erro
 type ReleaseTestOpts struct {
 	Regexes     []string
 	InvertRegex bool
+	ResultsFile string
 }
 
 func (c *Config) ReleaseTest(ctx context.Context, opts ReleaseTestOpts) error {

pkg/hhfab/vlabrunner.go

@@ -562,7 +562,7 @@ func (c *Config) VLABRun(ctx context.Context, vlab *VLAB, opts VLABRunOpts) erro
 						return fmt.Errorf("inspecting: %w", err)
 					}
 				case OnReadyReleaseTest:
-					if err := c.ReleaseTest(ctx, ReleaseTestOpts{}); err != nil {
+					if err := c.ReleaseTest(ctx, ReleaseTestOpts{ResultsFile: "test-results/release-test.xml"}); err != nil {
 						slog.Warn("Failed to run release test", "err", err)
 
 						if opts.CollectShowTech {