adding required annotations check and necessary tests

Signed-off-by: Adam D. Cornett <adc@redhat.com>

certification/results.go

@@ -22,4 +22,5 @@ type Results struct {
 	Passed            []Result
 	Failed            []Result
 	Errors            []Result
+	Warned            []Result
 }

internal/check/certification.go

@@ -6,6 +6,13 @@ import (
 	"github.com/redhat-openshift-ecosystem/openshift-preflight/internal/image"
 )
 
+// Indicates the possible levels that can be utilized in the implementation of a check.
+const (
+	LevelBest     = "best"
+	LevelOptional = "optional"
+	LevelWarn     = "warn"
+)
+
 // Check as an interface containing all methods necessary
 // to use and identify a given check.
 type Check interface {

internal/csv/csv.go

@@ -10,7 +10,19 @@ import (
 	corev1 "k8s.io/api/core/v1"
 )
 
-const InfrastructureFeaturesAnnotation = "operators.openshift.io/infrastructure-features"
+const (
+	InfrastructureFeaturesAnnotation = "operators.openshift.io/infrastructure-features"
+	DisconnectedAnnotation           = "features.operators.openshift.io/disconnected"
+	FIPSCompliantAnnotation          = "features.operators.openshift.io/fips-compliant"
+	ProxyAwareAnnotation             = "features.operators.openshift.io/proxy-aware"
+	TLSProfilesAnnotation            = "features.operators.openshift.io/tls-profiles"
+	TokenAuthAWSAnnotation           = "features.operators.openshift.io/token-auth-aws"
+	TokenAuthAzureAnnotation         = "features.operators.openshift.io/token-auth-azure"
+	TokenAuthGCPAnnotation           = "features.operators.openshift.io/token-auth-gcp"
+	CNFAnnotation                    = "features.operators.openshift.io/cnf"
+	CNIAnnotation                    = "features.operators.openshift.io/cni"
+	CSIAnnotation                    = "features.operators.openshift.io/csi"
+)
 
 // SupportsDisconnected accepts a stringified list of supported features
 // and returns true if "disconnected" is listed as a supported feature.

internal/engine/engine.go

@@ -216,33 +216,39 @@ func (c *craneEngine) ExecuteChecks(ctx context.Context) error {
 
 	// execute checks
 	logger.V(log.DBG).Info("executing checks")
-	for _, check := range c.checks {
+	for _, executedCheck := range c.checks {
 		c.results.TestedImage = c.image
 
-		logger.V(log.DBG).Info("running check", "check", check.Name())
-		if check.Metadata().Level == "optional" {
-			logger.Info(fmt.Sprintf("Check %s is not currently being enforced.", check.Name()))
+		logger.V(log.DBG).Info("running check", "check", executedCheck.Name())
+		if executedCheck.Metadata().Level == check.LevelOptional || executedCheck.Metadata().Level == check.LevelWarn {
+			logger.Info(fmt.Sprintf("Check %s is not currently being enforced.", executedCheck.Name()))
 		}
 
 		// run the validation
 		checkStartTime := time.Now()
-		checkPassed, err := check.Validate(ctx, c.imageRef)
+		checkPassed, err := executedCheck.Validate(ctx, c.imageRef)
 		checkElapsedTime := time.Since(checkStartTime)
 
 		if err != nil {
-			logger.WithValues("result", "ERROR", "err", err.Error()).Info("check completed", "check", check.Name())
-			c.results.Errors = appendUnlessOptional(c.results.Errors, certification.Result{Check: check, ElapsedTime: checkElapsedTime})
+			logger.WithValues("result", "ERROR", "err", err.Error()).Info("check completed", "check", executedCheck.Name())
+			c.results.Errors = appendUnlessOptional(c.results.Errors, certification.Result{Check: executedCheck, ElapsedTime: checkElapsedTime})
 			continue
 		}
 
 		if !checkPassed {
-			logger.WithValues("result", "FAILED").Info("check completed", "check", check.Name())
-			c.results.Failed = appendUnlessOptional(c.results.Failed, certification.Result{Check: check, ElapsedTime: checkElapsedTime})
+			// if a test doesn't pass but is of level warn include it in warning results, instead of failed results
+			if executedCheck.Metadata().Level == check.LevelWarn {
+				logger.WithValues("result", "WARNING").Info("check completed", "check", executedCheck.Name())
+				c.results.Warned = appendUnlessOptional(c.results.Warned, certification.Result{Check: executedCheck, ElapsedTime: checkElapsedTime})
+				continue
+			}
+			logger.WithValues("result", "FAILED").Info("check completed", "check", executedCheck.Name())
+			c.results.Failed = appendUnlessOptional(c.results.Failed, certification.Result{Check: executedCheck, ElapsedTime: checkElapsedTime})
 			continue
 		}
 
-		logger.WithValues("result", "PASSED").Info("check completed", "check", check.Name())
-		c.results.Passed = appendUnlessOptional(c.results.Passed, certification.Result{Check: check, ElapsedTime: checkElapsedTime})
+		logger.WithValues("result", "PASSED").Info("check completed", "check", executedCheck.Name())
+		c.results.Passed = appendUnlessOptional(c.results.Passed, certification.Result{Check: executedCheck, ElapsedTime: checkElapsedTime})
 	}
 
 	if len(c.results.Errors) > 0 || len(c.results.Failed) > 0 {
@@ -674,6 +680,7 @@ func InitializeOperatorChecks(ctx context.Context, p policy.Policy, cfg Operator
 			operatorpol.NewSecurityContextConstraintsCheck(),
 			&operatorpol.RelatedImagesCheck{},
 			operatorpol.FollowsRestrictedNetworkEnablementGuidelines{},
+			operatorpol.RequiredAnnotations{},
 		}, nil
 	}
 

internal/engine/engine_test.go

@@ -108,6 +108,24 @@ var _ = Describe("Execute Checks tests", func() {
 			check.HelpText{},
 		)
 
+		warningCheckPassing := check.NewGenericCheck(
+			"warnCheckPassing",
+			func(context.Context, image.ImageReference) (bool, error) {
+				return true, nil
+			},
+			check.Metadata{Level: check.LevelWarn},
+			check.HelpText{},
+		)
+
+		warningCheckFailing := check.NewGenericCheck(
+			"warnCheckFailing",
+			func(context.Context, image.ImageReference) (bool, error) {
+				return false, nil
+			},
+			check.Metadata{Level: check.LevelWarn},
+			check.HelpText{},
+		)
+
 		emptyConfig := runtime.Config{}
 		engine = craneEngine{
 			dockerConfig: emptyConfig.DockerConfig,
@@ -118,6 +136,8 @@ var _ = Describe("Execute Checks tests", func() {
 				failedCheck,
 				optionalCheckPassing,
 				optionalCheckFailing,
+				warningCheckPassing,
+				warningCheckFailing,
 			},
 			isBundle:  false,
 			isScratch: false,
@@ -127,9 +147,10 @@ var _ = Describe("Execute Checks tests", func() {
 		It("should succeed", func() {
 			err := engine.ExecuteChecks(testcontext)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(engine.results.Passed).To(HaveLen(1))
+			Expect(engine.results.Passed).To(HaveLen(2))
 			Expect(engine.results.Failed).To(HaveLen(1))
 			Expect(engine.results.Errors).To(HaveLen(1))
+			Expect(engine.results.Warned).To(HaveLen(1))
 			Expect(engine.results.CertificationHash).To(BeEmpty())
 		})
 		Context("it is a bundle", func() {
@@ -330,6 +351,7 @@ var _ = Describe("Check Name Queries", func() {
 			"SecurityContextConstraintsInCSV",
 			"AllImageRefsInRelatedImages",
 			"FollowsRestrictedNetworkEnablementGuidelines",
+			"RequiredAnnotations",
 		}),
 		Entry("scratch container policy", ScratchContainerPolicy, []string{
 			"HasLicense",

internal/formatters/junitxml.go

@@ -18,6 +18,7 @@ type JUnitTestSuite struct {
 	XMLName    xml.Name        `xml:"testsuite"`
 	Tests      int             `xml:"tests,attr"`
 	Failures   int             `xml:"failures,attr"`
+	Warnings   int             `xml:"warnings,attr"`
 	Time       string          `xml:"time,attr"`
 	Name       string          `xml:"name,attr"`
 	Properties []JUnitProperty `xml:"properties>property,omitempty"`
@@ -30,7 +31,8 @@ type JUnitTestCase struct {
 	Name        string            `xml:"name,attr"`
 	Time        string            `xml:"time,attr"`
 	SkipMessage *JUnitSkipMessage `xml:"skipped,omitempty"`
-	Failure     *JUnitFailure     `xml:"failure,omitempty"`
+	Failure     *JUnitMessage     `xml:"failure,omitempty"`
+	Warning     *JUnitMessage     `xml:"warning,omitempty"`
 	SystemOut   string            `xml:"system-out,omitempty"`
 	Message     string            `xml:",chardata"`
 }
@@ -44,18 +46,19 @@ type JUnitProperty struct {
 	Value string `xml:"value,attr"`
 }
 
-type JUnitFailure struct {
+type JUnitMessage struct {
 	Message  string `xml:"message,attr"`
 	Type     string `xml:"type,attr"`
 	Contents string `xml:",chardata"`
 }
 
-func junitXMLFormatter(ctx context.Context, r certification.Results) ([]byte, error) {
+func junitXMLFormatter(_ context.Context, r certification.Results) ([]byte, error) {
 	response := getResponse(r)
 	suites := JUnitTestSuites{}
 	testsuite := JUnitTestSuite{
-		Tests:      len(r.Errors) + len(r.Failed) + len(r.Passed),
+		Tests:      len(r.Errors) + len(r.Failed) + len(r.Passed) + len(r.Warned),
 		Failures:   len(r.Errors) + len(r.Failed),
+		Warnings:   len(r.Warned),
 		Time:       "0s",
 		Name:       "Red Hat Certification",
 		Properties: []JUnitProperty{},
@@ -80,7 +83,7 @@ func junitXMLFormatter(ctx context.Context, r certification.Results) ([]byte, er
 			Classname: response.Image,
 			Name:      result.Name(),
 			Time:      result.ElapsedTime.String(),
-			Failure: &JUnitFailure{
+			Failure: &JUnitMessage{
 				Message:  "Failed",
 				Type:     "",
 				Contents: fmt.Sprintf("%s: Suggested Fix: %s", result.Help().Message, result.Help().Suggestion),
@@ -90,6 +93,21 @@ func junitXMLFormatter(ctx context.Context, r certification.Results) ([]byte, er
 		totalDuration += result.ElapsedTime
 	}
 
+	for _, result := range r.Warned {
+		testCase := JUnitTestCase{
+			Classname: response.Image,
+			Name:      result.Name(),
+			Time:      result.ElapsedTime.String(),
+			Warning: &JUnitMessage{
+				Message:  "Warn",
+				Type:     "",
+				Contents: fmt.Sprintf("%s: Suggested Fix: %s", result.Help().Message, result.Help().Suggestion),
+			},
+		}
+		testsuite.TestCases = append(testsuite.TestCases, testCase)
+		totalDuration += result.ElapsedTime
+	}
+
 	testsuite.Time = fmt.Sprintf("%f", totalDuration.Seconds())
 	suites.Suites = append(suites.Suites, testsuite)
 

internal/formatters/junitxml_test.go

@@ -78,6 +78,38 @@ var _ = Describe("JUnitXML Formatter", func() {
 						ElapsedTime: 0,
 					},
 				},
+				Warned: []certification.Result{
+					{
+						Check: check.NewGenericCheck(
+							"WarningCheckPass",
+							func(ctx context.Context, ir image.ImageReference) (bool, error) { return true, nil },
+							check.Metadata{
+								Description:      "description",
+								KnowledgeBaseURL: "kburl",
+								CheckURL:         "checkurl",
+							},
+							check.HelpText{
+								Message:    "helptext",
+								Suggestion: "suggestion",
+							}),
+						ElapsedTime: 0,
+					},
+					{
+						Check: check.NewGenericCheck(
+							"WarningCheckFail",
+							func(ctx context.Context, ir image.ImageReference) (bool, error) { return false, nil },
+							check.Metadata{
+								Description:      "description",
+								KnowledgeBaseURL: "kburl",
+								CheckURL:         "checkurl",
+							},
+							check.HelpText{
+								Message:    "helptext",
+								Suggestion: "suggestion",
+							}),
+						ElapsedTime: 0,
+					},
+				},
 			}
 		})
 		It("should format without error", func() {

internal/formatters/util.go

@@ -11,6 +11,7 @@ func getResponse(r certification.Results) UserResponse {
 	passedChecks := make([]checkExecutionInfo, 0, len(r.Passed))
 	failedChecks := make([]checkExecutionInfo, 0, len(r.Failed))
 	erroredChecks := make([]checkExecutionInfo, 0, len(r.Errors))
+	warnedChecks := make([]checkExecutionInfo, 0, len(r.Warned))
 
 	if len(r.Passed) > 0 {
 		for _, check := range r.Passed {
@@ -47,15 +48,30 @@ func getResponse(r certification.Results) UserResponse {
 		}
 	}
 
+	if len(r.Warned) > 0 {
+		for _, check := range r.Warned {
+			warnedChecks = append(warnedChecks, checkExecutionInfo{
+				Name:             check.Name(),
+				ElapsedTime:      float64(check.ElapsedTime.Milliseconds()),
+				Description:      check.Metadata().Description,
+				Help:             check.Help().Message,
+				Suggestion:       check.Help().Suggestion,
+				KnowledgeBaseURL: check.Metadata().KnowledgeBaseURL,
+				CheckURL:         check.Metadata().CheckURL,
+			})
+		}
+	}
+
 	response := UserResponse{
 		Image:             r.TestedImage,
 		Passed:            r.PassedOverall,
 		LibraryInfo:       version.Version,
 		CertificationHash: r.CertificationHash,
 		Results: resultsText{
-			Passed: passedChecks,
-			Failed: failedChecks,
-			Errors: erroredChecks,
+			Passed:   passedChecks,
+			Failed:   failedChecks,
+			Errors:   erroredChecks,
+			Warnings: warnedChecks,
 		},
 	}
 
@@ -73,9 +89,10 @@ type UserResponse struct {
 
 // resultsText represents the results of check execution against the asset.
 type resultsText struct {
-	Passed []checkExecutionInfo `json:"passed" xml:"passed"`
-	Failed []checkExecutionInfo `json:"failed" xml:"failed"`
-	Errors []checkExecutionInfo `json:"errors" xml:"errors"`
+	Passed   []checkExecutionInfo `json:"passed" xml:"passed"`
+	Failed   []checkExecutionInfo `json:"failed" xml:"failed"`
+	Errors   []checkExecutionInfo `json:"errors" xml:"errors"`
+	Warnings []checkExecutionInfo `json:"warning,omitempty" xml:"warning,omitempty"`
 }
 
 // checkExecutionInfo contains all possible output fields that a user might see in their result.