Return error when processing conformance results (#326)

tools/protovalidate-conformance/internal/suites/suite.go

@@ -15,6 +15,7 @@
 package suites
 
 import (
+	"fmt"
 	"regexp"
 
 	"github.com/bufbuild/protovalidate/tools/internal/gen/buf/validate/conformance/harness"
@@ -79,12 +80,12 @@ func (s Suite) ProcessResults(
 	resp *harness.TestConformanceResponse,
 	options *harness.ResultOptions,
 	skippedCases []string,
-) *results.SuiteResults {
+) (*results.SuiteResults, error) {
 	out := &results.SuiteResults{
 		Name: suiteName,
 	}
 	respResults := resp.GetResults()
-	_ = s.Range(filter, func(caseName string, testCase Case) error {
+	err := s.Range(filter, func(caseName string, testCase Case) error {
 		var actual results.Result
 		if result, ok := respResults[caseName]; ok {
 			actual = results.FromProto(result)
@@ -114,7 +115,11 @@ func (s Suite) ProcessResults(
 		return nil
 	})
 
-	return out
+	if err != nil {
+		return nil, fmt.Errorf("failed to process results for suite \"%s\": %w", suiteName, err)
+	}
+
+	return out, nil
 }
 
 func isSkipped(caseName string, cases []string) bool {

tools/protovalidate-conformance/main.go

@@ -57,13 +57,16 @@ func main() {
 				return err
 			}
 		}
-		res := suite.ProcessResults(
+		res, err := suite.ProcessResults(
 			suiteName,
 			cfg.caseFilter,
 			resp,
 			options,
 			cfg.expectedFailures[suiteName],
 		)
+		if err != nil {
+			return err
+		}
 		res.Fdset = req.GetFdset()
 		resultSet.AddSuite(res, cfg.verbose)
 		return nil

tools/protovalidate-conformance/main_test.go

@@ -0,0 +1,39 @@
+// Copyright 2023-2025 Buf Technologies, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"testing"
+
+	"github.com/bufbuild/protovalidate/tools/protovalidate-conformance/internal/cases"
+	"github.com/bufbuild/protovalidate/tools/protovalidate-conformance/internal/suites"
+)
+
+func TestProcessAllSuites(t *testing.T) {
+	t.Parallel()
+	// Conformance test cases can be invalid: For example, they can define a bad
+	// field path, or a message that fails to marshal.
+	err := cases.GlobalSuites().Range(nil, func(suiteName string, suite suites.Suite) error {
+		_, err := suite.ToRequestProto(nil)
+		if err != nil {
+			return err
+		}
+		_, err = suite.ProcessResults(suiteName, nil, nil, nil, nil)
+		return err
+	})
+	if err != nil {
+		t.Error(err)
+	}
+}