fix 68: fix the Random seed and overhaul thresholds (#155)

Signed-off-by: Anthony Petrov <anthony@swirldslabs.com>

pbj-integration-tests/build.gradle.kts

@@ -65,6 +65,15 @@ testing {
             useJUnitPlatform { includeTags("FUZZ_TEST") }
             enableAssertions = false
         }
+
+        tasks.register<Test>("randomFuzzTest") {
+            testClassesDirs = sources.output.classesDirs
+            classpath = sources.runtimeClasspath
+
+            useJUnitPlatform { includeTags("FUZZ_TEST") }
+            enableAssertions = false
+            systemProperties["com.hedera.pbj.intergration.test.fuzz.useRandomSeed"] = true
+        }
     }
 }
 

pbj-integration-tests/src/main/java/com/hedera/pbj/integration/fuzz/FuzzTest.java

@@ -30,11 +30,10 @@ public class FuzzTest {
      * for the most desirable DESERIALIZATION_FAILED outcome to determine
      * if the test passed or not.
      */
-    public static <T> FuzzTestResult<T> fuzzTest(final T object, final double threshold) {
+    public static <T> FuzzTestResult<T> fuzzTest(final T object, final double threshold, final Random random) {
         final long startNanoTime = System.nanoTime();
 
         final Codec<T> codec = getCodec(object);
-        final Random random = new Random();
         final int repeatCount = estimateRepeatCount(object, codec);
 
         if (repeatCount == 0) {
@@ -50,7 +49,8 @@ public static <T> FuzzTestResult<T> fuzzTest(final T object, final double thresh
         }
 
         final Map<SingleFuzzTestResult, Long> resultCounts = IntStream.range(0, repeatCount)
-                .parallel()
+                // Note that we must run this stream sequentially to enable
+                // reproducing the tests for a given random seed.
                 .mapToObj(n -> SingleFuzzTest.fuzzTest(object, codec, random))
                 .collect(Collectors.groupingBy(Function.identity(), Collectors.counting()));
 

pbj-integration-tests/src/test/java/com/hedera/pbj/intergration/test/SampleFuzzTest.java

@@ -4,6 +4,7 @@
 import com.hedera.hapi.node.base.tests.ContractIDTest;
 import com.hedera.pbj.integration.fuzz.FuzzTest;
 import com.hedera.pbj.integration.fuzz.FuzzTestResult;
+import com.hedera.pbj.integration.fuzz.SingleFuzzTestResult;
 import com.hedera.pbj.test.proto.pbj.tests.EverythingTest;
 import com.hedera.pbj.test.proto.pbj.tests.HashevalTest;
 import com.hedera.pbj.test.proto.pbj.tests.InnerEverythingTest;
@@ -13,10 +14,12 @@
 import com.hedera.pbj.test.proto.pbj.tests.TimestampTestSecondsTest;
 import com.hedera.pbj.test.proto.pbj.tests.TimestampTestTest;
 import org.junit.jupiter.api.Tag;
-import org.junit.jupiter.params.ParameterizedTest;
-import org.junit.jupiter.params.provider.MethodSource;
+import org.junit.jupiter.api.Test;
 
+import java.text.NumberFormat;
 import java.util.List;
+import java.util.Random;
+import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -26,24 +29,65 @@
  * This is a sample fuzz test just to demonstrate the usage of the FuzzTest class.
  * It will be replaced with a more elaborate fuzz testing framework in the future.
  * See javadoc for FuzzTest for more details.
+ *
+ * Three thresholds defined at the beginning of the class below
+ * determine whether an individual test for a specific model object
+ * is considered passed and whether the fuzz test as a whole
+ * is considered passed or failed.
  */
 public class SampleFuzzTest {
-    public static final String FUZZ_TEST = "FUZZ_TEST";
-
-    // A percentage threshold for the DESERIALIZATION_FAILED outcomes.
-    // Note that we still encounter runs that result in less than 60%
-    // of the desirable outcome. Interestingly, this occurs with small
-    // objects mostly, for example an object with a single field
-    // of type string/byte array. When the string is too long, many a time
-    // the random data gets written into the string bytes w/o affecting
-    // the validity of the object. This may be addressed by increasing
-    // the number of bytes that we modify for a single test run.
-    // However, this may decrease the number of subtle modifications
-    // which we also want to test.
-    // For now, we keep the threshold at 60%.
-    // This will be refactored to gather the statistics across all the
-    // test cases and pass or fail the overall test based on the statistics.
-    private static final double THRESHOLD = 0.6;
+    // Flip to true to print out results stats for every tested model object.
+    // When false, only the fuzz test summary is printed to stdout.
+    private static final boolean debug = false;
+
+    /**
+     * A percentage threshold for the share of DESERIALIZATION_FAILED outcomes
+     * when running tests for a given model object.
+     *
+     * A test for that specific model object is considered passed
+     * if random modifications of the object's payload produce
+     * that many DESERIALIZATION_FAILED outcomes.
+     */
+    private static final double THRESHOLD = .8;
+
+    /**
+     * A percentage threshold for the pass rate across tests
+     * for all model objects.
+     *
+     * The fuzz test as a whole is considered passed
+     * if that many individual model tests pass.
+     */
+    private static final double PASS_RATE_THRESHOLD = .9;
+
+    /**
+     * A threshold for the mean value of the shares of DESERIALIZATION_FAILED
+     * outcomes across tests for all model objects.
+     *
+     * The fuzz test as a whole is considered passed
+     * if the mean value of all the individual DESERIALIZATION_FAILED
+     * shares is greater than this threshold.
+     */
+    private static final double DESERIALIZATION_FAILED_MEAN_THRESHOLD = .9;
+
+    /**
+     * Fuzz tests are tagged with this tag to allow Gradle/JUnit
+     * to disable assertions when running these tests.
+     * This enables us to catch the actual codec failures.
+     */
+    private static final String FUZZ_TEST_TAG = "FUZZ_TEST";
+
+    /**
+     * A fixed seed for a random numbers generator when
+     * we want to run the tests in a reproducible way.
+     *
+     * Use the randomFuzzTest Gradle target to use a random seed
+     * instead, which will run the tests in a random way
+     * allowing one to potentially discover new and unknown issues.
+     *
+     * This number is completely random. However, the threshold
+     * values above may need changing if this value changes.
+     */
+    private static final long FIXED_RANDOM_SEED = 837582698436792L;
 
     private static final List<List<?>> MODEL_TEST_OBJECTS = List.of(
             AccountIDTest.ARGUMENTS,
@@ -63,18 +107,76 @@ private static Stream<?> objectTestCases() {
                 .flatMap(List::stream);
     }
 
-    @ParameterizedTest
-    @MethodSource("objectTestCases")
-    @Tag(SampleFuzzTest.FUZZ_TEST)
-    void testMethod(Object object) throws Exception {
+    private static record ResultStats(
+            double passRate,
+            double deserializationFailedMean
+    ) {
+        private static final NumberFormat PERCENTAGE_FORMAT = NumberFormat.getPercentInstance();
+
+        boolean passed() {
+            return passRate > PASS_RATE_THRESHOLD
+                    && deserializationFailedMean > DESERIALIZATION_FAILED_MEAN_THRESHOLD;
+        }
+
+        String format() {
+            return "Fuzz tests " + (passed() ? "PASSED" : "FAILED")
+                    + " with passRate = " + PERCENTAGE_FORMAT.format(passRate)
+                    + " and deserializationFailedMean = " + PERCENTAGE_FORMAT.format(deserializationFailedMean);
+        }
+    }
+
+    @Test
+    @Tag(SampleFuzzTest.FUZZ_TEST_TAG)
+    void fuzzTest() {
         assumeFalse(
                 this.getClass().desiredAssertionStatus(),
                 "Fuzz tests run with assertions disabled only. Use the fuzzTest Gradle target."
         );
 
-        FuzzTestResult<?> fuzzTestResult = FuzzTest.fuzzTest(object, THRESHOLD);
-        String resultDescription = fuzzTestResult.format();
-        System.out.println(resultDescription);
-        assertTrue(fuzzTestResult.passed(), resultDescription);
+        final Random random = buildRandom();
+
+        final List<? extends FuzzTestResult<?>> results = objectTestCases()
+                // Note that we must run this stream sequentially to enable
+                // reproducing the tests for a given random seed.
+                .map(object -> FuzzTest.fuzzTest(object, THRESHOLD, random))
+                .peek(result -> { if (debug) System.out.println(result.format()); })
+                .collect(Collectors.toList());
+
+        final ResultStats resultStats = results.stream()
+                .map(result -> new ResultStats(
+                                result.passed() ? 1. : 0.,
+                                result.percentageMap().getOrDefault(SingleFuzzTestResult.DESERIALIZATION_FAILED, 0.)
+                        )
+                )
+                .reduce(
+                        (r1, r2) -> new ResultStats(
+                                r1.passRate() + r2.passRate(),
+                                r1.deserializationFailedMean() + r2.deserializationFailedMean())
+                )
+                .map(stats -> new ResultStats(
+                                stats.passRate() / (double) results.size(),
+                                stats.deserializationFailedMean() / (double) results.size()
+                        )
+                )
+                .orElse(new ResultStats(0., 0.));
+
+        final String statsMessage = resultStats.format();
+        System.out.println(statsMessage);
+        assertTrue(resultStats.passed(), statsMessage);
     }
+
+    private Random buildRandom() {
+        final boolean useRandomSeed
+                = Boolean.valueOf(System.getProperty("com.hedera.pbj.intergration.test.fuzz.useRandomSeed"));
+        final long seed = useRandomSeed ? new Random().nextLong() : FIXED_RANDOM_SEED;
+
+        System.out.println("Fuzz tests are configured to use a "
+                + (useRandomSeed ? "RANDOM" : "FIXED")
+                + " seed for `new Random(seed)`, and the seed value for this run is: "
+                + seed
+        );
+
+        return new Random(seed);
+    }
+
 }