From fe7eda861bab0438a1ef48fd07dbea68396ed6ea Mon Sep 17 00:00:00 2001 From: anthony-swirldslabs <152534762+anthony-swirldslabs@users.noreply.github.com> Date: Wed, 20 Dec 2023 11:08:15 -0800 Subject: [PATCH] fix 68: fix the Random seed and overhaul thresholds (#155) Signed-off-by: Anthony Petrov --- pbj-integration-tests/build.gradle.kts | 9 + .../hedera/pbj/integration/fuzz/FuzzTest.java | 6 +- .../pbj/intergration/test/SampleFuzzTest.java | 154 +++++++++++++++--- 3 files changed, 140 insertions(+), 29 deletions(-) diff --git a/pbj-integration-tests/build.gradle.kts b/pbj-integration-tests/build.gradle.kts index e54362b5..62e6ab51 100644 --- a/pbj-integration-tests/build.gradle.kts +++ b/pbj-integration-tests/build.gradle.kts @@ -65,6 +65,15 @@ testing { useJUnitPlatform { includeTags("FUZZ_TEST") } enableAssertions = false } + + tasks.register("randomFuzzTest") { + testClassesDirs = sources.output.classesDirs + classpath = sources.runtimeClasspath + + useJUnitPlatform { includeTags("FUZZ_TEST") } + enableAssertions = false + systemProperties["com.hedera.pbj.intergration.test.fuzz.useRandomSeed"] = true + } } } diff --git a/pbj-integration-tests/src/main/java/com/hedera/pbj/integration/fuzz/FuzzTest.java b/pbj-integration-tests/src/main/java/com/hedera/pbj/integration/fuzz/FuzzTest.java index 1c1ebb7b..049695f2 100644 --- a/pbj-integration-tests/src/main/java/com/hedera/pbj/integration/fuzz/FuzzTest.java +++ b/pbj-integration-tests/src/main/java/com/hedera/pbj/integration/fuzz/FuzzTest.java @@ -30,11 +30,10 @@ public class FuzzTest { * for the most desirable DESERIALIZATION_FAILED outcome to determine * if the test passed or not. */ - public static FuzzTestResult fuzzTest(final T object, final double threshold) { + public static FuzzTestResult fuzzTest(final T object, final double threshold, final Random random) { final long startNanoTime = System.nanoTime(); final Codec codec = getCodec(object); - final Random random = new Random(); final int repeatCount = estimateRepeatCount(object, codec); if (repeatCount == 0) { @@ -50,7 +49,8 @@ public static FuzzTestResult fuzzTest(final T object, final double thresh } final Map resultCounts = IntStream.range(0, repeatCount) - .parallel() + // Note that we must run this stream sequentially to enable + // reproducing the tests for a given random seed. .mapToObj(n -> SingleFuzzTest.fuzzTest(object, codec, random)) .collect(Collectors.groupingBy(Function.identity(), Collectors.counting())); diff --git a/pbj-integration-tests/src/test/java/com/hedera/pbj/intergration/test/SampleFuzzTest.java b/pbj-integration-tests/src/test/java/com/hedera/pbj/intergration/test/SampleFuzzTest.java index db3c92d2..6ecaf7f6 100644 --- a/pbj-integration-tests/src/test/java/com/hedera/pbj/intergration/test/SampleFuzzTest.java +++ b/pbj-integration-tests/src/test/java/com/hedera/pbj/intergration/test/SampleFuzzTest.java @@ -4,6 +4,7 @@ import com.hedera.hapi.node.base.tests.ContractIDTest; import com.hedera.pbj.integration.fuzz.FuzzTest; import com.hedera.pbj.integration.fuzz.FuzzTestResult; +import com.hedera.pbj.integration.fuzz.SingleFuzzTestResult; import com.hedera.pbj.test.proto.pbj.tests.EverythingTest; import com.hedera.pbj.test.proto.pbj.tests.HashevalTest; import com.hedera.pbj.test.proto.pbj.tests.InnerEverythingTest; @@ -13,10 +14,12 @@ import com.hedera.pbj.test.proto.pbj.tests.TimestampTestSecondsTest; import com.hedera.pbj.test.proto.pbj.tests.TimestampTestTest; import org.junit.jupiter.api.Tag; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.api.Test; +import java.text.NumberFormat; import java.util.List; +import java.util.Random; +import java.util.stream.Collectors; import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -26,24 +29,65 @@ * This is a sample fuzz test just to demonstrate the usage of the FuzzTest class. * It will be replaced with a more elaborate fuzz testing framework in the future. * See javadoc for FuzzTest for more details. + * + * Three thresholds defined at the beginning of the class below + * determine whether an individual test for a specific model object + * is considered passed and whether the fuzz test as a whole + * is considered passed or failed. */ public class SampleFuzzTest { - public static final String FUZZ_TEST = "FUZZ_TEST"; - - // A percentage threshold for the DESERIALIZATION_FAILED outcomes. - // Note that we still encounter runs that result in less than 60% - // of the desirable outcome. Interestingly, this occurs with small - // objects mostly, for example an object with a single field - // of type string/byte array. When the string is too long, many a time - // the random data gets written into the string bytes w/o affecting - // the validity of the object. This may be addressed by increasing - // the number of bytes that we modify for a single test run. - // However, this may decrease the number of subtle modifications - // which we also want to test. - // For now, we keep the threshold at 60%. - // This will be refactored to gather the statistics across all the - // test cases and pass or fail the overall test based on the statistics. - private static final double THRESHOLD = 0.6; + // Flip to true to print out results stats for every tested model object. + // When false, only the fuzz test summary is printed to stdout. + private static final boolean debug = false; + + /** + * A percentage threshold for the share of DESERIALIZATION_FAILED outcomes + * when running tests for a given model object. + * + * A test for that specific model object is considered passed + * if random modifications of the object's payload produce + * that many DESERIALIZATION_FAILED outcomes. + */ + private static final double THRESHOLD = .8; + + /** + * A percentage threshold for the pass rate across tests + * for all model objects. + * + * The fuzz test as a whole is considered passed + * if that many individual model tests pass. + */ + private static final double PASS_RATE_THRESHOLD = .9; + + /** + * A threshold for the mean value of the shares of DESERIALIZATION_FAILED + * outcomes across tests for all model objects. + * + * The fuzz test as a whole is considered passed + * if the mean value of all the individual DESERIALIZATION_FAILED + * shares is greater than this threshold. + */ + private static final double DESERIALIZATION_FAILED_MEAN_THRESHOLD = .9; + + /** + * Fuzz tests are tagged with this tag to allow Gradle/JUnit + * to disable assertions when running these tests. + * This enables us to catch the actual codec failures. + */ + private static final String FUZZ_TEST_TAG = "FUZZ_TEST"; + + /** + * A fixed seed for a random numbers generator when + * we want to run the tests in a reproducible way. + * + * Use the randomFuzzTest Gradle target to use a random seed + * instead, which will run the tests in a random way + * allowing one to potentially discover new and unknown issues. + * + * This number is completely random. However, the threshold + * values above may need changing if this value changes. + */ + private static final long FIXED_RANDOM_SEED = 837582698436792L; private static final List> MODEL_TEST_OBJECTS = List.of( AccountIDTest.ARGUMENTS, @@ -63,18 +107,76 @@ private static Stream objectTestCases() { .flatMap(List::stream); } - @ParameterizedTest - @MethodSource("objectTestCases") - @Tag(SampleFuzzTest.FUZZ_TEST) - void testMethod(Object object) throws Exception { + private static record ResultStats( + double passRate, + double deserializationFailedMean + ) { + private static final NumberFormat PERCENTAGE_FORMAT = NumberFormat.getPercentInstance(); + + boolean passed() { + return passRate > PASS_RATE_THRESHOLD + && deserializationFailedMean > DESERIALIZATION_FAILED_MEAN_THRESHOLD; + } + + String format() { + return "Fuzz tests " + (passed() ? "PASSED" : "FAILED") + + " with passRate = " + PERCENTAGE_FORMAT.format(passRate) + + " and deserializationFailedMean = " + PERCENTAGE_FORMAT.format(deserializationFailedMean); + } + } + + @Test + @Tag(SampleFuzzTest.FUZZ_TEST_TAG) + void fuzzTest() { assumeFalse( this.getClass().desiredAssertionStatus(), "Fuzz tests run with assertions disabled only. Use the fuzzTest Gradle target." ); - FuzzTestResult fuzzTestResult = FuzzTest.fuzzTest(object, THRESHOLD); - String resultDescription = fuzzTestResult.format(); - System.out.println(resultDescription); - assertTrue(fuzzTestResult.passed(), resultDescription); + final Random random = buildRandom(); + + final List> results = objectTestCases() + // Note that we must run this stream sequentially to enable + // reproducing the tests for a given random seed. + .map(object -> FuzzTest.fuzzTest(object, THRESHOLD, random)) + .peek(result -> { if (debug) System.out.println(result.format()); }) + .collect(Collectors.toList()); + + final ResultStats resultStats = results.stream() + .map(result -> new ResultStats( + result.passed() ? 1. : 0., + result.percentageMap().getOrDefault(SingleFuzzTestResult.DESERIALIZATION_FAILED, 0.) + ) + ) + .reduce( + (r1, r2) -> new ResultStats( + r1.passRate() + r2.passRate(), + r1.deserializationFailedMean() + r2.deserializationFailedMean()) + ) + .map(stats -> new ResultStats( + stats.passRate() / (double) results.size(), + stats.deserializationFailedMean() / (double) results.size() + ) + ) + .orElse(new ResultStats(0., 0.)); + + final String statsMessage = resultStats.format(); + System.out.println(statsMessage); + assertTrue(resultStats.passed(), statsMessage); } + + private Random buildRandom() { + final boolean useRandomSeed + = Boolean.valueOf(System.getProperty("com.hedera.pbj.intergration.test.fuzz.useRandomSeed")); + final long seed = useRandomSeed ? new Random().nextLong() : FIXED_RANDOM_SEED; + + System.out.println("Fuzz tests are configured to use a " + + (useRandomSeed ? "RANDOM" : "FIXED") + + " seed for `new Random(seed)`, and the seed value for this run is: " + + seed + ); + + return new Random(seed); + } + }