Add option to output all results to disk (#4344)

test/sdk-benchmarks/pom.xml

@@ -205,6 +205,11 @@
             <version>${awsjavasdk.version}</version>
             <scope>compile</scope>
         </dependency>
+        <dependency>
+            <groupId>commons-cli</groupId>
+            <artifactId>commons-cli</artifactId>
+            <scope>compile</scope>
+        </dependency>
     </dependencies>
 
     <dependencyManagement>
@@ -368,6 +373,8 @@
                         <argument>-classpath</argument>
                         <classpath/>
                         <argument>software.amazon.awssdk.benchmark.BenchmarkRunner</argument>
+                        <!-- Fail process on failed benchmarks -->
+                        <argument>-c</argument>
                     </arguments>
                 </configuration>
             </plugin>

test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/BenchmarkResultProcessor.java

@@ -24,6 +24,7 @@
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -34,6 +35,7 @@
 import software.amazon.awssdk.benchmark.stats.SdkBenchmarkParams;
 import software.amazon.awssdk.benchmark.stats.SdkBenchmarkResult;
 import software.amazon.awssdk.benchmark.stats.SdkBenchmarkStatistics;
+import software.amazon.awssdk.benchmark.utils.BenchmarkProcessorOutput;
 import software.amazon.awssdk.utils.Logger;
 
 
@@ -66,15 +68,18 @@ class BenchmarkResultProcessor {
      * Process benchmark results
      *
      * @param results the results of the benchmark
-     * @return the benchmark Id that failed the regression
+     * @return the benchmark results
      */
-    List<String> processBenchmarkResult(Collection<RunResult> results) {
-        List<SdkBenchmarkResult> currentData = new ArrayList<>();
+    BenchmarkProcessorOutput processBenchmarkResult(Collection<RunResult> results) {
+        Map<String, SdkBenchmarkResult> benchmarkResults = new HashMap<>();
+
         for (RunResult result : results) {
             String benchmarkId = getBenchmarkId(result.getParams());
+            SdkBenchmarkResult sdkBenchmarkData = constructSdkBenchmarkResult(result);
+
+            benchmarkResults.put(benchmarkId, sdkBenchmarkData);
 
             SdkBenchmarkResult baselineResult = baseline.get(benchmarkId);
-            SdkBenchmarkResult sdkBenchmarkData = constructSdkBenchmarkResult(result);
 
             if (baselineResult == null) {
                 log.warn(() -> {
@@ -90,15 +95,14 @@ List<String> processBenchmarkResult(Collection<RunResult> results) {
                 continue;
             }
 
-            currentData.add(sdkBenchmarkData);
-
             if (!validateBenchmarkResult(sdkBenchmarkData, baselineResult)) {
                 failedBenchmarkIds.add(benchmarkId);
             }
         }
 
-        log.info(() -> "Current result: " + serializeResult(currentData));
-        return failedBenchmarkIds;
+        BenchmarkProcessorOutput output = new BenchmarkProcessorOutput(benchmarkResults, failedBenchmarkIds);
+        log.info(() -> "Current result: " + serializeResult(output));
+        return output;
     }
 
     private SdkBenchmarkResult constructSdkBenchmarkResult(RunResult runResult) {
@@ -169,9 +173,9 @@ private boolean validateBenchmarkParams(SdkBenchmarkParams current, SdkBenchmark
         return current.getMode() == baseline.getMode();
     }
 
-    private String serializeResult(List<SdkBenchmarkResult> currentData) {
+    private String serializeResult(BenchmarkProcessorOutput processorOutput) {
         try {
-            return OBJECT_MAPPER.writeValueAsString(currentData);
+            return OBJECT_MAPPER.writeValueAsString(processorOutput);
         } catch (JsonProcessingException e) {
             log.error(() -> "Failed to serialize current result", e);
         }

test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/BenchmarkRunner.java

@@ -15,11 +15,23 @@
 
 package software.amazon.awssdk.benchmark;
 
-import com.fasterxml.jackson.core.JsonProcessingException;
+import static software.amazon.awssdk.benchmark.utils.BenchmarkConstant.OBJECT_MAPPER;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
+import java.util.stream.Collectors;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
 import org.openjdk.jmh.results.RunResult;
 import org.openjdk.jmh.runner.Runner;
 import org.openjdk.jmh.runner.RunnerException;
@@ -45,6 +57,8 @@
 import software.amazon.awssdk.benchmark.enhanced.dynamodb.EnhancedClientQueryV1MapperComparisonBenchmark;
 import software.amazon.awssdk.benchmark.enhanced.dynamodb.EnhancedClientScanV1MapperComparisonBenchmark;
 import software.amazon.awssdk.benchmark.enhanced.dynamodb.EnhancedClientUpdateV1MapperComparisonBenchmark;
+import software.amazon.awssdk.benchmark.stats.SdkBenchmarkResult;
+import software.amazon.awssdk.benchmark.utils.BenchmarkProcessorOutput;
 import software.amazon.awssdk.utils.Logger;
 
 
@@ -84,13 +98,15 @@ public class BenchmarkRunner {
 
     private final List<String> benchmarksToRun;
     private final BenchmarkResultProcessor resultProcessor;
+    private final BenchmarkRunnerOptions options;
 
-    private BenchmarkRunner(List<String> benchmarksToRun) {
+    private BenchmarkRunner(List<String> benchmarksToRun, BenchmarkRunnerOptions options) {
         this.benchmarksToRun = benchmarksToRun;
         this.resultProcessor = new BenchmarkResultProcessor();
+        this.options = options;
     }
 
-    public static void main(String... args) throws RunnerException, JsonProcessingException {
+    public static void main(String... args) throws Exception {
         List<String> benchmarksToRun = new ArrayList<>();
         benchmarksToRun.addAll(SYNC_BENCHMARKS);
         benchmarksToRun.addAll(ASYNC_BENCHMARKS);
@@ -99,13 +115,14 @@ public static void main(String... args) throws RunnerException, JsonProcessingEx
 
         log.info(() -> "Skipping tests, to reduce benchmark times: \n" + MAPPER_BENCHMARKS + "\n" + METRIC_BENCHMARKS);
 
-
-        BenchmarkRunner runner = new BenchmarkRunner(benchmarksToRun);
+        BenchmarkRunner runner = new BenchmarkRunner(benchmarksToRun, parseOptions(args));
 
         runner.runBenchmark();
     }
 
     private void runBenchmark() throws RunnerException {
+        log.info(() -> "Running with options: " + options);
+
         ChainedOptionsBuilder optionsBuilder = new OptionsBuilder();
 
         benchmarksToRun.forEach(optionsBuilder::include);
@@ -114,11 +131,70 @@ private void runBenchmark() throws RunnerException {
 
         Collection<RunResult> results = new Runner(optionsBuilder.build()).run();
 
-        List<String> failedResult = resultProcessor.processBenchmarkResult(results);
+        BenchmarkProcessorOutput processedResults = resultProcessor.processBenchmarkResult(results);
+        List<String> failedResults = processedResults.getFailedBenchmarks();
+
+        if (options.outputPath != null) {
+            log.info(() -> "Writing results to " + options.outputPath);
+            writeResults(processedResults, options.outputPath);
+        }
+
+        if (options.check && !failedResults.isEmpty()) {
+            log.info(() -> "Failed perf regression tests: " + failedResults);
+            throw new RuntimeException("Perf regression tests failed: " + failedResults);
+        }
+    }
+
+    private static BenchmarkRunnerOptions parseOptions(String[] args) throws ParseException {
+        Options cliOptions = new Options();
+        cliOptions.addOption("o", "output", true,
+                                     "The path to write the benchmark results to.");
+        cliOptions.addOption("c", "check", false,
+                             "If specified, exit with error code 1 if the results are not within the baseline.");
+
+        CommandLineParser parser = new DefaultParser();
+        CommandLine cmdLine = parser.parse(cliOptions, args);
+
+        BenchmarkRunnerOptions options = new BenchmarkRunnerOptions()
+            .check(cmdLine.hasOption("c"));
+
+        if (cmdLine.hasOption("o")) {
+            options.outputPath(Paths.get(cmdLine.getOptionValue("o")));
+        }
+
+        return options;
+    }
+
+    private static void writeResults(BenchmarkProcessorOutput output, Path outputPath) {
+        List<SdkBenchmarkResult> results = output.getBenchmarkResults().values().stream().collect(Collectors.toList());
+        try (OutputStream os = Files.newOutputStream(outputPath)) {
+            OBJECT_MAPPER.writeValue(os, results);
+        } catch (IOException e) {
+            log.error(() -> "Failed to write the results to " + outputPath, e);
+            throw new RuntimeException(e);
+        }
+    }
+
+    private static class BenchmarkRunnerOptions {
+        private Path outputPath;
+        private boolean check;
+
+        public BenchmarkRunnerOptions outputPath(Path outputPath) {
+            this.outputPath = outputPath;
+            return this;
+        }
+
+        public BenchmarkRunnerOptions check(boolean check) {
+            this.check = check;
+            return this;
+        }
 
-        if (!failedResult.isEmpty()) {
-            log.info(() -> "Failed perf regression tests: " + failedResult);
-            throw new RuntimeException("Perf regression tests failed: " + failedResult);
+        @Override
+        public String toString() {
+            return "BenchmarkRunnerOptions{" +
+                   "outputPath=" + outputPath +
+                   ", check=" + check +
+                   '}';
         }
     }
 }

test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/utils/BenchmarkProcessorOutput.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.benchmark.utils;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import java.util.List;
+import java.util.Map;
+import software.amazon.awssdk.benchmark.stats.SdkBenchmarkResult;
+
+/**
+ * The output object of the benchmark processor. This contains the results of the all the benchmarks that were run, and the
+ * list of benchmarks that failed.
+ */
+public final class BenchmarkProcessorOutput {
+    private final Map<String, SdkBenchmarkResult> benchmarkResults;
+    private final List<String> failedBenchmarks;
+
+    @JsonCreator
+    public BenchmarkProcessorOutput(Map<String, SdkBenchmarkResult> benchmarkResults, List<String> failedBenchmarks) {
+        this.benchmarkResults = benchmarkResults;
+        this.failedBenchmarks = failedBenchmarks;
+    }
+
+    public Map<String, SdkBenchmarkResult> getBenchmarkResults() {
+        return benchmarkResults;
+    }
+
+    public List<String> getFailedBenchmarks() {
+        return failedBenchmarks;
+    }
+}