From f4ed0fcac4c284580263f0c548664dd548c9cca3 Mon Sep 17 00:00:00 2001
From: Leaf Petersen <leafpetersen@users.noreply.github.com>
Date: Wed, 6 Jul 2022 03:45:31 -0700
Subject: [PATCH] Fix harness to not call timer repeatedly in the measured
 loop. (#38)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Ömer Sinan Ağacan <omersa@google.com>
Co-authored-by: Vyacheslav Egorov <vegorov@google.com>
---
 CHANGELOG.md                |  6 +++
 lib/benchmark_harness.dart  |  1 +
 lib/src/benchmark_base.dart | 82 ++++++++++++++++++++++++++-----------
 pubspec.yaml                |  2 +-
 4 files changed, 66 insertions(+), 25 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index c6d581f..7c808cf 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,9 @@
+## 2.2.0
+
+- Change measuring algorithm in `BenchmarkBase` to avoid calling stopwatch
+methods repeatedly in the measuring loop. This makes measurement work better
+for `run` methods which are small themselves.
+
 ## 2.1.0
 
 - Add AsyncBenchmarkBase.
diff --git a/lib/benchmark_harness.dart b/lib/benchmark_harness.dart
index 2809777..6170115 100644
--- a/lib/benchmark_harness.dart
+++ b/lib/benchmark_harness.dart
@@ -5,6 +5,7 @@
 library benchmark_harness;
 
 import 'dart:async';
+import 'dart:math' as math;
 
 part 'src/benchmark_base.dart';
 part 'src/async_benchmark_base.dart';
diff --git a/lib/src/benchmark_base.dart b/lib/src/benchmark_base.dart
index b8a08a7..4a6acfa 100644
--- a/lib/src/benchmark_base.dart
+++ b/lib/src/benchmark_base.dart
@@ -4,63 +4,97 @@
 
 part of benchmark_harness;
 
+const int _minimumMeasureDurationMillis = 2000;
+
 class BenchmarkBase {
   final String name;
   final ScoreEmitter emitter;
 
-  // Empty constructor.
   const BenchmarkBase(this.name, {this.emitter = const PrintEmitter()});
 
-  // The benchmark code.
-  // This function is not used, if both [warmup] and [exercise] are overwritten.
+  /// The benchmark code.
+  ///
+  /// This function is not used, if both [warmup] and [exercise] are overwritten.
   void run() {}
 
-  // Runs a short version of the benchmark. By default invokes [run] once.
+  /// Runs a short version of the benchmark. By default invokes [run] once.
   void warmup() {
     run();
   }
 
-  // Exercises the benchmark. By default invokes [run] 10 times.
+  /// Exercises the benchmark. By default invokes [run] 10 times.
   void exercise() {
     for (var i = 0; i < 10; i++) {
       run();
     }
   }
 
-  // Not measured setup code executed prior to the benchmark runs.
+  /// Not measured setup code executed prior to the benchmark runs.
   void setup() {}
 
-  // Not measures teardown code executed after the benchmark runs.
+  /// Not measured teardown code executed after the benchmark runs.
   void teardown() {}
 
-  // Measures the score for this benchmark by executing it repeatedly until
-  // time minimum has been reached.
-  static double measureFor(void Function() f, int minimumMillis) {
-    var minimumMicros = minimumMillis * 1000;
-    var iter = 0;
-    var watch = Stopwatch();
-    watch.start();
-    var elapsed = 0;
-    while (elapsed < minimumMicros) {
-      f();
-      elapsed = watch.elapsedMicroseconds;
-      iter++;
+  /// Measures the score for this benchmark by executing it enough times
+  /// to reach [minimumMillis].
+  static _Measurement _measureForImpl(void Function() f, int minimumMillis) {
+    final minimumMicros = minimumMillis * 1000;
+    var iter = 2;
+    final watch = Stopwatch()..start();
+    while (true) {
+      watch.reset();
+      for (var i = 0; i < iter; i++) {
+        f();
+      }
+      final elapsed = watch.elapsedMicroseconds;
+      final measurement = _Measurement(elapsed, iter);
+      if (measurement.elapsedMicros >= minimumMicros) {
+        return measurement;
+      }
+
+      iter = measurement.estimateIterationsNeededToReach(
+          minimumMicros: minimumMicros);
     }
-    return elapsed / iter;
   }
 
-  // Measures the score for the benchmark and returns it.
+  /// Measures the score for this benchmark by executing it repeatedly until
+  /// time minimum has been reached.
+  static double measureFor(void Function() f, int minimumMillis) =>
+      _measureForImpl(f, minimumMillis).score;
+
+  /// Measures the score for the benchmark and returns it.
   double measure() {
     setup();
     // Warmup for at least 100ms. Discard result.
-    measureFor(warmup, 100);
+    _measureForImpl(warmup, 100);
     // Run the benchmark for at least 2000ms.
-    var result = measureFor(exercise, 2000);
+    var result = _measureForImpl(exercise, _minimumMeasureDurationMillis);
     teardown();
-    return result;
+    return result.score;
   }
 
   void report() {
     emitter.emit(name, measure());
   }
 }
+
+class _Measurement {
+  final int elapsedMicros;
+  final int iterations;
+
+  _Measurement(this.elapsedMicros, this.iterations);
+
+  double get score => elapsedMicros / iterations;
+
+  int estimateIterationsNeededToReach({required int minimumMicros}) {
+    final elapsed = roundDownToMillisecond(elapsedMicros);
+    return elapsed == 0
+        ? iterations * 1000
+        : (iterations * math.max(minimumMicros / elapsed, 1.5)).ceil();
+  }
+
+  static int roundDownToMillisecond(int micros) => (micros ~/ 1000) * 1000;
+
+  @override
+  String toString() => '$elapsedMicros in $iterations iterations';
+}
diff --git a/pubspec.yaml b/pubspec.yaml
index 4c436c6..62675b9 100644
--- a/pubspec.yaml
+++ b/pubspec.yaml
@@ -1,5 +1,5 @@
 name: benchmark_harness
-version: 2.1.0
+version: 2.2.0
 description: The official Dart project benchmark harness.
 repository: https://github.com/dart-lang/benchmark_harness