Merge branch '19-benchmark-runInTxAsync'

Author: greenrobot-team

benchmark/analysis_options.yaml

@@ -0,0 +1,2 @@
+# https://dart.dev/guides/language/analysis-options
+include: package:lints/recommended.yaml

benchmark/bin/write.dart

@@ -1,4 +1,7 @@
+import 'dart:async';
+
 import 'package:objectbox_benchmark/benchmark.dart';
+import 'package:objectbox_benchmark/model.dart';
 import 'package:objectbox_benchmark/objectbox.g.dart';
 
 const count = 10000;
@@ -11,6 +14,8 @@ void main() async {
   await PutAsync2().report();
   await PutAsync3().report();
   await PutQueued().report();
+  await RunInTx().report();
+  await RunInTxAsync().report();
 }
 
 class Put extends DbBenchmark {
@@ -93,3 +98,29 @@ class PutQueued extends DbBenchmark {
     store.awaitAsyncSubmitted();
   }
 }
+
+class RunInTx extends DbBenchmark {
+  final items = prepareTestEntities(count, assignedIds: true);
+
+  RunInTx() : super('$RunInTx');
+
+  @override
+  void runIteration(int i) {
+    store.runInTransaction(TxMode.write, () => items.forEach(box.put));
+  }
+}
+
+class RunInTxAsync extends DbBenchmark {
+  final items = prepareTestEntities(count, assignedIds: true);
+
+  RunInTxAsync() : super('$RunInTxAsync');
+
+  @override
+  Future<void> runIteration(int iteration) {
+    return store.runInTransactionAsync(
+        TxMode.write,
+        (Store store, List<TestEntity> items) =>
+            items.forEach(store.box<TestEntity>().put),
+        items);
+  }
+}

benchmark/lib/benchmark.dart

@@ -1,3 +1,4 @@
+import 'dart:async';
 import 'dart:io';
 
 import 'package:meta/meta.dart';
@@ -12,40 +13,71 @@ class Benchmark {
   final watch = Stopwatch();
   final Emitter emitter;
 
+  /// Create a benchmark with the given [name], starts measuring total run time.
+  ///
+  /// ```dart
+  /// await Benchmark('Name', coefficient: 1 / count).report();
+  /// ```
+  ///
+  /// Call [report] on this to await results.
+  ///
+  /// Runs the [runIteration] function at least [iterations] times, defaults to
+  /// 1. If possible, leave this at the default. The benchmark is automatically
+  /// re-run a) to warm up until at least 100 ms have passed (e.g. to avoid
+  /// caching skewing results) and b) to measure until at least 2 seconds have
+  /// passed. This is similar to how Dart's benchmark_harness and various other
+  /// benchmarking libraries in other languages work.
+  ///
+  /// Set a fraction in [coefficient] to multiply the measured value of a run
+  /// with, defaults to 1. Use this if a run calls a to be measured function
+  /// multiple times (e.g. `1 / times`) to get the duration of a single call.
+  ///
+  /// Results are printed to the command line.
   Benchmark(this.name, {this.iterations = 1, this.coefficient = 1})
       : emitter = Emitter(iterations, coefficient) {
     print('-------------------------------------------------------------');
-    print('$name(iterations):       ' +
+    print('$name(iterations):          ' +
         Emitter._format(iterations.toDouble(), decimalPoints: 0));
-    print('$name(count):            ' +
-        Emitter._format(iterations / coefficient));
+    print('$name(unit count):          ' +
+        Emitter._format(iterations / coefficient, decimalPoints: 0));
     // Measure the total time of the test - if it's too high, you should
     // decrease the number of iterations. Expected time is between 2 and 3 sec.
     watch.start();
   }
 
-  // Not measured setup code executed prior to the benchmark runs.
+  /// Not measured setup code executed prior to [run] getting called.
   void setup() {}
 
+  /// Called after all [run] calls have completed, measures total time of the
+  /// benchmark.
+  ///
+  /// A method overriding this must call this.
   @mustCallSuper
   void teardown() {
     final millis = watch.elapsedMilliseconds;
     final color = millis > 3000 ? '\x1B[31m' : '';
-    print('$name(total time taken): '
+    print('$name(benchmark run time):     '
         '$color${Emitter._format(millis.toDouble(), suffix: ' ms')}\x1B[0m');
   }
 
+  /// Calls [runIteration] [iterations] of times.
   Future<void> run() async {
-    for (var i = 0; i < iterations; i++) runIteration(i);
+    for (var i = 0; i < iterations; i++) {
+      final result = runIteration(i);
+      if (result is Future) {
+        await result;
+      }
+    }
     return Future.value();
   }
 
-  void runIteration(int iteration) {
+  /// A single test iteration, given [iteration] index starting from 0.
+  FutureOr<void> runIteration(int iteration) {
     throw UnimplementedError('Please override runIteration() or run()');
   }
 
-  // Runs [f] for at least [minimumMillis] milliseconds.
-  static Future<double> _measureFor(Function f, int minimumMillis) async {
+  /// Runs [f] for at least [minimumMillis] milliseconds.
+  Future<double> _measureFor(Function f, int minimumMillis, bool warmUp) async {
     final minimumMicros = minimumMillis * 1000;
     var iter = 0;
     final watch = Stopwatch()..start();
@@ -55,21 +87,38 @@ class Benchmark {
       elapsed = watch.elapsedMicroseconds;
       iter++;
     }
+    if (!warmUp) {
+      // Print how often f had to be re-run to reach minimum run time.
+      final reruns = iter - 1;
+      print('$name(re-runs):             '
+          '${Emitter._format(reruns.toDouble(), decimalPoints: 0)}');
+    }
     return elapsed / iter;
   }
 
-  // Measures the score for the benchmark and returns it.
+  /// Measures the score for the benchmark and returns it.
+  ///
+  /// See [report] for details.
   @nonVirtual
   Future<double> _measure() async {
     setup();
     // Warmup for at least 100ms. Discard result.
-    await _measureFor(run, 100);
+    await _measureFor(run, 100, true);
     // Run the benchmark for at least 2000ms.
-    var result = await _measureFor(run, 2000);
+    var result = await _measureFor(run, 2000, false);
     teardown();
     return result;
   }
 
+  /// Starts the benchmark and waits for the result.
+  ///
+  /// - Calls [setup], then
+  /// - repeatedly calls [run] for at least 100 ms to warm up to avoid effects
+  /// e.g. due to caching,
+  /// - then calls [run] repeatedly until at least 2000 ms have passed to ensure
+  /// stable results and collects the average elapsed time of a call (if run
+  /// multiple times), then
+  /// - calls [teardown] and returns the result.
   @nonVirtual
   Future<void> report() async {
     emitter.emit(name, await _measure());
@@ -87,12 +136,14 @@ class Emitter {
   void emit(String testName, double value) {
     final timePerIter = value / iterations;
     final timePerUnit = timePerIter * coefficient;
-    print(
-        '$testName(Single iteration): ${_format(timePerIter, suffix: ' us')}');
-    print(
-        '$testName(Runtime per unit): ${_format(timePerUnit, suffix: ' us')}');
-    print('$testName(Runs per second):  ${_format(usInSec / timePerIter)}');
-    print('$testName(Units per second): ${_format(usInSec / timePerUnit)}');
+    print('$testName(single iteration):       '
+        '${_format(timePerIter, suffix: ' us')}');
+    print('$testName(per unit):               '
+        '${_format(timePerUnit, suffix: ' us')}');
+    print('$testName(iterations / second): '
+        '${_format(usInSec / timePerIter)}');
+    print('$testName(units / second):      '
+        '${_format(usInSec / timePerUnit)}');
   }
 
   // Simple number formatting, maybe use a lib?

benchmark/pubspec.yaml

@@ -10,6 +10,7 @@ dependencies:
   meta: ^1.3.0
 
 dev_dependencies:
+  lints: ^1.0.1
   objectbox_generator: any
   build_runner: any