update benchmarks

Author: tusharmath

README.md

@@ -8,8 +8,8 @@ Based on the benchmark results, pattern matching in Rust is significantly faster
 
 ```
 cargo bench
-   Compiling rust-dynamic-dispatch-performance v0.1.0 (/Users/tushar/Documents/Projects/temp/rust-dynamic-dispatch-performance)
-    Finished `bench` profile [optimized] target(s) in 1.66s
+   Compiling rust-dynamic-dispatch-performance v0.1.0 (/Users/tushar/Documents/Projects/rust-dynamic-dispatch-performance)
+    Finished `bench` profile [optimized] target(s) in 1.20s
      Running unittests src/main.rs (target/release/deps/rust_dynamic_dispatch_performance-5c61e2e8a62037f3)
 
 running 0 tests
@@ -18,21 +18,63 @@ test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; fini
 
      Running benches/benchmarks.rs (target/release/deps/benchmarks-738a32214e4b14ea)
 Dispatch vs Matching/Dynamic Dispatch
-                        time:   [22.946 ns 25.470 ns 28.807 ns]
-                        change: [+3.7542% +10.309% +18.945%] (p = 0.00 < 0.05)
-                        Performance has regressed.
-Found 16 outliers among 100 measurements (16.00%)
+                        time:   [2.5226 ns 2.5325 ns 2.5420 ns]
+                        change: [+0.3857% +0.8665% +1.3628%] (p = 0.00 < 0.05)
+                        Change within noise threshold.
+Found 5 outliers among 100 measurements (5.00%)
   4 (4.00%) high mild
-  12 (12.00%) high severe
+  1 (1.00%) high severe
 Dispatch vs Matching/Pattern Matching
-                        time:   [317.89 ps 320.03 ps 322.71 ps]
-                        change: [-5.1626% -2.1683% +0.0642%] (p = 0.12 > 0.05)
-                        No change in performance detected.
-Found 9 outliers among 100 measurements (9.00%)
-  3 (3.00%) high mild
+                        time:   [375.26 ps 377.67 ps 380.79 ps]
+                        change: [-76.135% -75.895% -75.626%] (p = 0.00 < 0.05)
+                        Performance has improved.
+Found 10 outliers among 100 measurements (10.00%)
+  4 (4.00%) high mild
   6 (6.00%) high severe
 ```
 
+## Update 1
+
+We were curious to see why pattern matching was significantly faster compared to dynamic dispatch, so we posted benchmarks on [Reddit](https://www.reddit.com/r/rust/comments/1cx7qvi/performance_pattern_matching_vs_dynamic_dispatch/). We received interesting feedback about the benchmark setup, such as the unnecessary heap allocation for dynamic dispatch. After addressing this, the performance difference was marginal.
+
+Digging deeper, especially after this [GitHub comment](https://github.com/tailcallhq/rust-benchmarks/issues/2) and further analysis on [Rust Compiler Explorer](https://rust.godbolt.org/), we discovered that the compiler optimizes the generated machine code based on the seed value set in the variable `output`. When set to `0`, the additions and multiplications would always return `0` regardless of the contents of the passed array. This optimization resulted in the pattern matching case reducing to (an optimization that was unavailable to the `dyn` implementation):
+
+```asm
+xor eax, eax
+ret
+```
+
+After correcting this by setting the seed value to `1`, the benchmarks showed a significant improvement, making pattern matching only **2.7x faster** than dynamic dispatch.
+
+```
+cargo bench
+   Compiling rust-dynamic-dispatch-performance v0.1.0 (/Users/tushar/Documents/Projects/rust-dynamic-dispatch-performance)
+    Finished `bench` profile [optimized] target(s) in 1.36s
+     Running unittests src/main.rs (target/release/deps/rust_dynamic_dispatch_performance-5c61e2e8a62037f3)
+
+running 0 tests
+
+test result: ok. 0 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.00s
+
+     Running benches/benchmarks.rs (target/release/deps/benchmarks-738a32214e4b14ea)
+Benchmarking Dispatch vs Matching/Dynamic Dispatch: Collecting 100 samples in
+Dispatch vs Matching/Dynamic Dispatch
+                        time:   [2.5552 ns 2.5824 ns 2.6199 ns]
+                        change: [+2.0878% +8.3408% +15.931%] (p = 0.01 < 0.05)
+                        Performance has regressed.
+Found 20 outliers among 100 measurements (20.00%)
+  1 (1.00%) high mild
+  19 (19.00%) high severe
+Benchmarking Dispatch vs Matching/Pattern Matching: Collecting 100 samples in
+Dispatch vs Matching/Pattern Matching
+                        time:   [947.52 ps 949.81 ps 952.50 ps]
+                        change: [+147.37% +149.83% +151.38%] (p = 0.00 < 0.05)
+                        Performance has regressed.
+Found 2 outliers among 100 measurements (2.00%)
+  1 (1.00%) high mild
+  1 (1.00%) high severe
+```
+
 Happy coding! 🦀
 
 PS: Feel free to raise a PR if you think there is something wrong with the way the benchmarks are designed.

benches/benchmarks.rs

@@ -1,5 +1,4 @@
 use criterion::{black_box, criterion_group, criterion_main, Criterion};
-use std::time::Duration;
 
 trait DoSomething {
     fn do_it(&self, i: i32) -> i32;
@@ -27,16 +26,14 @@ enum Action {
     Two(ActionTwo),
 }
 
-fn dynamic_dispatch(actions: &Vec<Box<dyn DoSomething>>) -> i32 {
-    let mut output = 0;
+fn dynamic_dispatch(actions: &Vec<Box<dyn DoSomething>>, mut output: i32) -> i32 {
     for action in actions {
         output = output + action.do_it(output);
     }
     output
 }
 
-fn pattern_matching(actions: &Vec<Action>) -> i32 {
-    let mut output = 0;
+fn pattern_matching(actions: &Vec<Action>, mut output: i32) -> i32 {
     for action in actions {
         match action {
             Action::One(a) => output = output + a.do_it(output),
@@ -52,18 +49,16 @@ fn benchmark(c: &mut Criterion) {
     let dynamic_actions: Vec<Box<dyn DoSomething>> = vec![Box::new(ActionOne), Box::new(ActionTwo)];
     let static_actions: Vec<Action> = vec![Action::One(ActionOne), Action::Two(ActionTwo)];
 
-    group.measurement_time(Duration::new(10, 0));
-
     group.bench_function("Dynamic Dispatch", |b| {
         b.iter(|| {
-            let output = dynamic_dispatch(&dynamic_actions);
+            let output = dynamic_dispatch(&dynamic_actions, 1);
             black_box(output)
         })
     });
 
     group.bench_function("Pattern Matching", |b| {
         b.iter(|| {
-            let output = pattern_matching(&static_actions);
+            let output = pattern_matching(&static_actions, 1);
             black_box(output)
         })
     });