From d3a6bd4437164cb66d63173a291c88f06102e116 Mon Sep 17 00:00:00 2001 From: JoJoJet Date: Tue, 7 Jun 2022 16:29:36 -0400 Subject: [PATCH 1/6] bench schedule dependency resolution No dependencies between systems * 100_schedule_noconstraints: [40.952 us 41.133 us 41.378 us] * 500_schedule_noconstraints: [112.43 us 112.62 us 112.95 us] 1000_schedule_noconstraints: [201.62 us 201.67 us 201.72 us] * 5000_schedule_noconstraints: [1.0837 ms 1.0994 ms 1.1203 ms] With dependencies * 100_schedule: [792.58 us 793.14 us 793.76 us] * 500_schedule: [16.538 ms 16.586 ms 16.632 ms] * 1000_schedule: [66.983 ms 67.225 ms 67.477 ms] * 5000_schedule: [3.1471 s 3.1800 s 3.2113 s] --- benches/Cargo.toml | 6 +++ benches/benches/bevy_ecs/schedule.rs | 74 ++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 benches/benches/bevy_ecs/schedule.rs diff --git a/benches/Cargo.toml b/benches/Cargo.toml index eeb69a2413e65..6ba2ad92809fe 100644 --- a/benches/Cargo.toml +++ b/benches/Cargo.toml @@ -11,6 +11,7 @@ glam = "0.20" rand = "0.8" rand_chacha = "0.3" criterion = { version = "0.3", features = ["html_reports"] } +bevy_app = { path = "../crates/bevy_app" } bevy_ecs = { path = "../crates/bevy_ecs" } bevy_reflect = { path = "../crates/bevy_reflect" } bevy_tasks = { path = "../crates/bevy_tasks" } @@ -46,6 +47,11 @@ name = "world_get" path = "benches/bevy_ecs/world_get.rs" harness = false +[[bench]] +name = "schedule" +path = "benches/bevy_ecs/schedule.rs" +harness = false + [[bench]] name = "reflect_list" path = "benches/bevy_reflect/list.rs" diff --git a/benches/benches/bevy_ecs/schedule.rs b/benches/benches/bevy_ecs/schedule.rs new file mode 100644 index 0000000000000..6eb7cc881cff3 --- /dev/null +++ b/benches/benches/bevy_ecs/schedule.rs @@ -0,0 +1,74 @@ +use bevy_app::App; +use bevy_ecs::prelude::*; +use criterion::{criterion_group, criterion_main, Criterion}; + +criterion_group!(benches, build_schedule); +criterion_main!(benches); + +fn build_schedule(criterion: &mut Criterion) { + let mut group = criterion.benchmark_group("build_schedule"); + group.warm_up_time(std::time::Duration::from_millis(500)); + group.measurement_time(std::time::Duration::from_secs(15)); + + // Benchmark graphs of different sizes. + for graph_size in [100, 500, 1000, 5000] { + // Basic benchmark without constraints. + group.bench_function(format!("{graph_size}_schedule_noconstraints"), |bencher| { + bencher.iter(|| { + let mut app = App::new(); + for i in 0..graph_size { + // empty system + fn sys() {} + app.add_system(sys); + } + }); + }); + // Benchmark with constraints. + group.bench_function(format!("{graph_size}_schedule"), |bencher| { + bencher.iter(|| { + // empty system + fn sys() {} + + // Use multiple different kinds of label; + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, SystemLabel)] + struct EvenLabel(usize); + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, SystemLabel)] + struct OddLabel(usize); + + // unique label + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, SystemLabel)] + struct NumLabel(usize); + + // Method: generate a set of `graph_size` systems which have a One True Ordering. + // Add system to the stage with full constraints. Hopefully this should be maximimally + // difficult for bevy to figure out. + + let mut app = App::new(); + for i in 0..graph_size { + let mut sys = if i % 2 == 0 { + sys.label(EvenLabel(i)) + } else { + sys.label(OddLabel(i)) + }; + for a in 0..i { + sys = if a % 2 == 0 { + sys.after(EvenLabel(a)) + } else { + sys.after(OddLabel(a)) + } + } + for b in 0..i { + sys = if b % 2 == 0 { + sys.before(EvenLabel(b)) + } else { + sys.before(OddLabel(b)) + } + } + app.add_system(sys); + } + }); + }); + } + + group.finish(); +} From 1af22765e97d22906e53e2154f79d37499b9d502 Mon Sep 17 00:00:00 2001 From: JoJoJet Date: Tue, 7 Jun 2022 18:09:02 -0400 Subject: [PATCH 2/6] actually run dependency resolver --- benches/benches/bevy_ecs/schedule.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/benches/benches/bevy_ecs/schedule.rs b/benches/benches/bevy_ecs/schedule.rs index 6eb7cc881cff3..171707bfe2e3e 100644 --- a/benches/benches/bevy_ecs/schedule.rs +++ b/benches/benches/bevy_ecs/schedule.rs @@ -16,7 +16,7 @@ fn build_schedule(criterion: &mut Criterion) { group.bench_function(format!("{graph_size}_schedule_noconstraints"), |bencher| { bencher.iter(|| { let mut app = App::new(); - for i in 0..graph_size { + for _ in 0..graph_size { // empty system fn sys() {} app.add_system(sys); @@ -57,7 +57,7 @@ fn build_schedule(criterion: &mut Criterion) { sys.after(OddLabel(a)) } } - for b in 0..i { + for b in i + 1..graph_size { sys = if b % 2 == 0 { sys.before(EvenLabel(b)) } else { @@ -66,6 +66,7 @@ fn build_schedule(criterion: &mut Criterion) { } app.add_system(sys); } + app.run(); }); }); } From a1f7f714463e86d568b47fcd68da95708e14a28c Mon Sep 17 00:00:00 2001 From: JoJoJet Date: Tue, 7 Jun 2022 18:28:56 -0400 Subject: [PATCH 3/6] lower the upper bound for graph sizes --- benches/benches/bevy_ecs/schedule.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benches/benches/bevy_ecs/schedule.rs b/benches/benches/bevy_ecs/schedule.rs index 171707bfe2e3e..2b56ea2d75383 100644 --- a/benches/benches/bevy_ecs/schedule.rs +++ b/benches/benches/bevy_ecs/schedule.rs @@ -11,7 +11,7 @@ fn build_schedule(criterion: &mut Criterion) { group.measurement_time(std::time::Duration::from_secs(15)); // Benchmark graphs of different sizes. - for graph_size in [100, 500, 1000, 5000] { + for graph_size in [100, 500, 1000] { // Basic benchmark without constraints. group.bench_function(format!("{graph_size}_schedule_noconstraints"), |bencher| { bencher.iter(|| { From 6301fd5567711f8bb707aaa142823842b70000a3 Mon Sep 17 00:00:00 2001 From: JoJoJet Date: Tue, 7 Jun 2022 22:24:02 -0400 Subject: [PATCH 4/6] move initialization code outside of the benchmark --- benches/benches/bevy_ecs/schedule.rs | 55 +++++++++++----------------- 1 file changed, 21 insertions(+), 34 deletions(-) diff --git a/benches/benches/bevy_ecs/schedule.rs b/benches/benches/bevy_ecs/schedule.rs index 2b56ea2d75383..2556bc5df62a0 100644 --- a/benches/benches/bevy_ecs/schedule.rs +++ b/benches/benches/bevy_ecs/schedule.rs @@ -6,10 +6,25 @@ criterion_group!(benches, build_schedule); criterion_main!(benches); fn build_schedule(criterion: &mut Criterion) { + // empty system + fn sys() {} + + // Use multiple different kinds of label to ensure that dynamic dispatch + // doesn't somehow get optimized away. + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, SystemLabel)] + struct NumLabel(usize); + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, SystemLabel)] + struct DummyLabel; + let mut group = criterion.benchmark_group("build_schedule"); group.warm_up_time(std::time::Duration::from_millis(500)); group.measurement_time(std::time::Duration::from_secs(15)); + // Method: generate a set of `graph_size` systems which have a One True Ordering. + // Add system to the stage with full constraints. Hopefully this should be maximimally + // difficult for bevy to figure out. + let labels: Vec<_> = (0..1000).map(NumLabel).collect(); + // Benchmark graphs of different sizes. for graph_size in [100, 500, 1000] { // Basic benchmark without constraints. @@ -17,52 +32,24 @@ fn build_schedule(criterion: &mut Criterion) { bencher.iter(|| { let mut app = App::new(); for _ in 0..graph_size { - // empty system - fn sys() {} app.add_system(sys); } + app.run(); }); }); + // Benchmark with constraints. group.bench_function(format!("{graph_size}_schedule"), |bencher| { bencher.iter(|| { - // empty system - fn sys() {} - - // Use multiple different kinds of label; - #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, SystemLabel)] - struct EvenLabel(usize); - #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, SystemLabel)] - struct OddLabel(usize); - - // unique label - #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, SystemLabel)] - struct NumLabel(usize); - - // Method: generate a set of `graph_size` systems which have a One True Ordering. - // Add system to the stage with full constraints. Hopefully this should be maximimally - // difficult for bevy to figure out. - let mut app = App::new(); + app.add_system(sys.label(DummyLabel)); for i in 0..graph_size { - let mut sys = if i % 2 == 0 { - sys.label(EvenLabel(i)) - } else { - sys.label(OddLabel(i)) - }; + let mut sys = sys.label(labels[i]).before(DummyLabel); for a in 0..i { - sys = if a % 2 == 0 { - sys.after(EvenLabel(a)) - } else { - sys.after(OddLabel(a)) - } + sys = sys.after(labels[a]); } for b in i + 1..graph_size { - sys = if b % 2 == 0 { - sys.before(EvenLabel(b)) - } else { - sys.before(OddLabel(b)) - } + sys = sys.before(labels[b]); } app.add_system(sys); } From 06b8dc68debf14273410d89ae00f3dc3f169fa89 Mon Sep 17 00:00:00 2001 From: JoJoJet Date: Wed, 8 Jun 2022 01:45:48 -0400 Subject: [PATCH 5/6] change `App.run` to `App.update` This should be equivalent with the default runner, but app.update() is much clearer that we're only running the first iteration. Co-authored-by: Alice Cecile --- benches/benches/bevy_ecs/schedule.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benches/benches/bevy_ecs/schedule.rs b/benches/benches/bevy_ecs/schedule.rs index 2556bc5df62a0..bd78efc0e6da2 100644 --- a/benches/benches/bevy_ecs/schedule.rs +++ b/benches/benches/bevy_ecs/schedule.rs @@ -34,7 +34,7 @@ fn build_schedule(criterion: &mut Criterion) { for _ in 0..graph_size { app.add_system(sys); } - app.run(); + app.update(); }); }); @@ -53,7 +53,7 @@ fn build_schedule(criterion: &mut Criterion) { } app.add_system(sys); } - app.run(); + app.update(); }); }); } From 3a5c231a5ba5ec0739a8a4d5dcfb2daa07fb87b9 Mon Sep 17 00:00:00 2001 From: JoJoJet Date: Wed, 8 Jun 2022 01:58:09 -0400 Subject: [PATCH 6/6] add comments and use better names --- benches/benches/bevy_ecs/schedule.rs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/benches/benches/bevy_ecs/schedule.rs b/benches/benches/bevy_ecs/schedule.rs index bd78efc0e6da2..0b8916e98b1e8 100644 --- a/benches/benches/bevy_ecs/schedule.rs +++ b/benches/benches/bevy_ecs/schedule.rs @@ -7,7 +7,7 @@ criterion_main!(benches); fn build_schedule(criterion: &mut Criterion) { // empty system - fn sys() {} + fn empty_system() {} // Use multiple different kinds of label to ensure that dynamic dispatch // doesn't somehow get optimized away. @@ -32,7 +32,7 @@ fn build_schedule(criterion: &mut Criterion) { bencher.iter(|| { let mut app = App::new(); for _ in 0..graph_size { - app.add_system(sys); + app.add_system(empty_system); } app.update(); }); @@ -42,9 +42,12 @@ fn build_schedule(criterion: &mut Criterion) { group.bench_function(format!("{graph_size}_schedule"), |bencher| { bencher.iter(|| { let mut app = App::new(); - app.add_system(sys.label(DummyLabel)); + app.add_system(empty_system.label(DummyLabel)); + + // Build a fully-connected dependency graph describing the One True Ordering. + // Not particularly realistic but this can be refined later. for i in 0..graph_size { - let mut sys = sys.label(labels[i]).before(DummyLabel); + let mut sys = empty_system.label(labels[i]).before(DummyLabel); for a in 0..i { sys = sys.after(labels[a]); } @@ -53,6 +56,10 @@ fn build_schedule(criterion: &mut Criterion) { } app.add_system(sys); } + // Run the app for a single frame. + // This is necessary since dependency resolution does not occur until the game runs. + // FIXME: Running the game clutters up the benchmarks, so ideally we'd be + // able to benchmark the dependency resolution directly. app.update(); }); });