diff --git a/Cargo.lock b/Cargo.lock index 7cd43d40..60ccb8f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -617,9 +617,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.7.2" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97" +checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" [[package]] name = "codspeed" @@ -652,10 +652,12 @@ name = "codspeed-criterion-compat" version = "4.0.0" dependencies = [ "async-std", + "clap", "codspeed", "codspeed-criterion-compat-walltime", "colored", "futures", + "regex", "smol 2.0.0", "tokio", ] @@ -698,9 +700,11 @@ dependencies = [ name = "codspeed-divan-compat" version = "4.0.0" dependencies = [ + "clap", "codspeed", "codspeed-divan-compat-macros", "codspeed-divan-compat-walltime", + "regex", ] [[package]] @@ -1608,9 +1612,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.5" +version = "1.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" +checksum = "8b5288124840bee7b386bc413c487869b360b2b4ec421ea56425128692f2a82c" dependencies = [ "aho-corasick", "memchr", @@ -1620,9 +1624,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.7" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +checksum = "833eb9ce86d40ef33cb1306d8accf7bc8ec2bfea4355cbdebb3df68b40925cad" dependencies = [ "aho-corasick", "memchr", @@ -1637,9 +1641,9 @@ checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" [[package]] name = "regex-syntax" -version = "0.8.4" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +checksum = "caf4aa5b0f434c91fe5c7f1ecb6a5ece2130b02ad2a590589dda5146df959001" [[package]] name = "rustc-demangle" diff --git a/crates/cargo-codspeed/tests/simple-criterion.rs b/crates/cargo-codspeed/tests/simple-criterion.rs index dc4b7dd7..35141b5f 100644 --- a/crates/cargo-codspeed/tests/simple-criterion.rs +++ b/crates/cargo-codspeed/tests/simple-criterion.rs @@ -1,10 +1,12 @@ use assert_cmd::assert::OutputAssertExt; -use predicates::str::contains; +use predicates::{prelude::PredicateBooleanExt, str::contains}; mod helpers; use helpers::*; const DIR: &str = "tests/simple-criterion.in"; +const FIB_BENCH_NAME: &str = "fib 20"; +const BUBBLE_SORT_BENCH_NAME: &str = "bubble sort"; #[test] fn test_criterion_run_without_build() { @@ -66,8 +68,8 @@ fn test_criterion_build_and_run_single() { .args(["--bench", "another_criterion_example"]) .assert() .success() - .stderr(contains("Finished running 1 benchmark suite(s)")) - .stderr(contains("another_criterion_example")); + .stdout(contains("another_criterion_example")) + .stderr(contains("Finished running 1 benchmark suite(s)")); teardown(dir); } @@ -80,20 +82,43 @@ fn test_criterion_build_and_run_filtered_by_name() { .arg("fib 20") .assert() .success() + .stdout(contains(FIB_BENCH_NAME)) + .stdout(contains(BUBBLE_SORT_BENCH_NAME).not()) + .stderr(contains("Finished running 2 benchmark suite(s)")); + cargo_codspeed(&dir) + .arg("run") + .arg("bu.*le") + .assert() + .success() + .stdout(contains(FIB_BENCH_NAME).not()) + .stdout(contains(BUBBLE_SORT_BENCH_NAME)) .stderr(contains("Finished running 2 benchmark suite(s)")); teardown(dir); } #[test] -fn test_criterion_build_and_run_filtered_by_partial_name() { +fn test_criterion_build_and_run_filtered_by_name_single() { let dir = setup(DIR, Project::Simple); cargo_codspeed(&dir).arg("build").assert().success(); cargo_codspeed(&dir) .arg("run") - .arg("bubble") + .arg("bu.*le") + .args(["--bench", "criterion_example"]) .assert() .success() - .stderr(contains("Finished running 2 benchmark suite(s)")); + .stdout(contains(BUBBLE_SORT_BENCH_NAME).not()) // We are filtering with a name that is not in the selected benchmark + .stdout(contains(FIB_BENCH_NAME).not()) + .stderr(contains("Finished running 1 benchmark suite(s)")); + + cargo_codspeed(&dir) + .arg("run") + .arg("fib") + .args(["--bench", "criterion_example"]) + .assert() + .success() + .stdout(contains(FIB_BENCH_NAME)) + .stdout(contains(BUBBLE_SORT_BENCH_NAME).not()) + .stderr(contains("Finished running 1 benchmark suite(s)")); teardown(dir); } diff --git a/crates/cargo-codspeed/tests/simple-divan.rs b/crates/cargo-codspeed/tests/simple-divan.rs index e7de4179..057a5686 100644 --- a/crates/cargo-codspeed/tests/simple-divan.rs +++ b/crates/cargo-codspeed/tests/simple-divan.rs @@ -1,10 +1,12 @@ use assert_cmd::assert::OutputAssertExt; -use predicates::str::contains; +use predicates::{prelude::PredicateBooleanExt, str::contains}; mod helpers; use helpers::*; const DIR: &str = "tests/simple-divan.in"; +const FIB_BENCH_NAME: &str = "fib_20"; +const BUBBLE_SORT_BENCH_NAME: &str = "bubble_sort_bench"; #[test] fn test_divan_run_without_build() { @@ -80,20 +82,42 @@ fn test_divan_build_and_run_filtered_by_name() { .arg("fib_20") .assert() .success() + .stdout(contains(FIB_BENCH_NAME)) + .stdout(contains(BUBBLE_SORT_BENCH_NAME).not()) + .stderr(contains("Finished running 2 benchmark suite(s)")); + cargo_codspeed(&dir) + .arg("run") + .arg("bu.*le_sort") + .assert() + .success() + .stdout(contains(FIB_BENCH_NAME).not()) + .stdout(contains(BUBBLE_SORT_BENCH_NAME)) .stderr(contains("Finished running 2 benchmark suite(s)")); teardown(dir); } #[test] -fn test_divan_build_and_run_filtered_by_partial_name() { +fn test_divan_build_and_run_filtered_by_name_single() { let dir = setup(DIR, Project::Simple); cargo_codspeed(&dir).arg("build").assert().success(); cargo_codspeed(&dir) .arg("run") - .arg("bubble_sort") + .arg("bu.*le_sort") + .args(["--bench", "divan_example"]) .assert() .success() - .stderr(contains("Finished running 2 benchmark suite(s)")); + .stdout(contains(FIB_BENCH_NAME).not()) + .stdout(contains(BUBBLE_SORT_BENCH_NAME).not()) // We are filtering with a name that is not in the selected benchmark + .stderr(contains("Finished running 1 benchmark suite(s)")); + cargo_codspeed(&dir) + .arg("run") + .arg("fib") + .args(["--bench", "divan_example"]) + .assert() + .success() + .stdout(contains(FIB_BENCH_NAME)) + .stdout(contains(BUBBLE_SORT_BENCH_NAME).not()) + .stderr(contains("Finished running 1 benchmark suite(s)")); teardown(dir); } diff --git a/crates/criterion_compat/Cargo.toml b/crates/criterion_compat/Cargo.toml index 36472ff8..8465fd27 100644 --- a/crates/criterion_compat/Cargo.toml +++ b/crates/criterion_compat/Cargo.toml @@ -20,6 +20,8 @@ keywords = ["codspeed", "benchmark", "criterion"] criterion = { package = "codspeed-criterion-compat-walltime", path = "./criterion_fork", version = "=4.0.0", default-features = false } codspeed = { path = "../codspeed", version = "=4.0.0" } colored = "2.1.0" +clap = { version = "4", default-features = false, features = ["std"] } +regex = { version = "1.5", default-features = false, features = ["std"] } futures = { version = "0.3", default-features = false, optional = true } smol = { version = "2.0", default-features = false, optional = true } diff --git a/crates/criterion_compat/src/compat/criterion.rs b/crates/criterion_compat/src/compat/criterion.rs index a4a9d409..a8b85c1a 100644 --- a/crates/criterion_compat/src/compat/criterion.rs +++ b/crates/criterion_compat/src/compat/criterion.rs @@ -6,13 +6,15 @@ use criterion::{ profiler::Profiler, PlottingBackend, }; +use regex::Regex; -use crate::{Bencher, BenchmarkGroup, BenchmarkId}; +use crate::{Bencher, BenchmarkFilter, BenchmarkGroup, BenchmarkId}; pub struct Criterion { pub codspeed: Option>>, pub current_file: String, pub macro_group: String, + pub filter: BenchmarkFilter, phantom: PhantomData<*const M>, } @@ -23,19 +25,57 @@ impl Criterion { "Harness: codspeed-criterion-compat v{}", env!("CARGO_PKG_VERSION"), ); + + // Parse CLI arguments to extract filter + let filter = Self::parse_filter(); + Criterion { codspeed: Some(Rc::new(RefCell::new(CodSpeed::new()))), current_file: String::new(), macro_group: String::new(), + filter, phantom: PhantomData, } } + fn parse_filter() -> BenchmarkFilter { + use clap::{Arg, Command}; + + let matches = Command::new("Criterion Benchmark") + .arg( + Arg::new("FILTER") + .help("Skip benchmarks whose names do not contain FILTER.") + .index(1), + ) + .arg( + Arg::new("exact") + .long("exact") + .num_args(0) + .help("Run benchmarks that exactly match the provided filter"), + ) + .get_matches(); + + if let Some(filter) = matches.get_one::("FILTER") { + if matches.get_flag("exact") { + BenchmarkFilter::Exact(filter.to_owned()) + } else { + let regex = Regex::new(filter).unwrap_or_else(|err| { + eprintln!("Unable to parse '{filter}' as a regular expression: {err}"); + std::process::exit(1); + }); + BenchmarkFilter::Regex(regex) + } + } else { + BenchmarkFilter::AcceptAll + } + } + pub fn with_patched_measurement(&mut self, _: Criterion) -> Criterion { Criterion { codspeed: self.codspeed.clone(), current_file: self.current_file.clone(), macro_group: self.macro_group.clone(), + filter: self.filter.clone(), phantom: PhantomData, } } @@ -92,6 +132,7 @@ impl Default for Criterion { codspeed: None, current_file: String::new(), macro_group: String::new(), + filter: BenchmarkFilter::AcceptAll, phantom: PhantomData, } } @@ -104,6 +145,7 @@ impl Criterion { codspeed: self.codspeed, current_file: self.current_file, macro_group: self.macro_group, + filter: self.filter, phantom: PhantomData::<*const M2>, } } diff --git a/crates/criterion_compat/src/compat/filter.rs b/crates/criterion_compat/src/compat/filter.rs new file mode 100644 index 00000000..2a10e41f --- /dev/null +++ b/crates/criterion_compat/src/compat/filter.rs @@ -0,0 +1,26 @@ +use regex::Regex; + +/// Benchmark filtering support - re-exported from criterion fork. +#[derive(Clone, Debug)] +pub enum BenchmarkFilter { + /// Run all benchmarks. + AcceptAll, + /// Run benchmarks matching this regex. + Regex(Regex), + /// Run the benchmark matching this string exactly. + Exact(String), + /// Do not run any benchmarks. + RejectAll, +} + +impl BenchmarkFilter { + /// Returns true if a string matches this filter. + pub fn is_match(&self, id: &str) -> bool { + match self { + Self::AcceptAll => true, + Self::Regex(r) => r.is_match(id), + Self::Exact(e) => e == id, + Self::RejectAll => false, + } + } +} diff --git a/crates/criterion_compat/src/compat/group.rs b/crates/criterion_compat/src/compat/group.rs index 75ae283d..90e52877 100644 --- a/crates/criterion_compat/src/compat/group.rs +++ b/crates/criterion_compat/src/compat/group.rs @@ -5,7 +5,7 @@ use codspeed::{codspeed::CodSpeed, utils::get_git_relative_path}; use criterion::measurement::WallTime; use criterion::{measurement::Measurement, PlotConfiguration, SamplingMode, Throughput}; -use crate::{Bencher, Criterion}; +use crate::{Bencher, BenchmarkFilter, Criterion}; /// Deprecated: using the default measurement will be removed in the next major version. /// Defaulting to WallTime differs from the original BenchmarkGroup implementation but avoids creating a breaking change @@ -14,6 +14,7 @@ pub struct BenchmarkGroup<'a, M: Measurement = WallTime> { current_file: String, macro_group: String, group_name: String, + filter: BenchmarkFilter, _marker: PhantomData<&'a M>, } @@ -29,6 +30,7 @@ impl<'a, M: Measurement> BenchmarkGroup<'a, M> { current_file: criterion.current_file.clone(), macro_group: criterion.macro_group.clone(), group_name, + filter: criterion.filter.clone(), _marker: PhantomData, } } @@ -73,6 +75,12 @@ impl<'a, M: Measurement> BenchmarkGroup<'a, M> { if let Some(parameter) = id.parameter { uri = format!("{uri}[{parameter}]"); } + + // Apply filter - skip benchmark if it doesn't match + if !self.filter.is_match(&uri) { + return; + } + let mut codspeed = self.codspeed.borrow_mut(); let mut b = Bencher::new(&mut codspeed, uri); f(&mut b, input); diff --git a/crates/criterion_compat/src/compat/mod.rs b/crates/criterion_compat/src/compat/mod.rs index cc7f709e..f09eb761 100644 --- a/crates/criterion_compat/src/compat/mod.rs +++ b/crates/criterion_compat/src/compat/mod.rs @@ -1,8 +1,10 @@ mod bencher; mod criterion; +mod filter; mod group; mod macros; pub use self::bencher::*; pub use self::criterion::*; +pub use self::filter::*; pub use self::group::*; diff --git a/crates/divan_compat/Cargo.toml b/crates/divan_compat/Cargo.toml index d38d68d3..cf383ebd 100644 --- a/crates/divan_compat/Cargo.toml +++ b/crates/divan_compat/Cargo.toml @@ -21,6 +21,8 @@ keywords = ["codspeed", "benchmark", "divan"] codspeed = { path = "../codspeed", version = "=4.0.0" } divan = { package = "codspeed-divan-compat-walltime", path = "./divan_fork", version = "=4.0.0" } codspeed-divan-compat-macros = { version = "=4.0.0", path = './macros' } +regex = "1.11.3" +clap = { version = "4", default-features = false, features = ["std", "env"] } [[bench]] name = "basic_example" diff --git a/crates/divan_compat/src/compat/cli.rs b/crates/divan_compat/src/compat/cli.rs new file mode 100644 index 00000000..02e04a6a --- /dev/null +++ b/crates/divan_compat/src/compat/cli.rs @@ -0,0 +1,20 @@ +use clap::{Arg, ArgAction, Command}; + +pub(crate) fn command() -> Command { + fn option(name: &'static str) -> Arg { + Arg::new(name).long(name) + } + + fn flag(name: &'static str) -> Arg { + option(name).action(ArgAction::SetTrue) + } + + Command::new("divan") + .arg( + Arg::new("filter") + .value_name("FILTER") + .help("Only run benchmarks whose names match this pattern") + .action(ArgAction::Append), + ) + .arg(flag("exact").help("Filter benchmarks by exact name rather than by pattern")) +} diff --git a/crates/divan_compat/src/compat/config.rs b/crates/divan_compat/src/compat/config.rs new file mode 100644 index 00000000..fd547735 --- /dev/null +++ b/crates/divan_compat/src/compat/config.rs @@ -0,0 +1,17 @@ +use regex::Regex; + +/// Filters which benchmark to run based on name. +pub(crate) enum Filter { + Regex(Regex), + Exact(String), +} + +impl Filter { + /// Returns `true` if a string matches this filter. + pub fn is_match(&self, s: &str) -> bool { + match self { + Self::Regex(r) => r.is_match(s), + Self::Exact(e) => e == s, + } + } +} diff --git a/crates/divan_compat/src/compat/mod.rs b/crates/divan_compat/src/compat/mod.rs index 479bc59b..99b695b6 100644 --- a/crates/divan_compat/src/compat/mod.rs +++ b/crates/divan_compat/src/compat/mod.rs @@ -13,15 +13,18 @@ pub mod __private { } mod bench; +mod cli; +mod config; mod entry; mod uri; mod util; -use std::{cell::RefCell, rc::Rc}; - pub use bench::*; use codspeed::codspeed::CodSpeed; +use config::Filter; use entry::AnyBenchEntry; +use regex::Regex; +use std::{cell::RefCell, rc::Rc}; pub fn main() { // Outlined steps of original divan::main and their equivalent in codspeed instrumented mode @@ -46,8 +49,37 @@ pub fn main() { // codspeed URI from entry metadata directly. // 3. Filtering - // We do not support finer filtering that bench targets for now, do nothing here, bench - // filtering is managed by the `cargo-codspeed` wrappers before we reach this point. + let should_run_benchmark_from_filters = { + let mut command = cli::command(); + let matches = command.get_matches_mut(); + let is_exact = matches.get_flag("exact"); + + let parse_filter = |filter: &String| { + if is_exact { + Filter::Exact(filter.to_owned()) + } else { + match Regex::new(filter) { + Ok(r) => Filter::Regex(r), + Err(error) => { + let kind = clap::error::ErrorKind::ValueValidation; + command.error(kind, error).exit(); + } + } + } + }; + + let filters: Option> = matches + .get_many::("filter") + .map(|arg_filters| arg_filters.map(parse_filter).collect()); + + move |uri: &str| { + if let Some(filters) = filters.as_ref() { + filters.iter().any(|filter| filter.is_match(uri)) + } else { + true + } + } + }; // 4. Scan the tree and execute benchmarks let codspeed = Rc::new(RefCell::new(CodSpeed::new())); @@ -66,6 +98,10 @@ pub fn main() { entry::BenchEntryRunner::Plain(bench_fn) => { let uri = uri::generate(&entry, entry.display_name()); + if !should_run_benchmark_from_filters(&uri) { + continue; + } + bench_fn(bench::Bencher::new(&codspeed, uri)); } entry::BenchEntryRunner::Args(bench_runner) => { @@ -74,6 +110,10 @@ pub fn main() { for (arg_index, arg_name) in bench_runner.arg_names().iter().enumerate() { let uri = uri::generate(&entry, arg_name); + if !should_run_benchmark_from_filters(&uri) { + continue; + } + let bencher = bench::Bencher::new(&codspeed, uri); bench_runner.bench(bencher, arg_index);