From 6fcffb35422a5805082aaf672db2ac517accb32d Mon Sep 17 00:00:00 2001 From: Simon Date: Mon, 22 Sep 2025 10:03:40 -0700 Subject: [PATCH 1/3] Add basic benchmark --- Cargo.lock | 207 +++++++++++++++++- aws_secretsmanager_caching/Cargo.toml | 10 +- .../benches/benchmark.rs | 117 ++++++++++ 3 files changed, 328 insertions(+), 6 deletions(-) create mode 100644 aws_secretsmanager_caching/benches/benchmark.rs diff --git a/Cargo.lock b/Cargo.lock index 68ab71c..ad7c455 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -53,6 +53,18 @@ dependencies = [ "libc", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstyle" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "862ed96ca487e809f1c8e5a8447f6ee2cf102f846893800b20cebdf541fc6bbd" + [[package]] name = "anyhow" version = "1.0.100" @@ -419,13 +431,15 @@ dependencies = [ ] [[package]] -name = "aws-smithy-mocks-experimental" -version = "0.2.4" +name = "aws-smithy-mocks" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ce8a35535906a8a9ceadbe7ff70ae8686a36f7df03b288b1256c084a5c45c69" +checksum = "178b1ad961028a58d48ce857f86ffbd5233a4b7e2c7b56d026fb1c1afe46696e" dependencies = [ + "aws-smithy-http-client", "aws-smithy-runtime-api", "aws-smithy-types", + "http 1.3.1", ] [[package]] @@ -590,13 +604,15 @@ version = "2.0.0" dependencies = [ "aws-config", "aws-sdk-secretsmanager", - "aws-smithy-mocks-experimental", + "aws-smithy-mocks", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", + "criterion", "http 0.2.12", "linked-hash-map", "log", + "rand", "rustls 0.23.32", "serde", "serde_json", @@ -735,6 +751,12 @@ dependencies = [ "either", ] +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cbor-diag" version = "0.1.12" @@ -831,6 +853,31 @@ dependencies = [ "libloading", ] +[[package]] +name = "clap" +version = "4.5.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2134bb3ea021b78629caa971416385309e0131b351b25e01dc16fb54e1b5fae" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.5.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2ba64afa3c0a6df7fa517765e31314e983f51dda798ffba27b988194fb65dc9" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675" + [[package]] name = "cmake" version = "0.1.54" @@ -932,6 +979,65 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "criterion" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1c047a62b0cc3e145fa84415a3191f628e980b194c2755aa12300a4e6cbd928" +dependencies = [ + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "itertools 0.13.0", + "num-traits", + "oorandom", + "plotters", + "rayon", + "regex", + "serde", + "serde_json", + "tinytemplate", + "tokio", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b1bcc0dc7dfae599d84ad0b1a55f80cde8af3725da8313b528da95ef783e338" +dependencies = [ + "cast", + "itertools 0.13.0", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + [[package]] name = "crunchy" version = "0.2.4" @@ -2003,6 +2109,12 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + [[package]] name = "openssl-probe" version = "0.1.6" @@ -2125,6 +2237,34 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + [[package]] name = "potential_utf" version = "0.1.3" @@ -2232,6 +2372,26 @@ dependencies = [ "getrandom 0.3.3", ] +[[package]] +name = "rayon" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.5.17" @@ -2484,6 +2644,15 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.28" @@ -2894,6 +3063,16 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinyvec" version = "1.10.0" @@ -3251,6 +3430,16 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -3343,6 +3532,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "web-sys" +version = "0.3.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbe734895e869dc429d78c4b433f8d17d95f8d05317440b4fad5ab2d33e596dc" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "which" version = "4.4.2" diff --git a/aws_secretsmanager_caching/Cargo.toml b/aws_secretsmanager_caching/Cargo.toml index 631ab7c..2f9a2e0 100644 --- a/aws_secretsmanager_caching/Cargo.toml +++ b/aws_secretsmanager_caching/Cargo.toml @@ -23,12 +23,18 @@ rustls = "0" log = "0.4.20" [dev-dependencies] -aws-smithy-mocks-experimental = "0" +aws-smithy-mocks = "0.1" aws-smithy-runtime = { version = "1", features = ["test-util", "wire-mock"] } aws-sdk-secretsmanager = { version = "1", features = ["test-util"] } -tokio = { version = "1", features = ["macros", "rt", "sync", "test-util"] } +tokio = { version = "1", features = ["macros", "rt", "rt-multi-thread", "sync", "test-util"] } http = "0" tokio-test = "0.4.4" +criterion = { version = "0.7.0", features = ["async_tokio", "html_reports"] } +rand = "0.9.2" [features] fips = ["rustls/fips"] + +[[bench]] +name = "benchmark" +harness = false diff --git a/aws_secretsmanager_caching/benches/benchmark.rs b/aws_secretsmanager_caching/benches/benchmark.rs new file mode 100644 index 0000000..ed31a65 --- /dev/null +++ b/aws_secretsmanager_caching/benches/benchmark.rs @@ -0,0 +1,117 @@ +use std::num::NonZeroUsize; +use std::time::Duration; + +use aws_sdk_secretsmanager::operation::describe_secret; +use aws_sdk_secretsmanager::operation::get_secret_value::GetSecretValueOutput; +use aws_secretsmanager_caching::SecretsManagerCachingClient; +use aws_smithy_mocks::{mock, mock_client, RuleMode}; +use criterion::{criterion_group, criterion_main, Criterion}; +use rand::distr::Alphabetic; +use rand::Rng; + +fn random_string(len: usize) -> String { + rand::rng() + .sample_iter(&Alphabetic) + .take(len) + .map(char::from) + .collect() +} + +/// Benchmark cache hits by retrieving the same secret over and over. +fn cache_hit(c: &mut Criterion) { + const CACHE_SIZE: NonZeroUsize = NonZeroUsize::new(1000).unwrap(); + + let gsv = mock!(aws_sdk_secretsmanager::Client::get_secret_value).then_output(move || { + GetSecretValueOutput::builder() + .name("secretid") + .arn("somearn") + .secret_string("hunter2") + .version_id("thisisaversionid") + .build() + }); + + let describe_secret = + mock!(aws_sdk_secretsmanager::Client::describe_secret).then_output(move || { + describe_secret::DescribeSecretOutput::builder() + .name("secretid") + .arn("somearn") + .version_ids_to_stages("thisisaversionid", vec!["AWSCURRENT".to_string()]) + .build() + }); + + let asm = mock_client!( + aws_sdk_secretsmanager, + RuleMode::MatchAny, + [&gsv, &describe_secret] + ); + + let cache = SecretsManagerCachingClient::new(asm, CACHE_SIZE, Duration::MAX, true).unwrap(); + + let rt = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .unwrap(); + + // Warm up the cache. + rt.block_on(cache.get_secret_value("secretid", None, None, false)) + .unwrap(); + + c.bench_function("CacheHit", |b| { + b.to_async(&rt).iter(async || { + // for _ in 0..10_000 { + cache + .get_secret_value("secretid", None, None, false) + .await + .unwrap(); + // } + }); + }); +} + +/// Benchmark cache eviction using a very small cache +fn cache_eviction(c: &mut Criterion) { + const CACHE_SIZE: NonZeroUsize = NonZeroUsize::new(1).unwrap(); + + let gsv = mock!(aws_sdk_secretsmanager::Client::get_secret_value).then_output(move || { + GetSecretValueOutput::builder() + .name("secretid") + .arn("somearn") + .secret_string("hunter2") + .version_id("thisisaversionid") + .build() + }); + + let describe_secret = + mock!(aws_sdk_secretsmanager::Client::describe_secret).then_output(move || { + describe_secret::DescribeSecretOutput::builder() + .name("secretid") + .arn("somearn") + .version_ids_to_stages("thisisaversionid", vec!["AWSCURRENT".to_string()]) + .build() + }); + + let asm = mock_client!( + aws_sdk_secretsmanager, + RuleMode::MatchAny, + [&gsv, &describe_secret] + ); + + let cache = SecretsManagerCachingClient::new(asm, CACHE_SIZE, Duration::MAX, true).unwrap(); + + let rt = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .unwrap(); + + c.bench_function("GetSecretValue", |b| { + b.to_async(&rt).iter(async || { + cache + .get_secret_value(&random_string(10), None, None, false) + .await + .unwrap(); + }); + }); +} + +criterion_group!(benches, cache_hit, cache_eviction); +criterion_main!(benches); From 9a311ed69592a43e3c884e2798e22dca5e6226d7 Mon Sep 17 00:00:00 2001 From: Simon Marty Date: Thu, 2 Oct 2025 13:54:42 -0700 Subject: [PATCH 2/3] Remove loop --- aws_secretsmanager_caching/benches/benchmark.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/aws_secretsmanager_caching/benches/benchmark.rs b/aws_secretsmanager_caching/benches/benchmark.rs index ed31a65..129a0f8 100644 --- a/aws_secretsmanager_caching/benches/benchmark.rs +++ b/aws_secretsmanager_caching/benches/benchmark.rs @@ -58,12 +58,10 @@ fn cache_hit(c: &mut Criterion) { c.bench_function("CacheHit", |b| { b.to_async(&rt).iter(async || { - // for _ in 0..10_000 { cache .get_secret_value("secretid", None, None, false) .await .unwrap(); - // } }); }); } From b8b41437f5dc52002bc5dd1ab7d0bc89222917c9 Mon Sep 17 00:00:00 2001 From: Simon Marty Date: Thu, 2 Oct 2025 13:55:37 -0700 Subject: [PATCH 3/3] Rename cache eviction bench function. --- aws_secretsmanager_caching/benches/benchmark.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_secretsmanager_caching/benches/benchmark.rs b/aws_secretsmanager_caching/benches/benchmark.rs index 129a0f8..d19363a 100644 --- a/aws_secretsmanager_caching/benches/benchmark.rs +++ b/aws_secretsmanager_caching/benches/benchmark.rs @@ -101,7 +101,7 @@ fn cache_eviction(c: &mut Criterion) { .build() .unwrap(); - c.bench_function("GetSecretValue", |b| { + c.bench_function("CacheEviction", |b| { b.to_async(&rt).iter(async || { cache .get_secret_value(&random_string(10), None, None, false)