From e3d215d78d1ac6978e82c5da8fc373d358474f58 Mon Sep 17 00:00:00 2001 From: Lucas Vieira Date: Tue, 28 Apr 2026 08:21:14 -0300 Subject: [PATCH] =?UTF-8?q?fix(s3):=20ListBuckets=20v2=20=E2=80=94=20Bucke?= =?UTF-8?q?tRegion=20+=20prefix/region/pagination=20filters?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #816. ListBuckets response previously omitted per entry, which the VS Code AWS Toolkit uses to filter visible buckets — buckets created through fakecloud were hidden in the IDE explorer. The AWS CLI tolerates the missing field, but real S3 always emits it. This fixes the symptom and rounds out the full ListBuckets v2 surface that AWS shipped alongside BucketRegion: - Per-bucket in every response - prefix query parameter — filter buckets by name prefix - bucket-region query parameter — filter buckets by region - max-buckets query parameter (1..=10000, default 10000) — page size - continuation-token query parameter — opaque base64 cursor - response element — next-page cursor when truncated - / echoed in response when input had them - InvalidArgument errors for out-of-range max-buckets and malformed tokens Coverage: 7 unit tests in fakecloud-s3 (region in body, region filter, prefix filter, pagination, continuation resume, invalid max-buckets, invalid token) and 4 e2e tests via aws-sdk-s3 (region populated, pagination across pages, prefix filter, region filter with mixed-region buckets). Conformance ListBuckets checksum unchanged — Smithy model already declared the v2 fields. --- crates/fakecloud-e2e/tests/s3.rs | 124 ++++++++++++++++++ crates/fakecloud-s3/src/service/buckets.rs | 102 ++++++++++++++- crates/fakecloud-s3/src/service/mod.rs | 141 +++++++++++++++++++++ website/static/llms-full.txt | 2 +- 4 files changed, 363 insertions(+), 6 deletions(-) diff --git a/crates/fakecloud-e2e/tests/s3.rs b/crates/fakecloud-e2e/tests/s3.rs index 0b706080..1ace0d9b 100644 --- a/crates/fakecloud-e2e/tests/s3.rs +++ b/crates/fakecloud-e2e/tests/s3.rs @@ -39,6 +39,130 @@ async fn s3_create_list_delete_bucket() { assert!(resp.buckets().is_empty()); } +#[tokio::test] +async fn s3_list_buckets_returns_region() { + let server = TestServer::start().await; + let client = server.s3_client().await; + + client + .create_bucket() + .bucket("region-test") + .send() + .await + .unwrap(); + + let resp = client.list_buckets().send().await.unwrap(); + let entry = resp + .buckets() + .iter() + .find(|b| b.name() == Some("region-test")) + .expect("bucket present"); + assert_eq!(entry.bucket_region(), Some("us-east-1")); +} + +#[tokio::test] +async fn s3_list_buckets_pagination() { + let server = TestServer::start().await; + let client = server.s3_client().await; + + for n in &["page-a", "page-b", "page-c"] { + client.create_bucket().bucket(*n).send().await.unwrap(); + } + + let page1 = client + .list_buckets() + .max_buckets(2) + .prefix("page-") + .send() + .await + .unwrap(); + let names1: Vec<&str> = page1 + .buckets() + .iter() + .map(|b| b.name().unwrap_or_default()) + .collect(); + assert_eq!(names1, vec!["page-a", "page-b"]); + let token = page1 + .continuation_token() + .expect("continuation token present"); + + let page2 = client + .list_buckets() + .max_buckets(2) + .prefix("page-") + .continuation_token(token) + .send() + .await + .unwrap(); + let names2: Vec<&str> = page2 + .buckets() + .iter() + .map(|b| b.name().unwrap_or_default()) + .collect(); + assert_eq!(names2, vec!["page-c"]); + assert!(page2.continuation_token().is_none()); +} + +#[tokio::test] +async fn s3_list_buckets_prefix_filter() { + let server = TestServer::start().await; + let client = server.s3_client().await; + + for n in &["alpha-1", "alpha-2", "beta"] { + client.create_bucket().bucket(*n).send().await.unwrap(); + } + + let resp = client.list_buckets().prefix("alpha-").send().await.unwrap(); + let names: Vec<&str> = resp + .buckets() + .iter() + .map(|b| b.name().unwrap_or_default()) + .collect(); + assert!(names.contains(&"alpha-1")); + assert!(names.contains(&"alpha-2")); + assert!(!names.contains(&"beta")); + assert_eq!(resp.prefix(), Some("alpha-")); +} + +#[tokio::test] +async fn s3_list_buckets_region_filter() { + use aws_sdk_s3::types::{BucketLocationConstraint, CreateBucketConfiguration}; + + let server = TestServer::start().await; + let client = server.s3_client().await; + + client + .create_bucket() + .bucket("east-only") + .send() + .await + .unwrap(); + client + .create_bucket() + .bucket("west-only") + .create_bucket_configuration( + CreateBucketConfiguration::builder() + .location_constraint(BucketLocationConstraint::UsWest2) + .build(), + ) + .send() + .await + .unwrap(); + + let resp = client + .list_buckets() + .bucket_region("us-west-2") + .send() + .await + .unwrap(); + let names: Vec<&str> = resp + .buckets() + .iter() + .map(|b| b.name().unwrap_or_default()) + .collect(); + assert_eq!(names, vec!["west-only"]); +} + #[tokio::test] async fn s3_head_bucket() { let server = TestServer::start().await; diff --git a/crates/fakecloud-s3/src/service/buckets.rs b/crates/fakecloud-s3/src/service/buckets.rs index e90422c8..31e64503 100644 --- a/crates/fakecloud-s3/src/service/buckets.rs +++ b/crates/fakecloud-s3/src/service/buckets.rs @@ -1,3 +1,5 @@ +use base64::engine::general_purpose::STANDARD as BASE64; +use base64::Engine as _; use http::{HeaderMap, StatusCode}; use bytes::Bytes; @@ -15,26 +17,116 @@ impl S3Service { pub(super) fn list_buckets( &self, account_id: &str, - _req: &AwsRequest, + req: &AwsRequest, ) -> Result { + let prefix = req.query_params.get("prefix").cloned(); + let bucket_region_filter = req.query_params.get("bucket-region").cloned(); + + let max_buckets: usize = match req.query_params.get("max-buckets") { + Some(v) => match v.parse::() { + Ok(n) if (1..=10_000).contains(&n) => n as usize, + _ => { + return Err(AwsServiceError::aws_error( + StatusCode::BAD_REQUEST, + "InvalidArgument", + "max-buckets must be between 1 and 10000", + )); + } + }, + None => 10_000, + }; + + let continuation_token = req.query_params.get("continuation-token").cloned(); + let token_after: Option = match continuation_token.as_deref() { + None => None, + Some("") => { + return Err(AwsServiceError::aws_error( + StatusCode::BAD_REQUEST, + "InvalidArgument", + "The continuation token provided is incorrect", + )); + } + Some(tok) => match BASE64 + .decode(tok.as_bytes()) + .ok() + .and_then(|d| String::from_utf8(d).ok()) + { + Some(s) => Some(s), + None => { + return Err(AwsServiceError::aws_error( + StatusCode::BAD_REQUEST, + "InvalidArgument", + "The continuation token provided is incorrect", + )); + } + }, + }; + let accts = self.state.read(); let __empty = crate::state::S3State::new(account_id, "us-east-1"); let state = accts.get(account_id).unwrap_or(&__empty); + + let mut filtered: Vec<&S3Bucket> = state + .buckets + .values() + .filter(|b| { + if let Some(p) = &prefix { + if !b.name.starts_with(p) { + return false; + } + } + if let Some(r) = &bucket_region_filter { + if &b.region != r { + return false; + } + } + true + }) + .collect(); + filtered.sort_by(|a, b| a.name.cmp(&b.name)); + + let start_index = match &token_after { + Some(after) => filtered.partition_point(|b| b.name.as_str() <= after.as_str()), + None => 0, + }; + let end_index = (start_index + max_buckets).min(filtered.len()); + let page = &filtered[start_index..end_index]; + let next_continuation = if end_index < filtered.len() { + page.last().map(|b| BASE64.encode(b.name.as_bytes())) + } else { + None + }; + let mut buckets_xml = String::new(); - let mut sorted: Vec<_> = state.buckets.values().collect(); - sorted.sort_by_key(|b| &b.name); - for b in sorted { + for b in page { buckets_xml.push_str(&format!( - "{}{}", + "{}{}{}", xml_escape(&b.name), b.creation_date.format("%Y-%m-%dT%H:%M:%S%.3fZ"), + xml_escape(&b.region), + )); + } + + let mut tail_xml = String::new(); + if let Some(p) = &prefix { + tail_xml.push_str(&format!("{}", xml_escape(p))); + } + if let Some(r) = &bucket_region_filter { + tail_xml.push_str(&format!("{}", xml_escape(r))); + } + if let Some(nct) = &next_continuation { + tail_xml.push_str(&format!( + "{}", + xml_escape(nct), )); } + let body = format!( "\ \ {account}{account}\ {buckets_xml}\ + {tail_xml}\ ", account = account_id, ); diff --git a/crates/fakecloud-s3/src/service/mod.rs b/crates/fakecloud-s3/src/service/mod.rs index 26972177..22d09d6c 100644 --- a/crates/fakecloud-s3/src/service/mod.rs +++ b/crates/fakecloud-s3/src/service/mod.rs @@ -5287,6 +5287,147 @@ mod tests { assert!(a < m && m < z, "buckets must be sorted"); } + fn seed_bucket_in_region(svc: &S3Service, name: &str, region: &str) { + let mut mas = svc.state.write(); + let state = mas.default_mut(); + state + .buckets + .insert(name.to_string(), S3Bucket::new(name, region, "owner")); + } + + #[test] + fn list_buckets_includes_bucket_region() { + let svc = make_service(); + seed_bucket(&svc, "alpha"); + + let req = make_request(Method::GET, "/", &[], b""); + let resp = svc.list_buckets("123456789012", &req).unwrap(); + let body = std::str::from_utf8(resp.body.expect_bytes()).unwrap(); + assert!( + body.contains("us-east-1"), + "response should include BucketRegion per-bucket: {body}" + ); + } + + #[test] + fn list_buckets_filter_by_bucket_region() { + let svc = make_service(); + seed_bucket_in_region(&svc, "east-bucket", "us-east-1"); + seed_bucket_in_region(&svc, "west-bucket", "us-west-2"); + + let req = make_request(Method::GET, "/", &[("bucket-region", "us-west-2")], b""); + let resp = svc.list_buckets("123456789012", &req).unwrap(); + let body = std::str::from_utf8(resp.body.expect_bytes()).unwrap(); + assert!(body.contains("west-bucket")); + assert!(!body.contains("east-bucket")); + assert!(body.contains("us-west-2")); + } + + #[test] + fn list_buckets_filter_by_prefix() { + let svc = make_service(); + seed_bucket(&svc, "foo-1"); + seed_bucket(&svc, "foo-2"); + seed_bucket(&svc, "bar"); + + let req = make_request(Method::GET, "/", &[("prefix", "foo-")], b""); + let resp = svc.list_buckets("123456789012", &req).unwrap(); + let body = std::str::from_utf8(resp.body.expect_bytes()).unwrap(); + assert!(body.contains("foo-1")); + assert!(body.contains("foo-2")); + assert!(!body.contains("bar")); + assert!(body.contains("foo-")); + } + + #[test] + fn list_buckets_max_buckets_paginates() { + let svc = make_service(); + for n in &["a", "b", "c", "d", "e"] { + seed_bucket(&svc, n); + } + + let req = make_request(Method::GET, "/", &[("max-buckets", "2")], b""); + let resp = svc.list_buckets("123456789012", &req).unwrap(); + let body = std::str::from_utf8(resp.body.expect_bytes()).unwrap(); + assert!(body.contains("a")); + assert!(body.contains("b")); + assert!(!body.contains("c")); + assert!(body.contains("")); + } + + #[test] + fn list_buckets_continuation_token_resumes() { + let svc = make_service(); + for n in &["a", "b", "c", "d", "e"] { + seed_bucket(&svc, n); + } + + let req = make_request(Method::GET, "/", &[("max-buckets", "2")], b""); + let resp = svc.list_buckets("123456789012", &req).unwrap(); + let body = std::str::from_utf8(resp.body.expect_bytes()).unwrap(); + let start = body.find("").unwrap() + "".len(); + let end = body.find("").unwrap(); + let token = body[start..end].to_string(); + + let req2 = make_request( + Method::GET, + "/", + &[("max-buckets", "2"), ("continuation-token", &token)], + b"", + ); + let resp2 = svc.list_buckets("123456789012", &req2).unwrap(); + let body2 = std::str::from_utf8(resp2.body.expect_bytes()).unwrap(); + assert!(body2.contains("c")); + assert!(body2.contains("d")); + assert!(!body2.contains("a")); + assert!(!body2.contains("b")); + // page 2 has more (e remains) so still emits a token + assert!(body2.contains("")); + + // page 3: should be e + no continuation + let start = body2.find("").unwrap() + "".len(); + let end = body2.find("").unwrap(); + let token2 = body2[start..end].to_string(); + let req3 = make_request( + Method::GET, + "/", + &[("max-buckets", "2"), ("continuation-token", &token2)], + b"", + ); + let resp3 = svc.list_buckets("123456789012", &req3).unwrap(); + let body3 = std::str::from_utf8(resp3.body.expect_bytes()).unwrap(); + assert!(body3.contains("e")); + assert!(!body3.contains("")); + } + + #[test] + fn list_buckets_invalid_max_buckets_errors() { + let svc = make_service(); + let req = make_request(Method::GET, "/", &[("max-buckets", "0")], b""); + assert_aws_err(svc.list_buckets("123456789012", &req), "InvalidArgument"); + + let req2 = make_request(Method::GET, "/", &[("max-buckets", "20000")], b""); + assert_aws_err(svc.list_buckets("123456789012", &req2), "InvalidArgument"); + + let req3 = make_request(Method::GET, "/", &[("max-buckets", "abc")], b""); + assert_aws_err(svc.list_buckets("123456789012", &req3), "InvalidArgument"); + } + + #[test] + fn list_buckets_invalid_continuation_token_errors() { + let svc = make_service(); + let req = make_request( + Method::GET, + "/", + &[("continuation-token", "!!!notb64!!!")], + b"", + ); + assert_aws_err(svc.list_buckets("123456789012", &req), "InvalidArgument"); + + let req2 = make_request(Method::GET, "/", &[("continuation-token", "")], b""); + assert_aws_err(svc.list_buckets("123456789012", &req2), "InvalidArgument"); + } + #[test] fn create_bucket_invalid_name_errors() { let svc = make_service(); diff --git a/website/static/llms-full.txt b/website/static/llms-full.txt index 9bd731d6..73235c58 100644 --- a/website/static/llms-full.txt +++ b/website/static/llms-full.txt @@ -249,7 +249,7 @@ Protocol: JSON (JSON body, `X-Amz-Target` header, JSON responses) **Bucket Configuration:** PutBucketTagging, GetBucketTagging, DeleteBucketTagging, PutBucketAcl, GetBucketAcl, PutBucketVersioning, GetBucketVersioning, PutBucketCors, GetBucketCors, DeleteBucketCors, PutBucketNotificationConfiguration, GetBucketNotificationConfiguration, PutBucketWebsite, GetBucketWebsite, DeleteBucketWebsite, PutBucketAccelerateConfiguration, GetBucketAccelerateConfiguration, PutPublicAccessBlock, GetPublicAccessBlock, DeletePublicAccessBlock, PutBucketEncryption, GetBucketEncryption, DeleteBucketEncryption, PutBucketLifecycleConfiguration, GetBucketLifecycleConfiguration, DeleteBucketLifecycleConfiguration, PutBucketLogging, GetBucketLogging, PutBucketPolicy, GetBucketPolicy, DeleteBucketPolicy, PutObjectLockConfiguration, GetObjectLockConfiguration, PutBucketReplication, GetBucketReplication, DeleteBucketReplication, PutBucketOwnershipControls, GetBucketOwnershipControls, DeleteBucketOwnershipControls, PutBucketInventoryConfiguration, GetBucketInventoryConfiguration, DeleteBucketInventoryConfiguration **Multipart Uploads:** CreateMultipartUpload, UploadPart, UploadPartCopy, CompleteMultipartUpload, AbortMultipartUpload, ListParts, ListMultipartUploads -Features: path-style addressing, prefix/delimiter listing, pagination, user metadata, multipart uploads, versioning, CORS, bucket notifications (SNS/SQS delivery), lifecycle rules with background expiration, object lock (retention and legal hold), encryption, replication config, website config. +Features: path-style addressing, prefix/delimiter listing, pagination (including ListBuckets v2 with `prefix`, `bucket-region`, `max-buckets`, `continuation-token`), user metadata, multipart uploads, versioning, CORS, bucket notifications (SNS/SQS delivery), lifecycle rules with background expiration, object lock (retention and legal hold), encryption, replication config, website config. Protocol: REST (HTTP method + path-based routing, XML responses)