diff --git a/crates/fakecloud-e2e/tests/s3.rs b/crates/fakecloud-e2e/tests/s3.rs index 0b706080..1ace0d9b 100644 --- a/crates/fakecloud-e2e/tests/s3.rs +++ b/crates/fakecloud-e2e/tests/s3.rs @@ -39,6 +39,130 @@ async fn s3_create_list_delete_bucket() { assert!(resp.buckets().is_empty()); } +#[tokio::test] +async fn s3_list_buckets_returns_region() { + let server = TestServer::start().await; + let client = server.s3_client().await; + + client + .create_bucket() + .bucket("region-test") + .send() + .await + .unwrap(); + + let resp = client.list_buckets().send().await.unwrap(); + let entry = resp + .buckets() + .iter() + .find(|b| b.name() == Some("region-test")) + .expect("bucket present"); + assert_eq!(entry.bucket_region(), Some("us-east-1")); +} + +#[tokio::test] +async fn s3_list_buckets_pagination() { + let server = TestServer::start().await; + let client = server.s3_client().await; + + for n in &["page-a", "page-b", "page-c"] { + client.create_bucket().bucket(*n).send().await.unwrap(); + } + + let page1 = client + .list_buckets() + .max_buckets(2) + .prefix("page-") + .send() + .await + .unwrap(); + let names1: Vec<&str> = page1 + .buckets() + .iter() + .map(|b| b.name().unwrap_or_default()) + .collect(); + assert_eq!(names1, vec!["page-a", "page-b"]); + let token = page1 + .continuation_token() + .expect("continuation token present"); + + let page2 = client + .list_buckets() + .max_buckets(2) + .prefix("page-") + .continuation_token(token) + .send() + .await + .unwrap(); + let names2: Vec<&str> = page2 + .buckets() + .iter() + .map(|b| b.name().unwrap_or_default()) + .collect(); + assert_eq!(names2, vec!["page-c"]); + assert!(page2.continuation_token().is_none()); +} + +#[tokio::test] +async fn s3_list_buckets_prefix_filter() { + let server = TestServer::start().await; + let client = server.s3_client().await; + + for n in &["alpha-1", "alpha-2", "beta"] { + client.create_bucket().bucket(*n).send().await.unwrap(); + } + + let resp = client.list_buckets().prefix("alpha-").send().await.unwrap(); + let names: Vec<&str> = resp + .buckets() + .iter() + .map(|b| b.name().unwrap_or_default()) + .collect(); + assert!(names.contains(&"alpha-1")); + assert!(names.contains(&"alpha-2")); + assert!(!names.contains(&"beta")); + assert_eq!(resp.prefix(), Some("alpha-")); +} + +#[tokio::test] +async fn s3_list_buckets_region_filter() { + use aws_sdk_s3::types::{BucketLocationConstraint, CreateBucketConfiguration}; + + let server = TestServer::start().await; + let client = server.s3_client().await; + + client + .create_bucket() + .bucket("east-only") + .send() + .await + .unwrap(); + client + .create_bucket() + .bucket("west-only") + .create_bucket_configuration( + CreateBucketConfiguration::builder() + .location_constraint(BucketLocationConstraint::UsWest2) + .build(), + ) + .send() + .await + .unwrap(); + + let resp = client + .list_buckets() + .bucket_region("us-west-2") + .send() + .await + .unwrap(); + let names: Vec<&str> = resp + .buckets() + .iter() + .map(|b| b.name().unwrap_or_default()) + .collect(); + assert_eq!(names, vec!["west-only"]); +} + #[tokio::test] async fn s3_head_bucket() { let server = TestServer::start().await; diff --git a/crates/fakecloud-s3/src/service/buckets.rs b/crates/fakecloud-s3/src/service/buckets.rs index e90422c8..31e64503 100644 --- a/crates/fakecloud-s3/src/service/buckets.rs +++ b/crates/fakecloud-s3/src/service/buckets.rs @@ -1,3 +1,5 @@ +use base64::engine::general_purpose::STANDARD as BASE64; +use base64::Engine as _; use http::{HeaderMap, StatusCode}; use bytes::Bytes; @@ -15,26 +17,116 @@ impl S3Service { pub(super) fn list_buckets( &self, account_id: &str, - _req: &AwsRequest, + req: &AwsRequest, ) -> Result { + let prefix = req.query_params.get("prefix").cloned(); + let bucket_region_filter = req.query_params.get("bucket-region").cloned(); + + let max_buckets: usize = match req.query_params.get("max-buckets") { + Some(v) => match v.parse::() { + Ok(n) if (1..=10_000).contains(&n) => n as usize, + _ => { + return Err(AwsServiceError::aws_error( + StatusCode::BAD_REQUEST, + "InvalidArgument", + "max-buckets must be between 1 and 10000", + )); + } + }, + None => 10_000, + }; + + let continuation_token = req.query_params.get("continuation-token").cloned(); + let token_after: Option = match continuation_token.as_deref() { + None => None, + Some("") => { + return Err(AwsServiceError::aws_error( + StatusCode::BAD_REQUEST, + "InvalidArgument", + "The continuation token provided is incorrect", + )); + } + Some(tok) => match BASE64 + .decode(tok.as_bytes()) + .ok() + .and_then(|d| String::from_utf8(d).ok()) + { + Some(s) => Some(s), + None => { + return Err(AwsServiceError::aws_error( + StatusCode::BAD_REQUEST, + "InvalidArgument", + "The continuation token provided is incorrect", + )); + } + }, + }; + let accts = self.state.read(); let __empty = crate::state::S3State::new(account_id, "us-east-1"); let state = accts.get(account_id).unwrap_or(&__empty); + + let mut filtered: Vec<&S3Bucket> = state + .buckets + .values() + .filter(|b| { + if let Some(p) = &prefix { + if !b.name.starts_with(p) { + return false; + } + } + if let Some(r) = &bucket_region_filter { + if &b.region != r { + return false; + } + } + true + }) + .collect(); + filtered.sort_by(|a, b| a.name.cmp(&b.name)); + + let start_index = match &token_after { + Some(after) => filtered.partition_point(|b| b.name.as_str() <= after.as_str()), + None => 0, + }; + let end_index = (start_index + max_buckets).min(filtered.len()); + let page = &filtered[start_index..end_index]; + let next_continuation = if end_index < filtered.len() { + page.last().map(|b| BASE64.encode(b.name.as_bytes())) + } else { + None + }; + let mut buckets_xml = String::new(); - let mut sorted: Vec<_> = state.buckets.values().collect(); - sorted.sort_by_key(|b| &b.name); - for b in sorted { + for b in page { buckets_xml.push_str(&format!( - "{}{}", + "{}{}{}", xml_escape(&b.name), b.creation_date.format("%Y-%m-%dT%H:%M:%S%.3fZ"), + xml_escape(&b.region), + )); + } + + let mut tail_xml = String::new(); + if let Some(p) = &prefix { + tail_xml.push_str(&format!("{}", xml_escape(p))); + } + if let Some(r) = &bucket_region_filter { + tail_xml.push_str(&format!("{}", xml_escape(r))); + } + if let Some(nct) = &next_continuation { + tail_xml.push_str(&format!( + "{}", + xml_escape(nct), )); } + let body = format!( "\ \ {account}{account}\ {buckets_xml}\ + {tail_xml}\ ", account = account_id, ); diff --git a/crates/fakecloud-s3/src/service/mod.rs b/crates/fakecloud-s3/src/service/mod.rs index 26972177..22d09d6c 100644 --- a/crates/fakecloud-s3/src/service/mod.rs +++ b/crates/fakecloud-s3/src/service/mod.rs @@ -5287,6 +5287,147 @@ mod tests { assert!(a < m && m < z, "buckets must be sorted"); } + fn seed_bucket_in_region(svc: &S3Service, name: &str, region: &str) { + let mut mas = svc.state.write(); + let state = mas.default_mut(); + state + .buckets + .insert(name.to_string(), S3Bucket::new(name, region, "owner")); + } + + #[test] + fn list_buckets_includes_bucket_region() { + let svc = make_service(); + seed_bucket(&svc, "alpha"); + + let req = make_request(Method::GET, "/", &[], b""); + let resp = svc.list_buckets("123456789012", &req).unwrap(); + let body = std::str::from_utf8(resp.body.expect_bytes()).unwrap(); + assert!( + body.contains("us-east-1"), + "response should include BucketRegion per-bucket: {body}" + ); + } + + #[test] + fn list_buckets_filter_by_bucket_region() { + let svc = make_service(); + seed_bucket_in_region(&svc, "east-bucket", "us-east-1"); + seed_bucket_in_region(&svc, "west-bucket", "us-west-2"); + + let req = make_request(Method::GET, "/", &[("bucket-region", "us-west-2")], b""); + let resp = svc.list_buckets("123456789012", &req).unwrap(); + let body = std::str::from_utf8(resp.body.expect_bytes()).unwrap(); + assert!(body.contains("west-bucket")); + assert!(!body.contains("east-bucket")); + assert!(body.contains("us-west-2")); + } + + #[test] + fn list_buckets_filter_by_prefix() { + let svc = make_service(); + seed_bucket(&svc, "foo-1"); + seed_bucket(&svc, "foo-2"); + seed_bucket(&svc, "bar"); + + let req = make_request(Method::GET, "/", &[("prefix", "foo-")], b""); + let resp = svc.list_buckets("123456789012", &req).unwrap(); + let body = std::str::from_utf8(resp.body.expect_bytes()).unwrap(); + assert!(body.contains("foo-1")); + assert!(body.contains("foo-2")); + assert!(!body.contains("bar")); + assert!(body.contains("foo-")); + } + + #[test] + fn list_buckets_max_buckets_paginates() { + let svc = make_service(); + for n in &["a", "b", "c", "d", "e"] { + seed_bucket(&svc, n); + } + + let req = make_request(Method::GET, "/", &[("max-buckets", "2")], b""); + let resp = svc.list_buckets("123456789012", &req).unwrap(); + let body = std::str::from_utf8(resp.body.expect_bytes()).unwrap(); + assert!(body.contains("a")); + assert!(body.contains("b")); + assert!(!body.contains("c")); + assert!(body.contains("")); + } + + #[test] + fn list_buckets_continuation_token_resumes() { + let svc = make_service(); + for n in &["a", "b", "c", "d", "e"] { + seed_bucket(&svc, n); + } + + let req = make_request(Method::GET, "/", &[("max-buckets", "2")], b""); + let resp = svc.list_buckets("123456789012", &req).unwrap(); + let body = std::str::from_utf8(resp.body.expect_bytes()).unwrap(); + let start = body.find("").unwrap() + "".len(); + let end = body.find("").unwrap(); + let token = body[start..end].to_string(); + + let req2 = make_request( + Method::GET, + "/", + &[("max-buckets", "2"), ("continuation-token", &token)], + b"", + ); + let resp2 = svc.list_buckets("123456789012", &req2).unwrap(); + let body2 = std::str::from_utf8(resp2.body.expect_bytes()).unwrap(); + assert!(body2.contains("c")); + assert!(body2.contains("d")); + assert!(!body2.contains("a")); + assert!(!body2.contains("b")); + // page 2 has more (e remains) so still emits a token + assert!(body2.contains("")); + + // page 3: should be e + no continuation + let start = body2.find("").unwrap() + "".len(); + let end = body2.find("").unwrap(); + let token2 = body2[start..end].to_string(); + let req3 = make_request( + Method::GET, + "/", + &[("max-buckets", "2"), ("continuation-token", &token2)], + b"", + ); + let resp3 = svc.list_buckets("123456789012", &req3).unwrap(); + let body3 = std::str::from_utf8(resp3.body.expect_bytes()).unwrap(); + assert!(body3.contains("e")); + assert!(!body3.contains("")); + } + + #[test] + fn list_buckets_invalid_max_buckets_errors() { + let svc = make_service(); + let req = make_request(Method::GET, "/", &[("max-buckets", "0")], b""); + assert_aws_err(svc.list_buckets("123456789012", &req), "InvalidArgument"); + + let req2 = make_request(Method::GET, "/", &[("max-buckets", "20000")], b""); + assert_aws_err(svc.list_buckets("123456789012", &req2), "InvalidArgument"); + + let req3 = make_request(Method::GET, "/", &[("max-buckets", "abc")], b""); + assert_aws_err(svc.list_buckets("123456789012", &req3), "InvalidArgument"); + } + + #[test] + fn list_buckets_invalid_continuation_token_errors() { + let svc = make_service(); + let req = make_request( + Method::GET, + "/", + &[("continuation-token", "!!!notb64!!!")], + b"", + ); + assert_aws_err(svc.list_buckets("123456789012", &req), "InvalidArgument"); + + let req2 = make_request(Method::GET, "/", &[("continuation-token", "")], b""); + assert_aws_err(svc.list_buckets("123456789012", &req2), "InvalidArgument"); + } + #[test] fn create_bucket_invalid_name_errors() { let svc = make_service(); diff --git a/website/static/llms-full.txt b/website/static/llms-full.txt index 9bd731d6..73235c58 100644 --- a/website/static/llms-full.txt +++ b/website/static/llms-full.txt @@ -249,7 +249,7 @@ Protocol: JSON (JSON body, `X-Amz-Target` header, JSON responses) **Bucket Configuration:** PutBucketTagging, GetBucketTagging, DeleteBucketTagging, PutBucketAcl, GetBucketAcl, PutBucketVersioning, GetBucketVersioning, PutBucketCors, GetBucketCors, DeleteBucketCors, PutBucketNotificationConfiguration, GetBucketNotificationConfiguration, PutBucketWebsite, GetBucketWebsite, DeleteBucketWebsite, PutBucketAccelerateConfiguration, GetBucketAccelerateConfiguration, PutPublicAccessBlock, GetPublicAccessBlock, DeletePublicAccessBlock, PutBucketEncryption, GetBucketEncryption, DeleteBucketEncryption, PutBucketLifecycleConfiguration, GetBucketLifecycleConfiguration, DeleteBucketLifecycleConfiguration, PutBucketLogging, GetBucketLogging, PutBucketPolicy, GetBucketPolicy, DeleteBucketPolicy, PutObjectLockConfiguration, GetObjectLockConfiguration, PutBucketReplication, GetBucketReplication, DeleteBucketReplication, PutBucketOwnershipControls, GetBucketOwnershipControls, DeleteBucketOwnershipControls, PutBucketInventoryConfiguration, GetBucketInventoryConfiguration, DeleteBucketInventoryConfiguration **Multipart Uploads:** CreateMultipartUpload, UploadPart, UploadPartCopy, CompleteMultipartUpload, AbortMultipartUpload, ListParts, ListMultipartUploads -Features: path-style addressing, prefix/delimiter listing, pagination, user metadata, multipart uploads, versioning, CORS, bucket notifications (SNS/SQS delivery), lifecycle rules with background expiration, object lock (retention and legal hold), encryption, replication config, website config. +Features: path-style addressing, prefix/delimiter listing, pagination (including ListBuckets v2 with `prefix`, `bucket-region`, `max-buckets`, `continuation-token`), user metadata, multipart uploads, versioning, CORS, bucket notifications (SNS/SQS delivery), lifecycle rules with background expiration, object lock (retention and legal hold), encryption, replication config, website config. Protocol: REST (HTTP method + path-based routing, XML responses)