From e65230b011198f182eb33c7cb3266eec1c425ec5 Mon Sep 17 00:00:00 2001 From: Heath Hopkins Date: Fri, 15 Aug 2025 11:19:47 -0400 Subject: [PATCH] Refactor credential handling in aws_utils to allow S3 credentials from sources other than environment variables by using the AWS default credential chain. Preserved credential checking before first use to prevent logs from filling up. Also added `aws-credential-types` and `tokio` to dependencies in Cargo.toml and aws_utils/Cargo.toml --- Cargo.lock | 51 ++++++++++++++ Cargo.toml | 3 +- crates/aws_utils/Cargo.toml | 2 + .../src/bin/demo_test_credentials.rs | 48 +++++++++++++ crates/aws_utils/src/lib.rs | 69 +++++++++++++------ crates/aws_utils/src/s3.rs | 2 +- 6 files changed, 153 insertions(+), 22 deletions(-) create mode 100644 crates/aws_utils/src/bin/demo_test_credentials.rs diff --git a/Cargo.lock b/Cargo.lock index a4f24ddc4..15b0774d9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -531,6 +531,8 @@ checksum = "b6fcc63c9860579e4cb396239570e979376e70aab79e496621748a09913f8b36" dependencies = [ "aws-credential-types", "aws-runtime", + "aws-sdk-sso", + "aws-sdk-ssooidc", "aws-sdk-sts", "aws-smithy-async", "aws-smithy-http", @@ -541,11 +543,14 @@ dependencies = [ "aws-types", "bytes", "fastrand", + "hex", "http 1.3.1", + "ring", "time", "tokio", "tracing", "url", + "zeroize", ] [[package]] @@ -656,6 +661,50 @@ dependencies = [ "url", ] +[[package]] +name = "aws-sdk-sso" +version = "1.72.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13118ad30741222f67b1a18e5071385863914da05124652b38e172d6d3d9ce31" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.9", + "regex-lite", + "tracing", +] + +[[package]] +name = "aws-sdk-ssooidc" +version = "1.73.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f879a8572b4683a8f84f781695bebf2f25cf11a81a2693c31fc0e0215c2c1726" +dependencies = [ + "aws-credential-types", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "http 0.2.9", + "regex-lite", + "tracing", +] + [[package]] name = "aws-sdk-sts" version = "1.66.0" @@ -956,10 +1005,12 @@ version = "0.1.0" dependencies = [ "anyhow", "aws-config", + "aws-credential-types", "aws-sdk-s3", "aws-smithy-types-convert", "aws-types", "futures", + "tokio", "tracing", ] diff --git a/Cargo.toml b/Cargo.toml index ec6efda08..4edb1ad3f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -76,7 +76,8 @@ async-trait = "0.1" async_zip = { version = "0.0.17", default-features = false, features = [ "deflate", "tokio", "zstd" ] } async_zip_0_0_9 = { package = "async_zip", version = "0.0.9", default-features = false, features = [ "zstd", "deflate" ] } atomic_refcell = "0.1.13" -aws-config = { version = "1.6", default-features = false, features = [ "client-hyper", "default-https-client", "rustls", "rt-tokio" ] } +aws-config = { version = "1.6", default-features = false, features = [ "client-hyper", "default-https-client", "rustls", "rt-tokio", "sso" ] } +aws-credential-types = { version = "1" } aws-lc-rs = { version = "1.13", default-features = false, features = [ "aws-lc-sys", "prebuilt-nasm" ] } aws-sdk-s3 = { version = "1.83", default-features = false, features = [ "default-https-client", "rt-tokio", "sigv4a" ] } aws-smithy-http = "0.62.0" diff --git a/crates/aws_utils/Cargo.toml b/crates/aws_utils/Cargo.toml index b34bb4e13..05f7cfefe 100644 --- a/crates/aws_utils/Cargo.toml +++ b/crates/aws_utils/Cargo.toml @@ -7,10 +7,12 @@ license = "LicenseRef-FSL-1.1-Apache-2.0" [dependencies] anyhow = { workspace = true } aws-config = { workspace = true } +aws-credential-types = { workspace = true } aws-sdk-s3 = { workspace = true } aws-smithy-types-convert = { workspace = true } aws-types = { workspace = true } futures = { workspace = true } +tokio = { workspace = true } tracing = { workspace = true } [lints] diff --git a/crates/aws_utils/src/bin/demo_test_credentials.rs b/crates/aws_utils/src/bin/demo_test_credentials.rs new file mode 100644 index 000000000..a796f97de --- /dev/null +++ b/crates/aws_utils/src/bin/demo_test_credentials.rs @@ -0,0 +1,48 @@ +use std::env; + +use anyhow::{Context, Result}; +use aws_config::BehaviorVersion; +use aws_sdk_s3 as s3; +use aws_utils::preflight_credentials; + +#[tokio::main] +async fn main() -> Result<()> { + // 1) Preflight: try to resolve credentials using the standard chain. + let _creds = preflight_credentials().await?; + + println!( + "✅ Credentials resolved{}", + match env::var("AWS_PROFILE") { + Ok(p) => format!(" (AWS_PROFILE={})", p), + Err(_) => String::new(), + } + ); + + // 2) Load full config explicitly setting profile if available + let mut config_loader = aws_config::defaults(BehaviorVersion::latest()); + if let Ok(profile) = env::var("AWS_PROFILE") { + config_loader = config_loader.profile_name(&profile); + } + let conf = config_loader.load().await; + + // 3) Use S3 client safely now that we know creds exist. + let client = s3::Client::new(&conf); + + // Example: list buckets + println!("Testing S3 access by listing buckets..."); + let resp = client.list_buckets().send().await + .context("S3 call failed (credentials may be invalid/expired or region/network misconfigured)")?; + + println!("✅ S3 access successful!"); + println!("Buckets:"); + let buckets = resp.buckets(); + if buckets.is_empty() { + println!(" (no buckets found)"); + } else { + for b in buckets { + println!(" - {}", b.name().unwrap_or("")); + } + } + + Ok(()) +} diff --git a/crates/aws_utils/src/lib.rs b/crates/aws_utils/src/lib.rs index 3a22d10d7..8a8ebbf51 100644 --- a/crates/aws_utils/src/lib.rs +++ b/crates/aws_utils/src/lib.rs @@ -1,15 +1,16 @@ -#![feature(coroutines)] -#![feature(exit_status_error)] +// #![feature(coroutines)] +// #![feature(exit_status_error)] use std::{ env, sync::LazyLock, }; use aws_config::{ - environment::credentials::EnvironmentVariableCredentialsProvider, + default_provider::credentials::DefaultCredentialsChain, BehaviorVersion, ConfigLoader, }; +use aws_credential_types::provider::ProvideCredentials; use aws_sdk_s3::config::Builder as S3ConfigBuilder; use aws_types::region::Region; @@ -18,12 +19,6 @@ pub mod s3; static S3_ENDPOINT_URL: LazyLock> = LazyLock::new(|| env::var("S3_ENDPOINT_URL").ok()); -static AWS_ACCESS_KEY_ID: LazyLock> = - LazyLock::new(|| env::var("AWS_ACCESS_KEY_ID").ok()); - -static AWS_SECRET_ACCESS_KEY: LazyLock> = - LazyLock::new(|| env::var("AWS_SECRET_ACCESS_KEY").ok()); - static AWS_REGION: LazyLock> = LazyLock::new(|| env::var("AWS_REGION").ok()); static AWS_S3_FORCE_PATH_STYLE: LazyLock = LazyLock::new(|| { @@ -36,25 +31,21 @@ static AWS_S3_FORCE_PATH_STYLE: LazyLock = LazyLock::new(|| { /// Similar aws_config::from_env but returns an error if credentials or /// region is are not. It also doesn't spew out log lines every time /// credentials are accessed. -pub fn must_config_from_env() -> anyhow::Result { +pub async fn must_config_from_env() -> anyhow::Result { let Some(region) = AWS_REGION.clone() else { anyhow::bail!("AWS_REGION env variable must be set"); }; let region = Region::new(region); - let Some(_) = AWS_ACCESS_KEY_ID.clone() else { - anyhow::bail!("AWS_ACCESS_KEY_ID env variable must be set"); - }; - let Some(_) = AWS_SECRET_ACCESS_KEY.clone() else { - anyhow::bail!("AWS_SECRET_ACCESS_KEY env variable must be set"); - }; - let credentials = EnvironmentVariableCredentialsProvider::new(); + + // Check for credentials using the default provider chain + let _creds = preflight_credentials().await?; + Ok(aws_config::defaults(BehaviorVersion::v2025_01_17()) - .region(region) - .credentials_provider(credentials)) + .region(region)) } pub async fn must_s3_config_from_env() -> anyhow::Result { - let base_config = must_config_from_env()?.load().await; + let base_config = must_config_from_env().await?.load().await; let mut s3_config_builder = S3ConfigBuilder::from(&base_config); if let Some(s3_endpoint_url) = S3_ENDPOINT_URL.clone() { s3_config_builder = s3_config_builder.endpoint_url(s3_endpoint_url); @@ -62,3 +53,41 @@ pub async fn must_s3_config_from_env() -> anyhow::Result { s3_config_builder = s3_config_builder.force_path_style(*AWS_S3_FORCE_PATH_STYLE); Ok(s3_config_builder) } + +/// Attempts to resolve credentials using the default chain: +/// env vars -> shared config/credentials (incl. SSO) -> web identity -> container creds -> EC2 IMDSv2. +/// Returns early with a helpful error if nothing is available. +pub async fn preflight_credentials() -> anyhow::Result { + let chain = DefaultCredentialsChain::builder().build().await; + + match chain.provide_credentials().await { + Ok(creds) => Ok(creds), + Err(err) => { + // Give actionable hints based on common setups. + let profile = env::var("AWS_PROFILE").unwrap_or_else(|_| "default".to_string()); + let mut help = String::new(); + help.push_str("No AWS credentials were found by the default provider chain.\n\n"); + help.push_str("Tried in this order:\n"); + help.push_str(" 1) Environment: AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY [/ AWS_SESSION_TOKEN]\n"); + help.push_str(" 2) Shared config/credentials files (~/.aws/config, ~/.aws/credentials) "); + help.push_str(&format!("(profile: {})\n", profile)); + help.push_str(" - If you use IAM Identity Center (SSO), run: aws sso login"); + if profile != "default" { + help.push_str(&format!(" --profile {}", profile)); + } + help.push_str("\n"); + help.push_str(" 3) Web identity (AssumeRoleWithWebIdentity; env/profiles with role_arn & web_identity_token_file)\n"); + help.push_str(" 4) Container credentials (ECS/EKS env: AWS_CONTAINER_CREDENTIALS_* or Pod Identity)\n"); + help.push_str(" 5) EC2 Instance Metadata (IMDSv2; instance role)\n\n"); + + help.push_str("Fixes:\n"); + help.push_str(" • For access keys: set AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, and AWS_SESSION_TOKEN (optional)\n"); + help.push_str(" • For profiles: set AWS_PROFILE or add a [profile] with credentials in ~/.aws/credentials\n"); + help.push_str(" • For SSO: aws configure sso && aws sso login\n"); + help.push_str(" • For web identity: ensure web_identity_token_file and role_arn are set\n"); + help.push_str(" • For containers/EC2: attach the proper task/IRSA/instance role\n"); + + anyhow::bail!("{}Underlying error: {}", help, err) + } + } +} diff --git a/crates/aws_utils/src/s3.rs b/crates/aws_utils/src/s3.rs index 6cc6aa1ac..b159333c8 100644 --- a/crates/aws_utils/src/s3.rs +++ b/crates/aws_utils/src/s3.rs @@ -33,7 +33,7 @@ impl S3Client { }; let config = must_s3_config_from_env() .await - .context("AWS env variables are required when using AWS Lambda")? + .context("Failed to create S3 configuration. Check AWS env variables or IAM permissions.")? .retry_config(retry_config) .build();