From 003f811292c68e40cf208ee5c1f52d484bef1bfd Mon Sep 17 00:00:00 2001 From: vzpd Date: Wed, 29 Apr 2026 11:51:01 +0800 Subject: [PATCH] feat(storage): support IAM authentication for S3 storage When access-key / secret-key are left blank, fall back to the AWS DefaultCredentialsProvider chain so that deployments on EC2, ECS, and EKS can authenticate via instance profile, task role, or IRSA without static credentials. - Extract buildCredentialsProvider() in S3StorageService - Add sts dependency for Web Identity Token (EKS) support - Add unit tests for credential provider selection - Update storage-spi docs (zh + en) and env example --- .env.release.example | 2 + .../docs/04-developer/plugins/storage-spi.md | 23 +++++++++ .../04-developer/plugins/storage-spi.md | 23 +++++++++ server/skillhub-storage/pom.xml | 5 ++ .../skillhub/storage/S3StorageService.java | 28 ++++++++--- .../storage/S3StorageServiceTest.java | 49 +++++++++++++++++++ 6 files changed, 123 insertions(+), 7 deletions(-) diff --git a/.env.release.example b/.env.release.example index 3f020fc75..930ab826f 100644 --- a/.env.release.example +++ b/.env.release.example @@ -34,6 +34,8 @@ SKILLHUB_STORAGE_PROVIDER=local SKILLHUB_STORAGE_S3_ENDPOINT=https://oss-cn-example.aliyuncs.com SKILLHUB_STORAGE_S3_PUBLIC_ENDPOINT= SKILLHUB_STORAGE_S3_BUCKET=skillhub-prod +# Static credentials for S3-compatible storage (MinIO, Alibaba OSS, etc.). +# Leave both blank to use IAM authentication (EC2 instance profile, ECS task role, EKS IRSA). SKILLHUB_STORAGE_S3_ACCESS_KEY=replace-me SKILLHUB_STORAGE_S3_SECRET_KEY=replace-me SKILLHUB_STORAGE_S3_REGION=cn-shanghai diff --git a/document/docs/04-developer/plugins/storage-spi.md b/document/docs/04-developer/plugins/storage-spi.md index f81189e17..3b7f4dbaa 100644 --- a/document/docs/04-developer/plugins/storage-spi.md +++ b/document/docs/04-developer/plugins/storage-spi.md @@ -34,6 +34,8 @@ S3 协议兼容实现，支持： ## 配置 +### 静态凭据（Access Key / Secret Key） + ```bash # 选择存储提供方 SKILLHUB_STORAGE_PROVIDER=s3 @@ -45,6 +47,27 @@ SKILLHUB_STORAGE_S3_ACCESS_KEY=xxx SKILLHUB_STORAGE_S3_SECRET_KEY=xxx ``` +### IAM 认证 + +部署在 AWS 上时，可以不配置 Access Key / Secret Key，让 SDK 自动使用 IAM 角色认证（[Default Credentials Provider Chain](https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/credentials-chain.html)）： + +```bash +SKILLHUB_STORAGE_PROVIDER=s3 +SKILLHUB_STORAGE_S3_BUCKET=skillhub +SKILLHUB_STORAGE_S3_REGION=us-east-1 +# 留空或不设置 ACCESS_KEY / SECRET_KEY，SDK 自动使用 IAM 认证 +SKILLHUB_STORAGE_S3_ACCESS_KEY= +SKILLHUB_STORAGE_S3_SECRET_KEY= +``` + +支持的 IAM 认证方式（按 SDK 优先级）： +- 环境变量（`AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY`） +- Java 系统属性 +- Web Identity Token（EKS IRSA） +- AWS 配置文件（`~/.aws/credentials`） +- EC2 Instance Profile +- ECS Task Role + ## 自定义实现 实现 `ObjectStorageService` 接口，注册为 Spring Bean 即可。 diff --git a/document/i18n/en/docusaurus-plugin-content-docs/current/04-developer/plugins/storage-spi.md b/document/i18n/en/docusaurus-plugin-content-docs/current/04-developer/plugins/storage-spi.md index f3a0a57d1..2667e6155 100644 --- a/document/i18n/en/docusaurus-plugin-content-docs/current/04-developer/plugins/storage-spi.md +++ b/document/i18n/en/docusaurus-plugin-content-docs/current/04-developer/plugins/storage-spi.md @@ -34,6 +34,8 @@ S3 protocol compatible implementation, supports: ## Configuration +### Static Credentials (Access Key / Secret Key) + ```bash # Select storage provider SKILLHUB_STORAGE_PROVIDER=s3 @@ -45,6 +47,27 @@ SKILLHUB_STORAGE_S3_ACCESS_KEY=xxx SKILLHUB_STORAGE_S3_SECRET_KEY=xxx ``` +### IAM Authentication + +When deployed on AWS, you can omit the Access Key / Secret Key and let the SDK use IAM role authentication via the [Default Credentials Provider Chain](https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/credentials-chain.html): + +```bash +SKILLHUB_STORAGE_PROVIDER=s3 +SKILLHUB_STORAGE_S3_BUCKET=skillhub +SKILLHUB_STORAGE_S3_REGION=us-east-1 +# Leave ACCESS_KEY / SECRET_KEY blank to use IAM authentication +SKILLHUB_STORAGE_S3_ACCESS_KEY= +SKILLHUB_STORAGE_S3_SECRET_KEY= +``` + +Supported IAM authentication methods (in SDK priority order): +- Environment variables (`AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY`) +- Java system properties +- Web Identity Token (EKS IRSA) +- AWS config file (`~/.aws/credentials`) +- EC2 Instance Profile +- ECS Task Role + ## Custom Implementation Implement `ObjectStorageService` interface and register as Spring Bean. diff --git a/server/skillhub-storage/pom.xml b/server/skillhub-storage/pom.xml index ec1674c31..939c0514b 100644 --- a/server/skillhub-storage/pom.xml +++ b/server/skillhub-storage/pom.xml @@ -30,6 +30,11 @@ apache-client 2.20.26 + + software.amazon.awssdk + sts + 2.20.26 + org.springframework.boot spring-boot-starter-test diff --git a/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/S3StorageService.java b/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/S3StorageService.java index d6a6327fd..e13adb402 100644 --- a/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/S3StorageService.java +++ b/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/S3StorageService.java @@ -6,6 +6,8 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.http.apache.ApacheHttpClient; @@ -50,19 +52,32 @@ void init() { .connectionAcquisitionTimeout(properties.getConnectionAcquisitionTimeout()); this.s3Client = buildS3Client(httpClientBuilder); this.s3Presigner = buildPresigner(); + String authMode = isStaticCredentials() ? "static credentials" : "IAM credentials (default provider chain)"; if (properties.isAutoCreateBucket()) { - log.info("Initialized S3 storage client for bucket '{}' (bucket auto-creation is deferred until first write)", - properties.getBucket()); + log.info("Initialized S3 storage client for bucket '{}' using {} (bucket auto-creation is deferred until first write)", + properties.getBucket(), authMode); } else { - log.info("Initialized S3 storage client for bucket '{}'", properties.getBucket()); + log.info("Initialized S3 storage client for bucket '{}' using {}", properties.getBucket(), authMode); } } + private boolean isStaticCredentials() { + return properties.getAccessKey() != null && !properties.getAccessKey().isBlank() + && properties.getSecretKey() != null && !properties.getSecretKey().isBlank(); + } + + AwsCredentialsProvider buildCredentialsProvider() { + if (isStaticCredentials()) { + return StaticCredentialsProvider.create( + AwsBasicCredentials.create(properties.getAccessKey(), properties.getSecretKey())); + } + return DefaultCredentialsProvider.create(); + } + protected S3Client buildS3Client(ApacheHttpClient.Builder httpClientBuilder) { var builder = S3Client.builder() .region(Region.of(properties.getRegion())) - .credentialsProvider(StaticCredentialsProvider.create( - AwsBasicCredentials.create(properties.getAccessKey(), properties.getSecretKey()))) + .credentialsProvider(buildCredentialsProvider()) .forcePathStyle(properties.isForcePathStyle()) .httpClientBuilder(httpClientBuilder) .overrideConfiguration(config -> config @@ -77,8 +92,7 @@ protected S3Client buildS3Client(ApacheHttpClient.Builder httpClientBuilder) { S3Presigner buildPresigner() { var presignerBuilder = S3Presigner.builder() .region(Region.of(properties.getRegion())) - .credentialsProvider(StaticCredentialsProvider.create( - AwsBasicCredentials.create(properties.getAccessKey(), properties.getSecretKey()))) + .credentialsProvider(buildCredentialsProvider()) .serviceConfiguration(S3Configuration.builder() .pathStyleAccessEnabled(properties.isForcePathStyle()) .build()); diff --git a/server/skillhub-storage/src/test/java/com/iflytek/skillhub/storage/S3StorageServiceTest.java b/server/skillhub-storage/src/test/java/com/iflytek/skillhub/storage/S3StorageServiceTest.java index 4e2439521..b18c737ce 100644 --- a/server/skillhub-storage/src/test/java/com/iflytek/skillhub/storage/S3StorageServiceTest.java +++ b/server/skillhub-storage/src/test/java/com/iflytek/skillhub/storage/S3StorageServiceTest.java @@ -1,6 +1,9 @@ package com.iflytek.skillhub.storage; import org.junit.jupiter.api.Test; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.http.apache.ApacheHttpClient; @@ -183,6 +186,52 @@ void putObjectShouldRetryWhenBucketWasCreatedConcurrently() { verify(client, times(2)).putObject(any(PutObjectRequest.class), any(RequestBody.class)); } + @Test + void shouldUseStaticCredentialsWhenAccessKeyAndSecretKeyProvided() { + S3StorageProperties props = createProperties(true); + S3StorageService service = new S3StorageService(props); + + AwsCredentialsProvider provider = service.buildCredentialsProvider(); + + assertThat(provider).isInstanceOf(StaticCredentialsProvider.class); + } + + @Test + void shouldUseDefaultCredentialsProviderWhenAccessKeyIsBlank() { + S3StorageProperties props = createProperties(true); + props.setAccessKey(""); + props.setSecretKey(""); + S3StorageService service = new S3StorageService(props); + + AwsCredentialsProvider provider = service.buildCredentialsProvider(); + + assertThat(provider).isInstanceOf(DefaultCredentialsProvider.class); + } + + @Test + void shouldUseDefaultCredentialsProviderWhenAccessKeyIsNull() { + S3StorageProperties props = createProperties(true); + props.setAccessKey(null); + props.setSecretKey(null); + S3StorageService service = new S3StorageService(props); + + AwsCredentialsProvider provider = service.buildCredentialsProvider(); + + assertThat(provider).isInstanceOf(DefaultCredentialsProvider.class); + } + + @Test + void shouldUseDefaultCredentialsProviderWhenOnlyAccessKeyIsSet() { + S3StorageProperties props = createProperties(true); + props.setAccessKey("some-key"); + props.setSecretKey(null); + S3StorageService service = new S3StorageService(props); + + AwsCredentialsProvider provider = service.buildCredentialsProvider(); + + assertThat(provider).isInstanceOf(DefaultCredentialsProvider.class); + } + private S3StorageProperties properties(boolean autoCreateBucket) { S3StorageProperties properties = createProperties(true); properties.setBucket("skillhub");