diff --git a/.env.release.example b/.env.release.example index 3f020fc75..930ab826f 100644 --- a/.env.release.example +++ b/.env.release.example @@ -34,6 +34,8 @@ SKILLHUB_STORAGE_PROVIDER=local SKILLHUB_STORAGE_S3_ENDPOINT=https://oss-cn-example.aliyuncs.com SKILLHUB_STORAGE_S3_PUBLIC_ENDPOINT= SKILLHUB_STORAGE_S3_BUCKET=skillhub-prod +# Static credentials for S3-compatible storage (MinIO, Alibaba OSS, etc.). +# Leave both blank to use IAM authentication (EC2 instance profile, ECS task role, EKS IRSA). SKILLHUB_STORAGE_S3_ACCESS_KEY=replace-me SKILLHUB_STORAGE_S3_SECRET_KEY=replace-me SKILLHUB_STORAGE_S3_REGION=cn-shanghai diff --git a/document/docs/04-developer/plugins/storage-spi.md b/document/docs/04-developer/plugins/storage-spi.md index f81189e17..3b7f4dbaa 100644 --- a/document/docs/04-developer/plugins/storage-spi.md +++ b/document/docs/04-developer/plugins/storage-spi.md @@ -34,6 +34,8 @@ S3 协议兼容实现，支持： ## 配置 +### 静态凭据（Access Key / Secret Key） + ```bash # 选择存储提供方 SKILLHUB_STORAGE_PROVIDER=s3 @@ -45,6 +47,27 @@ SKILLHUB_STORAGE_S3_ACCESS_KEY=xxx SKILLHUB_STORAGE_S3_SECRET_KEY=xxx ``` +### IAM 认证 + +部署在 AWS 上时，可以不配置 Access Key / Secret Key，让 SDK 自动使用 IAM 角色认证（[Default Credentials Provider Chain](https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/credentials-chain.html)）： + +```bash +SKILLHUB_STORAGE_PROVIDER=s3 +SKILLHUB_STORAGE_S3_BUCKET=skillhub +SKILLHUB_STORAGE_S3_REGION=us-east-1 +# 留空或不设置 ACCESS_KEY / SECRET_KEY，SDK 自动使用 IAM 认证 +SKILLHUB_STORAGE_S3_ACCESS_KEY= +SKILLHUB_STORAGE_S3_SECRET_KEY= +``` + +支持的 IAM 认证方式（按 SDK 优先级）： +- 环境变量（`AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY`） +- Java 系统属性 +- Web Identity Token（EKS IRSA） +- AWS 配置文件（`~/.aws/credentials`） +- EC2 Instance Profile +- ECS Task Role + ## 自定义实现 实现 `ObjectStorageService` 接口，注册为 Spring Bean 即可。 diff --git a/document/i18n/en/docusaurus-plugin-content-docs/current/04-developer/plugins/storage-spi.md b/document/i18n/en/docusaurus-plugin-content-docs/current/04-developer/plugins/storage-spi.md index f3a0a57d1..2667e6155 100644 --- a/document/i18n/en/docusaurus-plugin-content-docs/current/04-developer/plugins/storage-spi.md +++ b/document/i18n/en/docusaurus-plugin-content-docs/current/04-developer/plugins/storage-spi.md @@ -34,6 +34,8 @@ S3 protocol compatible implementation, supports: ## Configuration +### Static Credentials (Access Key / Secret Key) + ```bash # Select storage provider SKILLHUB_STORAGE_PROVIDER=s3 @@ -45,6 +47,27 @@ SKILLHUB_STORAGE_S3_ACCESS_KEY=xxx SKILLHUB_STORAGE_S3_SECRET_KEY=xxx ``` +### IAM Authentication + +When deployed on AWS, you can omit the Access Key / Secret Key and let the SDK use IAM role authentication via the [Default Credentials Provider Chain](https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/credentials-chain.html): + +```bash +SKILLHUB_STORAGE_PROVIDER=s3 +SKILLHUB_STORAGE_S3_BUCKET=skillhub +SKILLHUB_STORAGE_S3_REGION=us-east-1 +# Leave ACCESS_KEY / SECRET_KEY blank to use IAM authentication +SKILLHUB_STORAGE_S3_ACCESS_KEY= +SKILLHUB_STORAGE_S3_SECRET_KEY= +``` + +Supported IAM authentication methods (in SDK priority order): +- Environment variables (`AWS_ACCESS_KEY_ID` / `AWS_SECRET_ACCESS_KEY`) +- Java system properties +- Web Identity Token (EKS IRSA) +- AWS config file (`~/.aws/credentials`) +- EC2 Instance Profile +- ECS Task Role + ## Custom Implementation Implement `ObjectStorageService` interface and register as Spring Bean. diff --git a/server/skillhub-storage/pom.xml b/server/skillhub-storage/pom.xml index ec1674c31..939c0514b 100644 --- a/server/skillhub-storage/pom.xml +++ b/server/skillhub-storage/pom.xml @@ -30,6 +30,11 @@ apache-client 2.20.26 + + software.amazon.awssdk + sts + 2.20.26 + org.springframework.boot spring-boot-starter-test diff --git a/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/S3StorageService.java b/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/S3StorageService.java index d6a6327fd..e13adb402 100644 --- a/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/S3StorageService.java +++ b/server/skillhub-storage/src/main/java/com/iflytek/skillhub/storage/S3StorageService.java @@ -6,6 +6,8 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Service; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.http.apache.ApacheHttpClient; @@ -50,19 +52,32 @@ void init() { .connectionAcquisitionTimeout(properties.getConnectionAcquisitionTimeout()); this.s3Client = buildS3Client(httpClientBuilder); this.s3Presigner = buildPresigner(); + String authMode = isStaticCredentials() ? "static credentials" : "IAM credentials (default provider chain)"; if (properties.isAutoCreateBucket()) { - log.info("Initialized S3 storage client for bucket '{}' (bucket auto-creation is deferred until first write)", - properties.getBucket()); + log.info("Initialized S3 storage client for bucket '{}' using {} (bucket auto-creation is deferred until first write)", + properties.getBucket(), authMode); } else { - log.info("Initialized S3 storage client for bucket '{}'", properties.getBucket()); + log.info("Initialized S3 storage client for bucket '{}' using {}", properties.getBucket(), authMode); } } + private boolean isStaticCredentials() { + return properties.getAccessKey() != null && !properties.getAccessKey().isBlank() + && properties.getSecretKey() != null && !properties.getSecretKey().isBlank(); + } + + AwsCredentialsProvider buildCredentialsProvider() { + if (isStaticCredentials()) { + return StaticCredentialsProvider.create( + AwsBasicCredentials.create(properties.getAccessKey(), properties.getSecretKey())); + } + return DefaultCredentialsProvider.create(); + } + protected S3Client buildS3Client(ApacheHttpClient.Builder httpClientBuilder) { var builder = S3Client.builder() .region(Region.of(properties.getRegion())) - .credentialsProvider(StaticCredentialsProvider.create( - AwsBasicCredentials.create(properties.getAccessKey(), properties.getSecretKey()))) + .credentialsProvider(buildCredentialsProvider()) .forcePathStyle(properties.isForcePathStyle()) .httpClientBuilder(httpClientBuilder) .overrideConfiguration(config -> config @@ -77,8 +92,7 @@ protected S3Client buildS3Client(ApacheHttpClient.Builder httpClientBuilder) { S3Presigner buildPresigner() { var presignerBuilder = S3Presigner.builder() .region(Region.of(properties.getRegion())) - .credentialsProvider(StaticCredentialsProvider.create( - AwsBasicCredentials.create(properties.getAccessKey(), properties.getSecretKey()))) + .credentialsProvider(buildCredentialsProvider()) .serviceConfiguration(S3Configuration.builder() .pathStyleAccessEnabled(properties.isForcePathStyle()) .build()); diff --git a/server/skillhub-storage/src/test/java/com/iflytek/skillhub/storage/S3StorageServiceTest.java b/server/skillhub-storage/src/test/java/com/iflytek/skillhub/storage/S3StorageServiceTest.java index 4e2439521..b18c737ce 100644 --- a/server/skillhub-storage/src/test/java/com/iflytek/skillhub/storage/S3StorageServiceTest.java +++ b/server/skillhub-storage/src/test/java/com/iflytek/skillhub/storage/S3StorageServiceTest.java @@ -1,6 +1,9 @@ package com.iflytek.skillhub.storage; import org.junit.jupiter.api.Test; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.http.apache.ApacheHttpClient; @@ -183,6 +186,52 @@ void putObjectShouldRetryWhenBucketWasCreatedConcurrently() { verify(client, times(2)).putObject(any(PutObjectRequest.class), any(RequestBody.class)); } + @Test + void shouldUseStaticCredentialsWhenAccessKeyAndSecretKeyProvided() { + S3StorageProperties props = createProperties(true); + S3StorageService service = new S3StorageService(props); + + AwsCredentialsProvider provider = service.buildCredentialsProvider(); + + assertThat(provider).isInstanceOf(StaticCredentialsProvider.class); + } + + @Test + void shouldUseDefaultCredentialsProviderWhenAccessKeyIsBlank() { + S3StorageProperties props = createProperties(true); + props.setAccessKey(""); + props.setSecretKey(""); + S3StorageService service = new S3StorageService(props); + + AwsCredentialsProvider provider = service.buildCredentialsProvider(); + + assertThat(provider).isInstanceOf(DefaultCredentialsProvider.class); + } + + @Test + void shouldUseDefaultCredentialsProviderWhenAccessKeyIsNull() { + S3StorageProperties props = createProperties(true); + props.setAccessKey(null); + props.setSecretKey(null); + S3StorageService service = new S3StorageService(props); + + AwsCredentialsProvider provider = service.buildCredentialsProvider(); + + assertThat(provider).isInstanceOf(DefaultCredentialsProvider.class); + } + + @Test + void shouldUseDefaultCredentialsProviderWhenOnlyAccessKeyIsSet() { + S3StorageProperties props = createProperties(true); + props.setAccessKey("some-key"); + props.setSecretKey(null); + S3StorageService service = new S3StorageService(props); + + AwsCredentialsProvider provider = service.buildCredentialsProvider(); + + assertThat(provider).isInstanceOf(DefaultCredentialsProvider.class); + } + private S3StorageProperties properties(boolean autoCreateBucket) { S3StorageProperties properties = createProperties(true); properties.setBucket("skillhub");