From 198403ecd6e76614cba323bc4d71a4d0c1ae71f4 Mon Sep 17 00:00:00 2001 From: yuta519 Date: Sat, 14 Feb 2026 11:05:19 -0500 Subject: [PATCH 1/4] fix: remove prefix for driver's CreateUploadURL functions --- src/pkg/clouds/aws/ecs/upload.go | 14 ++++++-------- src/pkg/clouds/do/appPlatform/setup.go | 14 ++++++-------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/src/pkg/clouds/aws/ecs/upload.go b/src/pkg/clouds/aws/ecs/upload.go index 9a9c05e7b..c0b65ab5a 100644 --- a/src/pkg/clouds/aws/ecs/upload.go +++ b/src/pkg/clouds/aws/ecs/upload.go @@ -13,22 +13,20 @@ import ( // From https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html var s3InvalidCharsRegexp = regexp.MustCompile(`[^a-zA-Z0-9!_.*'()-]`) -const prefix = "uploads/" - -func (a *AwsEcs) CreateUploadURL(ctx context.Context, name string) (string, error) { +func (a *AwsEcs) CreateUploadURL(ctx context.Context, objectKeyName string) (string, error) { cfg, err := a.LoadConfig(ctx) if err != nil { return "", err } - if name == "" { - name = uuid.NewString() + if objectKeyName == "" { + objectKeyName = uuid.NewString() } else { - if len(name) > 64 { + if len(objectKeyName) > 64 { return "", errors.New("name must be less than 64 characters") } // Sanitize the digest so it's safe to use as a file name - name = s3InvalidCharsRegexp.ReplaceAllString(name, "_") + objectKeyName = s3InvalidCharsRegexp.ReplaceAllString(objectKeyName, "_") // name = path.Join(buildsPath, tenantName.String(), digest); TODO: avoid collisions between tenants } @@ -36,7 +34,7 @@ func (a *AwsEcs) CreateUploadURL(ctx context.Context, name string) (string, erro // Use S3 SDK to create a presigned URL for uploading a file. req, err := s3.NewPresignClient(s3Client).PresignPutObject(ctx, &s3.PutObjectInput{ Bucket: &a.BucketName, - Key: ptr.String(prefix + name), + Key: ptr.String(objectKeyName), }) if err != nil { return "", err diff --git a/src/pkg/clouds/do/appPlatform/setup.go b/src/pkg/clouds/do/appPlatform/setup.go index 632f18147..6fb085cc3 100644 --- a/src/pkg/clouds/do/appPlatform/setup.go +++ b/src/pkg/clouds/do/appPlatform/setup.go @@ -233,22 +233,20 @@ func NewClient(ctx context.Context) *godo.Client { var s3InvalidCharsRegexp = regexp.MustCompile(`[^a-zA-Z0-9!_.*'()-]`) -func (d DoApp) CreateUploadURL(ctx context.Context, name string) (string, error) { +func (d DoApp) CreateUploadURL(ctx context.Context, objectKeyName string) (string, error) { s3Client, err := d.CreateS3Client() if err != nil { return "", err } - prefix := "uploads/" - - if name == "" { - name = uuid.NewString() + if objectKeyName == "" { + objectKeyName = uuid.NewString() } else { - if len(name) > 64 { + if len(objectKeyName) > 64 { return "", errors.New("name must be less than 64 characters") } // Sanitize the digest so it's safe to use as a file name - name = s3InvalidCharsRegexp.ReplaceAllString(name, "_") + objectKeyName = s3InvalidCharsRegexp.ReplaceAllString(objectKeyName, "_") // name = path.Join(buildsPath, tenantName.String(), digest); TODO: avoid collisions between tenants } @@ -257,7 +255,7 @@ func (d DoApp) CreateUploadURL(ctx context.Context, name string) (string, error) // so should we just stick to the S3 SDK for all S3 operations, instead of using presigned URLs? req, err := s3.NewPresignClient(s3Client).PresignPutObject(ctx, &s3.PutObjectInput{ Bucket: &d.BucketName, - Key: ptr.String(prefix + name), + Key: &objectKeyName, }) if err != nil { From db5ade7ec495c9de248444eae95fad8d0e986c70 Mon Sep 17 00:00:00 2001 From: yuta519 Date: Sat, 14 Feb 2026 11:06:10 -0500 Subject: [PATCH 2/4] feat: consolidate UploadPrefix for all byoc functions --- src/pkg/cli/client/byoc/aws/byoc.go | 5 +++-- src/pkg/cli/client/byoc/common.go | 1 + src/pkg/cli/client/byoc/do/byoc.go | 5 +++-- src/pkg/cli/client/byoc/gcp/byoc.go | 5 ++--- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/pkg/cli/client/byoc/aws/byoc.go b/src/pkg/cli/client/byoc/aws/byoc.go index 39807bba8..ff3626983 100644 --- a/src/pkg/cli/client/byoc/aws/byoc.go +++ b/src/pkg/cli/client/byoc/aws/byoc.go @@ -10,6 +10,7 @@ import ( "io" "iter" "os" + "path" "strconv" "strings" "sync" @@ -238,7 +239,7 @@ func (b *ByocAws) deploy(ctx context.Context, req *client.DeployRequest, cmd str payloadString = base64.StdEncoding.EncodeToString(data) // TODO: consider making this a proper Data URL: "data:application/protobuf;base64,abcd…" } else { - payloadUrl, err := b.driver.CreateUploadURL(ctx, etag) + payloadUrl, err := b.driver.CreateUploadURL(ctx, path.Join(byoc.UploadPrefix, etag)) if err != nil { return nil, err } @@ -670,7 +671,7 @@ func (b *ByocAws) CreateUploadURL(ctx context.Context, req *defangv1.UploadURLRe return nil, err } - url, err := b.driver.CreateUploadURL(ctx, req.Digest) + url, err := b.driver.CreateUploadURL(ctx, path.Join(byoc.UploadPrefix, req.Digest)) if err != nil { return nil, err } diff --git a/src/pkg/cli/client/byoc/common.go b/src/pkg/cli/client/byoc/common.go index 224c628c2..dcb55425f 100644 --- a/src/pkg/cli/client/byoc/common.go +++ b/src/pkg/cli/client/byoc/common.go @@ -14,6 +14,7 @@ import ( const ( CdTaskPrefix = "defang-cd" // WARNING: renaming this practically deletes the Pulumi state + UploadPrefix = "uploads/" ) var ( diff --git a/src/pkg/cli/client/byoc/do/byoc.go b/src/pkg/cli/client/byoc/do/byoc.go index a70ae8d5a..ea0b2db8a 100644 --- a/src/pkg/cli/client/byoc/do/byoc.go +++ b/src/pkg/cli/client/byoc/do/byoc.go @@ -10,6 +10,7 @@ import ( "iter" "net/url" "os" + "path" "regexp" "strings" "time" @@ -162,7 +163,7 @@ func (b *ByocDo) deploy(ctx context.Context, req *client.DeployRequest, cmd stri return nil, err } - payloadUrl, err := b.driver.CreateUploadURL(ctx, etag) + payloadUrl, err := b.driver.CreateUploadURL(ctx, path.Join(byoc.UploadPrefix, etag)) if err != nil { return nil, err } @@ -281,7 +282,7 @@ func (b *ByocDo) CreateUploadURL(ctx context.Context, req *defangv1.UploadURLReq return nil, err } - url, err := b.driver.CreateUploadURL(ctx, req.Digest) + url, err := b.driver.CreateUploadURL(ctx, path.Join(byoc.UploadPrefix, req.Digest)) if err != nil { return nil, err diff --git a/src/pkg/cli/client/byoc/gcp/byoc.go b/src/pkg/cli/client/byoc/gcp/byoc.go index af008ef61..7106ec380 100644 --- a/src/pkg/cli/client/byoc/gcp/byoc.go +++ b/src/pkg/cli/client/byoc/gcp/byoc.go @@ -41,7 +41,6 @@ var _ client.Provider = (*ByocGcp)(nil) const ( DefangCDProjectName = "defang-cd" DefangUploadServiceAccountName = "defang-upload" - UploadPrefix = "uploads/" ) var ( @@ -484,7 +483,7 @@ func (b *ByocGcp) CreateUploadURL(ctx context.Context, req *defangv1.UploadURLRe return nil, err } - url, err := b.driver.CreateUploadURL(ctx, b.bucket, path.Join(UploadPrefix, req.Digest), b.uploadServiceAccount) + url, err := b.driver.CreateUploadURL(ctx, b.bucket, path.Join(byoc.UploadPrefix, req.Digest), b.uploadServiceAccount) if err != nil { if strings.Contains(err.Error(), "Permission 'iam.serviceAccounts.signBlob' denied on resource") { return nil, errors.New("current user does not have 'iam.serviceAccounts.signBlob' permission. If it has been recently added, please wait a few minutes then try again") @@ -538,7 +537,7 @@ func (b *ByocGcp) deploy(ctx context.Context, req *client.DeployRequest, command if len(data) < 1000 { payload = base64.StdEncoding.EncodeToString(data) } else { - payloadUrl, err := b.driver.CreateUploadURL(ctx, b.bucket, path.Join(UploadPrefix, etag), b.uploadServiceAccount) + payloadUrl, err := b.driver.CreateUploadURL(ctx, b.bucket, path.Join(byoc.UploadPrefix, etag), b.uploadServiceAccount) if err != nil { return nil, err } From 3010d2ec1d92859099de73012d528f5cd15456bd Mon Sep 17 00:00:00 2001 From: yuta519 Date: Sat, 14 Feb 2026 13:16:18 -0500 Subject: [PATCH 3/4] feat: change CreateUploadURL signature to add prefix --- src/pkg/clouds/driver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pkg/clouds/driver.go b/src/pkg/clouds/driver.go index 71a16f1f2..4d98ac4b3 100644 --- a/src/pkg/clouds/driver.go +++ b/src/pkg/clouds/driver.go @@ -50,7 +50,7 @@ type Driver interface { PutSecret(ctx context.Context, name, value string) error // DeleteSecrets(ctx context.Context, names ...string) error ListSecrets(ctx context.Context) ([]string, error) // no values - CreateUploadURL(ctx context.Context, name string) (string, error) + CreateUploadURL(ctx context.Context, prefix string, name string) (string, error) } type TaskInfo struct { From bfd9328c22e816f7abe9d95f71f73366072f2786 Mon Sep 17 00:00:00 2001 From: yuta519 Date: Sat, 14 Feb 2026 13:18:15 -0500 Subject: [PATCH 4/4] feat: update CreateUploadURL args --- src/pkg/cli/client/byoc/aws/byoc.go | 5 ++--- src/pkg/cli/client/byoc/do/byoc.go | 5 ++--- src/pkg/clouds/aws/ecs/upload.go | 12 ++++++------ src/pkg/clouds/do/appPlatform/setup.go | 12 ++++++------ src/pkg/crun/docker/common.go | 2 +- src/pkg/crun/local/local.go | 2 +- 6 files changed, 18 insertions(+), 20 deletions(-) diff --git a/src/pkg/cli/client/byoc/aws/byoc.go b/src/pkg/cli/client/byoc/aws/byoc.go index ff3626983..9d24df7f3 100644 --- a/src/pkg/cli/client/byoc/aws/byoc.go +++ b/src/pkg/cli/client/byoc/aws/byoc.go @@ -10,7 +10,6 @@ import ( "io" "iter" "os" - "path" "strconv" "strings" "sync" @@ -239,7 +238,7 @@ func (b *ByocAws) deploy(ctx context.Context, req *client.DeployRequest, cmd str payloadString = base64.StdEncoding.EncodeToString(data) // TODO: consider making this a proper Data URL: "data:application/protobuf;base64,abcd…" } else { - payloadUrl, err := b.driver.CreateUploadURL(ctx, path.Join(byoc.UploadPrefix, etag)) + payloadUrl, err := b.driver.CreateUploadURL(ctx, byoc.UploadPrefix, etag) if err != nil { return nil, err } @@ -671,7 +670,7 @@ func (b *ByocAws) CreateUploadURL(ctx context.Context, req *defangv1.UploadURLRe return nil, err } - url, err := b.driver.CreateUploadURL(ctx, path.Join(byoc.UploadPrefix, req.Digest)) + url, err := b.driver.CreateUploadURL(ctx, byoc.UploadPrefix, req.Digest) if err != nil { return nil, err } diff --git a/src/pkg/cli/client/byoc/do/byoc.go b/src/pkg/cli/client/byoc/do/byoc.go index ea0b2db8a..a7571a476 100644 --- a/src/pkg/cli/client/byoc/do/byoc.go +++ b/src/pkg/cli/client/byoc/do/byoc.go @@ -10,7 +10,6 @@ import ( "iter" "net/url" "os" - "path" "regexp" "strings" "time" @@ -163,7 +162,7 @@ func (b *ByocDo) deploy(ctx context.Context, req *client.DeployRequest, cmd stri return nil, err } - payloadUrl, err := b.driver.CreateUploadURL(ctx, path.Join(byoc.UploadPrefix, etag)) + payloadUrl, err := b.driver.CreateUploadURL(ctx, byoc.UploadPrefix, etag) if err != nil { return nil, err } @@ -282,7 +281,7 @@ func (b *ByocDo) CreateUploadURL(ctx context.Context, req *defangv1.UploadURLReq return nil, err } - url, err := b.driver.CreateUploadURL(ctx, path.Join(byoc.UploadPrefix, req.Digest)) + url, err := b.driver.CreateUploadURL(ctx, byoc.UploadPrefix, req.Digest) if err != nil { return nil, err diff --git a/src/pkg/clouds/aws/ecs/upload.go b/src/pkg/clouds/aws/ecs/upload.go index c0b65ab5a..1296d5320 100644 --- a/src/pkg/clouds/aws/ecs/upload.go +++ b/src/pkg/clouds/aws/ecs/upload.go @@ -13,20 +13,20 @@ import ( // From https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html var s3InvalidCharsRegexp = regexp.MustCompile(`[^a-zA-Z0-9!_.*'()-]`) -func (a *AwsEcs) CreateUploadURL(ctx context.Context, objectKeyName string) (string, error) { +func (a *AwsEcs) CreateUploadURL(ctx context.Context, prefix string, filename string) (string, error) { cfg, err := a.LoadConfig(ctx) if err != nil { return "", err } - if objectKeyName == "" { - objectKeyName = uuid.NewString() + if filename == "" { + filename = uuid.NewString() } else { - if len(objectKeyName) > 64 { + if len(filename) > 64 { return "", errors.New("name must be less than 64 characters") } // Sanitize the digest so it's safe to use as a file name - objectKeyName = s3InvalidCharsRegexp.ReplaceAllString(objectKeyName, "_") + filename = s3InvalidCharsRegexp.ReplaceAllString(filename, "_") // name = path.Join(buildsPath, tenantName.String(), digest); TODO: avoid collisions between tenants } @@ -34,7 +34,7 @@ func (a *AwsEcs) CreateUploadURL(ctx context.Context, objectKeyName string) (str // Use S3 SDK to create a presigned URL for uploading a file. req, err := s3.NewPresignClient(s3Client).PresignPutObject(ctx, &s3.PutObjectInput{ Bucket: &a.BucketName, - Key: ptr.String(objectKeyName), + Key: ptr.String(prefix + filename), }) if err != nil { return "", err diff --git a/src/pkg/clouds/do/appPlatform/setup.go b/src/pkg/clouds/do/appPlatform/setup.go index 6fb085cc3..97fe7a5f3 100644 --- a/src/pkg/clouds/do/appPlatform/setup.go +++ b/src/pkg/clouds/do/appPlatform/setup.go @@ -233,20 +233,20 @@ func NewClient(ctx context.Context) *godo.Client { var s3InvalidCharsRegexp = regexp.MustCompile(`[^a-zA-Z0-9!_.*'()-]`) -func (d DoApp) CreateUploadURL(ctx context.Context, objectKeyName string) (string, error) { +func (d DoApp) CreateUploadURL(ctx context.Context, prefix string, filename string) (string, error) { s3Client, err := d.CreateS3Client() if err != nil { return "", err } - if objectKeyName == "" { - objectKeyName = uuid.NewString() + if filename == "" { + filename = uuid.NewString() } else { - if len(objectKeyName) > 64 { + if len(filename) > 64 { return "", errors.New("name must be less than 64 characters") } // Sanitize the digest so it's safe to use as a file name - objectKeyName = s3InvalidCharsRegexp.ReplaceAllString(objectKeyName, "_") + filename = s3InvalidCharsRegexp.ReplaceAllString(filename, "_") // name = path.Join(buildsPath, tenantName.String(), digest); TODO: avoid collisions between tenants } @@ -255,7 +255,7 @@ func (d DoApp) CreateUploadURL(ctx context.Context, objectKeyName string) (strin // so should we just stick to the S3 SDK for all S3 operations, instead of using presigned URLs? req, err := s3.NewPresignClient(s3Client).PresignPutObject(ctx, &s3.PutObjectInput{ Bucket: &d.BucketName, - Key: &objectKeyName, + Key: ptr.String(prefix + filename), }) if err != nil { diff --git a/src/pkg/crun/docker/common.go b/src/pkg/crun/docker/common.go index c5384c1b8..860423d86 100644 --- a/src/pkg/crun/docker/common.go +++ b/src/pkg/crun/docker/common.go @@ -39,6 +39,6 @@ func (Docker) ListSecrets(ctx context.Context) ([]string, error) { return nil, errors.New("docker does not support secrets") } -func (Docker) CreateUploadURL(ctx context.Context, name string) (string, error) { +func (Docker) CreateUploadURL(ctx context.Context, prefix, name string) (string, error) { return "", errors.New("docker does not support uploads") } diff --git a/src/pkg/crun/local/local.go b/src/pkg/crun/local/local.go index c0b0bcd6f..ed666ea19 100644 --- a/src/pkg/crun/local/local.go +++ b/src/pkg/crun/local/local.go @@ -113,6 +113,6 @@ func (l *Local) ListSecrets(ctx context.Context) ([]string, error) { return nil, client.ErrNotImplemented("not implemented for local driver") } -func (l *Local) CreateUploadURL(ctx context.Context, name string) (string, error) { +func (l *Local) CreateUploadURL(ctx context.Context, prefix, name string) (string, error) { return "", client.ErrNotImplemented("not implemented for local driver") }