Get urls and pass to CD for states and events uploading

src/pkg/cli/cd.go

@@ -3,6 +3,8 @@ package cli
 import (
 	"context"
 	"errors"
+	"fmt"
+	"os"
 	"strings"
 	"time"
 
@@ -26,13 +28,34 @@ func CdCommand(ctx context.Context, projectName string, provider client.Provider
 		return "", dryrun.ErrDryRun
 	}
 
-	etag, err := provider.CdCommand(ctx, client.CdCommandRequest{Project: projectName, Command: command})
+	var statesUrl, eventsUrl string
+	if _, ok := provider.(*client.PlaygroundProvider); !ok { // Do not need upload URLs for Playground
+		var err error
+		statesUrl, eventsUrl, err = GetStatesAndEventsUploadUrls(ctx, projectName, provider, fabric)
+		if err != nil {
+			return "", err
+		}
+	}
+
+	etag, err := provider.CdCommand(ctx, client.CdCommandRequest{Project: projectName, Command: command, StatesUrl: statesUrl, EventsUrl: eventsUrl})
 	if err != nil || etag == "" {
 		return "", err
 	}
 
+	// Update deployment table to mark deployment as destroyed only after successful deletion of the subdomain
+	err = putDeployment(ctx, provider, fabric, putDeploymentParams{
+		Action:      defangv1.DeploymentAction_DEPLOYMENT_ACTION_DOWN,
+		ETag:        etag,
+		ProjectName: projectName,
+		StatesUrl:   statesUrl,
+		EventsUrl:   eventsUrl,
+	})
+	if err != nil {
+		return "", err
+	}
+
 	if command == client.CdCommandDestroy || command == client.CdCommandDown {
-		err := postDown(ctx, projectName, provider, fabric, etag)
+		err := postDown(ctx, projectName, provider, fabric)
 		if err != nil {
 			term.Debugf("postDown failed: %v", err)
 			term.Warn("Unable to update deployment history; deployment will proceed anyway.")
@@ -41,7 +64,7 @@ func CdCommand(ctx context.Context, projectName string, provider client.Provider
 	return etag, nil
 }
 
-func postDown(ctx context.Context, projectName string, provider client.Provider, fabric client.FabricClient, etag types.ETag) error {
+func postDown(ctx context.Context, projectName string, provider client.Provider, fabric client.FabricClient) error {
 	// Special bookkeeping for "down" commands: delete the subdomain zone and mark deployment as destroyed
 	err := fabric.DeleteSubdomainZone(ctx, &defangv1.DeleteSubdomainZoneRequest{
 		Project: projectName,
@@ -55,17 +78,6 @@ func postDown(ctx context.Context, projectName string, provider client.Provider,
 		}
 		return err
 	}
-
-	// Update deployment table to mark deployment as destroyed only after successful deletion of the subdomain
-	err = putDeployment(ctx, provider, fabric, putDeploymentParams{
-		Action:      defangv1.DeploymentAction_DEPLOYMENT_ACTION_DOWN,
-		ETag:        etag,
-		ProjectName: projectName,
-	})
-	if err != nil {
-		term.Debug("Failed to record deployment:", err)
-		return err
-	}
 	return nil
 }
 
@@ -144,3 +156,32 @@ func CdListLocal(ctx context.Context, provider client.Provider, allRegions bool)
 	}
 	return nil
 }
+
+func GetStatesAndEventsUploadUrls(ctx context.Context, projectName string, provider client.Provider, fabric client.FabricClient) (statesUrl string, eventsUrl string, err error) {
+	// Allow overriding upload URLs via environment variables
+	statesUrl, eventsUrl = os.Getenv("DEFANG_STATES_UPLOAD_URL"), os.Getenv("DEFANG_EVENTS_UPLOAD_URL")
+
+	if statesUrl == "" {
+		statesResp, err := fabric.CreateUploadURL(ctx, &defangv1.UploadURLRequest{
+			Project:  projectName,
+			Stack:    provider.GetStackName(),
+			Filename: "states.json",
+		})
+		if err != nil {
+			return "", "", fmt.Errorf("failed to create states upload URL: %w", err)
+		}
+		statesUrl = statesResp.Url
+	}
+	if eventsUrl == "" {
+		eventsResp, err := fabric.CreateUploadURL(ctx, &defangv1.UploadURLRequest{
+			Project:  projectName,
+			Stack:    provider.GetStackName(),
+			Filename: "events.log",
+		})
+		if err != nil {
+			return "", "", fmt.Errorf("failed to create events upload URL: %w", err)
+		}
+		eventsUrl = eventsResp.Url
+	}
+	return statesUrl, eventsUrl, nil
+}

src/pkg/cli/client/byoc/aws/byoc.go

@@ -172,15 +172,15 @@ func (b *ByocAws) GetDeploymentStatus(ctx context.Context) error {
 	return nil
 }
 
-func (b *ByocAws) Deploy(ctx context.Context, req *defangv1.DeployRequest) (*defangv1.DeployResponse, error) {
+func (b *ByocAws) Deploy(ctx context.Context, req *client.DeployRequest) (*defangv1.DeployResponse, error) {
 	return b.deploy(ctx, req, "up")
 }
 
-func (b *ByocAws) Preview(ctx context.Context, req *defangv1.DeployRequest) (*defangv1.DeployResponse, error) {
+func (b *ByocAws) Preview(ctx context.Context, req *client.DeployRequest) (*defangv1.DeployResponse, error) {
 	return b.deploy(ctx, req, "preview")
 }
 
-func (b *ByocAws) deploy(ctx context.Context, req *defangv1.DeployRequest, cmd string) (*defangv1.DeployResponse, error) {
+func (b *ByocAws) deploy(ctx context.Context, req *client.DeployRequest, cmd string) (*defangv1.DeployResponse, error) {
 	cfg, err := b.driver.LoadConfig(ctx)
 	if err != nil {
 		return nil, AnnotateAwsError(err)
@@ -246,6 +246,8 @@ func (b *ByocAws) deploy(ctx context.Context, req *defangv1.DeployRequest, cmd s
 		delegationSetId: req.DelegationSetId,
 		mode:            req.Mode,
 		project:         project.Name,
+		statesUrl:       req.StatesUrl,
+		eventsUrl:       req.EventsUrl,
 	}
 
 	if b.needDockerHubCreds {
@@ -502,6 +504,9 @@ type cdCommand struct {
 
 	dockerHubUsername    string
 	dockerHubAccessToken string
+
+	statesUrl string
+	eventsUrl string
 }
 
 func (b *ByocAws) runCdCommand(ctx context.Context, cmd cdCommand) (ecs.TaskArn, error) {
@@ -542,6 +547,15 @@ func (b *ByocAws) runCdCommand(ctx context.Context, cmd cdCommand) (ecs.TaskArn,
 			return nil, err
 		}
 	}
+
+	if cmd.statesUrl != "" {
+		env["DEFANG_STATES_UPLOAD_URL"] = cmd.statesUrl
+	}
+
+	if cmd.eventsUrl != "" {
+		env["DEFANG_EVENTS_UPLOAD_URL"] = cmd.eventsUrl
+	}
+
 	return b.driver.Run(ctx, env, cmd.command...)
 }
 
@@ -904,8 +918,10 @@ func (b *ByocAws) CdCommand(ctx context.Context, req client.CdCommandRequest) (s
 		return "", err
 	}
 	cmd := cdCommand{
-		project: req.Project,
-		command: []string{string(req.Command)},
+		project:   req.Project,
+		command:   []string{string(req.Command)},
+		statesUrl: req.StatesUrl,
+		eventsUrl: req.EventsUrl,
 	}
 	cdTaskArn, err := b.runCdCommand(ctx, cmd) // TODO: make domain optional for defang cd
 	if err != nil {

src/pkg/cli/client/byoc/aws/byoc_integration_test.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/DefangLabs/defang/src/pkg/cli/client"
 	"github.com/DefangLabs/defang/src/pkg/cli/client/byoc"
 	"github.com/DefangLabs/defang/src/pkg/cli/compose"
 	"github.com/DefangLabs/defang/src/pkg/clouds/aws"
@@ -21,9 +22,9 @@ func TestDeploy(t *testing.T) {
 	t.Run("multiple ingress without domain", func(t *testing.T) {
 		t.Skip("skipping test: delegation enabled")
 
-		_, err := b.Deploy(context.Background(), &defangv1.DeployRequest{
+		_, err := b.Deploy(context.Background(), &client.DeployRequest{DeployRequest: defangv1.DeployRequest{
 			Project: "byoc_integration_test",
-		})
+		}})
 		if err == nil || !strings.Contains(err.Error(), "duplicate endpoint:") {
 			t.Error("expected error")
 		}

src/pkg/cli/client/byoc/aws/validation_test.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"testing"
 
+	"github.com/DefangLabs/defang/src/pkg/cli/client"
 	"github.com/DefangLabs/defang/src/pkg/cli/client/byoc"
 	"github.com/DefangLabs/defang/src/pkg/clouds/aws"
 	"github.com/DefangLabs/defang/src/pkg/clouds/aws/ecs/cfn"
@@ -131,7 +132,7 @@ func TestDeployValidateGPUResources(t *testing.T) {
 	b.ByocBaseClient.SetupDone = true
 
 	t.Run("no errors", func(t *testing.T) {
-		testDeploy := defangv1.DeployRequest{
+		testDeploy := client.DeployRequest{DeployRequest: defangv1.DeployRequest{
 			Compose: []byte(
 				`name: project
 services:
@@ -144,7 +145,7 @@ services:
             - capabilities: [gpu]
               count: 1
 `),
-		}
+		}}
 
 		quotaClient = nil
 		_, err := b.Deploy(ctx, &testDeploy)
@@ -154,7 +155,7 @@ services:
 	})
 
 	t.Run("error on too many gpu", func(t *testing.T) {
-		testDeploy := defangv1.DeployRequest{
+		testDeploy := client.DeployRequest{DeployRequest: defangv1.DeployRequest{
 			Compose: []byte(
 				`name: project
 services:
@@ -167,7 +168,7 @@ services:
             - capabilities: [gpu]
               count: 24
 `),
-		}
+		}}
 
 		_, err := b.Deploy(ctx, &testDeploy)
 		if err != nil && !errors.Is(err, ErrGPUQuotaZero) && !errors.As(err, &errAWSOperation) {
@@ -176,7 +177,7 @@ services:
 	})
 
 	t.Run("no error on non-gpu resource", func(t *testing.T) {
-		testDeploy := defangv1.DeployRequest{
+		testDeploy := client.DeployRequest{DeployRequest: defangv1.DeployRequest{
 			Compose: []byte(
 				`name: project
 services:
@@ -189,7 +190,7 @@ services:
             - capabilities: [not-gpu]
               count: 24
 `),
-		}
+		}}
 
 		_, err := b.Deploy(ctx, &testDeploy)
 		if err != nil && errors.Is(err, ErrGPUQuotaZero) {

src/pkg/cli/client/byoc/do/byoc.go

@@ -126,15 +126,15 @@ func (b *ByocDo) GetProjectUpdate(ctx context.Context, projectName string) (*def
 	return &projUpdate, nil
 }
 
-func (b *ByocDo) Preview(ctx context.Context, req *defangv1.DeployRequest) (*defangv1.DeployResponse, error) {
+func (b *ByocDo) Preview(ctx context.Context, req *client.DeployRequest) (*defangv1.DeployResponse, error) {
 	return b.deploy(ctx, req, "preview")
 }
 
-func (b *ByocDo) Deploy(ctx context.Context, req *defangv1.DeployRequest) (*defangv1.DeployResponse, error) {
+func (b *ByocDo) Deploy(ctx context.Context, req *client.DeployRequest) (*defangv1.DeployResponse, error) {
 	return b.deploy(ctx, req, "up")
 }
 
-func (b *ByocDo) deploy(ctx context.Context, req *defangv1.DeployRequest, cmd string) (*defangv1.DeployResponse, error) {
+func (b *ByocDo) deploy(ctx context.Context, req *client.DeployRequest, cmd string) (*defangv1.DeployResponse, error) {
 	// If multiple Compose files were provided, req.Compose is the merged representation of all the files
 	project, err := compose.LoadFromContent(ctx, req.Compose, "")
 	if err != nil {
@@ -192,6 +192,8 @@ func (b *ByocDo) deploy(ctx context.Context, req *defangv1.DeployRequest, cmd st
 		delegateDomain: req.DelegateDomain,
 		mode:           req.Mode,
 		project:        project.Name,
+		statesUrl:      req.StatesUrl,
+		eventsUrl:      req.EventsUrl,
 	}
 	_, err = b.runCdCommand(ctx, cdCmd)
 	if err != nil {
@@ -230,6 +232,8 @@ func (b *ByocDo) CdCommand(ctx context.Context, req client.CdCommandRequest) (st
 		command:        []string{string(req.Command)},
 		delegateDomain: "dummy.domain",
 		project:        req.Project,
+		statesUrl:      req.StatesUrl,
+		eventsUrl:      req.EventsUrl,
 	}
 	_, err := b.runCdCommand(ctx, cmd)
 	if err != nil {
@@ -605,6 +609,8 @@ type cdCommand struct {
 	project        string
 	delegateDomain string
 	mode           defangv1.DeploymentMode
+	statesUrl      string
+	eventsUrl      string
 }
 
 //nolint:unparam
@@ -613,6 +619,15 @@ func (b *ByocDo) runCdCommand(ctx context.Context, cmd cdCommand) (*godo.App, er
 	if err != nil {
 		return nil, err
 	}
+
+	if cmd.statesUrl != "" {
+		env = append(env, &godo.AppVariableDefinition{Key: "DEFANG_STATES_UPLOAD_URL", Value: cmd.statesUrl, Type: godo.AppVariableType_Secret})
+	}
+
+	if cmd.eventsUrl != "" {
+		env = append(env, &godo.AppVariableDefinition{Key: "DEFANG_EVENTS_UPLOAD_URL", Value: cmd.eventsUrl, Type: godo.AppVariableType_Secret})
+	}
+
 	if term.DoDebug() {
 		// Convert the environment to a human-readable array of KEY=VALUE strings for debugging
 		debugEnv := make([]string, len(env))

src/pkg/cli/client/byoc/gcp/byoc.go

@@ -314,8 +314,10 @@ func (b *ByocGcp) CdCommand(ctx context.Context, req client.CdCommandRequest) (t
 		return "", err
 	}
 	cmd := cdCommand{
-		project: req.Project,
-		command: []string{string(req.Command)},
+		project:   req.Project,
+		command:   []string{string(req.Command)},
+		statesUrl: req.StatesUrl,
+		eventsUrl: req.EventsUrl,
 	}
 	cdExecutionId, err := b.runCdCommand(ctx, cmd) // TODO: make domain optional for defang cd
 	if err != nil {
@@ -330,6 +332,8 @@ type cdCommand struct {
 	etag           types.ETag
 	mode           defangv1.DeploymentMode
 	project        string
+	statesUrl      string
+	eventsUrl      string
 }
 
 type CloudBuildStep struct {
@@ -374,6 +378,14 @@ func (b *ByocGcp) runCdCommand(ctx context.Context, cmd cdCommand) (string, erro
 		env["DOMAIN"] = "dummy.domain"
 	}
 
+	if cmd.statesUrl != "" {
+		env["DEFANG_STATES_UPLOAD_URL"] = cmd.statesUrl
+	}
+
+	if cmd.eventsUrl != "" {
+		env["DEFANG_EVENTS_UPLOAD_URL"] = cmd.eventsUrl
+	}
+
 	if cmd.etag != "" {
 		env["DEFANG_ETAG"] = cmd.etag
 	}
@@ -438,19 +450,19 @@ func (b *ByocGcp) CreateUploadURL(ctx context.Context, req *defangv1.UploadURLRe
 	}
 	return &defangv1.UploadURLResponse{Url: url}, nil
 }
-func (b *ByocGcp) Deploy(ctx context.Context, req *defangv1.DeployRequest) (*defangv1.DeployResponse, error) {
+func (b *ByocGcp) Deploy(ctx context.Context, req *client.DeployRequest) (*defangv1.DeployResponse, error) {
 	return b.deploy(ctx, req, "up")
 }
 
-func (b *ByocGcp) Preview(ctx context.Context, req *defangv1.DeployRequest) (*defangv1.DeployResponse, error) {
+func (b *ByocGcp) Preview(ctx context.Context, req *client.DeployRequest) (*defangv1.DeployResponse, error) {
 	return b.deploy(ctx, req, "preview")
 }
 
 func (b *ByocGcp) GetDeploymentStatus(ctx context.Context) error {
 	return b.driver.GetBuildStatus(ctx, b.cdExecution)
 }
 
-func (b *ByocGcp) deploy(ctx context.Context, req *defangv1.DeployRequest, command string) (*defangv1.DeployResponse, error) {
+func (b *ByocGcp) deploy(ctx context.Context, req *client.DeployRequest, command string) (*defangv1.DeployResponse, error) {
 	// If multiple Compose files were provided, req.Compose is the merged representation of all the files
 	project, err := compose.LoadFromContent(ctx, req.Compose, "")
 	if err != nil {
@@ -507,6 +519,8 @@ func (b *ByocGcp) deploy(ctx context.Context, req *defangv1.DeployRequest, comma
 		etag:           etag,
 		mode:           req.Mode,
 		project:        project.Name,
+		statesUrl:      req.StatesUrl,
+		eventsUrl:      req.EventsUrl,
 	}
 	execution, err := b.runCdCommand(ctx, cdCmd)
 	if err != nil {

src/pkg/cli/client/client.go

@@ -13,6 +13,7 @@ type FabricClient interface {
 	CanIUse(context.Context, *defangv1.CanIUseRequest) (*defangv1.CanIUseResponse, error)
 	CheckLoginAndToS(context.Context) error
 	CreateDelegateSubdomainZone(context.Context, *defangv1.DelegateSubdomainZoneRequest) (*defangv1.DelegateSubdomainZoneResponse, error)
+	CreateUploadURL(context.Context, *defangv1.UploadURLRequest) (*defangv1.UploadURLResponse, error)
 	Debug(context.Context, *defangv1.DebugRequest) (*defangv1.DebugResponse, error)
 	DeleteSubdomainZone(context.Context, *defangv1.DeleteSubdomainZoneRequest) error
 	Estimate(context.Context, *defangv1.EstimateRequest) (*defangv1.EstimateResponse, error)