fix: missing digest in default mode

src/pkg/cli/client/mock.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"net/http"
+	"net/url"
 	"path"
 
 	"github.com/DefangLabs/defang/src/pkg/dns"
@@ -21,7 +22,8 @@ type MockProvider struct {
 }
 
 func (m MockProvider) CreateUploadURL(ctx context.Context, req *defangv1.UploadURLRequest) (*defangv1.UploadURLResponse, error) {
-	return &defangv1.UploadURLResponse{Url: m.UploadUrl + req.Digest}, nil
+	url, err := url.JoinPath(m.UploadUrl, req.Project, req.Stack, req.Digest)
+	return &defangv1.UploadURLResponse{Url: url}, err
 }
 
 func (m MockProvider) ListConfig(ctx context.Context, req *defangv1.ListConfigsRequest) (*defangv1.Secrets, error) {

src/pkg/cli/compose/context.go

@@ -118,9 +118,13 @@ func (tw *tarFactory) CreateHeader(info fs.FileInfo, slashPath string) (io.Write
 	}
 
 	// Make reproducible; WalkDir walks files in lexical order.
+	header.AccessTime = time.Time{}
+	header.ChangeTime = time.Time{}
 	header.ModTime = time.Unix(sourceDateEpoch, 0)
 	header.Gid = 0
 	header.Uid = 0
+	header.Gname = ""
+	header.Uname = ""
 	header.Name = slashPath
 	err = tw.WriteHeader(header)
 	return tw.Writer, err
@@ -191,7 +195,7 @@ var (
 	ContextSizeHardLimit = parseContextLimit(os.Getenv("DEFANG_BUILD_CONTEXT_LIMIT"), DefaultContextSizeHardLimit)
 )
 
-func getRemoteBuildContext(ctx context.Context, provider client.Provider, project, name string, build *types.BuildConfig, upload UploadMode) (string, error) {
+func getRemoteBuildContext(ctx context.Context, provider client.Provider, projectName, service string, build *types.BuildConfig, upload UploadMode) (string, error) {
 	root, err := filepath.Abs(build.Context)
 	if err != nil {
 		return "", fmt.Errorf("invalid build context: %w", err) // already checked in ValidateProject
@@ -212,51 +216,50 @@ func getRemoteBuildContext(ctx context.Context, provider client.Provider, projec
 		return root, nil
 	case UploadModeEstimate:
 		// For estimation, we don't bother packaging the files, we just return a placeholder URL
-		return fmt.Sprintf("s3://cd-preview/%v", time.Now().Unix()), nil
+		return fmt.Sprintf("s3://cd-preview/%s%s", service, archiveType.Extension), nil
 	}
 
-	term.Info("Packaging the project files for", name, "at", root)
+	term.Info("Packaging the project files for", service, "at", root)
 	buffer, err := createArchive(ctx, build.Context, build.Dockerfile, archiveType)
 	if err != nil {
 		return "", err
 	}
 
 	var digest string
 	switch upload {
-	case UploadModeDefault:
-	case UploadModeDigest:
+	case UploadModeDefault, UploadModeDigest:
 		// Calculate the digest of the tarball and pass it to the fabric controller (to avoid building the same image twice)
-		sha := sha256.Sum256(buffer.Bytes())
-		digest = "sha256-" + base64.StdEncoding.EncodeToString(sha[:]) // same as Nix
-		term.Debug("Digest:", digest)
+		digest = calcDigest(buffer.Bytes())
+		term.Debugf("Digest for %q: %s", service, digest)
 	case UploadModePreview:
-		// For preview, we invoke the CD "preview" command, which will want a valid (S3) URL, even though it won't be used
-		return fmt.Sprintf("s3://cd-preview/%v", time.Now().Unix()), nil
+		// For preview, we invoke the CD "preview" command, which will want a valid (S3) URL for diff, even though it won't be used
+		digest = calcDigest(buffer.Bytes())
+		return fmt.Sprintf("s3://cd-preview/%s%s", digest, archiveType.Extension), nil
 	case UploadModeForce:
-		// Force: always upload the tarball (to a random URL), triggering a new build
+		// Force: empty digest = always upload the tarball (to a random URL), triggering a new build
 	default:
 		panic("unexpected UploadMode value")
 	}
 
-	term.Info("Uploading the project files for", name)
-	return uploadArchive(ctx, provider, project, buffer, archiveType, digest)
+	term.Info("Uploading the project files for", service)
+	return uploadArchive(ctx, provider, projectName, buffer, archiveType, digest)
 }
 
-func uploadArchive(ctx context.Context, provider client.Provider, project string, body io.Reader, contentType ArchiveType, digest string) (string, error) {
-	// Upload the archive to the fabric controller storage;; TODO: use a streaming API
-	if contentType.MimeType == ArchiveTypeZip.MimeType {
-		digest = digest + ArchiveTypeZip.Extension
-	} else {
-		digest = digest + ArchiveTypeGzip.Extension
-	}
-	ureq := &defangv1.UploadURLRequest{Digest: digest, Project: project}
+func calcDigest(data []byte) string {
+	sha := sha256.Sum256(data)
+	return "sha256-" + base64.StdEncoding.EncodeToString(sha[:]) // same as Nix
+}
+
+func uploadArchive(ctx context.Context, provider client.Provider, projectName string, body io.Reader, archiveType ArchiveType, digest string) (string, error) {
+	// Upload the archive to the fabric controller storage; TODO: use a streaming API
+	ureq := &defangv1.UploadURLRequest{Digest: digest + archiveType.Extension, Project: projectName}
 	res, err := provider.CreateUploadURL(ctx, ureq)
 	if err != nil {
 		return "", err
 	}
 
 	// Do an HTTP PUT to the generated URL
-	resp, err := http.Put(ctx, res.Url, string(contentType.MimeType), body)
+	resp, err := http.Put(ctx, res.Url, string(archiveType.MimeType), body)
 	if err != nil {
 		return "", err
 	}
@@ -428,8 +431,8 @@ func walkContextFolder(root, dockerfile string, writeIgnore writeIgnoreFile, fn
 
 func createArchive(ctx context.Context, root string, dockerfile string, contentType ArchiveType) (*bytes.Buffer, error) {
 	fileCount := 0
-	// TODO: use io.Pipe and do proper streaming (instead of buffering everything in memory)
 
+	// TODO: use io.Pipe and do proper streaming (instead of buffering everything in memory)
 	buf := &bytes.Buffer{}
 	var factory WriterFactory
 	if contentType == ArchiveTypeZip {

src/pkg/cli/compose/context_test.go

@@ -8,12 +8,14 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"os"
+	"path"
 	"path/filepath"
 	"reflect"
 	"strings"
 	"testing"
 
 	"github.com/DefangLabs/defang/src/pkg/cli/client"
+	"github.com/compose-spec/compose-go/v2/types"
 	"github.com/moby/patternmatcher/ignorefile"
 )
 
@@ -38,82 +40,88 @@ func Test_parseContextLimit(t *testing.T) {
 }
 
 func TestUploadArchive(t *testing.T) {
+	const testproj = "testproj"
 	const path = "/upload/x/"
 	const digest = "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="
 
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if r.Method != "PUT" {
 			t.Errorf("Expected PUT request, got %v", r.Method)
 		}
-		if !strings.HasPrefix(r.URL.Path, path) {
-			t.Errorf("Expected prefix %v, got %v", path, r.URL.Path)
+		if !strings.HasPrefix(r.URL.Path, path+testproj) {
+			t.Errorf("Expected prefix %v, got %v", path+testproj, r.URL.Path)
 		}
 		if !(r.Header.Get("Content-Type") == string(ArchiveTypeGzip.MimeType) || r.Header.Get("Content-Type") == string(ArchiveTypeZip.MimeType)) {
 			t.Errorf("Expected Content-Type: application/gzip or application/zip, got %v", r.Header.Get("Content-Type"))
 		}
 		w.WriteHeader(200)
 	}))
-	defer server.Close()
+	t.Cleanup(server.Close)
 
+	uploadUrl := server.URL + path
 	t.Run("upload tar with digest", func(t *testing.T) {
-		url, err := uploadArchive(t.Context(), client.MockProvider{UploadUrl: server.URL + path}, "testproj", &bytes.Buffer{}, ArchiveTypeGzip, digest)
+		url, err := uploadArchive(t.Context(), client.MockProvider{UploadUrl: uploadUrl}, testproj, &bytes.Buffer{}, ArchiveTypeGzip, digest)
 		if err != nil {
 			t.Fatalf("uploadArchive() failed: %v", err)
 		}
-		var expectedPath = path + digest + ArchiveTypeGzip.Extension
+		var expectedPath = path + testproj + "/" + digest + ArchiveTypeGzip.Extension
 		if url != server.URL+expectedPath {
 			t.Errorf("Expected %v, got %v", server.URL+expectedPath, url)
 		}
 	})
 
 	t.Run("upload zip with digest", func(t *testing.T) {
-		url, err := uploadArchive(t.Context(), client.MockProvider{UploadUrl: server.URL + path}, "testproj", &bytes.Buffer{}, ArchiveTypeZip, digest)
+		url, err := uploadArchive(t.Context(), client.MockProvider{UploadUrl: uploadUrl}, testproj, &bytes.Buffer{}, ArchiveTypeZip, digest)
 		if err != nil {
 			t.Fatalf("uploadArchive() failed: %v", err)
 		}
-		var expectedPath = path + digest + ArchiveTypeZip.Extension
+		var expectedPath = path + testproj + "/" + digest + ArchiveTypeZip.Extension
 		if url != server.URL+expectedPath {
 			t.Errorf("Expected %v, got %v", server.URL+expectedPath, url)
 		}
 	})
 
 	t.Run("upload with zip", func(t *testing.T) {
-		url, err := uploadArchive(t.Context(), client.MockProvider{UploadUrl: server.URL + path}, "testproj", &bytes.Buffer{}, ArchiveTypeZip, "")
+		url, err := uploadArchive(t.Context(), client.MockProvider{UploadUrl: uploadUrl}, testproj, &bytes.Buffer{}, ArchiveTypeZip, "")
 		if err != nil {
 			t.Fatalf("uploadContent() failed: %v", err)
 		}
-		if url != server.URL+path+ArchiveTypeZip.Extension {
-			t.Errorf("Expected %v, got %v", server.URL+path+ArchiveTypeZip.Extension, url)
+		var expectedPath = path + testproj + "/" + ArchiveTypeZip.Extension
+		if url != server.URL+expectedPath {
+			t.Errorf("Expected %v, got %v", server.URL+expectedPath, url)
 		}
 	})
 
 	t.Run("upload with tar", func(t *testing.T) {
-		url, err := uploadArchive(t.Context(), client.MockProvider{UploadUrl: server.URL + path}, "testproj", &bytes.Buffer{}, ArchiveTypeGzip, "")
+		url, err := uploadArchive(t.Context(), client.MockProvider{UploadUrl: uploadUrl}, testproj, &bytes.Buffer{}, ArchiveTypeGzip, "")
 		if err != nil {
 			t.Fatalf("uploadContent() failed: %v", err)
 		}
-		if url != server.URL+path+ArchiveTypeGzip.Extension {
-			t.Errorf("Expected %v, got %v", server.URL+path+ArchiveTypeGzip.Extension, url)
+		var expectedPath = path + testproj + "/" + ArchiveTypeGzip.Extension
+		if url != server.URL+expectedPath {
+			t.Errorf("Expected %v, got %v", server.URL+expectedPath, url)
 		}
 	})
 
 	t.Run("force upload tar without digest", func(t *testing.T) {
-		url, err := uploadArchive(t.Context(), client.MockProvider{UploadUrl: server.URL + path}, "testproj", &bytes.Buffer{}, ArchiveTypeGzip, "")
+		url, err := uploadArchive(t.Context(), client.MockProvider{UploadUrl: uploadUrl}, testproj, &bytes.Buffer{}, ArchiveTypeGzip, "")
 		if err != nil {
 			t.Fatalf("uploadArchive() failed: %v", err)
 		}
-		if url != server.URL+path+ArchiveTypeGzip.Extension {
-			t.Errorf("Expected %v, got %v", server.URL+path+ArchiveTypeGzip.Extension, url)
+		var expectedPath = path + testproj + "/" + ArchiveTypeGzip.Extension
+		if url != server.URL+expectedPath {
+			t.Errorf("Expected %v, got %v", server.URL+expectedPath, url)
 		}
 	})
 
 	t.Run("force upload zip without digest", func(t *testing.T) {
-		url, err := uploadArchive(t.Context(), client.MockProvider{UploadUrl: server.URL + path}, "testproj", &bytes.Buffer{}, ArchiveTypeZip, "")
+		url, err := uploadArchive(t.Context(), client.MockProvider{UploadUrl: uploadUrl}, testproj, &bytes.Buffer{}, ArchiveTypeZip, "")
 		if err != nil {
 			t.Fatalf("uploadArchive() failed: %v", err)
 		}
-		if url != server.URL+path+ArchiveTypeZip.Extension {
-			t.Errorf("Expected %v, got %v", server.URL+path+ArchiveTypeZip.Extension, url)
+		var expectedPath = path + testproj + "/" + ArchiveTypeZip.Extension
+		if url != server.URL+expectedPath {
+			t.Errorf("Expected %v, got %v", server.URL+expectedPath, url)
 		}
 	})
 }
@@ -172,6 +180,101 @@ func TestWalkContextFolder(t *testing.T) {
 	})
 }
 
+func Test_getRemoteBuildContext(t *testing.T) {
+	tests := []struct {
+		name       string
+		uploadMode UploadMode
+		expectUrl  string
+		expectFile string
+	}{
+		{
+			name:       "Default UploadMode",
+			uploadMode: UploadModeDefault,
+			expectUrl:  "https://mock-bucket.s3.amazonaws.com/project1/sha256-B+3Dq6U37SrlbnrfS4uIk3CDwrPJ+Q15TqUCPBEMQuA=.tar.gz", // same as Digest mode
+			expectFile: "sha256-B+3Dq6U37SrlbnrfS4uIk3CDwrPJ+Q15TqUCPBEMQuA=.tar.gz",
+		},
+		{
+			name:       "Force UploadMode",
+			uploadMode: UploadModeForce,
+			expectUrl:  "https://mock-bucket.s3.amazonaws.com/project1/.tar.gz", // server decides name
+			expectFile: ".tar.gz",
+		},
+		{
+			name:       "Digest UploadMode",
+			uploadMode: UploadModeDigest,
+			expectUrl:  "https://mock-bucket.s3.amazonaws.com/project1/sha256-B+3Dq6U37SrlbnrfS4uIk3CDwrPJ+Q15TqUCPBEMQuA=.tar.gz",
+			expectFile: "sha256-B+3Dq6U37SrlbnrfS4uIk3CDwrPJ+Q15TqUCPBEMQuA=.tar.gz",
+		},
+		{
+			name:       "Ignore UploadMode",
+			uploadMode: UploadModeIgnore,
+			expectUrl:  "$SRC/testdata/testproj", // show local paths in "defang config"
+		},
+		{
+			name:       "Preview UploadMode",
+			uploadMode: UploadModePreview,
+			expectUrl:  "s3://cd-preview/sha256-B+3Dq6U37SrlbnrfS4uIk3CDwrPJ+Q15TqUCPBEMQuA=.tar.gz", // like digest but fake bucket
+		},
+		{
+			name:       "Estimate UploadMode",
+			uploadMode: UploadModeEstimate,
+			expectUrl:  "s3://cd-preview/service1.tar.gz", // like preview but skip digest calculation
+		},
+	}
+
+	tmpDir := t.TempDir()
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != "PUT" {
+			http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
+			return
+		}
+		defer r.Body.Close()
+		if dst, err := os.Create(filepath.Join(tmpDir, path.Base(r.URL.Path))); err != nil {
+			t.Errorf("Failed to create file: %v", err)
+		} else {
+			defer dst.Close()
+			if _, err := io.Copy(dst, r.Body); err != nil {
+				t.Errorf("Failed to write file: %v", err)
+			}
+		}
+		w.WriteHeader(200)
+	}))
+	t.Cleanup(server.Close)
+
+	src, err := filepath.Abs("../../..")
+	if err != nil {
+		t.Fatalf("Failed to get working directory: %v", err)
+	}
+
+	normalizer := strings.NewReplacer(src, "$SRC", server.URL, "https://mock-bucket.s3.amazonaws.com")
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			provider := client.MockProvider{UploadUrl: server.URL}
+			url, err := getRemoteBuildContext(t.Context(), provider, "project1", "service1", &types.BuildConfig{
+				Context: "../../../testdata/testproj",
+			}, tt.uploadMode)
+			if err != nil {
+				t.Fatalf("getRemoteBuildContext() failed: %v", err)
+			}
+			if got := normalizer.Replace(url); got != tt.expectUrl {
+				t.Errorf("Expected %v, got: %v", tt.expectUrl, got)
+			}
+			if tt.expectFile != "" {
+				// Check that the file was uploaded correctly
+				uploadedFile := filepath.Join(tmpDir, tt.expectFile)
+				all, err := os.ReadFile(uploadedFile)
+				if err != nil {
+					t.Fatalf("Failed to read uploaded file %v: %v", uploadedFile, err)
+				}
+				if calcDigest(all) != "sha256-B+3Dq6U37SrlbnrfS4uIk3CDwrPJ+Q15TqUCPBEMQuA=" {
+					t.Errorf("Uploaded file has unexpected digest: %v", calcDigest(all))
+				}
+			}
+		})
+	}
+}
+
 func TestCreateTarballReader(t *testing.T) {
 	t.Run("Default Dockerfile", func(t *testing.T) {
 		buffer, err := createArchive(t.Context(), "../../../testdata/testproj", "", ArchiveTypeGzip)
@@ -183,7 +286,7 @@ func TestCreateTarballReader(t *testing.T) {
 		if err != nil {
 			t.Fatalf("gzip.NewReader() failed: %v", err)
 		}
-		defer g.Close()
+		t.Cleanup(func() { g.Close() })
 
 		expected := []string{".dockerignore", ".env", "Dockerfile", "fileName.env"}
 		var actual []string

src/pkg/dns/check_test.go

@@ -135,7 +135,6 @@ func TestGetIPInSync(t *testing.T) {
 }
 
 func TestCheckDomainDNSReady(t *testing.T) {
-	term.SetDebug(true)
 	emptyResolver := MockResolver{}
 	hasARecordResolver := MockResolver{Records: map[DNSRequest]DNSResponse{
 		{Type: "NS", Domain: "api.test.com"}:       {Records: []string{"ns1.example.com", "ns2.example.com"}, Error: nil},
@@ -154,9 +153,12 @@ func TestCheckDomainDNSReady(t *testing.T) {
 	}}
 	resolver = hasARecordResolver
 
+	oldResolver, oldDebug := ResolverAt, term.DoDebug()
 	t.Cleanup(func() {
-		ResolverAt = DirectResolverAt
+		ResolverAt = oldResolver
+		term.SetDebug(oldDebug)
 	})
+	term.SetDebug(true)
 
 	t.Run("CNAME and A records not found", func(t *testing.T) {
 		ResolverAt = func(_ string) Resolver { return emptyResolver }