diff --git a/src/pkg/cli/client/mock.go b/src/pkg/cli/client/mock.go index c708b7cb4..40fab8218 100644 --- a/src/pkg/cli/client/mock.go +++ b/src/pkg/cli/client/mock.go @@ -4,6 +4,7 @@ import ( "context" "errors" "net/http" + "net/url" "path" "github.com/DefangLabs/defang/src/pkg/dns" @@ -21,7 +22,8 @@ type MockProvider struct { } func (m MockProvider) CreateUploadURL(ctx context.Context, req *defangv1.UploadURLRequest) (*defangv1.UploadURLResponse, error) { - return &defangv1.UploadURLResponse{Url: m.UploadUrl + req.Digest}, nil + url, err := url.JoinPath(m.UploadUrl, req.Project, req.Stack, req.Digest) + return &defangv1.UploadURLResponse{Url: url}, err } func (m MockProvider) ListConfig(ctx context.Context, req *defangv1.ListConfigsRequest) (*defangv1.Secrets, error) { diff --git a/src/pkg/cli/compose/context.go b/src/pkg/cli/compose/context.go index ffbd799f5..40d1106a1 100644 --- a/src/pkg/cli/compose/context.go +++ b/src/pkg/cli/compose/context.go @@ -118,9 +118,13 @@ func (tw *tarFactory) CreateHeader(info fs.FileInfo, slashPath string) (io.Write } // Make reproducible; WalkDir walks files in lexical order. + header.AccessTime = time.Time{} + header.ChangeTime = time.Time{} header.ModTime = time.Unix(sourceDateEpoch, 0) header.Gid = 0 header.Uid = 0 + header.Gname = "" + header.Uname = "" header.Name = slashPath err = tw.WriteHeader(header) return tw.Writer, err @@ -191,7 +195,7 @@ var ( ContextSizeHardLimit = parseContextLimit(os.Getenv("DEFANG_BUILD_CONTEXT_LIMIT"), DefaultContextSizeHardLimit) ) -func getRemoteBuildContext(ctx context.Context, provider client.Provider, project, name string, build *types.BuildConfig, upload UploadMode) (string, error) { +func getRemoteBuildContext(ctx context.Context, provider client.Provider, projectName, service string, build *types.BuildConfig, upload UploadMode) (string, error) { root, err := filepath.Abs(build.Context) if err != nil { return "", fmt.Errorf("invalid build context: %w", err) // already checked in ValidateProject @@ -212,10 +216,10 @@ func getRemoteBuildContext(ctx context.Context, provider client.Provider, projec return root, nil case UploadModeEstimate: // For estimation, we don't bother packaging the files, we just return a placeholder URL - return fmt.Sprintf("s3://cd-preview/%v", time.Now().Unix()), nil + return fmt.Sprintf("s3://cd-preview/%s%s", service, archiveType.Extension), nil } - term.Info("Packaging the project files for", name, "at", root) + term.Info("Packaging the project files for", service, "at", root) buffer, err := createArchive(ctx, build.Context, build.Dockerfile, archiveType) if err != nil { return "", err @@ -223,40 +227,39 @@ func getRemoteBuildContext(ctx context.Context, provider client.Provider, projec var digest string switch upload { - case UploadModeDefault: - case UploadModeDigest: + case UploadModeDefault, UploadModeDigest: // Calculate the digest of the tarball and pass it to the fabric controller (to avoid building the same image twice) - sha := sha256.Sum256(buffer.Bytes()) - digest = "sha256-" + base64.StdEncoding.EncodeToString(sha[:]) // same as Nix - term.Debug("Digest:", digest) + digest = calcDigest(buffer.Bytes()) + term.Debugf("Digest for %q: %s", service, digest) case UploadModePreview: - // For preview, we invoke the CD "preview" command, which will want a valid (S3) URL, even though it won't be used - return fmt.Sprintf("s3://cd-preview/%v", time.Now().Unix()), nil + // For preview, we invoke the CD "preview" command, which will want a valid (S3) URL for diff, even though it won't be used + digest = calcDigest(buffer.Bytes()) + return fmt.Sprintf("s3://cd-preview/%s%s", digest, archiveType.Extension), nil case UploadModeForce: - // Force: always upload the tarball (to a random URL), triggering a new build + // Force: empty digest = always upload the tarball (to a random URL), triggering a new build default: panic("unexpected UploadMode value") } - term.Info("Uploading the project files for", name) - return uploadArchive(ctx, provider, project, buffer, archiveType, digest) + term.Info("Uploading the project files for", service) + return uploadArchive(ctx, provider, projectName, buffer, archiveType, digest) } -func uploadArchive(ctx context.Context, provider client.Provider, project string, body io.Reader, contentType ArchiveType, digest string) (string, error) { - // Upload the archive to the fabric controller storage;; TODO: use a streaming API - if contentType.MimeType == ArchiveTypeZip.MimeType { - digest = digest + ArchiveTypeZip.Extension - } else { - digest = digest + ArchiveTypeGzip.Extension - } - ureq := &defangv1.UploadURLRequest{Digest: digest, Project: project} +func calcDigest(data []byte) string { + sha := sha256.Sum256(data) + return "sha256-" + base64.StdEncoding.EncodeToString(sha[:]) // same as Nix +} + +func uploadArchive(ctx context.Context, provider client.Provider, projectName string, body io.Reader, archiveType ArchiveType, digest string) (string, error) { + // Upload the archive to the fabric controller storage; TODO: use a streaming API + ureq := &defangv1.UploadURLRequest{Digest: digest + archiveType.Extension, Project: projectName} res, err := provider.CreateUploadURL(ctx, ureq) if err != nil { return "", err } // Do an HTTP PUT to the generated URL - resp, err := http.Put(ctx, res.Url, string(contentType.MimeType), body) + resp, err := http.Put(ctx, res.Url, string(archiveType.MimeType), body) if err != nil { return "", err } @@ -428,8 +431,8 @@ func walkContextFolder(root, dockerfile string, writeIgnore writeIgnoreFile, fn func createArchive(ctx context.Context, root string, dockerfile string, contentType ArchiveType) (*bytes.Buffer, error) { fileCount := 0 - // TODO: use io.Pipe and do proper streaming (instead of buffering everything in memory) + // TODO: use io.Pipe and do proper streaming (instead of buffering everything in memory) buf := &bytes.Buffer{} var factory WriterFactory if contentType == ArchiveTypeZip { diff --git a/src/pkg/cli/compose/context_test.go b/src/pkg/cli/compose/context_test.go index ed59dc7fb..3899e47af 100644 --- a/src/pkg/cli/compose/context_test.go +++ b/src/pkg/cli/compose/context_test.go @@ -8,12 +8,14 @@ import ( "net/http" "net/http/httptest" "os" + "path" "path/filepath" "reflect" "strings" "testing" "github.com/DefangLabs/defang/src/pkg/cli/client" + "github.com/compose-spec/compose-go/v2/types" "github.com/moby/patternmatcher/ignorefile" ) @@ -38,6 +40,7 @@ func Test_parseContextLimit(t *testing.T) { } func TestUploadArchive(t *testing.T) { + const testproj = "testproj" const path = "/upload/x/" const digest = "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" @@ -45,75 +48,80 @@ func TestUploadArchive(t *testing.T) { if r.Method != "PUT" { t.Errorf("Expected PUT request, got %v", r.Method) } - if !strings.HasPrefix(r.URL.Path, path) { - t.Errorf("Expected prefix %v, got %v", path, r.URL.Path) + if !strings.HasPrefix(r.URL.Path, path+testproj) { + t.Errorf("Expected prefix %v, got %v", path+testproj, r.URL.Path) } if !(r.Header.Get("Content-Type") == string(ArchiveTypeGzip.MimeType) || r.Header.Get("Content-Type") == string(ArchiveTypeZip.MimeType)) { t.Errorf("Expected Content-Type: application/gzip or application/zip, got %v", r.Header.Get("Content-Type")) } w.WriteHeader(200) })) - defer server.Close() + t.Cleanup(server.Close) + uploadUrl := server.URL + path t.Run("upload tar with digest", func(t *testing.T) { - url, err := uploadArchive(t.Context(), client.MockProvider{UploadUrl: server.URL + path}, "testproj", &bytes.Buffer{}, ArchiveTypeGzip, digest) + url, err := uploadArchive(t.Context(), client.MockProvider{UploadUrl: uploadUrl}, testproj, &bytes.Buffer{}, ArchiveTypeGzip, digest) if err != nil { t.Fatalf("uploadArchive() failed: %v", err) } - var expectedPath = path + digest + ArchiveTypeGzip.Extension + var expectedPath = path + testproj + "/" + digest + ArchiveTypeGzip.Extension if url != server.URL+expectedPath { t.Errorf("Expected %v, got %v", server.URL+expectedPath, url) } }) t.Run("upload zip with digest", func(t *testing.T) { - url, err := uploadArchive(t.Context(), client.MockProvider{UploadUrl: server.URL + path}, "testproj", &bytes.Buffer{}, ArchiveTypeZip, digest) + url, err := uploadArchive(t.Context(), client.MockProvider{UploadUrl: uploadUrl}, testproj, &bytes.Buffer{}, ArchiveTypeZip, digest) if err != nil { t.Fatalf("uploadArchive() failed: %v", err) } - var expectedPath = path + digest + ArchiveTypeZip.Extension + var expectedPath = path + testproj + "/" + digest + ArchiveTypeZip.Extension if url != server.URL+expectedPath { t.Errorf("Expected %v, got %v", server.URL+expectedPath, url) } }) t.Run("upload with zip", func(t *testing.T) { - url, err := uploadArchive(t.Context(), client.MockProvider{UploadUrl: server.URL + path}, "testproj", &bytes.Buffer{}, ArchiveTypeZip, "") + url, err := uploadArchive(t.Context(), client.MockProvider{UploadUrl: uploadUrl}, testproj, &bytes.Buffer{}, ArchiveTypeZip, "") if err != nil { t.Fatalf("uploadContent() failed: %v", err) } - if url != server.URL+path+ArchiveTypeZip.Extension { - t.Errorf("Expected %v, got %v", server.URL+path+ArchiveTypeZip.Extension, url) + var expectedPath = path + testproj + "/" + ArchiveTypeZip.Extension + if url != server.URL+expectedPath { + t.Errorf("Expected %v, got %v", server.URL+expectedPath, url) } }) t.Run("upload with tar", func(t *testing.T) { - url, err := uploadArchive(t.Context(), client.MockProvider{UploadUrl: server.URL + path}, "testproj", &bytes.Buffer{}, ArchiveTypeGzip, "") + url, err := uploadArchive(t.Context(), client.MockProvider{UploadUrl: uploadUrl}, testproj, &bytes.Buffer{}, ArchiveTypeGzip, "") if err != nil { t.Fatalf("uploadContent() failed: %v", err) } - if url != server.URL+path+ArchiveTypeGzip.Extension { - t.Errorf("Expected %v, got %v", server.URL+path+ArchiveTypeGzip.Extension, url) + var expectedPath = path + testproj + "/" + ArchiveTypeGzip.Extension + if url != server.URL+expectedPath { + t.Errorf("Expected %v, got %v", server.URL+expectedPath, url) } }) t.Run("force upload tar without digest", func(t *testing.T) { - url, err := uploadArchive(t.Context(), client.MockProvider{UploadUrl: server.URL + path}, "testproj", &bytes.Buffer{}, ArchiveTypeGzip, "") + url, err := uploadArchive(t.Context(), client.MockProvider{UploadUrl: uploadUrl}, testproj, &bytes.Buffer{}, ArchiveTypeGzip, "") if err != nil { t.Fatalf("uploadArchive() failed: %v", err) } - if url != server.URL+path+ArchiveTypeGzip.Extension { - t.Errorf("Expected %v, got %v", server.URL+path+ArchiveTypeGzip.Extension, url) + var expectedPath = path + testproj + "/" + ArchiveTypeGzip.Extension + if url != server.URL+expectedPath { + t.Errorf("Expected %v, got %v", server.URL+expectedPath, url) } }) t.Run("force upload zip without digest", func(t *testing.T) { - url, err := uploadArchive(t.Context(), client.MockProvider{UploadUrl: server.URL + path}, "testproj", &bytes.Buffer{}, ArchiveTypeZip, "") + url, err := uploadArchive(t.Context(), client.MockProvider{UploadUrl: uploadUrl}, testproj, &bytes.Buffer{}, ArchiveTypeZip, "") if err != nil { t.Fatalf("uploadArchive() failed: %v", err) } - if url != server.URL+path+ArchiveTypeZip.Extension { - t.Errorf("Expected %v, got %v", server.URL+path+ArchiveTypeZip.Extension, url) + var expectedPath = path + testproj + "/" + ArchiveTypeZip.Extension + if url != server.URL+expectedPath { + t.Errorf("Expected %v, got %v", server.URL+expectedPath, url) } }) } @@ -172,6 +180,101 @@ func TestWalkContextFolder(t *testing.T) { }) } +func Test_getRemoteBuildContext(t *testing.T) { + tests := []struct { + name string + uploadMode UploadMode + expectUrl string + expectFile string + }{ + { + name: "Default UploadMode", + uploadMode: UploadModeDefault, + expectUrl: "https://mock-bucket.s3.amazonaws.com/project1/sha256-B+3Dq6U37SrlbnrfS4uIk3CDwrPJ+Q15TqUCPBEMQuA=.tar.gz", // same as Digest mode + expectFile: "sha256-B+3Dq6U37SrlbnrfS4uIk3CDwrPJ+Q15TqUCPBEMQuA=.tar.gz", + }, + { + name: "Force UploadMode", + uploadMode: UploadModeForce, + expectUrl: "https://mock-bucket.s3.amazonaws.com/project1/.tar.gz", // server decides name + expectFile: ".tar.gz", + }, + { + name: "Digest UploadMode", + uploadMode: UploadModeDigest, + expectUrl: "https://mock-bucket.s3.amazonaws.com/project1/sha256-B+3Dq6U37SrlbnrfS4uIk3CDwrPJ+Q15TqUCPBEMQuA=.tar.gz", + expectFile: "sha256-B+3Dq6U37SrlbnrfS4uIk3CDwrPJ+Q15TqUCPBEMQuA=.tar.gz", + }, + { + name: "Ignore UploadMode", + uploadMode: UploadModeIgnore, + expectUrl: "$SRC/testdata/testproj", // show local paths in "defang config" + }, + { + name: "Preview UploadMode", + uploadMode: UploadModePreview, + expectUrl: "s3://cd-preview/sha256-B+3Dq6U37SrlbnrfS4uIk3CDwrPJ+Q15TqUCPBEMQuA=.tar.gz", // like digest but fake bucket + }, + { + name: "Estimate UploadMode", + uploadMode: UploadModeEstimate, + expectUrl: "s3://cd-preview/service1.tar.gz", // like preview but skip digest calculation + }, + } + + tmpDir := t.TempDir() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != "PUT" { + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } + defer r.Body.Close() + if dst, err := os.Create(filepath.Join(tmpDir, path.Base(r.URL.Path))); err != nil { + t.Errorf("Failed to create file: %v", err) + } else { + defer dst.Close() + if _, err := io.Copy(dst, r.Body); err != nil { + t.Errorf("Failed to write file: %v", err) + } + } + w.WriteHeader(200) + })) + t.Cleanup(server.Close) + + src, err := filepath.Abs("../../..") + if err != nil { + t.Fatalf("Failed to get working directory: %v", err) + } + + normalizer := strings.NewReplacer(src, "$SRC", server.URL, "https://mock-bucket.s3.amazonaws.com") + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + provider := client.MockProvider{UploadUrl: server.URL} + url, err := getRemoteBuildContext(t.Context(), provider, "project1", "service1", &types.BuildConfig{ + Context: "../../../testdata/testproj", + }, tt.uploadMode) + if err != nil { + t.Fatalf("getRemoteBuildContext() failed: %v", err) + } + if got := normalizer.Replace(url); got != tt.expectUrl { + t.Errorf("Expected %v, got: %v", tt.expectUrl, got) + } + if tt.expectFile != "" { + // Check that the file was uploaded correctly + uploadedFile := filepath.Join(tmpDir, tt.expectFile) + all, err := os.ReadFile(uploadedFile) + if err != nil { + t.Fatalf("Failed to read uploaded file %v: %v", uploadedFile, err) + } + if calcDigest(all) != "sha256-B+3Dq6U37SrlbnrfS4uIk3CDwrPJ+Q15TqUCPBEMQuA=" { + t.Errorf("Uploaded file has unexpected digest: %v", calcDigest(all)) + } + } + }) + } +} + func TestCreateTarballReader(t *testing.T) { t.Run("Default Dockerfile", func(t *testing.T) { buffer, err := createArchive(t.Context(), "../../../testdata/testproj", "", ArchiveTypeGzip) @@ -183,7 +286,7 @@ func TestCreateTarballReader(t *testing.T) { if err != nil { t.Fatalf("gzip.NewReader() failed: %v", err) } - defer g.Close() + t.Cleanup(func() { g.Close() }) expected := []string{".dockerignore", ".env", "Dockerfile", "fileName.env"} var actual []string diff --git a/src/pkg/dns/check_test.go b/src/pkg/dns/check_test.go index 8dfc6a038..904b0ac4d 100644 --- a/src/pkg/dns/check_test.go +++ b/src/pkg/dns/check_test.go @@ -135,7 +135,6 @@ func TestGetIPInSync(t *testing.T) { } func TestCheckDomainDNSReady(t *testing.T) { - term.SetDebug(true) emptyResolver := MockResolver{} hasARecordResolver := MockResolver{Records: map[DNSRequest]DNSResponse{ {Type: "NS", Domain: "api.test.com"}: {Records: []string{"ns1.example.com", "ns2.example.com"}, Error: nil}, @@ -154,9 +153,12 @@ func TestCheckDomainDNSReady(t *testing.T) { }} resolver = hasARecordResolver + oldResolver, oldDebug := ResolverAt, term.DoDebug() t.Cleanup(func() { - ResolverAt = DirectResolverAt + ResolverAt = oldResolver + term.SetDebug(oldDebug) }) + term.SetDebug(true) t.Run("CNAME and A records not found", func(t *testing.T) { ResolverAt = func(_ string) Resolver { return emptyResolver }