feat: copy from host to container

Add CopyHostPathTo to container methods, which is capable of copying files
and directories from the host to a container. It identifies the correct
copy semantics based on inspecting the source and target, replicating
the behaviour of docker cp and other OS copy tools.

This deprecates CopyDirToContainer and CopyFileToContainer while still
correcting their behaviour to also match docker cp behaviour.

Replace nonamedreturns linter with nakedret, as nonamedreturns prevents
naming returned parameters which has a number of valid uses including:
disambiguating return values and error checking.

Copying files to the container now doesn't compression as this is
slower and consumes more resources for the typical local transfer case.

Fix docker copy tests to they validate the correct behaviour by ensuring
that done is reported to the container log.

Clean up some error wrapping.

Fix testdata wait scripts.

Fix invalid FileMode values, so tests don't fail with the new FileMode
validation.

Switch to exists check for copy instead of running it as a shell.

Disable linter check for deprecated methods as we use them internally
and test them.

Fixes #2780

Author: stevenh

.golangci.yml

@@ -5,7 +5,7 @@ linters:
     - gocritic
     - gofumpt
     - misspell
-    - nonamedreturns
+    - nakedret
     - testifylint
     - errcheck
     - nolintlint

container.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/fs"
 	"os"
 	"path/filepath"
 	"strings"
@@ -63,13 +64,83 @@ type Container interface {
 	Exec(ctx context.Context, cmd []string, options ...tcexec.ProcessOption) (int, io.Reader, error)
 	ContainerIP(context.Context) (string, error)    // get container ip
 	ContainerIPs(context.Context) ([]string, error) // get all container IPs
-	CopyToContainer(ctx context.Context, fileContent []byte, containerFilePath string, fileMode int64) error
-	CopyDirToContainer(ctx context.Context, hostDirPath string, containerParentPath string, fileMode int64) error
-	CopyFileToContainer(ctx context.Context, hostFilePath string, containerFilePath string, fileMode int64) error
+
+	// CopyHostPathTo copies the contents of a hostPath to containerPath in the container
+	// with the given options.
+	// If the parent of the containerPath does not exist an error is returned.
+	CopyHostPathTo(ctx context.Context, hostPath, containerPath string, options ...CopyToOption) error
+
+	// CopyToContainer copies a file with contents of fileContent to containerPath in the
+	// container with the given fileMode.
+	// If fileMode contains bits not part of [fs.ModePerm] | [fs.ModeSetuid] | [fs.ModeSetgid] |
+	// [fs.ModeSticky] an error is returned.
+	CopyToContainer(ctx context.Context, fileContent []byte, containerPath string, fileMode int64) error
+
+	// CopyDirToContainer copies the contents of hostPath to containerPath in the container.
+	// If fileMode is non-zero all files will have their file permissions set to that of fileMode
+	// otherwise the file permissions will be copied from the host.
+	// If fileMode contains bits not part of [fs.ModePerm] | [fs.ModeSetuid] | [fs.ModeSetgid] |
+	// [fs.ModeSticky] an error is returned.
+	// If the parent of the containerPath does not exist an error is returned.
+	//
+	// Deprecated: use [DockerContainer.CopyHostPathTo] instead.
+	CopyDirToContainer(ctx context.Context, hostPath, containerPath string, fileMode int64) error
+
+	// CopyFileToContainer copies hostPath to containerPath in the container.
+	// If fileMode is non-zero the files permissions will be set to that of fileMode
+	// otherwise the file permissions will be set to that of the file in hostPath.
+	// If fileMode contains bits not part of [fs.ModePerm] | [fs.ModeSetuid] | [fs.ModeSetgid] |
+	// [fs.ModeSticky] an error is returned.
+	// If the parent of the containerPath does not exist an error is returned.
+	// If hostPath is a directory this is equivalent to [DockerContainer.CopyDirToContainer].
+	//
+	// Deprecated: use [DockerContainer.CopyHostPathTo] instead.
+	CopyFileToContainer(ctx context.Context, hostPath string, containerPath string, fileMode int64) error
+
 	CopyFileFromContainer(ctx context.Context, filePath string) (io.ReadCloser, error)
 	GetLogProductionErrorChannel() <-chan error
 }
 
+// copyToOptions contains options for the copy operation.
+type copyToOptions struct {
+	// followLink instructs the copy operation to follow symlinks.
+	followLink bool
+
+	// copyUIDGID instructs the copy operation to copy the UID and GID of the source file.
+	copyUIDGID bool
+
+	// allowOverwriteDirWithFile instructs the copy operation to allow overwriting a directory with a file.
+	allowOverwriteDirWithFile bool
+
+	// fileMode if not zero, instructs the copy operation to override the file permissions.
+	fileMode fs.FileMode
+}
+
+// CopyToOption represents a option for CopyTo methods.
+type CopyToOption func(*copyToOptions)
+
+// CopyToFollowLink instructs the copy operation to follow symlinks
+// when identifying the source.
+func CopyToFollowLink() CopyToOption {
+	return func(o *copyToOptions) {
+		o.followLink = true
+	}
+}
+
+// CopyToUIDGID instructs the copy operation to copy the UID and GID of the source.
+func CopyToUIDGID() CopyToOption {
+	return func(o *copyToOptions) {
+		o.copyUIDGID = true
+	}
+}
+
+// CopyToAllowOverwriteDirWithFile instructs the copy operation to allow overwriting a directory with a file.
+func CopyToAllowOverwriteDirWithFile() CopyToOption {
+	return func(o *copyToOptions) {
+		o.allowOverwriteDirWithFile = true
+	}
+}
+
 // ImageBuildInfo defines what is needed to build an image
 type ImageBuildInfo interface {
 	BuildOptions() (types.ImageBuildOptions, error) // converts the ImageBuildInfo to a types.ImageBuildOptions
@@ -104,11 +175,29 @@ type FromDockerfile struct {
 	BuildOptionsModifier func(*types.ImageBuildOptions)
 }
 
+// ContainerMount represents a file or directory to be copied into a container on startup.
 type ContainerFile struct {
-	HostFilePath      string    // If Reader is present, HostFilePath is ignored
-	Reader            io.Reader // If Reader is present, HostFilePath is ignored
+	// HostFilePath is the path to the file on the host machine.
+	// If Reader is present it is ignored.
+	// TODO: Rename to HostPath as HostFilePath infers its a file and it could be a
+	// directory.
+	HostFilePath string
+
+	// Reader provides the file content to be copied to the container.
+	// If present, HostFilePath is ignored.
+	Reader io.Reader
+
+	// ContainerFilePath is the path where this file will be copied to in the container.
+	// TODO: Rename to ContainerPath as ContainerFilePath infers its a file and it could
+	// be a directory.
 	ContainerFilePath string
-	FileMode          int64
+
+	// FileMode is the file mode to set on the file in the container.
+	// Must be set if Reader is present.
+	// If zero or not set, the file mode will be that of the host file.
+	// TODO: Should we only use FileMode for Reader, as it makes more sense to use the
+	// source file permissions when using a host path source?
+	FileMode int64
 }
 
 // validate validates the ContainerFile

docker.go

@@ -14,7 +14,6 @@ import (
 	"net"
 	"net/url"
 	"os"
-	"path/filepath"
 	"regexp"
 	"strings"
 	"sync"
@@ -602,92 +601,87 @@ func (c *DockerContainer) CopyFileFromContainer(ctx context.Context, filePath st
 	return ret, nil
 }
 
-// CopyDirToContainer copies the contents of a directory to a parent path in the container. This parent path must exist in the container first
-// as we cannot create it
-func (c *DockerContainer) CopyDirToContainer(ctx context.Context, hostDirPath string, containerParentPath string, fileMode int64) error {
-	dir, err := isDir(hostDirPath)
-	if err != nil {
+// CopyDirToContainer copies the contents of hostPath to containerPath in the container.
+// If fileMode is non-zero all files will have their file permissions set to that of fileMode
+// otherwise the file permissions will be copied from the host.
+// If fileMode contains bits not part of [fs.ModePerm] | [fs.ModeSetuid] | [fs.ModeSetgid] |
+// [fs.ModeSticky] an error is returned.
+// If the parent of the containerPath does not exist an error is returned.
+//
+// Deprecated: use [DockerContainer.CopyHostPathTo] instead.
+func (c *DockerContainer) CopyDirToContainer(ctx context.Context, hostPath string, containerPath string, fileMode int64) error {
+	if err := validateFileMode(fileMode); err != nil {
 		return err
 	}
 
-	if !dir {
-		// it's not a dir: let the consumer to handle an error
-		return fmt.Errorf("path %s is not a directory", hostDirPath)
-	}
-
-	buff, err := tarDir(hostDirPath, fileMode)
+	dir, err := isDir(hostPath)
 	if err != nil {
 		return err
 	}
 
-	// create the directory under its parent
-	parent := filepath.Dir(containerParentPath)
-
-	err = c.provider.client.CopyToContainer(ctx, c.ID, parent, buff, container.CopyToContainerOptions{})
-	if err != nil {
-		return err
+	if !dir {
+		// It's not a dir: let the consumer to handle an error.
+		return fmt.Errorf("host dir path %q is not a directory", hostPath)
 	}
-	defer c.provider.Close()
 
-	return nil
+	return c.CopyHostPathTo(ctx, hostPath, containerPath, copyToFileMode(fileMode))
 }
 
-func (c *DockerContainer) CopyFileToContainer(ctx context.Context, hostFilePath string, containerFilePath string, fileMode int64) error {
-	dir, err := isDir(hostFilePath)
+// CopyFileToContainer copies hostPath to containerPath in the container.
+// If fileMode is non-zero the files permissions will be set to that of fileMode
+// otherwise the file permissions will be set to that of the file in hostPath.
+// If fileMode contains bits not part of [fs.ModePerm] | [fs.ModeSetuid] | [fs.ModeSetgid] |
+// [fs.ModeSticky] an error is returned.
+// If the parent of the containerPath does not exist an error is returned.
+// If hostPath is a directory this is equivalent to [DockerContainer.CopyDirToContainer].
+//
+// Deprecated: use [DockerContainer.CopyHostPathTo] instead.
+func (c *DockerContainer) CopyFileToContainer(ctx context.Context, hostPath string, containerPath string, fileMode int64) error {
+	dir, err := isDir(hostPath)
 	if err != nil {
 		return err
 	}
 
 	if dir {
-		return c.CopyDirToContainer(ctx, hostFilePath, containerFilePath, fileMode)
-	}
-
-	f, err := os.Open(hostFilePath)
-	if err != nil {
-		return err
+		return c.CopyDirToContainer(ctx, hostPath, containerPath, fileMode)
 	}
-	defer f.Close()
 
-	info, err := f.Stat()
+	data, err := os.ReadFile(hostPath)
 	if err != nil {
-		return err
+		return fmt.Errorf("read file: %w", err)
 	}
 
-	// In Go 1.22 os.File is always an io.WriterTo. However, testcontainers
-	// currently allows Go 1.21, so we need to trick the compiler a little.
-	var file fs.File = f
-	return c.copyToContainer(ctx, func(tw io.Writer) error {
-		// Attempt optimized writeTo, implemented in linux
-		if wt, ok := file.(io.WriterTo); ok {
-			_, err := wt.WriteTo(tw)
-			return err
+	if fileMode == 0 {
+		fi, err := os.Stat(hostPath)
+		if err != nil {
+			return fmt.Errorf("stat file: %w", err)
 		}
-		_, err := io.Copy(tw, f)
-		return err
-	}, info.Size(), containerFilePath, fileMode)
-}
 
-// CopyToContainer copies fileContent data to a file in container
-func (c *DockerContainer) CopyToContainer(ctx context.Context, fileContent []byte, containerFilePath string, fileMode int64) error {
-	return c.copyToContainer(ctx, func(tw io.Writer) error {
-		_, err := tw.Write(fileContent)
-		return err
-	}, int64(len(fileContent)), containerFilePath, fileMode)
+		fileMode = int64(fi.Mode())
+	}
+
+	return c.CopyToContainer(ctx, data, containerPath, fileMode)
 }
 
-func (c *DockerContainer) copyToContainer(ctx context.Context, fileContent func(tw io.Writer) error, fileContentSize int64, containerFilePath string, fileMode int64) error {
-	buffer, err := tarFile(containerFilePath, fileContent, fileContentSize, fileMode)
-	if err != nil {
+// CopyToContainer copies a file with contents of fileContent to containerPath in the
+// container with the given fileMode.
+// If fileMode contains bits not part of [fs.ModePerm] | [fs.ModeSetuid] | [fs.ModeSetgid] |
+// [fs.ModeSticky] an error is returned.
+func (c *DockerContainer) CopyToContainer(ctx context.Context, fileContent []byte, containerPath string, fileMode int64) error {
+	if err := validateFileMode(fileMode); err != nil {
 		return err
 	}
 
-	err = c.provider.client.CopyToContainer(ctx, c.ID, "/", buffer, container.CopyToContainerOptions{})
+	// Create a tar with the single file containing the a file with the
+	// fully qualified container path as the name.
+	tar, err := tarFile(containerPath, fileContent, fs.FileMode(fileMode))
 	if err != nil {
 		return err
 	}
-	defer c.provider.Close()
 
-	return nil
+	// As the name of the file in the tar is the fully qualified container path
+	// it should by extracted in the container's root directory.
+	return c.copyTarTo(ctx, tar, "/")
 }
 
 type LogProductionOption func(*DockerContainer)