diff --git a/toolkit/go.mod b/toolkit/go.mod index d0b8f2ee2..dae3e03e9 100644 --- a/toolkit/go.mod +++ b/toolkit/go.mod @@ -1,5 +1,8 @@ module github.com/quay/claircore/toolkit -go 1.24 +go 1.25.0 -require github.com/google/go-cmp v0.7.0 +require ( + github.com/google/go-cmp v0.7.0 + golang.org/x/sys v0.37.0 +) diff --git a/toolkit/go.sum b/toolkit/go.sum index 40e761ae7..409b8c59f 100644 --- a/toolkit/go.sum +++ b/toolkit/go.sum @@ -1,2 +1,4 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= +golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= diff --git a/toolkit/spool/os_linux.go b/toolkit/spool/os_linux.go new file mode 100644 index 000000000..12ce81729 --- /dev/null +++ b/toolkit/spool/os_linux.go @@ -0,0 +1,93 @@ +package spool + +import ( + "fmt" + "io/fs" + "os" + "runtime" + "sync" + + "golang.org/x/sys/unix" +) + +// Init initializes [root]. +func init() { + var p string + + // If the environment was explicitly set, use it. + var ok bool + if p, ok = os.LookupEnv("TMPDIR"); ok { + goto Open + } + + // Try to honor file-hierarchy(7). + for _, name := range []string{`/var/tmp`, os.TempDir()} { + fi, err := os.Stat(name) + if err == nil && fi.IsDir() { + p = name + goto Open + } + } + +Open: + var err error + root, err = os.OpenRoot(p) + if err != nil { + panic(err) + } +} + +func checkRootTmpFile() bool { + f, err := root.OpenFile(".", os.O_WRONLY|unix.O_TMPFILE, 0o600) + if err != nil { + return false + } + f.Close() + return true +} + +var haveTmpFile = sync.OnceValue(checkRootTmpFile) + +func osAdjustName(name string) string { + if haveTmpFile() { + return "." + } + return name +} + +func osAdjustFlag(flag int) int { + if haveTmpFile() && (flag&os.O_CREATE != 0) { + // If we can use tmp, do so. + flag &= ^os.O_CREATE + flag |= unix.O_TMPFILE + } + return flag +} + +func osAddCleanup(f *os.File) { + // If not opened with O_TMPFILE (or there was an error), arrange for the + // file to be removed. + flags, err := unix.FcntlInt(f.Fd(), unix.F_GETFL, 0) + if err != nil || flags&unix.O_TMPFILE == 0 { + runtime.AddCleanup(f, func(name string) { root.Remove(name) }, f.Name()) + } +} + +// Reopen provides or emulates re-opening a file and obtaining an independent file description. +// +// The Linux implementation reopens files via [magic symlinks] in [proc]. +// +// [magic symlinks]: https://www.man7.org/linux/man-pages/man7/symlink.7.html +// [proc]: https://man7.org/linux/man-pages/man5/proc.5.html +func Reopen(f *os.File, flag int) (*os.File, error) { + if flag&os.O_CREATE != 0 { + return nil, fmt.Errorf("spool: cannot pass O_CREATE to Reopen: %w", fs.ErrInvalid) + } + fd := int(f.Fd()) + if fd == -1 { + return nil, fs.ErrClosed + } + p := fmt.Sprintf("/proc/self/fd/%d", fd) + + return os.OpenFile(p, flag, 0) +} diff --git a/toolkit/spool/os_unix.go b/toolkit/spool/os_unix.go new file mode 100644 index 000000000..e4512e591 --- /dev/null +++ b/toolkit/spool/os_unix.go @@ -0,0 +1,55 @@ +//go:build unix && !linux + +package spool + +import ( + "fmt" + "io/fs" + "os" + "runtime" +) + +// Init initializes [root]. +func init() { + var p string + + // If the environment was explicitly set, use it. + var ok bool + if p, ok = os.LookupEnv("TMPDIR"); ok { + goto Open + } + + // Try to honor file-hierarchy(7). + for _, name := range []string{`/var/tmp`, os.TempDir()} { + fi, err := os.Stat(name) + if err == nil && fi.IsDir() { + p = name + goto Open + } + } + +Open: + var err error + root, err = os.OpenRoot(p) + if err != nil { + panic(err) + } +} + +func osAdjustName(name string) string { return name } + +func osAdjustFlag(flag int) int { + return flag & os.O_CREATE +} + +func osAddCleanup(f *os.File) { + runtime.AddCleanup(f, func(name string) { root.Remove(name) }, f.Name()) +} + +// Reopen provides or emulates re-opening a file and obtaining an independent file description. +func Reopen(f *os.File, flag int) (*os.File, error) { + if flag&os.O_CREATE != 0 { + return nil, fmt.Errorf("spool: cannot pass O_CREATE to Reopen: %w", fs.ErrInvalid) + } + return root.OpenFile(f.Name(), flag, 0) +} diff --git a/toolkit/spool/os_windows.go b/toolkit/spool/os_windows.go new file mode 100644 index 000000000..2b4374a55 --- /dev/null +++ b/toolkit/spool/os_windows.go @@ -0,0 +1 @@ +package spool diff --git a/toolkit/spool/spool.go b/toolkit/spool/spool.go new file mode 100644 index 000000000..2f36176e7 --- /dev/null +++ b/toolkit/spool/spool.go @@ -0,0 +1,94 @@ +// Package spool provides utilities for managing "spool files". +// +// Files returned by this package can be counted on to be removed when all open descriptors are closed. +package spool + +import ( + "cmp" + "io/fs" + "math/rand/v2" + "os" + "runtime" + "strconv" +) + +// Root is the location all the spool files go into. +// +// This is initialized in os-specific files. +var root *os.Root + +// Mkname generates a unique file name based on the provided prefix. +func mkname(prefix string) string { + return cmp.Or(prefix, "tmp") + "." + + strconv.FormatUint(uint64(rand.Uint32()), 10) +} + +// OpenFile opens a temporary file with the provided prefix. +// +// If "prefix" is not provided, "tmp" will be used. +// Returned files cannot be opened by path. Callers should use [Reopen]. +func OpenFile(prefix string, flag int, perm fs.FileMode) (*os.File, error) { + name := osAdjustName(mkname(prefix)) + flag = osAdjustFlag(flag) + f, err := root.OpenFile(name, flag, perm) + if f != nil { + osAddCleanup(f) + } + return f, err +} + +// Create returns an [*os.File] that cannot be opened by path and will be +// removed when closed. +func Create() (*os.File, error) { + return OpenFile("", os.O_CREATE|os.O_RDWR, 0o600) +} + +// Mkdir creates a directory with the provided prefix. +// +// The directory will have its contents removed when the returned [*os.Root] is +// garbage collected. +func Mkdir(prefix string, perm fs.FileMode) (*os.Root, error) { + name := mkname(prefix) + if err := root.Mkdir(name, perm); err != nil { + return nil, err + } + r, err := root.OpenRoot(name) + if err == nil { // NB If successful + runtime.AddCleanup(r, func(name string) { + root.RemoveAll(name) + }, name) + } + return r, err +} + +/* +This package needs a few parts implemented in OS-specific ways. +Below is a quick rundown of them, along with a ready-to-use documentation comment. + +# Exported + +The documentation for these implementations should add additional paragraphs explaining the OS-specific parts starting "The ${OS} implementation [...]". + +See the Linux implementations in os_linux.go for an example. + + // Reopen provides or emulates re-opening a file and obtaining an independent file description. + func Reopen(f *os.File, flag int) (*os.File, error) + +The [Reopen] API is not possible with dup(2), which returns another file descriptor to the same file description. +An implementation using dup(2) would not provide independent offsets. + +# Unexported + + osAdjustName(string) string + +AdjustName should modify the passed file name as needed and return the result. + + osAdjustFlag(int) int + +AdjustFlag should modify the passed flags as needed and return the result. + + osAddCleanup(*os.File) + +AddCleanup should use [runtime.AddCleanup] to attach any needed cleanup functions to the passed [*os.File]. +An implementation will not be called with a nil pointer. +*/ diff --git a/toolkit/spool/spool_test.go b/toolkit/spool/spool_test.go new file mode 100644 index 000000000..5ed2c8667 --- /dev/null +++ b/toolkit/spool/spool_test.go @@ -0,0 +1,173 @@ +package spool + +import ( + "bytes" + crand "crypto/rand" + "io" + "math/rand/v2" + "os" + "testing" +) + +func resetRoot(t *testing.T) { + if root == nil { + root.Close() + root = nil + } + var err error + root, err = os.OpenRoot(t.TempDir()) + if err != nil { + t.Fatal(err) + } +} + +func TestCreate(t *testing.T) { + resetRoot(t) + + f, err := Create() + if err != nil { + t.Fatal(err) + } + defer f.Close() + + const x = `testing` + if _, err := io.WriteString(f, x); err != nil { + t.Error(err) + } + + want, got := []byte(x), make([]byte, len(x)) + if _, err := f.ReadAt(got, 0); err != nil { + t.Error(err) + } + t.Logf("got: %q, want: %q", string(got), string(want)) + if !bytes.Equal(got, want) { + t.Fail() + } +} + +func TestOpenFile(t *testing.T) { + resetRoot(t) + + f, err := OpenFile("TestOpenFile", os.O_CREATE|os.O_EXCL|os.O_RDWR, 0o600) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + const x = `testing` + if _, err := io.WriteString(f, x); err != nil { + t.Error(err) + } + + want, got := []byte(x), make([]byte, len(x)) + if _, err := f.ReadAt(got, 0); err != nil { + t.Error(err) + } + t.Logf("got: %q, want: %q", string(got), string(want)) + if !bytes.Equal(got, want) { + t.Fail() + } +} + +func TestReopen(t *testing.T) { + resetRoot(t) + + f1, err := Create() + if err != nil { + t.Fatal(err) + } + defer f1.Close() + + f2, err := Reopen(f1, os.O_RDONLY) + if err != nil { + t.Fatal(err) + } + defer f2.Close() + + if _, err := io.CopyN(f1, crand.Reader, 4096); err != nil { + t.Error(err) + } + + off1, err := f1.Seek(0, io.SeekCurrent) + if err != nil { + t.Error(err) + } + off2, err := f2.Seek(0, io.SeekCurrent) + if err != nil { + t.Error(err) + } + t.Logf("f1: %d", off1) + t.Logf("f2: %d", off2) + if off1 != 4096 || off2 != 0 { + t.Errorf("file descriptors are not independent") + } + + const sz = 32 + pos := rand.Int64N(4096 - sz) + b1, b2 := make([]byte, sz), make([]byte, sz) + if _, err := f1.ReadAt(b1, pos); err != nil { + t.Error(err) + } + if _, err := f2.ReadAt(b2, pos); err != nil { + t.Error(err) + } + t.Logf("f1: %x", b1) + t.Logf("f2: %x", b2) + if !bytes.Equal(b1, b2) { + t.Errorf("file descriptors have different backing") + } +} + +func TestMkdir(t *testing.T) { + resetRoot(t) + + dir, err := Mkdir("mkdir", 0o755) + if err != nil { + t.Fatal(err) + } + defer dir.Close() + + f1, err := dir.Create("file") + if err != nil { + t.Fatal(err) + } + defer f1.Close() + if _, err := io.CopyN(f1, crand.Reader, 4096); err != nil { + t.Error(err) + } + + f2, err := dir.Open("file") + if err != nil { + t.Fatal(err) + } + defer f2.Close() + + off1, err := f1.Seek(0, io.SeekCurrent) + if err != nil { + t.Error(err) + } + off2, err := f2.Seek(0, io.SeekCurrent) + if err != nil { + t.Error(err) + } + t.Logf("f1: %d", off1) + t.Logf("f2: %d", off2) + if off1 != 4096 || off2 != 0 { + t.Errorf("file descriptors are not independent") + } + + const sz = 32 + pos := rand.Int64N(4096 - sz) + b1, b2 := make([]byte, sz), make([]byte, sz) + if _, err := f1.ReadAt(b1, pos); err != nil { + t.Error(err) + } + if _, err := f2.ReadAt(b2, pos); err != nil { + t.Error(err) + } + t.Logf("f1: %x", b1) + t.Logf("f2: %x", b2) + if !bytes.Equal(b1, b2) { + t.Errorf("file descriptors have different backing") + } +}