diff --git a/cmd/chisel/cmd_cut.go b/cmd/chisel/cmd_cut.go index 35c81a79..cca9055f 100644 --- a/cmd/chisel/cmd_cut.go +++ b/cmd/chisel/cmd_cut.go @@ -11,6 +11,7 @@ import ( "github.com/canonical/chisel/internal/cache" "github.com/canonical/chisel/internal/setup" "github.com/canonical/chisel/internal/slicer" + "github.com/canonical/chisel/public/manifest" ) var shortCutHelp = "Cut a tree with selected slices" @@ -73,6 +74,21 @@ func (cmd *cmdCut) Execute(args []string) error { } } + mfest, err := slicer.Inspect(cmd.RootDir, release) + if err != nil { + return err + } + if mfest != nil { + mfest.IterateSlices("", func(slice *manifest.Slice) error { + sk, err := setup.ParseSliceKey(slice.Name) + if err != nil { + return err + } + sliceKeys = append(sliceKeys, sk) + return nil + }) + } + selection, err := setup.Select(release, sliceKeys, cmd.Arch) if err != nil { return err diff --git a/internal/manifestutil/manifestutil.go b/internal/manifestutil/manifestutil.go index 16b05402..f6bb7e60 100644 --- a/internal/manifestutil/manifestutil.go +++ b/internal/manifestutil/manifestutil.go @@ -18,22 +18,44 @@ import ( const DefaultFilename = "manifest.wall" +func collectManifests(slice *setup.Slice, collector func(path string, slice *setup.Slice)) { + for path, info := range slice.Contents { + if info.Generate == setup.GenerateManifest { + dir := strings.TrimSuffix(path, "**") + path = filepath.Join(dir, DefaultFilename) + collector(path, slice) + } + } +} + // FindPaths finds the paths marked with "generate:manifest" and // returns a map from the manifest path to all the slices that declare it. func FindPaths(slices []*setup.Slice) map[string][]*setup.Slice { manifestSlices := make(map[string][]*setup.Slice) + collector := func(path string, slice *setup.Slice) { + manifestSlices[path] = append(manifestSlices[path], slice) + } for _, slice := range slices { - for path, info := range slice.Contents { - if info.Generate == setup.GenerateManifest { - dir := strings.TrimSuffix(path, "**") - path = filepath.Join(dir, DefaultFilename) - manifestSlices[path] = append(manifestSlices[path], slice) - } - } + collectManifests(slice, collector) } return manifestSlices } +// FindPathsInRelease finds all the paths marked with "generate:manifest" +// for the given release. +func FindPathsInRelease(r *setup.Release) []string { + manifestPaths := make([]string,0) + collector := func(path string, slice *setup.Slice) { + manifestPaths = append(manifestPaths, path) + } + for _, pkg := range r.Packages { + for _, slice := range pkg.Slices { + collectManifests(slice, collector) + } + } + return manifestPaths +} + type WriteOptions struct { PackageInfo []*archive.PackageInfo Selection []*setup.Slice @@ -340,3 +362,12 @@ func Validate(mfest *manifest.Manifest) (err error) { } return nil } + +// CompareSchemas compares two manifest schema strings. +func CompareSchemas(va, vb string) int { + if va == manifest.Schema && va == vb { + return 0 + } + return -1 +} + diff --git a/internal/slicer/check.go b/internal/slicer/check.go new file mode 100644 index 00000000..ccf97605 --- /dev/null +++ b/internal/slicer/check.go @@ -0,0 +1,184 @@ +package slicer + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "syscall" + + "github.com/klauspost/compress/zstd" + + "github.com/canonical/chisel/internal/manifestutil" + "github.com/canonical/chisel/public/manifest" +) + +type pathInfo struct { + mode string + size int64 + link string + hash string +} + +func unixPerm(mode fs.FileMode) (perm uint32) { + perm = uint32(mode.Perm()) + if mode&fs.ModeSticky != 0 { + perm |= 0o1000 + } + return perm +} + +// checkRootDir checks the content of the target directory matches with +// the manifest. Files not managed by chisel are ignored. +// This function works under the assumption the manifest is valid. +func checkRootDir(mfest *manifest.Manifest, rootDir string) error { + singlePathsByFSInode := make(map[uint64]string) + fsInodeByManifestInode := make(map[uint64]uint64) + manifestInfos := make(map[string]*pathInfo) + err := mfest.IteratePaths("", func(path *manifest.Path) error { + pathHash := path.FinalSHA256 + if pathHash == "" { + pathHash = path.SHA256 + } + recordedPathInfo := &pathInfo{ + mode: path.Mode, + size: int64(path.Size), + link: path.Link, + hash: pathHash, + } + + fsInfo := &pathInfo{} + fullPath := filepath.Join(rootDir, path.Path) + info, err := os.Lstat(fullPath) + if err != nil { + return err + } + mode := info.Mode() + fsInfo.mode = fmt.Sprintf("0%o", unixPerm(mode)) + ftype := mode & fs.ModeType + switch ftype { + case fs.ModeDir: + // Nothing to do. + case fs.ModeSymlink: + fsInfo.link, err = os.Readlink(fullPath) + if err != nil { + return fmt.Errorf("cannot read symlink %q: %w", fullPath, err) + } + case 0: // Regular file. + h, err := contentHash(fullPath) + if err != nil { + return fmt.Errorf("cannot compute hash for %q: %w", fullPath, err) + } + fsInfo.hash = hex.EncodeToString(h) + fsInfo.size = info.Size() + default: + return fmt.Errorf("cannot check %q: unrecognized type %s", fullPath, mode.String()) + } + + // Collect manifests for tailored checking later. Adjust observed hash and + // size to still compare in a generic way. + if filepath.Base(path.Path) == manifestutil.DefaultFilename && recordedPathInfo.size == 0 && recordedPathInfo.hash == "" { + mfestInfo := *fsInfo + manifestInfos[path.Path] = &mfestInfo + fsInfo.size = 0 + fsInfo.hash = "" + } + + if recordedPathInfo.mode != fsInfo.mode { + return fmt.Errorf("inconsistent mode at %q: recorded %v, observed %v", path.Path, recordedPathInfo.mode, fsInfo.mode) + } + if recordedPathInfo.size != fsInfo.size { + return fmt.Errorf("inconsistent size at %q: recorded %v, observed %v", path.Path, recordedPathInfo.size, fsInfo.size) + } + if recordedPathInfo.link != fsInfo.link { + return fmt.Errorf("inconsistent link at %q: recorded %v, observed %v", path.Path, recordedPathInfo.link, fsInfo.link) + } + if recordedPathInfo.hash != fsInfo.hash { + return fmt.Errorf("inconsistent hash at %q: recorded %v, observed %v", path.Path, recordedPathInfo.hash, fsInfo.hash) + } + // Check hardlink. + if ftype != fs.ModeDir { + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return fmt.Errorf("cannot get syscall stat info for %q", info.Name()) + } + inode := stat.Ino + + if path.Inode == 0 { + // This path must not be linked to any other. + singlePath, ok := singlePathsByFSInode[inode] + if ok { + return fmt.Errorf("inconsistent content at %q: recorded no hardlink, observed hardlinked to %q", path.Path, singlePath) + } + singlePathsByFSInode[inode] = path.Path + } else { + recordedInode, ok := fsInodeByManifestInode[path.Inode] + if !ok { + fsInodeByManifestInode[path.Inode] = inode + } else if recordedInode != inode { + return fmt.Errorf("inconsistent content at %q: file hardlinked to a different inode", path.Path) + } + } + } + return nil + }) + if err != nil { + return err + } + + // Check manifests. + // They must all be valid manifests and be consistent per schema version. + schemaManifestInfos := make(map[string]*pathInfo) + for path, info := range manifestInfos { + fullPath := filepath.Join(rootDir, path) + f, err := os.Open(fullPath) + if err != nil { + return err + } + defer f.Close() + r, err := zstd.NewReader(f) + if err != nil { + return err + } + defer r.Close() + mfest, err = manifest.Read(r) + if err != nil { + return err + } + err = manifestutil.Validate(mfest) + if err != nil { + return err + } + schema := mfest.Schema() + refInfo, ok := schemaManifestInfos[schema] + if !ok { + schemaManifestInfos[schema] = info + continue + } + + if refInfo.size != info.size { + return fmt.Errorf("inconsistent manifest size for version %s at %q: recorded %v, observed %v", schema, path, refInfo.size, info.size) + } + if refInfo.hash != info.hash { + return fmt.Errorf("inconsistent manifest hash for version %s at %q: recorded %v, observed %v", schema, path, refInfo.hash, info.hash) + } + } + return nil +} + +func contentHash(path string) ([]byte, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return nil, err + } + return h.Sum(nil), nil +} diff --git a/internal/slicer/slicer.go b/internal/slicer/slicer.go index 9d3447fb..48eb0e11 100644 --- a/internal/slicer/slicer.go +++ b/internal/slicer/slicer.go @@ -3,10 +3,12 @@ package slicer import ( "archive/tar" "bytes" + "encoding/hex" "fmt" "io" "io/fs" "os" + "path" "path/filepath" "slices" "sort" @@ -21,6 +23,7 @@ import ( "github.com/canonical/chisel/internal/manifestutil" "github.com/canonical/chisel/internal/scripts" "github.com/canonical/chisel/internal/setup" + "github.com/canonical/chisel/public/manifest" ) const manifestMode fs.FileMode = 0644 @@ -537,3 +540,92 @@ func selectPkgArchives(archives map[string]archive.Archive, selection *setup.Sel } return pkgArchive, nil } + +// Inspect examines and validates the targetDir. It returns, if found and valid +// the manifest representing the content in the targetDir. +func Inspect(targetDir string, release *setup.Release) (*manifest.Manifest, error) { + var mfest *manifest.Manifest + manifestPaths := manifestutil.FindPathsInRelease(release) + if len(manifestPaths) > 0 { + logf("Inspecting root directory...") + var err error + mfest, err = selectValidManifest(targetDir, manifestPaths) + if err != nil { + return nil, err + } + if mfest != nil { + err = checkRootDir(mfest, targetDir) + if err != nil { + return nil, err + } + } + } + return mfest, nil +} + +// selectValidManifest returns, if found, a valid manifest with the latest +// schema. Consistency with all other manifests with the same schema is verified +// so the selection is deterministic. +func selectValidManifest(targetDir string, manifestPaths []string) (*manifest.Manifest, error) { + targetDir = filepath.Clean(targetDir) + if !filepath.IsAbs(targetDir) { + dir, err := os.Getwd() + if err != nil { + return nil, fmt.Errorf("cannot obtain current directory: %w", err) + } + targetDir = filepath.Join(dir, targetDir) + } + + type manifestHash struct { + path string + hash string + } + var selected *manifest.Manifest + schemaManifest := make(map[string]manifestHash) + for _, mfestPath := range manifestPaths { + err := func() error { + mfestFullPath := path.Join(targetDir, mfestPath) + f, err := os.Open(mfestFullPath) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + defer f.Close() + r, err := zstd.NewReader(f) + if err != nil { + return err + } + defer r.Close() + mfest, err := manifest.Read(r) + if err != nil { + return nil + } + err = manifestutil.Validate(mfest) + if err != nil { + return nil + } + + if selected == nil || manifestutil.CompareSchemas(mfest.Schema(), selected.Schema()) > 0 { + h, err := contentHash(mfestFullPath) + if err != nil { + return fmt.Errorf("cannot compute hash for %q: %w", mfestFullPath, err) + } + mfestHash := hex.EncodeToString(h) + refMfest, ok := schemaManifest[mfest.Schema()] + if !ok { + schemaManifest[mfest.Schema()] = manifestHash{mfestPath, mfestHash} + } else if refMfest.hash != mfestHash { + return fmt.Errorf("inconsistent manifests: %q and %q", refMfest.path, mfestPath) + } + selected = mfest + } + return nil + }() + if err != nil { + return nil, err + } + } + return selected, nil +} diff --git a/public/manifest/manifest.go b/public/manifest/manifest.go index 1e4809b8..65b362d5 100644 --- a/public/manifest/manifest.go +++ b/public/manifest/manifest.go @@ -68,6 +68,10 @@ func Read(reader io.Reader) (manifest *Manifest, err error) { return manifest, nil } +func (manifest *Manifest) Schema() string { + return manifest.db.Schema() +} + func (manifest *Manifest) IteratePaths(pathPrefix string, onMatch func(*Path) error) (err error) { return iteratePrefix(manifest, &Path{Kind: "path", Path: pathPrefix}, onMatch) }