OCI indexer

Dockerfile

@@ -12,7 +12,7 @@ RUN go mod download
 
 COPY . .
 
-RUN go build -o /workspace/bin/clip /workspace/cmd/main.go
+RUN go build -v ./pkg/...
 
 RUN mkdir -p /tmp/test
 

Makefile

@@ -3,7 +3,7 @@ imageVersion := latest
 .PHONY: e2e
 
 build:
-	okteto build --build-arg BUILD_ENV=okteto -f ./Dockerfile -t localhost:5001/beam-clip:$(imageVersion)
+	docker build -f ./Dockerfile -t localhost:5001/beam-clip:$(imageVersion) .
 
 start:
 	cd hack; okteto up --file okteto.yml
@@ -12,5 +12,4 @@ stop:
 	cd hack; okteto down --file okteto.yml
 
 e2e:
-	go build -o ./bin/e2e ./e2e/main.go
-
+	go build -o ./bin/e2e ./e2e/main.go

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/config v1.27.5
 	github.com/aws/aws-sdk-go-v2/credentials v1.17.5
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.15.15
+	github.com/aws/aws-sdk-go-v2/service/ecr v1.27.0
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.51.2
 	github.com/gofrs/flock v0.8.1
 	github.com/google/go-containerregistry v0.19.1

go.sum

@@ -26,6 +26,8 @@ github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7
 github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.2 h1:en92G0Z7xlksoOylkUhuBSfJgijC7rHVLRdnIlHEs0E=
 github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.2/go.mod h1:HgtQ/wN5G+8QSlK62lbOtNwQ3wTSByJ4wH2rCkPt+AE=
+github.com/aws/aws-sdk-go-v2/service/ecr v1.27.0 h1:e9RAM6FgxAN3ca3LKaCr20+YnMqg8vhX/k6WDA8BpT8=
+github.com/aws/aws-sdk-go-v2/service/ecr v1.27.0/go.mod h1:Fa36Bp93PNtMtKHoyIvQnJY8EGTR0UQqRo3NfjW0hT0=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1 h1:EyBZibRTVAs6ECHZOw5/wlylS9OcTzwyjeQMudmREjE=
 github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.1/go.mod h1:JKpmtYhhPs7D97NL/ltqz7yCkERFW5dOlHyVl66ZYF8=
 github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.3 h1:fpFzBoro/MetYBk+8kxoQGMeKSkXbymnbUh2gy6nVgk=

pkg/clip/archive.go

@@ -16,6 +16,7 @@ import (
 	"path/filepath"
 	"strings"
 	"syscall"
+	"time"
 
 	"github.com/hanwen/go-fuse/v2/fuse"
 	log "github.com/rs/zerolog/log"
@@ -31,10 +32,15 @@ func init() {
 	gob.Register(&common.ClipNode{})
 	gob.Register(&common.StorageInfoWrapper{})
 	gob.Register(&common.S3StorageInfo{})
+	gob.Register(&common.OCIStorageInfo{})
+	gob.Register(&common.RemoteRef{})
+	gob.Register(&common.GzipCheckpoint{})
+	gob.Register(&common.GzipIndex{})
+	gob.Register(&common.ZstdFrame{})
+	gob.Register(&common.ZstdIndex{})
 }
 
 type ClipArchiverOptions struct {
-	Verbose     bool
 	Compress    bool
 	ArchivePath string
 	SourcePath  string
@@ -68,11 +74,27 @@ func (ig *InodeGenerator) Next() uint64 {
 
 // populateIndex creates a representation of the filesystem/folder being archived
 func (ca *ClipArchiver) populateIndex(index *btree.BTree, sourcePath string) error {
+	// Create root directory
+	now := time.Now()
 	root := &common.ClipNode{
 		Path:     "/",
 		NodeType: common.DirNode,
 		Attr: fuse.Attr{
-			Mode: uint32(os.ModeDir | 0755),
+			Ino:       1,
+			Size:      0,
+			Blocks:    0,
+			Atime:     uint64(now.Unix()),
+			Atimensec: uint32(now.Nanosecond()),
+			Mtime:     uint64(now.Unix()),
+			Mtimensec: uint32(now.Nanosecond()),
+			Ctime:     uint64(now.Unix()),
+			Ctimensec: uint32(now.Nanosecond()),
+			Mode:      uint32(syscall.S_IFDIR | 0755),
+			Nlink:     2, // Directories start with link count of 2 (. and ..)
+			Owner: fuse.Owner{
+				Uid: 0, // root
+				Gid: 0, // root
+			},
 		},
 	}
 	index.Set(root)
@@ -431,6 +453,12 @@ func (ca *ClipArchiver) ExtractMetadata(archivePath string) (*common.ClipArchive
 				return nil, fmt.Errorf("error decoding s3 storage info: %v", err)
 			}
 			storageInfo = s3Info
+		case string(common.StorageModeOCI):
+			var ociInfo common.OCIStorageInfo
+			if err := gob.NewDecoder(bytes.NewReader(wrapper.Data)).Decode(&ociInfo); err != nil {
+				return nil, fmt.Errorf("error decoding oci storage info: %v", err)
+			}
+			storageInfo = ociInfo
 		default:
 			return nil, fmt.Errorf("unsupported storage info type: %s", wrapper.Type)
 		}
@@ -496,10 +524,7 @@ func (ca *ClipArchiver) Extract(opts ClipArchiverOptions) error {
 	// Iterate over the index and extract every node
 	index.Ascend(index.Min(), func(a interface{}) bool {
 		node := a.(*common.ClipNode)
-
-		if opts.Verbose {
-			log.Info().Msgf("Extracting... %s", node.Path)
-		}
+		log.Debug().Str("path", node.Path).Msg("Extracting")
 
 		if node.NodeType == common.FileNode {
 			// Seek to the position of the file in the archive
@@ -599,9 +624,7 @@ func (ca *ClipArchiver) writeBlocks(index *btree.BTree, sourcePath string, outFi
 }
 
 func (ca *ClipArchiver) processNode(node *common.ClipNode, writer *bufio.Writer, sourcePath string, pos *int64, opts ClipArchiverOptions) bool {
-	if opts.Verbose {
-		log.Info().Msgf("Archiving... %s", node.Path)
-	}
+	log.Debug().Str("path", node.Path).Msg("Archiving")
 
 	f, err := os.Open(path.Join(sourcePath, node.Path))
 	if err != nil {

pkg/clip/archive_test.go

@@ -77,7 +77,6 @@ func TestCreateArchive(t *testing.T) {
 	options := CreateOptions{
 		InputPath:  tempDir,
 		OutputPath: archiveFile.Name(),
-		Verbose:    true,
 	}
 
 	err = CreateArchive(options)
@@ -107,7 +106,6 @@ func TestCreateArchive(t *testing.T) {
 	extractOptions := ExtractOptions{
 		InputFile:  archiveFile.Name(),
 		OutputPath: extractDir,
-		Verbose:    true,
 	}
 
 	err = ExtractArchive(extractOptions)
@@ -243,7 +241,6 @@ func BenchmarkCreateArchiveFromOCIImage(b *testing.B) {
 		options := CreateOptions{
 			InputPath:  tmpDir,
 			OutputPath: archiveFile.Name(),
-			Verbose:    false,
 		}
 
 		start := time.Now()