From 5cdfe2e1f575b6c44f614c89c4f2eb6ec7543142 Mon Sep 17 00:00:00 2001 From: Arjun Raja Yogidas Date: Tue, 21 May 2024 17:10:56 +0000 Subject: [PATCH 1/4] add codebuild integration to github runners Signed-off-by: Arjun Raja Yogidas --- .github/workflows/build.yml | 5 +++-- integration/pull_test.go | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 724f3813e..b46a2fd36 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,7 +24,7 @@ env: jobs: test: - runs-on: ubuntu-20.04 + runs-on: codebuild-soci-github-ci-pipeline-${{ github.run_id }}-${{ github.run_attempt }}-ubuntu-7.0-xlarge timeout-minutes: 15 steps: - uses: actions/checkout@v4 @@ -34,7 +34,7 @@ jobs: - run: make - run: make test integration: - runs-on: ubuntu-20.04 + runs-on: codebuild-soci-github-ci-pipeline-${{ github.run_id }}-${{ github.run_attempt }}-ubuntu-7.0-xlarge timeout-minutes: 40 strategy: fail-fast: false @@ -47,4 +47,5 @@ jobs: - uses: actions/setup-go@v5 with: go-version: ${{ env.GO_VERSION }} + # - run: goenv global 1.22.1 - run: make integration diff --git a/integration/pull_test.go b/integration/pull_test.go index 0bfe1d372..d9929c470 100644 --- a/integration/pull_test.go +++ b/integration/pull_test.go @@ -236,7 +236,7 @@ func TestLazyPullWithSparseIndex(t *testing.T) { } func checkFuseMounts(t *testing.T, sh *shell.Shell, remoteSnapshotsExpectedCount int) { - mounts := string(sh.O("mount")) + mounts := string(sh.O("cat", "/proc/mounts")) remoteSnapshotsActualCount := strings.Count(mounts, "fuse.rawBridge") if remoteSnapshotsExpectedCount != remoteSnapshotsActualCount { t.Fatalf("incorrect number of remote snapshots; expected=%d, actual=%d", From 951a2f04223505e8d2d603cf0484964e6e433702 Mon Sep 17 00:00:00 2001 From: Arjun Raja Yogidas Date: Tue, 21 May 2024 17:14:38 +0000 Subject: [PATCH 2/4] test fix for fuse test failures Signed-off-by: Arjun Raja Yogidas --- cmd/soci/commands/create.go | 7 +------ soci/soci_index.go | 39 ++++++++++++++++++++++++++++++------- 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/cmd/soci/commands/create.go b/cmd/soci/commands/create.go index e4d8aa615..fd55c2a83 100644 --- a/cmd/soci/commands/create.go +++ b/cmd/soci/commands/create.go @@ -122,12 +122,7 @@ var CreateCommand = cli.Command{ return err } - sociIndexWithMetadata, err := builder.Build(ctx, srcImg) - if err != nil { - return err - } - - err = soci.WriteSociIndex(ctx, sociIndexWithMetadata, blobStore, builder.ArtifactsDb) + _, err = builder.BuildAndPush(ctx, blobStore, srcImg) if err != nil { return err } diff --git a/soci/soci_index.go b/soci/soci_index.go index f29d9c7f4..4c2cdd406 100644 --- a/soci/soci_index.go +++ b/soci/soci_index.go @@ -305,18 +305,42 @@ func NewIndexBuilder(contentStore content.Store, blobStore orascontent.Storage, }, nil } +// BuildAndPush builds a SOCI index, pushes and labels the artifacts, and returns the SOCI index. +func (b *IndexBuilder) BuildAndPush(ctx context.Context, blobStore store.Store, img images.Image) (*IndexWithMetadata, error) { + sociIndexWithMetadata, done, err := b.Build(ctx, blobStore, img) + if err != nil { + return nil, err + } + defer done(ctx) + + err = WriteSociIndex(ctx, sociIndexWithMetadata, blobStore, b.ArtifactsDb) + if err != nil { + return nil, err + } + return sociIndexWithMetadata, nil +} + // Build builds a soci index for `img` and return the index with metadata. -func (b *IndexBuilder) Build(ctx context.Context, img images.Image) (*IndexWithMetadata, error) { +// To prevent garbage collection of the zTOCs pushed by this command, +// we obtain a lease from containerd and also return it. +// It is the responsibility of the caller to terminate the lease. +// Ideally the lease ends after labelling and pushing the index and related zTOCs. +func (b *IndexBuilder) Build(ctx context.Context, blobStore store.Store, img images.Image) (*IndexWithMetadata, store.CleanupFunc, error) { // we get manifest descriptor before calling images.Manifest, since after calling // images.Manifest, images.Children will error out when reading the manifest blob (this happens on containerd side) imgManifestDesc, err := GetImageManifestDescriptor(ctx, b.contentStore, img.Target, platforms.OnlyStrict(b.config.platform)) if err != nil { - return nil, err + return nil, nil, err } manifest, err := images.Manifest(ctx, b.contentStore, img.Target, platforms.OnlyStrict(b.config.platform)) if err != nil { - return nil, err + return nil, nil, err + } + + ctx, done, err := blobStore.BatchOpen(ctx) + if err != nil { + return nil, nil, err } // attempt to build a ztoc for each layer @@ -357,8 +381,8 @@ func (b *IndexBuilder) Build(ctx context.Context, img images.Image) (*IndexWithM for _, err := range errs { errWrap = fmt.Errorf("%w; %v", errWrap, err) } - - return nil, errWrap + done(ctx) + return nil, nil, errWrap } ztocsDesc := make([]ocispec.Descriptor, 0, len(sociLayersDesc)) @@ -369,7 +393,8 @@ func (b *IndexBuilder) Build(ctx context.Context, img images.Image) (*IndexWithM } if len(ztocsDesc) == 0 { - return nil, ErrEmptyIndex + done(ctx) + return nil, nil, ErrEmptyIndex } annotations := map[string]string{ @@ -388,7 +413,7 @@ func (b *IndexBuilder) Build(ctx context.Context, img images.Image) (*IndexWithM Platform: &b.config.platform, ImageDigest: img.Target.Digest, CreatedAt: time.Now(), - }, nil + }, done, nil } // buildSociLayer builds a ztoc for an image layer (`desc`) and returns ztoc descriptor. From 1dcf0bfb15c01d033d8c758a6a4131cf0a57ce3d Mon Sep 17 00:00:00 2001 From: Arjun Raja Yogidas Date: Tue, 21 May 2024 18:52:42 +0000 Subject: [PATCH 3/4] test fix for fuse test failures - add nerdctl Signed-off-by: Arjun Raja Yogidas --- .github/workflows/build.yml | 12 ++++++++---- Dockerfile | 3 ++- fs/layer/node.go | 4 ++++ integration/metrics_test.go | 7 +++++++ integration/util_soci_test.go | 1 + integration/util_test.go | 9 +++++++++ 6 files changed, 31 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b46a2fd36..ffa52db14 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -24,7 +24,7 @@ env: jobs: test: - runs-on: codebuild-soci-github-ci-pipeline-${{ github.run_id }}-${{ github.run_attempt }}-ubuntu-7.0-xlarge + runs-on: codebuild-new_soci_CLI-${{ github.run_id }}-${{ github.run_attempt }}-ubuntu-7.0-xlarge timeout-minutes: 15 steps: - uses: actions/checkout@v4 @@ -34,7 +34,7 @@ jobs: - run: make - run: make test integration: - runs-on: codebuild-soci-github-ci-pipeline-${{ github.run_id }}-${{ github.run_attempt }}-ubuntu-7.0-xlarge + runs-on: codebuild-new_soci_CLI-${{ github.run_id }}-${{ github.run_attempt }}-ubuntu-7.0-xlarge timeout-minutes: 40 strategy: fail-fast: false @@ -47,5 +47,9 @@ jobs: - uses: actions/setup-go@v5 with: go-version: ${{ env.GO_VERSION }} - # - run: goenv global 1.22.1 - - run: make integration + # - run: wget https://github.com/containerd/nerdctl/releases/download/v1.7.6/nerdctl-1.7.6-linux-amd64.tar.gz + # - run: sudo tar Cxzvf /usr/local/bin nerdctl-1.7.6-linux-amd64.tar.gz + # - run: sudo apt -y update && sudo apt -y upgrade && sudo apt-get -y install gcc-multilib + # - run: uname -a + # - run: GO_TEST_FLAGS="-run TestLazyPullWithSparseIndex -count=1" make integration + - run: GO_TEST_FLAGS="-short -count=1" make integration diff --git a/Dockerfile b/Dockerfile index 9f525472d..bdcdac6aa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,7 +35,8 @@ RUN apk add --no-cache \ libc6-compat \ libseccomp-dev \ pigz \ - zlib-dev + zlib-dev \ + strace RUN cp $GOPATH/src/github.com/awslabs/soci-snapshotter/out/soci /usr/local/bin/ && \ cp $GOPATH/src/github.com/awslabs/soci-snapshotter/out/soci-snapshotter-grpc /usr/local/bin/ && \ mkdir /etc/soci-snapshotter-grpc && \ diff --git a/fs/layer/node.go b/fs/layer/node.go index 6459c1a48..7acbb540e 100644 --- a/fs/layer/node.go +++ b/fs/layer/node.go @@ -303,6 +303,8 @@ func (n *node) Readdir(ctx context.Context) (fusefs.DirStream, syscall.Errno) { if errno != 0 { return nil, errno } + + log.G(ctx).Debugf("Inside Readdir : %+v\n", ents) return fusefs.NewListDirStream(ents), 0 } @@ -541,6 +543,8 @@ func (n *node) Listxattr(ctx context.Context, dest []byte) (uint32, syscall.Errn for k := range ent.Xattrs { attrs = append(attrs, []byte(k+"\x00")...) } + + log.G(ctx).Debugf("Inside Listxattr : %+v\n", ent) if len(dest) < len(attrs) { return uint32(len(attrs)), syscall.ERANGE } diff --git a/integration/metrics_test.go b/integration/metrics_test.go index 0476b435d..5c41becc7 100644 --- a/integration/metrics_test.go +++ b/integration/metrics_test.go @@ -143,6 +143,10 @@ func TestOverlayFallbackMetric(t *testing.T) { } func TestFuseOperationFailureMetrics(t *testing.T) { + if testing.Short() { + t.Skip("skipping testing in short mode") + } + const logFuseOperationConfig = ` [fuse] log_fuse_operations = true @@ -397,6 +401,9 @@ func checkFuseOperationFailureMetrics(t *testing.T, output string, metricToCheck metricCountSum := 0 lines := strings.Split(output, "\n") + // t.Log("\n---------------------------- printinf outpurt of lines------------------------- \n") + // t.Log(output) + // t.Log("\n---------------------------- end------------------------- \n") for _, line := range lines { // skip non-fuse and fuse_mount_failure_count metrics if !strings.Contains(line, "fuse") || strings.Contains(line, commonmetrics.FuseMountFailureCount) { diff --git a/integration/util_soci_test.go b/integration/util_soci_test.go index cbf2675d8..5db727916 100644 --- a/integration/util_soci_test.go +++ b/integration/util_soci_test.go @@ -160,6 +160,7 @@ func buildIndex(sh *shell.Shell, src imageInfo, opt ...indexBuildOption) string "--platform", platforms.Format(src.platform), // this will make SOCI artifact available locally ) if err != nil { + fmt.Println("error line 163:", err) return "" } diff --git a/integration/util_test.go b/integration/util_test.go index e3d102c46..e933c2ffd 100644 --- a/integration/util_test.go +++ b/integration/util_test.go @@ -717,6 +717,7 @@ func rebootContainerd(t *testing.T, sh *shell.Shell, customContainerdConfig, cus sh.Gox(containerdCmds...) snapshotterCmds := shell.C("/usr/local/bin/soci-snapshotter-grpc", "--log-level", sociLogLevel, "--address", snapshotterSocket) + if customSnapshotterConfig != "" { snapshotterCmds = addConfig(t, sh, customSnapshotterConfig, snapshotterCmds...) } @@ -724,6 +725,14 @@ func rebootContainerd(t *testing.T, sh *shell.Shell, customContainerdConfig, cus if err != nil { t.Fatalf("failed to create pipe: %v", err) } + time.Sleep(2 * time.Second) + pids, err := sh.OLog("pgrep", "-f", "soci") + t.Log(string(pids)) + if err != nil { + t.Fatal(err) + } + // sh.Gox("strace", "-f", "-p", string(pids)) + reporter := testutil.NewTestingReporter(t) var m *testutil.LogMonitor = testutil.NewLogMonitor(reporter, outR, errR) From c509b82d4a7be10bf01084a1a9a9cfc1171ca39c Mon Sep 17 00:00:00 2001 From: Arjun Raja Yogidas Date: Fri, 24 May 2024 21:22:40 +0000 Subject: [PATCH 4/4] undo testing changes Signed-off-by: Arjun Raja Yogidas --- .github/workflows/build.yml | 7 +------ Dockerfile | 3 +-- fs/layer/node.go | 4 ---- integration/metrics_test.go | 7 ------- integration/util_soci_test.go | 1 - integration/util_test.go | 9 --------- 6 files changed, 2 insertions(+), 29 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ffa52db14..b2f0ceedb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -47,9 +47,4 @@ jobs: - uses: actions/setup-go@v5 with: go-version: ${{ env.GO_VERSION }} - # - run: wget https://github.com/containerd/nerdctl/releases/download/v1.7.6/nerdctl-1.7.6-linux-amd64.tar.gz - # - run: sudo tar Cxzvf /usr/local/bin nerdctl-1.7.6-linux-amd64.tar.gz - # - run: sudo apt -y update && sudo apt -y upgrade && sudo apt-get -y install gcc-multilib - # - run: uname -a - # - run: GO_TEST_FLAGS="-run TestLazyPullWithSparseIndex -count=1" make integration - - run: GO_TEST_FLAGS="-short -count=1" make integration + - run: make integration diff --git a/Dockerfile b/Dockerfile index bdcdac6aa..9f525472d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,8 +35,7 @@ RUN apk add --no-cache \ libc6-compat \ libseccomp-dev \ pigz \ - zlib-dev \ - strace + zlib-dev RUN cp $GOPATH/src/github.com/awslabs/soci-snapshotter/out/soci /usr/local/bin/ && \ cp $GOPATH/src/github.com/awslabs/soci-snapshotter/out/soci-snapshotter-grpc /usr/local/bin/ && \ mkdir /etc/soci-snapshotter-grpc && \ diff --git a/fs/layer/node.go b/fs/layer/node.go index 7acbb540e..6459c1a48 100644 --- a/fs/layer/node.go +++ b/fs/layer/node.go @@ -303,8 +303,6 @@ func (n *node) Readdir(ctx context.Context) (fusefs.DirStream, syscall.Errno) { if errno != 0 { return nil, errno } - - log.G(ctx).Debugf("Inside Readdir : %+v\n", ents) return fusefs.NewListDirStream(ents), 0 } @@ -543,8 +541,6 @@ func (n *node) Listxattr(ctx context.Context, dest []byte) (uint32, syscall.Errn for k := range ent.Xattrs { attrs = append(attrs, []byte(k+"\x00")...) } - - log.G(ctx).Debugf("Inside Listxattr : %+v\n", ent) if len(dest) < len(attrs) { return uint32(len(attrs)), syscall.ERANGE } diff --git a/integration/metrics_test.go b/integration/metrics_test.go index 5c41becc7..0476b435d 100644 --- a/integration/metrics_test.go +++ b/integration/metrics_test.go @@ -143,10 +143,6 @@ func TestOverlayFallbackMetric(t *testing.T) { } func TestFuseOperationFailureMetrics(t *testing.T) { - if testing.Short() { - t.Skip("skipping testing in short mode") - } - const logFuseOperationConfig = ` [fuse] log_fuse_operations = true @@ -401,9 +397,6 @@ func checkFuseOperationFailureMetrics(t *testing.T, output string, metricToCheck metricCountSum := 0 lines := strings.Split(output, "\n") - // t.Log("\n---------------------------- printinf outpurt of lines------------------------- \n") - // t.Log(output) - // t.Log("\n---------------------------- end------------------------- \n") for _, line := range lines { // skip non-fuse and fuse_mount_failure_count metrics if !strings.Contains(line, "fuse") || strings.Contains(line, commonmetrics.FuseMountFailureCount) { diff --git a/integration/util_soci_test.go b/integration/util_soci_test.go index 5db727916..cbf2675d8 100644 --- a/integration/util_soci_test.go +++ b/integration/util_soci_test.go @@ -160,7 +160,6 @@ func buildIndex(sh *shell.Shell, src imageInfo, opt ...indexBuildOption) string "--platform", platforms.Format(src.platform), // this will make SOCI artifact available locally ) if err != nil { - fmt.Println("error line 163:", err) return "" } diff --git a/integration/util_test.go b/integration/util_test.go index e933c2ffd..e3d102c46 100644 --- a/integration/util_test.go +++ b/integration/util_test.go @@ -717,7 +717,6 @@ func rebootContainerd(t *testing.T, sh *shell.Shell, customContainerdConfig, cus sh.Gox(containerdCmds...) snapshotterCmds := shell.C("/usr/local/bin/soci-snapshotter-grpc", "--log-level", sociLogLevel, "--address", snapshotterSocket) - if customSnapshotterConfig != "" { snapshotterCmds = addConfig(t, sh, customSnapshotterConfig, snapshotterCmds...) } @@ -725,14 +724,6 @@ func rebootContainerd(t *testing.T, sh *shell.Shell, customContainerdConfig, cus if err != nil { t.Fatalf("failed to create pipe: %v", err) } - time.Sleep(2 * time.Second) - pids, err := sh.OLog("pgrep", "-f", "soci") - t.Log(string(pids)) - if err != nil { - t.Fatal(err) - } - // sh.Gox("strace", "-f", "-p", string(pids)) - reporter := testutil.NewTestingReporter(t) var m *testutil.LogMonitor = testutil.NewLogMonitor(reporter, outR, errR)