From 436c413b1d91451840ca0efde03467c4aead02dc Mon Sep 17 00:00:00 2001 From: Logan Bussell Date: Fri, 27 Mar 2026 15:49:26 -0700 Subject: [PATCH] Handle 404 in GetAllImageDigestsFromRegistryAsync during parallel enumeration When enumerating manifests from the registry, a manifest can be deleted between the time it is listed by GetAllManifestPropertiesAsync and when GetManifestAsync attempts to fetch it. This race condition occurs when the registry has retention policies or concurrent cleanup operations. Catch RequestFailedException with status 404 and skip the manifest instead of crashing the entire command. This was the root cause of the cleanup pipeline (cleanup-acr-images-official) failing consistently. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...EolAnnotationDataForPublishCommandTests.cs | 99 +++++++++++++++++++ .../GenerateEolAnnotationDataCommandBase.cs | 17 +++- 2 files changed, 115 insertions(+), 1 deletion(-) diff --git a/src/ImageBuilder.Tests/GenerateEolAnnotationDataForPublishCommandTests.cs b/src/ImageBuilder.Tests/GenerateEolAnnotationDataForPublishCommandTests.cs index 106aa7b38..f8d3af0d9 100644 --- a/src/ImageBuilder.Tests/GenerateEolAnnotationDataForPublishCommandTests.cs +++ b/src/ImageBuilder.Tests/GenerateEolAnnotationDataForPublishCommandTests.cs @@ -8,6 +8,7 @@ using System.IO; using System.Text.Json.Nodes; using System.Threading.Tasks; +using Azure; using Microsoft.DotNet.ImageBuilder.Commands; using Microsoft.DotNet.ImageBuilder.Models.Annotations; using Microsoft.DotNet.ImageBuilder.Models.Image; @@ -1031,6 +1032,104 @@ public async Task GenerateEolAnnotationData_PlatformRemoved() Assert.Equal(expectedEolAnnotationsJson, actualEolDigestsJson); } + [Fact] + public async Task GenerateEolAnnotationData_ManifestDeletedDuringEnumeration_Skipped() + { + using TempFolderContext tempFolderContext = TestHelper.UseTempFolder(); + string repo1Image1DockerfilePath = DockerfileHelper.CreateDockerfile("1.0/runtime/os", tempFolderContext); + + ImageArtifactDetails imageArtifactDetails = new() + { + Repos = + { + new RepoData + { + Repo = "repo1", + Images = + { + new ImageData + { + Platforms = + { + Helpers.ImageInfoHelper.CreatePlatform(repo1Image1DockerfilePath, + simpleTags: ["tag"], + digest: DockerHelper.GetImageName(McrName, "repo1", digest: "platformdigest101")) + }, + ProductVersion = "1.0", + Manifest = new ManifestData + { + SharedTags = ["1.0"], + Digest = DockerHelper.GetImageName(McrName, "repo1", digest: "imagedigest101") + } + } + } + } + } + }; + + string oldImageInfoPath = Path.Combine(tempFolderContext.Path, "old-image-info.json"); + File.WriteAllText(oldImageInfoPath, JsonHelper.SerializeObject(imageArtifactDetails)); + + // Remove all images so everything in the registry is considered unsupported + imageArtifactDetails.Repos.Clear(); + + string newImageInfoPath = Path.Combine(tempFolderContext.Path, "new-image-info.json"); + File.WriteAllText(newImageInfoPath, JsonHelper.SerializeObject(imageArtifactDetails)); + + string newEolDigestsListPath = Path.Combine(tempFolderContext.Path, "eolDigests.json"); + + // Registry lists three manifests, but one will return 404 when fetched (simulating concurrent deletion) + Mock registryClientMock = CreateAcrClientMock( + [ + CreateContainerRepository($"{DefaultRepoPrefix}repo1", + manifestProperties: [ + CreateArtifactManifestProperties(digest: "platformdigest101", tags: ["tag"]), + CreateArtifactManifestProperties(digest: "deleteddigest", tags: ["old"]), + CreateArtifactManifestProperties(digest: "imagedigest101", tags: ["1.0"]), + ]) + ]); + IAcrClientFactory registryClientFactory = CreateAcrClientFactory( + AcrName, registryClientMock.Object); + + // Set up content client mock where "deleteddigest" throws a 404 + Mock contentClientMock = CreateAcrContentClientMock($"{DefaultRepoPrefix}repo1", + imageNameToQueryResultsMapping: new Dictionary + { + { "platformdigest101", new ManifestQueryResult(string.Empty, []) }, + { "imagedigest101", new ManifestQueryResult(string.Empty, []) }, + }); + contentClientMock + .Setup(o => o.GetManifestAsync("deleteddigest")) + .ThrowsAsync(new RequestFailedException(404, "manifest not found")); + + IAcrContentClientFactory registryContentClientFactory = CreateAcrContentClientFactory(AcrName, + [contentClientMock]); + + GenerateEolAnnotationDataForPublishCommand command = + InitializeCommand( + oldImageInfoPath, + newImageInfoPath, + newEolDigestsListPath, + registryClientFactory, + registryContentClientFactory); + await command.ExecuteAsync(); + + EolAnnotationsData expectedEolAnnotations = new() + { + EolDate = _globalDate, + EolDigests = + [ + new(DockerHelper.GetImageName(AcrName, $"{DefaultRepoPrefix}repo1", digest: "imagedigest101")) { Tag = "1.0" }, + new(DockerHelper.GetImageName(AcrName, $"{DefaultRepoPrefix}repo1", digest: "platformdigest101")) { Tag = "tag" }, + ] + }; + + string expectedEolAnnotationsJson = JsonConvert.SerializeObject(expectedEolAnnotations, Formatting.Indented, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore }); + string actualEolDigestsJson = File.ReadAllText(newEolDigestsListPath); + + Assert.Equal(expectedEolAnnotationsJson, actualEolDigestsJson); + } + private static GenerateEolAnnotationDataForPublishCommand InitializeCommand( string oldImageInfoPath, string newImageInfoPath, diff --git a/src/ImageBuilder/Commands/GenerateEolAnnotationDataCommandBase.cs b/src/ImageBuilder/Commands/GenerateEolAnnotationDataCommandBase.cs index 9c2f1494e..61a8f8c0e 100644 --- a/src/ImageBuilder/Commands/GenerateEolAnnotationDataCommandBase.cs +++ b/src/ImageBuilder/Commands/GenerateEolAnnotationDataCommandBase.cs @@ -7,6 +7,7 @@ using System.IO; using System.Linq; using System.Threading.Tasks; +using Azure; using Azure.Containers.ContainerRegistry; using Microsoft.DotNet.ImageBuilder.Configuration; using Microsoft.DotNet.ImageBuilder.Models.Annotations; @@ -83,7 +84,21 @@ await Parallel.ForEachAsync(repositoryNames, async (repositoryName, outerCT) => IAsyncEnumerable manifests = repo.GetAllManifestPropertiesAsync(); await Parallel.ForEachAsync(manifests, outerCT, async (manifestProps, innerCT) => { - ManifestQueryResult manifestResult = await contentClient.GetManifestAsync(manifestProps.Digest); + ManifestQueryResult manifestResult; + try + { + manifestResult = await contentClient.GetManifestAsync(manifestProps.Digest); + } + catch (RequestFailedException ex) when (ex.Status == 404) + { + // The manifest was listed but deleted before we could fetch it. This can happen + // when images are being cleaned up concurrently. Skip it. + _logger.LogWarning( + "Manifest {Digest} in {Repository} was listed but no longer exists. Skipping.", + manifestProps.Digest, + repositoryName); + return; + } // We only want to return image or manifest list digests here. But the registry will also contain // digests for annotations. These annotation digests should not be returned as we don't want to