Handle 404 in GetAllImageDigestsFromRegistryAsync

src/ImageBuilder.Tests/GenerateEolAnnotationDataForPublishCommandTests.cs

@@ -8,6 +8,7 @@
 using System.IO;
 using System.Text.Json.Nodes;
 using System.Threading.Tasks;
+using Azure;
 using Microsoft.DotNet.ImageBuilder.Commands;
 using Microsoft.DotNet.ImageBuilder.Models.Annotations;
 using Microsoft.DotNet.ImageBuilder.Models.Image;
@@ -1031,6 +1032,104 @@ public async Task GenerateEolAnnotationData_PlatformRemoved()
             Assert.Equal(expectedEolAnnotationsJson, actualEolDigestsJson);
         }
 
+        [Fact]
+        public async Task GenerateEolAnnotationData_ManifestDeletedDuringEnumeration_Skipped()
+        {
+            using TempFolderContext tempFolderContext = TestHelper.UseTempFolder();
+            string repo1Image1DockerfilePath = DockerfileHelper.CreateDockerfile("1.0/runtime/os", tempFolderContext);
+
+            ImageArtifactDetails imageArtifactDetails = new()
+            {
+                Repos =
+                {
+                    new RepoData
+                    {
+                        Repo = "repo1",
+                        Images =
+                        {
+                            new ImageData
+                            {
+                                Platforms =
+                                {
+                                    Helpers.ImageInfoHelper.CreatePlatform(repo1Image1DockerfilePath,
+                                        simpleTags: ["tag"],
+                                        digest: DockerHelper.GetImageName(McrName, "repo1", digest: "platformdigest101"))
+                                },
+                                ProductVersion = "1.0",
+                                Manifest = new ManifestData
+                                {
+                                    SharedTags = ["1.0"],
+                                    Digest = DockerHelper.GetImageName(McrName, "repo1", digest: "imagedigest101")
+                                }
+                            }
+                        }
+                    }
+                }
+            };
+
+            string oldImageInfoPath = Path.Combine(tempFolderContext.Path, "old-image-info.json");
+            File.WriteAllText(oldImageInfoPath, JsonHelper.SerializeObject(imageArtifactDetails));
+
+            // Remove all images so everything in the registry is considered unsupported
+            imageArtifactDetails.Repos.Clear();
+
+            string newImageInfoPath = Path.Combine(tempFolderContext.Path, "new-image-info.json");
+            File.WriteAllText(newImageInfoPath, JsonHelper.SerializeObject(imageArtifactDetails));
+
+            string newEolDigestsListPath = Path.Combine(tempFolderContext.Path, "eolDigests.json");
+
+            // Registry lists three manifests, but one will return 404 when fetched (simulating concurrent deletion)
+            Mock<IAcrClient> registryClientMock = CreateAcrClientMock(
+                [
+                    CreateContainerRepository($"{DefaultRepoPrefix}repo1",
+                        manifestProperties: [
+                            CreateArtifactManifestProperties(digest: "platformdigest101", tags: ["tag"]),
+                            CreateArtifactManifestProperties(digest: "deleteddigest", tags: ["old"]),
+                            CreateArtifactManifestProperties(digest: "imagedigest101", tags: ["1.0"]),
+                        ])
+                ]);
+            IAcrClientFactory registryClientFactory = CreateAcrClientFactory(
+                AcrName, registryClientMock.Object);
+
+            // Set up content client mock where "deleteddigest" throws a 404
+            Mock<IAcrContentClient> contentClientMock = CreateAcrContentClientMock($"{DefaultRepoPrefix}repo1",
+                imageNameToQueryResultsMapping: new Dictionary<string, ManifestQueryResult>
+                {
+                    { "platformdigest101", new ManifestQueryResult(string.Empty, []) },
+                    { "imagedigest101", new ManifestQueryResult(string.Empty, []) },
+                });
+            contentClientMock
+                .Setup(o => o.GetManifestAsync("deleteddigest"))
+                .ThrowsAsync(new RequestFailedException(404, "manifest not found"));
+
+            IAcrContentClientFactory registryContentClientFactory = CreateAcrContentClientFactory(AcrName,
+                [contentClientMock]);
+
+            GenerateEolAnnotationDataForPublishCommand command =
+                InitializeCommand(
+                    oldImageInfoPath,
+                    newImageInfoPath,
+                    newEolDigestsListPath,
+                    registryClientFactory,
+                    registryContentClientFactory);
+            await command.ExecuteAsync();
+
+            EolAnnotationsData expectedEolAnnotations = new()
+            {
+                EolDate = _globalDate,
+                EolDigests =
+                [
+                    new(DockerHelper.GetImageName(AcrName, $"{DefaultRepoPrefix}repo1", digest: "imagedigest101")) { Tag = "1.0" },
+                    new(DockerHelper.GetImageName(AcrName, $"{DefaultRepoPrefix}repo1", digest: "platformdigest101")) { Tag = "tag" },
+                ]
+            };
+
+            string expectedEolAnnotationsJson = JsonConvert.SerializeObject(expectedEolAnnotations, Formatting.Indented, new JsonSerializerSettings { NullValueHandling = NullValueHandling.Ignore });
+            string actualEolDigestsJson = File.ReadAllText(newEolDigestsListPath);
+
+            Assert.Equal(expectedEolAnnotationsJson, actualEolDigestsJson);
+        }
+
         private static GenerateEolAnnotationDataForPublishCommand InitializeCommand(
             string oldImageInfoPath,
             string newImageInfoPath,

src/ImageBuilder/Commands/GenerateEolAnnotationDataCommandBase.cs

@@ -7,6 +7,7 @@
 using System.IO;
 using System.Linq;
 using System.Threading.Tasks;
+using Azure;
 using Azure.Containers.ContainerRegistry;
 using Microsoft.DotNet.ImageBuilder.Configuration;
 using Microsoft.DotNet.ImageBuilder.Models.Annotations;
@@ -83,7 +84,21 @@ await Parallel.ForEachAsync(repositoryNames, async (repositoryName, outerCT) =>
             IAsyncEnumerable<ArtifactManifestProperties> manifests = repo.GetAllManifestPropertiesAsync();
             await Parallel.ForEachAsync(manifests, outerCT, async (manifestProps, innerCT) =>
             {
-                ManifestQueryResult manifestResult = await contentClient.GetManifestAsync(manifestProps.Digest);
+                ManifestQueryResult manifestResult;
+                try
+                {
+                    manifestResult = await contentClient.GetManifestAsync(manifestProps.Digest);
+                }
+                catch (RequestFailedException ex) when (ex.Status == 404)
+                {
+                    // The manifest was listed but deleted before we could fetch it. This can happen
+                    // when images are being cleaned up concurrently. Skip it.
+                    _logger.LogWarning(
+                        "Manifest {Digest} in {Repository} was listed but no longer exists. Skipping.",
+                        manifestProps.Digest,
+                        repositoryName);
+                    return;
+                }
 
                 // We only want to return image or manifest list digests here. But the registry will also contain
                 // digests for annotations. These annotation digests should not be returned as we don't want to