From 1ce149c7b9aba31591f1490747c19175134ac803 Mon Sep 17 00:00:00 2001 From: Adam Zippor Date: Mon, 15 Sep 2025 16:27:08 +0200 Subject: [PATCH 01/19] Add controller for codeflow history --- .../VirtualMonoRepo/CodeflowHistory.cs | 22 ++++++++++ .../Controllers/SubscriptionsController.cs | 41 +++++++++++++++++++ .../Controllers/SubscriptionsController.cs | 8 ++++ .../Controllers/SubscriptionsController.cs | 8 ++++ 4 files changed, 79 insertions(+) create mode 100644 src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/CodeflowHistory.cs diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/CodeflowHistory.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/CodeflowHistory.cs new file mode 100644 index 0000000000..501078bc97 --- /dev/null +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/CodeflowHistory.cs @@ -0,0 +1,22 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; + +namespace Microsoft.DotNet.DarcLib.VirtualMonoRepo; + +public class CodeflowHistory +{ + public string SorceRepoUrl { get; set; } + public string TargetRepoUrl { get; set; } + public List RepoCommits { get; set; } + public List VmrCommits { get; set; } + public List ForwardFlows {get; set; } + public List Backflows { get; set; } +} + +public class Codeflow +{ + public string SourceCommitSha { get; set; } + public string TargetCommitSha { get; set; } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs index 4a7b058148..4ed0344bde 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs @@ -8,6 +8,8 @@ using Microsoft.AspNetCore.ApiVersioning; using Microsoft.AspNetCore.ApiVersioning.Swashbuckle; using Microsoft.AspNetCore.Mvc; +using Microsoft.DotNet.DarcLib; +using Microsoft.DotNet.DarcLib.VirtualMonoRepo; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Options; using ProductConstructionService.Api.v2018_07_16.Models; @@ -27,6 +29,7 @@ public class SubscriptionsController : ControllerBase private readonly BuildAssetRegistryContext _context; private readonly IWorkItemProducerFactory _workItemProducerFactory; private readonly IGitHubInstallationIdResolver _installationIdResolver; + private readonly IRemoteFactory _remoteFactory; private readonly ILogger _logger; protected readonly IOptions _environmentNamespaceOptions; @@ -36,12 +39,14 @@ public SubscriptionsController( IWorkItemProducerFactory workItemProducerFactory, IGitHubInstallationIdResolver installationIdResolver, IOptions environmentNamespaceOptions, + IRemoteFactory remoteFactory, ILogger logger) { _context = context; _workItemProducerFactory = workItemProducerFactory; _installationIdResolver = installationIdResolver; _environmentNamespaceOptions = environmentNamespaceOptions; + _remoteFactory = remoteFactory; _logger = logger; } @@ -104,6 +109,14 @@ public virtual async Task GetSubscription(Guid id) return Ok(new Subscription(subscription)); } + [HttpPost("{id}/codeflowhistory")] + [SwaggerApiResponse(HttpStatusCode.Accepted, Type = typeof(Subscription), Description = "Subscription update has been triggered")] + [ValidateModelState] + public virtual async Task GetCodeflowHistory(Guid id) + { + return await GetCodeflowHistoryCore(id); + } + /// /// Trigger a manually by id /// @@ -156,6 +169,34 @@ protected async Task TriggerSubscriptionCore(Guid id, int buildId return Accepted(new Subscription(subscription)); } + protected async Task GetCodeflowHistoryCore(Guid id) + { + Maestro.Data.Models.Subscription? subscription = await _context.Subscriptions.Include(sub => sub.LastAppliedBuild) + .Include(sub => sub.Channel) + .Include(sub => sub.LastAppliedBuild) + .Include(sub => sub.ExcludedAssets) + .FirstOrDefaultAsync(sub => sub.Id == id); + + if (subscription == null) + { + return NotFound(); + } + + IRemote sourceRemote = await _remoteFactory.CreateRemoteAsync(subscription.SourceRepository); + + IRemote targetRemote = await _remoteFactory.CreateRemoteAsync(subscription.TargetRepository); + + var result = new CodeflowHistory + { + RepoCommits = [], + VmrCommits = [], + ForwardFlows = [], + Backflows = [], + }; + + return Ok(result); + } + private async Task EnqueueUpdateSubscriptionWorkItemAsync(Guid subscriptionId, int buildId, bool force = false) { Maestro.Data.Models.Subscription? subscriptionToUpdate; diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2019_01_16/Controllers/SubscriptionsController.cs b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2019_01_16/Controllers/SubscriptionsController.cs index 1435bcbbed..658b4e3912 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2019_01_16/Controllers/SubscriptionsController.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2019_01_16/Controllers/SubscriptionsController.cs @@ -93,6 +93,14 @@ public override async Task GetSubscription(Guid id) return Ok(new Subscription(subscription)); } + [HttpPost("{id}/codeflowhistory")] + [SwaggerApiResponse(HttpStatusCode.Accepted, Type = typeof(Subscription), Description = "Subscription update has been triggered")] + [ValidateModelState] + public override async Task GetCodeflowHistory(Guid id) + { + return await GetCodeflowHistoryCore(id); + } + /// /// Trigger a manually by id /// diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/SubscriptionsController.cs b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/SubscriptionsController.cs index 64a8bc1ca0..e993a236f9 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/SubscriptionsController.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/SubscriptionsController.cs @@ -146,6 +146,14 @@ public override async Task TriggerSubscription(Guid id, [FromQuer return await TriggerSubscriptionCore(id, buildId, force); } + [HttpPost("{id}/codeflowhistory")] + [SwaggerApiResponse(HttpStatusCode.Accepted, Type = typeof(Subscription), Description = "Subscription update has been triggered")] + [ValidateModelState] + public override async Task GetCodeflowHistory(Guid id) + { + return await GetCodeflowHistoryCore(id); + } + [ApiRemoved] public sealed override Task UpdateSubscription(Guid id, [FromBody] ProductConstructionService.Api.v2018_07_16.Models.SubscriptionUpdate update) { From 29ba1291bf72603a57438508b6e6e1a3133f8490 Mon Sep 17 00:00:00 2001 From: Adam Zippor Date: Thu, 18 Sep 2025 13:57:24 +0200 Subject: [PATCH 02/19] update codeflow history controller --- .../VirtualMonoRepo/CodeflowHistory.cs | 22 ------- .../Controllers/SubscriptionsController.cs | 64 +++++++++++++++---- .../Controllers/SubscriptionsController.cs | 6 +- .../Controllers/SubscriptionsController.cs | 6 +- .../Models/CodeflowHistoryResult.cs | 13 ++++ .../CodeflowHistory.cs | 16 +++++ .../CodeflowHistoryManager.cs | 39 +++++++++++ 7 files changed, 128 insertions(+), 38 deletions(-) delete mode 100644 src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/CodeflowHistory.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Api/Controllers/Models/CodeflowHistoryResult.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistory.cs create mode 100644 src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistoryManager.cs diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/CodeflowHistory.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/CodeflowHistory.cs deleted file mode 100644 index 501078bc97..0000000000 --- a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/CodeflowHistory.cs +++ /dev/null @@ -1,22 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Collections.Generic; - -namespace Microsoft.DotNet.DarcLib.VirtualMonoRepo; - -public class CodeflowHistory -{ - public string SorceRepoUrl { get; set; } - public string TargetRepoUrl { get; set; } - public List RepoCommits { get; set; } - public List VmrCommits { get; set; } - public List ForwardFlows {get; set; } - public List Backflows { get; set; } -} - -public class Codeflow -{ - public string SourceCommitSha { get; set; } - public string TargetCommitSha { get; set; } -} diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs index 4ed0344bde..8db1f4166b 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs @@ -8,14 +8,16 @@ using Microsoft.AspNetCore.ApiVersioning; using Microsoft.AspNetCore.ApiVersioning.Swashbuckle; using Microsoft.AspNetCore.Mvc; -using Microsoft.DotNet.DarcLib; -using Microsoft.DotNet.DarcLib.VirtualMonoRepo; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Options; +using ProductConstructionService.Common; +using ProductConstructionService.Api.Controllers.Models; using ProductConstructionService.Api.v2018_07_16.Models; +using Microsoft.DotNet.DarcLib; using ProductConstructionService.DependencyFlow.WorkItems; using ProductConstructionService.WorkItems; using Channel = Maestro.Data.Models.Channel; +using SubscriptionDAO = Maestro.Data.Models.Subscription; namespace ProductConstructionService.Api.Api.v2018_07_16.Controllers; @@ -29,17 +31,18 @@ public class SubscriptionsController : ControllerBase private readonly BuildAssetRegistryContext _context; private readonly IWorkItemProducerFactory _workItemProducerFactory; private readonly IGitHubInstallationIdResolver _installationIdResolver; + private readonly ICodeflowHistoryManager _codeflowHistoryManager; private readonly IRemoteFactory _remoteFactory; private readonly ILogger _logger; protected readonly IOptions _environmentNamespaceOptions; - public SubscriptionsController( BuildAssetRegistryContext context, IWorkItemProducerFactory workItemProducerFactory, IGitHubInstallationIdResolver installationIdResolver, IOptions environmentNamespaceOptions, IRemoteFactory remoteFactory, + ICodeflowHistoryManager codeflowHistoryManager, ILogger logger) { _context = context; @@ -47,6 +50,7 @@ public SubscriptionsController( _installationIdResolver = installationIdResolver; _environmentNamespaceOptions = environmentNamespaceOptions; _remoteFactory = remoteFactory; + _codeflowHistoryManager = codeflowHistoryManager; _logger = logger; } @@ -171,32 +175,64 @@ protected async Task TriggerSubscriptionCore(Guid id, int buildId protected async Task GetCodeflowHistoryCore(Guid id) { - Maestro.Data.Models.Subscription? subscription = await _context.Subscriptions.Include(sub => sub.LastAppliedBuild) - .Include(sub => sub.Channel) + var subscription = await _context.Subscriptions .Include(sub => sub.LastAppliedBuild) - .Include(sub => sub.ExcludedAssets) .FirstOrDefaultAsync(sub => sub.Id == id); - if (subscription == null) + if (subscription == null || !subscription.SourceEnabled) { return NotFound(); } - IRemote sourceRemote = await _remoteFactory.CreateRemoteAsync(subscription.SourceRepository); + var oppositeDirectionSubscription = await _context.Subscriptions + .Include(sub => sub.LastAppliedBuild) + .Include(sub => sub.Channel) + .Where(sub => + sub.SourceRepository == subscription.TargetRepository || + sub.TargetRepository == subscription.SourceRepository) + .FirstOrDefaultAsync(); - IRemote targetRemote = await _remoteFactory.CreateRemoteAsync(subscription.TargetRepository); + if (oppositeDirectionSubscription?.SourceEnabled != true) + { + oppositeDirectionSubscription = null; + } + + bool isForwardFlow = !string.IsNullOrEmpty(subscription.TargetDirectory); + + var cachedFlows = await _codeflowHistoryManager.GetCachedCodeflowHistory(id); + + var oppositeCachedFlows = oppositeDirectionSubscription != null + ? await _codeflowHistoryManager.GetCachedCodeflowHistory(oppositeDirectionSubscription.Id) + : null; + + var lastCommit = subscription.LastAppliedBuild.Commit; - var result = new CodeflowHistory + bool resultIsOutdated = IsCodeflowHistoryOutdated(subscription, cachedFlows) || + IsCodeflowHistoryOutdated(oppositeDirectionSubscription, oppositeCachedFlows); + + var result = new CodeflowHistoryResult { - RepoCommits = [], - VmrCommits = [], - ForwardFlows = [], - Backflows = [], + ResultIsOutdated = resultIsOutdated, + ForwardFlowHistory = isForwardFlow + ? cachedFlows + : oppositeCachedFlows, + BackflowHistory = isForwardFlow + ? oppositeCachedFlows + : cachedFlows, }; return Ok(result); } + private static bool IsCodeflowHistoryOutdated( + SubscriptionDAO? subscription, + CodeflowHistory? cachedFlows) + { + string? lastCachedCodeflow = cachedFlows?.Codeflows.LastOrDefault()?.SourceCommitSha; + string? lastAppliedCommit = subscription?.LastAppliedBuild?.Commit; + return !string.Equals(lastCachedCodeflow, lastAppliedCommit, StringComparison.Ordinal); + } + private async Task EnqueueUpdateSubscriptionWorkItemAsync(Guid subscriptionId, int buildId, bool force = false) { Maestro.Data.Models.Subscription? subscriptionToUpdate; diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2019_01_16/Controllers/SubscriptionsController.cs b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2019_01_16/Controllers/SubscriptionsController.cs index 658b4e3912..447d8cf0f6 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2019_01_16/Controllers/SubscriptionsController.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2019_01_16/Controllers/SubscriptionsController.cs @@ -7,9 +7,11 @@ using Microsoft.AspNetCore.ApiVersioning; using Microsoft.AspNetCore.ApiVersioning.Swashbuckle; using Microsoft.AspNetCore.Mvc; +using Microsoft.DotNet.DarcLib; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Options; using ProductConstructionService.Api.v2019_01_16.Models; +using ProductConstructionService.Common; using ProductConstructionService.WorkItems; namespace ProductConstructionService.Api.Api.v2019_01_16.Controllers; @@ -27,9 +29,11 @@ public SubscriptionsController( BuildAssetRegistryContext context, IWorkItemProducerFactory workItemProducerFactory, IGitHubInstallationIdResolver gitHubInstallationRetriever, + IRemoteFactory remoteFactory, + ICodeflowHistoryManager codeflowHistoryManager, IOptions environmentNamespaceOptions, ILogger logger) - : base(context, workItemProducerFactory, gitHubInstallationRetriever, environmentNamespaceOptions, logger) + : base(context, workItemProducerFactory, gitHubInstallationRetriever, environmentNamespaceOptions, remoteFactory, codeflowHistoryManager, logger) { _context = context; } diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/SubscriptionsController.cs b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/SubscriptionsController.cs index e993a236f9..26da258832 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/SubscriptionsController.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/SubscriptionsController.cs @@ -7,10 +7,12 @@ using Microsoft.AspNetCore.ApiVersioning; using Microsoft.AspNetCore.ApiVersioning.Swashbuckle; using Microsoft.AspNetCore.Mvc; +using Microsoft.DotNet.DarcLib; using Microsoft.DotNet.GitHub.Authentication; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Options; using ProductConstructionService.Api.v2020_02_20.Models; +using ProductConstructionService.Common; using ProductConstructionService.WorkItems; namespace ProductConstructionService.Api.Api.v2020_02_20.Controllers; @@ -33,8 +35,10 @@ public SubscriptionsController( IGitHubInstallationIdResolver gitHubInstallationRetriever, IWorkItemProducerFactory workItemProducerFactory, IOptions environmentNamespaceOptions, + IRemoteFactory remoteFactory, + ICodeflowHistoryManager codeflowHistoryManager, ILogger logger) - : base(context, workItemProducerFactory, gitHubInstallationRetriever, environmentNamespaceOptions, logger) + : base(context, workItemProducerFactory, gitHubInstallationRetriever, remoteFactory, codeflowHistoryManager, environmentNamespaceOptions, logger) { _context = context; _gitHubClientFactory = gitHubClientFactory; diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Controllers/Models/CodeflowHistoryResult.cs b/src/ProductConstructionService/ProductConstructionService.Api/Controllers/Models/CodeflowHistoryResult.cs new file mode 100644 index 0000000000..91a08b8422 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Api/Controllers/Models/CodeflowHistoryResult.cs @@ -0,0 +1,13 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using ProductConstructionService.Common; + +namespace ProductConstructionService.Api.Controllers.Models; + +public class CodeflowHistoryResult +{ + public CodeflowHistory? ForwardFlowHistory { get; set; } + public CodeflowHistory? BackflowHistory { get; set; } + public bool ResultIsOutdated { get; set; } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistory.cs b/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistory.cs new file mode 100644 index 0000000000..4a12487238 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistory.cs @@ -0,0 +1,16 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.DotNet.DarcLib; + +namespace ProductConstructionService.Common; + +public record CodeflowHistory( + List Commits, + List Codeflows); + + +public record CodeflowRecord( + string SourceCommitSha, + string TargetCommitSha, + DateTimeOffset CodeflowMergeDate); diff --git a/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistoryManager.cs b/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistoryManager.cs new file mode 100644 index 0000000000..4f9ce0e476 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistoryManager.cs @@ -0,0 +1,39 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.DotNet.DarcLib.VirtualMonoRepo; + +namespace ProductConstructionService.Common; + +public interface ICodeflowHistoryManager +{ + void RefreshCodeflowHistory(string repo, string branch); + Task GetCachedCodeflowHistory(Guid subscriptionId); +} + +public class CodeflowHistoryManager : ICodeflowHistoryManager +{ + private readonly IRedisCacheFactory _cacheFactory; + + public CodeflowHistoryManager(IRedisCacheFactory cacheFactory) + { + _cacheFactory = cacheFactory; + } + + public async void RefreshCodeflowHistory(string repo, string branch) + { + //0. Fetch latest commit from local git repo if exists + //1. Fetch new parts from GitHub API + //2. Fetch old parts from disk + //3. Stitch them together + //4. Write contents to cache + await Task.CompletedTask; + } + + public async Task GetCachedCodeflowHistory(Guid subscriptionId) + { + var cache = _cacheFactory.Create(subscriptionId.ToString()); + var cachedHistory = await cache.TryGetStateAsync(); + return cachedHistory; + } +} From e208448bf3be3a8f2e4e0523fa9d9a35cfba77aa Mon Sep 17 00:00:00 2001 From: Adam Zippor Date: Thu, 18 Sep 2025 15:19:00 +0200 Subject: [PATCH 03/19] Rename Maestro.Data.Models.Subscription references --- .../Controllers/SubscriptionsController.cs | 22 +- .../CodeflowHistory/CodeflowHistoryManager.cs | 209 ++++++++++++++++++ 2 files changed, 220 insertions(+), 11 deletions(-) create mode 100644 src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistory/CodeflowHistoryManager.cs diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs index 8db1f4166b..688d259304 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs @@ -66,7 +66,7 @@ public virtual IActionResult ListSubscriptions( int? channelId = null, bool? enabled = null) { - IQueryable query = _context.Subscriptions.Include(s => s.Channel); + IQueryable query = _context.Subscriptions.Include(s => s.Channel); if (!string.IsNullOrEmpty(sourceRepository)) { @@ -101,7 +101,7 @@ public virtual IActionResult ListSubscriptions( [ValidateModelState] public virtual async Task GetSubscription(Guid id) { - Maestro.Data.Models.Subscription? subscription = await _context.Subscriptions.Include(sub => sub.LastAppliedBuild) + SubscriptionDAO? subscription = await _context.Subscriptions.Include(sub => sub.LastAppliedBuild) .Include(sub => sub.Channel) .FirstOrDefaultAsync(sub => sub.Id == id); @@ -137,7 +137,7 @@ public virtual async Task TriggerSubscription(Guid id, [FromQuery protected async Task TriggerSubscriptionCore(Guid id, int buildId, bool force = false) { - Maestro.Data.Models.Subscription? subscription = await _context.Subscriptions + SubscriptionDAO? subscription = await _context.Subscriptions .Include(sub => sub.LastAppliedBuild) .Include(sub => sub.Channel) .FirstOrDefaultAsync(sub => sub.Id == id); @@ -235,7 +235,7 @@ private static bool IsCodeflowHistoryOutdated( private async Task EnqueueUpdateSubscriptionWorkItemAsync(Guid subscriptionId, int buildId, bool force = false) { - Maestro.Data.Models.Subscription? subscriptionToUpdate; + SubscriptionDAO? subscriptionToUpdate; if (buildId != 0) { // Update using a specific build @@ -310,7 +310,7 @@ public virtual async Task TriggerDailyUpdateAsync() foreach (var subscription in enabledSubscriptionsWithTargetFrequency) { - Maestro.Data.Models.Subscription? subscriptionWithBuilds = await _context.Subscriptions + SubscriptionDAO? subscriptionWithBuilds = await _context.Subscriptions .Where(s => s.Id == subscription.Id) .Include(s => s.Channel) .ThenInclude(c => c.BuildChannels) @@ -355,7 +355,7 @@ await workitemProducer.ProduceWorkItemAsync(new() [ValidateModelState] public virtual async Task UpdateSubscription(Guid id, [FromBody] SubscriptionUpdate update) { - Maestro.Data.Models.Subscription? subscription = await _context.Subscriptions.Where(sub => sub.Id == id) + SubscriptionDAO? subscription = await _context.Subscriptions.Where(sub => sub.Id == id) .FirstOrDefaultAsync(); if (subscription == null) @@ -401,7 +401,7 @@ public virtual async Task UpdateSubscription(Guid id, [FromBody] if (doUpdate) { - Maestro.Data.Models.Subscription? equivalentSubscription = await FindEquivalentSubscription(subscription); + SubscriptionDAO? equivalentSubscription = await FindEquivalentSubscription(subscription); if (equivalentSubscription != null) { return BadRequest( @@ -429,7 +429,7 @@ public virtual async Task UpdateSubscription(Guid id, [FromBody] [ValidateModelState] public virtual async Task DeleteSubscription(Guid id) { - Maestro.Data.Models.Subscription? subscription = + SubscriptionDAO? subscription = await _context.Subscriptions.FirstOrDefaultAsync(sub => sub.Id == id); if (subscription == null) @@ -460,7 +460,7 @@ public virtual async Task DeleteSubscription(Guid id) [Paginated(typeof(SubscriptionHistoryItem))] public virtual async Task GetSubscriptionHistory(Guid id) { - Maestro.Data.Models.Subscription? subscription = await _context.Subscriptions.Where(sub => sub.Id == id) + SubscriptionDAO? subscription = await _context.Subscriptions.Where(sub => sub.Id == id) .FirstOrDefaultAsync(); if (subscription == null) @@ -537,7 +537,7 @@ public virtual async Task Create([FromBody, Required] Subscriptio subscriptionModel.Id = Guid.NewGuid(); subscriptionModel.Namespace = defaultNamespace; - Maestro.Data.Models.Subscription? equivalentSubscription = await FindEquivalentSubscription(subscriptionModel); + SubscriptionDAO? equivalentSubscription = await FindEquivalentSubscription(subscriptionModel); if (equivalentSubscription != null) { return Conflict( @@ -624,7 +624,7 @@ protected async Task EnsureRepositoryRegistration(string repoUri) /// /// Subscription model with updated data. /// Subscription if it is found, null otherwise - private async Task FindEquivalentSubscription(Maestro.Data.Models.Subscription updatedOrNewSubscription) + private async Task FindEquivalentSubscription(SubscriptionDAO updatedOrNewSubscription) { // Compare subscriptions based on the 4 key elements: // - Channel diff --git a/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistory/CodeflowHistoryManager.cs b/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistory/CodeflowHistoryManager.cs new file mode 100644 index 0000000000..3a2ce3e63f --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistory/CodeflowHistoryManager.cs @@ -0,0 +1,209 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text.Json; +using Maestro.Data.Models; +using Microsoft.DotNet.DarcLib; +using Microsoft.DotNet.DarcLib.Models.VirtualMonoRepo; +using Pipelines.Sockets.Unofficial.Arenas; +using StackExchange.Redis; + +namespace ProductConstructionService.Common.CodeflowHistory; + +public interface ICodeflowHistoryManager +{ + Task> GetCachedCodeflowHistoryAsync( + string subscriptionId, + int commitFetchCount = 100); + + Task> FetchLatestCodeflowHistoryAsync( + Subscription subscription, + int commitFetchCount = 100); +} + +public record CodeflowGraphCommit( + string CommitSha, + string Author, + string Description, + string? SourceRepoFlowSha, + int? redisScore); + +public class CodeflowHistoryManager( + IRemoteFactory remoteFactory, + IConnectionMultiplexer connection) : ICodeflowHistoryManager +{ + private readonly IRemoteFactory _remoteFactory = remoteFactory; + private readonly IConnectionMultiplexer _connection = connection; + + private static RedisKey GetCodeflowGraphCommitKey(string id) => $"CodeflowGraphCommit_{id}"; + private static RedisKey GetSortedSetKey(string id) => $"CodeflowHistory_{id}"; + + private const int MaxCommitFetchCount = 500; + private const int MaxCommitsCached = 3000; + + public async Task> GetCachedCodeflowHistoryAsync( + string subscriptionId, + int commitFetchCount = 100) + { + if (commitFetchCount < 1) + { + throw new ArgumentOutOfRangeException(nameof(commitFetchCount)); + } + + var cache = _connection.GetDatabase(); + + var res = await cache.SortedSetRangeByRankWithScoresAsync( + key: subscriptionId, + start: 0, + stop: commitFetchCount - 1, + order: Order.Descending); + + var commitKeys = res + .Select(e => new RedisKey(e.Element.ToString())) + .ToArray(); + + var commitValues = await cache.StringGetAsync(commitKeys); + + if (commitValues.Any(val => !val.HasValue)) + throw new InvalidOperationException($"Corrupted commit data encountered for subscription `{subscriptionId}`."); + + return [.. commitValues + .Select(commit => JsonSerializer.Deserialize(commit.ToString())) + .OfType()]; + } + + public async Task> FetchLatestCodeflowHistoryAsync( + Subscription subscription, + int commitFetchCount = 100) + { + var cachedCommits = await GetCachedCodeflowHistoryAsync(subscription.Id.ToString()); + + var latestCachedCommit = cachedCommits.FirstOrDefault(); + + var newCommits = await FetchNewCommits( + subscription.TargetRepository, + subscription.TargetBranch, + latestCachedCommit?.CommitSha); + + if (newCommits.LastOrDefault()?.CommitSha == latestCachedCommit?.CommitSha) + { + newCommits.RemoveLast(); + } + else + { + // the fetched commits do not connect to the cached commits + // we don't know how many commits are missing, so clear the cache and start over + await ClearCodeflowCacheAsync(subscription.Id.ToString()); + } + + var graphCommits = await EnrichCommitsWithCodeflowDataAsync( + newCommits, + subscription); + + await CacheCommitsAsync(subscription.Id.ToString(), graphCommits); + + return [.. graphCommits + .Concat(cachedCommits) + .Take(commitFetchCount)]; + } + + private async Task> EnrichCommitsWithCodeflowDataAsync( + LinkedList commits, + Subscription subscription) + { + var remote = await _remoteFactory.CreateRemoteAsync(subscription.TargetRepository); + + var current = commits.First; + + while (current != null) + { + Codeflow? lastFlow = !string.IsNullOrEmpty(subscription.TargetDirectory) + ? await remote.GetLastIncomingForwardFlowAsync( + subscription.TargetBranch, + subscription.TargetDirectory, + current.Value.CommitSha) + : await remote.GetLastIncomingBackflowAsync( + subscription.TargetBranch, + current.Value.CommitSha); + + var target = current; + + while (target != null && target.Value.CommitSha != lastFlow?.TargetSha) + target = target.Next; + + if (target == null) + break; + + target.Value = target.Value with + { + SourceRepoFlowSha = lastFlow?.SourceSha, + }; + + current = target.Next; + } + return commits; + } + + private async Task CacheCommitsAsync( + string subscriptionId, + IEnumerable commits, + int latestCachedCommitScore = 0) + { + if (!commits.Any()) + { + return; + } + + var cache = _connection.GetDatabase(); + + var sortedSetEntries = commits + .Select(c => new SortedSetEntry(c.CommitSha, latestCachedCommitScore++)) + .ToArray(); + + await cache.SortedSetAddAsync(subscriptionId, sortedSetEntries); + + var commitGraphEntries = commits + .Select(c => new KeyValuePair( + GetCodeflowGraphCommitKey(c.CommitSha), + JsonSerializer.Serialize(c))) + .ToArray(); + + await cache.StringSetAsync(commitGraphEntries); + + if (latestCachedCommitScore > MaxCommitsCached) + { + await cache.SortedSetRemoveRangeByScoreAsync( + key: subscriptionId, + start: 0, + stop: latestCachedCommitScore - MaxCommitsCached); + } + } + + private async Task ClearCodeflowCacheAsync(string subscriptionId) + { + var cache = _connection.GetDatabase(); + await cache.KeyDeleteAsync(GetSortedSetKey(subscriptionId)); + } + + private async Task> FetchNewCommits( + string targetRepository, + string targetBranch, + string? latestCachedCommitSha) + { + var remote = await _remoteFactory.CreateRemoteAsync(targetRepository); + + var newCommits = await remote.FetchNewerRepoCommitsAsync( + targetRepository, + targetBranch, + latestCachedCommitSha, + MaxCommitFetchCount); + + return new LinkedList( + newCommits.Select(commit => new CodeflowGraphCommit( + CommitSha: commit.Sha, + Author: commit.Author, + Description: commit.Message, + SourceRepoFlowSha: null, + redisScore: null))); + } +} From 46702951e486c9bd9f6eb6106c720b8e37e81c73 Mon Sep 17 00:00:00 2001 From: Adam Zippor Date: Mon, 22 Sep 2025 14:12:56 +0200 Subject: [PATCH 04/19] Codeflow graphs WIP --- .../DarcLib/AzureDevOpsClient.cs | 11 + .../DarcLib/GitHubClient.cs | 193 ++++++++++++++++++ .../DarcLib/Helpers/VersionDetailsParser.cs | 9 +- src/Microsoft.DotNet.Darc/DarcLib/IRemote.cs | 28 ++- .../DarcLib/IRemoteGitRepo.cs | 21 ++ src/Microsoft.DotNet.Darc/DarcLib/Remote.cs | 24 +++ .../DarcLib/VirtualMonoRepo/VmrCodeflower.cs | 13 +- .../Controllers/SubscriptionsController.cs | 13 +- .../CodeflowHistory.cs | 10 - .../CodeflowHistoryManager.cs | 90 +++++++- .../CodeflowHistoryManager.cs | 78 +++++++ 11 files changed, 457 insertions(+), 33 deletions(-) create mode 100644 src/ProductConstructionService/ProductConstructionService.DependencyFlow/CodeflowHistoryManager.cs diff --git a/src/Microsoft.DotNet.Darc/DarcLib/AzureDevOpsClient.cs b/src/Microsoft.DotNet.Darc/DarcLib/AzureDevOpsClient.cs index 181b3bbeea..cc7ac932b1 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/AzureDevOpsClient.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/AzureDevOpsClient.cs @@ -1944,4 +1944,15 @@ public async Task> GetGitTreeNames(string path, stri UpdatedAt = DateTimeOffset.UtcNow, HeadBranchSha = pr.LastMergeSourceCommit.CommitId, }; + + public Task> FetchLatestRepoCommits(string repoUrl, string branch, int maxCount) + => throw new NotImplementedException(); + + public Task> FetchLatestFetchNewerRepoCommitsAsyncRepoCommits( + string repoUrl, + string branch, + string commitSha, + int maxCount) + => throw new NotImplementedException(); +>>>>>>> 0973d0ff2 (Codeflow graphs WIP) } diff --git a/src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs b/src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs index 7337d9a870..3117beb289 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Collections; using System.Collections.Generic; using System.Collections.Immutable; using System.IO; @@ -10,20 +11,26 @@ using System.Net.Http; using System.Reflection; using System.Text; +using System.Text.Json; using System.Text.RegularExpressions; using System.Threading.Tasks; +using LibGit2Sharp; using Maestro.Common; using Maestro.MergePolicyEvaluation; using Microsoft.DotNet.DarcLib.Helpers; using Microsoft.DotNet.DarcLib.Models; using Microsoft.DotNet.DarcLib.Models.GitHub; +using Microsoft.DotNet.DarcLib.Models.VirtualMonoRepo; +using Microsoft.DotNet.DarcLib.VirtualMonoRepo; using Microsoft.DotNet.Services.Utility; using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; +using Microsoft.TeamFoundation.TestManagement.WebApi; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using Newtonsoft.Json.Serialization; using Octokit; +using static System.Net.WebRequestMethods; #nullable enable namespace Microsoft.DotNet.DarcLib; @@ -1490,5 +1497,191 @@ private static PullRequest ToDarcLibPullRequest(Octokit.PullRequest pr) UpdatedAt = pr.UpdatedAt, HeadBranchSha = pr.Head.Sha, }; + public Task> FetchLatestRepoCommitsAsync(string repoUrl, string branch, int maxCount) + => throw new NotImplementedException(); + + public async Task> FetchNewerRepoCommitsAsync( + string repoUrl, + string branch, + string commitSha, + int maxCount) + { + if (maxCount <= 0) + { + maxCount = 100; + } + + (string owner, string repo) = ParseRepoUri(repoUrl); + + var request = new CommitRequest + { + Sha = branch ?? "main", + }; + + var options = new ApiOptions + { + PageSize = maxCount, + PageCount = 1, + StartPage = 1 + }; + + var allCommits = new List(); + + while (allCommits.Count < maxCount) + { + var commits = await GetClient(owner, repo) + .Repository + .Commit + .GetAll(owner, repo, request, options); + + foreach (Octokit.GitHubCommit c in commits) + { + var convertedCommit = new Commit( + c.Author?.Login, + c.Commit.Sha, + c.Commit.Message); + + allCommits.Add(convertedCommit); + if (convertedCommit.Sha.Equals(commitSha)) + { + break; + } + } + + if (commits.Count < options.PageSize) + { + break; + } + + options.StartPage++; + } + + return [.. allCommits.Take(maxCount)]; + } + + public async Task GetLastIncomingForwardFlow(string vmrUrl, string mappingName, string commit) + { + var content = await GetFileContentAtCommit( + vmrUrl, + commit, + VmrInfo.DefaultRelativeSourceManifestPath); + + var lastForwardFlowRepoSha = SourceManifest + .FromJson(content)? + .GetRepoVersion(mappingName) + .CommitSha; + + if (lastForwardFlowRepoSha == null) + { + return null; + } + + int lineNumber = content.Split(Environment.NewLine) + .ToList() + .FindIndex(line => line.Contains(lastForwardFlowRepoSha)); + + + string lastForwardFlowVmrSha = await BlameLineAsync( + vmrUrl, + commit, + VmrInfo.DefaultRelativeSourceManifestPath, + lineNumber); + + return new ForwardFlow(lastForwardFlowRepoSha, lastForwardFlowVmrSha); + } + + public async Task GetLastIncomingBackflow(string repoUrl, string commit) + { + var content = await GetFileContentAtCommit( + repoUrl, + commit, + VersionFiles.VersionDetailsXml); + + var lastBackflowVmrSha = VersionDetailsParser + .ParseVersionDetailsXml(content)? + .Source? + .Sha; + + if (lastBackflowVmrSha == null) + { + return null; + } + + int lineNumber = content + .Split(Environment.NewLine) + .ToList() + .FindIndex(line => + line.Contains(VersionDetailsParser.SourceElementName) && + line.Contains(lastBackflowVmrSha)); + + string lastBackflowRepoSha = await BlameLineAsync( + repoUrl, + commit, + VersionFiles.VersionDetailsXml, + lineNumber); + + return new Backflow(lastBackflowVmrSha, lastBackflowRepoSha); + } + + private async Task BlameLineAsync(string repoUrl, string commitOrBranch, string filePath, int lineNumber) + { + (string owner, string repo) = ParseRepoUri(repoUrl); + + string query = $@""" + {{ + repository(owner: {owner}, name: {repo}) {{ + object(expression: ""{commitOrBranch}:{filePath}"") {{ + ... on Blob {{ + blame {{ + ranges {{ + startingLine + endingLine + commit {{ oid }} + }} + }} + }} + }} + }} + }}"""; + + var client = CreateHttpClient(repoUrl); + + var requestBody = new { query }; + var content = new StringContent(JsonConvert.SerializeObject(requestBody, _serializerSettings)); + + var response = await client.PostAsync("", content); + response.EnsureSuccessStatusCode(); + + using var stream = await response.Content.ReadAsStreamAsync(); + using var doc = await JsonDocument.ParseAsync(stream); + + var ranges = doc.RootElement + .GetProperty("data") + .GetProperty("repository") + .GetProperty("object") + .GetProperty("blame") + .GetProperty("ranges") + .EnumerateArray(); + + foreach (var range in ranges) + { + int start = range.GetProperty("startingLine").GetInt32(); + int end = range.GetProperty("endingLine").GetInt32(); + + if (lineNumber >= start && lineNumber <= end) + { + return range.GetProperty("commit").GetProperty("oid").GetString()!; + } + } + + throw new InvalidOperationException($"Line {lineNumber} not found in blame data."); + } + + private async Task GetFileContentAtCommit(string repoUrl, string commit, string filePath) + { + (string owner, string repo) = ParseRepoUri(repoUrl); + var file = await GetClient(repoUrl).Repository.Content.GetAllContentsByRef(owner, repo, filePath, commit); + return Encoding.UTF8.GetString(Convert.FromBase64String(file[0].Content)); +>>>>>>> 0973d0ff2 (Codeflow graphs WIP) } } diff --git a/src/Microsoft.DotNet.Darc/DarcLib/Helpers/VersionDetailsParser.cs b/src/Microsoft.DotNet.Darc/DarcLib/Helpers/VersionDetailsParser.cs index 1438d6a98d..8997ccad0b 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/Helpers/VersionDetailsParser.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/Helpers/VersionDetailsParser.cs @@ -5,9 +5,14 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Threading.Tasks; using System.Xml; +using LibGit2Sharp; using Microsoft.DotNet.DarcLib.Models; using Microsoft.DotNet.DarcLib.Models.Darc; +using Microsoft.DotNet.DarcLib.Models.VirtualMonoRepo; +using Microsoft.TeamFoundation.Work.WebApi; +using static System.Net.Mime.MediaTypeNames; #nullable enable namespace Microsoft.DotNet.DarcLib.Helpers; @@ -56,13 +61,13 @@ public VersionDetails ParseVersionDetailsFile(string path, bool includePinned = return ParseVersionDetailsXml(content, includePinned: includePinned); } - public VersionDetails ParseVersionDetailsXml(string fileContents, bool includePinned = true) + public static VersionDetails ParseVersionDetailsXml(string fileContents, bool includePinned = true) { XmlDocument document = GetXmlDocument(fileContents); return ParseVersionDetailsXml(document, includePinned: includePinned); } - public VersionDetails ParseVersionDetailsXml(XmlDocument document, bool includePinned = true) + public static VersionDetails ParseVersionDetailsXml(XmlDocument document, bool includePinned = true) { XmlNodeList? dependencyNodes = (document?.DocumentElement?.SelectNodes($"//{DependencyElementName}")) ?? throw new Exception($"There was an error while reading '{VersionFiles.VersionDetailsXml}' and it came back empty. " + diff --git a/src/Microsoft.DotNet.Darc/DarcLib/IRemote.cs b/src/Microsoft.DotNet.Darc/DarcLib/IRemote.cs index ef94240fa3..00c87cb0b1 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/IRemote.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/IRemote.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System; using System.Collections.Generic; using System.Threading.Tasks; using Maestro.MergePolicyEvaluation; @@ -260,5 +261,30 @@ Task CommitUpdatesWithNoCloningAsync( /// Task> GetGitTreeNames(string path, string repoUri, string branch); - #endregion + /// + /// Fetches the latest commits from a repository branch, up to maxCount + /// + /// Full url of the git repository + /// Name of the git branch in the repo + /// Maximum count of commits to fetch + /// List of commits + Task> FetchLatestRepoCommitsAsync(string repoUrl, string branch, int maxCount = 100); + + + /// + /// Fetches the latest commits from a repository branch that are newer than the specified + /// commit, fetching up to maxCount commits in total. + /// + /// Full url of the git repository + /// Name of the git branch in the repo + /// Sha of the commit to fetch newer commits than + /// Maximum count of commits to fetch + /// List of commits + Task> FetchNewerRepoCommitsAsync(string repoUrl, string branch, string commitSha, int maxCount = 100); + + + Task> GetLastIncomingForwardflow(string vmrUrl, string commit); + Task> GetLastIncomingBackflow(string repoUrl, string commit); + + #endregion } diff --git a/src/Microsoft.DotNet.Darc/DarcLib/IRemoteGitRepo.cs b/src/Microsoft.DotNet.Darc/DarcLib/IRemoteGitRepo.cs index 5d5eb6c976..e76476de10 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/IRemoteGitRepo.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/IRemoteGitRepo.cs @@ -167,6 +167,27 @@ Task> SearchPullRequestsAsync( /// Returns a list of tree names (directories) under a given path in a given branch /// Task> GetGitTreeNames(string path, string repoUri, string branch); + + /// + /// Returns the latest commits for a given repository and branch, up to maxCount + /// + Task> FetchLatestRepoCommitsAsync(string repoUrl, string branch, int maxCount); + + /// + /// Returns the latest commits for a given repository and branch that are newer than + /// the given commit, fetching up to maxCount commits in total. + /// + Task> FetchNewerRepoCommitsAsync(string repoUrl, string branch, string commitSha, int maxCount); + + /// + /// Get the last forward flow that was merged onto the given VMR at the specified commit + /// + Task?> GetLastIncomingForwardFlow(string vmrUrl, string branch, string commit); + + /// + /// Get the last back flow that was merged onto the given repo at the specified commit + /// + Task?> GetLastIncomingBackflow(string repoUrl, string commit); } #nullable disable diff --git a/src/Microsoft.DotNet.Darc/DarcLib/Remote.cs b/src/Microsoft.DotNet.Darc/DarcLib/Remote.cs index 4810d40d26..918319cb00 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/Remote.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/Remote.cs @@ -501,4 +501,28 @@ public async Task> GetGitTreeNames(string path, stri { return await _remoteGitClient.GetGitTreeNames(path, repoUri, branch); } + + public Task> FetchLatestRepoCommitsAsync(string repoUrl, string branch, int maxCount = 100) + { + return _remoteGitClient.FetchLatestRepoCommitsAsync(repoUrl, branch, maxCount); + } + public Task> FetchNewerRepoCommitsAsync( + string repoUrl, + string branch, + string commitSha, + int maxCount = 100) + { + return _remoteGitClient.FetchNewerRepoCommitsAsync(repoUrl, branch, commitSha, maxCount); + } + + public async Task> GetLastIncomingForwardflow(string vmrUrl, string commit) + { + await Task.CompletedTask; + return null; + } + public async Task> GetLastIncomingBackflow(string repoUrl, string commit) + { + await Task.CompletedTask; + return null; + } } diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrCodeflower.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrCodeflower.cs index 24c45d1ed9..a25c9c5bff 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrCodeflower.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrCodeflower.cs @@ -508,19 +508,16 @@ await HandleRevertedFiles( /// /// Finds the last backflow between a repo and a VMR. /// - private async Task GetLastBackflow(NativePath repoPath) + private async Task GetLastBackflow(string repoPath) { - // Last backflow SHA comes from Version.Details.xml in the repo - SourceDependency? source = _versionDetailsParser.ParseVersionDetailsFile(repoPath / VersionFiles.VersionDetailsXml).Source; - if (source is null) + var versionDetailsContent = await _localGitClient.GetFileFromGitAsync(repoPath, VersionFiles.VersionDetailsXml); + if (versionDetailsContent == null) { return null; } - string lastBackflowVmrSha = source.Sha; - string lastBackflowRepoSha = await _localGitClient.BlameLineAsync( - repoPath / VersionFiles.VersionDetailsXml, - line => line.Contains(VersionDetailsParser.SourceElementName) && line.Contains(lastBackflowVmrSha)); + var lineNumber = VersionDetailsParser.SourceDependencyLineNumber(versionDetailsContent); + string lastBackflowRepoSha = await _localGitClient.BlameLineAsync(repoPath, VersionFiles.VersionDetailsXml, lineNumber); return new Backflow(lastBackflowVmrSha, lastBackflowRepoSha); } diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs index 688d259304..3a3c91699a 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs @@ -173,7 +173,7 @@ protected async Task TriggerSubscriptionCore(Guid id, int buildId return Accepted(new Subscription(subscription)); } - protected async Task GetCodeflowHistoryCore(Guid id) + protected async Task GetCodeflowHistoryCore(Guid id, bool fetchNewChanges) { var subscription = await _context.Subscriptions .Include(sub => sub.LastAppliedBuild) @@ -199,11 +199,14 @@ protected async Task GetCodeflowHistoryCore(Guid id) bool isForwardFlow = !string.IsNullOrEmpty(subscription.TargetDirectory); - var cachedFlows = await _codeflowHistoryManager.GetCachedCodeflowHistory(id); + CodeflowHistory? cachedFlows; + CodeflowHistory? oppositeCachedFlows; + + cachedFlows = await _codeflowHistoryManager.FetchLatestCodeflowHistoryAsync(id); + + oppositeCachedFlows = await _codeflowHistoryManager.FetchLatestCodeflowHistoryAsync( + oppositeDirectionSubscription?.Id); - var oppositeCachedFlows = oppositeDirectionSubscription != null - ? await _codeflowHistoryManager.GetCachedCodeflowHistory(oppositeDirectionSubscription.Id) - : null; var lastCommit = subscription.LastAppliedBuild.Commit; diff --git a/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistory.cs b/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistory.cs index 4a12487238..50609f7cf5 100644 --- a/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistory.cs +++ b/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistory.cs @@ -4,13 +4,3 @@ using Microsoft.DotNet.DarcLib; namespace ProductConstructionService.Common; - -public record CodeflowHistory( - List Commits, - List Codeflows); - - -public record CodeflowRecord( - string SourceCommitSha, - string TargetCommitSha, - DateTimeOffset CodeflowMergeDate); diff --git a/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistoryManager.cs b/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistoryManager.cs index 4f9ce0e476..c729b649df 100644 --- a/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistoryManager.cs +++ b/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistoryManager.cs @@ -1,23 +1,50 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using Microsoft.DotNet.DarcLib.VirtualMonoRepo; +using Microsoft.DotNet.DarcLib; +using Maestro.Data.Models; namespace ProductConstructionService.Common; public interface ICodeflowHistoryManager { void RefreshCodeflowHistory(string repo, string branch); - Task GetCachedCodeflowHistory(Guid subscriptionId); + Task GetCodeflowHistory(Guid? subscriptionId); + Task FetchLatestCodeflowHistoryAsync(Guid? subscriptionId); } +public record CodeflowHistory( + List Commits, + List Codeflows); + +public record CodeflowHistoryResult( + CodeflowHistory? ForwardFlowHistory, + CodeflowHistory? BackflowHistory, + bool ResultIsOutdated); + +public record CodeflowRecord( + string SourceCommitSha, + string TargetCommitSha, + DateTimeOffset CodeflowMergeDate); + +public record CodeflowGraphCommit( + string CommitSha, + DateTimeOffset CommitDate, + string Author, + string Description, + List IncomingCodeflows); + public class CodeflowHistoryManager : ICodeflowHistoryManager { - private readonly IRedisCacheFactory _cacheFactory; + private readonly IRedisCacheFactory _redisCacheFactory; + private readonly IRemoteFactory _remoteFactory; - public CodeflowHistoryManager(IRedisCacheFactory cacheFactory) + public CodeflowHistoryManager( + IRedisCacheFactory cacheFactory, + IRemoteFactory remoteFactory) { - _cacheFactory = cacheFactory; + _redisCacheFactory = cacheFactory; + _remoteFactory = remoteFactory; } public async void RefreshCodeflowHistory(string repo, string branch) @@ -30,10 +57,59 @@ public async void RefreshCodeflowHistory(string repo, string branch) await Task.CompletedTask; } - public async Task GetCachedCodeflowHistory(Guid subscriptionId) + public async Task GetCodeflowHistory(Subscription subscription, bool fetchLatest) + { + + } + + public async Task GetCachedCodeflowHistoryAsync(Guid? subscriptionId) { - var cache = _cacheFactory.Create(subscriptionId.ToString()); + string id = subscriptionId.ToString()!; + + var cache = _redisCacheFactory.Create(id); + var cachedHistory = await cache.TryGetStateAsync(); return cachedHistory; } + + + public async Task FetchLatestCodeflowHistoryAsync(Subscription subscription) + { + var cachedCommits = await GetCachedCodeflowHistoryAsync(subscription.Id); + + var remote = await _remoteFactory.CreateRemoteAsync(subscription.TargetRepository); + + var latestCommits = await remote.FetchNewerRepoCommitsAsync( + subscription.TargetBranch, + subscription.TargetBranch, + cachedCommits?.Commits.FirstOrDefault()?.CommitSha, + 500); + + var latestCachedCodeflow = cachedCommits?.Commits.FirstOrDefault( + x => x.IncomingCodeflows.Count > 0); + + var codeFlows = await FetchLatestIncomingCodeflows( + subscription.TargetRepository, + subscription.TargetBranch, + latestCachedCodeflow, + remote); + + return null; + } + + private async Task> FetchLatestIncomingCodeflows( + string repo, + string branch, + CodeflowGraphCommit? latestCachedCommit, + IRemote? remote) + { + if (remote == null) + { + remote = await _remoteFactory.CreateRemoteAsync(repo); + } + + var lastFlow = remote.GetLastIncomingCodeflow(branch, latestCachedCommit?.CommitSha); + + return null; + } } diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/CodeflowHistoryManager.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/CodeflowHistoryManager.cs new file mode 100644 index 0000000000..22a7171556 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/CodeflowHistoryManager.cs @@ -0,0 +1,78 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using ProductConstructionService.Common; + +namespace ProductConstructionService.DependencyFlow; + +public interface ICodeflowHistoryManager +{ + void RefreshCodeflowHistory(string repo, string branch); + Task GetCodeflowHistory(Guid? subscriptionId); + Task FetchLatestCodeflowHistoryAsync(Guid? subscriptionId); +} + +public record CodeflowHistory( + List Commits, + List Codeflows); + +public record CodeflowHistoryResult( + CodeflowHistory? ForwardFlowHistory, + CodeflowHistory? BackflowHistory, + bool ResultIsOutdated); + +public record CodeflowRecord( + string SourceCommitSha, + string TargetCommitSha, + DateTimeOffset CodeflowMergeDate); + +public record CodeflowGraphCommit( + string CommitSha, + DateTimeOffset CommitDate, + string Author, + string Description, + List OutgoingFlows); + +public class CodeflowHistoryManager : ICodeflowHistoryManager +{ + private readonly IRedisCacheFactory _redisCacheFactory; + private readonly IRemote _remote; + + public CodeflowHistoryManager(IRedisCacheFactory cacheFactory) + { + _redisCacheFactory = cacheFactory; + } + + public async void RefreshCodeflowHistory(string repo, string branch) + { + //0. Fetch latest commit from local git repo if exists + //1. Fetch new parts from GitHub API + //2. Fetch old parts from disk + //3. Stitch them together + //4. Write contents to cache + await Task.CompletedTask; + } + + public async Task GetCodeflowHistory(Guid? subscriptionId, bool fetchLatest) + { + } + + public async Task GetCachedCodeflowHistoryAsync(Guid? subscriptionId) + { + + string id = subscriptionId.ToString()!; + + var cache = _redisCacheFactory.Create(id); + + var cachedHistory = await cache.TryGetStateAsync(); + return cachedHistory; + } + + + public async Task FetchLatestCodeflowHistoryAsync(Guid? subscriptionId) + { + + await Task.CompletedTask; + return null; + } +} From e20a1aa528baf3e19435a0a8c2524455c99164ab Mon Sep 17 00:00:00 2001 From: Adam Zippor Date: Tue, 23 Sep 2025 12:08:05 +0200 Subject: [PATCH 05/19] remove unused file --- .../CodeflowHistoryManager.cs | 3 +- .../CodeflowHistoryManager.cs | 78 ------------------- 2 files changed, 2 insertions(+), 79 deletions(-) delete mode 100644 src/ProductConstructionService/ProductConstructionService.DependencyFlow/CodeflowHistoryManager.cs diff --git a/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistoryManager.cs b/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistoryManager.cs index c729b649df..83b0b01c28 100644 --- a/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistoryManager.cs +++ b/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistoryManager.cs @@ -59,7 +59,7 @@ public async void RefreshCodeflowHistory(string repo, string branch) public async Task GetCodeflowHistory(Subscription subscription, bool fetchLatest) { - + //todo: implement this method } public async Task GetCachedCodeflowHistoryAsync(Guid? subscriptionId) @@ -110,6 +110,7 @@ private async Task> FetchLatestIncomingCodeflows( var lastFlow = remote.GetLastIncomingCodeflow(branch, latestCachedCommit?.CommitSha); + //todo: implement this method return null; } } diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/CodeflowHistoryManager.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/CodeflowHistoryManager.cs deleted file mode 100644 index 22a7171556..0000000000 --- a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/CodeflowHistoryManager.cs +++ /dev/null @@ -1,78 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using ProductConstructionService.Common; - -namespace ProductConstructionService.DependencyFlow; - -public interface ICodeflowHistoryManager -{ - void RefreshCodeflowHistory(string repo, string branch); - Task GetCodeflowHistory(Guid? subscriptionId); - Task FetchLatestCodeflowHistoryAsync(Guid? subscriptionId); -} - -public record CodeflowHistory( - List Commits, - List Codeflows); - -public record CodeflowHistoryResult( - CodeflowHistory? ForwardFlowHistory, - CodeflowHistory? BackflowHistory, - bool ResultIsOutdated); - -public record CodeflowRecord( - string SourceCommitSha, - string TargetCommitSha, - DateTimeOffset CodeflowMergeDate); - -public record CodeflowGraphCommit( - string CommitSha, - DateTimeOffset CommitDate, - string Author, - string Description, - List OutgoingFlows); - -public class CodeflowHistoryManager : ICodeflowHistoryManager -{ - private readonly IRedisCacheFactory _redisCacheFactory; - private readonly IRemote _remote; - - public CodeflowHistoryManager(IRedisCacheFactory cacheFactory) - { - _redisCacheFactory = cacheFactory; - } - - public async void RefreshCodeflowHistory(string repo, string branch) - { - //0. Fetch latest commit from local git repo if exists - //1. Fetch new parts from GitHub API - //2. Fetch old parts from disk - //3. Stitch them together - //4. Write contents to cache - await Task.CompletedTask; - } - - public async Task GetCodeflowHistory(Guid? subscriptionId, bool fetchLatest) - { - } - - public async Task GetCachedCodeflowHistoryAsync(Guid? subscriptionId) - { - - string id = subscriptionId.ToString()!; - - var cache = _redisCacheFactory.Create(id); - - var cachedHistory = await cache.TryGetStateAsync(); - return cachedHistory; - } - - - public async Task FetchLatestCodeflowHistoryAsync(Guid? subscriptionId) - { - - await Task.CompletedTask; - return null; - } -} From a8ef01a6c23e99e6ee01606cf27ea72b9e1de661 Mon Sep 17 00:00:00 2001 From: Adam Zippor Date: Mon, 6 Oct 2025 08:48:39 +0200 Subject: [PATCH 06/19] get codeflows WIP --- .../CodeflowHistoryManager.cs | 134 +++++++++++++----- 1 file changed, 101 insertions(+), 33 deletions(-) diff --git a/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistoryManager.cs b/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistoryManager.cs index 83b0b01c28..a6669c0342 100644 --- a/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistoryManager.cs +++ b/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistoryManager.cs @@ -8,7 +8,6 @@ namespace ProductConstructionService.Common; public interface ICodeflowHistoryManager { - void RefreshCodeflowHistory(string repo, string branch); Task GetCodeflowHistory(Guid? subscriptionId); Task FetchLatestCodeflowHistoryAsync(Guid? subscriptionId); } @@ -32,7 +31,7 @@ public record CodeflowGraphCommit( DateTimeOffset CommitDate, string Author, string Description, - List IncomingCodeflows); + CodeflowGraphCommit? IncomingCodeflow); public class CodeflowHistoryManager : ICodeflowHistoryManager { @@ -47,70 +46,139 @@ public CodeflowHistoryManager( _remoteFactory = remoteFactory; } - public async void RefreshCodeflowHistory(string repo, string branch) + public async Task GetCachedCodeflowHistoryAsync(Subscription subscription) { - //0. Fetch latest commit from local git repo if exists - //1. Fetch new parts from GitHub API - //2. Fetch old parts from disk - //3. Stitch them together - //4. Write contents to cache - await Task.CompletedTask; + string id = subscription.Id.ToString()!; + var cache = _redisCacheFactory.Create(id); + return await cache.TryGetStateAsync(); } - public async Task GetCodeflowHistory(Subscription subscription, bool fetchLatest) + public async Task GetCachedCodeflowHistoryAsync( + Subscription subscription, + string commitSha, + int commitFetchCount) { - //todo: implement this method + // todo this method returns the codeflow history starting from commitSha. + // It only reads from redis and never modifies the cache } - public async Task GetCachedCodeflowHistoryAsync(Guid? subscriptionId) - { - string id = subscriptionId.ToString()!; - - var cache = _redisCacheFactory.Create(id); - - var cachedHistory = await cache.TryGetStateAsync(); - return cachedHistory; - } - - public async Task FetchLatestCodeflowHistoryAsync(Subscription subscription) + // get cached commits + // fetched fresh commits & fresh codeflows + // erase old if no connection + // persist new + public async Task FetchLatestCodeflowHistoryAsync( + Subscription subscription, + int commitFetchCount) { + //todo acquire lock on the redis Zset here + // (or not ? Maybe the unique commit SHA as the zset key ensures that commits can't be added twice) + // in that case, we'd only have to check that when a write fails due to the commit already being cached, + // we don't fail the flow var cachedCommits = await GetCachedCodeflowHistoryAsync(subscription.Id); var remote = await _remoteFactory.CreateRemoteAsync(subscription.TargetRepository); + latestCachedCommitSha = cachedCommits?.Commits.FirstOrDefault()?.CommitSha; + var latestCommits = await remote.FetchNewerRepoCommitsAsync( subscription.TargetBranch, subscription.TargetBranch, - cachedCommits?.Commits.FirstOrDefault()?.CommitSha, - 500); + latestCachedCommitSha, + commitFetchCount); + + if (latestCommits.Count == commitFetchCount && + latestCommits.LastOrDefault()?.CommitSha != latestCachedCommitSha) + { + // we have a gap in the history - throw away cache because we can't form a continuous history + cachedCommits = []; + } + else + { + latestCommits = latestCommits + .Where(commit => commit.CommitSha != latestCachedCommitSha) + .ToList(); + } var latestCachedCodeflow = cachedCommits?.Commits.FirstOrDefault( - x => x.IncomingCodeflows.Count > 0); + commit => commit.IncomingCodeflows != null); var codeFlows = await FetchLatestIncomingCodeflows( subscription.TargetRepository, subscription.TargetBranch, - latestCachedCodeflow, + !string.IsNullOrEmpty(subscription.TargetDirectory), + latestCommits, remote); + foreach (var commit in latestCommits) + { + string? sourceCommitSha = codeflows.GetCodeflowSourceCommit(commit.CommitSha); + commit.IncomingCodeflow = sourceCommitSha; + } + + // todo cache fresh commits and release lock on the Zset + await CacheCommits(latestCommits); + return null; } - private async Task> FetchLatestIncomingCodeflows( + private async Task FetchLatestIncomingCodeflows( string repo, string branch, - CodeflowGraphCommit? latestCachedCommit, + bool isForwardFlow, + List latestCommits, IRemote? remote) { - if (remote == null) + remote ??= await _remoteFactory.CreateRemoteAsync(repo); + + string? lastFlowSha = null; + string? lastCachedFlowSha = latestCommits + .FirstOrDefault(commit => commit.IncomingCodeflow != null) + ?.IncomingCodeflow + ?.CommitSha; + + while (last) + if (isForwardFlow) + { + var lastFlow = remote.GetVmrLastIncomingCodeflowAsync(branch, latestCachedCommit?.CommitSha); + } + else + { + var lastFlow = remote.GetRepoLastIncomingCodeflowAsync(branch, latestCachedCommit?.CommitSha); + } + + + return null; + } + + private async Task CacheCommits(List commits) + { + // Cache the commits as part of the subscription's redis ZSet of CodeflowGraphCommit objects + if (commits.Count == 0) { - remote = await _remoteFactory.CreateRemoteAsync(repo); + return; } + var cache = _redisCacheFactory.Create(subscription.Id.ToString()!); + await cache.SetStateAsync(new CodeflowHistory(commits, codeflows)); + } +} - var lastFlow = remote.GetLastIncomingCodeflow(branch, latestCachedCommit?.CommitSha); - - //todo: implement this method +class GraphCodeflows +{ + // keys: target repo commits that have incoing codeflows + // values: commit SHAs of those codeflows in the source repo + public Dictionary Codeflows { get; set; } = []; + + /// + /// Returns the source commit of the codeflow if targetCommitSha is a target commit of a codeflow. + /// Otherwise, return null + /// + public string? GetCodeflowSourceCommit(string targetCommitSha) + { + if (Codeflows.TryGetValue(targetCommitSha, out var sourceCommit)) + { + return sourceCommit; + } return null; } } From 6f2aa6590f70a37611b4ad3c681a07d85c4865bf Mon Sep 17 00:00:00 2001 From: Adam Zippor Date: Sun, 16 Nov 2025 16:15:26 +0100 Subject: [PATCH 07/19] codeflow graph WIP --- .../DarcLib/GitHubClient.cs | 6 +- .../CodeflowHistoryManager.cs | 164 +++++++++--------- 2 files changed, 89 insertions(+), 81 deletions(-) diff --git a/src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs b/src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs index 3117beb289..9a2f9f485e 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs @@ -1515,7 +1515,7 @@ public async Task> FetchNewerRepoCommitsAsync( var request = new CommitRequest { - Sha = branch ?? "main", + Sha = branch, }; var options = new ApiOptions @@ -1559,7 +1559,7 @@ public async Task> FetchNewerRepoCommitsAsync( return [.. allCommits.Take(maxCount)]; } - public async Task GetLastIncomingForwardFlow(string vmrUrl, string mappingName, string commit) + public async Task GetLastIncomingForwardFlowAtCommitAsync(string vmrUrl, string mappingName, string commit) { var content = await GetFileContentAtCommit( vmrUrl, @@ -1590,7 +1590,7 @@ public async Task> FetchNewerRepoCommitsAsync( return new ForwardFlow(lastForwardFlowRepoSha, lastForwardFlowVmrSha); } - public async Task GetLastIncomingBackflow(string repoUrl, string commit) + public async Task GetLastIncomingBackflowAtCommitAsync(string repoUrl, string commit) { var content = await GetFileContentAtCommit( repoUrl, diff --git a/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistoryManager.cs b/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistoryManager.cs index a6669c0342..4b9443226b 100644 --- a/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistoryManager.cs +++ b/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistoryManager.cs @@ -31,135 +31,143 @@ public record CodeflowGraphCommit( DateTimeOffset CommitDate, string Author, string Description, - CodeflowGraphCommit? IncomingCodeflow); + string? IncomingCodeflowSha); -public class CodeflowHistoryManager : ICodeflowHistoryManager +public class CodeflowHistoryManager( + IRemoteFactory remoteFactory, + IConnectionMultiplexer connection) : ICodeflowHistoryManager { - private readonly IRedisCacheFactory _redisCacheFactory; - private readonly IRemoteFactory _remoteFactory; + private readonly IRemoteFactory _remoteFactory = remoteFactory; + private readonly IConnectionMultiplexer _connection = connection; - public CodeflowHistoryManager( - IRedisCacheFactory cacheFactory, - IRemoteFactory remoteFactory) + public async Task GetCachedCodeflowHistoryAsync(string subscriptionId, int commitFetchCount) { - _redisCacheFactory = cacheFactory; - _remoteFactory = remoteFactory; - } + if (commitFetchCount < 1) + { + throw new ArgumentOutOfRangeException(nameof(commitFetchCount)); + } - public async Task GetCachedCodeflowHistoryAsync(Subscription subscription) - { - string id = subscription.Id.ToString()!; - var cache = _redisCacheFactory.Create(id); - return await cache.TryGetStateAsync(); - } + var cache = _connection.GetDatabase(); - public async Task GetCachedCodeflowHistoryAsync( - Subscription subscription, - string commitSha, - int commitFetchCount) - { - // todo this method returns the codeflow history starting from commitSha. - // It only reads from redis and never modifies the cache - } + var commitShas = await cache.SortedSetRangeByRankWithScores( + key: subscriptionId, + start: 0, + stop: commitFetchCount - 1, + order: Order.Descending()) + .Select(e => (string)e.Element) + .ToList(); + return await cache.StringGetAsync(commitShas) + .Select() + } - // get cached commits - // fetched fresh commits & fresh codeflows - // erase old if no connection - // persist new public async Task FetchLatestCodeflowHistoryAsync( Subscription subscription, int commitFetchCount) { - //todo acquire lock on the redis Zset here - // (or not ? Maybe the unique commit SHA as the zset key ensures that commits can't be added twice) - // in that case, we'd only have to check that when a write fails due to the commit already being cached, - // we don't fail the flow var cachedCommits = await GetCachedCodeflowHistoryAsync(subscription.Id); var remote = await _remoteFactory.CreateRemoteAsync(subscription.TargetRepository); - latestCachedCommitSha = cachedCommits?.Commits.FirstOrDefault()?.CommitSha; + latestCachedCommitSha = cachedCommits? + .Commits + .FirstOrDefault()? + .CommitSha; - var latestCommits = await remote.FetchNewerRepoCommitsAsync( + var newCommits = await remote.FetchNewerRepoCommitsAsync( subscription.TargetBranch, subscription.TargetBranch, latestCachedCommitSha, commitFetchCount); - if (latestCommits.Count == commitFetchCount && - latestCommits.LastOrDefault()?.CommitSha != latestCachedCommitSha) - { - // we have a gap in the history - throw away cache because we can't form a continuous history - cachedCommits = []; - } - else + if (newCommits.Count == commitFetchCount + && latestCommits.LastOrDefault()?.CommitSha != latestCachedCommitSha) { - latestCommits = latestCommits - .Where(commit => commit.CommitSha != latestCachedCommitSha) - .ToList(); + // there's a gap between the new and cached commits. clear the cache and start from scratch. + ClearCodeflowCacheAsync(subscription.Id); } - var latestCachedCodeflow = cachedCommits?.Commits.FirstOrDefault( - commit => commit.IncomingCodeflows != null); + newCommits.Remove(latestCachedCommitSha); - var codeFlows = await FetchLatestIncomingCodeflows( + var codeFlows = await EnrichCommitsWithCodeflowDataAsync( subscription.TargetRepository, subscription.TargetBranch, !string.IsNullOrEmpty(subscription.TargetDirectory), latestCommits, remote); - foreach (var commit in latestCommits) - { - string? sourceCommitSha = codeflows.GetCodeflowSourceCommit(commit.CommitSha); - commit.IncomingCodeflow = sourceCommitSha; - } - - // todo cache fresh commits and release lock on the Zset - await CacheCommits(latestCommits); + await CacheCommitsAsync(latestCommits); return null; } - private async Task FetchLatestIncomingCodeflows( + private async Task EnrichCommitsWithCodeflowDataAsync( string repo, string branch, bool isForwardFlow, - List latestCommits, - IRemote? remote) + List commits) { - remote ??= await _remoteFactory.CreateRemoteAsync(repo); - - string? lastFlowSha = null; - string? lastCachedFlowSha = latestCommits - .FirstOrDefault(commit => commit.IncomingCodeflow != null) - ?.IncomingCodeflow - ?.CommitSha; - - while (last) - if (isForwardFlow) + if (commits.Count == 0) + { + return []; + } + + remote = await _remoteFactory.CreateRemoteAsync(repo); + + var lastCommitSha = commits + .First() + .CommitSha; + + var commitLookups = commits.ToDictionary(c => c.CommitSha, c => c); + + while (true) + { + var lastFlow = isForwardFlow + ? await _remoteFactory.GetLastVmrIncomingCodeflowAsync(branch, lastCommitSha) + : await _remoteFactory.GetLastRepoIncomingCodeflowAsync(branch, lastCommitSha); + + if (commitLookups.Contains(lastFlow.TargetCommitSha)) { - var lastFlow = remote.GetVmrLastIncomingCodeflowAsync(branch, latestCachedCommit?.CommitSha); + commitLookups[lastFlow.TargetCommitSha].IncomingCodeflowSha = lastFlow.SourceCommitSha; + commitLookups.Remove(lastFlow.TargetCommitSha); // prevent the possibility of infinite loops + lastCommitSha = lastFlow.TargetCommitSha; } else { - var lastFlow = remote.GetRepoLastIncomingCodeflowAsync(branch, latestCachedCommit?.CommitSha); + break; } - - - return null; + } + return commits; } - private async Task CacheCommits(List commits) + private async Task CacheCommitsAsync( + string subscriptionId, + List commits, + int latestCachedCommitScore) { - // Cache the commits as part of the subscription's redis ZSet of CodeflowGraphCommit objects if (commits.Count == 0) { return; } - var cache = _redisCacheFactory.Create(subscription.Id.ToString()!); - await cache.SetStateAsync(new CodeflowHistory(commits, codeflows)); + var cache = _connection.GetDatabase(); + + int i = latestCachedCommitScore ?? 0; + + var sortedSetEntries = commits + .Select(c => new SortedSetEntry(c.CommitSha, i++)) + .ToArray(); + + await cache.SortedSetAddAsync(subscriptionId, sortedSetEntries); + + // todo key must either be unique to mapping, or contain last flow info for all mappings + // ..... or not! any one single commit is relevant only to one mapping + var commitGraphEntries = commits + .Select(c => new KeyValuePair("CodeflowGraphCommit_" + c.CommitSha, c)) + .ToArray(); + + await cache.StringSetAsync(commits); + + ClearCacheTail(); // remove any elements after the 3000th or so? } } @@ -180,5 +188,5 @@ class GraphCodeflows return sourceCommit; } return null; - } + } } From bb2fd0f2e94704e7c542218b58b7ebc59b850b5a Mon Sep 17 00:00:00 2001 From: Adam Zippor Date: Mon, 17 Nov 2025 17:10:30 +0100 Subject: [PATCH 08/19] codeflow graph improvements --- .../CodeflowHistoryManager.cs | 180 +++++++++--------- 1 file changed, 90 insertions(+), 90 deletions(-) diff --git a/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistoryManager.cs b/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistoryManager.cs index 4b9443226b..741b21ed54 100644 --- a/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistoryManager.cs +++ b/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistoryManager.cs @@ -1,37 +1,25 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using Microsoft.DotNet.DarcLib; +using System.Text.Json; using Maestro.Data.Models; +using Microsoft.DotNet.DarcLib; +using StackExchange.Redis; namespace ProductConstructionService.Common; public interface ICodeflowHistoryManager { - Task GetCodeflowHistory(Guid? subscriptionId); - Task FetchLatestCodeflowHistoryAsync(Guid? subscriptionId); + Task> GetCachedCodeflowHistoryAsync(string subscriptionId, int commitFetchCount); + Task> FetchLatestCodeflowHistoryAsync(Subscription subscription, int commitFetchCount); } -public record CodeflowHistory( - List Commits, - List Codeflows); - -public record CodeflowHistoryResult( - CodeflowHistory? ForwardFlowHistory, - CodeflowHistory? BackflowHistory, - bool ResultIsOutdated); - -public record CodeflowRecord( - string SourceCommitSha, - string TargetCommitSha, - DateTimeOffset CodeflowMergeDate); - public record CodeflowGraphCommit( string CommitSha, - DateTimeOffset CommitDate, string Author, string Description, - string? IncomingCodeflowSha); + string? IncomingCodeflowSha, + int? redisScore); public class CodeflowHistoryManager( IRemoteFactory remoteFactory, @@ -40,7 +28,14 @@ public class CodeflowHistoryManager( private readonly IRemoteFactory _remoteFactory = remoteFactory; private readonly IConnectionMultiplexer _connection = connection; - public async Task GetCachedCodeflowHistoryAsync(string subscriptionId, int commitFetchCount) + private static RedisKey GetCommitKey(string commitSha) => $"CodeflowGraphCommit_{commitSha}"; + + private const int MaxCommitFetchCount = 500; + + + public async Task> GetCachedCodeflowHistoryAsync( + string subscriptionId, + int commitFetchCount = 100) { if (commitFetchCount < 1) { @@ -49,59 +44,65 @@ public class CodeflowHistoryManager( var cache = _connection.GetDatabase(); - var commitShas = await cache.SortedSetRangeByRankWithScores( + var res = await cache.SortedSetRangeByRankWithScoresAsync( key: subscriptionId, start: 0, stop: commitFetchCount - 1, - order: Order.Descending()) - .Select(e => (string)e.Element) - .ToList(); + order: Order.Descending); - return await cache.StringGetAsync(commitShas) - .Select() + var commitKeys = res + .Select(e => new RedisKey(e.Element.ToString())) + .ToArray(); + + var commitValues = await cache.StringGetAsync(commitKeys); + + if (commitValues.Any(val => !val.HasValue)) + { + throw new InvalidOperationException($"Corrupted commit data encountered."); + } + + return [.. commitValues + .Select(commit => JsonSerializer.Deserialize(commit.ToString())) + .OfType()]; } - public async Task FetchLatestCodeflowHistoryAsync( + public async Task> FetchLatestCodeflowHistoryAsync( Subscription subscription, - int commitFetchCount) + int commitFetchCount = 100) { - var cachedCommits = await GetCachedCodeflowHistoryAsync(subscription.Id); + var cachedCommits = await GetCachedCodeflowHistoryAsync(subscription.Id.ToString()); - var remote = await _remoteFactory.CreateRemoteAsync(subscription.TargetRepository); + var latestCachedCommit = cachedCommits.FirstOrDefault(); - latestCachedCommitSha = cachedCommits? - .Commits - .FirstOrDefault()? - .CommitSha; - - var newCommits = await remote.FetchNewerRepoCommitsAsync( - subscription.TargetBranch, + var newCommits = await FetchNewCommits( + subscription.TargetRepository, subscription.TargetBranch, - latestCachedCommitSha, - commitFetchCount); + latestCachedCommit?.CommitSha); - if (newCommits.Count == commitFetchCount - && latestCommits.LastOrDefault()?.CommitSha != latestCachedCommitSha) + if (newCommits.LastOrDefault()?.CommitSha == latestCachedCommit?.CommitSha) + { + newCommits.RemoveAt(newCommits.Count - 1); + } + else { // there's a gap between the new and cached commits. clear the cache and start from scratch. - ClearCodeflowCacheAsync(subscription.Id); + await ClearCodeflowCacheAsync(subscription.Id); } - newCommits.Remove(latestCachedCommitSha); - - var codeFlows = await EnrichCommitsWithCodeflowDataAsync( + var graphCommits = await EnrichCommitsWithCodeflowDataAsync( subscription.TargetRepository, subscription.TargetBranch, !string.IsNullOrEmpty(subscription.TargetDirectory), - latestCommits, - remote); + newCommits); - await CacheCommitsAsync(latestCommits); + await CacheCommitsAsync(subscription.Id.ToString(), graphCommits); - return null; + return [.. graphCommits + .Concat(cachedCommits) + .Take(commitFetchCount)]; } - private async Task EnrichCommitsWithCodeflowDataAsync( + private async Task> EnrichCommitsWithCodeflowDataAsync( string repo, string branch, bool isForwardFlow, @@ -112,81 +113,80 @@ private async Task EnrichCommitsWithCodeflowDataAsync( return []; } - remote = await _remoteFactory.CreateRemoteAsync(repo); + var remote = await _remoteFactory.CreateRemoteAsync(repo); - var lastCommitSha = commits - .First() - .CommitSha; + var lastCommitSha = commits.First().CommitSha; var commitLookups = commits.ToDictionary(c => c.CommitSha, c => c); while (true) { var lastFlow = isForwardFlow - ? await _remoteFactory.GetLastVmrIncomingCodeflowAsync(branch, lastCommitSha) - : await _remoteFactory.GetLastRepoIncomingCodeflowAsync(branch, lastCommitSha); + ? await remote.GetLastVmrIncomingCodeflowAsync(branch, lastCommitSha) + : await remote.GetLastRepoIncomingCodeflowAsync(branch, lastCommitSha); - if (commitLookups.Contains(lastFlow.TargetCommitSha)) - { - commitLookups[lastFlow.TargetCommitSha].IncomingCodeflowSha = lastFlow.SourceCommitSha; - commitLookups.Remove(lastFlow.TargetCommitSha); // prevent the possibility of infinite loops - lastCommitSha = lastFlow.TargetCommitSha; - } - else + if (!commitLookups.Contains(lastFlow.TargetCommitSha)) { + // there are no more incoming codeflows within the commit range break; } + + commitLookups[lastFlow.TargetCommitSha].IncomingCodeflowSha = lastFlow.SourceCommitSha; + commitLookups.Remove(lastFlow.TargetCommitSha); + lastCommitSha = lastFlow.TargetCommitSha; } + return commits; } private async Task CacheCommitsAsync( string subscriptionId, List commits, - int latestCachedCommitScore) + int latestCachedCommitScore = 0) { if (commits.Count == 0) { return; } - var cache = _connection.GetDatabase(); - int i = latestCachedCommitScore ?? 0; + var cache = _connection.GetDatabase(); var sortedSetEntries = commits - .Select(c => new SortedSetEntry(c.CommitSha, i++)) + .Select(c => new SortedSetEntry(c.CommitSha, latestCachedCommitScore++)) .ToArray(); await cache.SortedSetAddAsync(subscriptionId, sortedSetEntries); - // todo key must either be unique to mapping, or contain last flow info for all mappings - // ..... or not! any one single commit is relevant only to one mapping var commitGraphEntries = commits - .Select(c => new KeyValuePair("CodeflowGraphCommit_" + c.CommitSha, c)) + .Select(c => new KeyValuePair( + GetCommitKey(c.CommitSha), + JsonSerializer.Serialize(c))) .ToArray(); - await cache.StringSetAsync(commits); + await cache.StringSetAsync(commitGraphEntries); - ClearCacheTail(); // remove any elements after the 3000th or so? + //todo remove any elements after the 3000th or so? to keep the cache from growing indefinitely } -} -class GraphCodeflows -{ - // keys: target repo commits that have incoing codeflows - // values: commit SHAs of those codeflows in the source repo - public Dictionary Codeflows { get; set; } = []; - - /// - /// Returns the source commit of the codeflow if targetCommitSha is a target commit of a codeflow. - /// Otherwise, return null - /// - public string? GetCodeflowSourceCommit(string targetCommitSha) + private async Task> FetchNewCommits( + string targetRepository, + string targetBranch, + string? latestCachedCommitSha) { - if (Codeflows.TryGetValue(targetCommitSha, out var sourceCommit)) - { - return sourceCommit; - } - return null; - } + var remote = await _remoteFactory.CreateRemoteAsync(targetRepository); + + var newCommits = await remote.FetchNewerRepoCommitsAsync( + targetRepository, + targetBranch, + latestCachedCommitSha, + MaxCommitFetchCount); + + return [.. newCommits + .Select(commit => new CodeflowGraphCommit( + CommitSha: commit.Sha, + Author: commit.Author, + Description: commit.Message, + IncomingCodeflowSha: null, + redisScore: null))]; + } } From c566f9ddcfcb530a2f9ec72799aef041b5c776ce Mon Sep 17 00:00:00 2001 From: Adam Zippor Date: Tue, 18 Nov 2025 09:57:04 +0100 Subject: [PATCH 09/19] add codeflow cache clear --- .../CodeflowHistoryManager.cs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistoryManager.cs b/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistoryManager.cs index 741b21ed54..ea0f97d878 100644 --- a/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistoryManager.cs +++ b/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistoryManager.cs @@ -28,7 +28,8 @@ public class CodeflowHistoryManager( private readonly IRemoteFactory _remoteFactory = remoteFactory; private readonly IConnectionMultiplexer _connection = connection; - private static RedisKey GetCommitKey(string commitSha) => $"CodeflowGraphCommit_{commitSha}"; + private static RedisKey GetCodeflowGraphCommitKey(string id) => $"CodeflowGraphCommit_{id}"; + private static RedisKey GetSortedSetKey(string id) => $"CodeflowHistory_{id}"; private const int MaxCommitFetchCount = 500; @@ -86,7 +87,7 @@ public async Task> FetchLatestCodeflowHistoryAsync( else { // there's a gap between the new and cached commits. clear the cache and start from scratch. - await ClearCodeflowCacheAsync(subscription.Id); + await ClearCodeflowCacheAsync(subscription.Id.ToString()); } var graphCommits = await EnrichCommitsWithCodeflowDataAsync( @@ -159,7 +160,7 @@ private async Task CacheCommitsAsync( var commitGraphEntries = commits .Select(c => new KeyValuePair( - GetCommitKey(c.CommitSha), + GetCodeflowGraphCommitKey(c.CommitSha), JsonSerializer.Serialize(c))) .ToArray(); @@ -168,6 +169,12 @@ private async Task CacheCommitsAsync( //todo remove any elements after the 3000th or so? to keep the cache from growing indefinitely } + private async Task ClearCodeflowCacheAsync(string subscriptionId) + { + var cache = _connection.GetDatabase(); + await cache.KeyDeleteAsync(GetSortedSetKey(subscriptionId)); + } + private async Task> FetchNewCommits( string targetRepository, string targetBranch, From f5df5ef4d391d8cb878cb99fc94e3a8c03d4f6b8 Mon Sep 17 00:00:00 2001 From: Adam Zippor Date: Tue, 18 Nov 2025 11:30:53 +0100 Subject: [PATCH 10/19] Trim off tail end of cache --- .../CodeflowHistoryManager.cs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistoryManager.cs b/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistoryManager.cs index ea0f97d878..f885e5480a 100644 --- a/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistoryManager.cs +++ b/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistoryManager.cs @@ -32,6 +32,7 @@ public class CodeflowHistoryManager( private static RedisKey GetSortedSetKey(string id) => $"CodeflowHistory_{id}"; private const int MaxCommitFetchCount = 500; + private const int MaxCommitsCached = 3000; public async Task> GetCachedCodeflowHistoryAsync( @@ -166,7 +167,13 @@ private async Task CacheCommitsAsync( await cache.StringSetAsync(commitGraphEntries); - //todo remove any elements after the 3000th or so? to keep the cache from growing indefinitely + if (latestCachedCommitScore > MaxCommitsCached) + { + await cache.SortedSetRemoveRangeByScoreAsync( + key: subscriptionId, + start: 0, + stop: latestCachedCommitScore - MaxCommitsCached); + } } private async Task ClearCodeflowCacheAsync(string subscriptionId) From 3c52313b62004e596266e5ed8bb293dd793ba79a Mon Sep 17 00:00:00 2001 From: Adam Zippor Date: Mon, 1 Dec 2025 10:31:56 +0100 Subject: [PATCH 11/19] Fix codeflow loop --- .../DarcLib/GitHubClient.cs | 20 ++--- src/Microsoft.DotNet.Darc/DarcLib/IRemote.cs | 5 +- .../DarcLib/IRemoteGitRepo.cs | 5 +- src/Microsoft.DotNet.Darc/DarcLib/Remote.cs | 22 +++-- .../CodeflowHistoryManager.cs | 80 ++++++++++--------- .../PullRequestUpdater.cs | 5 ++ 6 files changed, 74 insertions(+), 63 deletions(-) diff --git a/src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs b/src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs index 9a2f9f485e..1a76a11067 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs @@ -2,7 +2,6 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; -using System.Collections; using System.Collections.Generic; using System.Collections.Immutable; using System.IO; @@ -14,7 +13,6 @@ using System.Text.Json; using System.Text.RegularExpressions; using System.Threading.Tasks; -using LibGit2Sharp; using Maestro.Common; using Maestro.MergePolicyEvaluation; using Microsoft.DotNet.DarcLib.Helpers; @@ -25,7 +23,6 @@ using Microsoft.DotNet.Services.Utility; using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; -using Microsoft.TeamFoundation.TestManagement.WebApi; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using Newtonsoft.Json.Serialization; @@ -1559,7 +1556,7 @@ public async Task> FetchNewerRepoCommitsAsync( return [.. allCommits.Take(maxCount)]; } - public async Task GetLastIncomingForwardFlowAtCommitAsync(string vmrUrl, string mappingName, string commit) + public async Task GetLastIncomingForwardFlowAtCommitAsync(string vmrUrl, string mappingName, string commit) { var content = await GetFileContentAtCommit( vmrUrl, @@ -1590,7 +1587,7 @@ public async Task> FetchNewerRepoCommitsAsync( return new ForwardFlow(lastForwardFlowRepoSha, lastForwardFlowVmrSha); } - public async Task GetLastIncomingBackflowAtCommitAsync(string repoUrl, string commit) + public async Task GetLastIncomingBackflowAsync(string repoUrl, string commit) { var content = await GetFileContentAtCommit( repoUrl, @@ -1598,22 +1595,19 @@ public async Task> FetchNewerRepoCommitsAsync( VersionFiles.VersionDetailsXml); var lastBackflowVmrSha = VersionDetailsParser - .ParseVersionDetailsXml(content)? + .ParseVersionDetailsXml(content) .Source? - .Sha; - - if (lastBackflowVmrSha == null) - { - return null; - } + .Sha + ?? throw new ("Could not parse version details"); + // todo: we can skip this call if the last flown SHA is one that we already cached int lineNumber = content .Split(Environment.NewLine) .ToList() .FindIndex(line => line.Contains(VersionDetailsParser.SourceElementName) && line.Contains(lastBackflowVmrSha)); - + string lastBackflowRepoSha = await BlameLineAsync( repoUrl, commit, diff --git a/src/Microsoft.DotNet.Darc/DarcLib/IRemote.cs b/src/Microsoft.DotNet.Darc/DarcLib/IRemote.cs index 00c87cb0b1..00c95dba42 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/IRemote.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/IRemote.cs @@ -283,8 +283,9 @@ Task CommitUpdatesWithNoCloningAsync( Task> FetchNewerRepoCommitsAsync(string repoUrl, string branch, string commitSha, int maxCount = 100); - Task> GetLastIncomingForwardflow(string vmrUrl, string commit); - Task> GetLastIncomingBackflow(string repoUrl, string commit); + Task GetLastIncomingForwardFlowAsync(string vmrUrl, string commit); + + Task GetLastIncomingBackflowAsync(string repoUrl, string commit); #endregion } diff --git a/src/Microsoft.DotNet.Darc/DarcLib/IRemoteGitRepo.cs b/src/Microsoft.DotNet.Darc/DarcLib/IRemoteGitRepo.cs index e76476de10..2cd8a3d3fc 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/IRemoteGitRepo.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/IRemoteGitRepo.cs @@ -4,6 +4,7 @@ using Maestro.MergePolicyEvaluation; using Microsoft.DotNet.DarcLib.Helpers; using Microsoft.DotNet.DarcLib.Models; +using Microsoft.DotNet.DarcLib.Models.VirtualMonoRepo; using System; using System.Collections.Generic; using System.Threading.Tasks; @@ -182,12 +183,12 @@ Task> SearchPullRequestsAsync( /// /// Get the last forward flow that was merged onto the given VMR at the specified commit /// - Task?> GetLastIncomingForwardFlow(string vmrUrl, string branch, string commit); + Task GetLastIncomingForwardFlowAsync(string vmrUrl, string commit); /// /// Get the last back flow that was merged onto the given repo at the specified commit /// - Task?> GetLastIncomingBackflow(string repoUrl, string commit); + Task GetLastIncomingBackflowAsync(string repoUrl, string commit); } #nullable disable diff --git a/src/Microsoft.DotNet.Darc/DarcLib/Remote.cs b/src/Microsoft.DotNet.Darc/DarcLib/Remote.cs index 918319cb00..f6bb97f3fb 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/Remote.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/Remote.cs @@ -453,7 +453,6 @@ public async Task> GetPullRequestCommentsAsync(string pullRequestUr return await _remoteGitClient.GetPullRequestCommentsAsync(pullRequestUrl); } - public async Task GetSourceManifestAsync(string vmrUri, string branchOrCommit) { var fileContent = await _remoteGitClient.GetFileContentsAsync( @@ -502,10 +501,14 @@ public async Task> GetGitTreeNames(string path, stri return await _remoteGitClient.GetGitTreeNames(path, repoUri, branch); } - public Task> FetchLatestRepoCommitsAsync(string repoUrl, string branch, int maxCount = 100) + public Task> FetchLatestRepoCommitsAsync( + string repoUrl, + string branch, + int maxCount = 100) { return _remoteGitClient.FetchLatestRepoCommitsAsync(repoUrl, branch, maxCount); } + public Task> FetchNewerRepoCommitsAsync( string repoUrl, string branch, @@ -515,14 +518,17 @@ public Task> FetchNewerRepoCommitsAsync( return _remoteGitClient.FetchNewerRepoCommitsAsync(repoUrl, branch, commitSha, maxCount); } - public async Task> GetLastIncomingForwardflow(string vmrUrl, string commit) + public async Task GetLastIncomingForwardFlowAsync( + string vmrUrl, + string commit) { - await Task.CompletedTask; - return null; + return await _remoteGitClient.GetLastIncomingForwardFlowAsync(vmrUrl, commit); } - public async Task> GetLastIncomingBackflow(string repoUrl, string commit) + + public async Task GetLastIncomingBackflowAsync( + string repoUrl, + string commit) { - await Task.CompletedTask; - return null; + return await _remoteGitClient.GetLastIncomingBackflowAsync(repoUrl, commit); } } diff --git a/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistoryManager.cs b/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistoryManager.cs index f885e5480a..bd04c45701 100644 --- a/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistoryManager.cs +++ b/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistoryManager.cs @@ -4,14 +4,21 @@ using System.Text.Json; using Maestro.Data.Models; using Microsoft.DotNet.DarcLib; +using Microsoft.DotNet.DarcLib.Models.VirtualMonoRepo; +using Pipelines.Sockets.Unofficial.Arenas; using StackExchange.Redis; namespace ProductConstructionService.Common; public interface ICodeflowHistoryManager { - Task> GetCachedCodeflowHistoryAsync(string subscriptionId, int commitFetchCount); - Task> FetchLatestCodeflowHistoryAsync(Subscription subscription, int commitFetchCount); + Task> GetCachedCodeflowHistoryAsync( + string subscriptionId, + int commitFetchCount); + + Task> FetchLatestCodeflowHistoryAsync( + Subscription subscription, + int commitFetchCount); } public record CodeflowGraphCommit( @@ -34,8 +41,7 @@ public class CodeflowHistoryManager( private const int MaxCommitFetchCount = 500; private const int MaxCommitsCached = 3000; - - public async Task> GetCachedCodeflowHistoryAsync( + public async Task> GetCachedCodeflowHistoryAsync( string subscriptionId, int commitFetchCount = 100) { @@ -68,7 +74,7 @@ public async Task> GetCachedCodeflowHistoryAsync( .OfType()]; } - public async Task> FetchLatestCodeflowHistoryAsync( + public async Task> FetchLatestCodeflowHistoryAsync( Subscription subscription, int commitFetchCount = 100) { @@ -83,19 +89,20 @@ public async Task> FetchLatestCodeflowHistoryAsync( if (newCommits.LastOrDefault()?.CommitSha == latestCachedCommit?.CommitSha) { - newCommits.RemoveAt(newCommits.Count - 1); + newCommits.RemoveLast(); } else { - // there's a gap between the new and cached commits. clear the cache and start from scratch. + // the fetched commits do not connect to the cached commits + // we don't know how many commits are missing, so clear the cache and start over await ClearCodeflowCacheAsync(subscription.Id.ToString()); } var graphCommits = await EnrichCommitsWithCodeflowDataAsync( + newCommits, subscription.TargetRepository, subscription.TargetBranch, - !string.IsNullOrEmpty(subscription.TargetDirectory), - newCommits); + !string.IsNullOrEmpty(subscription.TargetDirectory)); await CacheCommitsAsync(subscription.Id.ToString(), graphCommits); @@ -104,49 +111,46 @@ public async Task> FetchLatestCodeflowHistoryAsync( .Take(commitFetchCount)]; } - private async Task> EnrichCommitsWithCodeflowDataAsync( + private async Task> EnrichCommitsWithCodeflowDataAsync( + LinkedList commits, string repo, string branch, - bool isForwardFlow, - List commits) + bool isForwardFlow) { - if (commits.Count == 0) - { - return []; - } - var remote = await _remoteFactory.CreateRemoteAsync(repo); - var lastCommitSha = commits.First().CommitSha; + var current = commits.First; - var commitLookups = commits.ToDictionary(c => c.CommitSha, c => c); - - while (true) + while (current != null) { - var lastFlow = isForwardFlow - ? await remote.GetLastVmrIncomingCodeflowAsync(branch, lastCommitSha) - : await remote.GetLastRepoIncomingCodeflowAsync(branch, lastCommitSha); + Codeflow lastFlow = isForwardFlow + ? await remote.GetLastIncomingForwardFlowAsync(branch, current.Value.CommitSha) + : await remote.GetLastIncomingBackflowAsync(branch, current.Value.CommitSha); - if (!commitLookups.Contains(lastFlow.TargetCommitSha)) - { - // there are no more incoming codeflows within the commit range + var target = current; + + while (target != null && target.Value.CommitSha != lastFlow.TargetSha) + target = target.Next; + + if (target == null) break; - } - commitLookups[lastFlow.TargetCommitSha].IncomingCodeflowSha = lastFlow.SourceCommitSha; - commitLookups.Remove(lastFlow.TargetCommitSha); - lastCommitSha = lastFlow.TargetCommitSha; - } + target.Value = target.Value with + { + IncomingCodeflowSha = lastFlow.SourceSha, + }; + current = target.Next; + } return commits; } private async Task CacheCommitsAsync( string subscriptionId, - List commits, + IEnumerable commits, int latestCachedCommitScore = 0) { - if (commits.Count == 0) + if (!commits.Any()) { return; } @@ -182,7 +186,7 @@ private async Task ClearCodeflowCacheAsync(string subscriptionId) await cache.KeyDeleteAsync(GetSortedSetKey(subscriptionId)); } - private async Task> FetchNewCommits( + private async Task> FetchNewCommits( string targetRepository, string targetBranch, string? latestCachedCommitSha) @@ -195,12 +199,12 @@ private async Task> FetchNewCommits( latestCachedCommitSha, MaxCommitFetchCount); - return [.. newCommits - .Select(commit => new CodeflowGraphCommit( + return new LinkedList( + newCommits.Select(commit => new CodeflowGraphCommit( CommitSha: commit.Sha, Author: commit.Author, Description: commit.Message, IncomingCodeflowSha: null, - redisScore: null))]; + redisScore: null))); } } diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestUpdater.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestUpdater.cs index 8d853f2498..19a057aa51 100644 --- a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestUpdater.cs +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestUpdater.cs @@ -453,6 +453,11 @@ private async Task TryMergingPrAsync( { await remote.MergeDependencyPullRequestAsync(pr.Url, new MergePullRequestParameters()); + //todo: add codeflow history synchronization here + //todo: we should be able to obtain the last flown sha & squashed PR sha here, and cache the codeflow + // without any github requests. + + //todo: fire these in a new process - exceptions shouldn't affect main thread foreach (SubscriptionPullRequestUpdate subscription in pr.ContainedSubscriptions) { await RegisterSubscriptionUpdateAction(SubscriptionUpdateAction.MergingPullRequest, subscription.SubscriptionId); From cbe3261d3e900beb0cb235031ae5836c5dd257b8 Mon Sep 17 00:00:00 2001 From: Adam Zippor Date: Thu, 18 Dec 2025 18:25:31 +0100 Subject: [PATCH 12/19] fix build --- .../Maestro.DataProviders/RemoteFactory.cs | 3 + .../DarcLib/AzureDevOpsClient.cs | 10 + .../DarcLib/GitHubClient.cs | 22 +- .../DarcLib/GitRepoFactory.cs | 4 + .../DarcLib/Helpers/VersionDetailsParser.cs | 11 +- src/Microsoft.DotNet.Darc/DarcLib/IRemote.cs | 2 +- .../DarcLib/IRemoteGitRepo.cs | 4 +- src/Microsoft.DotNet.Darc/DarcLib/Remote.cs | 3 +- .../DarcLib/VirtualMonoRepo/VmrCodeflower.cs | 13 +- .../Controllers/SubscriptionsController.cs | 22 +- .../Controllers/SubscriptionsController.cs | 2 +- .../Controllers/SubscriptionsController.cs | 3 +- .../Models/CodeflowHistoryResult.cs | 6 +- .../CodeflowHistory.cs | 6 - .../CodeflowHistoryManager.cs | 210 ------------------ .../PullRequestUpdater.cs | 1 + 16 files changed, 64 insertions(+), 258 deletions(-) delete mode 100644 src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistory.cs delete mode 100644 src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistoryManager.cs diff --git a/src/Maestro/Maestro.DataProviders/RemoteFactory.cs b/src/Maestro/Maestro.DataProviders/RemoteFactory.cs index a9f74a9517..ee81e1e5ab 100644 --- a/src/Maestro/Maestro.DataProviders/RemoteFactory.cs +++ b/src/Maestro/Maestro.DataProviders/RemoteFactory.cs @@ -25,6 +25,7 @@ public class RemoteFactory : IRemoteFactory private readonly IGitHubTokenProvider _gitHubTokenProvider; private readonly IAzureDevOpsTokenProvider _azdoTokenProvider; private readonly IServiceProvider _serviceProvider; + private readonly IVersionDetailsParser _versionDetailsParser; public RemoteFactory( BuildAssetRegistryContext context, @@ -45,6 +46,7 @@ public RemoteFactory( _azdoTokenProvider = azdoTokenProvider; _cache = memoryCache; _serviceProvider = serviceProvider; + _versionDetailsParser = versionDetailsParser; } public async Task CreateRemoteAsync(string repoUrl) @@ -87,6 +89,7 @@ private async Task GetRemoteGitClient(string repoUrl) : new GitHubClient( new Microsoft.DotNet.DarcLib.GitHubTokenProvider(_gitHubTokenProvider), _processManager, + _versionDetailsParser, _loggerFactory.CreateLogger(), _cache.Cache), diff --git a/src/Microsoft.DotNet.Darc/DarcLib/AzureDevOpsClient.cs b/src/Microsoft.DotNet.Darc/DarcLib/AzureDevOpsClient.cs index cc7ac932b1..04af78647f 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/AzureDevOpsClient.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/AzureDevOpsClient.cs @@ -16,6 +16,7 @@ using Microsoft.DotNet.DarcLib.Helpers; using Microsoft.DotNet.DarcLib.Models; using Microsoft.DotNet.DarcLib.Models.AzureDevOps; +using Microsoft.DotNet.DarcLib.Models.VirtualMonoRepo; using Microsoft.DotNet.Services.Utility; using Microsoft.Extensions.Logging; using Microsoft.TeamFoundation.SourceControl.WebApi; @@ -1954,5 +1955,14 @@ public Task> FetchLatestFetchNewerRepoCommitsAsyncRepoCommits( string commitSha, int maxCount) => throw new NotImplementedException(); +<<<<<<< HEAD >>>>>>> 0973d0ff2 (Codeflow graphs WIP) +======= + + public Task> FetchLatestRepoCommitsAsync(string repoUrl, string branch, int maxCount) => throw new NotImplementedException(); + public Task> FetchNewerRepoCommitsAsync(string repoUrl, string branch, string commitSha, int maxCount) => throw new NotImplementedException(); + public Task GetLastIncomingForwardFlowAsync(string vmrUrl, string commit) => throw new NotImplementedException(); + public Task GetLastIncomingBackflowAsync(string repoUrl, string commit) => throw new NotImplementedException(); + public Task GetLastIncomingForwardFlowAsync(string vmrUrl, string mappingName, string commit) => throw new NotImplementedException(); +>>>>>>> e3300f865 (fix build) } diff --git a/src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs b/src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs index 1a76a11067..7f5246e5a8 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs @@ -27,7 +27,6 @@ using Newtonsoft.Json.Linq; using Newtonsoft.Json.Serialization; using Octokit; -using static System.Net.WebRequestMethods; #nullable enable namespace Microsoft.DotNet.DarcLib; @@ -52,6 +51,7 @@ public class GitHubClient : RemoteRepoBase, IRemoteGitRepo private readonly string _userAgent = $"DarcLib-{DarcLibVersion}"; private IGitHubClient? _lazyClient = null; private readonly Dictionary<(string, string, string?), string> _gitRefCommitCache; + private readonly IVersionDetailsParser _versionDetailsParser; static GitHubClient() { @@ -64,9 +64,10 @@ static GitHubClient() public GitHubClient( IRemoteTokenProvider remoteTokenProvider, IProcessManager processManager, + IVersionDetailsParser versionDetailsParser, ILogger logger, IMemoryCache? cache) - : this(remoteTokenProvider, processManager, logger, null, cache) + : this(remoteTokenProvider, processManager, logger, null, versionDetailsParser, cache) { } @@ -75,6 +76,7 @@ public GitHubClient( IProcessManager processManager, ILogger logger, string? temporaryRepositoryPath, + IVersionDetailsParser versionDetailsParser, IMemoryCache? cache) : base(remoteTokenProvider, processManager, temporaryRepositoryPath, cache, logger) { @@ -86,6 +88,7 @@ public GitHubClient( NullValueHandling = NullValueHandling.Ignore }; _gitRefCommitCache = []; + _versionDetailsParser = versionDetailsParser; } public bool AllowRetries { get; set; } = true; @@ -1556,7 +1559,7 @@ public async Task> FetchNewerRepoCommitsAsync( return [.. allCommits.Take(maxCount)]; } - public async Task GetLastIncomingForwardFlowAtCommitAsync(string vmrUrl, string mappingName, string commit) + public async Task GetLastIncomingForwardFlowAsync(string vmrUrl, string mappingName, string commit) { var content = await GetFileContentAtCommit( vmrUrl, @@ -1587,18 +1590,21 @@ public async Task GetLastIncomingForwardFlowAtCommitAsync(string vm return new ForwardFlow(lastForwardFlowRepoSha, lastForwardFlowVmrSha); } - public async Task GetLastIncomingBackflowAsync(string repoUrl, string commit) + public async Task GetLastIncomingBackflowAsync(string repoUrl, string commit) { var content = await GetFileContentAtCommit( repoUrl, commit, VersionFiles.VersionDetailsXml); - var lastBackflowVmrSha = VersionDetailsParser - .ParseVersionDetailsXml(content) + var lastBackflowVmrSha = _versionDetailsParser.ParseVersionDetailsXml(content) .Source? - .Sha - ?? throw new ("Could not parse version details"); + .Sha; + + if (lastBackflowVmrSha == null) + { + return null; + } // todo: we can skip this call if the last flown SHA is one that we already cached int lineNumber = content diff --git a/src/Microsoft.DotNet.Darc/DarcLib/GitRepoFactory.cs b/src/Microsoft.DotNet.Darc/DarcLib/GitRepoFactory.cs index e827178929..6f0ae0b199 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/GitRepoFactory.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/GitRepoFactory.cs @@ -22,6 +22,7 @@ public class GitRepoFactory : IGitRepoFactory private readonly ITelemetryRecorder _telemetryRecorder; private readonly IProcessManager _processManager; private readonly IFileSystem _fileSystem; + private readonly IVersionDetailsParser _versionDetailsParser; private readonly ILoggerFactory _loggerFactory; private readonly string? _temporaryPath = null; @@ -31,6 +32,7 @@ public GitRepoFactory( ITelemetryRecorder telemetryRecorder, IProcessManager processManager, IFileSystem fileSystem, + IVersionDetailsParser versionDetailsParser, ILoggerFactory loggerFactory, string temporaryPath) { @@ -39,6 +41,7 @@ public GitRepoFactory( _telemetryRecorder = telemetryRecorder; _processManager = processManager; _fileSystem = fileSystem; + _versionDetailsParser = versionDetailsParser; _loggerFactory = loggerFactory; _temporaryPath = temporaryPath; } @@ -56,6 +59,7 @@ public GitRepoFactory( _processManager, _loggerFactory.CreateLogger(), _temporaryPath, + _versionDetailsParser, // Caching not in use for Darc local client. null), diff --git a/src/Microsoft.DotNet.Darc/DarcLib/Helpers/VersionDetailsParser.cs b/src/Microsoft.DotNet.Darc/DarcLib/Helpers/VersionDetailsParser.cs index 8997ccad0b..ad0658c438 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/Helpers/VersionDetailsParser.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/Helpers/VersionDetailsParser.cs @@ -5,14 +5,9 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using System.Threading.Tasks; using System.Xml; -using LibGit2Sharp; using Microsoft.DotNet.DarcLib.Models; using Microsoft.DotNet.DarcLib.Models.Darc; -using Microsoft.DotNet.DarcLib.Models.VirtualMonoRepo; -using Microsoft.TeamFoundation.Work.WebApi; -using static System.Net.Mime.MediaTypeNames; #nullable enable namespace Microsoft.DotNet.DarcLib.Helpers; @@ -61,13 +56,13 @@ public VersionDetails ParseVersionDetailsFile(string path, bool includePinned = return ParseVersionDetailsXml(content, includePinned: includePinned); } - public static VersionDetails ParseVersionDetailsXml(string fileContents, bool includePinned = true) + public VersionDetails ParseVersionDetailsXml(string fileContents, bool includePinned = true) { XmlDocument document = GetXmlDocument(fileContents); return ParseVersionDetailsXml(document, includePinned: includePinned); } - public static VersionDetails ParseVersionDetailsXml(XmlDocument document, bool includePinned = true) + public VersionDetails ParseVersionDetailsXml(XmlDocument document, bool includePinned = true) { XmlNodeList? dependencyNodes = (document?.DocumentElement?.SelectNodes($"//{DependencyElementName}")) ?? throw new Exception($"There was an error while reading '{VersionFiles.VersionDetailsXml}' and it came back empty. " + @@ -82,7 +77,7 @@ public static VersionDetails ParseVersionDetailsXml(XmlDocument document, bool i return new VersionDetails(dependencies, vmrCodeflow); } - private static List ParseDependencyDetails(XmlNodeList dependencies) + private List ParseDependencyDetails(XmlNodeList dependencies) { List dependencyDetails = []; diff --git a/src/Microsoft.DotNet.Darc/DarcLib/IRemote.cs b/src/Microsoft.DotNet.Darc/DarcLib/IRemote.cs index 00c95dba42..54a32e0346 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/IRemote.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/IRemote.cs @@ -283,7 +283,7 @@ Task CommitUpdatesWithNoCloningAsync( Task> FetchNewerRepoCommitsAsync(string repoUrl, string branch, string commitSha, int maxCount = 100); - Task GetLastIncomingForwardFlowAsync(string vmrUrl, string commit); + Task GetLastIncomingForwardFlowAsync(string vmrUrl, string mappingNAme, string commit); Task GetLastIncomingBackflowAsync(string repoUrl, string commit); diff --git a/src/Microsoft.DotNet.Darc/DarcLib/IRemoteGitRepo.cs b/src/Microsoft.DotNet.Darc/DarcLib/IRemoteGitRepo.cs index 2cd8a3d3fc..c32a1c13ce 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/IRemoteGitRepo.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/IRemoteGitRepo.cs @@ -183,12 +183,12 @@ Task> SearchPullRequestsAsync( /// /// Get the last forward flow that was merged onto the given VMR at the specified commit /// - Task GetLastIncomingForwardFlowAsync(string vmrUrl, string commit); + Task GetLastIncomingForwardFlowAsync(string vmrUrl, string mappingName, string commit); /// /// Get the last back flow that was merged onto the given repo at the specified commit /// - Task GetLastIncomingBackflowAsync(string repoUrl, string commit); + Task GetLastIncomingBackflowAsync(string repoUrl, string commit); } #nullable disable diff --git a/src/Microsoft.DotNet.Darc/DarcLib/Remote.cs b/src/Microsoft.DotNet.Darc/DarcLib/Remote.cs index f6bb97f3fb..045d1f8bf2 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/Remote.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/Remote.cs @@ -520,9 +520,10 @@ public Task> FetchNewerRepoCommitsAsync( public async Task GetLastIncomingForwardFlowAsync( string vmrUrl, + string mappingName, string commit) { - return await _remoteGitClient.GetLastIncomingForwardFlowAsync(vmrUrl, commit); + return await _remoteGitClient.GetLastIncomingForwardFlowAsync(vmrUrl, mappingName, commit); } public async Task GetLastIncomingBackflowAsync( diff --git a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrCodeflower.cs b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrCodeflower.cs index a25c9c5bff..24c45d1ed9 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrCodeflower.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrCodeflower.cs @@ -508,16 +508,19 @@ await HandleRevertedFiles( /// /// Finds the last backflow between a repo and a VMR. /// - private async Task GetLastBackflow(string repoPath) + private async Task GetLastBackflow(NativePath repoPath) { - var versionDetailsContent = await _localGitClient.GetFileFromGitAsync(repoPath, VersionFiles.VersionDetailsXml); - if (versionDetailsContent == null) + // Last backflow SHA comes from Version.Details.xml in the repo + SourceDependency? source = _versionDetailsParser.ParseVersionDetailsFile(repoPath / VersionFiles.VersionDetailsXml).Source; + if (source is null) { return null; } - var lineNumber = VersionDetailsParser.SourceDependencyLineNumber(versionDetailsContent); - string lastBackflowRepoSha = await _localGitClient.BlameLineAsync(repoPath, VersionFiles.VersionDetailsXml, lineNumber); + string lastBackflowVmrSha = source.Sha; + string lastBackflowRepoSha = await _localGitClient.BlameLineAsync( + repoPath / VersionFiles.VersionDetailsXml, + line => line.Contains(VersionDetailsParser.SourceElementName) && line.Contains(lastBackflowVmrSha)); return new Backflow(lastBackflowVmrSha, lastBackflowRepoSha); } diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs index 3a3c91699a..244557f4b9 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs @@ -10,10 +10,10 @@ using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Options; -using ProductConstructionService.Common; using ProductConstructionService.Api.Controllers.Models; using ProductConstructionService.Api.v2018_07_16.Models; using Microsoft.DotNet.DarcLib; +using ProductConstructionService.Common.CodeflowHistory; using ProductConstructionService.DependencyFlow.WorkItems; using ProductConstructionService.WorkItems; using Channel = Maestro.Data.Models.Channel; @@ -118,7 +118,7 @@ public virtual async Task GetSubscription(Guid id) [ValidateModelState] public virtual async Task GetCodeflowHistory(Guid id) { - return await GetCodeflowHistoryCore(id); + return await GetCodeflowHistoryCore(id, false); } /// @@ -173,7 +173,7 @@ protected async Task TriggerSubscriptionCore(Guid id, int buildId return Accepted(new Subscription(subscription)); } - protected async Task GetCodeflowHistoryCore(Guid id, bool fetchNewChanges) + protected async Task GetCodeflowHistoryCore(Guid id, bool fetchNewChanges = false) { var subscription = await _context.Subscriptions .Include(sub => sub.LastAppliedBuild) @@ -199,14 +199,14 @@ protected async Task GetCodeflowHistoryCore(Guid id, bool fetchNe bool isForwardFlow = !string.IsNullOrEmpty(subscription.TargetDirectory); - CodeflowHistory? cachedFlows; - CodeflowHistory? oppositeCachedFlows; + IReadOnlyCollection? cachedFlows; + IReadOnlyCollection? oppositeCachedFlows; - cachedFlows = await _codeflowHistoryManager.FetchLatestCodeflowHistoryAsync(id); - - oppositeCachedFlows = await _codeflowHistoryManager.FetchLatestCodeflowHistoryAsync( - oppositeDirectionSubscription?.Id); + cachedFlows = await _codeflowHistoryManager.FetchLatestCodeflowHistoryAsync(subscription); + oppositeCachedFlows = oppositeDirectionSubscription != null + ? await _codeflowHistoryManager.FetchLatestCodeflowHistoryAsync(oppositeDirectionSubscription) + : []; var lastCommit = subscription.LastAppliedBuild.Commit; @@ -229,9 +229,9 @@ protected async Task GetCodeflowHistoryCore(Guid id, bool fetchNe private static bool IsCodeflowHistoryOutdated( SubscriptionDAO? subscription, - CodeflowHistory? cachedFlows) + IReadOnlyCollection? cachedFlows) { - string? lastCachedCodeflow = cachedFlows?.Codeflows.LastOrDefault()?.SourceCommitSha; + string? lastCachedCodeflow = cachedFlows?.LastOrDefault()?.SourceRepoFlowSha; string? lastAppliedCommit = subscription?.LastAppliedBuild?.Commit; return !string.Equals(lastCachedCodeflow, lastAppliedCommit, StringComparison.Ordinal); } diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2019_01_16/Controllers/SubscriptionsController.cs b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2019_01_16/Controllers/SubscriptionsController.cs index 447d8cf0f6..d8eb1e2281 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2019_01_16/Controllers/SubscriptionsController.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2019_01_16/Controllers/SubscriptionsController.cs @@ -11,7 +11,7 @@ using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Options; using ProductConstructionService.Api.v2019_01_16.Models; -using ProductConstructionService.Common; +using ProductConstructionService.Common.CodeflowHistory; using ProductConstructionService.WorkItems; namespace ProductConstructionService.Api.Api.v2019_01_16.Controllers; diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/SubscriptionsController.cs b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/SubscriptionsController.cs index 26da258832..03536e79a6 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/SubscriptionsController.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/SubscriptionsController.cs @@ -7,12 +7,11 @@ using Microsoft.AspNetCore.ApiVersioning; using Microsoft.AspNetCore.ApiVersioning.Swashbuckle; using Microsoft.AspNetCore.Mvc; -using Microsoft.DotNet.DarcLib; using Microsoft.DotNet.GitHub.Authentication; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Options; using ProductConstructionService.Api.v2020_02_20.Models; -using ProductConstructionService.Common; +using ProductConstructionService.Common.CodeflowHistory; using ProductConstructionService.WorkItems; namespace ProductConstructionService.Api.Api.v2020_02_20.Controllers; diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Controllers/Models/CodeflowHistoryResult.cs b/src/ProductConstructionService/ProductConstructionService.Api/Controllers/Models/CodeflowHistoryResult.cs index 91a08b8422..f9dd2687ca 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Controllers/Models/CodeflowHistoryResult.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Controllers/Models/CodeflowHistoryResult.cs @@ -1,13 +1,13 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using ProductConstructionService.Common; +using ProductConstructionService.Common.CodeflowHistory; namespace ProductConstructionService.Api.Controllers.Models; public class CodeflowHistoryResult { - public CodeflowHistory? ForwardFlowHistory { get; set; } - public CodeflowHistory? BackflowHistory { get; set; } + public IReadOnlyCollection ForwardFlowHistory { get; set; } = []; + public IReadOnlyCollection BackflowHistory { get; set; } = []; public bool ResultIsOutdated { get; set; } } diff --git a/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistory.cs b/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistory.cs deleted file mode 100644 index 50609f7cf5..0000000000 --- a/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistory.cs +++ /dev/null @@ -1,6 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using Microsoft.DotNet.DarcLib; - -namespace ProductConstructionService.Common; diff --git a/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistoryManager.cs b/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistoryManager.cs deleted file mode 100644 index bd04c45701..0000000000 --- a/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistoryManager.cs +++ /dev/null @@ -1,210 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Text.Json; -using Maestro.Data.Models; -using Microsoft.DotNet.DarcLib; -using Microsoft.DotNet.DarcLib.Models.VirtualMonoRepo; -using Pipelines.Sockets.Unofficial.Arenas; -using StackExchange.Redis; - -namespace ProductConstructionService.Common; - -public interface ICodeflowHistoryManager -{ - Task> GetCachedCodeflowHistoryAsync( - string subscriptionId, - int commitFetchCount); - - Task> FetchLatestCodeflowHistoryAsync( - Subscription subscription, - int commitFetchCount); -} - -public record CodeflowGraphCommit( - string CommitSha, - string Author, - string Description, - string? IncomingCodeflowSha, - int? redisScore); - -public class CodeflowHistoryManager( - IRemoteFactory remoteFactory, - IConnectionMultiplexer connection) : ICodeflowHistoryManager -{ - private readonly IRemoteFactory _remoteFactory = remoteFactory; - private readonly IConnectionMultiplexer _connection = connection; - - private static RedisKey GetCodeflowGraphCommitKey(string id) => $"CodeflowGraphCommit_{id}"; - private static RedisKey GetSortedSetKey(string id) => $"CodeflowHistory_{id}"; - - private const int MaxCommitFetchCount = 500; - private const int MaxCommitsCached = 3000; - - public async Task> GetCachedCodeflowHistoryAsync( - string subscriptionId, - int commitFetchCount = 100) - { - if (commitFetchCount < 1) - { - throw new ArgumentOutOfRangeException(nameof(commitFetchCount)); - } - - var cache = _connection.GetDatabase(); - - var res = await cache.SortedSetRangeByRankWithScoresAsync( - key: subscriptionId, - start: 0, - stop: commitFetchCount - 1, - order: Order.Descending); - - var commitKeys = res - .Select(e => new RedisKey(e.Element.ToString())) - .ToArray(); - - var commitValues = await cache.StringGetAsync(commitKeys); - - if (commitValues.Any(val => !val.HasValue)) - { - throw new InvalidOperationException($"Corrupted commit data encountered."); - } - - return [.. commitValues - .Select(commit => JsonSerializer.Deserialize(commit.ToString())) - .OfType()]; - } - - public async Task> FetchLatestCodeflowHistoryAsync( - Subscription subscription, - int commitFetchCount = 100) - { - var cachedCommits = await GetCachedCodeflowHistoryAsync(subscription.Id.ToString()); - - var latestCachedCommit = cachedCommits.FirstOrDefault(); - - var newCommits = await FetchNewCommits( - subscription.TargetRepository, - subscription.TargetBranch, - latestCachedCommit?.CommitSha); - - if (newCommits.LastOrDefault()?.CommitSha == latestCachedCommit?.CommitSha) - { - newCommits.RemoveLast(); - } - else - { - // the fetched commits do not connect to the cached commits - // we don't know how many commits are missing, so clear the cache and start over - await ClearCodeflowCacheAsync(subscription.Id.ToString()); - } - - var graphCommits = await EnrichCommitsWithCodeflowDataAsync( - newCommits, - subscription.TargetRepository, - subscription.TargetBranch, - !string.IsNullOrEmpty(subscription.TargetDirectory)); - - await CacheCommitsAsync(subscription.Id.ToString(), graphCommits); - - return [.. graphCommits - .Concat(cachedCommits) - .Take(commitFetchCount)]; - } - - private async Task> EnrichCommitsWithCodeflowDataAsync( - LinkedList commits, - string repo, - string branch, - bool isForwardFlow) - { - var remote = await _remoteFactory.CreateRemoteAsync(repo); - - var current = commits.First; - - while (current != null) - { - Codeflow lastFlow = isForwardFlow - ? await remote.GetLastIncomingForwardFlowAsync(branch, current.Value.CommitSha) - : await remote.GetLastIncomingBackflowAsync(branch, current.Value.CommitSha); - - var target = current; - - while (target != null && target.Value.CommitSha != lastFlow.TargetSha) - target = target.Next; - - if (target == null) - break; - - target.Value = target.Value with - { - IncomingCodeflowSha = lastFlow.SourceSha, - }; - - current = target.Next; - } - return commits; - } - - private async Task CacheCommitsAsync( - string subscriptionId, - IEnumerable commits, - int latestCachedCommitScore = 0) - { - if (!commits.Any()) - { - return; - } - - var cache = _connection.GetDatabase(); - - var sortedSetEntries = commits - .Select(c => new SortedSetEntry(c.CommitSha, latestCachedCommitScore++)) - .ToArray(); - - await cache.SortedSetAddAsync(subscriptionId, sortedSetEntries); - - var commitGraphEntries = commits - .Select(c => new KeyValuePair( - GetCodeflowGraphCommitKey(c.CommitSha), - JsonSerializer.Serialize(c))) - .ToArray(); - - await cache.StringSetAsync(commitGraphEntries); - - if (latestCachedCommitScore > MaxCommitsCached) - { - await cache.SortedSetRemoveRangeByScoreAsync( - key: subscriptionId, - start: 0, - stop: latestCachedCommitScore - MaxCommitsCached); - } - } - - private async Task ClearCodeflowCacheAsync(string subscriptionId) - { - var cache = _connection.GetDatabase(); - await cache.KeyDeleteAsync(GetSortedSetKey(subscriptionId)); - } - - private async Task> FetchNewCommits( - string targetRepository, - string targetBranch, - string? latestCachedCommitSha) - { - var remote = await _remoteFactory.CreateRemoteAsync(targetRepository); - - var newCommits = await remote.FetchNewerRepoCommitsAsync( - targetRepository, - targetBranch, - latestCachedCommitSha, - MaxCommitFetchCount); - - return new LinkedList( - newCommits.Select(commit => new CodeflowGraphCommit( - CommitSha: commit.Sha, - Author: commit.Author, - Description: commit.Message, - IncomingCodeflowSha: null, - redisScore: null))); - } -} diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestUpdater.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestUpdater.cs index 19a057aa51..7adf5f8dbe 100644 --- a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestUpdater.cs +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestUpdater.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Collections.Immutable; +using Kusto.Ingest; using Maestro.Data.Models; using Maestro.DataProviders; using Maestro.MergePolicies; From 64b31126af2ee4c4657310de947316c27e137492 Mon Sep 17 00:00:00 2001 From: Adam Zippor Date: Thu, 18 Dec 2025 19:41:32 +0100 Subject: [PATCH 13/19] fix build 2 --- src/Microsoft.DotNet.Darc/Darc/Helpers/RemoteFactory.cs | 4 ++++ src/Microsoft.DotNet.Darc/DarcLib/AzureDevOpsClient.cs | 4 ---- src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs | 3 ++- .../DarcLib/Helpers/VersionDetailsParser.cs | 2 +- .../Api/v2020_02_20/Controllers/SubscriptionsController.cs | 1 + 5 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/Microsoft.DotNet.Darc/Darc/Helpers/RemoteFactory.cs b/src/Microsoft.DotNet.Darc/Darc/Helpers/RemoteFactory.cs index e82288b051..5853e2dd6e 100644 --- a/src/Microsoft.DotNet.Darc/Darc/Helpers/RemoteFactory.cs +++ b/src/Microsoft.DotNet.Darc/Darc/Helpers/RemoteFactory.cs @@ -21,12 +21,14 @@ internal class RemoteFactory : IRemoteFactory private readonly IServiceProvider _serviceProvider; private readonly IAzureDevOpsTokenProvider _azdoTokenProvider; private readonly IRemoteTokenProvider _githubTokenProvider; + private readonly IVersionDetailsParser _versionDetailsParser; public RemoteFactory( IAzureDevOpsTokenProvider azdoTokenProvider, [FromKeyedServices("github")]IRemoteTokenProvider githubTokenProvider, ILoggerFactory loggerFactory, ICommandLineOptions options, + IVersionDetailsParser versionDetailsParser, IServiceProvider serviceProvider) { _loggerFactory = loggerFactory; @@ -34,6 +36,7 @@ public RemoteFactory( _serviceProvider = serviceProvider; _azdoTokenProvider = azdoTokenProvider; _githubTokenProvider = githubTokenProvider; + _versionDetailsParser = versionDetailsParser; } public Task CreateRemoteAsync(string repoUrl) @@ -62,6 +65,7 @@ private IRemoteGitRepo CreateRemoteGitClient(ICommandLineOptions options, string new ProcessManager(_loggerFactory.CreateLogger(), options.GitLocation), _loggerFactory.CreateLogger(), temporaryRepositoryRoot, + _versionDetailsParser, // Caching not in use for Darc local client. null), diff --git a/src/Microsoft.DotNet.Darc/DarcLib/AzureDevOpsClient.cs b/src/Microsoft.DotNet.Darc/DarcLib/AzureDevOpsClient.cs index 04af78647f..a4dff27d70 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/AzureDevOpsClient.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/AzureDevOpsClient.cs @@ -1955,14 +1955,10 @@ public Task> FetchLatestFetchNewerRepoCommitsAsyncRepoCommits( string commitSha, int maxCount) => throw new NotImplementedException(); -<<<<<<< HEAD ->>>>>>> 0973d0ff2 (Codeflow graphs WIP) -======= public Task> FetchLatestRepoCommitsAsync(string repoUrl, string branch, int maxCount) => throw new NotImplementedException(); public Task> FetchNewerRepoCommitsAsync(string repoUrl, string branch, string commitSha, int maxCount) => throw new NotImplementedException(); public Task GetLastIncomingForwardFlowAsync(string vmrUrl, string commit) => throw new NotImplementedException(); public Task GetLastIncomingBackflowAsync(string repoUrl, string commit) => throw new NotImplementedException(); public Task GetLastIncomingForwardFlowAsync(string vmrUrl, string mappingName, string commit) => throw new NotImplementedException(); ->>>>>>> e3300f865 (fix build) } diff --git a/src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs b/src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs index 7f5246e5a8..0c479c0c6c 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs @@ -1497,6 +1497,8 @@ private static PullRequest ToDarcLibPullRequest(Octokit.PullRequest pr) UpdatedAt = pr.UpdatedAt, HeadBranchSha = pr.Head.Sha, }; + } + public Task> FetchLatestRepoCommitsAsync(string repoUrl, string branch, int maxCount) => throw new NotImplementedException(); @@ -1682,6 +1684,5 @@ private async Task GetFileContentAtCommit(string repoUrl, string commit, (string owner, string repo) = ParseRepoUri(repoUrl); var file = await GetClient(repoUrl).Repository.Content.GetAllContentsByRef(owner, repo, filePath, commit); return Encoding.UTF8.GetString(Convert.FromBase64String(file[0].Content)); ->>>>>>> 0973d0ff2 (Codeflow graphs WIP) } } diff --git a/src/Microsoft.DotNet.Darc/DarcLib/Helpers/VersionDetailsParser.cs b/src/Microsoft.DotNet.Darc/DarcLib/Helpers/VersionDetailsParser.cs index ad0658c438..1438d6a98d 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/Helpers/VersionDetailsParser.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/Helpers/VersionDetailsParser.cs @@ -77,7 +77,7 @@ public VersionDetails ParseVersionDetailsXml(XmlDocument document, bool includeP return new VersionDetails(dependencies, vmrCodeflow); } - private List ParseDependencyDetails(XmlNodeList dependencies) + private static List ParseDependencyDetails(XmlNodeList dependencies) { List dependencyDetails = []; diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/SubscriptionsController.cs b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/SubscriptionsController.cs index 03536e79a6..b732ba3bc8 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/SubscriptionsController.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/SubscriptionsController.cs @@ -4,6 +4,7 @@ using System.ComponentModel.DataAnnotations; using System.Net; using Maestro.Data; +using Microsoft.DotNet.DarcLib; using Microsoft.AspNetCore.ApiVersioning; using Microsoft.AspNetCore.ApiVersioning.Swashbuckle; using Microsoft.AspNetCore.Mvc; From 326ce937e675d5841125132bdb73bf28714aa63a Mon Sep 17 00:00:00 2001 From: Adam Zippor Date: Sat, 3 Jan 2026 18:41:19 +0100 Subject: [PATCH 14/19] Final changes --- .../DarcLib/GitHubClient.cs | 83 ++++++++++++------- .../Controllers/SubscriptionsController.cs | 13 +-- .../Controllers/SubscriptionsController.cs | 2 +- .../PcsStartup.cs | 2 + .../CodeflowHistory/CodeflowHistoryManager.cs | 18 ++-- 5 files changed, 69 insertions(+), 49 deletions(-) diff --git a/src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs b/src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs index 0c479c0c6c..715b9fdffd 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs @@ -1540,11 +1540,12 @@ public async Task> FetchNewerRepoCommitsAsync( { var convertedCommit = new Commit( c.Author?.Login, - c.Commit.Sha, + c.Sha, c.Commit.Message); allCommits.Add(convertedCommit); - if (convertedCommit.Sha.Equals(commitSha)) + + if (convertedCommit.Sha == commitSha) { break; } @@ -1578,11 +1579,12 @@ public async Task> FetchNewerRepoCommitsAsync( return null; } - int lineNumber = content.Split(Environment.NewLine) + int lineNumber = content.Split('\n') .ToList() - .FindIndex(line => line.Contains(lastForwardFlowRepoSha)); - + .FindIndex(line => line.Contains(lastForwardFlowRepoSha)) + + 1; // result is 0-indexed, but github lines are 1-indexed; + // todo: we can skip this call if the last flown SHA is one that we already cached string lastForwardFlowVmrSha = await BlameLineAsync( vmrUrl, commit, @@ -1608,14 +1610,15 @@ public async Task> FetchNewerRepoCommitsAsync( return null; } - // todo: we can skip this call if the last flown SHA is one that we already cached int lineNumber = content - .Split(Environment.NewLine) + .Split('\n') .ToList() .FindIndex(line => line.Contains(VersionDetailsParser.SourceElementName) && - line.Contains(lastBackflowVmrSha)); + line.Contains(lastBackflowVmrSha)) + + 1; // result is 0-indexed, but github lines are 1-indexed + // todo: we can skip this call if the last flown SHA is one that we already cached string lastBackflowRepoSha = await BlameLineAsync( repoUrl, commit, @@ -1629,41 +1632,54 @@ private async Task BlameLineAsync(string repoUrl, string commitOrBranch, { (string owner, string repo) = ParseRepoUri(repoUrl); - string query = $@""" - {{ - repository(owner: {owner}, name: {repo}) {{ - object(expression: ""{commitOrBranch}:{filePath}"") {{ - ... on Blob {{ - blame {{ - ranges {{ - startingLine - endingLine - commit {{ oid }} + var query = $@" + query {{ + repository(owner: {JsonConvert.SerializeObject(owner)}, name: {JsonConvert.SerializeObject(repo)}) {{ + object(expression: {JsonConvert.SerializeObject(commitOrBranch)}) {{ + ... on Commit {{ + blame(path: {JsonConvert.SerializeObject(filePath)}) {{ + ranges {{ + startingLine + endingLine + commit {{ + oid + }} + }} + }} }} }} }} - }} - }} - }}"""; - - var client = CreateHttpClient(repoUrl); + }}"; + + using var client = CreateHttpClient(repoUrl); var requestBody = new { query }; - var content = new StringContent(JsonConvert.SerializeObject(requestBody, _serializerSettings)); + var content = new StringContent( + JsonConvert.SerializeObject(requestBody, _serializerSettings), + Encoding.UTF8, + "application/json" + ); - var response = await client.PostAsync("", content); + var response = await client.PostAsync("graphql", content); response.EnsureSuccessStatusCode(); using var stream = await response.Content.ReadAsStreamAsync(); using var doc = await JsonDocument.ParseAsync(stream); - var ranges = doc.RootElement + if (doc.RootElement.TryGetProperty("errors", out var errors)) + throw new InvalidOperationException($"GitHub GraphQL error: {errors}"); + + var obj = doc.RootElement .GetProperty("data") .GetProperty("repository") - .GetProperty("object") - .GetProperty("blame") - .GetProperty("ranges") - .EnumerateArray(); + .GetProperty("object"); + + if (obj.ValueKind == JsonValueKind.Null) + throw new InvalidOperationException($"Commit or branch '{commitOrBranch}' not found."); + + // The blame field comes directly from the Commit node + var blame = obj.GetProperty("blame"); + var ranges = blame.GetProperty("ranges").EnumerateArray(); foreach (var range in ranges) { @@ -1672,17 +1688,20 @@ ... on Blob {{ if (lineNumber >= start && lineNumber <= end) { - return range.GetProperty("commit").GetProperty("oid").GetString()!; + var oid = range.GetProperty("commit").GetProperty("oid").GetString(); + return oid ?? throw new InvalidOperationException("Commit OID was null."); } } throw new InvalidOperationException($"Line {lineNumber} not found in blame data."); } + + private async Task GetFileContentAtCommit(string repoUrl, string commit, string filePath) { (string owner, string repo) = ParseRepoUri(repoUrl); var file = await GetClient(repoUrl).Repository.Content.GetAllContentsByRef(owner, repo, filePath, commit); - return Encoding.UTF8.GetString(Convert.FromBase64String(file[0].Content)); + return file[0].Content; } } diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs index 244557f4b9..5e1360ca9c 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs @@ -177,9 +177,9 @@ protected async Task GetCodeflowHistoryCore(Guid id, bool fetchNe { var subscription = await _context.Subscriptions .Include(sub => sub.LastAppliedBuild) - .FirstOrDefaultAsync(sub => sub.Id == id); + .FirstOrDefaultAsync(sub => sub.Id == id && sub.SourceEnabled == true); - if (subscription == null || !subscription.SourceEnabled) + if (subscription == null) { return NotFound(); } @@ -190,12 +190,7 @@ protected async Task GetCodeflowHistoryCore(Guid id, bool fetchNe .Where(sub => sub.SourceRepository == subscription.TargetRepository || sub.TargetRepository == subscription.SourceRepository) - .FirstOrDefaultAsync(); - - if (oppositeDirectionSubscription?.SourceEnabled != true) - { - oppositeDirectionSubscription = null; - } + .FirstOrDefaultAsync(sub => sub.SourceEnabled == true); bool isForwardFlow = !string.IsNullOrEmpty(subscription.TargetDirectory); @@ -208,7 +203,7 @@ protected async Task GetCodeflowHistoryCore(Guid id, bool fetchNe ? await _codeflowHistoryManager.FetchLatestCodeflowHistoryAsync(oppositeDirectionSubscription) : []; - var lastCommit = subscription.LastAppliedBuild.Commit; + var lastCommit = subscription.LastAppliedBuild?.Commit; bool resultIsOutdated = IsCodeflowHistoryOutdated(subscription, cachedFlows) || IsCodeflowHistoryOutdated(oppositeDirectionSubscription, oppositeCachedFlows); diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/SubscriptionsController.cs b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/SubscriptionsController.cs index b732ba3bc8..a1e83ff639 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/SubscriptionsController.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/SubscriptionsController.cs @@ -150,7 +150,7 @@ public override async Task TriggerSubscription(Guid id, [FromQuer return await TriggerSubscriptionCore(id, buildId, force); } - [HttpPost("{id}/codeflowhistory")] + [HttpGet("{id}/codeflowhistory")] [SwaggerApiResponse(HttpStatusCode.Accepted, Type = typeof(Subscription), Description = "Subscription update has been triggered")] [ValidateModelState] public override async Task GetCodeflowHistory(Guid id) diff --git a/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs b/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs index 9a935cc062..ea586341f0 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs @@ -38,6 +38,7 @@ using ProductConstructionService.Api.Controllers; using Azure.Core; using Microsoft.DotNet.DarcLib.Helpers; +using ProductConstructionService.Common.CodeflowHistory; namespace ProductConstructionService.Api; @@ -138,6 +139,7 @@ internal static async Task ConfigurePcs( builder.Services.Configure(_ => { }); builder.Services.AddMemoryCache(); builder.Services.AddSingleton(builder.Configuration); + builder.Services.AddScoped(); // We do not use AddMemoryCache here. We use our own cache because we wish to // use a sized cache and some components, such as EFCore, do not implement their caching diff --git a/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistory/CodeflowHistoryManager.cs b/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistory/CodeflowHistoryManager.cs index 3a2ce3e63f..c1c8816b49 100644 --- a/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistory/CodeflowHistoryManager.cs +++ b/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistory/CodeflowHistoryManager.cs @@ -30,10 +30,10 @@ public record CodeflowGraphCommit( public class CodeflowHistoryManager( IRemoteFactory remoteFactory, - IConnectionMultiplexer connection) : ICodeflowHistoryManager + ConfigurationOptions options) : ICodeflowHistoryManager { private readonly IRemoteFactory _remoteFactory = remoteFactory; - private readonly IConnectionMultiplexer _connection = connection; + private readonly IConnectionMultiplexer _connection = ConnectionMultiplexer.Connect(options); private static RedisKey GetCodeflowGraphCommitKey(string id) => $"CodeflowGraphCommit_{id}"; private static RedisKey GetSortedSetKey(string id) => $"CodeflowHistory_{id}"; @@ -53,19 +53,22 @@ public async Task> GetCachedCodeflowHis var cache = _connection.GetDatabase(); var res = await cache.SortedSetRangeByRankWithScoresAsync( - key: subscriptionId, + key: GetSortedSetKey(subscriptionId), start: 0, stop: commitFetchCount - 1, order: Order.Descending); var commitKeys = res - .Select(e => new RedisKey(e.Element.ToString())) + .Select(e => new RedisKey(GetCodeflowGraphCommitKey(e.Element.ToString()))) .ToArray(); var commitValues = await cache.StringGetAsync(commitKeys); if (commitValues.Any(val => !val.HasValue)) + { + await ClearCodeflowCacheAsync(subscriptionId); throw new InvalidOperationException($"Corrupted commit data encountered for subscription `{subscriptionId}`."); + } return [.. commitValues .Select(commit => JsonSerializer.Deserialize(commit.ToString())) @@ -119,11 +122,11 @@ private async Task> EnrichCommitsWithCodeflowDat { Codeflow? lastFlow = !string.IsNullOrEmpty(subscription.TargetDirectory) ? await remote.GetLastIncomingForwardFlowAsync( - subscription.TargetBranch, + subscription.TargetRepository, subscription.TargetDirectory, current.Value.CommitSha) : await remote.GetLastIncomingBackflowAsync( - subscription.TargetBranch, + subscription.TargetRepository, current.Value.CommitSha); var target = current; @@ -157,10 +160,11 @@ private async Task CacheCommitsAsync( var cache = _connection.GetDatabase(); var sortedSetEntries = commits + .Reverse() .Select(c => new SortedSetEntry(c.CommitSha, latestCachedCommitScore++)) .ToArray(); - await cache.SortedSetAddAsync(subscriptionId, sortedSetEntries); + await cache.SortedSetAddAsync(GetSortedSetKey(subscriptionId), sortedSetEntries); var commitGraphEntries = commits .Select(c => new KeyValuePair( From b77dba50a4111943b108789482d8831158170688 Mon Sep 17 00:00:00 2001 From: Adam Zippor Date: Mon, 5 Jan 2026 09:47:32 +0100 Subject: [PATCH 15/19] Graphs frontend WIP --- .../Models/CodeflowHistoryResult.cs | 2 + .../Components/CodeflowHistoryGraph.razor | 193 ++++++++++++++++++ .../Components/SubscriptionDetailDialog.razor | 5 + 3 files changed, 200 insertions(+) create mode 100644 src/ProductConstructionService/ProductConstructionService.BarViz/Components/CodeflowHistoryGraph.razor diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Controllers/Models/CodeflowHistoryResult.cs b/src/ProductConstructionService/ProductConstructionService.Api/Controllers/Models/CodeflowHistoryResult.cs index f9dd2687ca..4e4a781dc2 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Controllers/Models/CodeflowHistoryResult.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Controllers/Models/CodeflowHistoryResult.cs @@ -9,5 +9,7 @@ public class CodeflowHistoryResult { public IReadOnlyCollection ForwardFlowHistory { get; set; } = []; public IReadOnlyCollection BackflowHistory { get; set; } = []; + public string RepoName { get; set; } = ""; + public string VmrName { get; set; } = ""; public bool ResultIsOutdated { get; set; } } diff --git a/src/ProductConstructionService/ProductConstructionService.BarViz/Components/CodeflowHistoryGraph.razor b/src/ProductConstructionService/ProductConstructionService.BarViz/Components/CodeflowHistoryGraph.razor new file mode 100644 index 0000000000..7bb98d0e06 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.BarViz/Components/CodeflowHistoryGraph.razor @@ -0,0 +1,193 @@ +@inherits ComponentBase + +

CodeflowHistoryGraph

+ + + + + + + + + + + + + + + + @for (int i = 0; i < leftColumn.Count; i++) + { + + @((MarkupString)$"{leftColumn[i]}") + + } + + + + + + @for (int i = 0; i < rightColumn.Count; i++) + { + + @((MarkupString)$"{rightColumn[i]}") + ; + } + + + + @foreach (var ff in ForwardFlows) + { + + } + + @foreach (var ff in Backflows) + { + + } + + +@code { + private const int BOX_HEIGHT = 50; + private const int BOX_WIDTH = 120; + private const int BOX_Y_MARGIN = 30; + private const int COL_SPACE = 500; + + private CodeflowHistory codeflowGraph = new CodeflowHistory(); + + private List leftColumn = ["abc123", "cbf123", "...[21 more \n commits]...", "xcv123", "zvn123"]; + private List rightColumn = ["wer865", "uwe765", "wet843", "wet845", "wry865", "iet854", "ery854"]; + + private List<(int, int)> ForwardFlows = [(1, 2), (4, 3)]; + private List<(int, int)> Backflows = [(6, 2)]; + + protected override void OnInitialized() + { + codeflowGraph = new CodeflowHistory(); + + } + public class CodeflowHistory + { + public IReadOnlyCollection ForwardFlowHistory { get; set; } = []; + public IReadOnlyCollection BackflowHistory { get; set; } = []; + public string RepoName { get; set; } = ""; + public string VmrName { get; set; } = ""; + public bool ResultIsOutdated { get; set; } + } + + private static CodeflowLayoutModel BuildLayout(CodeflowHistory history) + { + var leftCommits = history.ForwardFlowHistory + .Select((commit, index) => + new LayoutCommit( + commit.CommitSha, + commit.Author, + commit.Description, + RowIndex: index, + Column: GraphColumn.Left)) + .ToList(); + + var rightCommits = history.BackflowHistory + .Select((commit, index) => + new LayoutCommit( + commit.CommitSha, + commit.Author, + commit.Description, + RowIndex: index, + Column: GraphColumn.Right)) + .ToList(); + + var leftCommitBySha = leftCommits.ToDictionary(c => c.CommitSha); + var rightCommitBySha = rightCommits.ToDictionary(c => c.CommitSha); + + var leftToRightArrows = new List(); + var rightToLeftArrows = new List(); + + // Arrows originating from LEFT column + foreach (var commit in history.ForwardFlowHistory) + { + if (commit.SourceRepoFlowSha is null) + { + continue; + } + + if (rightCommitBySha.ContainsKey(commit.SourceRepoFlowSha)) + { + leftToRightArrows.Add(new LayoutArrow( + FromCommitSha: commit.SourceRepoFlowSha, + ToCommitSha: commit.CommitSha, + FromColumn: GraphColumn.Right, + ToColumn: GraphColumn.Left)); + } + } + + // Arrows originating from RIGHT column + foreach (var commit in history.BackflowHistory) + { + if (commit.SourceRepoFlowSha is null) + { + continue; + } + + if (leftCommitBySha.ContainsKey(commit.SourceRepoFlowSha)) + { + rightToLeftArrows.Add(new LayoutArrow( + FromCommitSha: commit.SourceRepoFlowSha, + ToCommitSha: commit.CommitSha, + FromColumn: GraphColumn.Left, + ToColumn: GraphColumn.Right)); + } + } + + return new CodeflowLayoutModel + { + LeftColumnName = history.RepoName, + RightColumnName = history.VmrName, + + LeftCommits = leftCommits, + RightCommits = rightCommits, + + LeftToRightArrows = leftToRightArrows, + RightToLeftArrows = rightToLeftArrows + }; + } + + public record CodeflowGraphCommit( + string CommitSha, + string Author, + string Description, + string? SourceRepoFlowSha); + + public enum GraphColumn + { + Left, + Right + } + + public record LayoutCommit( + string CommitSha, + string Author, + string Description, + int RowIndex, + GraphColumn Column); + + public record LayoutArrow( + string FromCommitSha, + string ToCommitSha, + GraphColumn FromColumn, + GraphColumn ToColumn); + + private class CodeflowLayoutModel + { + public string LeftColumnName { get; init; } = ""; + public string RightColumnName { get; init; } = ""; + + public IReadOnlyList LeftCommits { get; init; } = []; + public IReadOnlyList RightCommits { get; init; } = []; + + public IReadOnlyList LeftToRightArrows { get; init; } = []; + public IReadOnlyList RightToLeftArrows { get; init; } = []; + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.BarViz/Components/SubscriptionDetailDialog.razor b/src/ProductConstructionService/ProductConstructionService.BarViz/Components/SubscriptionDetailDialog.razor index e0b955617f..30474c77a3 100644 --- a/src/ProductConstructionService/ProductConstructionService.BarViz/Components/SubscriptionDetailDialog.razor +++ b/src/ProductConstructionService/ProductConstructionService.BarViz/Components/SubscriptionDetailDialog.razor @@ -137,6 +137,11 @@ + +
+ +
+ From 6b51f12d989578becd7fdb1b15303ac33fd1e7eb Mon Sep 17 00:00:00 2001 From: Adam Zippor Date: Sun, 11 Jan 2026 15:34:54 +0100 Subject: [PATCH 16/19] connect front and back --- .../Generated/Models/CodeflowHistory.cs | 67 ++++++ .../Generated/Subscriptions.cs | 75 +++++++ .../Controllers/SubscriptionsController.cs | 28 +-- .../Controllers/SubscriptionsController.cs | 8 +- .../Controllers/SubscriptionsController.cs | 5 +- .../Models/CodeflowHistoryResult.cs | 17 ++ .../appsettings.Development.json | 2 +- .../Components/CodeflowHistoryGraph.razor | 210 +++++++----------- .../Components/SubscriptionDetailDialog.razor | 2 +- 9 files changed, 258 insertions(+), 156 deletions(-) create mode 100644 src/ProductConstructionService/Microsoft.DotNet.ProductConstructionService.Client/Generated/Models/CodeflowHistory.cs diff --git a/src/ProductConstructionService/Microsoft.DotNet.ProductConstructionService.Client/Generated/Models/CodeflowHistory.cs b/src/ProductConstructionService/Microsoft.DotNet.ProductConstructionService.Client/Generated/Models/CodeflowHistory.cs new file mode 100644 index 0000000000..0ac85c1bd0 --- /dev/null +++ b/src/ProductConstructionService/Microsoft.DotNet.ProductConstructionService.Client/Generated/Models/CodeflowHistory.cs @@ -0,0 +1,67 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using Newtonsoft.Json; + +namespace Microsoft.DotNet.ProductConstructionService.Client.Models +{ + public partial class CodeflowHistory + { + public CodeflowHistory( + List forwardFlowHistory, + List backflowHistory, + string repoName, + string vmrName, + bool resultIsOutdated) + { + ForwardFlowHistory = forwardFlowHistory; + BackflowHistory = backflowHistory; + RepoName = repoName; + VmrName = vmrName; + ResultIsOutdated = resultIsOutdated; + } + + [JsonProperty("forwardFlowHistory")] + public List ForwardFlowHistory { get; } + + [JsonProperty("backflowHistory")] + public List BackflowHistory { get; } + + [JsonProperty("repoName")] + public string RepoName { get; } + + [JsonProperty("vmrName")] + public string VmrName { get; } + + [JsonProperty("resultIsOutdated")] + public bool ResultIsOutdated { get; } + } + + public partial class CodeflowGraphCommit + { + public CodeflowGraphCommit( + string commitSha, + string author, + string description, + string sourceRepoFlowSha) + { + CommitSha = commitSha; + Author = author; + Description = description; + SourceRepoFlowSha = sourceRepoFlowSha; + } + + [JsonProperty("commitSha")] + public string CommitSha { get; } + + [JsonProperty("author")] + public string Author { get; } + + [JsonProperty("description")] + public string Description { get; } + + [JsonProperty("sourceRepoFlowSha")] + public string SourceRepoFlowSha { get; } + } +} diff --git a/src/ProductConstructionService/Microsoft.DotNet.ProductConstructionService.Client/Generated/Subscriptions.cs b/src/ProductConstructionService/Microsoft.DotNet.ProductConstructionService.Client/Generated/Subscriptions.cs index 6dfe762d58..97169165a4 100644 --- a/src/ProductConstructionService/Microsoft.DotNet.ProductConstructionService.Client/Generated/Subscriptions.cs +++ b/src/ProductConstructionService/Microsoft.DotNet.ProductConstructionService.Client/Generated/Subscriptions.cs @@ -71,6 +71,11 @@ Task TriggerDailyUpdateAsync( CancellationToken cancellationToken = default ); + + Task GetCodeflowHistoryAsync( + Guid id, + CancellationToken cancellationToken = default + ); } internal partial class Subscriptions : IServiceOperations, ISubscriptions @@ -753,5 +758,75 @@ internal async Task OnGetSubscriptionHistoryFailed(Request req, Response res) Client.OnFailedRequest(ex); throw ex; } + + partial void HandleFailedGetCodeflowHistoryRequest(RestApiException ex); + + public async Task GetCodeflowHistoryAsync( + Guid id, + CancellationToken cancellationToken = default + ) + { + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/subscriptions/{id}/codeflowhistory".Replace("{id}", Uri.EscapeDataString(Client.Serialize(id))), + false); + + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Get; + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnGetCodeflowHistoryFailed(_req, _res).ConfigureAwait(false); + } + + if (_res.ContentStream == null) + { + await OnGetCodeflowHistoryFailed(_req, _res).ConfigureAwait(false); + } + + using (var _reader = new StreamReader(_res.ContentStream)) + { + var _content = await _reader.ReadToEndAsync().ConfigureAwait(false); + var _body = Client.Deserialize(_content); + return _body; + } + } + } + } + + internal async Task OnGetCodeflowHistoryFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedGetCodeflowHistoryRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } } } diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs index 5e1360ca9c..9aaa18dc53 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs @@ -18,6 +18,7 @@ using ProductConstructionService.WorkItems; using Channel = Maestro.Data.Models.Channel; using SubscriptionDAO = Maestro.Data.Models.Subscription; +using Microsoft.EntityFrameworkCore.Infrastructure; namespace ProductConstructionService.Api.Api.v2018_07_16.Controllers; @@ -112,15 +113,15 @@ public virtual async Task GetSubscription(Guid id) return Ok(new Subscription(subscription)); } - - [HttpPost("{id}/codeflowhistory")] - [SwaggerApiResponse(HttpStatusCode.Accepted, Type = typeof(Subscription), Description = "Subscription update has been triggered")] + /* + [HttpGet("{id}/codeflowhistory")] + [SwaggerApiResponse(HttpStatusCode.Accepted, Type = typeof(CodeflowHistoryResult), Description = "The codeflow history")] [ValidateModelState] public virtual async Task GetCodeflowHistory(Guid id) { return await GetCodeflowHistoryCore(id, false); } - + */ /// /// Trigger a manually by id /// @@ -208,16 +209,15 @@ protected async Task GetCodeflowHistoryCore(Guid id, bool fetchNe bool resultIsOutdated = IsCodeflowHistoryOutdated(subscription, cachedFlows) || IsCodeflowHistoryOutdated(oppositeDirectionSubscription, oppositeCachedFlows); - var result = new CodeflowHistoryResult - { - ResultIsOutdated = resultIsOutdated, - ForwardFlowHistory = isForwardFlow - ? cachedFlows - : oppositeCachedFlows, - BackflowHistory = isForwardFlow - ? oppositeCachedFlows - : cachedFlows, - }; + var forwardFlowHistory = isForwardFlow ? cachedFlows : oppositeCachedFlows; + var backflowHistory = isForwardFlow ? oppositeCachedFlows : cachedFlows; + + var result = new CodeflowHistoryResult( + forwardFlowHistory, + backflowHistory, + subscription.TargetDirectory ?? subscription.SourceDirectory, + "VMR", + resultIsOutdated); return Ok(result); } diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2019_01_16/Controllers/SubscriptionsController.cs b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2019_01_16/Controllers/SubscriptionsController.cs index d8eb1e2281..a50821de1a 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2019_01_16/Controllers/SubscriptionsController.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2019_01_16/Controllers/SubscriptionsController.cs @@ -10,6 +10,7 @@ using Microsoft.DotNet.DarcLib; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Options; +using ProductConstructionService.Api.Controllers.Models; using ProductConstructionService.Api.v2019_01_16.Models; using ProductConstructionService.Common.CodeflowHistory; using ProductConstructionService.WorkItems; @@ -96,14 +97,15 @@ public override async Task GetSubscription(Guid id) return Ok(new Subscription(subscription)); } - - [HttpPost("{id}/codeflowhistory")] - [SwaggerApiResponse(HttpStatusCode.Accepted, Type = typeof(Subscription), Description = "Subscription update has been triggered")] + /* + [HttpGet("{id}/codeflowhistory")] + [SwaggerApiResponse(HttpStatusCode.Accepted, Type = typeof(CodeflowHistoryResult), Description = "The codeflow history")] [ValidateModelState] public override async Task GetCodeflowHistory(Guid id) { return await GetCodeflowHistoryCore(id); } + */ /// /// Trigger a manually by id diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/SubscriptionsController.cs b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/SubscriptionsController.cs index a1e83ff639..7ee12b5dfe 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/SubscriptionsController.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/SubscriptionsController.cs @@ -14,6 +14,7 @@ using ProductConstructionService.Api.v2020_02_20.Models; using ProductConstructionService.Common.CodeflowHistory; using ProductConstructionService.WorkItems; +using ProductConstructionService.Api.Controllers.Models; namespace ProductConstructionService.Api.Api.v2020_02_20.Controllers; @@ -151,9 +152,9 @@ public override async Task TriggerSubscription(Guid id, [FromQuer } [HttpGet("{id}/codeflowhistory")] - [SwaggerApiResponse(HttpStatusCode.Accepted, Type = typeof(Subscription), Description = "Subscription update has been triggered")] + [SwaggerApiResponse(HttpStatusCode.Accepted, Type = typeof(Subscription), Description = "The codeflow history")] [ValidateModelState] - public override async Task GetCodeflowHistory(Guid id) + public async Task GetCodeflowHistory(Guid id) { return await GetCodeflowHistoryCore(id); } diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Controllers/Models/CodeflowHistoryResult.cs b/src/ProductConstructionService/ProductConstructionService.Api/Controllers/Models/CodeflowHistoryResult.cs index 4e4a781dc2..d7ccd783c9 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Controllers/Models/CodeflowHistoryResult.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Controllers/Models/CodeflowHistoryResult.cs @@ -5,8 +5,25 @@ namespace ProductConstructionService.Api.Controllers.Models; +/// +/// API model for codeflow history. +/// public class CodeflowHistoryResult { + public CodeflowHistoryResult( + IReadOnlyCollection forwardFlowHistory, + IReadOnlyCollection backflowHistory, + string repoName, + string vmrName, + bool resultIsOutdated) + { + ForwardFlowHistory = forwardFlowHistory; + BackflowHistory = backflowHistory; + RepoName = repoName; + VmrName = vmrName; + ResultIsOutdated = resultIsOutdated; + } + public IReadOnlyCollection ForwardFlowHistory { get; set; } = []; public IReadOnlyCollection BackflowHistory { get; set; } = []; public string RepoName { get; set; } = ""; diff --git a/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Development.json b/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Development.json index 63c3b0f7d6..a411992828 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Development.json +++ b/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Development.json @@ -28,7 +28,7 @@ "RedirectUri": "https://localhost:53180/signin-oidc" }, "ApiRedirect": { - // "Uri": "https://maestro.dot.net/" + // "Uri": "https://maestro.dot.net/" }, "EnvironmentNamespaceOptions": { "DefaultNamespaceName": "local" diff --git a/src/ProductConstructionService/ProductConstructionService.BarViz/Components/CodeflowHistoryGraph.razor b/src/ProductConstructionService/ProductConstructionService.BarViz/Components/CodeflowHistoryGraph.razor index 7bb98d0e06..4da937cf44 100644 --- a/src/ProductConstructionService/ProductConstructionService.BarViz/Components/CodeflowHistoryGraph.razor +++ b/src/ProductConstructionService/ProductConstructionService.BarViz/Components/CodeflowHistoryGraph.razor @@ -1,8 +1,19 @@ -@inherits ComponentBase +@using System.Linq.Expressions +@using System.Text +@using Microsoft.DotNet.ProductConstructionService.Client +@using Microsoft.DotNet.ProductConstructionService.Client.Models +@using Microsoft.FluentUI.AspNetCore.Components.Extensions +@using ProductConstructionService.BarViz.Code.Helpers +@using ProductConstructionService.BarViz.Components +@using TextCopy +@inject NavigationManager NavManager +@inject IProductConstructionServiceApi PcsApi + +@inherits ComponentBase

CodeflowHistoryGraph

- + @@ -15,179 +26,108 @@ - - @for (int i = 0; i < leftColumn.Count; i++) + @((MarkupString)$"{_leftColumnName}") + + @for (int i = 0; i < _leftColumn.Count; i++) { - @((MarkupString)$"{leftColumn[i]}") + @((MarkupString)$"{_leftColumn[i]}") } - - @for (int i = 0; i < rightColumn.Count; i++) + @((MarkupString)$"{_rightColumnName}") + + @for (int i = 0; i < _rightColumn.Count; i++) { - @((MarkupString)$"{rightColumn[i]}") + @((MarkupString)$"{_rightColumn[i]}") ; } - @foreach (var ff in ForwardFlows) + @foreach (var ff in _forwardFlows) { - } - @foreach (var ff in Backflows) + @foreach (var ff in _backflows) { - } @code { + [Parameter] + public Guid SubscriptionId { get; set; } = default!; + private const int BOX_HEIGHT = 50; private const int BOX_WIDTH = 120; private const int BOX_Y_MARGIN = 30; private const int COL_SPACE = 500; - private CodeflowHistory codeflowGraph = new CodeflowHistory(); - - private List leftColumn = ["abc123", "cbf123", "...[21 more \n commits]...", "xcv123", "zvn123"]; - private List rightColumn = ["wer865", "uwe765", "wet843", "wet845", "wry865", "iet854", "ery854"]; + private string _leftColumnName = "Repo"; + private string _rightColumnName = "VMR"; - private List<(int, int)> ForwardFlows = [(1, 2), (4, 3)]; - private List<(int, int)> Backflows = [(6, 2)]; + private List _leftColumn = ["abc123", "cbf123", "...[21 more \n commits]...", "xcv123", "zvn123"]; + private List _rightColumn = ["wer865", "uwe765", "wet843", "wet845", "wry865", "iet854", "ery854"]; - protected override void OnInitialized() - { - codeflowGraph = new CodeflowHistory(); + private List<(int, int)> _forwardFlows = [(1, 2), (4, 3)]; + private List<(int, int)> _backflows = [(6, 2)]; - } - public class CodeflowHistory + protected override async Task OnInitializedAsync() { - public IReadOnlyCollection ForwardFlowHistory { get; set; } = []; - public IReadOnlyCollection BackflowHistory { get; set; } = []; - public string RepoName { get; set; } = ""; - public string VmrName { get; set; } = ""; - public bool ResultIsOutdated { get; set; } + var codeflowHistory = await PcsApi.Subscriptions.GetCodeflowHistoryAsync(SubscriptionId); + //var codeflowHistory = new CodeflowHistory(_vmrCommits, _repoCommits, "SampleRepo", "SampleVMR", false); + BuildGraphVisualization(codeflowHistory); } - private static CodeflowLayoutModel BuildLayout(CodeflowHistory history) + public void BuildGraphVisualization(CodeflowHistory codeflowHistory) { - var leftCommits = history.ForwardFlowHistory - .Select((commit, index) => - new LayoutCommit( - commit.CommitSha, - commit.Author, - commit.Description, - RowIndex: index, - Column: GraphColumn.Left)) + _leftColumnName = codeflowHistory.RepoName; + _rightColumnName = codeflowHistory.VmrName; + + // 1. Columns + _leftColumn = codeflowHistory.BackflowHistory + .Where(c => !string.IsNullOrEmpty(c.CommitSha)) + .Select(c => c.CommitSha[..7]) + .ToList(); + + _rightColumn = codeflowHistory.ForwardFlowHistory + .Where(c => !string.IsNullOrEmpty(c.CommitSha)) + .Select(c => c.CommitSha[..7]) + .ToList(); + + HashSet codeflowCommits = codeflowHistory.BackflowHistory.Select(c => c.SourceRepoFlowSha) + .Concat(codeflowHistory.ForwardFlowHistory.Select(c => c.SourceRepoFlowSha)) + .Where(val => val is not null) + .ToHashSet(); + + // 2. SHA -> 0-based index maps + var leftMap = _leftColumn + .Select((sha, idx) => (sha, idx)) + .ToDictionary(t => t.sha, t => t.idx); + + var rightMap = _rightColumn + .Select((sha, idx) => (sha, idx)) + .ToDictionary(t => t.sha, t => t.idx); + + // 3. Build BF (vmr -> repo) + _backflows = codeflowHistory.BackflowHistory + .Where(c => !string.IsNullOrEmpty(c.SourceRepoFlowSha) && rightMap.ContainsKey(c.SourceRepoFlowSha)) + .Select(c => (leftMap[c.CommitSha], rightMap[c.SourceRepoFlowSha!])) .ToList(); - var rightCommits = history.BackflowHistory - .Select((commit, index) => - new LayoutCommit( - commit.CommitSha, - commit.Author, - commit.Description, - RowIndex: index, - Column: GraphColumn.Right)) + // 4. Build FF (repo -> vmr) + _forwardFlows = codeflowHistory.ForwardFlowHistory + .Where(c => !string.IsNullOrEmpty(c.SourceRepoFlowSha) && leftMap.ContainsKey(c.SourceRepoFlowSha)) + .Select(c => (rightMap[c.CommitSha], leftMap[c.SourceRepoFlowSha!])) .ToList(); - - var leftCommitBySha = leftCommits.ToDictionary(c => c.CommitSha); - var rightCommitBySha = rightCommits.ToDictionary(c => c.CommitSha); - - var leftToRightArrows = new List(); - var rightToLeftArrows = new List(); - - // Arrows originating from LEFT column - foreach (var commit in history.ForwardFlowHistory) - { - if (commit.SourceRepoFlowSha is null) - { - continue; - } - - if (rightCommitBySha.ContainsKey(commit.SourceRepoFlowSha)) - { - leftToRightArrows.Add(new LayoutArrow( - FromCommitSha: commit.SourceRepoFlowSha, - ToCommitSha: commit.CommitSha, - FromColumn: GraphColumn.Right, - ToColumn: GraphColumn.Left)); - } - } - - // Arrows originating from RIGHT column - foreach (var commit in history.BackflowHistory) - { - if (commit.SourceRepoFlowSha is null) - { - continue; - } - - if (leftCommitBySha.ContainsKey(commit.SourceRepoFlowSha)) - { - rightToLeftArrows.Add(new LayoutArrow( - FromCommitSha: commit.SourceRepoFlowSha, - ToCommitSha: commit.CommitSha, - FromColumn: GraphColumn.Left, - ToColumn: GraphColumn.Right)); - } - } - - return new CodeflowLayoutModel - { - LeftColumnName = history.RepoName, - RightColumnName = history.VmrName, - - LeftCommits = leftCommits, - RightCommits = rightCommits, - - LeftToRightArrows = leftToRightArrows, - RightToLeftArrows = rightToLeftArrows - }; - } - - public record CodeflowGraphCommit( - string CommitSha, - string Author, - string Description, - string? SourceRepoFlowSha); - - public enum GraphColumn - { - Left, - Right } - public record LayoutCommit( - string CommitSha, - string Author, - string Description, - int RowIndex, - GraphColumn Column); - - public record LayoutArrow( - string FromCommitSha, - string ToCommitSha, - GraphColumn FromColumn, - GraphColumn ToColumn); - - private class CodeflowLayoutModel - { - public string LeftColumnName { get; init; } = ""; - public string RightColumnName { get; init; } = ""; - - public IReadOnlyList LeftCommits { get; init; } = []; - public IReadOnlyList RightCommits { get; init; } = []; - - public IReadOnlyList LeftToRightArrows { get; init; } = []; - public IReadOnlyList RightToLeftArrows { get; init; } = []; - } } diff --git a/src/ProductConstructionService/ProductConstructionService.BarViz/Components/SubscriptionDetailDialog.razor b/src/ProductConstructionService/ProductConstructionService.BarViz/Components/SubscriptionDetailDialog.razor index 30474c77a3..ee2453d17f 100644 --- a/src/ProductConstructionService/ProductConstructionService.BarViz/Components/SubscriptionDetailDialog.razor +++ b/src/ProductConstructionService/ProductConstructionService.BarViz/Components/SubscriptionDetailDialog.razor @@ -139,7 +139,7 @@
- +
From e2943d4d116e3f78e0759e30b410a51be48f71f8 Mon Sep 17 00:00:00 2001 From: Adam Zippor Date: Mon, 12 Jan 2026 12:56:51 +0100 Subject: [PATCH 17/19] temp --- .../Controllers/SubscriptionsController.cs | 8 ++ .../Components/CodeflowHistoryGraph.razor | 119 +++++++++++++++--- .../Components/SubscriptionDetailDialog.razor | 97 +------------- 3 files changed, 112 insertions(+), 112 deletions(-) diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs index 9aaa18dc53..c765bb8a62 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs @@ -212,6 +212,14 @@ protected async Task GetCodeflowHistoryCore(Guid id, bool fetchNe var forwardFlowHistory = isForwardFlow ? cachedFlows : oppositeCachedFlows; var backflowHistory = isForwardFlow ? oppositeCachedFlows : cachedFlows; + forwardFlowHistory = forwardFlowHistory + .Select(commitGraph => commitGraph with { CommitSha = Commit.GetShortSha(commitGraph.CommitSha)}) + .ToList(); + + backflowHistory = backflowHistory + .Select(commitGraph => commitGraph with { CommitSha = Commit.GetShortSha(commitGraph.CommitSha) }) + .ToList(); + var result = new CodeflowHistoryResult( forwardFlowHistory, backflowHistory, diff --git a/src/ProductConstructionService/ProductConstructionService.BarViz/Components/CodeflowHistoryGraph.razor b/src/ProductConstructionService/ProductConstructionService.BarViz/Components/CodeflowHistoryGraph.razor index 4da937cf44..1aeb66279c 100644 --- a/src/ProductConstructionService/ProductConstructionService.BarViz/Components/CodeflowHistoryGraph.razor +++ b/src/ProductConstructionService/ProductConstructionService.BarViz/Components/CodeflowHistoryGraph.razor @@ -6,14 +6,13 @@ @using ProductConstructionService.BarViz.Code.Helpers @using ProductConstructionService.BarViz.Components @using TextCopy -@inject NavigationManager NavManager @inject IProductConstructionServiceApi PcsApi @inherits ComponentBase

CodeflowHistoryGraph

- + @@ -27,11 +26,11 @@ @((MarkupString)$"{_leftColumnName}") - + @for (int i = 0; i < _leftColumn.Count; i++) { - @((MarkupString)$"{_leftColumn[i]}") + @((MarkupString)$"{_leftColumn[i]}") } @@ -39,7 +38,7 @@ @((MarkupString)$"{_rightColumnName}") - + @for (int i = 0; i < _rightColumn.Count; i++) { @@ -82,8 +81,8 @@ protected override async Task OnInitializedAsync() { - var codeflowHistory = await PcsApi.Subscriptions.GetCodeflowHistoryAsync(SubscriptionId); - //var codeflowHistory = new CodeflowHistory(_vmrCommits, _repoCommits, "SampleRepo", "SampleVMR", false); + //var codeflowHistory = await PcsApi.Subscriptions.GetCodeflowHistoryAsync(SubscriptionId); + var codeflowHistory = new CodeflowHistory(_vmrCommits, _repoCommits, "SampleRepo", "SampleVMR", false); BuildGraphVisualization(codeflowHistory); } @@ -94,19 +93,24 @@ // 1. Columns _leftColumn = codeflowHistory.BackflowHistory - .Where(c => !string.IsNullOrEmpty(c.CommitSha)) - .Select(c => c.CommitSha[..7]) + .Select(c => c.CommitSha) .ToList(); _rightColumn = codeflowHistory.ForwardFlowHistory - .Where(c => !string.IsNullOrEmpty(c.CommitSha)) - .Select(c => c.CommitSha[..7]) + .Select(c => c.CommitSha) .ToList(); - HashSet codeflowCommits = codeflowHistory.BackflowHistory.Select(c => c.SourceRepoFlowSha) - .Concat(codeflowHistory.ForwardFlowHistory.Select(c => c.SourceRepoFlowSha)) - .Where(val => val is not null) - .ToHashSet(); + HashSet codeflowCommits = codeflowHistory.BackflowHistory + .Where(c => !string.IsNullOrEmpty(c.SourceRepoFlowSha)) + .SelectMany(c => new[] { c.SourceRepoFlowSha, c.CommitSha }) + .Concat(codeflowHistory.ForwardFlowHistory + .Where(c => !string.IsNullOrEmpty(c.SourceRepoFlowSha)) + .SelectMany(c => new[] { c.SourceRepoFlowSha, c.CommitSha })) + .Where(val => val is not null) + .ToHashSet(); + + _leftColumn = CompactifyCommitsColumn(_leftColumn, codeflowCommits); + _rightColumn = CompactifyCommitsColumn(_rightColumn, codeflowCommits); // 2. SHA -> 0-based index maps var leftMap = _leftColumn @@ -120,14 +124,95 @@ // 3. Build BF (vmr -> repo) _backflows = codeflowHistory.BackflowHistory .Where(c => !string.IsNullOrEmpty(c.SourceRepoFlowSha) && rightMap.ContainsKey(c.SourceRepoFlowSha)) - .Select(c => (leftMap[c.CommitSha], rightMap[c.SourceRepoFlowSha!])) + .Select(c => (rightMap[c.SourceRepoFlowSha!], leftMap[c.CommitSha])) .ToList(); // 4. Build FF (repo -> vmr) _forwardFlows = codeflowHistory.ForwardFlowHistory .Where(c => !string.IsNullOrEmpty(c.SourceRepoFlowSha) && leftMap.ContainsKey(c.SourceRepoFlowSha)) - .Select(c => (rightMap[c.CommitSha], leftMap[c.SourceRepoFlowSha!])) + .Select(c => (leftMap[c.SourceRepoFlowSha!], rightMap[c.CommitSha])) .ToList(); + } + public static List CompactifyCommitsColumn(List commits, HashSet codeflowCommits) + { + string CommitsHiddenString(string commitA, string commitB, int num) => $"({compactedCommits} commits hidden)"; + // $"[{firstCompactedCommit}]...[{lastCompactedCommit}]\n({compactedCommits} commits hidden)" + string firstCompactedCommit = ""; + string lastCompactedCommit = ""; + int compactedCommits = 0; + List result = []; + + for (int i = 0; i < commits.Count; i++) + { + var a = Math.Max(0, i - 1); + var b = Math.Max(0, i - 1); + var c = Math.Min(commits.Count - 1, i + 1); + var d = Math.Min(commits.Count - 1, i + 1); + + // check if there's no codeflows +/- 2 commits around current commit + if (!codeflowCommits.Contains(commits[i]) && + !codeflowCommits.Contains(commits[a]) && + !codeflowCommits.Contains(commits[b]) && + !codeflowCommits.Contains(commits[c]) && + !codeflowCommits.Contains(commits[d])) + { + if (compactedCommits == 0) + { + firstCompactedCommit = commits[i]; + } + compactedCommits++; + lastCompactedCommit = commits[i]; + } + else + { + if (compactedCommits == 1) + { + result.Add(firstCompactedCommit); + compactedCommits = 0; + firstCompactedCommit = ""; + } + else if (compactedCommits > 0) + { + result.Add(CommitsHiddenString(firstCompactedCommit, lastCompactedCommit, compactedCommits)); + compactedCommits = 0; + firstCompactedCommit = ""; + } + result.Add(commits[i]); + } + } + if (compactedCommits > 0) + { + result.Add(CommitsHiddenString(firstCompactedCommit, lastCompactedCommit, compactedCommits)); + } + return result; + } + + private List _repoCommits = new List + { + new("a1b2c3", "Alice", "Fix login bug", null), + new("d4e5f6", "Bob", "Add unit tests", "w1x2y3"), + new("g7h8i9", "Carol", "Refactor auth module", null), + new("dontcompactmeyet", "Carol", "Refactor auth module", null), + new("compactMePlease", "Carol", "Refactor auth module", null), + new("metoomenotsocompacted", "Carol", "Refactor auth module", null), + new("imnotcompacted:)", "Carol", "Refactor auth module", null), + new("j1k2l3", "Dave", "Update README", null), + new("m4n5o6", "Eve", "Optimize query", null), + new("p7q8r9", "Frank", "Fix typo", "w4x5y6"), + new("u6i7o7", "Grace", "Remove unused code", null), + new("b3n5n7", "Grace", "Remove unused code", null), + }; + + private List _vmrCommits = new List + { + new("w1x2y3", "Henry", "Merge feature branch", null), + new("w4x5y6", "Ivy", "Hotfix production bug", null), + new("w7x8y9", "Jack", "Update docs", "a1b2c3"), + new("w0x1y2", "Kate", "Add CI pipeline", null), + new("w3x4y5", "Leo", "Refactor build scripts", "g7h8i9"), + new("w6x7y8", "Mona", "Improve logging", null), + new("w9x0y1", "Nina", "Update dependencies", null), + }; } diff --git a/src/ProductConstructionService/ProductConstructionService.BarViz/Components/SubscriptionDetailDialog.razor b/src/ProductConstructionService/ProductConstructionService.BarViz/Components/SubscriptionDetailDialog.razor index ee2453d17f..acc058b6a5 100644 --- a/src/ProductConstructionService/ProductConstructionService.BarViz/Components/SubscriptionDetailDialog.razor +++ b/src/ProductConstructionService/ProductConstructionService.BarViz/Components/SubscriptionDetailDialog.razor @@ -43,102 +43,9 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @if (!string.IsNullOrEmpty(Content.TargetDirectory)) - { - - - - - } - @if (Content.SourceEnabled && !string.IsNullOrEmpty(Content.SourceDirectory)) - { - - - - - } - - - - - - - - - -
Id@Content.Id
Channel@Content.Channel.Name
Source repository@Content.SourceRepository
Target repository@Content.TargetRepository
Target branch@Content.TargetBranch
Update frequency@Content.Policy.UpdateFrequency
Enabled@(Content.Enabled ? "Yes" : "No")
Batchable@(Content.Policy.Batchable ? "Yes" : "No")
Merge policies - @if (Content.Policy.MergePolicies == null || Content.Policy.MergePolicies.Count == 0) - { - None - } - else - { -
    - @foreach (var policy in Content.Policy.MergePolicies) - { -
  • @policy.Name
    • - } -
- } -
Source-enabled@(Content.SourceEnabled ? (string.IsNullOrEmpty(Content.SourceDirectory) ? "Yes (Forward flow)" : "Yes (Backflow)") : "No")
Target directory@Content.TargetDirectory
Source directory@Content.SourceDirectory
ExcludedAssets - @if (Content.ExcludedAssets == null || Content.ExcludedAssets.Count == 0) - { - None - } - else - { -
    - @foreach (var asset in Content.ExcludedAssets) - { -
  • @asset
    • - } -
- } -
PR notification tags@Content.PullRequestFailureNotificationTags
-
+ +
From 709430669df2f2dd143b1d6dc5a29d7aa6051c61 Mon Sep 17 00:00:00 2001 From: Adam Zippor Date: Mon, 12 Jan 2026 18:24:35 +0100 Subject: [PATCH 18/19] controller bugfix + return short SHAs --- .../Controllers/SubscriptionsController.cs | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs index c765bb8a62..f11660651e 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs @@ -189,8 +189,8 @@ protected async Task GetCodeflowHistoryCore(Guid id, bool fetchNe .Include(sub => sub.LastAppliedBuild) .Include(sub => sub.Channel) .Where(sub => - sub.SourceRepository == subscription.TargetRepository || - sub.TargetRepository == subscription.SourceRepository) + sub.SourceRepository == subscription.TargetRepository + && sub.TargetRepository == subscription.SourceRepository) .FirstOrDefaultAsync(sub => sub.SourceEnabled == true); bool isForwardFlow = !string.IsNullOrEmpty(subscription.TargetDirectory); @@ -213,17 +213,29 @@ protected async Task GetCodeflowHistoryCore(Guid id, bool fetchNe var backflowHistory = isForwardFlow ? oppositeCachedFlows : cachedFlows; forwardFlowHistory = forwardFlowHistory - .Select(commitGraph => commitGraph with { CommitSha = Commit.GetShortSha(commitGraph.CommitSha)}) - .ToList(); + .Select(commitGraph => commitGraph with + { + CommitSha = Commit.GetShortSha(commitGraph.CommitSha), + SourceRepoFlowSha = !string.IsNullOrEmpty(commitGraph.SourceRepoFlowSha) + ? Commit.GetShortSha(commitGraph.SourceRepoFlowSha) + : "" + }).ToList(); backflowHistory = backflowHistory - .Select(commitGraph => commitGraph with { CommitSha = Commit.GetShortSha(commitGraph.CommitSha) }) + .Select(commitGraph => commitGraph with { + CommitSha = Commit.GetShortSha(commitGraph.CommitSha), + SourceRepoFlowSha = !string.IsNullOrEmpty(commitGraph.SourceRepoFlowSha) + ? Commit.GetShortSha(commitGraph.SourceRepoFlowSha) + : "" + }) .ToList(); var result = new CodeflowHistoryResult( forwardFlowHistory, backflowHistory, - subscription.TargetDirectory ?? subscription.SourceDirectory, + string.IsNullOrEmpty(subscription.TargetDirectory) + ? subscription.SourceDirectory + : subscription.TargetBranch, "VMR", resultIsOutdated); From ede1dd1975e09489954fce613594654ea43bce10 Mon Sep 17 00:00:00 2001 From: Adam Zippor Date: Mon, 12 Jan 2026 18:24:50 +0100 Subject: [PATCH 19/19] improve graph layout --- .../Components/CodeflowHistoryGraph.razor | 67 +++++++++++++------ 1 file changed, 47 insertions(+), 20 deletions(-) diff --git a/src/ProductConstructionService/ProductConstructionService.BarViz/Components/CodeflowHistoryGraph.razor b/src/ProductConstructionService/ProductConstructionService.BarViz/Components/CodeflowHistoryGraph.razor index 1aeb66279c..a0a63a9b96 100644 --- a/src/ProductConstructionService/ProductConstructionService.BarViz/Components/CodeflowHistoryGraph.razor +++ b/src/ProductConstructionService/ProductConstructionService.BarViz/Components/CodeflowHistoryGraph.razor @@ -21,29 +21,59 @@ + @((MarkupString)$"{_leftColumnName}") - + @for (int i = 0; i < _leftColumn.Count; i++) { - - @((MarkupString)$"{_leftColumn[i]}") + if (!_leftColumn[i].Contains("hidden")) + { + + @((MarkupString)$"{_leftColumn[i]}") + ; + } + else + { + + var firstLine = _leftColumn[i].Split('(')[0]; + var secondLine = "(" + _leftColumn[i].Split('(')[1]; + @((MarkupString)$"{firstLine}") + ; + @((MarkupString)$"{secondLine}") + ; + } } @((MarkupString)$"{_rightColumnName}") - + @for (int i = 0; i < _rightColumn.Count; i++) { - - @((MarkupString)$"{_rightColumn[i]}") - ; + if (!_rightColumn[i].Contains("hidden")) + { + + @((MarkupString)$"{_rightColumn[i]}") + ; + + } + else + { + + var firstLine = _rightColumn[i].Split('(')[0]; + var secondLine = "(" + _rightColumn[i].Split('(')[1]; + @((MarkupString)$"{firstLine}") + ; + @((MarkupString)$"{secondLine}") + ; + + } } @@ -65,8 +95,9 @@ [Parameter] public Guid SubscriptionId { get; set; } = default!; - private const int BOX_HEIGHT = 50; + private const int BOX_HEIGHT = 55; private const int BOX_WIDTH = 120; + private const int BOX_WIDTH_2 = 160; private const int BOX_Y_MARGIN = 30; private const int COL_SPACE = 500; @@ -81,8 +112,8 @@ protected override async Task OnInitializedAsync() { - //var codeflowHistory = await PcsApi.Subscriptions.GetCodeflowHistoryAsync(SubscriptionId); - var codeflowHistory = new CodeflowHistory(_vmrCommits, _repoCommits, "SampleRepo", "SampleVMR", false); + var codeflowHistory = await PcsApi.Subscriptions.GetCodeflowHistoryAsync(SubscriptionId); + // var codeflowHistory = new CodeflowHistory(_vmrCommits, _repoCommits, "SampleRepo", "SampleVMR", false); BuildGraphVisualization(codeflowHistory); } @@ -137,7 +168,7 @@ public static List CompactifyCommitsColumn(List commits, HashSet codeflowCommits) { - string CommitsHiddenString(string commitA, string commitB, int num) => $"({compactedCommits} commits hidden)"; + string CommitsHiddenString(string commitA, string commitB, int num) => $"{commitA}...{commitB} ({num} commits hidden)"; // $"[{firstCompactedCommit}]...[{lastCompactedCommit}]\n({compactedCommits} commits hidden)" string firstCompactedCommit = ""; string lastCompactedCommit = ""; @@ -152,11 +183,7 @@ var d = Math.Min(commits.Count - 1, i + 1); // check if there's no codeflows +/- 2 commits around current commit - if (!codeflowCommits.Contains(commits[i]) && - !codeflowCommits.Contains(commits[a]) && - !codeflowCommits.Contains(commits[b]) && - !codeflowCommits.Contains(commits[c]) && - !codeflowCommits.Contains(commits[d])) + if (!codeflowCommits.Contains(commits[i])) { if (compactedCommits == 0) { @@ -194,10 +221,10 @@ new("a1b2c3", "Alice", "Fix login bug", null), new("d4e5f6", "Bob", "Add unit tests", "w1x2y3"), new("g7h8i9", "Carol", "Refactor auth module", null), - new("dontcompactmeyet", "Carol", "Refactor auth module", null), - new("compactMePlease", "Carol", "Refactor auth module", null), - new("metoomenotsocompacted", "Carol", "Refactor auth module", null), - new("imnotcompacted:)", "Carol", "Refactor auth module", null), + new("k2d9n0", "Carol", "Refactor auth module", null), + new("sh7d66e", "Carol", "Refactor auth module", null), + new("uej367", "Carol", "Refactor auth module", null), + new("xj39tt)", "Carol", "Refactor auth module", null), new("j1k2l3", "Dave", "Update README", null), new("m4n5o6", "Eve", "Optimize query", null), new("p7q8r9", "Frank", "Fix typo", "w4x5y6"),