diff --git a/src/Maestro/Maestro.DataProviders/RemoteFactory.cs b/src/Maestro/Maestro.DataProviders/RemoteFactory.cs index a9f74a9517..ee81e1e5ab 100644 --- a/src/Maestro/Maestro.DataProviders/RemoteFactory.cs +++ b/src/Maestro/Maestro.DataProviders/RemoteFactory.cs @@ -25,6 +25,7 @@ public class RemoteFactory : IRemoteFactory private readonly IGitHubTokenProvider _gitHubTokenProvider; private readonly IAzureDevOpsTokenProvider _azdoTokenProvider; private readonly IServiceProvider _serviceProvider; + private readonly IVersionDetailsParser _versionDetailsParser; public RemoteFactory( BuildAssetRegistryContext context, @@ -45,6 +46,7 @@ public RemoteFactory( _azdoTokenProvider = azdoTokenProvider; _cache = memoryCache; _serviceProvider = serviceProvider; + _versionDetailsParser = versionDetailsParser; } public async Task CreateRemoteAsync(string repoUrl) @@ -87,6 +89,7 @@ private async Task GetRemoteGitClient(string repoUrl) : new GitHubClient( new Microsoft.DotNet.DarcLib.GitHubTokenProvider(_gitHubTokenProvider), _processManager, + _versionDetailsParser, _loggerFactory.CreateLogger(), _cache.Cache), diff --git a/src/Microsoft.DotNet.Darc/Darc/Helpers/RemoteFactory.cs b/src/Microsoft.DotNet.Darc/Darc/Helpers/RemoteFactory.cs index e82288b051..5853e2dd6e 100644 --- a/src/Microsoft.DotNet.Darc/Darc/Helpers/RemoteFactory.cs +++ b/src/Microsoft.DotNet.Darc/Darc/Helpers/RemoteFactory.cs @@ -21,12 +21,14 @@ internal class RemoteFactory : IRemoteFactory private readonly IServiceProvider _serviceProvider; private readonly IAzureDevOpsTokenProvider _azdoTokenProvider; private readonly IRemoteTokenProvider _githubTokenProvider; + private readonly IVersionDetailsParser _versionDetailsParser; public RemoteFactory( IAzureDevOpsTokenProvider azdoTokenProvider, [FromKeyedServices("github")]IRemoteTokenProvider githubTokenProvider, ILoggerFactory loggerFactory, ICommandLineOptions options, + IVersionDetailsParser versionDetailsParser, IServiceProvider serviceProvider) { _loggerFactory = loggerFactory; @@ -34,6 +36,7 @@ public RemoteFactory( _serviceProvider = serviceProvider; _azdoTokenProvider = azdoTokenProvider; _githubTokenProvider = githubTokenProvider; + _versionDetailsParser = versionDetailsParser; } public Task CreateRemoteAsync(string repoUrl) @@ -62,6 +65,7 @@ private IRemoteGitRepo CreateRemoteGitClient(ICommandLineOptions options, string new ProcessManager(_loggerFactory.CreateLogger(), options.GitLocation), _loggerFactory.CreateLogger(), temporaryRepositoryRoot, + _versionDetailsParser, // Caching not in use for Darc local client. null), diff --git a/src/Microsoft.DotNet.Darc/DarcLib/AzureDevOpsClient.cs b/src/Microsoft.DotNet.Darc/DarcLib/AzureDevOpsClient.cs index 181b3bbeea..a4dff27d70 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/AzureDevOpsClient.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/AzureDevOpsClient.cs @@ -16,6 +16,7 @@ using Microsoft.DotNet.DarcLib.Helpers; using Microsoft.DotNet.DarcLib.Models; using Microsoft.DotNet.DarcLib.Models.AzureDevOps; +using Microsoft.DotNet.DarcLib.Models.VirtualMonoRepo; using Microsoft.DotNet.Services.Utility; using Microsoft.Extensions.Logging; using Microsoft.TeamFoundation.SourceControl.WebApi; @@ -1944,4 +1945,20 @@ public async Task> GetGitTreeNames(string path, stri UpdatedAt = DateTimeOffset.UtcNow, HeadBranchSha = pr.LastMergeSourceCommit.CommitId, }; + + public Task> FetchLatestRepoCommits(string repoUrl, string branch, int maxCount) + => throw new NotImplementedException(); + + public Task> FetchLatestFetchNewerRepoCommitsAsyncRepoCommits( + string repoUrl, + string branch, + string commitSha, + int maxCount) + => throw new NotImplementedException(); + + public Task> FetchLatestRepoCommitsAsync(string repoUrl, string branch, int maxCount) => throw new NotImplementedException(); + public Task> FetchNewerRepoCommitsAsync(string repoUrl, string branch, string commitSha, int maxCount) => throw new NotImplementedException(); + public Task GetLastIncomingForwardFlowAsync(string vmrUrl, string commit) => throw new NotImplementedException(); + public Task GetLastIncomingBackflowAsync(string repoUrl, string commit) => throw new NotImplementedException(); + public Task GetLastIncomingForwardFlowAsync(string vmrUrl, string mappingName, string commit) => throw new NotImplementedException(); } diff --git a/src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs b/src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs index 7337d9a870..715b9fdffd 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs @@ -10,6 +10,7 @@ using System.Net.Http; using System.Reflection; using System.Text; +using System.Text.Json; using System.Text.RegularExpressions; using System.Threading.Tasks; using Maestro.Common; @@ -17,6 +18,8 @@ using Microsoft.DotNet.DarcLib.Helpers; using Microsoft.DotNet.DarcLib.Models; using Microsoft.DotNet.DarcLib.Models.GitHub; +using Microsoft.DotNet.DarcLib.Models.VirtualMonoRepo; +using Microsoft.DotNet.DarcLib.VirtualMonoRepo; using Microsoft.DotNet.Services.Utility; using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; @@ -48,6 +51,7 @@ public class GitHubClient : RemoteRepoBase, IRemoteGitRepo private readonly string _userAgent = $"DarcLib-{DarcLibVersion}"; private IGitHubClient? _lazyClient = null; private readonly Dictionary<(string, string, string?), string> _gitRefCommitCache; + private readonly IVersionDetailsParser _versionDetailsParser; static GitHubClient() { @@ -60,9 +64,10 @@ static GitHubClient() public GitHubClient( IRemoteTokenProvider remoteTokenProvider, IProcessManager processManager, + IVersionDetailsParser versionDetailsParser, ILogger logger, IMemoryCache? cache) - : this(remoteTokenProvider, processManager, logger, null, cache) + : this(remoteTokenProvider, processManager, logger, null, versionDetailsParser, cache) { } @@ -71,6 +76,7 @@ public GitHubClient( IProcessManager processManager, ILogger logger, string? temporaryRepositoryPath, + IVersionDetailsParser versionDetailsParser, IMemoryCache? cache) : base(remoteTokenProvider, processManager, temporaryRepositoryPath, cache, logger) { @@ -82,6 +88,7 @@ public GitHubClient( NullValueHandling = NullValueHandling.Ignore }; _gitRefCommitCache = []; + _versionDetailsParser = versionDetailsParser; } public bool AllowRetries { get; set; } = true; @@ -1491,4 +1498,210 @@ private static PullRequest ToDarcLibPullRequest(Octokit.PullRequest pr) HeadBranchSha = pr.Head.Sha, }; } + + public Task> FetchLatestRepoCommitsAsync(string repoUrl, string branch, int maxCount) + => throw new NotImplementedException(); + + public async Task> FetchNewerRepoCommitsAsync( + string repoUrl, + string branch, + string commitSha, + int maxCount) + { + if (maxCount <= 0) + { + maxCount = 100; + } + + (string owner, string repo) = ParseRepoUri(repoUrl); + + var request = new CommitRequest + { + Sha = branch, + }; + + var options = new ApiOptions + { + PageSize = maxCount, + PageCount = 1, + StartPage = 1 + }; + + var allCommits = new List(); + + while (allCommits.Count < maxCount) + { + var commits = await GetClient(owner, repo) + .Repository + .Commit + .GetAll(owner, repo, request, options); + + foreach (Octokit.GitHubCommit c in commits) + { + var convertedCommit = new Commit( + c.Author?.Login, + c.Sha, + c.Commit.Message); + + allCommits.Add(convertedCommit); + + if (convertedCommit.Sha == commitSha) + { + break; + } + } + + if (commits.Count < options.PageSize) + { + break; + } + + options.StartPage++; + } + + return [.. allCommits.Take(maxCount)]; + } + + public async Task GetLastIncomingForwardFlowAsync(string vmrUrl, string mappingName, string commit) + { + var content = await GetFileContentAtCommit( + vmrUrl, + commit, + VmrInfo.DefaultRelativeSourceManifestPath); + + var lastForwardFlowRepoSha = SourceManifest + .FromJson(content)? + .GetRepoVersion(mappingName) + .CommitSha; + + if (lastForwardFlowRepoSha == null) + { + return null; + } + + int lineNumber = content.Split('\n') + .ToList() + .FindIndex(line => line.Contains(lastForwardFlowRepoSha)) + + 1; // result is 0-indexed, but github lines are 1-indexed; + + // todo: we can skip this call if the last flown SHA is one that we already cached + string lastForwardFlowVmrSha = await BlameLineAsync( + vmrUrl, + commit, + VmrInfo.DefaultRelativeSourceManifestPath, + lineNumber); + + return new ForwardFlow(lastForwardFlowRepoSha, lastForwardFlowVmrSha); + } + + public async Task GetLastIncomingBackflowAsync(string repoUrl, string commit) + { + var content = await GetFileContentAtCommit( + repoUrl, + commit, + VersionFiles.VersionDetailsXml); + + var lastBackflowVmrSha = _versionDetailsParser.ParseVersionDetailsXml(content) + .Source? + .Sha; + + if (lastBackflowVmrSha == null) + { + return null; + } + + int lineNumber = content + .Split('\n') + .ToList() + .FindIndex(line => + line.Contains(VersionDetailsParser.SourceElementName) && + line.Contains(lastBackflowVmrSha)) + + 1; // result is 0-indexed, but github lines are 1-indexed + + // todo: we can skip this call if the last flown SHA is one that we already cached + string lastBackflowRepoSha = await BlameLineAsync( + repoUrl, + commit, + VersionFiles.VersionDetailsXml, + lineNumber); + + return new Backflow(lastBackflowVmrSha, lastBackflowRepoSha); + } + + private async Task BlameLineAsync(string repoUrl, string commitOrBranch, string filePath, int lineNumber) + { + (string owner, string repo) = ParseRepoUri(repoUrl); + + var query = $@" + query {{ + repository(owner: {JsonConvert.SerializeObject(owner)}, name: {JsonConvert.SerializeObject(repo)}) {{ + object(expression: {JsonConvert.SerializeObject(commitOrBranch)}) {{ + ... on Commit {{ + blame(path: {JsonConvert.SerializeObject(filePath)}) {{ + ranges {{ + startingLine + endingLine + commit {{ + oid + }} + }} + }} + }} + }} + }} + }}"; + + using var client = CreateHttpClient(repoUrl); + + var requestBody = new { query }; + var content = new StringContent( + JsonConvert.SerializeObject(requestBody, _serializerSettings), + Encoding.UTF8, + "application/json" + ); + + var response = await client.PostAsync("graphql", content); + response.EnsureSuccessStatusCode(); + + using var stream = await response.Content.ReadAsStreamAsync(); + using var doc = await JsonDocument.ParseAsync(stream); + + if (doc.RootElement.TryGetProperty("errors", out var errors)) + throw new InvalidOperationException($"GitHub GraphQL error: {errors}"); + + var obj = doc.RootElement + .GetProperty("data") + .GetProperty("repository") + .GetProperty("object"); + + if (obj.ValueKind == JsonValueKind.Null) + throw new InvalidOperationException($"Commit or branch '{commitOrBranch}' not found."); + + // The blame field comes directly from the Commit node + var blame = obj.GetProperty("blame"); + var ranges = blame.GetProperty("ranges").EnumerateArray(); + + foreach (var range in ranges) + { + int start = range.GetProperty("startingLine").GetInt32(); + int end = range.GetProperty("endingLine").GetInt32(); + + if (lineNumber >= start && lineNumber <= end) + { + var oid = range.GetProperty("commit").GetProperty("oid").GetString(); + return oid ?? throw new InvalidOperationException("Commit OID was null."); + } + } + + throw new InvalidOperationException($"Line {lineNumber} not found in blame data."); + } + + + + private async Task GetFileContentAtCommit(string repoUrl, string commit, string filePath) + { + (string owner, string repo) = ParseRepoUri(repoUrl); + var file = await GetClient(repoUrl).Repository.Content.GetAllContentsByRef(owner, repo, filePath, commit); + return file[0].Content; + } } diff --git a/src/Microsoft.DotNet.Darc/DarcLib/GitRepoFactory.cs b/src/Microsoft.DotNet.Darc/DarcLib/GitRepoFactory.cs index e827178929..6f0ae0b199 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/GitRepoFactory.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/GitRepoFactory.cs @@ -22,6 +22,7 @@ public class GitRepoFactory : IGitRepoFactory private readonly ITelemetryRecorder _telemetryRecorder; private readonly IProcessManager _processManager; private readonly IFileSystem _fileSystem; + private readonly IVersionDetailsParser _versionDetailsParser; private readonly ILoggerFactory _loggerFactory; private readonly string? _temporaryPath = null; @@ -31,6 +32,7 @@ public GitRepoFactory( ITelemetryRecorder telemetryRecorder, IProcessManager processManager, IFileSystem fileSystem, + IVersionDetailsParser versionDetailsParser, ILoggerFactory loggerFactory, string temporaryPath) { @@ -39,6 +41,7 @@ public GitRepoFactory( _telemetryRecorder = telemetryRecorder; _processManager = processManager; _fileSystem = fileSystem; + _versionDetailsParser = versionDetailsParser; _loggerFactory = loggerFactory; _temporaryPath = temporaryPath; } @@ -56,6 +59,7 @@ public GitRepoFactory( _processManager, _loggerFactory.CreateLogger(), _temporaryPath, + _versionDetailsParser, // Caching not in use for Darc local client. null), diff --git a/src/Microsoft.DotNet.Darc/DarcLib/IRemote.cs b/src/Microsoft.DotNet.Darc/DarcLib/IRemote.cs index ef94240fa3..54a32e0346 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/IRemote.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/IRemote.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System; using System.Collections.Generic; using System.Threading.Tasks; using Maestro.MergePolicyEvaluation; @@ -260,5 +261,31 @@ Task CommitUpdatesWithNoCloningAsync( /// Task> GetGitTreeNames(string path, string repoUri, string branch); - #endregion + /// + /// Fetches the latest commits from a repository branch, up to maxCount + /// + /// Full url of the git repository + /// Name of the git branch in the repo + /// Maximum count of commits to fetch + /// List of commits + Task> FetchLatestRepoCommitsAsync(string repoUrl, string branch, int maxCount = 100); + + + /// + /// Fetches the latest commits from a repository branch that are newer than the specified + /// commit, fetching up to maxCount commits in total. + /// + /// Full url of the git repository + /// Name of the git branch in the repo + /// Sha of the commit to fetch newer commits than + /// Maximum count of commits to fetch + /// List of commits + Task> FetchNewerRepoCommitsAsync(string repoUrl, string branch, string commitSha, int maxCount = 100); + + + Task GetLastIncomingForwardFlowAsync(string vmrUrl, string mappingNAme, string commit); + + Task GetLastIncomingBackflowAsync(string repoUrl, string commit); + + #endregion } diff --git a/src/Microsoft.DotNet.Darc/DarcLib/IRemoteGitRepo.cs b/src/Microsoft.DotNet.Darc/DarcLib/IRemoteGitRepo.cs index 5d5eb6c976..c32a1c13ce 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/IRemoteGitRepo.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/IRemoteGitRepo.cs @@ -4,6 +4,7 @@ using Maestro.MergePolicyEvaluation; using Microsoft.DotNet.DarcLib.Helpers; using Microsoft.DotNet.DarcLib.Models; +using Microsoft.DotNet.DarcLib.Models.VirtualMonoRepo; using System; using System.Collections.Generic; using System.Threading.Tasks; @@ -167,6 +168,27 @@ Task> SearchPullRequestsAsync( /// Returns a list of tree names (directories) under a given path in a given branch /// Task> GetGitTreeNames(string path, string repoUri, string branch); + + /// + /// Returns the latest commits for a given repository and branch, up to maxCount + /// + Task> FetchLatestRepoCommitsAsync(string repoUrl, string branch, int maxCount); + + /// + /// Returns the latest commits for a given repository and branch that are newer than + /// the given commit, fetching up to maxCount commits in total. + /// + Task> FetchNewerRepoCommitsAsync(string repoUrl, string branch, string commitSha, int maxCount); + + /// + /// Get the last forward flow that was merged onto the given VMR at the specified commit + /// + Task GetLastIncomingForwardFlowAsync(string vmrUrl, string mappingName, string commit); + + /// + /// Get the last back flow that was merged onto the given repo at the specified commit + /// + Task GetLastIncomingBackflowAsync(string repoUrl, string commit); } #nullable disable diff --git a/src/Microsoft.DotNet.Darc/DarcLib/Remote.cs b/src/Microsoft.DotNet.Darc/DarcLib/Remote.cs index 4810d40d26..045d1f8bf2 100644 --- a/src/Microsoft.DotNet.Darc/DarcLib/Remote.cs +++ b/src/Microsoft.DotNet.Darc/DarcLib/Remote.cs @@ -453,7 +453,6 @@ public async Task> GetPullRequestCommentsAsync(string pullRequestUr return await _remoteGitClient.GetPullRequestCommentsAsync(pullRequestUrl); } - public async Task GetSourceManifestAsync(string vmrUri, string branchOrCommit) { var fileContent = await _remoteGitClient.GetFileContentsAsync( @@ -501,4 +500,36 @@ public async Task> GetGitTreeNames(string path, stri { return await _remoteGitClient.GetGitTreeNames(path, repoUri, branch); } + + public Task> FetchLatestRepoCommitsAsync( + string repoUrl, + string branch, + int maxCount = 100) + { + return _remoteGitClient.FetchLatestRepoCommitsAsync(repoUrl, branch, maxCount); + } + + public Task> FetchNewerRepoCommitsAsync( + string repoUrl, + string branch, + string commitSha, + int maxCount = 100) + { + return _remoteGitClient.FetchNewerRepoCommitsAsync(repoUrl, branch, commitSha, maxCount); + } + + public async Task GetLastIncomingForwardFlowAsync( + string vmrUrl, + string mappingName, + string commit) + { + return await _remoteGitClient.GetLastIncomingForwardFlowAsync(vmrUrl, mappingName, commit); + } + + public async Task GetLastIncomingBackflowAsync( + string repoUrl, + string commit) + { + return await _remoteGitClient.GetLastIncomingBackflowAsync(repoUrl, commit); + } } diff --git a/src/ProductConstructionService/Microsoft.DotNet.ProductConstructionService.Client/Generated/Models/CodeflowHistory.cs b/src/ProductConstructionService/Microsoft.DotNet.ProductConstructionService.Client/Generated/Models/CodeflowHistory.cs new file mode 100644 index 0000000000..0ac85c1bd0 --- /dev/null +++ b/src/ProductConstructionService/Microsoft.DotNet.ProductConstructionService.Client/Generated/Models/CodeflowHistory.cs @@ -0,0 +1,67 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using Newtonsoft.Json; + +namespace Microsoft.DotNet.ProductConstructionService.Client.Models +{ + public partial class CodeflowHistory + { + public CodeflowHistory( + List forwardFlowHistory, + List backflowHistory, + string repoName, + string vmrName, + bool resultIsOutdated) + { + ForwardFlowHistory = forwardFlowHistory; + BackflowHistory = backflowHistory; + RepoName = repoName; + VmrName = vmrName; + ResultIsOutdated = resultIsOutdated; + } + + [JsonProperty("forwardFlowHistory")] + public List ForwardFlowHistory { get; } + + [JsonProperty("backflowHistory")] + public List BackflowHistory { get; } + + [JsonProperty("repoName")] + public string RepoName { get; } + + [JsonProperty("vmrName")] + public string VmrName { get; } + + [JsonProperty("resultIsOutdated")] + public bool ResultIsOutdated { get; } + } + + public partial class CodeflowGraphCommit + { + public CodeflowGraphCommit( + string commitSha, + string author, + string description, + string sourceRepoFlowSha) + { + CommitSha = commitSha; + Author = author; + Description = description; + SourceRepoFlowSha = sourceRepoFlowSha; + } + + [JsonProperty("commitSha")] + public string CommitSha { get; } + + [JsonProperty("author")] + public string Author { get; } + + [JsonProperty("description")] + public string Description { get; } + + [JsonProperty("sourceRepoFlowSha")] + public string SourceRepoFlowSha { get; } + } +} diff --git a/src/ProductConstructionService/Microsoft.DotNet.ProductConstructionService.Client/Generated/Subscriptions.cs b/src/ProductConstructionService/Microsoft.DotNet.ProductConstructionService.Client/Generated/Subscriptions.cs index 6dfe762d58..97169165a4 100644 --- a/src/ProductConstructionService/Microsoft.DotNet.ProductConstructionService.Client/Generated/Subscriptions.cs +++ b/src/ProductConstructionService/Microsoft.DotNet.ProductConstructionService.Client/Generated/Subscriptions.cs @@ -71,6 +71,11 @@ Task TriggerDailyUpdateAsync( CancellationToken cancellationToken = default ); + + Task GetCodeflowHistoryAsync( + Guid id, + CancellationToken cancellationToken = default + ); } internal partial class Subscriptions : IServiceOperations, ISubscriptions @@ -753,5 +758,75 @@ internal async Task OnGetSubscriptionHistoryFailed(Request req, Response res) Client.OnFailedRequest(ex); throw ex; } + + partial void HandleFailedGetCodeflowHistoryRequest(RestApiException ex); + + public async Task GetCodeflowHistoryAsync( + Guid id, + CancellationToken cancellationToken = default + ) + { + + const string apiVersion = "2020-02-20"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/subscriptions/{id}/codeflowhistory".Replace("{id}", Uri.EscapeDataString(Client.Serialize(id))), + false); + + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Get; + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnGetCodeflowHistoryFailed(_req, _res).ConfigureAwait(false); + } + + if (_res.ContentStream == null) + { + await OnGetCodeflowHistoryFailed(_req, _res).ConfigureAwait(false); + } + + using (var _reader = new StreamReader(_res.ContentStream)) + { + var _content = await _reader.ReadToEndAsync().ConfigureAwait(false); + var _body = Client.Deserialize(_content); + return _body; + } + } + } + } + + internal async Task OnGetCodeflowHistoryFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedGetCodeflowHistoryRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } } } diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs index 4a7b058148..f11660651e 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2018_07_16/Controllers/SubscriptionsController.cs @@ -10,10 +10,15 @@ using Microsoft.AspNetCore.Mvc; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Options; +using ProductConstructionService.Api.Controllers.Models; using ProductConstructionService.Api.v2018_07_16.Models; +using Microsoft.DotNet.DarcLib; +using ProductConstructionService.Common.CodeflowHistory; using ProductConstructionService.DependencyFlow.WorkItems; using ProductConstructionService.WorkItems; using Channel = Maestro.Data.Models.Channel; +using SubscriptionDAO = Maestro.Data.Models.Subscription; +using Microsoft.EntityFrameworkCore.Infrastructure; namespace ProductConstructionService.Api.Api.v2018_07_16.Controllers; @@ -27,21 +32,26 @@ public class SubscriptionsController : ControllerBase private readonly BuildAssetRegistryContext _context; private readonly IWorkItemProducerFactory _workItemProducerFactory; private readonly IGitHubInstallationIdResolver _installationIdResolver; + private readonly ICodeflowHistoryManager _codeflowHistoryManager; + private readonly IRemoteFactory _remoteFactory; private readonly ILogger _logger; protected readonly IOptions _environmentNamespaceOptions; - public SubscriptionsController( BuildAssetRegistryContext context, IWorkItemProducerFactory workItemProducerFactory, IGitHubInstallationIdResolver installationIdResolver, IOptions environmentNamespaceOptions, + IRemoteFactory remoteFactory, + ICodeflowHistoryManager codeflowHistoryManager, ILogger logger) { _context = context; _workItemProducerFactory = workItemProducerFactory; _installationIdResolver = installationIdResolver; _environmentNamespaceOptions = environmentNamespaceOptions; + _remoteFactory = remoteFactory; + _codeflowHistoryManager = codeflowHistoryManager; _logger = logger; } @@ -57,7 +67,7 @@ public virtual IActionResult ListSubscriptions( int? channelId = null, bool? enabled = null) { - IQueryable query = _context.Subscriptions.Include(s => s.Channel); + IQueryable query = _context.Subscriptions.Include(s => s.Channel); if (!string.IsNullOrEmpty(sourceRepository)) { @@ -92,7 +102,7 @@ public virtual IActionResult ListSubscriptions( [ValidateModelState] public virtual async Task GetSubscription(Guid id) { - Maestro.Data.Models.Subscription? subscription = await _context.Subscriptions.Include(sub => sub.LastAppliedBuild) + SubscriptionDAO? subscription = await _context.Subscriptions.Include(sub => sub.LastAppliedBuild) .Include(sub => sub.Channel) .FirstOrDefaultAsync(sub => sub.Id == id); @@ -103,7 +113,15 @@ public virtual async Task GetSubscription(Guid id) return Ok(new Subscription(subscription)); } - + /* + [HttpGet("{id}/codeflowhistory")] + [SwaggerApiResponse(HttpStatusCode.Accepted, Type = typeof(CodeflowHistoryResult), Description = "The codeflow history")] + [ValidateModelState] + public virtual async Task GetCodeflowHistory(Guid id) + { + return await GetCodeflowHistoryCore(id, false); + } + */ /// /// Trigger a manually by id /// @@ -120,7 +138,7 @@ public virtual async Task TriggerSubscription(Guid id, [FromQuery protected async Task TriggerSubscriptionCore(Guid id, int buildId, bool force = false) { - Maestro.Data.Models.Subscription? subscription = await _context.Subscriptions + SubscriptionDAO? subscription = await _context.Subscriptions .Include(sub => sub.LastAppliedBuild) .Include(sub => sub.Channel) .FirstOrDefaultAsync(sub => sub.Id == id); @@ -156,9 +174,86 @@ protected async Task TriggerSubscriptionCore(Guid id, int buildId return Accepted(new Subscription(subscription)); } + protected async Task GetCodeflowHistoryCore(Guid id, bool fetchNewChanges = false) + { + var subscription = await _context.Subscriptions + .Include(sub => sub.LastAppliedBuild) + .FirstOrDefaultAsync(sub => sub.Id == id && sub.SourceEnabled == true); + + if (subscription == null) + { + return NotFound(); + } + + var oppositeDirectionSubscription = await _context.Subscriptions + .Include(sub => sub.LastAppliedBuild) + .Include(sub => sub.Channel) + .Where(sub => + sub.SourceRepository == subscription.TargetRepository + && sub.TargetRepository == subscription.SourceRepository) + .FirstOrDefaultAsync(sub => sub.SourceEnabled == true); + + bool isForwardFlow = !string.IsNullOrEmpty(subscription.TargetDirectory); + + IReadOnlyCollection? cachedFlows; + IReadOnlyCollection? oppositeCachedFlows; + + cachedFlows = await _codeflowHistoryManager.FetchLatestCodeflowHistoryAsync(subscription); + + oppositeCachedFlows = oppositeDirectionSubscription != null + ? await _codeflowHistoryManager.FetchLatestCodeflowHistoryAsync(oppositeDirectionSubscription) + : []; + + var lastCommit = subscription.LastAppliedBuild?.Commit; + + bool resultIsOutdated = IsCodeflowHistoryOutdated(subscription, cachedFlows) || + IsCodeflowHistoryOutdated(oppositeDirectionSubscription, oppositeCachedFlows); + + var forwardFlowHistory = isForwardFlow ? cachedFlows : oppositeCachedFlows; + var backflowHistory = isForwardFlow ? oppositeCachedFlows : cachedFlows; + + forwardFlowHistory = forwardFlowHistory + .Select(commitGraph => commitGraph with + { + CommitSha = Commit.GetShortSha(commitGraph.CommitSha), + SourceRepoFlowSha = !string.IsNullOrEmpty(commitGraph.SourceRepoFlowSha) + ? Commit.GetShortSha(commitGraph.SourceRepoFlowSha) + : "" + }).ToList(); + + backflowHistory = backflowHistory + .Select(commitGraph => commitGraph with { + CommitSha = Commit.GetShortSha(commitGraph.CommitSha), + SourceRepoFlowSha = !string.IsNullOrEmpty(commitGraph.SourceRepoFlowSha) + ? Commit.GetShortSha(commitGraph.SourceRepoFlowSha) + : "" + }) + .ToList(); + + var result = new CodeflowHistoryResult( + forwardFlowHistory, + backflowHistory, + string.IsNullOrEmpty(subscription.TargetDirectory) + ? subscription.SourceDirectory + : subscription.TargetBranch, + "VMR", + resultIsOutdated); + + return Ok(result); + } + + private static bool IsCodeflowHistoryOutdated( + SubscriptionDAO? subscription, + IReadOnlyCollection? cachedFlows) + { + string? lastCachedCodeflow = cachedFlows?.LastOrDefault()?.SourceRepoFlowSha; + string? lastAppliedCommit = subscription?.LastAppliedBuild?.Commit; + return !string.Equals(lastCachedCodeflow, lastAppliedCommit, StringComparison.Ordinal); + } + private async Task EnqueueUpdateSubscriptionWorkItemAsync(Guid subscriptionId, int buildId, bool force = false) { - Maestro.Data.Models.Subscription? subscriptionToUpdate; + SubscriptionDAO? subscriptionToUpdate; if (buildId != 0) { // Update using a specific build @@ -233,7 +328,7 @@ public virtual async Task TriggerDailyUpdateAsync() foreach (var subscription in enabledSubscriptionsWithTargetFrequency) { - Maestro.Data.Models.Subscription? subscriptionWithBuilds = await _context.Subscriptions + SubscriptionDAO? subscriptionWithBuilds = await _context.Subscriptions .Where(s => s.Id == subscription.Id) .Include(s => s.Channel) .ThenInclude(c => c.BuildChannels) @@ -278,7 +373,7 @@ await workitemProducer.ProduceWorkItemAsync(new() [ValidateModelState] public virtual async Task UpdateSubscription(Guid id, [FromBody] SubscriptionUpdate update) { - Maestro.Data.Models.Subscription? subscription = await _context.Subscriptions.Where(sub => sub.Id == id) + SubscriptionDAO? subscription = await _context.Subscriptions.Where(sub => sub.Id == id) .FirstOrDefaultAsync(); if (subscription == null) @@ -324,7 +419,7 @@ public virtual async Task UpdateSubscription(Guid id, [FromBody] if (doUpdate) { - Maestro.Data.Models.Subscription? equivalentSubscription = await FindEquivalentSubscription(subscription); + SubscriptionDAO? equivalentSubscription = await FindEquivalentSubscription(subscription); if (equivalentSubscription != null) { return BadRequest( @@ -352,7 +447,7 @@ public virtual async Task UpdateSubscription(Guid id, [FromBody] [ValidateModelState] public virtual async Task DeleteSubscription(Guid id) { - Maestro.Data.Models.Subscription? subscription = + SubscriptionDAO? subscription = await _context.Subscriptions.FirstOrDefaultAsync(sub => sub.Id == id); if (subscription == null) @@ -383,7 +478,7 @@ public virtual async Task DeleteSubscription(Guid id) [Paginated(typeof(SubscriptionHistoryItem))] public virtual async Task GetSubscriptionHistory(Guid id) { - Maestro.Data.Models.Subscription? subscription = await _context.Subscriptions.Where(sub => sub.Id == id) + SubscriptionDAO? subscription = await _context.Subscriptions.Where(sub => sub.Id == id) .FirstOrDefaultAsync(); if (subscription == null) @@ -460,7 +555,7 @@ public virtual async Task Create([FromBody, Required] Subscriptio subscriptionModel.Id = Guid.NewGuid(); subscriptionModel.Namespace = defaultNamespace; - Maestro.Data.Models.Subscription? equivalentSubscription = await FindEquivalentSubscription(subscriptionModel); + SubscriptionDAO? equivalentSubscription = await FindEquivalentSubscription(subscriptionModel); if (equivalentSubscription != null) { return Conflict( @@ -547,7 +642,7 @@ protected async Task EnsureRepositoryRegistration(string repoUri) /// /// Subscription model with updated data. /// Subscription if it is found, null otherwise - private async Task FindEquivalentSubscription(Maestro.Data.Models.Subscription updatedOrNewSubscription) + private async Task FindEquivalentSubscription(SubscriptionDAO updatedOrNewSubscription) { // Compare subscriptions based on the 4 key elements: // - Channel diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2019_01_16/Controllers/SubscriptionsController.cs b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2019_01_16/Controllers/SubscriptionsController.cs index 1435bcbbed..a50821de1a 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2019_01_16/Controllers/SubscriptionsController.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2019_01_16/Controllers/SubscriptionsController.cs @@ -7,9 +7,12 @@ using Microsoft.AspNetCore.ApiVersioning; using Microsoft.AspNetCore.ApiVersioning.Swashbuckle; using Microsoft.AspNetCore.Mvc; +using Microsoft.DotNet.DarcLib; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Options; +using ProductConstructionService.Api.Controllers.Models; using ProductConstructionService.Api.v2019_01_16.Models; +using ProductConstructionService.Common.CodeflowHistory; using ProductConstructionService.WorkItems; namespace ProductConstructionService.Api.Api.v2019_01_16.Controllers; @@ -27,9 +30,11 @@ public SubscriptionsController( BuildAssetRegistryContext context, IWorkItemProducerFactory workItemProducerFactory, IGitHubInstallationIdResolver gitHubInstallationRetriever, + IRemoteFactory remoteFactory, + ICodeflowHistoryManager codeflowHistoryManager, IOptions environmentNamespaceOptions, ILogger logger) - : base(context, workItemProducerFactory, gitHubInstallationRetriever, environmentNamespaceOptions, logger) + : base(context, workItemProducerFactory, gitHubInstallationRetriever, environmentNamespaceOptions, remoteFactory, codeflowHistoryManager, logger) { _context = context; } @@ -92,6 +97,15 @@ public override async Task GetSubscription(Guid id) return Ok(new Subscription(subscription)); } + /* + [HttpGet("{id}/codeflowhistory")] + [SwaggerApiResponse(HttpStatusCode.Accepted, Type = typeof(CodeflowHistoryResult), Description = "The codeflow history")] + [ValidateModelState] + public override async Task GetCodeflowHistory(Guid id) + { + return await GetCodeflowHistoryCore(id); + } + */ /// /// Trigger a manually by id diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/SubscriptionsController.cs b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/SubscriptionsController.cs index 64a8bc1ca0..7ee12b5dfe 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/SubscriptionsController.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/Api/v2020_02_20/Controllers/SubscriptionsController.cs @@ -4,6 +4,7 @@ using System.ComponentModel.DataAnnotations; using System.Net; using Maestro.Data; +using Microsoft.DotNet.DarcLib; using Microsoft.AspNetCore.ApiVersioning; using Microsoft.AspNetCore.ApiVersioning.Swashbuckle; using Microsoft.AspNetCore.Mvc; @@ -11,7 +12,9 @@ using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Options; using ProductConstructionService.Api.v2020_02_20.Models; +using ProductConstructionService.Common.CodeflowHistory; using ProductConstructionService.WorkItems; +using ProductConstructionService.Api.Controllers.Models; namespace ProductConstructionService.Api.Api.v2020_02_20.Controllers; @@ -33,8 +36,10 @@ public SubscriptionsController( IGitHubInstallationIdResolver gitHubInstallationRetriever, IWorkItemProducerFactory workItemProducerFactory, IOptions environmentNamespaceOptions, + IRemoteFactory remoteFactory, + ICodeflowHistoryManager codeflowHistoryManager, ILogger logger) - : base(context, workItemProducerFactory, gitHubInstallationRetriever, environmentNamespaceOptions, logger) + : base(context, workItemProducerFactory, gitHubInstallationRetriever, remoteFactory, codeflowHistoryManager, environmentNamespaceOptions, logger) { _context = context; _gitHubClientFactory = gitHubClientFactory; @@ -146,6 +151,14 @@ public override async Task TriggerSubscription(Guid id, [FromQuer return await TriggerSubscriptionCore(id, buildId, force); } + [HttpGet("{id}/codeflowhistory")] + [SwaggerApiResponse(HttpStatusCode.Accepted, Type = typeof(Subscription), Description = "The codeflow history")] + [ValidateModelState] + public async Task GetCodeflowHistory(Guid id) + { + return await GetCodeflowHistoryCore(id); + } + [ApiRemoved] public sealed override Task UpdateSubscription(Guid id, [FromBody] ProductConstructionService.Api.v2018_07_16.Models.SubscriptionUpdate update) { diff --git a/src/ProductConstructionService/ProductConstructionService.Api/Controllers/Models/CodeflowHistoryResult.cs b/src/ProductConstructionService/ProductConstructionService.Api/Controllers/Models/CodeflowHistoryResult.cs new file mode 100644 index 0000000000..d7ccd783c9 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Api/Controllers/Models/CodeflowHistoryResult.cs @@ -0,0 +1,32 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using ProductConstructionService.Common.CodeflowHistory; + +namespace ProductConstructionService.Api.Controllers.Models; + +/// +/// API model for codeflow history. +/// +public class CodeflowHistoryResult +{ + public CodeflowHistoryResult( + IReadOnlyCollection forwardFlowHistory, + IReadOnlyCollection backflowHistory, + string repoName, + string vmrName, + bool resultIsOutdated) + { + ForwardFlowHistory = forwardFlowHistory; + BackflowHistory = backflowHistory; + RepoName = repoName; + VmrName = vmrName; + ResultIsOutdated = resultIsOutdated; + } + + public IReadOnlyCollection ForwardFlowHistory { get; set; } = []; + public IReadOnlyCollection BackflowHistory { get; set; } = []; + public string RepoName { get; set; } = ""; + public string VmrName { get; set; } = ""; + public bool ResultIsOutdated { get; set; } +} diff --git a/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs b/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs index 9a935cc062..ea586341f0 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs +++ b/src/ProductConstructionService/ProductConstructionService.Api/PcsStartup.cs @@ -38,6 +38,7 @@ using ProductConstructionService.Api.Controllers; using Azure.Core; using Microsoft.DotNet.DarcLib.Helpers; +using ProductConstructionService.Common.CodeflowHistory; namespace ProductConstructionService.Api; @@ -138,6 +139,7 @@ internal static async Task ConfigurePcs( builder.Services.Configure(_ => { }); builder.Services.AddMemoryCache(); builder.Services.AddSingleton(builder.Configuration); + builder.Services.AddScoped(); // We do not use AddMemoryCache here. We use our own cache because we wish to // use a sized cache and some components, such as EFCore, do not implement their caching diff --git a/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Development.json b/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Development.json index 63c3b0f7d6..a411992828 100644 --- a/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Development.json +++ b/src/ProductConstructionService/ProductConstructionService.Api/appsettings.Development.json @@ -28,7 +28,7 @@ "RedirectUri": "https://localhost:53180/signin-oidc" }, "ApiRedirect": { - // "Uri": "https://maestro.dot.net/" + // "Uri": "https://maestro.dot.net/" }, "EnvironmentNamespaceOptions": { "DefaultNamespaceName": "local" diff --git a/src/ProductConstructionService/ProductConstructionService.BarViz/Components/CodeflowHistoryGraph.razor b/src/ProductConstructionService/ProductConstructionService.BarViz/Components/CodeflowHistoryGraph.razor new file mode 100644 index 0000000000..a0a63a9b96 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.BarViz/Components/CodeflowHistoryGraph.razor @@ -0,0 +1,245 @@ +@using System.Linq.Expressions +@using System.Text +@using Microsoft.DotNet.ProductConstructionService.Client +@using Microsoft.DotNet.ProductConstructionService.Client.Models +@using Microsoft.FluentUI.AspNetCore.Components.Extensions +@using ProductConstructionService.BarViz.Code.Helpers +@using ProductConstructionService.BarViz.Components +@using TextCopy +@inject IProductConstructionServiceApi PcsApi + +@inherits ComponentBase + +

CodeflowHistoryGraph

+ + + + + + + + + + + + + + + + @((MarkupString)$"{_leftColumnName}") + + @for (int i = 0; i < _leftColumn.Count; i++) + { + if (!_leftColumn[i].Contains("hidden")) + { + + @((MarkupString)$"{_leftColumn[i]}") + ; + } + else + { + + + var firstLine = _leftColumn[i].Split('(')[0]; + var secondLine = "(" + _leftColumn[i].Split('(')[1]; + @((MarkupString)$"{firstLine}") + ; + @((MarkupString)$"{secondLine}") + ; + } + } + + + + + @((MarkupString)$"{_rightColumnName}") + + @for (int i = 0; i < _rightColumn.Count; i++) + { + if (!_rightColumn[i].Contains("hidden")) + { + + @((MarkupString)$"{_rightColumn[i]}") + ; + + } + else + { + + var firstLine = _rightColumn[i].Split('(')[0]; + var secondLine = "(" + _rightColumn[i].Split('(')[1]; + @((MarkupString)$"{firstLine}") + ; + @((MarkupString)$"{secondLine}") + ; + + } + } + + + + @foreach (var ff in _forwardFlows) + { + + } + + @foreach (var ff in _backflows) + { + + } + + +@code { + [Parameter] + public Guid SubscriptionId { get; set; } = default!; + + private const int BOX_HEIGHT = 55; + private const int BOX_WIDTH = 120; + private const int BOX_WIDTH_2 = 160; + private const int BOX_Y_MARGIN = 30; + private const int COL_SPACE = 500; + + private string _leftColumnName = "Repo"; + private string _rightColumnName = "VMR"; + + private List _leftColumn = ["abc123", "cbf123", "...[21 more \n commits]...", "xcv123", "zvn123"]; + private List _rightColumn = ["wer865", "uwe765", "wet843", "wet845", "wry865", "iet854", "ery854"]; + + private List<(int, int)> _forwardFlows = [(1, 2), (4, 3)]; + private List<(int, int)> _backflows = [(6, 2)]; + + protected override async Task OnInitializedAsync() + { + var codeflowHistory = await PcsApi.Subscriptions.GetCodeflowHistoryAsync(SubscriptionId); + // var codeflowHistory = new CodeflowHistory(_vmrCommits, _repoCommits, "SampleRepo", "SampleVMR", false); + BuildGraphVisualization(codeflowHistory); + } + + public void BuildGraphVisualization(CodeflowHistory codeflowHistory) + { + _leftColumnName = codeflowHistory.RepoName; + _rightColumnName = codeflowHistory.VmrName; + + // 1. Columns + _leftColumn = codeflowHistory.BackflowHistory + .Select(c => c.CommitSha) + .ToList(); + + _rightColumn = codeflowHistory.ForwardFlowHistory + .Select(c => c.CommitSha) + .ToList(); + + HashSet codeflowCommits = codeflowHistory.BackflowHistory + .Where(c => !string.IsNullOrEmpty(c.SourceRepoFlowSha)) + .SelectMany(c => new[] { c.SourceRepoFlowSha, c.CommitSha }) + .Concat(codeflowHistory.ForwardFlowHistory + .Where(c => !string.IsNullOrEmpty(c.SourceRepoFlowSha)) + .SelectMany(c => new[] { c.SourceRepoFlowSha, c.CommitSha })) + .Where(val => val is not null) + .ToHashSet(); + + _leftColumn = CompactifyCommitsColumn(_leftColumn, codeflowCommits); + _rightColumn = CompactifyCommitsColumn(_rightColumn, codeflowCommits); + + // 2. SHA -> 0-based index maps + var leftMap = _leftColumn + .Select((sha, idx) => (sha, idx)) + .ToDictionary(t => t.sha, t => t.idx); + + var rightMap = _rightColumn + .Select((sha, idx) => (sha, idx)) + .ToDictionary(t => t.sha, t => t.idx); + + // 3. Build BF (vmr -> repo) + _backflows = codeflowHistory.BackflowHistory + .Where(c => !string.IsNullOrEmpty(c.SourceRepoFlowSha) && rightMap.ContainsKey(c.SourceRepoFlowSha)) + .Select(c => (rightMap[c.SourceRepoFlowSha!], leftMap[c.CommitSha])) + .ToList(); + + // 4. Build FF (repo -> vmr) + _forwardFlows = codeflowHistory.ForwardFlowHistory + .Where(c => !string.IsNullOrEmpty(c.SourceRepoFlowSha) && leftMap.ContainsKey(c.SourceRepoFlowSha)) + .Select(c => (leftMap[c.SourceRepoFlowSha!], rightMap[c.CommitSha])) + .ToList(); + + } + + public static List CompactifyCommitsColumn(List commits, HashSet codeflowCommits) + { + string CommitsHiddenString(string commitA, string commitB, int num) => $"{commitA}...{commitB} ({num} commits hidden)"; + // $"[{firstCompactedCommit}]...[{lastCompactedCommit}]\n({compactedCommits} commits hidden)" + string firstCompactedCommit = ""; + string lastCompactedCommit = ""; + int compactedCommits = 0; + List result = []; + + for (int i = 0; i < commits.Count; i++) + { + var a = Math.Max(0, i - 1); + var b = Math.Max(0, i - 1); + var c = Math.Min(commits.Count - 1, i + 1); + var d = Math.Min(commits.Count - 1, i + 1); + + // check if there's no codeflows +/- 2 commits around current commit + if (!codeflowCommits.Contains(commits[i])) + { + if (compactedCommits == 0) + { + firstCompactedCommit = commits[i]; + } + compactedCommits++; + lastCompactedCommit = commits[i]; + } + else + { + if (compactedCommits == 1) + { + result.Add(firstCompactedCommit); + compactedCommits = 0; + firstCompactedCommit = ""; + } + else if (compactedCommits > 0) + { + result.Add(CommitsHiddenString(firstCompactedCommit, lastCompactedCommit, compactedCommits)); + compactedCommits = 0; + firstCompactedCommit = ""; + } + result.Add(commits[i]); + } + } + if (compactedCommits > 0) + { + result.Add(CommitsHiddenString(firstCompactedCommit, lastCompactedCommit, compactedCommits)); + } + return result; + } + + private List _repoCommits = new List + { + new("a1b2c3", "Alice", "Fix login bug", null), + new("d4e5f6", "Bob", "Add unit tests", "w1x2y3"), + new("g7h8i9", "Carol", "Refactor auth module", null), + new("k2d9n0", "Carol", "Refactor auth module", null), + new("sh7d66e", "Carol", "Refactor auth module", null), + new("uej367", "Carol", "Refactor auth module", null), + new("xj39tt)", "Carol", "Refactor auth module", null), + new("j1k2l3", "Dave", "Update README", null), + new("m4n5o6", "Eve", "Optimize query", null), + new("p7q8r9", "Frank", "Fix typo", "w4x5y6"), + new("u6i7o7", "Grace", "Remove unused code", null), + new("b3n5n7", "Grace", "Remove unused code", null), + }; + + private List _vmrCommits = new List + { + new("w1x2y3", "Henry", "Merge feature branch", null), + new("w4x5y6", "Ivy", "Hotfix production bug", null), + new("w7x8y9", "Jack", "Update docs", "a1b2c3"), + new("w0x1y2", "Kate", "Add CI pipeline", null), + new("w3x4y5", "Leo", "Refactor build scripts", "g7h8i9"), + new("w6x7y8", "Mona", "Improve logging", null), + new("w9x0y1", "Nina", "Update dependencies", null), + }; +} diff --git a/src/ProductConstructionService/ProductConstructionService.BarViz/Components/SubscriptionDetailDialog.razor b/src/ProductConstructionService/ProductConstructionService.BarViz/Components/SubscriptionDetailDialog.razor index e0b955617f..acc058b6a5 100644 --- a/src/ProductConstructionService/ProductConstructionService.BarViz/Components/SubscriptionDetailDialog.razor +++ b/src/ProductConstructionService/ProductConstructionService.BarViz/Components/SubscriptionDetailDialog.razor @@ -43,100 +43,12 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @if (!string.IsNullOrEmpty(Content.TargetDirectory)) - { - - - - - } - @if (Content.SourceEnabled && !string.IsNullOrEmpty(Content.SourceDirectory)) - { - - - - - } - - - - - - - - - -
Id@Content.Id
Channel@Content.Channel.Name
Source repository@Content.SourceRepository
Target repository@Content.TargetRepository
Target branch@Content.TargetBranch
Update frequency@Content.Policy.UpdateFrequency
Enabled@(Content.Enabled ? "Yes" : "No")
Batchable@(Content.Policy.Batchable ? "Yes" : "No")
Merge policies - @if (Content.Policy.MergePolicies == null || Content.Policy.MergePolicies.Count == 0) - { - None - } - else - { -
    - @foreach (var policy in Content.Policy.MergePolicies) - { -
  • @policy.Name
    • - } -
- } -
Source-enabled@(Content.SourceEnabled ? (string.IsNullOrEmpty(Content.SourceDirectory) ? "Yes (Forward flow)" : "Yes (Backflow)") : "No")
Target directory@Content.TargetDirectory
Source directory@Content.SourceDirectory
ExcludedAssets - @if (Content.ExcludedAssets == null || Content.ExcludedAssets.Count == 0) - { - None - } - else - { -
    - @foreach (var asset in Content.ExcludedAssets) - { -
  • @asset
    • - } -
- } -
PR notification tags@Content.PullRequestFailureNotificationTags
+ + +
+ +
+ diff --git a/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistory/CodeflowHistoryManager.cs b/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistory/CodeflowHistoryManager.cs new file mode 100644 index 0000000000..c1c8816b49 --- /dev/null +++ b/src/ProductConstructionService/ProductConstructionService.Common/CodeflowHistory/CodeflowHistoryManager.cs @@ -0,0 +1,213 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text.Json; +using Maestro.Data.Models; +using Microsoft.DotNet.DarcLib; +using Microsoft.DotNet.DarcLib.Models.VirtualMonoRepo; +using Pipelines.Sockets.Unofficial.Arenas; +using StackExchange.Redis; + +namespace ProductConstructionService.Common.CodeflowHistory; + +public interface ICodeflowHistoryManager +{ + Task> GetCachedCodeflowHistoryAsync( + string subscriptionId, + int commitFetchCount = 100); + + Task> FetchLatestCodeflowHistoryAsync( + Subscription subscription, + int commitFetchCount = 100); +} + +public record CodeflowGraphCommit( + string CommitSha, + string Author, + string Description, + string? SourceRepoFlowSha, + int? redisScore); + +public class CodeflowHistoryManager( + IRemoteFactory remoteFactory, + ConfigurationOptions options) : ICodeflowHistoryManager +{ + private readonly IRemoteFactory _remoteFactory = remoteFactory; + private readonly IConnectionMultiplexer _connection = ConnectionMultiplexer.Connect(options); + + private static RedisKey GetCodeflowGraphCommitKey(string id) => $"CodeflowGraphCommit_{id}"; + private static RedisKey GetSortedSetKey(string id) => $"CodeflowHistory_{id}"; + + private const int MaxCommitFetchCount = 500; + private const int MaxCommitsCached = 3000; + + public async Task> GetCachedCodeflowHistoryAsync( + string subscriptionId, + int commitFetchCount = 100) + { + if (commitFetchCount < 1) + { + throw new ArgumentOutOfRangeException(nameof(commitFetchCount)); + } + + var cache = _connection.GetDatabase(); + + var res = await cache.SortedSetRangeByRankWithScoresAsync( + key: GetSortedSetKey(subscriptionId), + start: 0, + stop: commitFetchCount - 1, + order: Order.Descending); + + var commitKeys = res + .Select(e => new RedisKey(GetCodeflowGraphCommitKey(e.Element.ToString()))) + .ToArray(); + + var commitValues = await cache.StringGetAsync(commitKeys); + + if (commitValues.Any(val => !val.HasValue)) + { + await ClearCodeflowCacheAsync(subscriptionId); + throw new InvalidOperationException($"Corrupted commit data encountered for subscription `{subscriptionId}`."); + } + + return [.. commitValues + .Select(commit => JsonSerializer.Deserialize(commit.ToString())) + .OfType()]; + } + + public async Task> FetchLatestCodeflowHistoryAsync( + Subscription subscription, + int commitFetchCount = 100) + { + var cachedCommits = await GetCachedCodeflowHistoryAsync(subscription.Id.ToString()); + + var latestCachedCommit = cachedCommits.FirstOrDefault(); + + var newCommits = await FetchNewCommits( + subscription.TargetRepository, + subscription.TargetBranch, + latestCachedCommit?.CommitSha); + + if (newCommits.LastOrDefault()?.CommitSha == latestCachedCommit?.CommitSha) + { + newCommits.RemoveLast(); + } + else + { + // the fetched commits do not connect to the cached commits + // we don't know how many commits are missing, so clear the cache and start over + await ClearCodeflowCacheAsync(subscription.Id.ToString()); + } + + var graphCommits = await EnrichCommitsWithCodeflowDataAsync( + newCommits, + subscription); + + await CacheCommitsAsync(subscription.Id.ToString(), graphCommits); + + return [.. graphCommits + .Concat(cachedCommits) + .Take(commitFetchCount)]; + } + + private async Task> EnrichCommitsWithCodeflowDataAsync( + LinkedList commits, + Subscription subscription) + { + var remote = await _remoteFactory.CreateRemoteAsync(subscription.TargetRepository); + + var current = commits.First; + + while (current != null) + { + Codeflow? lastFlow = !string.IsNullOrEmpty(subscription.TargetDirectory) + ? await remote.GetLastIncomingForwardFlowAsync( + subscription.TargetRepository, + subscription.TargetDirectory, + current.Value.CommitSha) + : await remote.GetLastIncomingBackflowAsync( + subscription.TargetRepository, + current.Value.CommitSha); + + var target = current; + + while (target != null && target.Value.CommitSha != lastFlow?.TargetSha) + target = target.Next; + + if (target == null) + break; + + target.Value = target.Value with + { + SourceRepoFlowSha = lastFlow?.SourceSha, + }; + + current = target.Next; + } + return commits; + } + + private async Task CacheCommitsAsync( + string subscriptionId, + IEnumerable commits, + int latestCachedCommitScore = 0) + { + if (!commits.Any()) + { + return; + } + + var cache = _connection.GetDatabase(); + + var sortedSetEntries = commits + .Reverse() + .Select(c => new SortedSetEntry(c.CommitSha, latestCachedCommitScore++)) + .ToArray(); + + await cache.SortedSetAddAsync(GetSortedSetKey(subscriptionId), sortedSetEntries); + + var commitGraphEntries = commits + .Select(c => new KeyValuePair( + GetCodeflowGraphCommitKey(c.CommitSha), + JsonSerializer.Serialize(c))) + .ToArray(); + + await cache.StringSetAsync(commitGraphEntries); + + if (latestCachedCommitScore > MaxCommitsCached) + { + await cache.SortedSetRemoveRangeByScoreAsync( + key: subscriptionId, + start: 0, + stop: latestCachedCommitScore - MaxCommitsCached); + } + } + + private async Task ClearCodeflowCacheAsync(string subscriptionId) + { + var cache = _connection.GetDatabase(); + await cache.KeyDeleteAsync(GetSortedSetKey(subscriptionId)); + } + + private async Task> FetchNewCommits( + string targetRepository, + string targetBranch, + string? latestCachedCommitSha) + { + var remote = await _remoteFactory.CreateRemoteAsync(targetRepository); + + var newCommits = await remote.FetchNewerRepoCommitsAsync( + targetRepository, + targetBranch, + latestCachedCommitSha, + MaxCommitFetchCount); + + return new LinkedList( + newCommits.Select(commit => new CodeflowGraphCommit( + CommitSha: commit.Sha, + Author: commit.Author, + Description: commit.Message, + SourceRepoFlowSha: null, + redisScore: null))); + } +} diff --git a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestUpdater.cs b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestUpdater.cs index 8d853f2498..7adf5f8dbe 100644 --- a/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestUpdater.cs +++ b/src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestUpdater.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Collections.Immutable; +using Kusto.Ingest; using Maestro.Data.Models; using Maestro.DataProviders; using Maestro.MergePolicies; @@ -453,6 +454,11 @@ private async Task TryMergingPrAsync( { await remote.MergeDependencyPullRequestAsync(pr.Url, new MergePullRequestParameters()); + //todo: add codeflow history synchronization here + //todo: we should be able to obtain the last flown sha & squashed PR sha here, and cache the codeflow + // without any github requests. + + //todo: fire these in a new process - exceptions shouldn't affect main thread foreach (SubscriptionPullRequestUpdate subscription in pr.ContainedSubscriptions) { await RegisterSubscriptionUpdateAction(SubscriptionUpdateAction.MergingPullRequest, subscription.SubscriptionId);