Codeflow history API endpoint

src/Maestro/Maestro.DataProviders/RemoteFactory.cs

@@ -25,6 +25,7 @@ public class RemoteFactory : IRemoteFactory
     private readonly IGitHubTokenProvider _gitHubTokenProvider;
     private readonly IAzureDevOpsTokenProvider _azdoTokenProvider;
     private readonly IServiceProvider _serviceProvider;
+    private readonly IVersionDetailsParser _versionDetailsParser;
 
     public RemoteFactory(
         BuildAssetRegistryContext context,
@@ -45,6 +46,7 @@ public RemoteFactory(
         _azdoTokenProvider = azdoTokenProvider;
         _cache = memoryCache;
         _serviceProvider = serviceProvider;
+        _versionDetailsParser = versionDetailsParser;
     }
 
     public async Task<IRemote> CreateRemoteAsync(string repoUrl)
@@ -87,6 +89,7 @@ private async Task<IRemoteGitRepo> GetRemoteGitClient(string repoUrl)
                 : new GitHubClient(
                     new Microsoft.DotNet.DarcLib.GitHubTokenProvider(_gitHubTokenProvider),
                     _processManager,
+                    _versionDetailsParser,
                     _loggerFactory.CreateLogger<GitHubClient>(),
                     _cache.Cache),
 

src/Microsoft.DotNet.Darc/Darc/Helpers/RemoteFactory.cs

@@ -21,19 +21,22 @@ internal class RemoteFactory : IRemoteFactory
     private readonly IServiceProvider _serviceProvider;
     private readonly IAzureDevOpsTokenProvider _azdoTokenProvider;
     private readonly IRemoteTokenProvider _githubTokenProvider;
+    private readonly IVersionDetailsParser _versionDetailsParser;
 
     public RemoteFactory(
         IAzureDevOpsTokenProvider azdoTokenProvider,
         [FromKeyedServices("github")]IRemoteTokenProvider githubTokenProvider,
         ILoggerFactory loggerFactory,
         ICommandLineOptions options,
+        IVersionDetailsParser versionDetailsParser,
         IServiceProvider serviceProvider)
     {
         _loggerFactory = loggerFactory;
         _options = options;
         _serviceProvider = serviceProvider;
         _azdoTokenProvider = azdoTokenProvider;
         _githubTokenProvider = githubTokenProvider;
+        _versionDetailsParser = versionDetailsParser;
     }
 
     public Task<IRemote> CreateRemoteAsync(string repoUrl)
@@ -62,6 +65,7 @@ private IRemoteGitRepo CreateRemoteGitClient(ICommandLineOptions options, string
                     new ProcessManager(_loggerFactory.CreateLogger<IProcessManager>(), options.GitLocation),
                     _loggerFactory.CreateLogger<GitHubClient>(),
                     temporaryRepositoryRoot,
+                    _versionDetailsParser,
                     // Caching not in use for Darc local client.
                     null),
 

src/Microsoft.DotNet.Darc/DarcLib/AzureDevOpsClient.cs

@@ -16,6 +16,7 @@
 using Microsoft.DotNet.DarcLib.Helpers;
 using Microsoft.DotNet.DarcLib.Models;
 using Microsoft.DotNet.DarcLib.Models.AzureDevOps;
+using Microsoft.DotNet.DarcLib.Models.VirtualMonoRepo;
 using Microsoft.DotNet.Services.Utility;
 using Microsoft.Extensions.Logging;
 using Microsoft.TeamFoundation.SourceControl.WebApi;
@@ -1944,4 +1945,20 @@ public async Task<IReadOnlyCollection<string>> GetGitTreeNames(string path, stri
         UpdatedAt = DateTimeOffset.UtcNow,
         HeadBranchSha = pr.LastMergeSourceCommit.CommitId,
     };
+
+    public Task<List<Commit>> FetchLatestRepoCommits(string repoUrl, string branch, int maxCount)
+        => throw new NotImplementedException();
+
+    public Task<List<Commit>> FetchLatestFetchNewerRepoCommitsAsyncRepoCommits(
+        string repoUrl,
+        string branch,
+        string commitSha,
+        int maxCount)
+        => throw new NotImplementedException();
+
+    public Task<List<Commit>> FetchLatestRepoCommitsAsync(string repoUrl, string branch, int maxCount) => throw new NotImplementedException();
+    public Task<List<Commit>> FetchNewerRepoCommitsAsync(string repoUrl, string branch, string commitSha, int maxCount) => throw new NotImplementedException();
+    public Task<ForwardFlow> GetLastIncomingForwardFlowAsync(string vmrUrl, string commit) => throw new NotImplementedException();
+    public Task<Backflow> GetLastIncomingBackflowAsync(string repoUrl, string commit) => throw new NotImplementedException();
+    public Task<ForwardFlow> GetLastIncomingForwardFlowAsync(string vmrUrl, string mappingName, string commit) => throw new NotImplementedException();
 }

src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs

@@ -10,13 +10,16 @@
 using System.Net.Http;
 using System.Reflection;
 using System.Text;
+using System.Text.Json;
 using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 using Maestro.Common;
 using Maestro.MergePolicyEvaluation;
 using Microsoft.DotNet.DarcLib.Helpers;
 using Microsoft.DotNet.DarcLib.Models;
 using Microsoft.DotNet.DarcLib.Models.GitHub;
+using Microsoft.DotNet.DarcLib.Models.VirtualMonoRepo;
+using Microsoft.DotNet.DarcLib.VirtualMonoRepo;
 using Microsoft.DotNet.Services.Utility;
 using Microsoft.Extensions.Caching.Memory;
 using Microsoft.Extensions.Logging;
@@ -48,6 +51,7 @@ public class GitHubClient : RemoteRepoBase, IRemoteGitRepo
     private readonly string _userAgent = $"DarcLib-{DarcLibVersion}";
     private IGitHubClient? _lazyClient = null;
     private readonly Dictionary<(string, string, string?), string> _gitRefCommitCache;
+    private readonly IVersionDetailsParser _versionDetailsParser;
 
     static GitHubClient()
     {
@@ -60,9 +64,10 @@ static GitHubClient()
     public GitHubClient(
         IRemoteTokenProvider remoteTokenProvider,
         IProcessManager processManager,
+        IVersionDetailsParser versionDetailsParser,
         ILogger logger,
         IMemoryCache? cache)
-        : this(remoteTokenProvider, processManager, logger, null, cache)
+        : this(remoteTokenProvider, processManager, logger, null, versionDetailsParser, cache)
     {
     }
 
@@ -71,6 +76,7 @@ public GitHubClient(
         IProcessManager processManager,
         ILogger logger,
         string? temporaryRepositoryPath,
+        IVersionDetailsParser versionDetailsParser,
         IMemoryCache? cache)
         : base(remoteTokenProvider, processManager, temporaryRepositoryPath, cache, logger)
     {
@@ -82,6 +88,7 @@ public GitHubClient(
             NullValueHandling = NullValueHandling.Ignore
         };
         _gitRefCommitCache = [];
+        _versionDetailsParser = versionDetailsParser;
     }
 
     public bool AllowRetries { get; set; } = true;
@@ -1491,4 +1498,210 @@ private static PullRequest ToDarcLibPullRequest(Octokit.PullRequest pr)
             HeadBranchSha = pr.Head.Sha,
         };
     }
+
+    public Task<List<Commit>> FetchLatestRepoCommitsAsync(string repoUrl, string branch, int maxCount)
+        => throw new NotImplementedException();
+
+    public async Task<List<Commit>> FetchNewerRepoCommitsAsync(
+        string repoUrl,
+        string branch,
+        string commitSha,
+        int maxCount)
+    {
+        if (maxCount <= 0)
+        {
+            maxCount = 100;
+        }
+
+        (string owner, string repo) = ParseRepoUri(repoUrl);
+
+        var request = new CommitRequest
+        {
+            Sha = branch,
+        };
+
+        var options = new ApiOptions
+        {
+            PageSize = maxCount,
+            PageCount = 1,
+            StartPage = 1
+        };
+
+        var allCommits = new List<Commit>();
+
+        while (allCommits.Count < maxCount)
+        {
+            var commits = await GetClient(owner, repo)
+                .Repository
+                .Commit
+                .GetAll(owner, repo, request, options);
+
+            foreach (Octokit.GitHubCommit c in commits)
+            {
+                var convertedCommit = new Commit(
+                    c.Author?.Login,
+                    c.Sha,
+                    c.Commit.Message);
+
+                allCommits.Add(convertedCommit);
+
+                if (convertedCommit.Sha == commitSha)
+                {
+                    break;
+                }
+            }
+
+            if (commits.Count < options.PageSize)
+            {
+                break;
+            }
+
+            options.StartPage++;
+        }
+
+        return [.. allCommits.Take(maxCount)];
+    }
+
+    public async Task<ForwardFlow?> GetLastIncomingForwardFlowAsync(string vmrUrl, string mappingName, string commit)
+    {
+        var content = await GetFileContentAtCommit(
+            vmrUrl,
+            commit,
+            VmrInfo.DefaultRelativeSourceManifestPath);
+
+        var lastForwardFlowRepoSha = SourceManifest
+            .FromJson(content)?
+            .GetRepoVersion(mappingName)
+            .CommitSha;
+
+        if (lastForwardFlowRepoSha == null)
+        {
+            return null;
+        }
+
+        int lineNumber = content.Split('\n')
+            .ToList()
+            .FindIndex(line => line.Contains(lastForwardFlowRepoSha))
+            + 1; // result is 0-indexed, but github lines are 1-indexed;
+
+        // todo: we can skip this call if the last flown SHA is one that we already cached
+        string lastForwardFlowVmrSha = await BlameLineAsync(
+            vmrUrl,
+            commit,
+            VmrInfo.DefaultRelativeSourceManifestPath,
+            lineNumber);
+
+        return new ForwardFlow(lastForwardFlowRepoSha, lastForwardFlowVmrSha);
+    }
+
+    public async Task<Backflow?> GetLastIncomingBackflowAsync(string repoUrl, string commit)
+    {
+        var content = await GetFileContentAtCommit(
+            repoUrl,
+            commit,
+            VersionFiles.VersionDetailsXml);
+
+        var lastBackflowVmrSha = _versionDetailsParser.ParseVersionDetailsXml(content)
+            .Source?
+            .Sha;
+
+        if (lastBackflowVmrSha == null)
+        {
+            return null;
+        }
+
+        int lineNumber = content
+            .Split('\n')
+            .ToList()
+            .FindIndex(line =>
+                line.Contains(VersionDetailsParser.SourceElementName) &&
+                line.Contains(lastBackflowVmrSha))
+            + 1; // result is 0-indexed, but github lines are 1-indexed
+
+        // todo: we can skip this call if the last flown SHA is one that we already cached
+        string lastBackflowRepoSha = await BlameLineAsync(
+            repoUrl,
+            commit,
+            VersionFiles.VersionDetailsXml,
+            lineNumber);
+
+        return new Backflow(lastBackflowVmrSha, lastBackflowRepoSha);
+    }
+
+    private async Task<string> BlameLineAsync(string repoUrl, string commitOrBranch, string filePath, int lineNumber)
+    {
+        (string owner, string repo) = ParseRepoUri(repoUrl);
+
+        var query = $@"
+            query {{
+              repository(owner: {JsonConvert.SerializeObject(owner)}, name: {JsonConvert.SerializeObject(repo)}) {{
+                object(expression: {JsonConvert.SerializeObject(commitOrBranch)}) {{
+                  ... on Commit {{
+                    blame(path: {JsonConvert.SerializeObject(filePath)}) {{
+                      ranges {{
+                        startingLine
+                        endingLine
+                        commit {{
+                          oid
+                        }}
+                      }}
+                    }}
+                  }}
+                }}
+              }}
+            }}";
+
+        using var client = CreateHttpClient(repoUrl);
+
+        var requestBody = new { query };
+        var content = new StringContent(
+            JsonConvert.SerializeObject(requestBody, _serializerSettings),
+            Encoding.UTF8,
+            "application/json"
+        );
+
+        var response = await client.PostAsync("graphql", content);
+        response.EnsureSuccessStatusCode();
+
+        using var stream = await response.Content.ReadAsStreamAsync();
+        using var doc = await JsonDocument.ParseAsync(stream);
+
+        if (doc.RootElement.TryGetProperty("errors", out var errors))
+            throw new InvalidOperationException($"GitHub GraphQL error: {errors}");
+
+        var obj = doc.RootElement
+            .GetProperty("data")
+            .GetProperty("repository")
+            .GetProperty("object");
+
+        if (obj.ValueKind == JsonValueKind.Null)
+            throw new InvalidOperationException($"Commit or branch '{commitOrBranch}' not found.");
+
+        // The blame field comes directly from the Commit node
+        var blame = obj.GetProperty("blame");
+        var ranges = blame.GetProperty("ranges").EnumerateArray();
+
+        foreach (var range in ranges)
+        {
+            int start = range.GetProperty("startingLine").GetInt32();
+            int end = range.GetProperty("endingLine").GetInt32();
+
+            if (lineNumber >= start && lineNumber <= end)
+            {
+                var oid = range.GetProperty("commit").GetProperty("oid").GetString();
+                return oid ?? throw new InvalidOperationException("Commit OID was null.");
+            }
+        }
+
+        throw new InvalidOperationException($"Line {lineNumber} not found in blame data.");
+    }
+
+
+
+    private async Task<string> GetFileContentAtCommit(string repoUrl, string commit, string filePath)
+    {
+        (string owner, string repo) = ParseRepoUri(repoUrl);
+        var file = await GetClient(repoUrl).Repository.Content.GetAllContentsByRef(owner, repo, filePath, commit);
+        return file[0].Content;
+    }
 }

src/Microsoft.DotNet.Darc/DarcLib/GitRepoFactory.cs

@@ -22,6 +22,7 @@ public class GitRepoFactory : IGitRepoFactory
     private readonly ITelemetryRecorder _telemetryRecorder;
     private readonly IProcessManager _processManager;
     private readonly IFileSystem _fileSystem;
+    private readonly IVersionDetailsParser _versionDetailsParser;
     private readonly ILoggerFactory _loggerFactory;
     private readonly string? _temporaryPath = null;
 
@@ -31,6 +32,7 @@ public GitRepoFactory(
         ITelemetryRecorder telemetryRecorder,
         IProcessManager processManager,
         IFileSystem fileSystem,
+        IVersionDetailsParser versionDetailsParser,
         ILoggerFactory loggerFactory,
         string temporaryPath)
     {
@@ -39,6 +41,7 @@ public GitRepoFactory(
         _telemetryRecorder = telemetryRecorder;
         _processManager = processManager;
         _fileSystem = fileSystem;
+        _versionDetailsParser = versionDetailsParser;
         _loggerFactory = loggerFactory;
         _temporaryPath = temporaryPath;
     }
@@ -56,6 +59,7 @@ public GitRepoFactory(
             _processManager,
             _loggerFactory.CreateLogger<GitHubClient>(),
             _temporaryPath,
+            _versionDetailsParser,
             // Caching not in use for Darc local client.
             null),