diff --git a/PatchPanda.Units/Services/VersionServiceTests.cs b/PatchPanda.Units/Services/VersionServiceTests.cs new file mode 100644 index 0000000..420c308 --- /dev/null +++ b/PatchPanda.Units/Services/VersionServiceTests.cs @@ -0,0 +1,124 @@ +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; +using Octokit; +using PatchPanda.Units.Helpers; +using PatchPanda.Web; +using PatchPanda.Web.Db; +using PatchPanda.Web.Services; + +namespace PatchPanda.Units.Services; + +public class VersionServiceTests +{ + private class TestableVersionService : VersionService + { + public Mock MockGitHubClient { get; } = new(); + + public TestableVersionService( + ILogger logger, + IConfiguration configuration, + IDbContextFactory dbContextFactory, + IAiService aiService + ) : base(logger, configuration, dbContextFactory, aiService) + { + } + + protected override IGitHubClient GetClient() + { + return MockGitHubClient.Object; + } + } + + private readonly Mock> _logger = new(); + private readonly Mock _configuration = new(); + private readonly Mock _aiService = new(); + private readonly IDbContextFactory _dbContextFactory = Helper.CreateInMemoryFactory(); + + [Fact] + public async Task SecurityCheck_Retries_On_Failure() + { + // Arrange + var service = new TestableVersionService( + _logger.Object, + _configuration.Object, + _dbContextFactory, + _aiService.Object + ); + + using var db = _dbContextFactory.CreateDbContext(); + + // Setup App Settings + db.AppSettings.Add(new AppSetting { Key = Constants.SettingsKeys.SECURITY_SCANNING_ENABLED, Value = "true" }); + await db.SaveChangesAsync(); + + var stack = Helper.GetTestStack(); + var app = stack.Apps[0]; + app.GitHubRepo = new Tuple(TestData.GITHUB_OWNER, TestData.GITHUB_REPO); + app.NewerVersions.Clear(); + + db.Containers.Add(app); + await db.SaveChangesAsync(); + + // Setup AI Service + _aiService.Setup(x => x.IsInitialized()).Returns(true); + + // Fail twice, then succeed + _aiService.SetupSequence(x => x.AnalyzeDiff(It.IsAny())) + .ThrowsAsync(new Exception(TestData.AI_ERROR)) + .ThrowsAsync(new Exception(TestData.AI_ERROR)) + .ReturnsAsync(new SecurityAnalysisResult { Analysis = TestData.SAFE_ANALYSIS, IsSuspectedMalicious = false }); + + // Setup GitHub Client + var mockRepo = new Mock(); + var mockRelease = new Mock(); + var mockCommit = new Mock(); + + service.MockGitHubClient.Setup(x => x.Repository).Returns(mockRepo.Object); + mockRepo.Setup(x => x.Release).Returns(mockRelease.Object); + mockRepo.Setup(x => x.Commit).Returns(mockCommit.Object); + + var releaseNew = new Release( + TestData.DUMMY_URL, TestData.DUMMY_URL, TestData.DUMMY_URL, TestData.DUMMY_URL, 1, "nodeId", TestData.NEW_VERSION, + "master", TestData.NEW_VERSION, TestData.BODY, false, false, DateTimeOffset.Now, + DateTimeOffset.Now, new Author(), TestData.DUMMY_URL, TestData.DUMMY_URL, null + ); + + var releaseCurrent = new Release( + TestData.DUMMY_URL, TestData.DUMMY_URL, TestData.DUMMY_URL, TestData.DUMMY_URL, 2, "nodeId", TestData.VERSION, + "master", TestData.VERSION, TestData.BODY, false, false, DateTimeOffset.Now, + DateTimeOffset.Now, new Author(), TestData.DUMMY_URL, TestData.DUMMY_URL, null + ); + + mockRelease.Setup(x => x.GetAll(TestData.GITHUB_OWNER, TestData.GITHUB_REPO, It.IsAny())) + .ReturnsAsync(new List { releaseNew, releaseCurrent }); + + // Construct GitHubCommitFile with necessary patch content + // GitHubCommitFile(filename, additions, deletions, changes, status, blobUrl, contentsUrl, rawUrl, sha, patch, previousFileName) + var commitFile = new GitHubCommitFile("filename", 0, 0, 0, "status", TestData.DUMMY_URL, TestData.DUMMY_URL, TestData.DUMMY_URL, "sha", TestData.PATCH_CONTENT, null); + + // Construct CompareResult + // CompareResult(url, htmlUrl, permalinkUrl, diffUrl, patchUrl, baseCommit, mergeBaseCommit, status, aheadBy, behindBy, totalCommits, commits, files) + var compareResult = new CompareResult( + TestData.DUMMY_URL, TestData.DUMMY_URL, TestData.DUMMY_URL, TestData.DUMMY_URL, TestData.DUMMY_URL, + null, // baseCommit + null, // mergeBaseCommit + "ahead", // status + 1, // aheadBy + 0, // behindBy + 1, // totalCommits + new List(), // commits + new List { commitFile } // files + ); + + mockCommit.Setup(x => x.Compare(TestData.GITHUB_OWNER, TestData.GITHUB_REPO, It.IsAny(), It.IsAny())) + .ReturnsAsync(compareResult); + + // Act + var result = await service.GetNewerVersions(app, []); + + // Assert + _aiService.Verify(x => x.AnalyzeDiff(It.IsAny()), Times.AtLeast(3)); + Assert.Single(result); + Assert.Equal(TestData.SAFE_ANALYSIS, result.First().SecurityAnalysis); + } +} diff --git a/PatchPanda.Units/TestData.cs b/PatchPanda.Units/TestData.cs index 0a3eb0f..e9cdfbd 100644 --- a/PatchPanda.Units/TestData.cs +++ b/PatchPanda.Units/TestData.cs @@ -28,4 +28,10 @@ public static class TestData public const string RELEASE_TAG_A = "v1.0.0"; public const string RELEASE_TAG_B = "v2.0.0"; public const string ALPINE_IMAGE = "alpine:3.16"; + + public const string SAFE_ANALYSIS = "Safe"; + public const string AI_ERROR = "AI Error"; + public const string PATCH_CONTENT = "patch_content"; + public const string DUMMY_URL = "http://dummy.url"; + public const string BODY = "body"; } diff --git a/PatchPanda.Web/Services/VersionService.cs b/PatchPanda.Web/Services/VersionService.cs index 158ec63..966decd 100644 --- a/PatchPanda.Web/Services/VersionService.cs +++ b/PatchPanda.Web/Services/VersionService.cs @@ -35,7 +35,7 @@ IAiService aiService _aiService = aiService; } - private GitHubClient GetClient() + protected virtual IGitHubClient GetClient() { var client = new GitHubClient(new ProductHeaderValue("PatchPanda")); @@ -174,58 +174,86 @@ await db.AppSettings.FirstOrDefaultAsync(x => if (securityScanningEnabled && _aiService.IsInitialized()) { - try - { - var client = GetClient(); + // Resolve the correct tag for the current version to ensure comparison works + // We extract the semantic version portion from the app version and use it to find the source tag + var adjustedRegex = app.GitHubVersionRegex.TrimStart('^', 'v').TrimEnd('$'); + var versionMatch = Regex.Match(app.Version, adjustedRegex); - // Resolve the correct tag for the current version to ensure comparison works - // We extract the semantic version portion from the app version and use it to find the source tag - var adjustedRegex = app.GitHubVersionRegex.TrimStart('^', 'v').TrimEnd('$'); - var versionMatch = Regex.Match(app.Version, adjustedRegex); + if (versionMatch.Success) + { + var currentRelease = allReleases.FirstOrDefault(r => + r.TagName is not null + && Regex.IsMatch(r.TagName, Regex.Escape(versionMatch.Value)) + ); - if (versionMatch.Success) + if (currentRelease != null) { - var currentRelease = allReleases.FirstOrDefault(r => - r.TagName is not null - && Regex.IsMatch(r.TagName, Regex.Escape(versionMatch.Value)) - ); + var baseTag = currentRelease.TagName; - if (currentRelease != null) + for (int i = 1; i <= Constants.Limits.MAX_OLLAMA_ATTEMPTS; i++) { - var baseTag = currentRelease.TagName; - - // Get the difference between the current version and the new version - var diff = await client.Repository.Commit.Compare( - repo.Item1, - repo.Item2, - baseTag, - notSeenNewVersion.VersionNumber - ); - - var textToAnalyze = string.Concat( - diff.Files.Select(f => f.Patch ?? "") - ); - - var analysis = await _aiService.AnalyzeDiff(textToAnalyze); - - if (analysis is not null) + try + { + var client = GetClient(); + + // Get the difference between the current version and the new version + var diff = await client.Repository.Commit.Compare( + repo.Item1, + repo.Item2, + baseTag, + notSeenNewVersion.VersionNumber + ); + + var textToAnalyze = string.Concat( + diff.Files.Select(f => f.Patch ?? "") + ); + + var analysis = await _aiService.AnalyzeDiff(textToAnalyze); + + if (analysis is not null) + { + notSeenNewVersion.SecurityAnalysis = analysis.Analysis; + notSeenNewVersion.IsSuspectedMalicious = + analysis.IsSuspectedMalicious; + break; + } + else + { + _logger.LogWarning( + "Attempting to perform security scan for {Repo} version {Version}, attempt {Count} out of {Max} resulted in null analysis.", + $"{repo.Item1}/{repo.Item2}", + notSeenNewVersion.VersionNumber, + i, + Constants.Limits.MAX_OLLAMA_ATTEMPTS + ); + } + } + catch (Exception ex) { - notSeenNewVersion.SecurityAnalysis = analysis.Analysis; - notSeenNewVersion.IsSuspectedMalicious = - analysis.IsSuspectedMalicious; + if (i == Constants.Limits.MAX_OLLAMA_ATTEMPTS) + { + _logger.LogError( + ex, + "Failed to perform security scan for {Repo} version {Version}", + $"{repo.Item1}/{repo.Item2}", + notSeenNewVersion.VersionNumber + ); + } + else + { + _logger.LogWarning( + ex, + "Attempting to perform security scan for {Repo} version {Version}, attempt {Count} out of {Max} failed.", + $"{repo.Item1}/{repo.Item2}", + notSeenNewVersion.VersionNumber, + i, + Constants.Limits.MAX_OLLAMA_ATTEMPTS + ); + } } } } } - catch (Exception ex) - { - _logger.LogError( - ex, - "Failed to perform security scan for {Repo} version {Version}", - $"{repo.Item1}/{repo.Item2}", - notSeenNewVersion.VersionNumber - ); - } } if (_aiService.IsInitialized())