diff --git a/src/gg.regression.unity.bots/Runtime/Scripts/StateRecorder/BotSegments/BotSegmentsPlaybackContainer.cs b/src/gg.regression.unity.bots/Runtime/Scripts/StateRecorder/BotSegments/BotSegmentsPlaybackContainer.cs index 8a3201e4..4695aef2 100644 --- a/src/gg.regression.unity.bots/Runtime/Scripts/StateRecorder/BotSegments/BotSegmentsPlaybackContainer.cs +++ b/src/gg.regression.unity.bots/Runtime/Scripts/StateRecorder/BotSegments/BotSegmentsPlaybackContainer.cs @@ -1,7 +1,9 @@ using System; using System.Collections.Generic; +using System.Linq; using RegressionGames.StateRecorder.BotSegments.Models; using StateRecorder.BotSegments.Models; +using StateRecorder.BotSegments.Models.SegmentValidations; namespace RegressionGames.StateRecorder.BotSegments { @@ -65,5 +67,25 @@ public BotSegment PeekBotSegment() return null; } + + + /** + * Collects all of the results from the top-level validations and individual bot segments + */ + public List GetAllValidationResults() + { + + // First add all the top level results + var results = Validations.Select(validation => validation.data.GetResults()).ToList(); + + // Then add the individual bot segment results + foreach (var botSegment in _botSegments) + { + results.AddRange(botSegment.validations.Select(v => v.data.GetResults())); + } + + return results; + } + } } diff --git a/src/gg.regression.unity.bots/Runtime/Scripts/StateRecorder/BotSegments/BotSegmentsPlaybackController.cs b/src/gg.regression.unity.bots/Runtime/Scripts/StateRecorder/BotSegments/BotSegmentsPlaybackController.cs index 631225bb..bb909e7d 100644 --- a/src/gg.regression.unity.bots/Runtime/Scripts/StateRecorder/BotSegments/BotSegmentsPlaybackController.cs +++ b/src/gg.regression.unity.bots/Runtime/Scripts/StateRecorder/BotSegments/BotSegmentsPlaybackController.cs @@ -1,12 +1,14 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using Newtonsoft.Json; using RegressionGames.StateRecorder.BotSegments.Models; using RegressionGames.StateRecorder.BotSegments.Models.BotActions.KeyMoments; using RegressionGames.StateRecorder.Models; using StateRecorder.BotSegments; using StateRecorder.BotSegments.Models; +using StateRecorder.BotSegments.Models.SegmentValidations; #if UNITY_EDITOR using UnityEditor; #endif @@ -664,6 +666,7 @@ public void UnloadSegmentsAndReset() _replaySuccessful = null; WaitingForKeyFrameConditions = null; + _screenRecorder.validationResults = _dataPlaybackContainer?.GetAllValidationResults() ?? new List(); _screenRecorder.StopRecording(); #if ENABLE_LEGACY_INPUT_MANAGER RGLegacyInputWrapper.StopSimulation(); @@ -704,6 +707,7 @@ public void Stop() WaitingForKeyFrameConditions = null; _lastSegmentPlaybackWarning = null; + _screenRecorder.validationResults = _dataPlaybackContainer?.GetAllValidationResults() ?? new List(); _screenRecorder.StopRecording(); #if ENABLE_LEGACY_INPUT_MANAGER RGLegacyInputWrapper.StopSimulation(); @@ -825,6 +829,7 @@ private void LogPlaybackWarning(string loggedMessage, Exception ex = null) RGDebug.LogWarning(loggedMessage); } } + FindObjectOfType()?.SetKeyFrameWarningText(loggedMessage); if (pauseEditorOnPlaybackWarning) { diff --git a/src/gg.regression.unity.bots/Runtime/Scripts/StateRecorder/BotSegments/Models/SegmentValidations/SegmentValidationResultContainer.cs b/src/gg.regression.unity.bots/Runtime/Scripts/StateRecorder/BotSegments/Models/SegmentValidations/SegmentValidationResultContainer.cs index bf91df2a..ffd4b2c0 100644 --- a/src/gg.regression.unity.bots/Runtime/Scripts/StateRecorder/BotSegments/Models/SegmentValidations/SegmentValidationResultContainer.cs +++ b/src/gg.regression.unity.bots/Runtime/Scripts/StateRecorder/BotSegments/Models/SegmentValidations/SegmentValidationResultContainer.cs @@ -1,5 +1,7 @@ using System; using JetBrains.Annotations; +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; namespace StateRecorder.BotSegments.Models.SegmentValidations { @@ -28,6 +30,7 @@ public class SegmentValidationResultContainer * The actual state of this validation result * */ + [JsonConverter(typeof(StringEnumConverter))] public SegmentValidationStatus result; public SegmentValidationResultContainer(string name, [CanBeNull] string description, SegmentValidationStatus result) diff --git a/src/gg.regression.unity.bots/Runtime/Scripts/StateRecorder/ScreenRecorder.cs b/src/gg.regression.unity.bots/Runtime/Scripts/StateRecorder/ScreenRecorder.cs index 24e6e843..30038c63 100644 --- a/src/gg.regression.unity.bots/Runtime/Scripts/StateRecorder/ScreenRecorder.cs +++ b/src/gg.regression.unity.bots/Runtime/Scripts/StateRecorder/ScreenRecorder.cs @@ -9,6 +9,7 @@ using System.Threading; using System.Threading.Tasks; using JetBrains.Annotations; +using Newtonsoft.Json; using RegressionGames.ActionManager; using RegressionGames.CodeCoverage; using RegressionGames.RemoteOrchestration; @@ -16,7 +17,9 @@ using RegressionGames.StateRecorder.BotSegments.Models.BotActions; using RegressionGames.StateRecorder.BotSegments.Models.BotCriteria; using RegressionGames.StateRecorder.Models; +using RegressionGames.Validation; using StateRecorder.BotSegments; +using StateRecorder.BotSegments.Models.SegmentValidations; #if UNITY_EDITOR using UnityEditor; #endif @@ -85,6 +88,7 @@ public class ScreenRecorder : MonoBehaviour private string _currentGameplaySessionGameMetadataPath; private string _currentGameplaySessionThumbnailPath; private string _currentGameplaySessionLogsDirectoryPrefix; + private string _currentGameplaySessionValidationsPrefix; private CancellationTokenSource _tokenSource; @@ -112,6 +116,8 @@ public class ScreenRecorder : MonoBehaviour private KeyMomentEvaluator _keyMomentEvaluator = new(); + public List validationResults = new(); + #if UNITY_EDITOR private bool _needToRefreshAssets; #endif @@ -182,6 +188,7 @@ private async Task HandleEndRecording(long tickCount, string thumbnailPath, string logsDirectoryPrefix, string gameMetadataPath, + string validationsPath, bool onDestroy = false) { if (!onDestroy) @@ -265,6 +272,9 @@ private async Task HandleEndRecording(long tickCount, ZipFile.CreateFromDirectory(keyMomentsDirectoryPrefix, keyMomentsDirectoryPrefix + ".zip"); RGDebug.LogInfo($"Finished zipping replay to file: {keyMomentsDirectoryPrefix}.zip"); }); + + // Save the validation results to the validations JSON file, if there are any + await File.WriteAllBytesAsync(_currentGameplaySessionValidationsPrefix, Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(validationResults ?? new List()))); // Finally, we also save a thumbnail, by choosing the middle file in the screenshots var screenshotFiles = Directory.GetFiles(screenshotsDirectoryPrefix); @@ -279,6 +289,9 @@ private async Task HandleEndRecording(long tickCount, // wait for the zip tasks to finish Task.WaitAll(zipTask1, zipTask2, zipTask3, zipTask4, zipTask5); + + // print validation results + RGValidateLoggerUtility.LogValidationResults(validationResults); if (!wasReplay) { @@ -575,6 +588,7 @@ private IEnumerator StartRecordingCoroutine(string referenceSessionId) _startTime = DateTime.Now; _tickQueue = new BlockingCollection<(TickDataToWriteToDisk, Action)>(new ConcurrentQueue<(TickDataToWriteToDisk, Action)>()); _tokenSource = new CancellationTokenSource(); + validationResults = new(); Directory.CreateDirectory(stateRecordingsDirectory); @@ -616,6 +630,7 @@ private IEnumerator StartRecordingCoroutine(string referenceSessionId) Directory.CreateDirectory(_currentGameplaySessionMetadataDirectoryPrefix); _currentGameplaySessionThumbnailPath = _currentGameplaySessionDirectoryPrefix + "/thumbnail.jpg"; + _currentGameplaySessionValidationsPrefix = _currentGameplaySessionDirectoryPrefix + "/validations.json"; // run the tick processor in the background, but don't hook it to the token source.. we'll manage cancelling this on our own so we don't miss processing ticks Task.Run(ProcessTicks); @@ -739,6 +754,7 @@ private void StopRecordingCleanupHelper(bool wasRecording, bool wasReplay) _currentGameplaySessionThumbnailPath, _currentGameplaySessionLogsDirectoryPrefix, _currentGameplaySessionGameMetadataPath, + _currentGameplaySessionValidationsPrefix, true); } else diff --git a/src/gg.regression.unity.bots/Runtime/Scripts/Validation/RGValidateLoggerUtility.cs b/src/gg.regression.unity.bots/Runtime/Scripts/Validation/RGValidateLoggerUtility.cs new file mode 100644 index 00000000..79032fbb --- /dev/null +++ b/src/gg.regression.unity.bots/Runtime/Scripts/Validation/RGValidateLoggerUtility.cs @@ -0,0 +1,86 @@ +using System.Collections.Generic; +using System.Text; +using JetBrains.Annotations; +using StateRecorder.BotSegments.Models.SegmentValidations; + +namespace RegressionGames.Validation +{ + + /** + * A set of utilities for logging results related to validations + */ + public class RGValidateLoggerUtility + { + + /** + * Prints out validation results in a clean way in the logs + */ + public static void LogValidationResults([CanBeNull] List results) + { + + if (results == null) + { + // For now, if there are no validations, we just print a small message + RGDebug.LogInfo("No validations were provided as part of this run"); + return; + } + + // Print out results while also collecting total results + var passed = 0; + var failed = 0; + var unknown = 0; + + var logBuilder = new StringBuilder(100_000); + + logBuilder.Append("--------------- VALIDATION RESULTS --------------- (If in the editor, click this to view more)\n\n"); + foreach (var resultSet in results) + { + logBuilder.Append("" + resultSet.name + "\n"); + foreach (var validation in resultSet.validationResults) + { + switch (validation.result) + { + case SegmentValidationStatus.PASSED: + logBuilder.Append(" [PASS] "); + passed++; + break; + case SegmentValidationStatus.FAILED: + logBuilder.Append(" [FAIL] "); + failed++; + break; + case SegmentValidationStatus.UNKNOWN: + logBuilder.Append(" [UNKNOWN] "); + unknown++; + break; + } + logBuilder.Append(validation.name + "\n"); + } + + logBuilder.Append("\n"); + } + + if (failed > 0) + { + logBuilder.Append("VALIDATIONS FAILED - "); + } + else + { + logBuilder.Append("VALIDATIONS PASSED - "); + } + + logBuilder.Append($"{failed} FAILED, {passed} PASSED ({unknown} UNKNOWN)\n\n"); + + // Finally log the results + if (failed > 0) + { + RGDebug.LogError(logBuilder.ToString()); + } + else + { + RGDebug.LogInfo(logBuilder.ToString()); + } + + } + + } +} \ No newline at end of file diff --git a/src/gg.regression.unity.bots/Runtime/Scripts/Validation/RGValidateLoggerUtility.cs.meta b/src/gg.regression.unity.bots/Runtime/Scripts/Validation/RGValidateLoggerUtility.cs.meta new file mode 100644 index 00000000..31a25e82 --- /dev/null +++ b/src/gg.regression.unity.bots/Runtime/Scripts/Validation/RGValidateLoggerUtility.cs.meta @@ -0,0 +1,3 @@ +fileFormatVersion: 2 +guid: 2c2c3e9500ae4570b6f70b0bbf1935c0 +timeCreated: 1734494119 \ No newline at end of file