diff --git a/CHANGELOG.md b/CHANGELOG.md index 673a9096..33660df6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to Stability Matrix will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning 2.0](https://semver.org/spec/v2.0.0.html). +## v2.15.7 +### Changed +- Settings file saves are now atomic to prevent corruption from interrupted writes +### Fixed +- Fixed [#1590](https://github.com/LykosAI/StabilityMatrix/issues/1590) - Startup crash when settings file is corrupted. Settings files are now self-healing with automatic recovery from null bytes, truncated JSON, and missing brackets + ## v2.15.6 ### Added - Added NVIDIA driver version warning when launching ComfyUI with CUDA 13.0 (cu130) and driver versions below 580.x diff --git a/StabilityMatrix.Core/Services/SettingsJsonSanitizer.cs b/StabilityMatrix.Core/Services/SettingsJsonSanitizer.cs new file mode 100644 index 00000000..6ed0e748 --- /dev/null +++ b/StabilityMatrix.Core/Services/SettingsJsonSanitizer.cs @@ -0,0 +1,328 @@ +using System.Reflection; +using System.Text; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Json.Serialization; +using Microsoft.Extensions.Logging; +using StabilityMatrix.Core.Models.Settings; + +namespace StabilityMatrix.Core.Services; + +/// +/// Provides methods to sanitize and recover corrupted settings JSON files. +/// +public static class SettingsJsonSanitizer +{ + private static readonly Dictionary SettingsProperties = typeof(Settings) + .GetProperties(BindingFlags.Public | BindingFlags.Instance) + .Where(property => property.CanWrite && property.GetCustomAttribute() is null) + .ToDictionary(GetJsonPropertyName, StringComparer.OrdinalIgnoreCase); + + /// + /// Strips null bytes (0x00) from raw file content. + /// + public static byte[] SanitizeBytes(byte[] rawBytes) + { + // Fast path for clean files + if (Array.IndexOf(rawBytes, (byte)0x00) < 0) + return rawBytes; + + return Array.FindAll(rawBytes, b => b != 0x00); + } + + /// + /// Ensures the JSON text has matching brackets by appending missing closing braces/brackets. + /// Uses a stack to correctly handle nested and mixed {} / [] pairs. + /// + public static string TryFixBraces(string jsonText) + { + var normalized = NormalizeClosures(jsonText, out var stack, out _, out _); + var trimmed = TrimIncompleteValue(normalized); + var rescanned = NormalizeClosures(trimmed, out stack, out var inString, out var escaped); + var sb = new StringBuilder(rescanned); + + // If truncated inside a string literal, close it first + if (inString) + { + if (escaped) + sb.Append('\\'); + sb.Append('"'); + } + + // Append missing closing brackets in correct LIFO order + while (stack.Count > 0) + { + sb.Append('\n').Append(stack.Pop()); + } + + return sb.ToString(); + } + + /// + /// Attempts to deserialize settings JSON with progressive recovery strategies. + /// Returns null if all recovery attempts fail. + /// + public static Settings? TryDeserializeWithRecovery(string jsonText, ILogger? logger = null) + { + // Step 1: Sanitize text (strip null bytes, fix braces) + var sanitized = jsonText.Replace("\0", ""); + sanitized = TryFixBraces(sanitized); + + // Step 2: Try direct deserialization of sanitized text + try + { + var settings = JsonSerializer.Deserialize(sanitized, SettingsSerializerContext.Default.Settings); + if (settings is not null) + { + logger?.LogInformation("Settings recovered after text sanitization"); + return settings; + } + } + catch (JsonException ex) + { + logger?.LogWarning( + ex, + "Sanitized text still failed to deserialize, attempting property-level recovery" + ); + } + + // Step 3: Property-level recovery using JsonNode + return TryPropertyLevelRecovery(sanitized, logger); + } + + /// + /// Attempts to parse JSON with JsonNode, remove corrupt properties, and re-deserialize. + /// + private static Settings? TryPropertyLevelRecovery(string jsonText, ILogger? logger) + { + JsonNode? rootNode; + try + { + rootNode = JsonNode.Parse( + jsonText, + documentOptions: new JsonDocumentOptions + { + AllowTrailingCommas = true, + CommentHandling = JsonCommentHandling.Skip, + } + ); + } + catch (JsonException) + { + // Try more aggressive cleanup: find the last valid closing brace + var lastBrace = jsonText.LastIndexOf('}'); + if (lastBrace <= 0) + { + logger?.LogWarning("Could not parse JSON even with JsonNode, no recoverable content found"); + return null; + } + + try + { + rootNode = JsonNode.Parse( + jsonText[..(lastBrace + 1)], + documentOptions: new JsonDocumentOptions + { + AllowTrailingCommas = true, + CommentHandling = JsonCommentHandling.Skip, + } + ); + } + catch (JsonException) + { + logger?.LogWarning("Could not parse JSON even after aggressive cleanup"); + return null; + } + } + + if (rootNode is not JsonObject rootObject) + { + logger?.LogWarning("Settings JSON root is not an object"); + return null; + } + + var settings = new Settings(); + var recoveredPropertyCount = 0; + + foreach (var property in rootObject) + { + if (property.Value is null) + continue; + + if (!SettingsProperties.TryGetValue(property.Key, out var targetProperty)) + continue; + + if (!TryDeserializePropertyValue(property.Value, targetProperty.PropertyType, out var value)) + { + logger?.LogWarning( + "Skipping corrupted settings property {PropertyName} during recovery", + property.Key + ); + continue; + } + + targetProperty.SetValue(settings, value); + recoveredPropertyCount++; + } + + logger?.LogInformation( + "Settings recovered via property-level recovery with {RecoveredPropertyCount} properties", + recoveredPropertyCount + ); + return settings; + } + + private static string NormalizeClosures( + string jsonText, + out Stack stack, + out bool inString, + out bool escaped + ) + { + stack = new Stack(); + inString = false; + escaped = false; + var normalized = new StringBuilder(jsonText.Length + 8); + + foreach (var c in jsonText) + { + if (escaped) + { + normalized.Append(c); + escaped = false; + continue; + } + + if (c == '\\' && inString) + { + normalized.Append(c); + escaped = true; + continue; + } + + if (c == '"') + { + normalized.Append(c); + inString = !inString; + continue; + } + + if (inString) + { + normalized.Append(c); + continue; + } + + switch (c) + { + case '{': + normalized.Append(c); + stack.Push('}'); + break; + case '[': + normalized.Append(c); + stack.Push(']'); + break; + case '}' + or ']': + ConsumeClosingToken(c, stack, normalized); + break; + default: + normalized.Append(c); + break; + } + } + + return normalized.ToString(); + } + + private static void ConsumeClosingToken(char token, Stack stack, StringBuilder normalized) + { + if (stack.Count == 0) + return; + + if (stack.Peek() == token) + { + normalized.Append(token); + stack.Pop(); + return; + } + + if (!stack.Contains(token)) + return; + + while (stack.Count > 0 && stack.Peek() != token) + { + normalized.Append(stack.Pop()); + } + + if (stack.Count == 0) + return; + + normalized.Append(token); + stack.Pop(); + } + + private static string TrimIncompleteValue(string jsonText) + { + var trimmed = jsonText.TrimEnd(); + if (trimmed.Length == 0) + return trimmed; + + var lastChar = trimmed[^1]; + if ( + lastChar != '"' + && lastChar != '}' + && lastChar != ']' + && !char.IsDigit(lastChar) + && lastChar != 'e' + && lastChar != 'l' + && lastChar != 's' + && lastChar != ',' + ) + { + var lastSafe = trimmed.LastIndexOfAny([',', '}', ']', '{', '[']); + if (lastSafe > 0) + { + trimmed = trimmed[..(lastSafe + 1)]; + } + } + + trimmed = trimmed.TrimEnd(); + if (trimmed.EndsWith(',')) + { + trimmed = trimmed[..^1]; + } + + return trimmed; + } + + private static string GetJsonPropertyName(PropertyInfo property) + { + return property.GetCustomAttribute()?.Name ?? property.Name; + } + + private static bool TryDeserializePropertyValue(JsonNode node, Type propertyType, out object? value) + { + try + { + value = JsonSerializer.Deserialize( + node.ToJsonString(), + propertyType, + SettingsSerializerContext.Default.Options + ); + + if (value is null && propertyType.IsValueType && Nullable.GetUnderlyingType(propertyType) is null) + { + return false; + } + + return true; + } + catch (JsonException) + { + value = null; + return false; + } + } +} diff --git a/StabilityMatrix.Core/Services/SettingsManager.cs b/StabilityMatrix.Core/Services/SettingsManager.cs index 13aa9f88..955d0ffc 100644 --- a/StabilityMatrix.Core/Services/SettingsManager.cs +++ b/StabilityMatrix.Core/Services/SettingsManager.cs @@ -4,6 +4,7 @@ using System.Reactive.Disposables; using System.Reactive.Linq; using System.Reflection; +using System.Text; using System.Text.Json; using AsyncAwaitBestPractices; using CompiledExpressions; @@ -468,23 +469,15 @@ protected virtual void LoadSettings(CancellationToken cancellationToken = defaul return; } - using var fileStream = SettingsFile.Info.OpenRead(); + var rawBytes = File.ReadAllBytes(SettingsFile); - if (fileStream.Length == 0) + if (rawBytes.Length == 0) { logger.LogWarning("Settings file is empty, using default settings"); return; } - var loadedSettings = JsonSerializer.Deserialize( - fileStream, - SettingsSerializerContext.Default.Settings - ); - - if (loadedSettings is not null) - { - Settings = loadedSettings; - } + Settings = DeserializeOrRecoverSettings(rawBytes); } finally { @@ -511,24 +504,15 @@ protected virtual async Task LoadSettingsAsync(CancellationToken cancellationTok return; } - await using var fileStream = SettingsFile.Info.OpenRead(); + var rawBytes = await File.ReadAllBytesAsync(SettingsFile, cancellationToken).ConfigureAwait(false); - if (fileStream.Length == 0) + if (rawBytes.Length == 0) { logger.LogWarning("Settings file is empty, using default settings"); return; } - var loadedSettings = await JsonSerializer - .DeserializeAsync(fileStream, SettingsSerializerContext.Default.Settings, cancellationToken) - .ConfigureAwait(false); - - if (loadedSettings is not null) - { - Settings = loadedSettings; - } - - Loaded?.Invoke(this, EventArgs.Empty); + Settings = DeserializeOrRecoverSettings(rawBytes); } finally { @@ -540,6 +524,61 @@ protected virtual async Task LoadSettingsAsync(CancellationToken cancellationTok } } + /// + /// Attempts to deserialize settings from raw bytes, falling back to sanitization + /// and recovery if the JSON is corrupted. Returns default settings as a last resort. + /// + private Settings DeserializeOrRecoverSettings(byte[] rawBytes) + { + // Try normal deserialization first + try + { + var loadedSettings = JsonSerializer.Deserialize( + rawBytes, + SettingsSerializerContext.Default.Settings + ); + + if (loadedSettings is not null) + { + return loadedSettings; + } + } + catch (JsonException ex) + { + logger.LogWarning(ex, "Failed to deserialize settings, attempting recovery"); + } + + // Recovery path: backup corrupted file, sanitize, and attempt recovery + BackupCorruptedFile(rawBytes); + + var jsonText = Encoding.UTF8.GetString(SettingsJsonSanitizer.SanitizeBytes(rawBytes)); + var recovered = SettingsJsonSanitizer.TryDeserializeWithRecovery(jsonText, logger); + + if (recovered is not null) + { + logger.LogInformation("Settings recovered from corrupted file"); + return recovered; + } + + logger.LogWarning("Could not recover settings from corrupted file, using defaults"); + return new Settings(); + } + + private void BackupCorruptedFile(byte[] rawBytes) + { + try + { + var timestamp = DateTime.Now.ToString("yyyyMMdd-HHmmss"); + var backupPath = SettingsFile + $".{timestamp}.bak"; + File.WriteAllBytes(backupPath, rawBytes); + logger.LogInformation("Backed up corrupted settings file to {BackupPath}", backupPath); + } + catch (Exception ex) + { + logger.LogWarning(ex, "Failed to create backup of corrupted settings file"); + } + } + protected virtual void SaveSettings(CancellationToken cancellationToken = default) { // Skip saving if not loaded yet @@ -550,15 +589,13 @@ protected virtual void SaveSettings(CancellationToken cancellationToken = defaul try { - // Create empty settings file if it doesn't exist - if (!SettingsFile.Exists) - { - SettingsFile.Directory?.Create(); - SettingsFile.Create(); - } + SettingsFile.Directory?.Create(); // Check disk space - if (SystemInfo.GetDiskFreeSpaceBytes(SettingsFile) is < 1 * SystemInfo.Mebibyte) + if ( + SettingsFile.Exists + && SystemInfo.GetDiskFreeSpaceBytes(SettingsFile) is < 1 * SystemInfo.Mebibyte + ) { logger.LogWarning("Not enough disk space to save settings"); return; @@ -575,12 +612,18 @@ protected virtual void SaveSettings(CancellationToken cancellationToken = defaul return; } - using var fs = File.Open(SettingsFile, FileMode.Open); - if (fs.CanWrite) + // Write to temp file then rename for atomic save + var tempPath = SettingsFile + ".tmp"; + File.WriteAllBytes(tempPath, jsonBytes); + try + { + File.Move(tempPath, SettingsFile, overwrite: true); + } + catch (Exception ex) { - fs.Write(jsonBytes, 0, jsonBytes.Length); - fs.Flush(); - fs.SetLength(jsonBytes.Length); + logger.LogWarning(ex, "Failed to move temp settings file, cleaning up"); + try { File.Delete(tempPath); } catch { /* best effort */ } + throw; } } finally @@ -599,15 +642,13 @@ protected virtual async Task SaveSettingsAsync(CancellationToken cancellationTok try { - // Create empty settings file if it doesn't exist - if (!SettingsFile.Exists) - { - SettingsFile.Directory?.Create(); - SettingsFile.Create(); - } + SettingsFile.Directory?.Create(); // Check disk space - if (SystemInfo.GetDiskFreeSpaceBytes(SettingsFile) is < 1 * SystemInfo.Mebibyte) + if ( + SettingsFile.Exists + && SystemInfo.GetDiskFreeSpaceBytes(SettingsFile) is < 1 * SystemInfo.Mebibyte + ) { logger.LogWarning("Not enough disk space to save settings"); return; @@ -624,12 +665,18 @@ protected virtual async Task SaveSettingsAsync(CancellationToken cancellationTok return; } - await using var fs = File.Open(SettingsFile, FileMode.Open); - if (fs.CanWrite) + // Write to temp file then rename for atomic save + var tempPath = SettingsFile + ".tmp"; + await File.WriteAllBytesAsync(tempPath, jsonBytes, cancellationToken).ConfigureAwait(false); + try + { + File.Move(tempPath, SettingsFile, overwrite: true); + } + catch (Exception ex) { - await fs.WriteAsync(jsonBytes, cancellationToken).ConfigureAwait(false); - await fs.FlushAsync(cancellationToken).ConfigureAwait(false); - fs.SetLength(jsonBytes.Length); + logger.LogWarning(ex, "Failed to move temp settings file, cleaning up"); + try { File.Delete(tempPath); } catch { /* best effort */ } + throw; } } finally diff --git a/StabilityMatrix.Tests/Core/SettingsJsonSanitizerTests.cs b/StabilityMatrix.Tests/Core/SettingsJsonSanitizerTests.cs new file mode 100644 index 00000000..b684a911 --- /dev/null +++ b/StabilityMatrix.Tests/Core/SettingsJsonSanitizerTests.cs @@ -0,0 +1,286 @@ +using System.Text; +using System.Text.Json; +using StabilityMatrix.Core.Models.Settings; +using StabilityMatrix.Core.Services; + +namespace StabilityMatrix.Tests.Core; + +[TestClass] +public class SettingsJsonSanitizerTests +{ + [TestMethod] + public void SanitizeBytes_RemovesNullBytes() + { + var input = Encoding.UTF8.GetBytes("{\"key\": 123}"); + // Insert null bytes + var corrupted = new byte[input.Length + 3]; + Array.Copy(input, 0, corrupted, 0, 5); + corrupted[5] = 0x00; + corrupted[6] = 0x00; + corrupted[7] = 0x00; + Array.Copy(input, 5, corrupted, 8, input.Length - 5); + + var result = SettingsJsonSanitizer.SanitizeBytes(corrupted); + var resultText = Encoding.UTF8.GetString(result); + + Assert.AreEqual("{\"key\": 123}", resultText); + } + + [TestMethod] + public void SanitizeBytes_CleanInput_ReturnsSameArray() + { + var input = Encoding.UTF8.GetBytes("{\"key\": 123}"); + + var result = SettingsJsonSanitizer.SanitizeBytes(input); + + // Should return the same reference (no copy) for clean input + Assert.AreSame(input, result); + } + + [TestMethod] + public void TryFixBraces_MissingClosingBrace_AppendsBrace() + { + var input = """{"key": "value" """; + + var result = SettingsJsonSanitizer.TryFixBraces(input); + + // Should be valid JSON now + Assert.IsNotNull(JsonDocument.Parse(result)); + } + + [TestMethod] + public void TryFixBraces_ValidJson_ReturnsUnchanged() + { + var input = """{"key": "value"}"""; + + var result = SettingsJsonSanitizer.TryFixBraces(input); + + Assert.AreEqual(input, result); + } + + [TestMethod] + public void TryFixBraces_NestedMissingBraces_AppendsMultiple() + { + var input = """{"outer": {"inner": "value" """; + + var result = SettingsJsonSanitizer.TryFixBraces(input); + + Assert.IsNotNull(JsonDocument.Parse(result)); + } + + [TestMethod] + public void TryFixBraces_MixedBrackets_ClosesInCorrectOrder() + { + // Truncated JSON with an open array inside an object + var input = """{"items": [1, 2, 3"""; + + var result = SettingsJsonSanitizer.TryFixBraces(input); + + // Should close ] then } in correct LIFO order + Assert.IsNotNull(JsonDocument.Parse(result)); + Assert.IsTrue(result.TrimEnd().EndsWith('}')); + } + + [TestMethod] + public void TryFixBraces_MissingArrayCloseBeforeObjectClose_InsertsArrayClose() + { + var input = """{"items": [1, 2, 3}"""; + + var result = SettingsJsonSanitizer.TryFixBraces(input); + + var document = JsonDocument.Parse(result); + var items = document.RootElement.GetProperty("items"); + Assert.AreEqual(JsonValueKind.Array, items.ValueKind); + Assert.AreEqual(3, items.GetArrayLength()); + } + + [TestMethod] + public void TryFixBraces_TruncatedInsideString_ClosesStringAndBraces() + { + // JSON truncated in the middle of a string value + var input = """{"Theme": "Da"""; + + var result = SettingsJsonSanitizer.TryFixBraces(input); + + // Should close the string literal, then the brace + Assert.IsNotNull(JsonDocument.Parse(result)); + } + + [TestMethod] + public void TryDeserializeWithRecovery_ValidJson_ReturnsSettings() + { + var json = """ + { + "Version": 1, + "Theme": "Dark", + "InferenceDimensionStepChange": 64 + } + """; + + var result = SettingsJsonSanitizer.TryDeserializeWithRecovery(json); + + Assert.IsNotNull(result); + Assert.AreEqual(1, result.Version); + Assert.AreEqual("Dark", result.Theme); + Assert.AreEqual(64, result.InferenceDimensionStepChange); + } + + [TestMethod] + public void TryDeserializeWithRecovery_NullBytesInNumber_Recovers() + { + // Simulate the exact issue from #1590: null byte in InferenceDimensionStepChange value + var json = "{\n \"Version\": 1,\n \"InferenceDimensionStepChange\": 12\08\n}"; + + var result = SettingsJsonSanitizer.TryDeserializeWithRecovery(json); + + Assert.IsNotNull(result); + Assert.AreEqual(1, result.Version); + // The value 128 is recovered after stripping the null byte + Assert.AreEqual(128, result.InferenceDimensionStepChange); + } + + [TestMethod] + public void TryDeserializeWithRecovery_NullBytesScattered_Recovers() + { + var json = "{\n \"Theme\": \"Da\0rk\",\n \"CheckForUpdates\": tr\0ue\n}"; + + var result = SettingsJsonSanitizer.TryDeserializeWithRecovery(json); + + Assert.IsNotNull(result); + Assert.AreEqual("Dark", result.Theme); + Assert.AreEqual(true, result.CheckForUpdates); + } + + [TestMethod] + public void TryDeserializeWithRecovery_TruncatedJson_RecoversSalvageableProperties() + { + var json = """ + { + "Version": 1, + "Theme": "Dark", + "InferenceDimensionStepChange": 64, + "CheckForUpdates": true, + "ConsoleFontSize": + """; + + var result = SettingsJsonSanitizer.TryDeserializeWithRecovery(json); + + Assert.IsNotNull(result); + Assert.AreEqual(1, result.Version); + Assert.AreEqual("Dark", result.Theme); + Assert.AreEqual(64, result.InferenceDimensionStepChange); + Assert.AreEqual(true, result.CheckForUpdates); + } + + [TestMethod] + public void TryDeserializeWithRecovery_TotallyCorrupt_ReturnsNull() + { + var json = "not json at all !@#$%^&*()"; + + var result = SettingsJsonSanitizer.TryDeserializeWithRecovery(json); + + Assert.IsNull(result); + } + + [TestMethod] + public void TryDeserializeWithRecovery_EmptyObject_ReturnsSettingsWithDefaults() + { + var json = "{}"; + + var result = SettingsJsonSanitizer.TryDeserializeWithRecovery(json); + + Assert.IsNotNull(result); + // Should have default values + Assert.AreEqual(128, result.InferenceDimensionStepChange); + Assert.AreEqual(14, result.ConsoleFontSize); + } + + [TestMethod] + public void TryDeserializeWithRecovery_PreservesValidProperties_WhenOthersCorrupt() + { + // JSON where Theme is valid but we have an invalid property type + var json = """ + { + "Version": 1, + "Theme": "Dark", + "FirstLaunchSetupComplete": true, + "InferenceDimensionStepChange": 64 + } + """; + + var result = SettingsJsonSanitizer.TryDeserializeWithRecovery(json); + + Assert.IsNotNull(result); + Assert.AreEqual("Dark", result.Theme); + Assert.AreEqual(true, result.FirstLaunchSetupComplete); + Assert.AreEqual(64, result.InferenceDimensionStepChange); + } + + [TestMethod] + public void TryDeserializeWithRecovery_InvalidPropertyType_SkipsBadPropertyAndPreservesOthers() + { + var json = """ + { + "Theme": "Dark", + "CheckForUpdates": true, + "InferenceDimensionStepChange": "oops" + } + """; + + var result = SettingsJsonSanitizer.TryDeserializeWithRecovery(json); + + Assert.IsNotNull(result); + Assert.AreEqual("Dark", result.Theme); + Assert.AreEqual(true, result.CheckForUpdates); + Assert.AreEqual(128, result.InferenceDimensionStepChange); + } + + [TestMethod] + public void TryDeserializeWithRecovery_MissingArrayCloseBeforeObjectClose_Recovers() + { + var json = """ + { + "Theme": "Dark", + "SelectedBaseModels": ["sdxl"} + """; + + var result = SettingsJsonSanitizer.TryDeserializeWithRecovery(json); + + Assert.IsNotNull(result); + Assert.AreEqual("Dark", result.Theme); + CollectionAssert.AreEqual(new[] { "sdxl" }, result.SelectedBaseModels); + } + + [TestMethod] + public void TryDeserializeWithRecovery_MissingClosingBrace_Recovers() + { + var json = """ + { + "Version": 1, + "Theme": "Dark" + """; + + var result = SettingsJsonSanitizer.TryDeserializeWithRecovery(json); + + Assert.IsNotNull(result); + Assert.AreEqual(1, result.Version); + Assert.AreEqual("Dark", result.Theme); + } + + [TestMethod] + public void TryDeserializeWithRecovery_TrailingComma_Recovers() + { + var json = """ + { + "Version": 1, + "Theme": "Dark", + } + """; + + var result = SettingsJsonSanitizer.TryDeserializeWithRecovery(json); + + Assert.IsNotNull(result); + Assert.AreEqual(1, result.Version); + Assert.AreEqual("Dark", result.Theme); + } +}