diff --git a/src/OpenClaw.Agent/AgentRuntime.cs b/src/OpenClaw.Agent/AgentRuntime.cs index 16e6f47..bf48be8 100644 --- a/src/OpenClaw.Agent/AgentRuntime.cs +++ b/src/OpenClaw.Agent/AgentRuntime.cs @@ -65,6 +65,7 @@ public sealed class AgentRuntime : IAgentRuntime private readonly SkillsConfig? _skillsConfig; private readonly string? _skillWorkspacePath; private readonly IReadOnlyList _pluginSkillDirs; + private readonly string? _memoryRecallPrefix; private readonly object _skillGate = new(); private string[] _loadedSkillNames = []; private int _skillPromptLength; @@ -158,6 +159,9 @@ public AgentRuntime( _isContractRuntimeBudgetExceeded = isContractRuntimeBudgetExceeded; _recordContractTurnUsage = recordContractTurnUsage; _appendContractSnapshot = appendContractSnapshot; + var projectId = gatewayConfig?.Memory.ProjectId + ?? Environment.GetEnvironmentVariable("OPENCLAW_PROJECT"); + _memoryRecallPrefix = string.IsNullOrWhiteSpace(projectId) ? null : $"project:{projectId.Trim()}:"; ApplySkills(skills ?? []); } @@ -632,9 +636,16 @@ private async ValueTask TryInjectRecallAsync(List messages, string try { var limit = Math.Clamp(_recall.MaxNotes, 1, 32); - var hits = await search.SearchNotesAsync(userMessage, prefix: null, limit, ct); + _metrics?.IncrementMemoryRecallSearches(); + var hits = await search.SearchNotesAsync(userMessage, _memoryRecallPrefix, limit, ct); + if (hits.Count == 0 && !string.IsNullOrWhiteSpace(_memoryRecallPrefix)) + { + _metrics?.IncrementMemoryRecallSearches(); + hits = await search.SearchNotesAsync(userMessage, prefix: null, limit, ct); + } if (hits.Count == 0) return; + _metrics?.AddMemoryRecallHits(hits.Count); var maxChars = Math.Clamp(_recall.MaxChars, 256, 100_000); @@ -1258,6 +1269,7 @@ public async Task CompactHistoryAsync(Session session, CancellationToken ct) if (!string.IsNullOrWhiteSpace(summary)) { + _metrics?.IncrementMemoryCompactions(); session.History.RemoveRange(0, toSummarizeCount); session.History.Insert(0, new ChatTurn { diff --git a/src/OpenClaw.Core/Memory/FileMemoryStore.cs b/src/OpenClaw.Core/Memory/FileMemoryStore.cs index c6e05b5..83ee806 100644 --- a/src/OpenClaw.Core/Memory/FileMemoryStore.cs +++ b/src/OpenClaw.Core/Memory/FileMemoryStore.cs @@ -6,6 +6,7 @@ using Microsoft.Extensions.Logging; using OpenClaw.Core.Abstractions; using OpenClaw.Core.Models; +using OpenClaw.Core.Observability; namespace OpenClaw.Core.Memory; @@ -37,12 +38,17 @@ public sealed class FileMemoryStore : IMemoryStore, IMemoryNoteSearch, IMemoryRe private readonly string _branchesPath; private readonly IMemoryCache _sessionCache; private readonly SemaphoreSlim[] _sessionLoadStripes; + private readonly SemaphoreSlim _noteIndexGate = new(1, 1); + private readonly ConcurrentDictionary _noteIndex = new(StringComparer.Ordinal); private readonly ILogger? _logger; + private readonly RuntimeMetrics? _metrics; + private int _noteIndexInitialized; - public FileMemoryStore(string basePath, int maxCachedSessions = 100, ILogger? logger = null) + public FileMemoryStore(string basePath, int maxCachedSessions = 100, ILogger? logger = null, RuntimeMetrics? metrics = null) { _basePath = basePath ?? throw new ArgumentNullException(nameof(basePath)); _logger = logger; + _metrics = metrics; _sessionsPath = Path.Combine(_basePath, "sessions"); _notesPath = Path.Combine(_basePath, "notes"); @@ -68,7 +74,11 @@ public FileMemoryStore(string basePath, int maxCachedSessions = 100, ILogger> ListNotesWithPrefixAsync(string prefix, CancellationToken ct) { - var results = new List(); - - try - { - var files = Directory.EnumerateFiles(_notesPath, "*.md"); - foreach (var file in files) - { - var encodedKey = Path.GetFileNameWithoutExtension(file); - var key = ResolveNoteKey(encodedKey); - - if (key.StartsWith(prefix, StringComparison.Ordinal)) - results.Add(key); - } - } - catch - { - // Return empty list on error - } - - return ValueTask.FromResult>(results); + return ListNotesWithPrefixCoreAsync(prefix ?? "", ct); } public async ValueTask> SearchNotesAsync(string query, string? prefix, int limit, CancellationToken ct) @@ -340,56 +335,48 @@ public async ValueTask> SearchNotesAsync(string que limit = Math.Clamp(limit, 1, 50); prefix ??= ""; - - var hits = new List(capacity: Math.Min(limit, 16)); try { - foreach (var file in Directory.EnumerateFiles(_notesPath, "*.md")) + await EnsureNoteIndexLoadedAsync(ct); + var normalizedQuery = NormalizeSearchText(query); + if (normalizedQuery.Length == 0) + return []; + + var terms = BuildQueryTerms(normalizedQuery); + var candidates = _noteIndex.Values + .Where(entry => string.IsNullOrEmpty(prefix) || entry.Key.StartsWith(prefix, StringComparison.Ordinal)) + .Select(entry => new { Entry = entry, Score = ScoreNoteEntry(entry, normalizedQuery, terms) }) + .Where(static item => item.Score > 0) + .OrderByDescending(static item => item.Score) + .ThenByDescending(static item => item.Entry.UpdatedAt) + .ThenBy(static item => item.Entry.Key, StringComparer.Ordinal) + .Take(Math.Min(limit * 4, 64)) + .ToArray(); + + var hits = new List(capacity: Math.Min(limit, candidates.Length)); + foreach (var candidate in candidates) { ct.ThrowIfCancellationRequested(); - var encodedKey = Path.GetFileNameWithoutExtension(file); - var key = ResolveNoteKey(encodedKey); - - if (!string.IsNullOrEmpty(prefix) && !key.StartsWith(prefix, StringComparison.Ordinal)) - continue; - - string content; - try - { - content = await File.ReadAllTextAsync(file, ct); - } - catch - { - continue; - } - - if (content.IndexOf(query, StringComparison.OrdinalIgnoreCase) < 0 && - key.IndexOf(query, StringComparison.OrdinalIgnoreCase) < 0) - { - continue; - } - - var updatedAt = File.GetLastWriteTimeUtc(file); - + var content = await LoadNoteAsync(candidate.Entry.Key, ct) ?? candidate.Entry.PreviewContent; hits.Add(new MemoryNoteHit { - Key = key, + Key = candidate.Entry.Key, Content = content, - UpdatedAt = new DateTimeOffset(updatedAt, TimeSpan.Zero), - Score = 1.0f + UpdatedAt = candidate.Entry.UpdatedAt, + Score = candidate.Score }); if (hits.Count >= limit) break; } + + return hits; } catch { return []; } - - return hits; } public async ValueTask SaveBranchAsync(SessionBranch branch, CancellationToken ct) @@ -794,6 +781,129 @@ private ValueTask AddToCacheAsync(string sessionId, Session session) return ValueTask.CompletedTask; } + private async ValueTask> ListNotesWithPrefixCoreAsync(string prefix, CancellationToken ct) + { + try + { + await EnsureNoteIndexLoadedAsync(ct); + return _noteIndex.Keys + .Where(key => key.StartsWith(prefix, StringComparison.Ordinal)) + .OrderBy(static key => key, StringComparer.Ordinal) + .ToArray(); + } + catch + { + return []; + } + } + + private async ValueTask EnsureNoteIndexLoadedAsync(CancellationToken ct) + { + if (Volatile.Read(ref _noteIndexInitialized) != 0) + return; + + await _noteIndexGate.WaitAsync(ct); + try + { + if (_noteIndexInitialized != 0) + return; + + _noteIndex.Clear(); + foreach (var file in Directory.EnumerateFiles(_notesPath, "*.md")) + { + ct.ThrowIfCancellationRequested(); + + var encodedKey = Path.GetFileNameWithoutExtension(file); + var key = ResolveNoteKey(encodedKey); + + string content; + try + { + content = await File.ReadAllTextAsync(file, ct); + } + catch + { + continue; + } + + var updatedAt = new DateTimeOffset(File.GetLastWriteTimeUtc(file), TimeSpan.Zero); + _noteIndex[key] = CreateNoteIndexEntry(key, content, updatedAt); + } + + Volatile.Write(ref _noteIndexInitialized, 1); + } + finally + { + _noteIndexGate.Release(); + } + } + + private void UpsertNoteIndexEntry(string key, string content, DateTimeOffset updatedAt) + { + if (Volatile.Read(ref _noteIndexInitialized) == 0) + return; + + _noteIndex[key] = CreateNoteIndexEntry(key, content, updatedAt); + } + + private static NoteIndexEntry CreateNoteIndexEntry(string key, string content, DateTimeOffset updatedAt) + { + content ??= ""; + return new NoteIndexEntry + { + Key = key, + PreviewContent = content.Length <= 4_096 ? content : content[..4_096] + "…", + SearchText = NormalizeSearchText($"{key}\n{content}"), + UpdatedAt = updatedAt + }; + } + + private static string NormalizeSearchText(string value) + { + if (string.IsNullOrWhiteSpace(value)) + return string.Empty; + + var normalized = value.Replace("\r\n", "\n", StringComparison.Ordinal) + .Replace('\r', '\n') + .ToLowerInvariant(); + + return normalized.Length <= 16_384 ? normalized : normalized[..16_384]; + } + + private static string[] BuildQueryTerms(string normalizedQuery) + { + return normalizedQuery + .Split([' ', '\n', '\t', ',', '.', ';', ':', '!', '?', '(', ')', '[', ']', '{', '}', '"', '\'', '/', '\\', '-', '_'], StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .Where(static term => term.Length >= 3) + .Distinct(StringComparer.Ordinal) + .Take(8) + .ToArray(); + } + + private static float ScoreNoteEntry(NoteIndexEntry entry, string normalizedQuery, IReadOnlyList terms) + { + var score = 0f; + if (entry.SearchText.Contains(normalizedQuery, StringComparison.Ordinal)) + score += 6f; + if (entry.Key.Contains(normalizedQuery, StringComparison.OrdinalIgnoreCase)) + score += 4f; + + foreach (var term in terms) + { + if (entry.Key.Contains(term, StringComparison.OrdinalIgnoreCase)) + score += 2f; + if (entry.SearchText.Contains(term, StringComparison.Ordinal)) + score += 1f; + } + + if (score <= 0f) + return 0f; + + var ageDays = Math.Max(0d, (DateTimeOffset.UtcNow - entry.UpdatedAt).TotalDays); + var recencyBoost = (float)Math.Max(0.1d, 1.5d - Math.Min(1.4d, ageDays / 14d)); + return score + recencyBoost; + } + private async ValueTask PersistOriginalNoteKeyAsync(string key, string keyPath, string keyTempPath, CancellationToken ct) { if (!RequiresKeySidecar(key)) @@ -876,6 +986,14 @@ private static string DecodeKey(string encoded) } } + private sealed class NoteIndexEntry + { + public required string Key { get; init; } + public required string PreviewContent { get; init; } + public required string SearchText { get; init; } + public required DateTimeOffset UpdatedAt { get; init; } + } + // ── ISessionAdminStore ──────────────────────────────────────────────── public async ValueTask ListSessionsAsync( diff --git a/src/OpenClaw.Core/Memory/MemoryRetentionArchive.cs b/src/OpenClaw.Core/Memory/MemoryRetentionArchive.cs index 309e8c1..2e9aaa0 100644 --- a/src/OpenClaw.Core/Memory/MemoryRetentionArchive.cs +++ b/src/OpenClaw.Core/Memory/MemoryRetentionArchive.cs @@ -106,25 +106,37 @@ public static (int DeletedFiles, int Errors, List ErrorMessages) PurgeEx { ct.ThrowIfCancellationRequested(); + var shouldDelete = false; try { - using var stream = File.OpenRead(file); - using var doc = JsonDocument.Parse(stream); - if (!doc.RootElement.TryGetProperty("sweptAtUtc", out var sweptAtElement) || - sweptAtElement.ValueKind != JsonValueKind.String || - !DateTime.TryParse( - sweptAtElement.GetString(), - provider: null, - System.Globalization.DateTimeStyles.RoundtripKind, - out var sweptAtUtc)) + if (TryGetArchiveSweepDayUtc(archiveRoot, file, out var archiveDayUtc)) { - var fallbackLastWriteUtc = File.GetLastWriteTimeUtc(file); - if (fallbackLastWriteUtc >= cutoff) + if (archiveDayUtc > cutoff.Date) continue; + if (archiveDayUtc < cutoff.Date) + shouldDelete = true; } - else if (sweptAtUtc >= cutoff) + + if (!shouldDelete) { - continue; + using var stream = File.OpenRead(file); + using var doc = JsonDocument.Parse(stream); + if (!doc.RootElement.TryGetProperty("sweptAtUtc", out var sweptAtElement) || + sweptAtElement.ValueKind != JsonValueKind.String || + !DateTime.TryParse( + sweptAtElement.GetString(), + provider: null, + System.Globalization.DateTimeStyles.RoundtripKind, + out var sweptAtUtc)) + { + var fallbackLastWriteUtc = File.GetLastWriteTimeUtc(file); + if (fallbackLastWriteUtc >= cutoff) + continue; + } + else if (sweptAtUtc >= cutoff) + { + continue; + } } } catch (Exception ex) @@ -159,6 +171,33 @@ private static string EncodeId(string id) return Convert.ToHexString(hash).ToLowerInvariant(); } + private static bool TryGetArchiveSweepDayUtc(string archiveRoot, string filePath, out DateTime archiveDayUtc) + { + archiveDayUtc = default; + + try + { + var relative = Path.GetRelativePath(archiveRoot, filePath); + var segments = relative.Split(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); + if (segments.Length < 4) + return false; + + if (!int.TryParse(segments[0], out var year) || + !int.TryParse(segments[1], out var month) || + !int.TryParse(segments[2], out var day)) + { + return false; + } + + archiveDayUtc = new DateTime(year, month, day, 0, 0, 0, DateTimeKind.Utc); + return true; + } + catch + { + return false; + } + } + private static void CleanupEmptyDirectories(string archiveRoot) { foreach (var dir in Directory.EnumerateDirectories(archiveRoot, "*", SearchOption.AllDirectories) diff --git a/src/OpenClaw.Core/Memory/SqliteMemoryStore.cs b/src/OpenClaw.Core/Memory/SqliteMemoryStore.cs index efc5189..2d14384 100644 --- a/src/OpenClaw.Core/Memory/SqliteMemoryStore.cs +++ b/src/OpenClaw.Core/Memory/SqliteMemoryStore.cs @@ -163,9 +163,14 @@ INSERT INTO notes_fts(key, content) { return JsonSerializer.Deserialize(json, CoreJsonContext.Default.Session); } - catch + catch (Exception ex) { - return null; + _logger?.LogError(ex, "Persisted sqlite session row for {SessionId} is corrupt or unreadable", sessionId); + throw new MemoryStoreCorruptionException( + $"Session '{sessionId}' could not be loaded because its persisted sqlite state is corrupt.", + sessionId, + $"{_dbPath}#sessions/{sessionId}", + ex); } } @@ -516,9 +521,10 @@ ON CONFLICT(branch_id) DO UPDATE SET { return JsonSerializer.Deserialize(json, CoreJsonContext.Default.SessionBranch); } - catch + catch (Exception ex) { - return null; + _logger?.LogError(ex, "Persisted sqlite branch row for {BranchId} is corrupt or unreadable", branchId); + throw new InvalidDataException($"Branch '{branchId}' could not be loaded because its persisted sqlite state is corrupt.", ex); } } @@ -955,11 +961,11 @@ public async Task BackfillEmbeddingsAsync(int batchSize = 50, CancellationToken if (!_enableVectors || _embeddingGenerator is null) return; + await using var conn = new SqliteConnection(ConnectionString); + await conn.OpenAsync(ct); + while (!ct.IsCancellationRequested) { - await using var conn = new SqliteConnection(ConnectionString); - await conn.OpenAsync(ct); - await using var cmd = conn.CreateCommand(); cmd.CommandText = "SELECT key, content FROM notes WHERE embedding IS NULL LIMIT $limit;"; cmd.Parameters.AddWithValue("$limit", batchSize); @@ -972,6 +978,8 @@ public async Task BackfillEmbeddingsAsync(int batchSize = 50, CancellationToken if (batch.Count == 0) break; + var updates = new List<(string Key, byte[] Embedding)>(batch.Count); + foreach (var (key, content) in batch) { try @@ -979,14 +987,7 @@ public async Task BackfillEmbeddingsAsync(int batchSize = 50, CancellationToken var result = await _embeddingGenerator.GenerateAsync([content], cancellationToken: ct); if (result is { Count: > 0 }) { - var blob = SerializeEmbedding(result[0]); - await using var updateConn = new SqliteConnection(ConnectionString); - await updateConn.OpenAsync(ct); - await using var updateCmd = updateConn.CreateCommand(); - updateCmd.CommandText = "UPDATE notes SET embedding = $embedding WHERE key = $key;"; - updateCmd.Parameters.AddWithValue("$embedding", blob); - updateCmd.Parameters.AddWithValue("$key", key); - await updateCmd.ExecuteNonQueryAsync(ct); + updates.Add((key, SerializeEmbedding(result[0]))); } } catch (Exception ex) @@ -994,6 +995,25 @@ public async Task BackfillEmbeddingsAsync(int batchSize = 50, CancellationToken _logger?.LogWarning(ex, "Backfill embedding failed for note '{Key}'", key); } } + + if (updates.Count == 0) + continue; + + await using var tx = (SqliteTransaction)await conn.BeginTransactionAsync(ct); + await using var updateCmd = conn.CreateCommand(); + updateCmd.Transaction = tx; + updateCmd.CommandText = "UPDATE notes SET embedding = $embedding WHERE key = $key;"; + var embeddingParam = updateCmd.Parameters.Add("$embedding", SqliteType.Blob); + var keyParam = updateCmd.Parameters.Add("$key", SqliteType.Text); + + foreach (var update in updates) + { + embeddingParam.Value = update.Embedding; + keyParam.Value = update.Key; + await updateCmd.ExecuteNonQueryAsync(ct); + } + + await tx.CommitAsync(ct); } } diff --git a/src/OpenClaw.Core/Models/GatewayConfig.cs b/src/OpenClaw.Core/Models/GatewayConfig.cs index 490828f..f76b7c4 100644 --- a/src/OpenClaw.Core/Models/GatewayConfig.cs +++ b/src/OpenClaw.Core/Models/GatewayConfig.cs @@ -113,7 +113,7 @@ public sealed class MemoryConfig public bool EnableCompaction { get; set; } = false; /// Number of history turns that triggers compaction (must exceed MaxHistoryTurns). - public int CompactionThreshold { get; set; } = 40; + public int CompactionThreshold { get; set; } = 80; /// Number of recent turns to keep verbatim during compaction. public int CompactionKeepRecent { get; set; } = 10; diff --git a/src/OpenClaw.Core/Observability/RuntimeMetrics.cs b/src/OpenClaw.Core/Observability/RuntimeMetrics.cs index bc0d8e6..91aa01e 100644 --- a/src/OpenClaw.Core/Observability/RuntimeMetrics.cs +++ b/src/OpenClaw.Core/Observability/RuntimeMetrics.cs @@ -38,6 +38,11 @@ public sealed class RuntimeMetrics private long _retentionSkippedProtectedSessions; private long _operatorAuditWriteFailures; private long _runtimeEventWriteFailures; + private long _sessionCacheHits; + private long _sessionCacheMisses; + private long _memoryRecallSearches; + private long _memoryRecallHits; + private long _memoryCompactions; // ── Gauges ──────────────────────────────────────────────────────────── private int _activeSessions; @@ -74,6 +79,11 @@ public sealed class RuntimeMetrics public long RetentionSkippedProtectedSessions => Interlocked.Read(ref _retentionSkippedProtectedSessions); public long OperatorAuditWriteFailures => Interlocked.Read(ref _operatorAuditWriteFailures); public long RuntimeEventWriteFailures => Interlocked.Read(ref _runtimeEventWriteFailures); + public long SessionCacheHits => Interlocked.Read(ref _sessionCacheHits); + public long SessionCacheMisses => Interlocked.Read(ref _sessionCacheMisses); + public long MemoryRecallSearches => Interlocked.Read(ref _memoryRecallSearches); + public long MemoryRecallHits => Interlocked.Read(ref _memoryRecallHits); + public long MemoryCompactions => Interlocked.Read(ref _memoryCompactions); public int ActiveSessions => Volatile.Read(ref _activeSessions); public int CircuitBreakerState => Volatile.Read(ref _circuitBreakerState); public long RetentionLastRunAtUnixSeconds => Interlocked.Read(ref _retentionLastRunAtUnixSeconds); @@ -108,6 +118,11 @@ public sealed class RuntimeMetrics public void AddRetentionSkippedProtectedSessions(long n) => Interlocked.Add(ref _retentionSkippedProtectedSessions, n); public void IncrementOperatorAuditWriteFailures() => Interlocked.Increment(ref _operatorAuditWriteFailures); public void IncrementRuntimeEventWriteFailures() => Interlocked.Increment(ref _runtimeEventWriteFailures); + public void IncrementSessionCacheHits() => Interlocked.Increment(ref _sessionCacheHits); + public void IncrementSessionCacheMisses() => Interlocked.Increment(ref _sessionCacheMisses); + public void IncrementMemoryRecallSearches() => Interlocked.Increment(ref _memoryRecallSearches); + public void AddMemoryRecallHits(long n) => Interlocked.Add(ref _memoryRecallHits, n); + public void IncrementMemoryCompactions() => Interlocked.Increment(ref _memoryCompactions); public void SetActiveSessions(int count) => Volatile.Write(ref _activeSessions, count); public void SetCircuitBreakerState(int state) => Volatile.Write(ref _circuitBreakerState, state); public void SetRetentionLastRun(DateTimeOffset runAtUtc, long durationMs, bool succeeded) @@ -150,6 +165,11 @@ public void SetRetentionLastRun(DateTimeOffset runAtUtc, long durationMs, bool s RetentionSkippedProtectedSessions = RetentionSkippedProtectedSessions, OperatorAuditWriteFailures = OperatorAuditWriteFailures, RuntimeEventWriteFailures = RuntimeEventWriteFailures, + SessionCacheHits = SessionCacheHits, + SessionCacheMisses = SessionCacheMisses, + MemoryRecallSearches = MemoryRecallSearches, + MemoryRecallHits = MemoryRecallHits, + MemoryCompactions = MemoryCompactions, RetentionLastRunAtUnixSeconds = RetentionLastRunAtUnixSeconds, RetentionLastRunDurationMs = RetentionLastRunDurationMs, RetentionLastRunSucceeded = RetentionLastRunSucceeded, @@ -188,6 +208,11 @@ public struct MetricsSnapshot public long RetentionSkippedProtectedSessions { get; set; } public long OperatorAuditWriteFailures { get; set; } public long RuntimeEventWriteFailures { get; set; } + public long SessionCacheHits { get; set; } + public long SessionCacheMisses { get; set; } + public long MemoryRecallSearches { get; set; } + public long MemoryRecallHits { get; set; } + public long MemoryCompactions { get; set; } public long RetentionLastRunAtUnixSeconds { get; set; } public long RetentionLastRunDurationMs { get; set; } public int RetentionLastRunSucceeded { get; set; } diff --git a/src/OpenClaw.Core/Validation/ConfigValidator.cs b/src/OpenClaw.Core/Validation/ConfigValidator.cs index e9a2f1e..1f759e2 100644 --- a/src/OpenClaw.Core/Validation/ConfigValidator.cs +++ b/src/OpenClaw.Core/Validation/ConfigValidator.cs @@ -55,6 +55,11 @@ public static IReadOnlyList Validate(Models.GatewayConfig config) ValidateModelProfiles(config, errors, pluginBackedProvidersPossible); // Memory + if (!string.Equals(config.Memory.Provider, "file", StringComparison.OrdinalIgnoreCase) && + !string.Equals(config.Memory.Provider, "sqlite", StringComparison.OrdinalIgnoreCase)) + { + errors.Add($"Memory.Provider '{config.Memory.Provider}' must be 'file' or 'sqlite'."); + } if (string.IsNullOrWhiteSpace(config.Memory.StoragePath)) errors.Add("Memory.StoragePath must be set."); if (config.Memory.MaxHistoryTurns < 1) diff --git a/src/OpenClaw.Gateway/Composition/CoreServicesExtensions.cs b/src/OpenClaw.Gateway/Composition/CoreServicesExtensions.cs index 2e3bd51..17aa7a0 100644 --- a/src/OpenClaw.Gateway/Composition/CoreServicesExtensions.cs +++ b/src/OpenClaw.Gateway/Composition/CoreServicesExtensions.cs @@ -30,7 +30,8 @@ public static IServiceCollection AddOpenClawCoreServices(this IServiceCollection services.AddSingleton(sp => new AllowlistManager(config.Memory.StoragePath, sp.GetRequiredService>())); - services.AddSingleton(_ => CreateMemoryStore(config)); + services.AddSingleton(); + services.AddSingleton(sp => CreateMemoryStore(config, sp.GetRequiredService())); services.AddSingleton(sp => { var memory = sp.GetRequiredService(); @@ -39,7 +40,6 @@ public static IServiceCollection AddOpenClawCoreServices(this IServiceCollection }); services.AddSingleton(sp => (ISessionSearchStore)sp.GetRequiredService()); AddFeatureStores(services, config); - services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); @@ -107,7 +107,13 @@ public static IServiceCollection AddOpenClawCoreServices(this IServiceCollection config, sp.GetRequiredService().CreateLogger("SessionManager"), sp.GetRequiredService())); - services.AddSingleton(); + services.AddSingleton(sp => new MemoryRetentionSweeperService( + config, + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService(), + sp.GetRequiredService>(), + sp.GetRequiredService().GetAll)); services.AddSingleton(sp => sp.GetRequiredService()); services.AddHostedService(sp => sp.GetRequiredService()); services.AddSingleton(); @@ -150,7 +156,7 @@ private static string ResolveSqliteDbPath(GatewayConfig config) return Path.GetFullPath(dbPath); } - private static IMemoryStore CreateMemoryStore(OpenClaw.Core.Models.GatewayConfig config) + private static IMemoryStore CreateMemoryStore(OpenClaw.Core.Models.GatewayConfig config, RuntimeMetrics metrics) { if (string.Equals(config.Memory.Provider, "sqlite", StringComparison.OrdinalIgnoreCase)) { @@ -181,6 +187,7 @@ private static IMemoryStore CreateMemoryStore(OpenClaw.Core.Models.GatewayConfig return new FileMemoryStore( config.Memory.StoragePath, - config.Memory.MaxCachedSessions ?? config.MaxConcurrentSessions); + config.Memory.MaxCachedSessions ?? config.MaxConcurrentSessions, + metrics: metrics); } } diff --git a/src/OpenClaw.Gateway/Extensions/MemoryRetentionSweeperService.cs b/src/OpenClaw.Gateway/Extensions/MemoryRetentionSweeperService.cs index 95e429d..31a2691 100644 --- a/src/OpenClaw.Gateway/Extensions/MemoryRetentionSweeperService.cs +++ b/src/OpenClaw.Gateway/Extensions/MemoryRetentionSweeperService.cs @@ -5,6 +5,7 @@ using OpenClaw.Core.Models; using OpenClaw.Core.Observability; using OpenClaw.Core.Sessions; +using OpenClaw.Gateway; namespace OpenClaw.Gateway.Extensions; @@ -20,11 +21,20 @@ public interface IMemoryRetentionCoordinator /// public sealed class MemoryRetentionSweeperService : BackgroundService, IMemoryRetentionCoordinator { + private static readonly HashSet ProtectedRetentionTags = new(StringComparer.OrdinalIgnoreCase) + { + "keep", + "pinned", + "retain", + "retention-exempt" + }; + private readonly GatewayConfig _config; private readonly SessionManager _sessionManager; private readonly RuntimeMetrics _metrics; private readonly ILogger _logger; private readonly IMemoryRetentionStore? _retentionStore; + private readonly Func>? _metadataSnapshotProvider; private readonly SemaphoreSlim _runGate = new(1, 1); private readonly object _statusLock = new(); private RetentionRunStatus _status; @@ -34,13 +44,15 @@ public MemoryRetentionSweeperService( SessionManager sessionManager, IMemoryStore memoryStore, RuntimeMetrics metrics, - ILogger logger) + ILogger logger, + Func>? metadataSnapshotProvider = null) { _config = config; _sessionManager = sessionManager; _metrics = metrics; _logger = logger; _retentionStore = memoryStore as IMemoryRetentionStore; + _metadataSnapshotProvider = metadataSnapshotProvider; _status = new RetentionRunStatus { @@ -216,6 +228,15 @@ private async ValueTask> BuildProtectedSetAsync(Cancellatio set.Add(session.Id); } + if (_metadataSnapshotProvider is not null) + { + foreach (var metadata in _metadataSnapshotProvider().Values) + { + if (metadata.Starred || metadata.Tags.Any(static tag => ProtectedRetentionTags.Contains(tag))) + set.Add(metadata.SessionId); + } + } + return set; } diff --git a/src/OpenClaw.Gateway/HeartbeatService.cs b/src/OpenClaw.Gateway/HeartbeatService.cs index d2f49aa..a7ad3c1 100644 --- a/src/OpenClaw.Gateway/HeartbeatService.cs +++ b/src/OpenClaw.Gateway/HeartbeatService.cs @@ -851,8 +851,13 @@ private async ValueTask> BuildSuggestionsA if (session is null) continue; - foreach (var turn in session.History.Where(static turn => string.Equals(turn.Role, "user", StringComparison.OrdinalIgnoreCase))) - texts.Add(($"session:{summary.Id}", turn.Content)); + foreach (var turn in session.History + .Where(static turn => string.Equals(turn.Role, "user", StringComparison.OrdinalIgnoreCase)) + .TakeLast(6)) + { + if (!string.IsNullOrWhiteSpace(turn.Content)) + texts.Add(($"session:{summary.Id}", Truncate(turn.Content, 2_000))); + } } } @@ -861,7 +866,7 @@ private async ValueTask> BuildSuggestionsA { var note = await _memoryStore.LoadNoteAsync(key, ct); if (!string.IsNullOrWhiteSpace(note)) - texts.Add(($"note:{key}", note!)); + texts.Add(($"note:{key}", Truncate(note!, 2_000))); } foreach (var (source, text) in texts) diff --git a/src/OpenClaw.Gateway/appsettings.Production.json b/src/OpenClaw.Gateway/appsettings.Production.json index dca5160..4d57831 100644 --- a/src/OpenClaw.Gateway/appsettings.Production.json +++ b/src/OpenClaw.Gateway/appsettings.Production.json @@ -13,9 +13,26 @@ }, "Memory": { + "Provider": "sqlite", "StoragePath": "/app/memory", "MaxHistoryTurns": 50, - "MaxCachedSessions": 128 + "MaxCachedSessions": 128, + "Sqlite": { + "DbPath": "/app/memory/openclaw.db", + "EnableFts": true, + "EnableVectors": false + }, + "Retention": { + "Enabled": true, + "RunOnStartup": true, + "SweepIntervalMinutes": 30, + "SessionTtlDays": 30, + "BranchTtlDays": 14, + "ArchiveEnabled": true, + "ArchivePath": "/app/memory/archive", + "ArchiveRetentionDays": 30, + "MaxItemsPerSweep": 1000 + } }, "Security": { diff --git a/src/OpenClaw.Gateway/wwwroot/admin.html b/src/OpenClaw.Gateway/wwwroot/admin.html index 2bb5e97..8d77a2d 100644 --- a/src/OpenClaw.Gateway/wwwroot/admin.html +++ b/src/OpenClaw.Gateway/wwwroot/admin.html @@ -1532,7 +1532,7 @@

Notes

allowBrowserEvaluate: settingsInputs.allowBrowserEvaluate.checked, maxHistoryTurns: toInt(settingsInputs.maxHistoryTurns.value, 50), enableCompaction: settingsInputs.enableCompaction.checked, - compactionThreshold: toInt(settingsInputs.compactionThreshold.value, 40), + compactionThreshold: toInt(settingsInputs.compactionThreshold.value, 80), compactionKeepRecent: toInt(settingsInputs.compactionKeepRecent.value, 10), retentionEnabled: settingsInputs.retentionEnabled.checked, retentionRunOnStartup: settingsInputs.retentionRunOnStartup.checked, diff --git a/src/OpenClaw.MicrosoftAgentFrameworkAdapter/MafAgentRuntime.cs b/src/OpenClaw.MicrosoftAgentFrameworkAdapter/MafAgentRuntime.cs index c6f4cf2..7ba08e8 100644 --- a/src/OpenClaw.MicrosoftAgentFrameworkAdapter/MafAgentRuntime.cs +++ b/src/OpenClaw.MicrosoftAgentFrameworkAdapter/MafAgentRuntime.cs @@ -43,6 +43,7 @@ public sealed class MafAgentRuntime : IAgentRuntime private readonly Func? _isContractTokenBudgetExceeded; private readonly Func? _isContractRuntimeBudgetExceeded; private readonly Action? _appendContractSnapshot; + private readonly string? _memoryRecallPrefix; private readonly object _skillGate = new(); private readonly IList _mafTools; private string _systemPrompt = string.Empty; @@ -93,6 +94,9 @@ public MafAgentRuntime( _isContractTokenBudgetExceeded = context.IsContractTokenBudgetExceeded; _isContractRuntimeBudgetExceeded = context.IsContractRuntimeBudgetExceeded; _appendContractSnapshot = context.AppendContractSnapshot; + var projectId = context.Config.Memory.ProjectId + ?? Environment.GetEnvironmentVariable("OPENCLAW_PROJECT"); + _memoryRecallPrefix = string.IsNullOrWhiteSpace(projectId) ? null : $"project:{projectId.Trim()}:"; _chatClient = new MafExecutionServiceChatClient( context.LlmExecutionService, context.RuntimeMetrics, @@ -507,9 +511,16 @@ private async ValueTask TryInjectRecallAsync(List messages, string try { var limit = Math.Clamp(_recall.MaxNotes, 1, 32); - var hits = await search.SearchNotesAsync(userMessage, prefix: null, limit, ct); + _metrics?.IncrementMemoryRecallSearches(); + var hits = await search.SearchNotesAsync(userMessage, _memoryRecallPrefix, limit, ct); + if (hits.Count == 0 && !string.IsNullOrWhiteSpace(_memoryRecallPrefix)) + { + _metrics?.IncrementMemoryRecallSearches(); + hits = await search.SearchNotesAsync(userMessage, prefix: null, limit, ct); + } if (hits.Count == 0) return; + _metrics?.AddMemoryRecallHits(hits.Count); var maxChars = Math.Clamp(_recall.MaxChars, 256, 100_000); var sb = new StringBuilder(); sb.AppendLine("[Relevant memory]"); @@ -612,6 +623,7 @@ private async Task CompactHistoryAsync(Session session, CancellationToken ct) return; } + _metrics?.IncrementMemoryCompactions(); session.History.RemoveRange(0, toSummarizeCount); session.History.Insert(0, new ChatTurn { diff --git a/src/OpenClaw.Tests/ConfigValidatorTests.cs b/src/OpenClaw.Tests/ConfigValidatorTests.cs index e047aff..752c70c 100644 --- a/src/OpenClaw.Tests/ConfigValidatorTests.cs +++ b/src/OpenClaw.Tests/ConfigValidatorTests.cs @@ -189,6 +189,21 @@ public void Validate_CompactionThresholdMustExceedMaxHistoryTurns_ReturnsError() Assert.Contains(errors, e => e.Contains("greater than MaxHistoryTurns", StringComparison.Ordinal)); } + [Fact] + public void Validate_InvalidMemoryProvider_ReturnsError() + { + var config = new GatewayConfig + { + Memory = new MemoryConfig + { + Provider = "redis" + } + }; + + var errors = ConfigValidator.Validate(config); + Assert.Contains(errors, e => e.Contains("Memory.Provider", StringComparison.Ordinal)); + } + [Fact] public void Validate_InvalidRuntimeMode_ReturnsError() { diff --git a/src/OpenClaw.Tests/FeatureParityTests.cs b/src/OpenClaw.Tests/FeatureParityTests.cs index dbf919f..cde20dd 100644 --- a/src/OpenClaw.Tests/FeatureParityTests.cs +++ b/src/OpenClaw.Tests/FeatureParityTests.cs @@ -879,7 +879,7 @@ public void GatewayConfig_MemoryConfig_HasCompaction() { var config = new MemoryConfig(); Assert.False(config.EnableCompaction); // Default false - Assert.Equal(40, config.CompactionThreshold); + Assert.Equal(80, config.CompactionThreshold); Assert.Equal(10, config.CompactionKeepRecent); } diff --git a/src/OpenClaw.Tests/FileMemoryStoreTests.cs b/src/OpenClaw.Tests/FileMemoryStoreTests.cs index 1120fa8..0edfadb 100644 --- a/src/OpenClaw.Tests/FileMemoryStoreTests.cs +++ b/src/OpenClaw.Tests/FileMemoryStoreTests.cs @@ -186,4 +186,29 @@ public async Task SearchNotesAsync_LongKeys_RespectPrefixFilter() Directory.Delete(storagePath, recursive: true); } } + + [Fact] + public async Task SearchNotesAsync_PrefersHigherScoringAndMoreRecentNotes() + { + var storagePath = Path.Combine(Path.GetTempPath(), "openclaw-file-memory-tests", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(storagePath); + + try + { + var store = new FileMemoryStore(storagePath, 4); + await store.SaveNoteAsync("project:demo:legacy", "architecture notes about migration", CancellationToken.None); + await Task.Delay(20); + await store.SaveNoteAsync("project:demo:architecture", "architecture migration checklist", CancellationToken.None); + + var hits = await store.SearchNotesAsync("architecture migration", "project:demo:", 2, CancellationToken.None); + + Assert.Equal(2, hits.Count); + Assert.Equal("project:demo:architecture", hits[0].Key); + Assert.True(hits[0].Score >= hits[1].Score); + } + finally + { + Directory.Delete(storagePath, recursive: true); + } + } } diff --git a/src/OpenClaw.Tests/MemoryRecallInjectionTests.cs b/src/OpenClaw.Tests/MemoryRecallInjectionTests.cs index 26fd7e9..81f0bc5 100644 --- a/src/OpenClaw.Tests/MemoryRecallInjectionTests.cs +++ b/src/OpenClaw.Tests/MemoryRecallInjectionTests.cs @@ -46,4 +46,44 @@ public async Task RunAsync_InsertsRelevantMemoryUserMessage_WhenEnabled() (m.Text ?? "").Contains("[Relevant memory]", StringComparison.Ordinal) && (m.Text ?? "").Contains("untrusted", StringComparison.OrdinalIgnoreCase)); } + + [Fact] + public async Task RunAsync_PrefersProjectScopedRecall_WhenProjectIdConfigured() + { + var chatClient = Substitute.For(); + chatClient.GetResponseAsync( + Arg.Any>(), + Arg.Any(), + Arg.Any()) + .Returns(Task.FromResult(new ChatResponse(new[] { new ChatMessage(ChatRole.Assistant, "ok") }))); + + var memory = Substitute.For(); + var search = (IMemoryNoteSearch)memory; + search.SearchNotesAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(ValueTask.FromResult>([])); + + var agent = new AgentRuntime( + chatClient, + tools: [], + memory, + new LlmProviderConfig { Provider = "openai", ApiKey = "test", Model = "gpt-4" }, + maxHistoryTurns: 5, + recall: new MemoryRecallConfig { Enabled = true, MaxNotes = 5, MaxChars = 4000 }, + gatewayConfig: new GatewayConfig + { + Memory = new MemoryConfig + { + ProjectId = "demo" + } + }); + + var session = new Session { Id = "s1", ChannelId = "test", SenderId = "u1" }; + _ = await agent.RunAsync(session, "what should I remember?", CancellationToken.None); + + await search.Received().SearchNotesAsync( + "what should I remember?", + "project:demo:", + Arg.Any(), + Arg.Any()); + } } diff --git a/src/OpenClaw.Tests/MemoryRetentionSweeperServiceTests.cs b/src/OpenClaw.Tests/MemoryRetentionSweeperServiceTests.cs index b4d8f22..c0781e4 100644 --- a/src/OpenClaw.Tests/MemoryRetentionSweeperServiceTests.cs +++ b/src/OpenClaw.Tests/MemoryRetentionSweeperServiceTests.cs @@ -3,6 +3,7 @@ using OpenClaw.Core.Models; using OpenClaw.Core.Observability; using OpenClaw.Core.Sessions; +using OpenClaw.Gateway; using OpenClaw.Gateway.Extensions; using Xunit; @@ -131,9 +132,54 @@ public async Task SweepNowAsync_RejectsOverlappingRuns() await first; } + [Fact] + public async Task SweepNowAsync_ProtectsStarredSessionsFromMetadataStore() + { + var root = Path.Combine(Path.GetTempPath(), "openclaw-retention-tests", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(root); + + try + { + var config = new GatewayConfig + { + Memory = new MemoryConfig + { + StoragePath = root, + Retention = new MemoryRetentionConfig + { + Enabled = true, + ArchiveEnabled = false + } + } + }; + + var metadataStore = new SessionMetadataStore(root, NullLogger.Instance); + metadataStore.Set("session-starred", new SessionMetadataUpdateRequest { Starred = true }); + + var store = new StubRetentionStore(); + var manager = new SessionManager(store, config); + var service = new MemoryRetentionSweeperService( + config, + manager, + store, + new RuntimeMetrics(), + NullLogger.Instance, + metadataStore.GetAll); + + _ = await service.SweepNowAsync(dryRun: true, CancellationToken.None); + + Assert.Contains("session-starred", store.LastProtectedSessionIds); + } + finally + { + Directory.Delete(root, recursive: true); + } + } + private sealed class StubRetentionStore : IMemoryStore, IMemoryRetentionStore { public Func>? NextResultFactory { get; set; } + public IReadOnlySet LastProtectedSessionIds { get; private set; } = new HashSet(StringComparer.Ordinal); public ValueTask GetSessionAsync(string sessionId, CancellationToken ct) => ValueTask.FromResult(null); public ValueTask SaveSessionAsync(Session session, CancellationToken ct) => ValueTask.CompletedTask; @@ -151,6 +197,7 @@ public async ValueTask SweepAsync( IReadOnlySet protectedSessionIds, CancellationToken ct) { + LastProtectedSessionIds = new HashSet(protectedSessionIds, StringComparer.Ordinal); if (NextResultFactory is null) { return new RetentionSweepResult diff --git a/src/OpenClaw.Tests/SqliteSessionSearchTests.cs b/src/OpenClaw.Tests/SqliteSessionSearchTests.cs index 5f61732..9a564a2 100644 --- a/src/OpenClaw.Tests/SqliteSessionSearchTests.cs +++ b/src/OpenClaw.Tests/SqliteSessionSearchTests.cs @@ -155,6 +155,39 @@ await store.SaveSessionAsync(new Session } } + [Fact] + public async Task GetSessionAsync_CorruptRow_ThrowsCorruptionException() + { + var root = CreateTempDirectory(); + try + { + var dbPath = Path.Combine(root, "memory.db"); + using var store = new SqliteMemoryStore(dbPath, enableFts: false); + + await store.SaveSessionAsync(new Session + { + Id = "session-corrupt", + ChannelId = "websocket", + SenderId = "alice" + }, CancellationToken.None); + + await using var conn = new Microsoft.Data.Sqlite.SqliteConnection(new Microsoft.Data.Sqlite.SqliteConnectionStringBuilder { DataSource = dbPath }.ToString()); + await conn.OpenAsync(); + await using var cmd = conn.CreateCommand(); + cmd.CommandText = "UPDATE sessions SET json = '{not valid json' WHERE id = $id;"; + cmd.Parameters.AddWithValue("$id", "session-corrupt"); + await cmd.ExecuteNonQueryAsync(); + + var ex = await Assert.ThrowsAsync(async () => + await store.GetSessionAsync("session-corrupt", CancellationToken.None)); + Assert.Equal("session-corrupt", ex.SessionId); + } + finally + { + Directory.Delete(root, recursive: true); + } + } + private static string CreateTempDirectory() { var path = Path.Combine(Path.GetTempPath(), "openclaw-tests", Guid.NewGuid().ToString("n"));