From 5a6d9e5aff9d30aeebbea0ef8aa74061679cbdb8 Mon Sep 17 00:00:00 2001 From: Benedikt Koehler Date: Tue, 28 Apr 2026 20:33:03 +0200 Subject: [PATCH 1/5] feat: add trajectory retention schema --- docs/content/extensibility/adaptive-skills.md | 18 +- docs/content/reference/configuration.md | 4 +- .../extensibility/adaptive-skills.md | 18 +- docs/development/reference/configuration.md | 4 +- src/config/runtime-config.ts | 33 ++ src/skills/adaptive-skills-types.ts | 2 + src/skills/skill-run-trajectories.ts | 283 +++++++++++++++++- src/skills/skills-inspection.ts | 31 ++ tests/config-reload.integration.test.ts | 25 ++ tests/skills-inspection.test.ts | 55 ++++ tests/skills-observation.test.ts | 43 ++- 11 files changed, 494 insertions(+), 22 deletions(-) diff --git a/docs/content/extensibility/adaptive-skills.md b/docs/content/extensibility/adaptive-skills.md index 6d5057b75..386df731c 100644 --- a/docs/content/extensibility/adaptive-skills.md +++ b/docs/content/extensibility/adaptive-skills.md @@ -44,7 +44,9 @@ is disabled by default. "rollbackImprovementThreshold": 0.05, "trajectoryCapture": { "enabledAgentIds": [], - "storeDir": "" + "storeDir": "", + "retentionDays": 365, + "retentionDaysByTenant": {} } } } @@ -66,6 +68,10 @@ Key settings: - `trajectoryCapture.storeDir`: optional trajectory store location; empty uses a `trajectories/` directory beside the runtime database, absolute paths are used as-is, and relative paths resolve under the runtime home directory +- `trajectoryCapture.retentionDays`: trajectory JSONL retention window; default + is 365 days and `0` disables trajectory pruning +- `trajectoryCapture.retentionDaysByTenant`: optional per-coworker retention + overrides keyed by tenant ID; trajectory tenants currently map to agent IDs Legacy `skillCognee` config input is still normalized into `adaptiveSkills` for backward compatibility. @@ -90,12 +96,14 @@ Feedback is attached to the most recent observation for the same session. ## Retention -Observation queries are windowed, but storage is also pruned now. On each +Observation queries are windowed, but storage is also pruned. On each inspection interval the heartbeat deletes `skill_observations` rows older than -`observationRetentionDays`. +`observationRetentionDays` and removes expired trajectory JSONL files according +to the trajectory retention policy. -This keeps high-traffic skills from accumulating unbounded observation history -while preserving the amendment history table as the durable review log. +This keeps high-traffic skills from accumulating unbounded observation and +trajectory history while preserving the amendment history table as the durable +review log. ## Operator Surfaces diff --git a/docs/content/reference/configuration.md b/docs/content/reference/configuration.md index 0cc3477cb..8bfe17a8d 100644 --- a/docs/content/reference/configuration.md +++ b/docs/content/reference/configuration.md @@ -151,7 +151,9 @@ saved revision history directly. `adaptiveSkills.trajectoryCapture.enabledAgentIds`; when `adaptiveSkills.trajectoryCapture.storeDir` is empty, trajectories are stored beside the runtime database, absolute paths are used as-is, and relative paths - resolve under the runtime home directory + resolve under the runtime home directory; trajectory retention defaults to + `adaptiveSkills.trajectoryCapture.retentionDays: 365` and can be overridden + per coworker with `adaptiveSkills.trajectoryCapture.retentionDaysByTenant` - `imessage.*` for the dual-backend local or BlueBubbles iMessage transport; prefer storing the BlueBubbles password as `IMESSAGE_PASSWORD` in the encrypted secret store instead of plaintext config diff --git a/docs/development/extensibility/adaptive-skills.md b/docs/development/extensibility/adaptive-skills.md index 6d5057b75..386df731c 100644 --- a/docs/development/extensibility/adaptive-skills.md +++ b/docs/development/extensibility/adaptive-skills.md @@ -44,7 +44,9 @@ is disabled by default. "rollbackImprovementThreshold": 0.05, "trajectoryCapture": { "enabledAgentIds": [], - "storeDir": "" + "storeDir": "", + "retentionDays": 365, + "retentionDaysByTenant": {} } } } @@ -66,6 +68,10 @@ Key settings: - `trajectoryCapture.storeDir`: optional trajectory store location; empty uses a `trajectories/` directory beside the runtime database, absolute paths are used as-is, and relative paths resolve under the runtime home directory +- `trajectoryCapture.retentionDays`: trajectory JSONL retention window; default + is 365 days and `0` disables trajectory pruning +- `trajectoryCapture.retentionDaysByTenant`: optional per-coworker retention + overrides keyed by tenant ID; trajectory tenants currently map to agent IDs Legacy `skillCognee` config input is still normalized into `adaptiveSkills` for backward compatibility. @@ -90,12 +96,14 @@ Feedback is attached to the most recent observation for the same session. ## Retention -Observation queries are windowed, but storage is also pruned now. On each +Observation queries are windowed, but storage is also pruned. On each inspection interval the heartbeat deletes `skill_observations` rows older than -`observationRetentionDays`. +`observationRetentionDays` and removes expired trajectory JSONL files according +to the trajectory retention policy. -This keeps high-traffic skills from accumulating unbounded observation history -while preserving the amendment history table as the durable review log. +This keeps high-traffic skills from accumulating unbounded observation and +trajectory history while preserving the amendment history table as the durable +review log. ## Operator Surfaces diff --git a/docs/development/reference/configuration.md b/docs/development/reference/configuration.md index 1479bf672..b60174f7c 100644 --- a/docs/development/reference/configuration.md +++ b/docs/development/reference/configuration.md @@ -125,7 +125,9 @@ leak into the saved revision metadata. `adaptiveSkills.trajectoryCapture.enabledAgentIds`; when `adaptiveSkills.trajectoryCapture.storeDir` is empty, trajectories are stored beside the runtime database, absolute paths are used as-is, and relative paths - resolve under the runtime home directory + resolve under the runtime home directory; trajectory retention defaults to + `adaptiveSkills.trajectoryCapture.retentionDays: 365` and can be overridden + per coworker with `adaptiveSkills.trajectoryCapture.retentionDaysByTenant` - `imessage.*` for the dual-backend local or BlueBubbles iMessage transport; prefer storing the BlueBubbles password as `IMESSAGE_PASSWORD` in the encrypted secret store instead of plaintext config diff --git a/src/config/runtime-config.ts b/src/config/runtime-config.ts index a2d337625..19e15cf8b 100644 --- a/src/config/runtime-config.ts +++ b/src/config/runtime-config.ts @@ -1049,6 +1049,8 @@ export const DEFAULT_RUNTIME_CONFIG: RuntimeConfig = { trajectoryCapture: { enabledAgentIds: [], storeDir: '', + retentionDays: 365, + retentionDaysByTenant: {}, }, inspectionIntervalMs: 3_600_000, observationRetentionDays: 30, @@ -1733,6 +1735,25 @@ function normalizeStringArray(value: unknown, fallback: string[]): string[] { return fallback; } +function normalizeRetentionDaysByTenant( + value: unknown, + fallback: Record, + defaultRetentionDays: number, +): Record { + if (!isRecord(value)) return { ...fallback }; + const normalized: Record = {}; + for (const [tenantId, rawDays] of Object.entries(value)) { + const normalizedTenantId = tenantId.trim(); + if (!normalizedTenantId) continue; + normalized[normalizedTenantId] = normalizeInteger( + rawDays, + fallback[normalizedTenantId] ?? defaultRetentionDays, + { min: 0 }, + ); + } + return normalized; +} + function normalizeOptionalBaseUrl(value: unknown, fallback: string): string { const candidate = normalizeString(value, fallback, { allowEmpty: true }); return candidate ? candidate.replace(/\/+$/, '') : ''; @@ -4716,6 +4737,11 @@ function normalizeRuntimeConfig( rawDiscord.commandMode, legacyCommandModeFallback, ); + const normalizedTrajectoryRetentionDays = normalizeInteger( + rawTrajectoryCapture.retentionDays, + DEFAULT_RUNTIME_CONFIG.adaptiveSkills.trajectoryCapture.retentionDays, + { min: 0 }, + ); return { version: CONFIG_VERSION, @@ -4803,6 +4829,13 @@ function normalizeRuntimeConfig( DEFAULT_RUNTIME_CONFIG.adaptiveSkills.trajectoryCapture.storeDir, { allowEmpty: true }, ), + retentionDays: normalizedTrajectoryRetentionDays, + retentionDaysByTenant: normalizeRetentionDaysByTenant( + rawTrajectoryCapture.retentionDaysByTenant, + DEFAULT_RUNTIME_CONFIG.adaptiveSkills.trajectoryCapture + .retentionDaysByTenant, + normalizedTrajectoryRetentionDays, + ), }, inspectionIntervalMs: normalizeInteger( rawAdaptiveSkills.inspectionIntervalMs, diff --git a/src/skills/adaptive-skills-types.ts b/src/skills/adaptive-skills-types.ts index f567ea875..b4aed5bae 100644 --- a/src/skills/adaptive-skills-types.ts +++ b/src/skills/adaptive-skills-types.ts @@ -139,6 +139,8 @@ export interface AdaptiveSkillsConfig { trajectoryCapture: { enabledAgentIds: string[]; storeDir: string; + retentionDays: number; + retentionDaysByTenant: Record; }; inspectionIntervalMs: number; observationRetentionDays: number; diff --git a/src/skills/skill-run-trajectories.ts b/src/skills/skill-run-trajectories.ts index 64fe5a341..fc408a9e5 100644 --- a/src/skills/skill-run-trajectories.ts +++ b/src/skills/skill-run-trajectories.ts @@ -6,18 +6,63 @@ import { } from '../config/runtime-config.js'; import { DEFAULT_RUNTIME_HOME_DIR } from '../config/runtime-paths.js'; import { logger } from '../logger.js'; +import { getAgentSkillScores } from '../memory/db.js'; import { expandHomePath } from '../utils/path.js'; -import type { SkillRunEvent } from './skill-run-events.js'; +import type { + AdaptiveSkillsConfig, + AgentSkillScore, +} from './adaptive-skills-types.js'; +import type { + SkillRunBoundedPayload, + SkillRunEvent, + SkillRunFullPayload, + SkillRunToolExecutionSummary, +} from './skill-run-events.js'; -export const SKILL_RUN_TRAJECTORY_SCHEMA_VERSION = 1; +export const SKILL_RUN_TRAJECTORY_SCHEMA_VERSION = 2; const SKILL_RUN_TRAJECTORY_DIR_MODE = 0o700; +const MS_PER_DAY = 24 * 60 * 60 * 1000; +const TRAJECTORY_DATE_DIR_PATTERN = /^\d{4}-\d{2}-\d{2}$/; let loggedTrajectoryCaptureConfigKey: string | null = null; +export interface SkillRunTrajectoryPayload { + content: string; + truncated: boolean; + source: 'bounded' | 'full'; +} + +export interface SkillRunTrajectoryToolUse + extends SkillRunToolExecutionSummary { + arguments: SkillRunTrajectoryPayload | null; + result: SkillRunTrajectoryPayload | null; +} + +export interface SkillRunTrajectoryScore { + run: number; + agent_skill: { + score: number; + quality_score: number; + reliability_score: number; + timing_score: number; + total_executions: number; + } | null; +} + export interface SkillRunTrajectoryRecord { schema_version: typeof SKILL_RUN_TRAJECTORY_SCHEMA_VERSION; captured_at: string; date: string; + tenant_id: string; agent_id: string; + skill_id: string; + session_id: string; + run_id: string; + input: SkillRunTrajectoryPayload | null; + output: SkillRunTrajectoryPayload | null; + model: string | null; + tools_used: SkillRunTrajectoryToolUse[]; + outcome: SkillRunEvent['outcome']; + score: SkillRunTrajectoryScore; event: SkillRunEvent; } @@ -26,8 +71,11 @@ function safeFilePart(raw: string): string { return normalized || 'unknown'; } -function resolveTrajectoryStoreDir(config: RuntimeConfig): string { - const configured = config.adaptiveSkills.trajectoryCapture.storeDir.trim(); +export function resolveSkillRunTrajectoryStoreDir( + config: RuntimeConfig, + adaptiveSkills: AdaptiveSkillsConfig = config.adaptiveSkills, +): string { + const configured = adaptiveSkills.trajectoryCapture.storeDir.trim(); if (!configured) { return path.join(path.dirname(config.ops.dbPath), 'trajectories'); } @@ -37,9 +85,10 @@ function resolveTrajectoryStoreDir(config: RuntimeConfig): string { return path.join(DEFAULT_RUNTIME_HOME_DIR, expanded); } -function normalizedTrajectoryCaptureAgentIds(config: RuntimeConfig): string[] { - const enabledAgentIds = - config.adaptiveSkills.trajectoryCapture.enabledAgentIds; +function normalizedTrajectoryCaptureAgentIds( + config: AdaptiveSkillsConfig, +): string[] { + const enabledAgentIds = config.trajectoryCapture.enabledAgentIds; if (enabledAgentIds.length === 0) return []; return enabledAgentIds.map((agentId) => agentId.trim()).filter(Boolean); } @@ -111,6 +160,80 @@ function ensurePrivateTrajectoryDirectories(input: { }); } +function buildTrajectoryPayload( + bounded: SkillRunBoundedPayload | null, + full: SkillRunFullPayload | null, +): SkillRunTrajectoryPayload | null { + if (full) { + return { + content: full.content, + truncated: false, + source: 'full', + }; + } + if (!bounded) return null; + return { + content: bounded.content, + truncated: bounded.truncated, + source: 'bounded', + }; +} + +function buildTrajectoryToolUse( + summary: SkillRunToolExecutionSummary, + event: SkillRunEvent, + index: number, +): SkillRunTrajectoryToolUse { + const full = event.tool_executions_full[index]; + return { + ...summary, + arguments: full ? buildTrajectoryPayload(null, full.arguments) : null, + result: full ? buildTrajectoryPayload(null, full.result) : null, + }; +} + +function scoreSkillRunOutcome(outcome: SkillRunEvent['outcome']): number { + if (outcome === 'success') return 1; + if (outcome === 'partial') return 0.5; + return 0; +} + +function buildTrajectoryScore( + event: SkillRunEvent & { agent_id: string }, +): SkillRunTrajectoryScore { + let agentSkillScore: AgentSkillScore | null = null; + try { + const [score] = getAgentSkillScores({ + agentId: event.agent_id, + skillName: event.skill_id, + limit: 1, + }); + agentSkillScore = score ?? null; + } catch (error) { + logger.warn( + { + agentId: event.agent_id, + skillId: event.skill_id, + runId: event.run_id, + error, + }, + 'Failed to read agent skill score for trajectory', + ); + } + return { + run: scoreSkillRunOutcome(event.outcome), + agent_skill: agentSkillScore + ? { + score: agentSkillScore.score, + quality_score: agentSkillScore.quality_score, + reliability_score: agentSkillScore.reliability_score, + timing_score: agentSkillScore.timing_score, + total_executions: agentSkillScore.total_executions, + } + : null, + }; +} + export function buildSkillRunTrajectoryRecord( event: SkillRunEvent & { agent_id: string }, capturedAt = new Date(), @@ -120,20 +243,34 @@ export function buildSkillRunTrajectoryRecord( schema_version: SKILL_RUN_TRAJECTORY_SCHEMA_VERSION, captured_at, date: captured_at.slice(0, 10), + tenant_id: event.agent_id, agent_id: event.agent_id, + skill_id: event.skill_id, + session_id: event.session_id, + run_id: event.run_id, + input: buildTrajectoryPayload(event.input, event.input_full), + output: buildTrajectoryPayload(event.output, event.output_full), + model: event.model, + tools_used: event.tool_executions.map((summary, index) => + buildTrajectoryToolUse(summary, event, index), + ), + outcome: event.outcome, + score: buildTrajectoryScore(event), event, }; } export function recordSkillRunTrajectory(event: SkillRunEvent): void { const config = getRuntimeConfig(); - const enabledAgentIds = normalizedTrajectoryCaptureAgentIds(config); + const enabledAgentIds = normalizedTrajectoryCaptureAgentIds( + config.adaptiveSkills, + ); if (enabledAgentIds.length === 0) { loggedTrajectoryCaptureConfigKey = null; return; } - const storeDir = resolveTrajectoryStoreDir(config); + const storeDir = resolveSkillRunTrajectoryStoreDir(config); logTrajectoryCaptureEnabledOnce({ agentIds: enabledAgentIds, storeDir, @@ -168,3 +305,131 @@ export function recordSkillRunTrajectory(event: SkillRunEvent): void { ); } } + +function tenantRetentionDays( + tenantId: string, + config: AdaptiveSkillsConfig, +): number { + const normalizedTenantId = tenantId.trim(); + const overrides = config.trajectoryCapture.retentionDaysByTenant; + const configured = + overrides[normalizedTenantId] ?? + overrides[safeFilePart(normalizedTenantId)]; + return configured ?? config.trajectoryCapture.retentionDays; +} + +function shouldPruneTrajectoryDate(input: { + date: string; + retentionDays: number; + now: Date; +}): boolean { + if (input.retentionDays <= 0) return false; + const cutoffDate = new Date( + input.now.getTime() - input.retentionDays * MS_PER_DAY, + ) + .toISOString() + .slice(0, 10); + return input.date < cutoffDate; +} + +function readTrajectoryTenantId(filePath: string): string | null { + let fd: number | null = null; + try { + fd = fs.openSync(filePath, 'r'); + const buffer = Buffer.alloc(8192); + const bytesRead = fs.readSync(fd, buffer, 0, buffer.length, 0); + const firstLine = buffer + .toString('utf-8', 0, bytesRead) + .split(/\r?\n/, 1)[0] + ?.trim(); + if (!firstLine) return null; + const record = JSON.parse(firstLine) as { + tenant_id?: unknown; + agent_id?: unknown; + event?: { agent_id?: unknown }; + }; + for (const candidate of [ + record.tenant_id, + record.agent_id, + record.event?.agent_id, + ]) { + if (typeof candidate !== 'string') continue; + const normalized = candidate.trim(); + if (normalized) return normalized; + } + return null; + } catch { + return null; + } finally { + if (fd != null) fs.closeSync(fd); + } +} + +function removeDateDirIfEmpty(dateDir: string): void { + try { + if (fs.readdirSync(dateDir).length === 0) { + fs.rmdirSync(dateDir); + } + } catch { + // Best-effort cleanup only; a concurrent writer may have added a file. + } +} + +export function pruneExpiredSkillRunTrajectories(input?: { + config?: RuntimeConfig; + adaptiveSkills?: AdaptiveSkillsConfig; + now?: Date; +}): number { + const runtimeConfig = input?.config ?? getRuntimeConfig(); + const adaptiveSkills = input?.adaptiveSkills ?? runtimeConfig.adaptiveSkills; + if (adaptiveSkills.trajectoryCapture.retentionDays <= 0) return 0; + + const storeDir = resolveSkillRunTrajectoryStoreDir( + runtimeConfig, + adaptiveSkills, + ); + if (!fs.existsSync(storeDir)) return 0; + + let prunedFiles = 0; + const now = input?.now ?? new Date(); + try { + for (const dateEntry of fs.readdirSync(storeDir, { withFileTypes: true })) { + if ( + !dateEntry.isDirectory() || + !TRAJECTORY_DATE_DIR_PATTERN.test(dateEntry.name) + ) { + continue; + } + const dateDir = path.join(storeDir, dateEntry.name); + for (const fileEntry of fs.readdirSync(dateDir, { + withFileTypes: true, + })) { + if (!fileEntry.isFile() || !fileEntry.name.endsWith('.jsonl')) { + continue; + } + const filePath = path.join(dateDir, fileEntry.name); + const fileTenantId = + readTrajectoryTenantId(filePath) ?? + path.basename(fileEntry.name, '.jsonl'); + if ( + !shouldPruneTrajectoryDate({ + date: dateEntry.name, + retentionDays: tenantRetentionDays(fileTenantId, adaptiveSkills), + now, + }) + ) { + continue; + } + fs.unlinkSync(filePath); + prunedFiles += 1; + } + removeDateDirIfEmpty(dateDir); + } + } catch (error) { + logger.warn( + { storeDir, error }, + 'Failed to prune expired skill run trajectories', + ); + } + return prunedFiles; +} diff --git a/src/skills/skills-inspection.ts b/src/skills/skills-inspection.ts index 5e16e12b7..c4181016b 100644 --- a/src/skills/skills-inspection.ts +++ b/src/skills/skills-inspection.ts @@ -15,10 +15,12 @@ import type { AdaptiveSkillsConfig, SkillHealthMetrics, } from './adaptive-skills-types.js'; +import { pruneExpiredSkillRunTrajectories } from './skill-run-trajectories.js'; import { applyAmendment, proposeAmendment } from './skills-amendment.js'; const LAST_INSPECTION_KEY = 'adaptive-skills:last-inspection-at'; const LAST_OBSERVATION_PRUNE_KEY = 'adaptive-skills:last-observation-prune-at'; +const LAST_TRAJECTORY_PRUNE_KEY = 'adaptive-skills:last-trajectory-prune-at'; const queuedSkillAmendments = new Set(); let queuedSkillAmendmentWork: Promise = Promise.resolve(); @@ -77,6 +79,28 @@ function runObservationPruneIfDue( return pruneSkillObservations({ createdBefore: cutoffIso }); } +function runTrajectoryPruneIfDue( + agentId: string, + config: AdaptiveSkillsConfig, + now: number, +): number { + if (config.trajectoryCapture.retentionDays <= 0) return 0; + if ( + !shouldRunScheduledWork( + agentId, + LAST_TRAJECTORY_PRUNE_KEY, + config.inspectionIntervalMs, + now, + ) + ) { + return 0; + } + return pruneExpiredSkillRunTrajectories({ + adaptiveSkills: config, + now: new Date(now), + }); +} + function inspectionSeverity(metrics: SkillHealthMetrics): number { if (!metrics.degraded) return 0; return ( @@ -243,6 +267,13 @@ export async function runPeriodicSkillInspection(input?: { 'Pruned expired adaptive skill observations', ); } + const prunedTrajectories = runTrajectoryPruneIfDue(agentId, config, now); + if (prunedTrajectories > 0) { + logger.info( + { prunedTrajectories }, + 'Pruned expired skill run trajectories', + ); + } if (!config.enabled) return []; diff --git a/tests/config-reload.integration.test.ts b/tests/config-reload.integration.test.ts index 21a49eea2..c965b89af 100644 --- a/tests/config-reload.integration.test.ts +++ b/tests/config-reload.integration.test.ts @@ -88,6 +88,31 @@ describe('config reload integration', () => { expect(cfg.ops.healthPort).toBe(7777); }); + it('reloadRuntimeConfig normalizes trajectory retention policy', () => { + writeConfig({ + adaptiveSkills: { + trajectoryCapture: { + retentionDays: 90, + retentionDaysByTenant: { + writer: '30', + reviewer: -5, + invalid: 'not-a-number', + }, + }, + }, + }); + + const cfg = configMod.reloadRuntimeConfig('test'); + expect(cfg.adaptiveSkills.trajectoryCapture.retentionDays).toBe(90); + expect( + cfg.adaptiveSkills.trajectoryCapture.retentionDaysByTenant, + ).toEqual({ + writer: 30, + reviewer: 0, + invalid: 90, + }); + }); + it('missing config.json yields default config after ensureRuntimeConfigFile', () => { // Remove any config.json that ensureRuntimeConfigFile may have seeded. if (fs.existsSync(configPath)) fs.unlinkSync(configPath); diff --git a/tests/skills-inspection.test.ts b/tests/skills-inspection.test.ts index 41fa88a4b..66288470e 100644 --- a/tests/skills-inspection.test.ts +++ b/tests/skills-inspection.test.ts @@ -1,3 +1,5 @@ +import fs from 'node:fs'; +import path from 'node:path'; import Database from 'better-sqlite3'; import { afterEach, expect, test, vi } from 'vitest'; import type { AdaptiveSkillsTestContext } from './helpers/adaptive-skills-test-setup.ts'; @@ -221,3 +223,56 @@ test('runPeriodicSkillInspection prunes observations older than the retention wi 'session-new', ]); }); + +test('runPeriodicSkillInspection prunes expired trajectory files by tenant policy', async () => { + context = await createAdaptiveSkillsTestContext(); + const storeDir = path.join(context.homeDir, 'trajectory-store'); + context.runtimeConfigModule.updateRuntimeConfig((draft) => { + draft.adaptiveSkills.enabled = false; + draft.adaptiveSkills.inspectionIntervalMs = 0; + draft.adaptiveSkills.trajectoryCapture.storeDir = storeDir; + draft.adaptiveSkills.trajectoryCapture.retentionDays = 365; + draft.adaptiveSkills.trajectoryCapture.retentionDaysByTenant = { + 'agent-short': 1, + 'agent-keep': 999, + }; + }); + + const dateDaysAgo = (days: number) => + new Date(Date.now() - days * 24 * 60 * 60 * 1000) + .toISOString() + .slice(0, 10); + const writeTrajectory = (date: string, agentId: string) => { + const filePath = path.join(storeDir, date, `${agentId}.jsonl`); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync( + filePath, + `${JSON.stringify({ + schema_version: 2, + tenant_id: agentId, + agent_id: agentId, + })}\n`, + 'utf-8', + ); + return filePath; + }; + + const staleDefaultPath = writeTrajectory(dateDaysAgo(400), 'agent-default'); + const staleShortPath = writeTrajectory(dateDaysAgo(5), 'agent-short'); + const retainedOverridePath = writeTrajectory(dateDaysAgo(400), 'agent-keep'); + const retainedDefaultPath = writeTrajectory(dateDaysAgo(2), 'agent-default'); + + const { runPeriodicSkillInspection } = await import( + '../src/skills/skills-inspection.ts' + ); + const result = await runPeriodicSkillInspection({ + agentId: 'trajectory-cleanup', + }); + expect(result).toEqual([]); + + expect(fs.existsSync(staleDefaultPath)).toBe(false); + expect(fs.existsSync(staleShortPath)).toBe(false); + expect(fs.existsSync(retainedOverridePath)).toBe(true); + expect(fs.existsSync(retainedDefaultPath)).toBe(true); + expect(fs.existsSync(path.dirname(staleShortPath))).toBe(false); +}); diff --git a/tests/skills-observation.test.ts b/tests/skills-observation.test.ts index 34cc1e52e..33d63db77 100644 --- a/tests/skills-observation.test.ts +++ b/tests/skills-observation.test.ts @@ -696,9 +696,50 @@ test('captures opt-in skill_run trajectories in append-only files keyed by date expect(rows).toHaveLength(2); expect(rows[0]).toMatchObject({ - schema_version: 1, + schema_version: 2, date, + tenant_id: 'agent-1', agent_id: 'agent-1', + skill_id: context.skillName, + session_id: 'session-trajectory-1', + run_id: 'run-trajectory-1', + input: { + truncated: false, + source: 'full', + content: expect.stringContaining('draft the note'), + }, + output: { + truncated: false, + source: 'full', + content: expect.stringContaining('o'.repeat(4_500)), + }, + model: null, + tools_used: [ + { + name: 'bash', + arguments: { + source: 'full', + truncated: false, + content: expect.stringContaining('printf data'), + }, + result: { + source: 'full', + truncated: false, + content: expect.stringContaining('tool-output-'), + }, + }, + ], + outcome: 'success', + score: { + run: 1, + agent_skill: { + score: expect.any(Number), + quality_score: expect.any(Number), + reliability_score: expect.any(Number), + timing_score: expect.any(Number), + total_executions: 1, + }, + }, event: { type: 'skill_run', skill_id: context.skillName, From 6b2c9ce5d90bb9e287b9e7a59ca7adf363b864cb Mon Sep 17 00:00:00 2001 From: Benedikt Koehler Date: Tue, 28 Apr 2026 23:06:56 +0200 Subject: [PATCH 2/5] fix: address trajectory review comments --- src/skills/skill-run-trajectories.ts | 41 ++---------- tests/skills-inspection.test.ts | 99 +++++++++++++++------------- tests/skills-observation.test.ts | 8 +-- 3 files changed, 59 insertions(+), 89 deletions(-) diff --git a/src/skills/skill-run-trajectories.ts b/src/skills/skill-run-trajectories.ts index fc408a9e5..cf735b154 100644 --- a/src/skills/skill-run-trajectories.ts +++ b/src/skills/skill-run-trajectories.ts @@ -6,12 +6,8 @@ import { } from '../config/runtime-config.js'; import { DEFAULT_RUNTIME_HOME_DIR } from '../config/runtime-paths.js'; import { logger } from '../logger.js'; -import { getAgentSkillScores } from '../memory/db.js'; import { expandHomePath } from '../utils/path.js'; -import type { - AdaptiveSkillsConfig, - AgentSkillScore, -} from './adaptive-skills-types.js'; +import type { AdaptiveSkillsConfig } from './adaptive-skills-types.js'; import type { SkillRunBoundedPayload, SkillRunEvent, @@ -199,38 +195,11 @@ function scoreSkillRunOutcome(outcome: SkillRunEvent['outcome']): number { } function buildTrajectoryScore( - event: SkillRunEvent & { agent_id: string }, + outcome: SkillRunEvent['outcome'], ): SkillRunTrajectoryScore { - let agentSkillScore: AgentSkillScore | null = null; - try { - const [score] = getAgentSkillScores({ - agentId: event.agent_id, - skillName: event.skill_id, - limit: 1, - }); - agentSkillScore = score ?? null; - } catch (error) { - logger.warn( - { - agentId: event.agent_id, - skillId: event.skill_id, - runId: event.run_id, - error, - }, - 'Failed to read agent skill score for trajectory', - ); - } return { - run: scoreSkillRunOutcome(event.outcome), - agent_skill: agentSkillScore - ? { - score: agentSkillScore.score, - quality_score: agentSkillScore.quality_score, - reliability_score: agentSkillScore.reliability_score, - timing_score: agentSkillScore.timing_score, - total_executions: agentSkillScore.total_executions, - } - : null, + run: scoreSkillRunOutcome(outcome), + agent_skill: null, }; } @@ -255,7 +224,7 @@ export function buildSkillRunTrajectoryRecord( buildTrajectoryToolUse(summary, event, index), ), outcome: event.outcome, - score: buildTrajectoryScore(event), + score: buildTrajectoryScore(event.outcome), event, }; } diff --git a/tests/skills-inspection.test.ts b/tests/skills-inspection.test.ts index 66288470e..3af147368 100644 --- a/tests/skills-inspection.test.ts +++ b/tests/skills-inspection.test.ts @@ -225,54 +225,61 @@ test('runPeriodicSkillInspection prunes observations older than the retention wi }); test('runPeriodicSkillInspection prunes expired trajectory files by tenant policy', async () => { - context = await createAdaptiveSkillsTestContext(); - const storeDir = path.join(context.homeDir, 'trajectory-store'); - context.runtimeConfigModule.updateRuntimeConfig((draft) => { - draft.adaptiveSkills.enabled = false; - draft.adaptiveSkills.inspectionIntervalMs = 0; - draft.adaptiveSkills.trajectoryCapture.storeDir = storeDir; - draft.adaptiveSkills.trajectoryCapture.retentionDays = 365; - draft.adaptiveSkills.trajectoryCapture.retentionDaysByTenant = { - 'agent-short': 1, - 'agent-keep': 999, - }; - }); + vi.useFakeTimers(); + vi.setSystemTime(new Date('2024-01-15T12:00:00.000Z')); - const dateDaysAgo = (days: number) => - new Date(Date.now() - days * 24 * 60 * 60 * 1000) - .toISOString() - .slice(0, 10); - const writeTrajectory = (date: string, agentId: string) => { - const filePath = path.join(storeDir, date, `${agentId}.jsonl`); - fs.mkdirSync(path.dirname(filePath), { recursive: true }); - fs.writeFileSync( - filePath, - `${JSON.stringify({ - schema_version: 2, - tenant_id: agentId, - agent_id: agentId, - })}\n`, - 'utf-8', - ); - return filePath; - }; + try { + context = await createAdaptiveSkillsTestContext(); + const storeDir = path.join(context.homeDir, 'trajectory-store'); + context.runtimeConfigModule.updateRuntimeConfig((draft) => { + draft.adaptiveSkills.enabled = false; + draft.adaptiveSkills.inspectionIntervalMs = 0; + draft.adaptiveSkills.trajectoryCapture.storeDir = storeDir; + draft.adaptiveSkills.trajectoryCapture.retentionDays = 365; + draft.adaptiveSkills.trajectoryCapture.retentionDaysByTenant = { + 'agent-short': 1, + 'agent-keep': 999, + }; + }); - const staleDefaultPath = writeTrajectory(dateDaysAgo(400), 'agent-default'); - const staleShortPath = writeTrajectory(dateDaysAgo(5), 'agent-short'); - const retainedOverridePath = writeTrajectory(dateDaysAgo(400), 'agent-keep'); - const retainedDefaultPath = writeTrajectory(dateDaysAgo(2), 'agent-default'); + const dateDaysAgo = (days: number) => + new Date(Date.now() - days * 24 * 60 * 60 * 1000) + .toISOString() + .slice(0, 10); + const writeTrajectory = (date: string, agentId: string) => { + const filePath = path.join(storeDir, date, `${agentId}.jsonl`); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync( + filePath, + `${JSON.stringify({ + schema_version: 2, + tenant_id: agentId, + agent_id: agentId, + })}\n`, + 'utf-8', + ); + return filePath; + }; - const { runPeriodicSkillInspection } = await import( - '../src/skills/skills-inspection.ts' - ); - const result = await runPeriodicSkillInspection({ - agentId: 'trajectory-cleanup', - }); - expect(result).toEqual([]); + const staleDefaultPath = writeTrajectory(dateDaysAgo(400), 'agent-default'); + const staleShortPath = writeTrajectory(dateDaysAgo(5), 'agent-short'); + const retainedOverridePath = writeTrajectory(dateDaysAgo(400), 'agent-keep'); + const retainedDefaultPath = writeTrajectory(dateDaysAgo(2), 'agent-default'); - expect(fs.existsSync(staleDefaultPath)).toBe(false); - expect(fs.existsSync(staleShortPath)).toBe(false); - expect(fs.existsSync(retainedOverridePath)).toBe(true); - expect(fs.existsSync(retainedDefaultPath)).toBe(true); - expect(fs.existsSync(path.dirname(staleShortPath))).toBe(false); + const { runPeriodicSkillInspection } = await import( + '../src/skills/skills-inspection.ts' + ); + const result = await runPeriodicSkillInspection({ + agentId: 'trajectory-cleanup', + }); + expect(result).toEqual([]); + + expect(fs.existsSync(staleDefaultPath)).toBe(false); + expect(fs.existsSync(staleShortPath)).toBe(false); + expect(fs.existsSync(retainedOverridePath)).toBe(true); + expect(fs.existsSync(retainedDefaultPath)).toBe(true); + expect(fs.existsSync(path.dirname(staleShortPath))).toBe(false); + } finally { + vi.useRealTimers(); + } }); diff --git a/tests/skills-observation.test.ts b/tests/skills-observation.test.ts index 33d63db77..9784a23b7 100644 --- a/tests/skills-observation.test.ts +++ b/tests/skills-observation.test.ts @@ -732,13 +732,7 @@ test('captures opt-in skill_run trajectories in append-only files keyed by date outcome: 'success', score: { run: 1, - agent_skill: { - score: expect.any(Number), - quality_score: expect.any(Number), - reliability_score: expect.any(Number), - timing_score: expect.any(Number), - total_executions: 1, - }, + agent_skill: null, }, event: { type: 'skill_run', From 3093571e97fbdd46024987b74df4df04724a1e72 Mon Sep 17 00:00:00 2001 From: Benedikt Koehler Date: Tue, 28 Apr 2026 23:20:03 +0200 Subject: [PATCH 3/5] refactor: dedupe trajectory agent checks --- src/skills/skill-run-trajectories.ts | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/skills/skill-run-trajectories.ts b/src/skills/skill-run-trajectories.ts index cf735b154..0a81247a0 100644 --- a/src/skills/skill-run-trajectories.ts +++ b/src/skills/skill-run-trajectories.ts @@ -107,27 +107,32 @@ function logTrajectoryCaptureEnabledOnce(input: { ); } -export function isTrajectoryCaptureEnabledForAgentId( +function isEnabledTrajectoryCaptureAgentId( agentId: string | null | undefined, - config: RuntimeConfig, + enabledAgentIds: string[], ): agentId is string { const normalizedAgentId = agentId?.trim(); - if (!normalizedAgentId) return false; - const enabledAgentIds = - config.adaptiveSkills.trajectoryCapture.enabledAgentIds; - if (enabledAgentIds.length === 0) return false; + if (!normalizedAgentId || enabledAgentIds.length === 0) return false; return enabledAgentIds.some((enabledAgentId) => { return enabledAgentId.trim() === normalizedAgentId; }); } +export function isTrajectoryCaptureEnabledForAgentId( + agentId: string | null | undefined, + config: RuntimeConfig, +): agentId is string { + return isEnabledTrajectoryCaptureAgentId( + agentId, + config.adaptiveSkills.trajectoryCapture.enabledAgentIds, + ); +} + function isTrajectoryCaptureEnabledForAgent( event: SkillRunEvent, enabledAgentIds: string[], ): event is SkillRunEvent & { agent_id: string } { - const agentId = event.agent_id?.trim(); - if (!agentId) return false; - return enabledAgentIds.includes(agentId); + return isEnabledTrajectoryCaptureAgentId(event.agent_id, enabledAgentIds); } export function skillRunTrajectoryFilePath(input: { From 4348ebaccc91dc7337c798ea0a5e793baa00356a Mon Sep 17 00:00:00 2001 From: Benedikt Koehler Date: Tue, 28 Apr 2026 23:22:08 +0200 Subject: [PATCH 4/5] fix: omit derived agent skill trajectory score --- src/skills/skill-run-trajectories.ts | 8 -------- tests/skills-observation.test.ts | 1 - 2 files changed, 9 deletions(-) diff --git a/src/skills/skill-run-trajectories.ts b/src/skills/skill-run-trajectories.ts index 0a81247a0..f45d78b28 100644 --- a/src/skills/skill-run-trajectories.ts +++ b/src/skills/skill-run-trajectories.ts @@ -35,13 +35,6 @@ export interface SkillRunTrajectoryToolUse export interface SkillRunTrajectoryScore { run: number; - agent_skill: { - score: number; - quality_score: number; - reliability_score: number; - timing_score: number; - total_executions: number; - } | null; } export interface SkillRunTrajectoryRecord { @@ -204,7 +197,6 @@ function buildTrajectoryScore( ): SkillRunTrajectoryScore { return { run: scoreSkillRunOutcome(outcome), - agent_skill: null, }; } diff --git a/tests/skills-observation.test.ts b/tests/skills-observation.test.ts index 9784a23b7..6b85a2ff9 100644 --- a/tests/skills-observation.test.ts +++ b/tests/skills-observation.test.ts @@ -732,7 +732,6 @@ test('captures opt-in skill_run trajectories in append-only files keyed by date outcome: 'success', score: { run: 1, - agent_skill: null, }, event: { type: 'skill_run', From abe032ce6d0de2e645714268987469edf81bac64 Mon Sep 17 00:00:00 2001 From: Benedikt Koehler Date: Tue, 28 Apr 2026 23:25:42 +0200 Subject: [PATCH 5/5] refactor: simplify trajectory pruning config --- src/skills/skill-run-trajectories.ts | 11 ++--- src/skills/skills-inspection.ts | 28 +++++++++-- tests/skills-inspection.test.ts | 73 +++++++++++++++++++++++++++- 3 files changed, 97 insertions(+), 15 deletions(-) diff --git a/src/skills/skill-run-trajectories.ts b/src/skills/skill-run-trajectories.ts index f45d78b28..b78bb2ad6 100644 --- a/src/skills/skill-run-trajectories.ts +++ b/src/skills/skill-run-trajectories.ts @@ -62,9 +62,8 @@ function safeFilePart(raw: string): string { export function resolveSkillRunTrajectoryStoreDir( config: RuntimeConfig, - adaptiveSkills: AdaptiveSkillsConfig = config.adaptiveSkills, ): string { - const configured = adaptiveSkills.trajectoryCapture.storeDir.trim(); + const configured = config.adaptiveSkills.trajectoryCapture.storeDir.trim(); if (!configured) { return path.join(path.dirname(config.ops.dbPath), 'trajectories'); } @@ -343,17 +342,13 @@ function removeDateDirIfEmpty(dateDir: string): void { export function pruneExpiredSkillRunTrajectories(input?: { config?: RuntimeConfig; - adaptiveSkills?: AdaptiveSkillsConfig; now?: Date; }): number { const runtimeConfig = input?.config ?? getRuntimeConfig(); - const adaptiveSkills = input?.adaptiveSkills ?? runtimeConfig.adaptiveSkills; + const adaptiveSkills = runtimeConfig.adaptiveSkills; if (adaptiveSkills.trajectoryCapture.retentionDays <= 0) return 0; - const storeDir = resolveSkillRunTrajectoryStoreDir( - runtimeConfig, - adaptiveSkills, - ); + const storeDir = resolveSkillRunTrajectoryStoreDir(runtimeConfig); if (!fs.existsSync(storeDir)) return 0; let prunedFiles = 0; diff --git a/src/skills/skills-inspection.ts b/src/skills/skills-inspection.ts index c4181016b..ccdc83b7d 100644 --- a/src/skills/skills-inspection.ts +++ b/src/skills/skills-inspection.ts @@ -1,6 +1,9 @@ import { DEFAULT_AGENT_ID } from '../agents/agent-types.js'; import { makeAuditRunId, recordAuditEvent } from '../audit/audit-events.js'; -import { getRuntimeConfig } from '../config/runtime-config.js'; +import { + getRuntimeConfig, + type RuntimeConfig, +} from '../config/runtime-config.js'; import { logger } from '../logger.js'; import { getLatestSkillAmendment, @@ -28,6 +31,15 @@ function resolveConfig(config?: AdaptiveSkillsConfig): AdaptiveSkillsConfig { return config || getRuntimeConfig().adaptiveSkills; } +function resolveRuntimeConfig(config?: AdaptiveSkillsConfig): RuntimeConfig { + const runtimeConfig = getRuntimeConfig(); + if (!config) return runtimeConfig; + return { + ...runtimeConfig, + adaptiveSkills: config, + }; +} + function windowStartIso(config: AdaptiveSkillsConfig): string { return new Date( Date.now() - config.trailingWindowHours * 60 * 60 * 1000, @@ -81,9 +93,10 @@ function runObservationPruneIfDue( function runTrajectoryPruneIfDue( agentId: string, - config: AdaptiveSkillsConfig, + runtimeConfig: RuntimeConfig, now: number, ): number { + const config = runtimeConfig.adaptiveSkills; if (config.trajectoryCapture.retentionDays <= 0) return 0; if ( !shouldRunScheduledWork( @@ -96,7 +109,7 @@ function runTrajectoryPruneIfDue( return 0; } return pruneExpiredSkillRunTrajectories({ - adaptiveSkills: config, + config: runtimeConfig, now: new Date(now), }); } @@ -257,7 +270,8 @@ export async function runPeriodicSkillInspection(input?: { agentId?: string; config?: AdaptiveSkillsConfig; }): Promise { - const config = resolveConfig(input?.config); + const runtimeConfig = resolveRuntimeConfig(input?.config); + const config = runtimeConfig.adaptiveSkills; const agentId = input?.agentId || DEFAULT_AGENT_ID; const now = Date.now(); const prunedObservations = runObservationPruneIfDue(agentId, config, now); @@ -267,7 +281,11 @@ export async function runPeriodicSkillInspection(input?: { 'Pruned expired adaptive skill observations', ); } - const prunedTrajectories = runTrajectoryPruneIfDue(agentId, config, now); + const prunedTrajectories = runTrajectoryPruneIfDue( + agentId, + runtimeConfig, + now, + ); if (prunedTrajectories > 0) { logger.info( { prunedTrajectories }, diff --git a/tests/skills-inspection.test.ts b/tests/skills-inspection.test.ts index 3af147368..ab87cdaab 100644 --- a/tests/skills-inspection.test.ts +++ b/tests/skills-inspection.test.ts @@ -263,8 +263,14 @@ test('runPeriodicSkillInspection prunes expired trajectory files by tenant polic const staleDefaultPath = writeTrajectory(dateDaysAgo(400), 'agent-default'); const staleShortPath = writeTrajectory(dateDaysAgo(5), 'agent-short'); - const retainedOverridePath = writeTrajectory(dateDaysAgo(400), 'agent-keep'); - const retainedDefaultPath = writeTrajectory(dateDaysAgo(2), 'agent-default'); + const retainedOverridePath = writeTrajectory( + dateDaysAgo(400), + 'agent-keep', + ); + const retainedDefaultPath = writeTrajectory( + dateDaysAgo(2), + 'agent-default', + ); const { runPeriodicSkillInspection } = await import( '../src/skills/skills-inspection.ts' @@ -283,3 +289,66 @@ test('runPeriodicSkillInspection prunes expired trajectory files by tenant polic vi.useRealTimers(); } }); + +test('pruneExpiredSkillRunTrajectories resolves empty store dir from provided runtime config', async () => { + context = await createAdaptiveSkillsTestContext(); + const runtimeConfig = context.runtimeConfigModule.getRuntimeConfig(); + const configuredDbPath = path.join( + context.homeDir, + 'configured-data', + 'test.db', + ); + const configuredStoreDir = path.join( + path.dirname(configuredDbPath), + 'trajectories', + ); + const runtimeStoreDir = path.join( + path.dirname(context.dbPath), + 'trajectories', + ); + + const writeTrajectory = (storeDir: string) => { + const filePath = path.join(storeDir, '2024-01-01', 'agent-default.jsonl'); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync( + filePath, + `${JSON.stringify({ + schema_version: 2, + tenant_id: 'agent-default', + agent_id: 'agent-default', + })}\n`, + 'utf-8', + ); + return filePath; + }; + + const configuredPath = writeTrajectory(configuredStoreDir); + const runtimePath = writeTrajectory(runtimeStoreDir); + const { pruneExpiredSkillRunTrajectories } = await import( + '../src/skills/skill-run-trajectories.ts' + ); + + const pruned = pruneExpiredSkillRunTrajectories({ + config: { + ...runtimeConfig, + ops: { + ...runtimeConfig.ops, + dbPath: configuredDbPath, + }, + adaptiveSkills: { + ...runtimeConfig.adaptiveSkills, + trajectoryCapture: { + ...runtimeConfig.adaptiveSkills.trajectoryCapture, + storeDir: '', + retentionDays: 1, + retentionDaysByTenant: {}, + }, + }, + }, + now: new Date('2024-01-15T12:00:00.000Z'), + }); + + expect(pruned).toBe(1); + expect(fs.existsSync(configuredPath)).toBe(false); + expect(fs.existsSync(runtimePath)).toBe(true); +});