diff --git a/docs/content/extensibility/adaptive-skills.md b/docs/content/extensibility/adaptive-skills.md index 6d5057b7..386df731 100644 --- a/docs/content/extensibility/adaptive-skills.md +++ b/docs/content/extensibility/adaptive-skills.md @@ -44,7 +44,9 @@ is disabled by default. "rollbackImprovementThreshold": 0.05, "trajectoryCapture": { "enabledAgentIds": [], - "storeDir": "" + "storeDir": "", + "retentionDays": 365, + "retentionDaysByTenant": {} } } } @@ -66,6 +68,10 @@ Key settings: - `trajectoryCapture.storeDir`: optional trajectory store location; empty uses a `trajectories/` directory beside the runtime database, absolute paths are used as-is, and relative paths resolve under the runtime home directory +- `trajectoryCapture.retentionDays`: trajectory JSONL retention window; default + is 365 days and `0` disables trajectory pruning +- `trajectoryCapture.retentionDaysByTenant`: optional per-coworker retention + overrides keyed by tenant ID; trajectory tenants currently map to agent IDs Legacy `skillCognee` config input is still normalized into `adaptiveSkills` for backward compatibility. @@ -90,12 +96,14 @@ Feedback is attached to the most recent observation for the same session. ## Retention -Observation queries are windowed, but storage is also pruned now. On each +Observation queries are windowed, but storage is also pruned. On each inspection interval the heartbeat deletes `skill_observations` rows older than -`observationRetentionDays`. +`observationRetentionDays` and removes expired trajectory JSONL files according +to the trajectory retention policy. -This keeps high-traffic skills from accumulating unbounded observation history -while preserving the amendment history table as the durable review log. +This keeps high-traffic skills from accumulating unbounded observation and +trajectory history while preserving the amendment history table as the durable +review log. ## Operator Surfaces diff --git a/docs/content/reference/configuration.md b/docs/content/reference/configuration.md index 0cc3477c..8bfe17a8 100644 --- a/docs/content/reference/configuration.md +++ b/docs/content/reference/configuration.md @@ -151,7 +151,9 @@ saved revision history directly. `adaptiveSkills.trajectoryCapture.enabledAgentIds`; when `adaptiveSkills.trajectoryCapture.storeDir` is empty, trajectories are stored beside the runtime database, absolute paths are used as-is, and relative paths - resolve under the runtime home directory + resolve under the runtime home directory; trajectory retention defaults to + `adaptiveSkills.trajectoryCapture.retentionDays: 365` and can be overridden + per coworker with `adaptiveSkills.trajectoryCapture.retentionDaysByTenant` - `imessage.*` for the dual-backend local or BlueBubbles iMessage transport; prefer storing the BlueBubbles password as `IMESSAGE_PASSWORD` in the encrypted secret store instead of plaintext config diff --git a/docs/development/extensibility/adaptive-skills.md b/docs/development/extensibility/adaptive-skills.md index 6d5057b7..386df731 100644 --- a/docs/development/extensibility/adaptive-skills.md +++ b/docs/development/extensibility/adaptive-skills.md @@ -44,7 +44,9 @@ is disabled by default. "rollbackImprovementThreshold": 0.05, "trajectoryCapture": { "enabledAgentIds": [], - "storeDir": "" + "storeDir": "", + "retentionDays": 365, + "retentionDaysByTenant": {} } } } @@ -66,6 +68,10 @@ Key settings: - `trajectoryCapture.storeDir`: optional trajectory store location; empty uses a `trajectories/` directory beside the runtime database, absolute paths are used as-is, and relative paths resolve under the runtime home directory +- `trajectoryCapture.retentionDays`: trajectory JSONL retention window; default + is 365 days and `0` disables trajectory pruning +- `trajectoryCapture.retentionDaysByTenant`: optional per-coworker retention + overrides keyed by tenant ID; trajectory tenants currently map to agent IDs Legacy `skillCognee` config input is still normalized into `adaptiveSkills` for backward compatibility. @@ -90,12 +96,14 @@ Feedback is attached to the most recent observation for the same session. ## Retention -Observation queries are windowed, but storage is also pruned now. On each +Observation queries are windowed, but storage is also pruned. On each inspection interval the heartbeat deletes `skill_observations` rows older than -`observationRetentionDays`. +`observationRetentionDays` and removes expired trajectory JSONL files according +to the trajectory retention policy. -This keeps high-traffic skills from accumulating unbounded observation history -while preserving the amendment history table as the durable review log. +This keeps high-traffic skills from accumulating unbounded observation and +trajectory history while preserving the amendment history table as the durable +review log. ## Operator Surfaces diff --git a/docs/development/reference/configuration.md b/docs/development/reference/configuration.md index 1479bf67..b60174f7 100644 --- a/docs/development/reference/configuration.md +++ b/docs/development/reference/configuration.md @@ -125,7 +125,9 @@ leak into the saved revision metadata. `adaptiveSkills.trajectoryCapture.enabledAgentIds`; when `adaptiveSkills.trajectoryCapture.storeDir` is empty, trajectories are stored beside the runtime database, absolute paths are used as-is, and relative paths - resolve under the runtime home directory + resolve under the runtime home directory; trajectory retention defaults to + `adaptiveSkills.trajectoryCapture.retentionDays: 365` and can be overridden + per coworker with `adaptiveSkills.trajectoryCapture.retentionDaysByTenant` - `imessage.*` for the dual-backend local or BlueBubbles iMessage transport; prefer storing the BlueBubbles password as `IMESSAGE_PASSWORD` in the encrypted secret store instead of plaintext config diff --git a/src/config/runtime-config.ts b/src/config/runtime-config.ts index a2d33762..19e15cf8 100644 --- a/src/config/runtime-config.ts +++ b/src/config/runtime-config.ts @@ -1049,6 +1049,8 @@ export const DEFAULT_RUNTIME_CONFIG: RuntimeConfig = { trajectoryCapture: { enabledAgentIds: [], storeDir: '', + retentionDays: 365, + retentionDaysByTenant: {}, }, inspectionIntervalMs: 3_600_000, observationRetentionDays: 30, @@ -1733,6 +1735,25 @@ function normalizeStringArray(value: unknown, fallback: string[]): string[] { return fallback; } +function normalizeRetentionDaysByTenant( + value: unknown, + fallback: Record, + defaultRetentionDays: number, +): Record { + if (!isRecord(value)) return { ...fallback }; + const normalized: Record = {}; + for (const [tenantId, rawDays] of Object.entries(value)) { + const normalizedTenantId = tenantId.trim(); + if (!normalizedTenantId) continue; + normalized[normalizedTenantId] = normalizeInteger( + rawDays, + fallback[normalizedTenantId] ?? defaultRetentionDays, + { min: 0 }, + ); + } + return normalized; +} + function normalizeOptionalBaseUrl(value: unknown, fallback: string): string { const candidate = normalizeString(value, fallback, { allowEmpty: true }); return candidate ? candidate.replace(/\/+$/, '') : ''; @@ -4716,6 +4737,11 @@ function normalizeRuntimeConfig( rawDiscord.commandMode, legacyCommandModeFallback, ); + const normalizedTrajectoryRetentionDays = normalizeInteger( + rawTrajectoryCapture.retentionDays, + DEFAULT_RUNTIME_CONFIG.adaptiveSkills.trajectoryCapture.retentionDays, + { min: 0 }, + ); return { version: CONFIG_VERSION, @@ -4803,6 +4829,13 @@ function normalizeRuntimeConfig( DEFAULT_RUNTIME_CONFIG.adaptiveSkills.trajectoryCapture.storeDir, { allowEmpty: true }, ), + retentionDays: normalizedTrajectoryRetentionDays, + retentionDaysByTenant: normalizeRetentionDaysByTenant( + rawTrajectoryCapture.retentionDaysByTenant, + DEFAULT_RUNTIME_CONFIG.adaptiveSkills.trajectoryCapture + .retentionDaysByTenant, + normalizedTrajectoryRetentionDays, + ), }, inspectionIntervalMs: normalizeInteger( rawAdaptiveSkills.inspectionIntervalMs, diff --git a/src/skills/adaptive-skills-types.ts b/src/skills/adaptive-skills-types.ts index f567ea87..b4aed5ba 100644 --- a/src/skills/adaptive-skills-types.ts +++ b/src/skills/adaptive-skills-types.ts @@ -139,6 +139,8 @@ export interface AdaptiveSkillsConfig { trajectoryCapture: { enabledAgentIds: string[]; storeDir: string; + retentionDays: number; + retentionDaysByTenant: Record; }; inspectionIntervalMs: number; observationRetentionDays: number; diff --git a/src/skills/skill-run-trajectories.ts b/src/skills/skill-run-trajectories.ts index 64fe5a34..b78bb2ad 100644 --- a/src/skills/skill-run-trajectories.ts +++ b/src/skills/skill-run-trajectories.ts @@ -7,17 +7,51 @@ import { import { DEFAULT_RUNTIME_HOME_DIR } from '../config/runtime-paths.js'; import { logger } from '../logger.js'; import { expandHomePath } from '../utils/path.js'; -import type { SkillRunEvent } from './skill-run-events.js'; +import type { AdaptiveSkillsConfig } from './adaptive-skills-types.js'; +import type { + SkillRunBoundedPayload, + SkillRunEvent, + SkillRunFullPayload, + SkillRunToolExecutionSummary, +} from './skill-run-events.js'; -export const SKILL_RUN_TRAJECTORY_SCHEMA_VERSION = 1; +export const SKILL_RUN_TRAJECTORY_SCHEMA_VERSION = 2; const SKILL_RUN_TRAJECTORY_DIR_MODE = 0o700; +const MS_PER_DAY = 24 * 60 * 60 * 1000; +const TRAJECTORY_DATE_DIR_PATTERN = /^\d{4}-\d{2}-\d{2}$/; let loggedTrajectoryCaptureConfigKey: string | null = null; +export interface SkillRunTrajectoryPayload { + content: string; + truncated: boolean; + source: 'bounded' | 'full'; +} + +export interface SkillRunTrajectoryToolUse + extends SkillRunToolExecutionSummary { + arguments: SkillRunTrajectoryPayload | null; + result: SkillRunTrajectoryPayload | null; +} + +export interface SkillRunTrajectoryScore { + run: number; +} + export interface SkillRunTrajectoryRecord { schema_version: typeof SKILL_RUN_TRAJECTORY_SCHEMA_VERSION; captured_at: string; date: string; + tenant_id: string; agent_id: string; + skill_id: string; + session_id: string; + run_id: string; + input: SkillRunTrajectoryPayload | null; + output: SkillRunTrajectoryPayload | null; + model: string | null; + tools_used: SkillRunTrajectoryToolUse[]; + outcome: SkillRunEvent['outcome']; + score: SkillRunTrajectoryScore; event: SkillRunEvent; } @@ -26,7 +60,9 @@ function safeFilePart(raw: string): string { return normalized || 'unknown'; } -function resolveTrajectoryStoreDir(config: RuntimeConfig): string { +export function resolveSkillRunTrajectoryStoreDir( + config: RuntimeConfig, +): string { const configured = config.adaptiveSkills.trajectoryCapture.storeDir.trim(); if (!configured) { return path.join(path.dirname(config.ops.dbPath), 'trajectories'); @@ -37,9 +73,10 @@ function resolveTrajectoryStoreDir(config: RuntimeConfig): string { return path.join(DEFAULT_RUNTIME_HOME_DIR, expanded); } -function normalizedTrajectoryCaptureAgentIds(config: RuntimeConfig): string[] { - const enabledAgentIds = - config.adaptiveSkills.trajectoryCapture.enabledAgentIds; +function normalizedTrajectoryCaptureAgentIds( + config: AdaptiveSkillsConfig, +): string[] { + const enabledAgentIds = config.trajectoryCapture.enabledAgentIds; if (enabledAgentIds.length === 0) return []; return enabledAgentIds.map((agentId) => agentId.trim()).filter(Boolean); } @@ -62,27 +99,32 @@ function logTrajectoryCaptureEnabledOnce(input: { ); } -export function isTrajectoryCaptureEnabledForAgentId( +function isEnabledTrajectoryCaptureAgentId( agentId: string | null | undefined, - config: RuntimeConfig, + enabledAgentIds: string[], ): agentId is string { const normalizedAgentId = agentId?.trim(); - if (!normalizedAgentId) return false; - const enabledAgentIds = - config.adaptiveSkills.trajectoryCapture.enabledAgentIds; - if (enabledAgentIds.length === 0) return false; + if (!normalizedAgentId || enabledAgentIds.length === 0) return false; return enabledAgentIds.some((enabledAgentId) => { return enabledAgentId.trim() === normalizedAgentId; }); } +export function isTrajectoryCaptureEnabledForAgentId( + agentId: string | null | undefined, + config: RuntimeConfig, +): agentId is string { + return isEnabledTrajectoryCaptureAgentId( + agentId, + config.adaptiveSkills.trajectoryCapture.enabledAgentIds, + ); +} + function isTrajectoryCaptureEnabledForAgent( event: SkillRunEvent, enabledAgentIds: string[], ): event is SkillRunEvent & { agent_id: string } { - const agentId = event.agent_id?.trim(); - if (!agentId) return false; - return enabledAgentIds.includes(agentId); + return isEnabledTrajectoryCaptureAgentId(event.agent_id, enabledAgentIds); } export function skillRunTrajectoryFilePath(input: { @@ -111,6 +153,52 @@ function ensurePrivateTrajectoryDirectories(input: { }); } +function buildTrajectoryPayload( + bounded: SkillRunBoundedPayload | null, + full: SkillRunFullPayload | null, +): SkillRunTrajectoryPayload | null { + if (full) { + return { + content: full.content, + truncated: false, + source: 'full', + }; + } + if (!bounded) return null; + return { + content: bounded.content, + truncated: bounded.truncated, + source: 'bounded', + }; +} + +function buildTrajectoryToolUse( + summary: SkillRunToolExecutionSummary, + event: SkillRunEvent, + index: number, +): SkillRunTrajectoryToolUse { + const full = event.tool_executions_full[index]; + return { + ...summary, + arguments: full ? buildTrajectoryPayload(null, full.arguments) : null, + result: full ? buildTrajectoryPayload(null, full.result) : null, + }; +} + +function scoreSkillRunOutcome(outcome: SkillRunEvent['outcome']): number { + if (outcome === 'success') return 1; + if (outcome === 'partial') return 0.5; + return 0; +} + +function buildTrajectoryScore( + outcome: SkillRunEvent['outcome'], +): SkillRunTrajectoryScore { + return { + run: scoreSkillRunOutcome(outcome), + }; +} + export function buildSkillRunTrajectoryRecord( event: SkillRunEvent & { agent_id: string }, capturedAt = new Date(), @@ -120,20 +208,34 @@ export function buildSkillRunTrajectoryRecord( schema_version: SKILL_RUN_TRAJECTORY_SCHEMA_VERSION, captured_at, date: captured_at.slice(0, 10), + tenant_id: event.agent_id, agent_id: event.agent_id, + skill_id: event.skill_id, + session_id: event.session_id, + run_id: event.run_id, + input: buildTrajectoryPayload(event.input, event.input_full), + output: buildTrajectoryPayload(event.output, event.output_full), + model: event.model, + tools_used: event.tool_executions.map((summary, index) => + buildTrajectoryToolUse(summary, event, index), + ), + outcome: event.outcome, + score: buildTrajectoryScore(event.outcome), event, }; } export function recordSkillRunTrajectory(event: SkillRunEvent): void { const config = getRuntimeConfig(); - const enabledAgentIds = normalizedTrajectoryCaptureAgentIds(config); + const enabledAgentIds = normalizedTrajectoryCaptureAgentIds( + config.adaptiveSkills, + ); if (enabledAgentIds.length === 0) { loggedTrajectoryCaptureConfigKey = null; return; } - const storeDir = resolveTrajectoryStoreDir(config); + const storeDir = resolveSkillRunTrajectoryStoreDir(config); logTrajectoryCaptureEnabledOnce({ agentIds: enabledAgentIds, storeDir, @@ -168,3 +270,127 @@ export function recordSkillRunTrajectory(event: SkillRunEvent): void { ); } } + +function tenantRetentionDays( + tenantId: string, + config: AdaptiveSkillsConfig, +): number { + const normalizedTenantId = tenantId.trim(); + const overrides = config.trajectoryCapture.retentionDaysByTenant; + const configured = + overrides[normalizedTenantId] ?? + overrides[safeFilePart(normalizedTenantId)]; + return configured ?? config.trajectoryCapture.retentionDays; +} + +function shouldPruneTrajectoryDate(input: { + date: string; + retentionDays: number; + now: Date; +}): boolean { + if (input.retentionDays <= 0) return false; + const cutoffDate = new Date( + input.now.getTime() - input.retentionDays * MS_PER_DAY, + ) + .toISOString() + .slice(0, 10); + return input.date < cutoffDate; +} + +function readTrajectoryTenantId(filePath: string): string | null { + let fd: number | null = null; + try { + fd = fs.openSync(filePath, 'r'); + const buffer = Buffer.alloc(8192); + const bytesRead = fs.readSync(fd, buffer, 0, buffer.length, 0); + const firstLine = buffer + .toString('utf-8', 0, bytesRead) + .split(/\r?\n/, 1)[0] + ?.trim(); + if (!firstLine) return null; + const record = JSON.parse(firstLine) as { + tenant_id?: unknown; + agent_id?: unknown; + event?: { agent_id?: unknown }; + }; + for (const candidate of [ + record.tenant_id, + record.agent_id, + record.event?.agent_id, + ]) { + if (typeof candidate !== 'string') continue; + const normalized = candidate.trim(); + if (normalized) return normalized; + } + return null; + } catch { + return null; + } finally { + if (fd != null) fs.closeSync(fd); + } +} + +function removeDateDirIfEmpty(dateDir: string): void { + try { + if (fs.readdirSync(dateDir).length === 0) { + fs.rmdirSync(dateDir); + } + } catch { + // Best-effort cleanup only; a concurrent writer may have added a file. + } +} + +export function pruneExpiredSkillRunTrajectories(input?: { + config?: RuntimeConfig; + now?: Date; +}): number { + const runtimeConfig = input?.config ?? getRuntimeConfig(); + const adaptiveSkills = runtimeConfig.adaptiveSkills; + if (adaptiveSkills.trajectoryCapture.retentionDays <= 0) return 0; + + const storeDir = resolveSkillRunTrajectoryStoreDir(runtimeConfig); + if (!fs.existsSync(storeDir)) return 0; + + let prunedFiles = 0; + const now = input?.now ?? new Date(); + try { + for (const dateEntry of fs.readdirSync(storeDir, { withFileTypes: true })) { + if ( + !dateEntry.isDirectory() || + !TRAJECTORY_DATE_DIR_PATTERN.test(dateEntry.name) + ) { + continue; + } + const dateDir = path.join(storeDir, dateEntry.name); + for (const fileEntry of fs.readdirSync(dateDir, { + withFileTypes: true, + })) { + if (!fileEntry.isFile() || !fileEntry.name.endsWith('.jsonl')) { + continue; + } + const filePath = path.join(dateDir, fileEntry.name); + const fileTenantId = + readTrajectoryTenantId(filePath) ?? + path.basename(fileEntry.name, '.jsonl'); + if ( + !shouldPruneTrajectoryDate({ + date: dateEntry.name, + retentionDays: tenantRetentionDays(fileTenantId, adaptiveSkills), + now, + }) + ) { + continue; + } + fs.unlinkSync(filePath); + prunedFiles += 1; + } + removeDateDirIfEmpty(dateDir); + } + } catch (error) { + logger.warn( + { storeDir, error }, + 'Failed to prune expired skill run trajectories', + ); + } + return prunedFiles; +} diff --git a/src/skills/skills-inspection.ts b/src/skills/skills-inspection.ts index 5e16e12b..ccdc83b7 100644 --- a/src/skills/skills-inspection.ts +++ b/src/skills/skills-inspection.ts @@ -1,6 +1,9 @@ import { DEFAULT_AGENT_ID } from '../agents/agent-types.js'; import { makeAuditRunId, recordAuditEvent } from '../audit/audit-events.js'; -import { getRuntimeConfig } from '../config/runtime-config.js'; +import { + getRuntimeConfig, + type RuntimeConfig, +} from '../config/runtime-config.js'; import { logger } from '../logger.js'; import { getLatestSkillAmendment, @@ -15,10 +18,12 @@ import type { AdaptiveSkillsConfig, SkillHealthMetrics, } from './adaptive-skills-types.js'; +import { pruneExpiredSkillRunTrajectories } from './skill-run-trajectories.js'; import { applyAmendment, proposeAmendment } from './skills-amendment.js'; const LAST_INSPECTION_KEY = 'adaptive-skills:last-inspection-at'; const LAST_OBSERVATION_PRUNE_KEY = 'adaptive-skills:last-observation-prune-at'; +const LAST_TRAJECTORY_PRUNE_KEY = 'adaptive-skills:last-trajectory-prune-at'; const queuedSkillAmendments = new Set(); let queuedSkillAmendmentWork: Promise = Promise.resolve(); @@ -26,6 +31,15 @@ function resolveConfig(config?: AdaptiveSkillsConfig): AdaptiveSkillsConfig { return config || getRuntimeConfig().adaptiveSkills; } +function resolveRuntimeConfig(config?: AdaptiveSkillsConfig): RuntimeConfig { + const runtimeConfig = getRuntimeConfig(); + if (!config) return runtimeConfig; + return { + ...runtimeConfig, + adaptiveSkills: config, + }; +} + function windowStartIso(config: AdaptiveSkillsConfig): string { return new Date( Date.now() - config.trailingWindowHours * 60 * 60 * 1000, @@ -77,6 +91,29 @@ function runObservationPruneIfDue( return pruneSkillObservations({ createdBefore: cutoffIso }); } +function runTrajectoryPruneIfDue( + agentId: string, + runtimeConfig: RuntimeConfig, + now: number, +): number { + const config = runtimeConfig.adaptiveSkills; + if (config.trajectoryCapture.retentionDays <= 0) return 0; + if ( + !shouldRunScheduledWork( + agentId, + LAST_TRAJECTORY_PRUNE_KEY, + config.inspectionIntervalMs, + now, + ) + ) { + return 0; + } + return pruneExpiredSkillRunTrajectories({ + config: runtimeConfig, + now: new Date(now), + }); +} + function inspectionSeverity(metrics: SkillHealthMetrics): number { if (!metrics.degraded) return 0; return ( @@ -233,7 +270,8 @@ export async function runPeriodicSkillInspection(input?: { agentId?: string; config?: AdaptiveSkillsConfig; }): Promise { - const config = resolveConfig(input?.config); + const runtimeConfig = resolveRuntimeConfig(input?.config); + const config = runtimeConfig.adaptiveSkills; const agentId = input?.agentId || DEFAULT_AGENT_ID; const now = Date.now(); const prunedObservations = runObservationPruneIfDue(agentId, config, now); @@ -243,6 +281,17 @@ export async function runPeriodicSkillInspection(input?: { 'Pruned expired adaptive skill observations', ); } + const prunedTrajectories = runTrajectoryPruneIfDue( + agentId, + runtimeConfig, + now, + ); + if (prunedTrajectories > 0) { + logger.info( + { prunedTrajectories }, + 'Pruned expired skill run trajectories', + ); + } if (!config.enabled) return []; diff --git a/tests/config-reload.integration.test.ts b/tests/config-reload.integration.test.ts index 21a49eea..c965b89a 100644 --- a/tests/config-reload.integration.test.ts +++ b/tests/config-reload.integration.test.ts @@ -88,6 +88,31 @@ describe('config reload integration', () => { expect(cfg.ops.healthPort).toBe(7777); }); + it('reloadRuntimeConfig normalizes trajectory retention policy', () => { + writeConfig({ + adaptiveSkills: { + trajectoryCapture: { + retentionDays: 90, + retentionDaysByTenant: { + writer: '30', + reviewer: -5, + invalid: 'not-a-number', + }, + }, + }, + }); + + const cfg = configMod.reloadRuntimeConfig('test'); + expect(cfg.adaptiveSkills.trajectoryCapture.retentionDays).toBe(90); + expect( + cfg.adaptiveSkills.trajectoryCapture.retentionDaysByTenant, + ).toEqual({ + writer: 30, + reviewer: 0, + invalid: 90, + }); + }); + it('missing config.json yields default config after ensureRuntimeConfigFile', () => { // Remove any config.json that ensureRuntimeConfigFile may have seeded. if (fs.existsSync(configPath)) fs.unlinkSync(configPath); diff --git a/tests/skills-inspection.test.ts b/tests/skills-inspection.test.ts index 41fa88a4..ab87cdaa 100644 --- a/tests/skills-inspection.test.ts +++ b/tests/skills-inspection.test.ts @@ -1,3 +1,5 @@ +import fs from 'node:fs'; +import path from 'node:path'; import Database from 'better-sqlite3'; import { afterEach, expect, test, vi } from 'vitest'; import type { AdaptiveSkillsTestContext } from './helpers/adaptive-skills-test-setup.ts'; @@ -221,3 +223,132 @@ test('runPeriodicSkillInspection prunes observations older than the retention wi 'session-new', ]); }); + +test('runPeriodicSkillInspection prunes expired trajectory files by tenant policy', async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2024-01-15T12:00:00.000Z')); + + try { + context = await createAdaptiveSkillsTestContext(); + const storeDir = path.join(context.homeDir, 'trajectory-store'); + context.runtimeConfigModule.updateRuntimeConfig((draft) => { + draft.adaptiveSkills.enabled = false; + draft.adaptiveSkills.inspectionIntervalMs = 0; + draft.adaptiveSkills.trajectoryCapture.storeDir = storeDir; + draft.adaptiveSkills.trajectoryCapture.retentionDays = 365; + draft.adaptiveSkills.trajectoryCapture.retentionDaysByTenant = { + 'agent-short': 1, + 'agent-keep': 999, + }; + }); + + const dateDaysAgo = (days: number) => + new Date(Date.now() - days * 24 * 60 * 60 * 1000) + .toISOString() + .slice(0, 10); + const writeTrajectory = (date: string, agentId: string) => { + const filePath = path.join(storeDir, date, `${agentId}.jsonl`); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync( + filePath, + `${JSON.stringify({ + schema_version: 2, + tenant_id: agentId, + agent_id: agentId, + })}\n`, + 'utf-8', + ); + return filePath; + }; + + const staleDefaultPath = writeTrajectory(dateDaysAgo(400), 'agent-default'); + const staleShortPath = writeTrajectory(dateDaysAgo(5), 'agent-short'); + const retainedOverridePath = writeTrajectory( + dateDaysAgo(400), + 'agent-keep', + ); + const retainedDefaultPath = writeTrajectory( + dateDaysAgo(2), + 'agent-default', + ); + + const { runPeriodicSkillInspection } = await import( + '../src/skills/skills-inspection.ts' + ); + const result = await runPeriodicSkillInspection({ + agentId: 'trajectory-cleanup', + }); + expect(result).toEqual([]); + + expect(fs.existsSync(staleDefaultPath)).toBe(false); + expect(fs.existsSync(staleShortPath)).toBe(false); + expect(fs.existsSync(retainedOverridePath)).toBe(true); + expect(fs.existsSync(retainedDefaultPath)).toBe(true); + expect(fs.existsSync(path.dirname(staleShortPath))).toBe(false); + } finally { + vi.useRealTimers(); + } +}); + +test('pruneExpiredSkillRunTrajectories resolves empty store dir from provided runtime config', async () => { + context = await createAdaptiveSkillsTestContext(); + const runtimeConfig = context.runtimeConfigModule.getRuntimeConfig(); + const configuredDbPath = path.join( + context.homeDir, + 'configured-data', + 'test.db', + ); + const configuredStoreDir = path.join( + path.dirname(configuredDbPath), + 'trajectories', + ); + const runtimeStoreDir = path.join( + path.dirname(context.dbPath), + 'trajectories', + ); + + const writeTrajectory = (storeDir: string) => { + const filePath = path.join(storeDir, '2024-01-01', 'agent-default.jsonl'); + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync( + filePath, + `${JSON.stringify({ + schema_version: 2, + tenant_id: 'agent-default', + agent_id: 'agent-default', + })}\n`, + 'utf-8', + ); + return filePath; + }; + + const configuredPath = writeTrajectory(configuredStoreDir); + const runtimePath = writeTrajectory(runtimeStoreDir); + const { pruneExpiredSkillRunTrajectories } = await import( + '../src/skills/skill-run-trajectories.ts' + ); + + const pruned = pruneExpiredSkillRunTrajectories({ + config: { + ...runtimeConfig, + ops: { + ...runtimeConfig.ops, + dbPath: configuredDbPath, + }, + adaptiveSkills: { + ...runtimeConfig.adaptiveSkills, + trajectoryCapture: { + ...runtimeConfig.adaptiveSkills.trajectoryCapture, + storeDir: '', + retentionDays: 1, + retentionDaysByTenant: {}, + }, + }, + }, + now: new Date('2024-01-15T12:00:00.000Z'), + }); + + expect(pruned).toBe(1); + expect(fs.existsSync(configuredPath)).toBe(false); + expect(fs.existsSync(runtimePath)).toBe(true); +}); diff --git a/tests/skills-observation.test.ts b/tests/skills-observation.test.ts index 34cc1e52..6b85a2ff 100644 --- a/tests/skills-observation.test.ts +++ b/tests/skills-observation.test.ts @@ -696,9 +696,43 @@ test('captures opt-in skill_run trajectories in append-only files keyed by date expect(rows).toHaveLength(2); expect(rows[0]).toMatchObject({ - schema_version: 1, + schema_version: 2, date, + tenant_id: 'agent-1', agent_id: 'agent-1', + skill_id: context.skillName, + session_id: 'session-trajectory-1', + run_id: 'run-trajectory-1', + input: { + truncated: false, + source: 'full', + content: expect.stringContaining('draft the note'), + }, + output: { + truncated: false, + source: 'full', + content: expect.stringContaining('o'.repeat(4_500)), + }, + model: null, + tools_used: [ + { + name: 'bash', + arguments: { + source: 'full', + truncated: false, + content: expect.stringContaining('printf data'), + }, + result: { + source: 'full', + truncated: false, + content: expect.stringContaining('tool-output-'), + }, + }, + ], + outcome: 'success', + score: { + run: 1, + }, event: { type: 'skill_run', skill_id: context.skillName,