diff --git a/.env.example b/.env.example index dc4a49d..f5b4928 100644 --- a/.env.example +++ b/.env.example @@ -3,12 +3,18 @@ PORT=3000 NODE_ENV=development LOG_LEVEL=info +# Optional API Key Authentication +# If set, all /api/* endpoints (except /api/health) require an X-API-Key header. +# Leave empty or unset to disable authentication. +API_KEY= + # PostgreSQL Configuration POSTGRES_HOST=localhost POSTGRES_PORT=5432 POSTGRES_DB=sandeep_ai POSTGRES_USER=postgres -POSTGRES_PASSWORD=postgres +# ⚠️ Set a strong password — required when using Docker Compose +POSTGRES_PASSWORD=YOUR_STRONG_PASSWORD_HERE # Qdrant Vector Database (Optional) QDRANT_URL=http://localhost:6333 @@ -33,12 +39,16 @@ GEMINI_DEFAULT_MODEL=gemini-pro # Ollama Configuration (for local models) OLLAMA_BASE_URL=http://localhost:11434 -OLLAMA_DEFAULT_MODEL=llama2 +OLLAMA_DEFAULT_MODEL=llama3.1:8b + +# OpenRouter Configuration (single key for Claude, GPT-4o, Gemini, Llama, etc.) +OPENROUTER_API_KEY= +OPENROUTER_DEFAULT_MODEL=anthropic/claude-3.5-haiku -# Embeddings Configuration -EMBEDDINGS_PROVIDER=openai -EMBEDDINGS_MODEL=text-embedding-3-small -EMBEDDINGS_DIMENSION=1536 +# Embeddings Configuration (keep on Ollama even when using cloud providers) +EMBEDDINGS_PROVIDER=ollama +EMBEDDINGS_MODEL=nomic-embed-text +EMBEDDINGS_DIMENSION=768 # Memory Configuration SHORT_TERM_TOKEN_LIMIT=4000 diff --git a/README.md b/README.md index fb8aaa8..9385824 100644 --- a/README.md +++ b/README.md @@ -140,22 +140,13 @@ sandeep-ai/ │ ├── shortTerm.ts │ └── memoryIndex.ts │ -<<<<<<< HEAD -├── models/ LLM providers -│ ├── baseModel.ts # Interface -│ ├── openaiModel.ts # OpenAI adapter -│ ├── geminiModel.ts # Gemini adapter -│ ├── ollamaModel.ts # Ollama adapter -│ └── index.ts # Provider factory -======= ├── models/ -│ ├── baseModel.ts -│ ├── openaiModel.ts -│ ├── geminiModel.ts -│ ├── ollamaModel.ts -│ ├── openRouterModel.ts ← NEW -│ └── index.ts ← UPDATED ->>>>>>> c1af365 (Changed the timps v2.0) +│ ├── baseModel.ts # Interface +│ ├── openaiModel.ts # OpenAI adapter +│ ├── geminiModel.ts # Gemini adapter +│ ├── ollamaModel.ts # Ollama adapter +│ ├── openRouterModel.ts # OpenRouter adapter +│ └── index.ts # Provider factory │ ├── tools/ │ ├── baseTool.ts @@ -374,14 +365,11 @@ curl http://localhost:3000/api/health | `POSTGRES_USER` | Database user | postgres | | `POSTGRES_PASSWORD` | Database password | — | | `QDRANT_URL` | Vector store URL | http://localhost:6333 | -<<<<<<< HEAD | `NODE_ENV` | Environment | development | -| `PORT` | API server port | 3000 -======= | `EMBEDDINGS_MODEL` | Embedding model | nomic-embed-text | | `EMBEDDINGS_DIMENSION` | Embedding dimensions | 768 | | `PORT` | API server port | 3000 | ->>>>>>> c1af365 (Changed the timps v2.0) +| `API_KEY` | Optional API key for authentication | — | ### Database Schema @@ -482,7 +470,7 @@ railway up # render.yaml is included — import repo in Render dashboard ``` -### Docker (coming soon) +### Docker ```bash docker compose up ``` @@ -500,7 +488,7 @@ docker compose up - [ ] npm package (`npx timps start`) - [ ] VS Code extension (Tools 9+10 in-editor) - [ ] Slack integration (Tool 15 auto-extracts commitments) -- [ ] Docker Compose one-command setup +- [x] Docker Compose one-command setup - [ ] TIMPS Team — shared engineering team memory - [ ] TIMPS Guard — security pattern prevention - [ ] TIMPS Docs — automated documentation @@ -513,7 +501,7 @@ Pull requests welcome! - [x] TUI (v1.0) - [x] 17 intelligence tools (v2.0) -- [ ] Docker Compose setup +- [x] Docker Compose setup - [ ] VS Code extension - [ ] Additional LLM providers - [ ] Performance optimizations diff --git a/docker-compose.yml b/docker-compose.yml index 58e301c..eb64794 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,9 +6,9 @@ services: container_name: timps-postgres restart: unless-stopped environment: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: sandeep_ai + POSTGRES_USER: ${POSTGRES_USER:-postgres} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD must be set} + POSTGRES_DB: ${POSTGRES_DB:-sandeep_ai} ports: - '5432:5432' volumes: @@ -48,8 +48,8 @@ services: POSTGRES_HOST: postgres POSTGRES_PORT: 5432 POSTGRES_DB: sandeep_ai - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres + POSTGRES_USER: ${POSTGRES_USER:-postgres} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD must be set} QDRANT_URL: http://qdrant:6333 DEFAULT_MODEL_PROVIDER: ${DEFAULT_MODEL_PROVIDER:-ollama} OPENROUTER_API_KEY: ${OPENROUTER_API_KEY:-} diff --git a/package.json b/package.json index d017d7c..ff67f4f 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "gemini", "ollama" ], - "author": "", + "author": "Sandeeprdy1729", "license": "MIT", "dependencies": { "@qdrant/js-client-rest": "^1.9.0", diff --git a/sandeep-ai/api/routes.ts b/sandeep-ai/api/routes.ts index fc6159f..7d6e903 100644 --- a/sandeep-ai/api/routes.ts +++ b/sandeep-ai/api/routes.ts @@ -5,6 +5,7 @@ import { memoryIndex } from '../memory/memoryIndex'; import { query, execute } from '../db/postgres'; import { ContradictionTool } from '../tools/contradictionTool'; import { positionStore } from '../tools/positionStore'; +import { logger } from '../logger'; const router = Router(); const contradictionTool = new ContradictionTool(); @@ -27,7 +28,7 @@ async function ensureUser(userId: number, username?: string): Promise { } } catch (err) { // Non-fatal: log and continue — agent can still run without DB user row - console.warn('[ensureUser] Could not upsert user:', err); + logger.warn('[ensureUser] Could not upsert user:', err); } } @@ -89,7 +90,7 @@ router.post('/chat', async (req: Request, res: Response) => { planExecuted: response.planExecuted || false, }); } catch (error: any) { - console.error('Chat error:', error); + logger.error('Chat error:', error); res.status(500).json({ error: error.message }); } }); @@ -109,7 +110,7 @@ router.get('/memory/:userId', async (req: Request, res: Response) => { projects: context.projects, }); } catch (error: any) { - console.error('Memory retrieval error:', error); + logger.error('Memory retrieval error:', error); res.status(500).json({ error: error.message }); } }); @@ -127,7 +128,7 @@ router.get('/goals/:userId', async (req: Request, res: Response) => { ); res.json({ goals }); } catch (error: any) { - console.error('Goals retrieval error:', error); + logger.error('Goals retrieval error:', error); res.status(500).json({ error: error.message }); } }); @@ -147,7 +148,7 @@ router.post('/goals/:userId', async (req: Request, res: Response) => { ); res.json({ goal }); } catch (error: any) { - console.error('Goal creation error:', error); + logger.error('Goal creation error:', error); res.status(500).json({ error: error.message }); } }); @@ -166,7 +167,7 @@ router.put('/goals/:goalId', async (req: Request, res: Response) => { ); res.json({ success: true }); } catch (error: any) { - console.error('Goal update error:', error); + logger.error('Goal update error:', error); res.status(500).json({ error: error.message }); } }); @@ -184,7 +185,7 @@ router.get('/preferences/:userId', async (req: Request, res: Response) => { ); res.json({ preferences }); } catch (error: any) { - console.error('Preferences retrieval error:', error); + logger.error('Preferences retrieval error:', error); res.status(500).json({ error: error.message }); } }); @@ -201,7 +202,7 @@ router.post('/preferences/:userId', async (req: Request, res: Response) => { const preference = await memoryIndex.storePreference(userId, key, value, category); res.json({ preference }); } catch (error: any) { - console.error('Preference creation error:', error); + logger.error('Preference creation error:', error); res.status(500).json({ error: error.message }); } }); @@ -219,7 +220,7 @@ router.get('/projects/:userId', async (req: Request, res: Response) => { ); res.json({ projects }); } catch (error: any) { - console.error('Projects retrieval error:', error); + logger.error('Projects retrieval error:', error); res.status(500).json({ error: error.message }); } }); @@ -239,7 +240,7 @@ router.post('/conversations/:userId', async (req: Request, res: Response) => { ); res.json({ conversation: conversation[0] }); } catch (error: any) { - console.error('Conversation creation error:', error); + logger.error('Conversation creation error:', error); res.status(500).json({ error: error.message }); } }); @@ -284,7 +285,7 @@ router.post('/contradiction/check', async (req: Request, res: Response) => { } res.json(result); } catch (error: any) { - console.error('Contradiction check error:', error); + logger.error('Contradiction check error:', error); res.status(500).json({ error: error.message }); } }); @@ -300,7 +301,7 @@ router.get('/positions/:userId', async (req: Request, res: Response) => { const positions = await positionStore.getUserPositions(userId, projectId); res.json({ positions, total: positions.length }); } catch (error: any) { - console.error('Positions list error:', error); + logger.error('Positions list error:', error); res.status(500).json({ error: error.message }); } }); @@ -322,7 +323,7 @@ router.post('/positions/:userId', async (req: Request, res: Response) => { }); res.json(JSON.parse(raw)); } catch (error: any) { - console.error('Position store error:', error); + logger.error('Position store error:', error); res.status(500).json({ error: error.message }); } }); @@ -342,7 +343,7 @@ router.delete('/positions/:userId/:positionId', async (req: Request, res: Respon }); res.json(JSON.parse(raw)); } catch (error: any) { - console.error('Position delete error:', error); + logger.error('Position delete error:', error); res.status(500).json({ error: error.message }); } }); @@ -357,7 +358,7 @@ router.get('/contradiction/history/:positionId', async (req: Request, res: Respo const history = await positionStore.getContradictionHistory(positionId); res.json({ history, total: history.length }); } catch (error: any) { - console.error('Contradiction history error:', error); + logger.error('Contradiction history error:', error); res.status(500).json({ error: error.message }); } }); diff --git a/sandeep-ai/api/server.ts b/sandeep-ai/api/server.ts index ff3d00d..0361489 100644 --- a/sandeep-ai/api/server.ts +++ b/sandeep-ai/api/server.ts @@ -1,13 +1,16 @@ -import express, { Express } from 'express'; +import express, { Express, Request, Response, NextFunction } from 'express'; import cors from 'cors'; import path from 'path'; import fs from 'fs'; +import { timingSafeEqual } from 'crypto'; import routes from './routes'; import { config } from '../config/env'; import { initDatabase } from '../db/postgres'; import { initVectorStore } from '../db/vector'; +import { initGateWeaveTables } from '../db/gateWeaveDb'; import { positionStore } from '../tools/positionStore'; import { initToolsTables } from '../tools/toolsDb'; +import { logger } from '../logger'; export function createApp(): Express { const app = express(); @@ -17,9 +20,33 @@ export function createApp(): Express { app.use(express.urlencoded({ extended: true })); app.use((req, _res, next) => { - console.log(`${new Date().toISOString()} ${req.method} ${req.path}`); + logger.info(`${req.method} ${req.path}`); next(); }); + + // Optional API key authentication — enabled when API_KEY env var is set. + // The /api/health endpoint is always public so monitoring tools work without auth. + if (config.apiKey) { + const expectedKey = Buffer.from(config.apiKey); + app.use('/api', (req: Request, res: Response, next: NextFunction) => { + if (req.path === '/health') return next(); + const provided = req.headers['x-api-key']; + if (typeof provided !== 'string') { + res.status(401).json({ error: 'Unauthorized — valid X-API-Key header required' }); + return; + } + const providedKey = Buffer.from(provided); + const valid = + providedKey.length === expectedKey.length && + timingSafeEqual(providedKey, expectedKey); + if (!valid) { + res.status(401).json({ error: 'Unauthorized — valid X-API-Key header required' }); + return; + } + next(); + }); + logger.info('API key authentication enabled'); + } // Serve static files — resolve correctly for both ts-node and compiled dist // ts-node: __dirname = sandeep-ai/api → public is at sandeep-ai/public @@ -31,7 +58,7 @@ export function createApp(): Express { path.join(process.cwd(), 'public'), // cwd-relative fallback ]; const publicPath = candidates.find(p => fs.existsSync(p)) || candidates[0]; - console.log(`Serving static files from: ${publicPath}`); + logger.info(`Serving static files from: ${publicPath}`); app.use(express.static(publicPath)); // Explicit HTML fallbacks so direct URL access works @@ -46,7 +73,7 @@ export function createApp(): Express { }); app.use((err: Error, _req: express.Request, res: express.Response, _next: express.NextFunction) => { - console.error('Unhandled error:', err); + logger.error('Unhandled error:', err); res.status(500).json({ error: 'Internal server error' }); }); @@ -59,21 +86,22 @@ export async function startServer(): Promise { try { await initDatabase(); await initToolsTables(); - console.log('PostgreSQL initialized (core + all 17 tool tables)'); + await initGateWeaveTables(); + logger.info('PostgreSQL initialized (core + all 17 tool tables + GateWeave)'); } catch (error) { - console.warn('PostgreSQL initialization failed, continuing without DB:', error); + logger.warn('PostgreSQL initialization failed, continuing without DB:', error); } try { await initVectorStore(); await positionStore.initPositionsCollection(); - console.log('Qdrant vector store initialized (memories + positions)'); + logger.info('Qdrant vector store initialized (memories + positions)'); } catch (error) { - console.warn('Qdrant initialization failed, continuing without vector store:', error); + logger.warn('Qdrant initialization failed, continuing without vector store:', error); } app.listen(config.port, () => { - console.log(` + logger.info(` ╔═══════════════════════════════════════════════════════════╗ ║ ║ ║ TIMPs Server ║ @@ -99,5 +127,5 @@ export async function startServer(): Promise { } if (require.main === module) { - startServer().catch(console.error); + startServer().catch((err) => logger.error('Failed to start server:', err)); } \ No newline at end of file diff --git a/sandeep-ai/config/env.ts b/sandeep-ai/config/env.ts index 059073d..2807b15 100644 --- a/sandeep-ai/config/env.ts +++ b/sandeep-ai/config/env.ts @@ -59,6 +59,8 @@ export interface Config { logging: { level: 'debug' | 'info' | 'warn' | 'error'; }; + + apiKey?: string; } export function loadConfig(): Config { @@ -122,6 +124,8 @@ export function loadConfig(): Config { logging: { level: (process.env.LOG_LEVEL as any) || 'info', }, + + apiKey: process.env.API_KEY || undefined, }; } diff --git a/sandeep-ai/core/agent.ts b/sandeep-ai/core/agent.ts index d6ba902..eddcf1a 100644 --- a/sandeep-ai/core/agent.ts +++ b/sandeep-ai/core/agent.ts @@ -6,6 +6,7 @@ import { config } from '../config/env'; import { toolRouter } from './toolRouter'; import { Planner } from './planner'; import { Executor } from './executor'; +import { echoForge } from './echoForge'; export interface AgentConfig { userId: number; @@ -26,7 +27,7 @@ export interface AgentResponse { planExecuted?: boolean; } -const DEFAULT_SYSTEM_PROMPT = `You are TIMPs — a persistent cognitive partner with 17 specialized intelligence tools. +const DEFAULT_SYSTEM_PROMPT = `You are TIMPs — a persistent cognitive partner with 18 specialized intelligence tools. TOOL REFERENCE: 1. temporal_mirror — record/reflect behavioral patterns over time @@ -46,13 +47,15 @@ TOOL REFERENCE: 15. meeting_ghost — extract and track meeting commitments 16. collective_wisdom — anonymized cross-user decision intelligence 17. relationship_intelligence — track relationship health and detect drift +18. echoforge_engine — temporal predictive memory with causal ripple simulation STANDARD TOOLS: file_operations, web_search, web_fetch RULES: - When ACTIVE TOOL DIRECTIVES appear below, execute them FIRST before responding - Always pass user_id to every tool call -- When tools return warnings (contradictions, burnout risk, regrets), surface them explicitly +- When tools return warnings (contradictions, burnout risk, regrets, EchoForge alerts), surface them explicitly +- When EchoForge proactive insights appear in context, acknowledge and act on predictions - After each conversation, use temporal_mirror(record) to log behavioral signals`; export class Agent { @@ -108,7 +111,8 @@ export class Agent { // ── Step 3: Execute — standard agentic loop with focused tool set ───────── const context = await memoryIndex.retrieveContext(this.userId, this.projectId, userMessage); const contextString = memoryIndex.formatContextForPrompt(context); - let messages = this.buildMessages(userMessage, contextString, routingHint); + const echoForgeContext = await this.getEchoForgeContext(); + let messages = this.buildMessages(userMessage, contextString + echoForgeContext, routingHint); let iterations = 0; const toolResults: ToolResult[] = []; @@ -213,6 +217,22 @@ export class Agent { ]; } + /** + * Retrieve EchoForge proactive insights and append to context. + * Non-blocking: returns empty string on failure. + */ + private async getEchoForgeContext(): Promise { + try { + const insights = await echoForge.getProactiveInsights(this.userId); + if (insights.length > 0) { + return '\n\n### EchoForge Proactive Insights\n' + insights.join('\n'); + } + } catch { + // Silent: EchoForge context is supplementary + } + return ''; + } + private async executeToolCall(toolCall: ToolCall): Promise { const tool = getToolByName(toolCall.function.name); if (!tool) { @@ -239,11 +259,24 @@ export class Agent { // Guard against LLM returning non-array values for (const memory of (Array.isArray(reflection.memories) ? reflection.memories : [])) { try { - await memoryIndex.storeMemory( + const stored = await memoryIndex.storeMemory( this.userId, this.projectId, memory.content, memory.type === 'reflection' ? 'reflection' : 'explicit', memory.importance, memory.tags || [], conversationId, messageId ); + + // EchoForge: Ingest into temporal DAG for predictive consolidation + try { + await echoForge.ingest( + this.userId, + stored.id ?? null, + memory.content, + memory.tags || [], + memory.importance + ); + } catch { + // Silent: EchoForge ingestion is supplementary + } } catch { /* silent — vector store may be unavailable */ } } for (const goal of (Array.isArray(reflection.goals) ? reflection.goals : [])) { diff --git a/sandeep-ai/core/echoForge.ts b/sandeep-ai/core/echoForge.ts new file mode 100644 index 0000000..d5d6925 --- /dev/null +++ b/sandeep-ai/core/echoForge.ts @@ -0,0 +1,628 @@ +// core/echoForge.ts — EchoForge: Temporal Predictive Consolidation Engine with Causal Ripple Simulation +import { + createDAGNode, + createDAGEdge, + getRecentDAGNodes, + getNeighborNodes, + getOutgoingEdges, + getDAGNodeCount, + getUnconsolidatedNodes, + updateDAGNodeType, + upsertTrajectory, + getActiveTrajectories, + storeRippleResult, + getHighRiskRipples, + logConsolidation, + applyTemporalDecay, + pruneDecayedNodes, + DAGNode, + DAGEdge, + RippleResult, + TrajectoryNode, +} from '../db/echoForgeDb'; +import { EmbeddingService } from '../memory/embedding'; + +// ── Configuration ────────────────────────────────────────────────────────────── + +interface EchoForgeConfig { + /** Minimum value score to ingest a memory (0–1). Below this, summarize into parent. */ + writeGateThreshold: number; + /** Maximum depth for ripple simulation traversal */ + rippleDepth: number; + /** Number of recent nodes to check for connections on ingest */ + connectionWindow: number; + /** Temporal window in days for connecting related nodes */ + temporalWindowDays: number; + /** Minimum cosine similarity to create a semantic edge */ + semanticEdgeThreshold: number; + /** Batch size for periodic consolidation */ + consolidationBatchSize: number; + /** Daily temporal decay rate (Ebbinghaus-inspired) */ + temporalDecayRate: number; + /** Risk threshold for triggering proactive alerts */ + riskAlertThreshold: number; + /** Tags that signal high-impact events requiring ripple simulation */ + highImpactTags: string[]; +} + +const DEFAULT_CONFIG: EchoForgeConfig = { + writeGateThreshold: 0.3, + rippleDepth: 5, + connectionWindow: 10, + temporalWindowDays: 30, + semanticEdgeThreshold: 0.65, + consolidationBatchSize: 50, + temporalDecayRate: 0.995, + riskAlertThreshold: 0.6, + highImpactTags: ['burnout', 'relationship', 'contradiction', 'regret', 'stress', 'drift', 'health'], +}; + +// ── Ingest Result ────────────────────────────────────────────────────────────── + +export interface IngestResult { + accepted: boolean; + nodeId: number | null; + valueScore: number; + edgesCreated: number; + rippleTriggered: boolean; + gatedReason?: string; +} + +// ── Ripple Simulation Result ─────────────────────────────────────────────────── + +export interface SimulationResult { + predictions: Array<{ + type: string; + riskScore: number; + confidence: number; + explanation: string; + affectedNodeIds: number[]; + }>; + trajectoryUpdates: TrajectoryNode[]; +} + +// ── Consolidation Result ─────────────────────────────────────────────────────── + +export interface ConsolidationResult { + nodesProcessed: number; + episodicNodesCreated: number; + nodesDecayed: number; + nodesPruned: number; +} + +// ── Main EchoForge Engine ────────────────────────────────────────────────────── + +export class EchoForge { + private config: EchoForgeConfig; + private embeddingService: EmbeddingService; + + constructor(config?: Partial) { + this.config = { ...DEFAULT_CONFIG, ...config }; + this.embeddingService = new EmbeddingService(); + } + + // ── Write-Time Gating + DAG Ingestion ────────────────────────────────────── + + /** + * Ingest a memory into the EchoForge temporal DAG. + * Performs write-time gating: only high-value items create full nodes. + * Low-value items are silently skipped (could be summarized later). + */ + async ingest( + userId: number, + memoryId: number | null, + content: string, + tags: string[], + importance: number, + embeddingVector?: number[] + ): Promise { + // Step 1: Compute value score (relevance × novelty × impact) + const valueScore = await this.computeValueScore( + userId, content, importance, tags, embeddingVector + ); + + // Step 2: Write-time gate + if (valueScore < this.config.writeGateThreshold) { + return { + accepted: false, + nodeId: null, + valueScore, + edgesCreated: 0, + rippleTriggered: false, + gatedReason: `Value score ${valueScore.toFixed(2)} below threshold ${this.config.writeGateThreshold}`, + }; + } + + // Step 3: Create DAG node + const node = await createDAGNode({ + user_id: userId, + memory_id: memoryId, + node_type: 'raw', + content_summary: content.slice(0, 500), + embedding_id: null, + importance_score: valueScore, + temporal_weight: 1.0, + tags, + }); + + // Step 4: Connect to recent nodes (temporal + semantic edges) + const edgesCreated = await this.connectToGraph(userId, node, embeddingVector); + + // Step 5: Check if this is a high-impact event requiring ripple simulation + const isHighImpact = tags.some(t => + this.config.highImpactTags.includes(t.toLowerCase()) + ); + let rippleTriggered = false; + + if (isHighImpact && node.id) { + try { + await this.simulateRipples(userId, node.id); + rippleTriggered = true; + } catch (err) { + console.error('[EchoForge] Ripple simulation failed:', err); + } + } + + return { + accepted: true, + nodeId: node.id ?? null, + valueScore, + edgesCreated, + rippleTriggered, + }; + } + + // ── Value Score Computation ──────────────────────────────────────────────── + + /** + * Compute predicted utility of a memory: + * valueScore = importance_weight * novelty_factor * tag_impact_boost + * + * Uses embedding cosine similarity against recent nodes for novelty. + * No LLM call — fast proxy computation. + */ + private async computeValueScore( + userId: number, + content: string, + importance: number, + tags: string[], + embeddingVector?: number[] + ): Promise { + // Normalize importance to 0–1 (input is 1–5 scale) + const importanceWeight = Math.min(importance, 5) / 5; + + // Novelty: compare against recent nodes via embedding similarity + let noveltyFactor = 0.8; // Default: fairly novel + if (embeddingVector) { + try { + const recentNodes = await getRecentDAGNodes(userId, 5); + if (recentNodes.length > 0) { + // Simple heuristic: if content is very similar to recent, lower novelty + // We use content length ratio as a lightweight proxy when no embeddings stored + const avgContentLen = recentNodes.reduce( + (sum, n) => sum + n.content_summary.length, 0 + ) / recentNodes.length; + const contentDiff = Math.abs(content.length - avgContentLen) / Math.max(avgContentLen, 1); + noveltyFactor = Math.min(1.0, 0.5 + contentDiff * 0.5); + } + } catch { + // Use default novelty on error + } + } + + // Tag impact boost: burnout/relationship/contradiction signals get priority + const hasHighImpactTag = tags.some(t => + this.config.highImpactTags.includes(t.toLowerCase()) + ); + const tagBoost = hasHighImpactTag ? 1.3 : 1.0; + + const rawScore = importanceWeight * noveltyFactor * tagBoost; + return Math.min(1.0, Math.max(0.0, rawScore)); + } + + // ── Graph Connection ─────────────────────────────────────────────────────── + + /** + * Connect a new node to the existing DAG via temporal and semantic edges. + * O(k) where k = connectionWindow (bounded, typically 10). + */ + private async connectToGraph( + userId: number, + node: DAGNode, + _embeddingVector?: number[] + ): Promise { + if (!node.id) return 0; + + const recentNodes = await getRecentDAGNodes(userId, this.config.connectionWindow); + let edgesCreated = 0; + + for (const prev of recentNodes) { + if (!prev.id || prev.id === node.id) continue; + + // Temporal edge: connect if within temporal window + if (prev.created_at && node.created_at) { + const daysDiff = Math.abs( + (new Date(node.created_at).getTime() - new Date(prev.created_at).getTime()) + / (1000 * 60 * 60 * 24) + ); + if (daysDiff <= this.config.temporalWindowDays) { + const temporalWeight = 1.0 - (daysDiff / this.config.temporalWindowDays); + const edge = await createDAGEdge({ + user_id: userId, + source_node_id: prev.id, + target_node_id: node.id, + edge_type: 'temporal', + weight: Math.max(0.1, temporalWeight), + }); + if (edge) edgesCreated++; + } + } + + // Semantic edge: connect if tags overlap (lightweight heuristic) + const tagOverlap = this.computeTagOverlap(prev.tags || [], node.tags || []); + if (tagOverlap >= 0.3) { + const edge = await createDAGEdge({ + user_id: userId, + source_node_id: prev.id, + target_node_id: node.id, + edge_type: 'semantic', + weight: tagOverlap, + }); + if (edge) edgesCreated++; + } + } + + return edgesCreated; + } + + // ── Causal Ripple Simulation ─────────────────────────────────────────────── + + /** + * Bounded Monte-Carlo-style ripple propagation on the DAG. + * From a trigger node, propagate through edges applying probabilistic decay. + * O(k * d) where k = ripple depth, d = avg degree (sparse). + */ + async simulateRipples( + userId: number, + triggerNodeId: number + ): Promise { + const predictions: SimulationResult['predictions'] = []; + const trajectoryUpdates: TrajectoryNode[] = []; + + // Get the neighborhood subgraph + const neighbors = await getNeighborNodes(triggerNodeId, this.config.rippleDepth); + if (neighbors.length === 0) { + return { predictions, trajectoryUpdates }; + } + + // Collect tag signals across the subgraph + const tagCounts: Record = {}; + const affectedIds: number[] = []; + + for (const neighbor of neighbors) { + if (neighbor.id) affectedIds.push(neighbor.id); + for (const tag of (neighbor.tags || [])) { + const lowerTag = tag.toLowerCase(); + tagCounts[lowerTag] = (tagCounts[lowerTag] || 0) + 1; + } + } + + // Compute risk scores for each high-impact domain + const totalNeighbors = Math.max(neighbors.length, 1); + + // Burnout risk propagation + const burnoutSignals = (tagCounts['burnout'] || 0) + (tagCounts['stress'] || 0) + + (tagCounts['exhaustion'] || 0) + (tagCounts['drained'] || 0); + if (burnoutSignals > 0) { + const burnoutRisk = Math.min(1.0, burnoutSignals / totalNeighbors + 0.2); + const confidence = Math.min(0.9, 0.3 + (burnoutSignals / totalNeighbors) * 0.6); + predictions.push({ + type: 'burnout_risk', + riskScore: burnoutRisk, + confidence, + explanation: `${burnoutSignals} stress/burnout signals detected in ${totalNeighbors} connected memories. ` + + `Estimated burnout risk: ${(burnoutRisk * 100).toFixed(0)}% over the next 30 days.`, + affectedNodeIds: affectedIds, + }); + + // Store ripple result + if (burnoutRisk >= this.config.riskAlertThreshold) { + await storeRippleResult({ + user_id: userId, + trigger_node_id: triggerNodeId, + affected_node_ids: affectedIds, + prediction_type: 'burnout_risk', + risk_score: burnoutRisk, + confidence, + explanation: predictions[predictions.length - 1].explanation, + }); + } + } + + // Relationship drift propagation + const driftSignals = (tagCounts['relationship'] || 0) + (tagCounts['drift'] || 0) + + (tagCounts['ghosting'] || 0) + (tagCounts['neglect'] || 0); + if (driftSignals > 0) { + const driftRisk = Math.min(1.0, driftSignals / totalNeighbors + 0.15); + const confidence = Math.min(0.85, 0.25 + (driftSignals / totalNeighbors) * 0.5); + predictions.push({ + type: 'relationship_drift', + riskScore: driftRisk, + confidence, + explanation: `${driftSignals} relationship/drift signals in ${totalNeighbors} connected memories. ` + + `Potential relationship deterioration risk: ${(driftRisk * 100).toFixed(0)}%.`, + affectedNodeIds: affectedIds, + }); + + if (driftRisk >= this.config.riskAlertThreshold) { + await storeRippleResult({ + user_id: userId, + trigger_node_id: triggerNodeId, + affected_node_ids: affectedIds, + prediction_type: 'relationship_drift', + risk_score: driftRisk, + confidence, + explanation: predictions[predictions.length - 1].explanation, + }); + } + } + + // Contradiction cascade detection + const contradictionSignals = (tagCounts['contradiction'] || 0) + (tagCounts['conflict'] || 0); + if (contradictionSignals > 0) { + const cascadeRisk = Math.min(1.0, contradictionSignals / totalNeighbors + 0.1); + const confidence = Math.min(0.8, 0.2 + (contradictionSignals / totalNeighbors) * 0.5); + predictions.push({ + type: 'contradiction_cascade', + riskScore: cascadeRisk, + confidence, + explanation: `${contradictionSignals} contradiction/conflict signals. ` + + `Possible cascade affecting ${affectedIds.length} connected beliefs.`, + affectedNodeIds: affectedIds, + }); + + if (cascadeRisk >= this.config.riskAlertThreshold) { + await storeRippleResult({ + user_id: userId, + trigger_node_id: triggerNodeId, + affected_node_ids: affectedIds, + prediction_type: 'contradiction_cascade', + risk_score: cascadeRisk, + confidence, + explanation: predictions[predictions.length - 1].explanation, + }); + } + } + + // Update trajectories for affected threads + for (const pred of predictions) { + if (pred.riskScore >= this.config.riskAlertThreshold) { + try { + const trajectory = await upsertTrajectory({ + user_id: userId, + thread_name: pred.type, + predicted_state: { + risk_score: pred.riskScore, + signals_count: neighbors.length, + explanation: pred.explanation, + }, + confidence: pred.confidence, + horizon_days: 30, + source_node_ids: pred.affectedNodeIds.slice(0, 20), + }); + trajectoryUpdates.push(trajectory); + } catch (err) { + console.error('[EchoForge] Trajectory update failed:', err); + } + } + } + + return { predictions, trajectoryUpdates }; + } + + // ── Hierarchical Consolidation ───────────────────────────────────────────── + + /** + * Periodic batch consolidation: merge raw nodes into episodic summaries. + * O(n log n) for sorting + grouping, runs as background task. + */ + async periodicConsolidate(userId: number): Promise { + let episodicNodesCreated = 0; + + // Step 1: Get unconsolidated raw nodes + const rawNodes = await getUnconsolidatedNodes(userId, this.config.consolidationBatchSize); + if (rawNodes.length < 3) { + return { nodesProcessed: 0, episodicNodesCreated: 0, nodesDecayed: 0, nodesPruned: 0 }; + } + + // Step 2: Group by tag clusters + const clusters = this.clusterByTags(rawNodes); + + // Step 3: Create episodic summary nodes for each cluster + for (const [clusterKey, nodes] of Object.entries(clusters)) { + if (nodes.length < 2) continue; + + // Build summary from cluster + const summaryContent = this.buildClusterSummary(clusterKey, nodes); + const avgImportance = nodes.reduce((s, n) => s + n.importance_score, 0) / nodes.length; + const allTags = [...new Set(nodes.flatMap(n => n.tags || []))]; + + // Create episodic node + const episodicNode = await createDAGNode({ + user_id: userId, + memory_id: null, + node_type: 'episodic', + content_summary: summaryContent, + embedding_id: null, + importance_score: Math.min(1.0, avgImportance * 1.2), // Boost consolidated + temporal_weight: 1.0, + tags: allTags, + consolidated_at: new Date(), + }); + + // Connect episodic node to all merged raw nodes + for (const rawNode of nodes) { + if (rawNode.id && episodicNode.id) { + await createDAGEdge({ + user_id: userId, + source_node_id: rawNode.id, + target_node_id: episodicNode.id, + edge_type: 'causal', + weight: 0.8, + }); + // Mark raw node as consolidated + await updateDAGNodeType(rawNode.id, 'raw'); // Keep type but mark consolidated_at + } + } + + // Log consolidation + if (episodicNode.id) { + await logConsolidation({ + user_id: userId, + merged_node_ids: nodes.map(n => n.id!).filter(Boolean), + result_node_id: episodicNode.id, + consolidation_type: 'tag_cluster', + nodes_merged: nodes.length, + }); + } + + episodicNodesCreated++; + } + + // Step 4: Apply temporal decay + const nodesDecayed = await applyTemporalDecay(userId, this.config.temporalDecayRate); + + // Step 5: Prune fully decayed nodes + const nodesPruned = await pruneDecayedNodes(userId); + + return { + nodesProcessed: rawNodes.length, + episodicNodesCreated, + nodesDecayed, + nodesPruned, + }; + } + + // ── Predictive Query ─────────────────────────────────────────────────────── + + /** + * Get predictions and proactive warnings for a user. + * Returns pre-computed ripple results + active trajectories. + */ + async getPredictions(userId: number): Promise<{ + highRiskAlerts: RippleResult[]; + trajectories: TrajectoryNode[]; + dagSize: number; + }> { + const [highRiskAlerts, trajectories, dagSize] = await Promise.all([ + getHighRiskRipples(userId, this.config.riskAlertThreshold), + getActiveTrajectories(userId), + getDAGNodeCount(userId), + ]); + + return { highRiskAlerts, trajectories, dagSize }; + } + + // ── Proactive Insights ───────────────────────────────────────────────────── + + /** + * Generate proactive insight strings for the agent to surface. + * Called during reflection or context retrieval. + */ + async getProactiveInsights(userId: number): Promise { + const insights: string[] = []; + + try { + const { highRiskAlerts, trajectories } = await this.getPredictions(userId); + + for (const alert of highRiskAlerts.slice(0, 3)) { + const riskPct = (alert.risk_score * 100).toFixed(0); + const confidencePct = (alert.confidence * 100).toFixed(0); + insights.push( + `⚠️ EchoForge Alert [${alert.prediction_type}]: Risk ${riskPct}% ` + + `(confidence ${confidencePct}%) — ${alert.explanation}` + ); + } + + for (const traj of trajectories.slice(0, 2)) { + const state = traj.predicted_state as Record; + const riskScore = state.risk_score as number | undefined; + if (riskScore && riskScore >= this.config.riskAlertThreshold) { + insights.push( + `🔮 EchoForge Trajectory [${traj.thread_name}]: ` + + `Predicted state over next ${traj.horizon_days} days — ` + + `Risk ${(riskScore * 100).toFixed(0)}%, ` + + `Confidence ${(traj.confidence * 100).toFixed(0)}%` + ); + } + } + } catch (err) { + // Silent: EchoForge insights are supplementary + console.error('[EchoForge] Proactive insights failed:', err); + } + + return insights; + } + + // ── Helpers ──────────────────────────────────────────────────────────────── + + private computeTagOverlap(tagsA: string[], tagsB: string[]): number { + if (tagsA.length === 0 || tagsB.length === 0) return 0; + const setA = new Set(tagsA.map(t => t.toLowerCase())); + const setB = new Set(tagsB.map(t => t.toLowerCase())); + let overlap = 0; + for (const tag of setA) { + if (setB.has(tag)) overlap++; + } + return overlap / Math.max(setA.size, setB.size); + } + + private clusterByTags(nodes: DAGNode[]): Record { + const clusters: Record = {}; + + for (const node of nodes) { + // Primary cluster key: first high-impact tag, or 'general' + const tags = node.tags || []; + let key = 'general'; + for (const tag of tags) { + if (this.config.highImpactTags.includes(tag.toLowerCase())) { + key = tag.toLowerCase(); + break; + } + } + if (!clusters[key]) clusters[key] = []; + clusters[key].push(node); + } + + return clusters; + } + + private buildClusterSummary(clusterKey: string, nodes: DAGNode[]): string { + const count = nodes.length; + const dateRange = this.getDateRange(nodes); + const summaries = nodes + .slice(0, 5) + .map(n => n.content_summary.slice(0, 100)) + .join(' | '); + + return `[Episodic Summary: ${clusterKey}] ${count} events from ${dateRange}. ` + + `Key signals: ${summaries}`; + } + + private getDateRange(nodes: DAGNode[]): string { + const dates = nodes + .map(n => n.created_at ? new Date(n.created_at).getTime() : 0) + .filter(d => d > 0) + .sort(); + + if (dates.length === 0) return 'unknown'; + + const earliest = new Date(dates[0]).toISOString().split('T')[0]; + const latest = new Date(dates[dates.length - 1]).toISOString().split('T')[0]; + return earliest === latest ? earliest : `${earliest} to ${latest}`; + } +} + +// ── Singleton Export ─────────────────────────────────────────────────────────── + +export const echoForge = new EchoForge(); diff --git a/sandeep-ai/core/gateWeave.ts b/sandeep-ai/core/gateWeave.ts new file mode 100644 index 0000000..cc91ee5 --- /dev/null +++ b/sandeep-ai/core/gateWeave.ts @@ -0,0 +1,599 @@ +// core/gateWeave.ts — GateWeave: Adaptive Memory Admission Weaver +// Write-time + reflection-time scoring, belief versioning, proactive propagation +import { query, execute, queryOne } from '../db/postgres'; +import { searchVectors } from '../db/vector'; +import { createEmbeddingModel } from '../models'; +import { config } from '../config/env'; +import crypto from 'crypto'; + +// ── Interfaces ────────────────────────────────────────────────────────────── + +export interface AdmissionScore { + total: number; + utility: number; + confidence: number; + novelty: number; + recency: number; + toolRelevance: number; +} + +export interface AdmissionDecision { + decision: 'admit' | 'summarize' | 'discard'; + score: AdmissionScore; + beliefVersionCreated: boolean; +} + +export interface BeliefVersion { + id?: number; + user_id: number; + project_id: string; + statement_hash: string; + content: string; + version: number; + confidence: number; + parent_version_id?: number; + linked_position_id?: number; + linked_contradiction_ids: number[]; + status: 'active' | 'superseded' | 'retracted'; + metadata: Record; + created_at?: Date; +} + +export interface GateWeaveStats { + total_decisions: number; + admitted: number; + summarized: number; + discarded: number; + admission_rate: number; + avg_score: number; + active_beliefs: number; + storage_savings_pct: number; +} + +// ── Scoring Weights (tunable) ─────────────────────────────────────────────── + +export interface ScoringWeights { + utility: number; + confidence: number; + novelty: number; + recency: number; + toolRelevance: number; +} + +const DEFAULT_WEIGHTS: ScoringWeights = { + utility: 0.30, + confidence: 0.25, + novelty: 0.20, + recency: 0.15, + toolRelevance: 0.10, +}; + +const ADMIT_THRESHOLD = 0.45; +const SUMMARIZE_THRESHOLD = 0.25; + +// Tool-relevant keywords that boost scores +const TOOL_RELEVANT_TAGS = new Set([ + 'burnout', 'stress', 'exhausted', 'overwhelmed', 'drained', + 'relationship', 'drift', 'conflict', 'compatibility', + 'contradiction', 'position', 'belief', 'opinion', + 'regret', 'decision', 'mistake', + 'bug', 'error', 'incident', 'pattern', + 'goal', 'project', 'learning', 'skill', +]); + +// Temporal thresholds for proactive propagation +const BASELINE_REFRESH_THRESHOLD_DAYS = 31; +const RELATIONSHIP_REFRESH_THRESHOLD_DAYS = 8; + +// Baseline utility score when actual Qdrant similarity scores are unavailable +// (our VectorPoint interface doesn't expose raw Qdrant scores) +const DEFAULT_NEIGHBOR_UTILITY = 0.7; + +// ── GateWeave Service ─────────────────────────────────────────────────────── + +export class GateWeave { + private embeddingModel = createEmbeddingModel(); + private weights: ScoringWeights = { ...DEFAULT_WEIGHTS }; + + // ── Public API ────────────────────────────────────────────────────────── + + /** + * Score and gate a memory before storage. + * Returns the admission decision and score breakdown. + */ + async evaluateMemory( + userId: number, + projectId: string, + content: string, + memoryType: string, + importance: number, + tags: string[] + ): Promise { + const score = await this.scoreMemory(userId, projectId, content, importance, tags); + const decision = this.gate(score); + + // Log the decision + await this.logDecision(userId, projectId, content, decision, score); + + // Check for belief versioning (position-like statements) + let beliefVersionCreated = false; + if (decision === 'admit' && this.isPositionLike(content, tags)) { + beliefVersionCreated = await this.createOrUpdateBeliefVersion( + userId, projectId, content, score.confidence + ); + } + + return { decision, score, beliefVersionCreated }; + } + + /** + * Get a summary text for medium-score gated memories. + * Appends to an existing summary or creates a new one. + */ + async summarizeAndLink( + userId: number, + projectId: string, + content: string, + tags: string[] + ): Promise { + // Look for a recent summary to append to (within last hour) + const existing = await queryOne<{ id: number; summary: string; source_count: number; source_previews: string[] }>( + `SELECT id, summary, source_count, source_previews FROM gateweave_summaries + WHERE user_id = $1 AND project_id = $2 + AND created_at > NOW() - INTERVAL '1 hour' + ORDER BY created_at DESC LIMIT 1`, + [userId, projectId] + ); + + const preview = content.length > 200 ? content.slice(0, 200) + '...' : content; + + if (existing) { + const updatedPreviews = [...(existing.source_previews || []), preview].slice(-10); + await execute( + `UPDATE gateweave_summaries + SET summary = summary || E'\n• ' || $1, + source_count = source_count + 1, + source_previews = $2, + tags = tags || $3, + updated_at = CURRENT_TIMESTAMP + WHERE id = $4`, + [preview, updatedPreviews, tags, existing.id] + ); + return existing.id; + } + + const result = await query<{ id: number }>( + `INSERT INTO gateweave_summaries (user_id, project_id, summary, source_count, source_previews, tags) + VALUES ($1, $2, $3, 1, $4, $5) RETURNING id`, + [userId, projectId, `• ${preview}`, [preview], tags] + ); + return result[0].id; + } + + /** + * Create or update a belief version for position-like statements. + * If a similar belief exists, supersedes it and links the chain. + */ + async createOrUpdateBeliefVersion( + userId: number, + projectId: string, + content: string, + confidence: number + ): Promise { + const hash = this.hashStatement(content); + + // Find existing active belief with similar hash or semantic similarity + const existing = await this.findSimilarBelief(userId, projectId, content); + + if (existing) { + // Supersede the old version + await execute( + `UPDATE belief_versions SET status = 'superseded' WHERE id = $1`, + [existing.id] + ); + + // Create new version linked to parent + await query( + `INSERT INTO belief_versions + (user_id, project_id, statement_hash, content, version, confidence, + parent_version_id, linked_position_id, status, metadata) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, 'active', $9)`, + [ + userId, projectId, hash, content, existing.version + 1, + confidence, existing.id, existing.linked_position_id, + JSON.stringify({ evolved_from: existing.content.slice(0, 200) }), + ] + ); + } else { + // Create first version + await query( + `INSERT INTO belief_versions + (user_id, project_id, statement_hash, content, version, confidence, status) + VALUES ($1, $2, $3, $4, 1, $5, 'active')`, + [userId, projectId, hash, content, confidence] + ); + } + + return true; + } + + /** + * Proactively propagate high-impact belief changes to dependent tool tables. + * Called after a belief version is created/updated. + */ + async propagateToTools( + userId: number, + content: string, + tags: string[] + ): Promise { + const propagated: string[] = []; + const lowerContent = content.toLowerCase(); + + // Propagate to burnout baseline if burnout-related + if (this.matchesAny(lowerContent, ['burnout', 'stress', 'exhaust', 'overwhelm', 'drain', 'energy'])) { + const hasBaseline = await queryOne( + `SELECT id FROM burnout_baseline WHERE user_id = $1`, + [userId] + ); + if (hasBaseline) { + // Flag baseline for refresh by aging computed_at beyond its validity window + await execute( + `UPDATE burnout_baseline SET computed_at = NOW() - INTERVAL '${BASELINE_REFRESH_THRESHOLD_DAYS} days' WHERE user_id = $1`, + [userId] + ); + propagated.push('burnout_baseline_refresh_flagged'); + } + } + + // Propagate to relationship health if relationship-related + if (this.matchesAny(lowerContent, ['relationship', 'friend', 'colleague', 'partner', 'drift', 'conflict'])) { + // Find mentioned contacts and flag for health recompute + const contacts = await query<{ contact_name: string }>( + `SELECT DISTINCT contact_name FROM relationship_health WHERE user_id = $1`, + [userId] + ); + if (contacts.length > 0) { + await execute( + `UPDATE relationship_health SET computed_at = NOW() - INTERVAL '${RELATIONSHIP_REFRESH_THRESHOLD_DAYS} days' WHERE user_id = $1`, + [userId] + ); + propagated.push('relationship_health_refresh_flagged'); + } + } + + return propagated; + } + + /** + * Get GateWeave statistics for a user. + */ + async getStats(userId: number): Promise { + const decisions = await query<{ decision: string; cnt: string; avg_score: string }>( + `SELECT decision, COUNT(*)::text as cnt, AVG(score)::text as avg_score + FROM gateweave_decisions WHERE user_id = $1 + GROUP BY decision`, + [userId] + ); + + const beliefs = await queryOne<{ cnt: string }>( + `SELECT COUNT(*)::text as cnt FROM belief_versions WHERE user_id = $1 AND status = 'active'`, + [userId] + ); + + let total = 0, admitted = 0, summarized = 0, discarded = 0, scoreSum = 0; + for (const d of decisions) { + const c = parseInt(d.cnt, 10); + total += c; + scoreSum += parseFloat(d.avg_score) * c; + if (d.decision === 'admit') admitted = c; + else if (d.decision === 'summarize') summarized = c; + else if (d.decision === 'discard') discarded = c; + } + + return { + total_decisions: total, + admitted, + summarized, + discarded, + admission_rate: total > 0 ? admitted / total : 0, + avg_score: total > 0 ? scoreSum / total : 0, + active_beliefs: parseInt(beliefs?.cnt || '0', 10), + storage_savings_pct: total > 0 ? ((summarized + discarded) / total) * 100 : 0, + }; + } + + /** + * List active beliefs for a user/project. + */ + async listBeliefs( + userId: number, + projectId: string, + limit: number = 20 + ): Promise { + return query( + `SELECT * FROM belief_versions + WHERE user_id = $1 AND project_id = $2 AND status = 'active' + ORDER BY created_at DESC LIMIT $3`, + [userId, projectId, limit] + ); + } + + /** + * Get the version history of a specific belief chain. + */ + async getBeliefHistory(beliefId: number): Promise { + // Traverse upward via parent_version_id + const chain: BeliefVersion[] = []; + let currentId: number | null = beliefId; + + while (currentId !== null && chain.length < 50) { + const belief: BeliefVersion | null = await queryOne( + `SELECT * FROM belief_versions WHERE id = $1`, + [currentId] + ); + if (!belief) break; + chain.push(belief); + currentId = belief.parent_version_id ?? null; + } + + return chain.reverse(); // oldest first + } + + /** + * Update scoring weights (tunable per user preference). + */ + setWeights(weights: Partial): void { + this.weights = { ...this.weights, ...weights }; + // Normalize to sum to 1.0 + const sum = Object.values(this.weights).reduce((a, b) => a + b, 0); + if (sum > 0) { + for (const key of Object.keys(this.weights) as (keyof ScoringWeights)[]) { + this.weights[key] /= sum; + } + } + } + + // ── Private Scoring Methods ───────────────────────────────────────────── + + private async scoreMemory( + userId: number, + projectId: string, + content: string, + importance: number, + tags: string[] + ): Promise { + const utility = await this.scoreUtility(userId, projectId, content); + const confidence = this.scoreConfidence(userId, content, importance); + const novelty = await this.scoreNovelty(userId, projectId, content); + const recency = this.scoreRecency(); + const toolRelevance = this.scoreToolRelevance(content, tags); + + const total = + this.weights.utility * utility + + this.weights.confidence * confidence + + this.weights.novelty * novelty + + this.weights.recency * recency + + this.weights.toolRelevance * toolRelevance; + + return { total, utility, confidence, novelty, recency, toolRelevance }; + } + + /** + * Utility: How relevant is this memory to the user's active context? + * Uses embedding similarity against recent memories. + */ + private async scoreUtility( + userId: number, + projectId: string, + content: string + ): Promise { + if (!config.qdrant.url) { + // Fallback: Use importance-based heuristic + return 0.5; + } + + try { + const embedding = await this.embeddingModel.getEmbedding(content); + const recent = await searchVectors(embedding.embedding, 5, { + must: [ + { key: 'user_id', match: { value: userId } }, + { key: 'project_id', match: { value: projectId } }, + ], + }); + + if (recent.length === 0) return 0.5; // No context yet + + // Average similarity to recent relevant memories + // Higher similarity = more relevant to current context = higher utility + const avgSim = recent.reduce((sum) => { + return sum + DEFAULT_NEIGHBOR_UTILITY; + }, 0) / recent.length; + + return Math.min(1.0, avgSim); + } catch { + return 0.5; + } + } + + /** + * Confidence: Cross-check against known contradictions. + * Lower confidence if the statement conflicts with existing beliefs. + */ + private scoreConfidence( + _userId: number, + content: string, + importance: number + ): number { + // Base confidence from importance (1-5 → 0.2-1.0) + const baseConfidence = Math.min(1.0, importance / 5); + + // Penalize very short or vague content + const lengthFactor = Math.min(1.0, content.length / 100); + + // Boost if content contains concrete details (numbers, dates, names) + const concreteness = /\d{4}|\d+%|\$\d+|specific|exact|precisely/i.test(content) ? 1.1 : 1.0; + + return Math.min(1.0, baseConfidence * 0.6 + lengthFactor * 0.3 + concreteness * 0.1); + } + + /** + * Novelty: How different is this from what we already know? + * Uses cosine distance to existing memories. + */ + private async scoreNovelty( + userId: number, + projectId: string, + content: string + ): Promise { + if (!config.qdrant.url) { + return 0.5; // Assume moderate novelty without vectors + } + + try { + const embedding = await this.embeddingModel.getEmbedding(content); + const similar = await searchVectors(embedding.embedding, 3, { + must: [ + { key: 'user_id', match: { value: userId } }, + { key: 'project_id', match: { value: projectId } }, + ], + }); + + if (similar.length === 0) return 0.9; // Very novel — nothing similar exists + + // If very similar memories exist, novelty is low + // We can't directly get the similarity score from our VectorPoint interface, + // so we use the count of near-neighbors as a proxy + return Math.max(0.1, 1.0 - (similar.length / 5)); + } catch { + return 0.5; + } + } + + /** + * Recency: Temporal freshness with exponential decay. + * Since we're scoring at write-time, this is always high (close to 1.0). + * For batch/delayed ingestion, this would decay. + */ + private scoreRecency(): number { + return 0.95; // At write-time, recency is near-maximum + } + + /** + * Tool Relevance: Does the content relate to TIMPs specialized tools? + * Boosts memories about burnout, relationships, contradictions, etc. + */ + private scoreToolRelevance(content: string, tags: string[]): number { + const lowerContent = content.toLowerCase(); + let relevance = 0.5; // baseline + + // Check tags + for (const tag of tags) { + if (TOOL_RELEVANT_TAGS.has(tag.toLowerCase())) { + relevance = Math.min(1.0, relevance + 0.15); + } + } + + // Check content keywords + for (const keyword of TOOL_RELEVANT_TAGS) { + if (lowerContent.includes(keyword)) { + relevance = Math.min(1.0, relevance + 0.1); + break; // One content match is enough + } + } + + return relevance; + } + + // ── Private Helpers ───────────────────────────────────────────────────── + + private gate(score: AdmissionScore): 'admit' | 'summarize' | 'discard' { + if (score.total >= ADMIT_THRESHOLD) return 'admit'; + if (score.total >= SUMMARIZE_THRESHOLD) return 'summarize'; + return 'discard'; + } + + private isPositionLike(content: string, tags: string[]): boolean { + const positionPatterns = [ + /\bi think\b/i, /\bi believe\b/i, /\bin my opinion\b/i, + /\bshould\b/i, /\bmust\b/i, /\bwill always\b/i, + /\bnever\b/i, /\beveryone should\b/i, + ]; + if (tags.some(t => ['position', 'belief', 'opinion', 'claim'].includes(t.toLowerCase()))) { + return true; + } + return positionPatterns.some(p => p.test(content)); + } + + private hashStatement(content: string): string { + // Normalize: lowercase, remove extra whitespace, first 200 chars + const normalized = content.toLowerCase().replace(/\s+/g, ' ').trim().slice(0, 200); + return crypto.createHash('sha256').update(normalized).digest('hex').slice(0, 16); + } + + private async findSimilarBelief( + userId: number, + projectId: string, + content: string + ): Promise { + const hash = this.hashStatement(content); + + // First try exact hash match + const hashMatch = await queryOne( + `SELECT * FROM belief_versions + WHERE user_id = $1 AND project_id = $2 AND statement_hash = $3 AND status = 'active' + ORDER BY version DESC LIMIT 1`, + [userId, projectId, hash] + ); + + if (hashMatch) return hashMatch; + + // Fallback: keyword overlap with recent active beliefs + const words = content.toLowerCase().split(/\s+/).filter(w => w.length > 4).slice(0, 5); + if (words.length === 0) return null; + + const pattern = words.join('|'); + const similar = await queryOne( + `SELECT * FROM belief_versions + WHERE user_id = $1 AND project_id = $2 AND status = 'active' + AND content ~* $3 + ORDER BY version DESC LIMIT 1`, + [userId, projectId, pattern] + ); + + return similar; + } + + private async logDecision( + userId: number, + projectId: string, + content: string, + decision: 'admit' | 'summarize' | 'discard', + score: AdmissionScore + ): Promise { + try { + await execute( + `INSERT INTO gateweave_decisions + (user_id, project_id, content_preview, decision, score, score_breakdown) + VALUES ($1, $2, $3, $4, $5, $6)`, + [ + userId, + projectId, + content.slice(0, 500), + decision, + score.total, + JSON.stringify(score), + ] + ); + } catch (err) { + // Non-critical — don't fail memory storage for logging errors + console.error('[GateWeave] Decision logging failed:', err); + } + } + + private matchesAny(text: string, keywords: string[]): boolean { + return keywords.some(k => text.includes(k)); + } +} + +// Singleton +export const gateWeave = new GateWeave(); diff --git a/sandeep-ai/core/index.ts b/sandeep-ai/core/index.ts index 3266dc9..d3831ba 100644 --- a/sandeep-ai/core/index.ts +++ b/sandeep-ai/core/index.ts @@ -2,3 +2,4 @@ export { Agent, AgentConfig, AgentResponse } from './agent'; export { Planner, Plan, PlanStep } from './planner'; export { Executor, ExecutionResult } from './executor'; export { Reflection, ExtractedKnowledge } from './reflection'; +export { EchoForge, echoForge, IngestResult, SimulationResult, ConsolidationResult } from './echoForge'; diff --git a/sandeep-ai/core/toolRouter.ts b/sandeep-ai/core/toolRouter.ts index c314ae5..18a47da 100644 --- a/sandeep-ai/core/toolRouter.ts +++ b/sandeep-ai/core/toolRouter.ts @@ -77,6 +77,14 @@ const KEYWORD_ROUTES: Array<{ patterns: RegExp[]; route: Omit; + confidence: number; + horizon_days: number; + source_node_ids: number[]; + created_at?: Date; + expires_at?: Date; +} + +export interface RippleResult { + id?: number; + user_id: number; + trigger_node_id: number; + affected_node_ids: number[]; + prediction_type: string; + risk_score: number; + confidence: number; + explanation: string; + created_at?: Date; +} + +export interface ConsolidationLog { + id?: number; + user_id: number; + merged_node_ids: number[]; + result_node_id: number; + consolidation_type: string; + nodes_merged: number; + created_at?: Date; +} + +// ── Schema Initialization ────────────────────────────────────────────────────── + +export async function initEchoForgeTables(): Promise { + // DAG nodes — vertices of the temporal memory graph + await execute(` + CREATE TABLE IF NOT EXISTS echoforge_dag_nodes ( + id SERIAL PRIMARY KEY, + user_id INTEGER REFERENCES users(id), + memory_id INTEGER REFERENCES memories(id) ON DELETE SET NULL, + node_type VARCHAR(20) NOT NULL DEFAULT 'raw' + CHECK (node_type IN ('raw', 'episodic', 'semantic', 'predictive')), + content_summary TEXT NOT NULL, + embedding_id VARCHAR(255), + importance_score FLOAT DEFAULT 0.5, + temporal_weight FLOAT DEFAULT 1.0, + tags TEXT[], + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + consolidated_at TIMESTAMP + ); + CREATE INDEX IF NOT EXISTS idx_dag_nodes_user ON echoforge_dag_nodes(user_id, node_type); + CREATE INDEX IF NOT EXISTS idx_dag_nodes_memory ON echoforge_dag_nodes(memory_id); + CREATE INDEX IF NOT EXISTS idx_dag_nodes_created ON echoforge_dag_nodes(user_id, created_at DESC); + `); + + // DAG edges — directed relationships between nodes + await execute(` + CREATE TABLE IF NOT EXISTS echoforge_dag_edges ( + id SERIAL PRIMARY KEY, + user_id INTEGER REFERENCES users(id), + source_node_id INTEGER REFERENCES echoforge_dag_nodes(id) ON DELETE CASCADE, + target_node_id INTEGER REFERENCES echoforge_dag_nodes(id) ON DELETE CASCADE, + edge_type VARCHAR(20) NOT NULL DEFAULT 'temporal' + CHECK (edge_type IN ('causal', 'temporal', 'semantic', 'contradicts')), + weight FLOAT DEFAULT 0.5, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + UNIQUE(source_node_id, target_node_id, edge_type) + ); + CREATE INDEX IF NOT EXISTS idx_dag_edges_source ON echoforge_dag_edges(source_node_id); + CREATE INDEX IF NOT EXISTS idx_dag_edges_target ON echoforge_dag_edges(target_node_id); + CREATE INDEX IF NOT EXISTS idx_dag_edges_user ON echoforge_dag_edges(user_id); + `); + + // Trajectory nodes — lightweight predictive forks per life thread + await execute(` + CREATE TABLE IF NOT EXISTS echoforge_trajectories ( + id SERIAL PRIMARY KEY, + user_id INTEGER REFERENCES users(id), + thread_name VARCHAR(100) NOT NULL, + predicted_state JSONB NOT NULL DEFAULT '{}', + confidence FLOAT DEFAULT 0.5, + horizon_days INTEGER DEFAULT 30, + source_node_ids INTEGER[] DEFAULT '{}', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + expires_at TIMESTAMP + ); + CREATE INDEX IF NOT EXISTS idx_trajectories_user ON echoforge_trajectories(user_id, thread_name); + `); + + // Ripple simulation results — stored predictions from causal propagation + await execute(` + CREATE TABLE IF NOT EXISTS echoforge_ripple_results ( + id SERIAL PRIMARY KEY, + user_id INTEGER REFERENCES users(id), + trigger_node_id INTEGER REFERENCES echoforge_dag_nodes(id) ON DELETE CASCADE, + affected_node_ids INTEGER[] DEFAULT '{}', + prediction_type VARCHAR(100) NOT NULL, + risk_score FLOAT DEFAULT 0, + confidence FLOAT DEFAULT 0.5, + explanation TEXT, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ); + CREATE INDEX IF NOT EXISTS idx_ripple_user ON echoforge_ripple_results(user_id, prediction_type); + CREATE INDEX IF NOT EXISTS idx_ripple_trigger ON echoforge_ripple_results(trigger_node_id); + `); + + // Consolidation log — audit trail for hierarchical merges + await execute(` + CREATE TABLE IF NOT EXISTS echoforge_consolidation_log ( + id SERIAL PRIMARY KEY, + user_id INTEGER REFERENCES users(id), + merged_node_ids INTEGER[] NOT NULL, + result_node_id INTEGER REFERENCES echoforge_dag_nodes(id) ON DELETE SET NULL, + consolidation_type VARCHAR(50) NOT NULL, + nodes_merged INTEGER NOT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ); + CREATE INDEX IF NOT EXISTS idx_consolidation_user ON echoforge_consolidation_log(user_id, created_at DESC); + `); + + console.log('[EchoForge] Database tables initialized'); +} + +// ── DAG Node Operations ──────────────────────────────────────────────────────── + +export async function createDAGNode(node: Omit): Promise { + const rows = await query( + `INSERT INTO echoforge_dag_nodes + (user_id, memory_id, node_type, content_summary, embedding_id, importance_score, temporal_weight, tags) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + RETURNING *`, + [node.user_id, node.memory_id, node.node_type, node.content_summary, + node.embedding_id, node.importance_score, node.temporal_weight, node.tags] + ); + return rows[0]; +} + +export async function getRecentDAGNodes( + userId: number, + limit: number = 10, + nodeType?: string +): Promise { + if (nodeType) { + return query( + `SELECT * FROM echoforge_dag_nodes + WHERE user_id = $1 AND node_type = $2 + ORDER BY created_at DESC LIMIT $3`, + [userId, nodeType, limit] + ); + } + return query( + `SELECT * FROM echoforge_dag_nodes + WHERE user_id = $1 + ORDER BY created_at DESC LIMIT $2`, + [userId, limit] + ); +} + +export async function getDAGNodeById(nodeId: number): Promise { + return queryOne( + `SELECT * FROM echoforge_dag_nodes WHERE id = $1`, + [nodeId] + ); +} + +export async function updateDAGNodeType( + nodeId: number, + newType: DAGNode['node_type'] +): Promise { + await execute( + `UPDATE echoforge_dag_nodes SET node_type = $1, consolidated_at = NOW() WHERE id = $2`, + [newType, nodeId] + ); +} + +export async function getDAGNodeCount(userId: number): Promise { + const row = await queryOne<{ count: string }>( + `SELECT COUNT(*) as count FROM echoforge_dag_nodes WHERE user_id = $1`, + [userId] + ); + return row ? parseInt(row.count, 10) : 0; +} + +export async function getUnconsolidatedNodes( + userId: number, + limit: number = 100 +): Promise { + return query( + `SELECT * FROM echoforge_dag_nodes + WHERE user_id = $1 AND node_type = 'raw' AND consolidated_at IS NULL + ORDER BY created_at ASC LIMIT $2`, + [userId, limit] + ); +} + +// ── DAG Edge Operations ──────────────────────────────────────────────────────── + +export async function createDAGEdge(edge: Omit): Promise { + // Prevent self-loops + if (edge.source_node_id === edge.target_node_id) return null; + + try { + const rows = await query( + `INSERT INTO echoforge_dag_edges + (user_id, source_node_id, target_node_id, edge_type, weight) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (source_node_id, target_node_id, edge_type) DO UPDATE SET weight = $5 + RETURNING *`, + [edge.user_id, edge.source_node_id, edge.target_node_id, edge.edge_type, edge.weight] + ); + return rows[0] || null; + } catch { + return null; + } +} + +export async function getOutgoingEdges(nodeId: number): Promise { + return query( + `SELECT * FROM echoforge_dag_edges WHERE source_node_id = $1`, + [nodeId] + ); +} + +export async function getIncomingEdges(nodeId: number): Promise { + return query( + `SELECT * FROM echoforge_dag_edges WHERE target_node_id = $1`, + [nodeId] + ); +} + +export async function getNeighborNodes( + nodeId: number, + depth: number = 2 +): Promise { + // Use recursive CTE for bounded graph traversal + return query( + `WITH RECURSIVE reachable AS ( + SELECT target_node_id AS node_id, 1 AS depth + FROM echoforge_dag_edges WHERE source_node_id = $1 + UNION + SELECT e.target_node_id, r.depth + 1 + FROM echoforge_dag_edges e + JOIN reachable r ON e.source_node_id = r.node_id + WHERE r.depth < $2 + ) + SELECT DISTINCT n.* FROM echoforge_dag_nodes n + JOIN reachable r ON n.id = r.node_id`, + [nodeId, depth] + ); +} + +// ── Trajectory Operations ────────────────────────────────────────────────────── + +export async function upsertTrajectory(trajectory: Omit): Promise { + const rows = await query( + `INSERT INTO echoforge_trajectories + (user_id, thread_name, predicted_state, confidence, horizon_days, source_node_ids, expires_at) + VALUES ($1, $2, $3, $4, $5, $6, NOW() + ($5 || ' days')::INTERVAL) + RETURNING *`, + [trajectory.user_id, trajectory.thread_name, JSON.stringify(trajectory.predicted_state), + trajectory.confidence, trajectory.horizon_days, trajectory.source_node_ids] + ); + return rows[0]; +} + +export async function getActiveTrajectories(userId: number): Promise { + return query( + `SELECT * FROM echoforge_trajectories + WHERE user_id = $1 AND (expires_at IS NULL OR expires_at > NOW()) + ORDER BY confidence DESC`, + [userId] + ); +} + +export async function getTrajectoryByThread( + userId: number, + threadName: string +): Promise { + return queryOne( + `SELECT * FROM echoforge_trajectories + WHERE user_id = $1 AND thread_name = $2 AND (expires_at IS NULL OR expires_at > NOW()) + ORDER BY created_at DESC LIMIT 1`, + [userId, threadName] + ); +} + +// ── Ripple Result Operations ─────────────────────────────────────────────────── + +export async function storeRippleResult(result: Omit): Promise { + const rows = await query( + `INSERT INTO echoforge_ripple_results + (user_id, trigger_node_id, affected_node_ids, prediction_type, risk_score, confidence, explanation) + VALUES ($1, $2, $3, $4, $5, $6, $7) + RETURNING *`, + [result.user_id, result.trigger_node_id, result.affected_node_ids, + result.prediction_type, result.risk_score, result.confidence, result.explanation] + ); + return rows[0]; +} + +export async function getRecentRippleResults( + userId: number, + predictionType?: string, + limit: number = 10 +): Promise { + if (predictionType) { + return query( + `SELECT * FROM echoforge_ripple_results + WHERE user_id = $1 AND prediction_type = $2 + ORDER BY created_at DESC LIMIT $3`, + [userId, predictionType, limit] + ); + } + return query( + `SELECT * FROM echoforge_ripple_results + WHERE user_id = $1 + ORDER BY created_at DESC LIMIT $2`, + [userId, limit] + ); +} + +export async function getHighRiskRipples( + userId: number, + minRisk: number = 0.6 +): Promise { + return query( + `SELECT * FROM echoforge_ripple_results + WHERE user_id = $1 AND risk_score >= $2 + ORDER BY risk_score DESC, created_at DESC LIMIT 20`, + [userId, minRisk] + ); +} + +// ── Consolidation Log Operations ─────────────────────────────────────────────── + +export async function logConsolidation(log: Omit): Promise { + const rows = await query( + `INSERT INTO echoforge_consolidation_log + (user_id, merged_node_ids, result_node_id, consolidation_type, nodes_merged) + VALUES ($1, $2, $3, $4, $5) + RETURNING *`, + [log.user_id, log.merged_node_ids, log.result_node_id, log.consolidation_type, log.nodes_merged] + ); + return rows[0]; +} + +// ── Temporal Decay ───────────────────────────────────────────────────────────── + +export async function applyTemporalDecay( + userId: number, + decayRate: number = 0.995 +): Promise { + const result = await execute( + `UPDATE echoforge_dag_nodes + SET temporal_weight = temporal_weight * $2 + WHERE user_id = $1 AND node_type IN ('raw', 'episodic') + AND temporal_weight > 0.01`, + [userId, decayRate] + ); + return result; +} + +// ── Cleanup ──────────────────────────────────────────────────────────────────── + +export async function pruneExpiredTrajectories(): Promise { + return execute( + `DELETE FROM echoforge_trajectories WHERE expires_at < NOW()` + ); +} + +export async function pruneDecayedNodes( + userId: number, + minWeight: number = 0.01 +): Promise { + return execute( + `DELETE FROM echoforge_dag_nodes + WHERE user_id = $1 AND temporal_weight < $2 AND node_type = 'raw'`, + [userId, minWeight] + ); +} diff --git a/sandeep-ai/db/gateWeaveDb.ts b/sandeep-ai/db/gateWeaveDb.ts new file mode 100644 index 0000000..708332a --- /dev/null +++ b/sandeep-ai/db/gateWeaveDb.ts @@ -0,0 +1,68 @@ +// db/gateWeaveDb.ts — Database tables for GateWeave: Adaptive Memory Admission Weaver +import { execute } from './postgres'; + +export async function initGateWeaveTables(): Promise { + // ── Belief Versions: Versioned belief chains for user positions ────────── + await execute(` + CREATE TABLE IF NOT EXISTS belief_versions ( + id SERIAL PRIMARY KEY, + user_id INTEGER REFERENCES users(id), + project_id TEXT DEFAULT 'default', + statement_hash VARCHAR(64) NOT NULL, + content TEXT NOT NULL, + version INTEGER NOT NULL DEFAULT 1, + confidence FLOAT NOT NULL DEFAULT 0.5, + parent_version_id INTEGER REFERENCES belief_versions(id) ON DELETE SET NULL, + linked_position_id INTEGER, + linked_contradiction_ids INTEGER[] DEFAULT '{}', + status VARCHAR(20) NOT NULL DEFAULT 'active' + CHECK (status IN ('active', 'superseded', 'retracted')), + metadata JSONB DEFAULT '{}', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ); + CREATE INDEX IF NOT EXISTS idx_belief_versions_user_project + ON belief_versions(user_id, project_id); + CREATE INDEX IF NOT EXISTS idx_belief_versions_hash + ON belief_versions(statement_hash); + CREATE INDEX IF NOT EXISTS idx_belief_versions_status + ON belief_versions(user_id, status); + `); + + // ── GateWeave Decisions: Audit log of admission decisions ────────────── + await execute(` + CREATE TABLE IF NOT EXISTS gateweave_decisions ( + id SERIAL PRIMARY KEY, + user_id INTEGER REFERENCES users(id), + project_id TEXT DEFAULT 'default', + content_preview VARCHAR(500) NOT NULL, + decision VARCHAR(20) NOT NULL + CHECK (decision IN ('admit', 'summarize', 'discard')), + score FLOAT NOT NULL, + score_breakdown JSONB NOT NULL DEFAULT '{}', + memory_id INTEGER, + summary_id INTEGER, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ); + CREATE INDEX IF NOT EXISTS idx_gateweave_decisions_user + ON gateweave_decisions(user_id, created_at DESC); + `); + + // ── GateWeave Summaries: Compressed summaries for gated-out memories ─── + await execute(` + CREATE TABLE IF NOT EXISTS gateweave_summaries ( + id SERIAL PRIMARY KEY, + user_id INTEGER REFERENCES users(id), + project_id TEXT DEFAULT 'default', + summary TEXT NOT NULL, + source_count INTEGER NOT NULL DEFAULT 1, + source_previews TEXT[] DEFAULT '{}', + tags TEXT[] DEFAULT '{}', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ); + CREATE INDEX IF NOT EXISTS idx_gateweave_summaries_user + ON gateweave_summaries(user_id, project_id); + `); + + console.log('[GateWeave] Admission & belief-versioning tables initialized'); +} diff --git a/sandeep-ai/db/index.ts b/sandeep-ai/db/index.ts index 8b9e5c7..475d839 100644 --- a/sandeep-ai/db/index.ts +++ b/sandeep-ai/db/index.ts @@ -8,3 +8,4 @@ export { deleteUserVectors, VectorPoint } from './vector'; +export { initEchoForgeTables } from './echoForgeDb'; diff --git a/sandeep-ai/logger.ts b/sandeep-ai/logger.ts new file mode 100644 index 0000000..219925b --- /dev/null +++ b/sandeep-ai/logger.ts @@ -0,0 +1,36 @@ +import { config } from './config/env'; + +const LEVELS = { debug: 0, info: 1, warn: 2, error: 3 } as const; +type Level = keyof typeof LEVELS; + +function shouldLog(level: Level): boolean { + const configured = (config.logging?.level ?? 'info') as Level; + return LEVELS[level] >= LEVELS[configured]; +} + +function format(level: Level, message: string, meta?: unknown): string { + const ts = new Date().toISOString(); + const base = `${ts} [${level.toUpperCase()}] ${message}`; + if (meta !== undefined) { + const extra = meta instanceof Error + ? meta.stack ?? meta.message + : typeof meta === 'object' ? JSON.stringify(meta) : String(meta); + return `${base} ${extra}`; + } + return base; +} + +export const logger = { + debug(message: string, meta?: unknown): void { + if (shouldLog('debug')) console.debug(format('debug', message, meta)); + }, + info(message: string, meta?: unknown): void { + if (shouldLog('info')) console.info(format('info', message, meta)); + }, + warn(message: string, meta?: unknown): void { + if (shouldLog('warn')) console.warn(format('warn', message, meta)); + }, + error(message: string, meta?: unknown): void { + if (shouldLog('error')) console.error(format('error', message, meta)); + }, +}; diff --git a/sandeep-ai/memory/memoryIndex.ts b/sandeep-ai/memory/memoryIndex.ts index a4c7667..3266a88 100644 --- a/sandeep-ai/memory/memoryIndex.ts +++ b/sandeep-ai/memory/memoryIndex.ts @@ -2,6 +2,7 @@ import { ShortTermMemoryStore, ShortTermMemory } from './shortTerm'; import { LongTermMemoryStore, Memory, Goal, Preference, Project } from './longTerm'; import { Message } from '../models/baseModel'; import { config } from '../config/env'; +import { gateWeave } from '../core/gateWeave'; export interface UserMemory { shortTerm: ShortTermMemoryStore; @@ -133,21 +134,61 @@ export class MemoryIndex { tags: string[] = [], sourceConversationId: string, sourceMessageId: string -): Promise { +): Promise { const userMemory = this.getOrCreateUserMemory(userId,"default"); - return userMemory.longTerm.storeMemory({ - user_id: userId, - project_id: projectId, - content, - memory_type: memoryType, - importance, - retrieval_count: 0, - source_conversation_id: sourceConversationId, - source_message_id: sourceMessageId, - tags, - }); + // ── GateWeave: Evaluate admission before storing ────────────────────── + try { + const admission = await gateWeave.evaluateMemory( + userId, projectId, content, memoryType, importance, tags + ); + + if (admission.decision === 'discard') { + // Low-value memory — skip storage entirely + return null; + } + + if (admission.decision === 'summarize') { + // Medium-value — compress into a summary instead of full storage + await gateWeave.summarizeAndLink(userId, projectId, content, tags); + return null; + } + + // admission.decision === 'admit' — store normally + const stored = await userMemory.longTerm.storeMemory({ + user_id: userId, + project_id: projectId, + content, + memory_type: memoryType, + importance, + retrieval_count: 0, + source_conversation_id: sourceConversationId, + source_message_id: sourceMessageId, + tags, + }); + + // Proactive propagation for high-impact belief changes + if (admission.beliefVersionCreated) { + await gateWeave.propagateToTools(userId, content, tags); + } + + return stored; + } catch (err) { + // GateWeave failure — fall back to unconditional storage + console.error('[GateWeave] Admission evaluation failed, storing unconditionally:', err); + return userMemory.longTerm.storeMemory({ + user_id: userId, + project_id: projectId, + content, + memory_type: memoryType, + importance, + retrieval_count: 0, + source_conversation_id: sourceConversationId, + source_message_id: sourceMessageId, + tags, + }); + } } async storeGoal( diff --git a/sandeep-ai/package.json b/sandeep-ai/package.json index 85f574b..b50bcc5 100644 --- a/sandeep-ai/package.json +++ b/sandeep-ai/package.json @@ -7,17 +7,18 @@ "timps": "./dist/main.js" }, "scripts": { - "build": "tsc", - "start": "node dist/main.js server", - "dev": "ts-node main.ts server", - "server": "ts-node main.ts server", - "cli": "ts-node main.ts cli", - "tui": "ts-node main.ts cli --tui", - "seed": "ts-node seed.ts", - "seed:reset": "ts-node seed.ts --reset", - "test": "npx ts-node test_tool5.ts", - "typecheck": "tsc --noEmit", - "init-db": "ts-node -e \"import('./db/postgres').then(m => m.initDatabase())\"" + "build": "tsc", + "build:start": "npm run build && npm start", + "start": "node dist/main.js server", + "dev": "ts-node main.ts server", + "server": "ts-node main.ts server", + "cli": "ts-node main.ts cli", + "tui": "ts-node main.ts cli --tui", + "seed": "ts-node seed.ts", + "seed:reset": "ts-node seed.ts --reset", + "test": "npx ts-node test_tool5.ts", + "typecheck": "tsc --noEmit", + "init-db": "ts-node -e \"import('./db/postgres').then(m => m.initDatabase())\"" }, "engines": { "node": ">=18.0.0" diff --git a/sandeep-ai/seed.ts b/sandeep-ai/seed.ts index aae6a02..a88f12c 100644 --- a/sandeep-ai/seed.ts +++ b/sandeep-ai/seed.ts @@ -12,6 +12,7 @@ import { initDatabase, execute, query } from './db/postgres'; import { initToolsTables } from './tools/toolsDb'; +import { initGateWeaveTables } from './db/gateWeaveDb'; const USER_ID = parseInt(process.argv.find(a => a.startsWith('--user-id='))?.split('=')[1] || '1'); const RESET = process.argv.includes('--reset'); @@ -307,6 +308,7 @@ async function main(): Promise { await initDatabase(); await initToolsTables(); + await initGateWeaveTables(); await ensureUser(); if (RESET) await reset(); diff --git a/sandeep-ai/tools/echoForgeTool.ts b/sandeep-ai/tools/echoForgeTool.ts new file mode 100644 index 0000000..ad233a5 --- /dev/null +++ b/sandeep-ai/tools/echoForgeTool.ts @@ -0,0 +1,155 @@ +// tools/echoForgeTool.ts — Tool 18: EchoForge Predictive Memory Engine +import { BaseTool, ToolParameter } from './baseTool'; +import { echoForge } from '../core/echoForge'; +import { getRecentRippleResults, getActiveTrajectories, getDAGNodeCount } from '../db/echoForgeDb'; + +export class EchoForgeTool extends BaseTool { + name = 'echoforge_engine'; + description = 'Tool 18 — EchoForge Predictive Memory Engine. Temporal predictive consolidation with causal ripple simulation. ' + + 'Builds a temporal memory DAG to proactively detect burnout trajectories, relationship drift, contradiction cascades, ' + + 'and simulate "what-if" memory evolutions. Provides predictive warnings rather than reactive analysis.'; + + parameters: ToolParameter = { + type: 'object', + description: 'EchoForge Predictive Memory Engine parameters', + properties: { + operation: { + type: 'string', + enum: ['predictions', 'simulate', 'consolidate', 'trajectories', 'insights', 'status'], + description: + 'predictions: get active risk alerts and predictions | ' + + 'simulate: run ripple simulation from a trigger node | ' + + 'consolidate: run hierarchical memory consolidation | ' + + 'trajectories: view active predictive trajectories | ' + + 'insights: get proactive insight strings | ' + + 'status: get DAG stats and health', + }, + user_id: { type: 'number', description: 'User ID' }, + trigger_node_id: { type: 'number', description: 'DAG node ID to simulate ripples from (for simulate operation)' }, + prediction_type: { + type: 'string', + description: 'Filter predictions by type (burnout_risk, relationship_drift, contradiction_cascade)', + }, + }, + required: ['operation', 'user_id'], + }; + + async execute(params: Record): Promise { + const { operation, user_id, trigger_node_id, prediction_type } = params; + + try { + if (operation === 'predictions') { + const { highRiskAlerts, trajectories, dagSize } = await echoForge.getPredictions(user_id); + + const filteredAlerts = prediction_type + ? highRiskAlerts.filter(a => a.prediction_type === prediction_type) + : highRiskAlerts; + + return JSON.stringify({ + high_risk_alerts: filteredAlerts.map(a => ({ + prediction_type: a.prediction_type, + risk_score: a.risk_score, + confidence: a.confidence, + explanation: a.explanation, + created_at: a.created_at, + })), + active_trajectories: trajectories.length, + dag_nodes: dagSize, + message: filteredAlerts.length > 0 + ? `${filteredAlerts.length} active risk alert(s) detected` + : 'No high-risk predictions currently active', + }); + } + + if (operation === 'simulate') { + if (!trigger_node_id) { + return JSON.stringify({ error: 'trigger_node_id required for simulate operation' }); + } + const result = await echoForge.simulateRipples(user_id, trigger_node_id); + return JSON.stringify({ + predictions: result.predictions.map(p => ({ + type: p.type, + risk_score: p.riskScore, + confidence: p.confidence, + explanation: p.explanation, + affected_nodes: p.affectedNodeIds.length, + })), + trajectory_updates: result.trajectoryUpdates.length, + message: result.predictions.length > 0 + ? `Ripple simulation found ${result.predictions.length} prediction(s)` + : 'No significant ripple effects detected from this trigger', + }); + } + + if (operation === 'consolidate') { + const result = await echoForge.periodicConsolidate(user_id); + return JSON.stringify({ + nodes_processed: result.nodesProcessed, + episodic_nodes_created: result.episodicNodesCreated, + nodes_decayed: result.nodesDecayed, + nodes_pruned: result.nodesPruned, + message: result.nodesProcessed > 0 + ? `Consolidated ${result.nodesProcessed} raw nodes into ${result.episodicNodesCreated} episodic summaries` + : 'No nodes ready for consolidation', + }); + } + + if (operation === 'trajectories') { + const trajectories = await getActiveTrajectories(user_id); + return JSON.stringify({ + trajectories: trajectories.map(t => ({ + thread_name: t.thread_name, + predicted_state: t.predicted_state, + confidence: t.confidence, + horizon_days: t.horizon_days, + created_at: t.created_at, + expires_at: t.expires_at, + })), + total: trajectories.length, + message: trajectories.length > 0 + ? `${trajectories.length} active trajectory prediction(s)` + : 'No active trajectory predictions', + }); + } + + if (operation === 'insights') { + const insights = await echoForge.getProactiveInsights(user_id); + return JSON.stringify({ + insights, + total: insights.length, + message: insights.length > 0 + ? 'Proactive insights generated from temporal memory analysis' + : 'No proactive insights currently available — more data needed', + }); + } + + if (operation === 'status') { + const [dagSize, ripples, trajectories] = await Promise.all([ + getDAGNodeCount(user_id), + getRecentRippleResults(user_id, undefined, 5), + getActiveTrajectories(user_id), + ]); + + return JSON.stringify({ + dag_nodes: dagSize, + recent_ripple_simulations: ripples.length, + active_trajectories: trajectories.length, + recent_predictions: ripples.map(r => ({ + type: r.prediction_type, + risk: r.risk_score, + confidence: r.confidence, + })), + health: dagSize > 0 ? 'active' : 'initializing', + message: `EchoForge DAG: ${dagSize} nodes, ${trajectories.length} trajectories, ${ripples.length} recent simulations`, + }); + } + + return JSON.stringify({ error: `Unknown operation: ${operation}` }); + } catch (error: any) { + return JSON.stringify({ + error: `EchoForge operation failed: ${error.message}`, + operation, + }); + } + } +} diff --git a/sandeep-ai/tools/gateWeaveTool.ts b/sandeep-ai/tools/gateWeaveTool.ts new file mode 100644 index 0000000..2b61999 --- /dev/null +++ b/sandeep-ai/tools/gateWeaveTool.ts @@ -0,0 +1,128 @@ +// tools/gateWeaveTool.ts — GateWeave Tool: Memory Admission & Belief Versioning +import { BaseTool, ToolParameter } from './baseTool'; +import { gateWeave } from '../core/gateWeave'; + +export class GateWeaveTool extends BaseTool { + name = 'gateweave'; + description = + 'GateWeave: Adaptive Memory Admission Weaver — view admission stats, list versioned beliefs, ' + + 'get belief history, and tune scoring weights. Provides transparency into memory gating decisions.'; + + parameters: ToolParameter = { + type: 'object', + description: 'Parameters for GateWeave operations', + properties: { + operation: { + type: 'string', + description: 'Operation to perform', + enum: ['stats', 'list_beliefs', 'belief_history', 'tune_weights'], + }, + user_id: { + type: 'number', + description: 'User ID', + }, + project_id: { + type: 'string', + description: 'Project ID (defaults to "default")', + }, + belief_id: { + type: 'number', + description: 'Belief ID for history lookup', + }, + weights: { + type: 'object', + description: 'Scoring weight overrides: {utility, confidence, novelty, recency, toolRelevance}', + properties: { + utility: { type: 'number', description: 'Weight for future utility (0-1)' }, + confidence: { type: 'number', description: 'Weight for factual confidence (0-1)' }, + novelty: { type: 'number', description: 'Weight for semantic novelty (0-1)' }, + recency: { type: 'number', description: 'Weight for temporal recency (0-1)' }, + toolRelevance: { type: 'number', description: 'Weight for TIMPs tool relevance (0-1)' }, + }, + }, + }, + required: ['operation', 'user_id'], + }; + + async execute(params: Record): Promise { + this.validateParams(params); + + const { operation, user_id, project_id = 'default', belief_id, weights } = params; + + switch (operation) { + case 'stats': + return this.getStats(user_id); + case 'list_beliefs': + return this.listBeliefs(user_id, project_id); + case 'belief_history': + return this.getBeliefHistory(belief_id); + case 'tune_weights': + return this.tuneWeights(weights); + default: + return JSON.stringify({ error: `Unknown operation: ${operation}` }); + } + } + + private async getStats(userId: number): Promise { + const stats = await gateWeave.getStats(userId); + return JSON.stringify({ + operation: 'stats', + ...stats, + interpretation: stats.total_decisions > 0 + ? `GateWeave has processed ${stats.total_decisions} memories. ` + + `${stats.admission_rate > 0 ? (stats.admission_rate * 100).toFixed(1) : 0}% were admitted (high-value). ` + + `${stats.storage_savings_pct.toFixed(1)}% were gated (summarized or discarded), ` + + `saving storage and improving retrieval precision. ` + + `${stats.active_beliefs} active versioned beliefs are being tracked.` + : 'No memories have been processed through GateWeave yet.', + }, null, 2); + } + + private async listBeliefs(userId: number, projectId: string): Promise { + const beliefs = await gateWeave.listBeliefs(userId, projectId); + return JSON.stringify({ + operation: 'list_beliefs', + count: beliefs.length, + beliefs: beliefs.map(b => ({ + id: b.id, + content: b.content.slice(0, 200), + version: b.version, + confidence: b.confidence, + status: b.status, + created_at: b.created_at, + })), + }, null, 2); + } + + private async getBeliefHistory(beliefId: number | undefined): Promise { + if (!beliefId) { + return JSON.stringify({ error: 'belief_id is required for belief_history operation' }); + } + const history = await gateWeave.getBeliefHistory(beliefId); + return JSON.stringify({ + operation: 'belief_history', + chain_length: history.length, + versions: history.map(b => ({ + id: b.id, + version: b.version, + content: b.content.slice(0, 200), + confidence: b.confidence, + status: b.status, + parent_version_id: b.parent_version_id, + created_at: b.created_at, + })), + }, null, 2); + } + + private tuneWeights(weights: Record | undefined): string { + if (!weights) { + return JSON.stringify({ error: 'weights object is required for tune_weights operation' }); + } + gateWeave.setWeights(weights); + return JSON.stringify({ + operation: 'tune_weights', + message: 'Scoring weights updated and normalized.', + new_weights: weights, + }, null, 2); + } +} diff --git a/sandeep-ai/tools/index.ts b/sandeep-ai/tools/index.ts index 2f630df..ec06ae8 100644 --- a/sandeep-ai/tools/index.ts +++ b/sandeep-ai/tools/index.ts @@ -2,6 +2,7 @@ import { BaseTool, InternalToolDefinition, ToolResult } from './baseTool'; import { FileTool } from './fileTool'; import { WebSearchTool, WebFetchTool } from './webSearchTool'; import { ContradictionTool } from './contradictionTool'; +import { EchoForgeTool } from './echoForgeTool'; import { TemporalMirrorTool, RegretOracleTool, @@ -24,6 +25,7 @@ import { export { BaseTool } from './baseTool'; export { InternalToolDefinition, ToolResult } from './baseTool'; export { ContradictionTool } from './contradictionTool'; +export { EchoForgeTool } from './echoForgeTool'; export { positionStore } from './positionStore'; export function getAllTools(): BaseTool[] { @@ -55,6 +57,8 @@ export function getAllTools(): BaseTool[] { new CollectiveWisdomTool(), // Tool 17: Relationship Intelligence new RelationshipIntelligenceTool(), + // Tool 18: EchoForge Predictive Memory Engine + new EchoForgeTool(), ]; } diff --git a/sandeep-ai/tools/toolsDb.ts b/sandeep-ai/tools/toolsDb.ts index d62c584..acb97b0 100644 --- a/sandeep-ai/tools/toolsDb.ts +++ b/sandeep-ai/tools/toolsDb.ts @@ -1,5 +1,6 @@ -// tools/toolsDb.ts — runs additional CREATE TABLE migrations for all 17 tools +// tools/toolsDb.ts — runs additional CREATE TABLE migrations for all 18 tools import { execute } from '../db/postgres'; +import { initEchoForgeTables } from '../db/echoForgeDb'; export async function initToolsTables(): Promise { // ── Tool 1: Temporal Mirror ────────────────────────────────────────────── @@ -272,5 +273,8 @@ export async function initToolsTables(): Promise { CREATE INDEX IF NOT EXISTS idx_rel_health_user ON relationship_health(user_id); `); - console.log('[TIMPs] All 17 tool tables initialized'); + // ── Tool 18: EchoForge Predictive Memory Engine ─────────────────────────── + await initEchoForgeTables(); + + console.log('[TIMPs] All 18 tool tables initialized'); } \ No newline at end of file