diff --git a/CHANGELOG.md b/CHANGELOG.md index 7718786..713dc4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,24 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] +## [0.1.11] — 2026-04-08 + +### Added + +- **Smart Approval Modals**: Automatically detects `[y/n]` and "Press Enter" prompts in any agent session. A native card slides up from the bottom of the screen with large, tap-friendly Approve and Deny buttons — no keyboard required. +- **Action Timeline**: A new Timeline tab intercepts agent tool-use events (Claude tool boxes, generic shell commands) and renders them as collapsible cards with status indicators, timestamps, and tool type icons. Provides a high-signal activity feed easy to skim on mobile. +- **Heuristics Engine**: A semantic PTY parser (`HeuristicsEngine`) runs a headless xterm instance per session to detect prompt boundaries and tool-use events with high precision. Agent-agnostic — works with Claude Code, Gemini CLI, and any CLI tool using standard terminal conventions. + +### Improved + +- **Serial chunk processing**: Heuristics engine now queues PTY chunks through a serial promise chain, eliminating potential race conditions when high-frequency output arrives faster than xterm can process it. +- **Resource cleanup**: Headless xterm instances are disposed on both WebSocket `close` and `error` events, preventing memory leaks in long-running or dropped sessions. +- **Timeline memory cap**: Action timeline is capped at 100 entries per session to bound memory growth during extended agent runs. + +### Fixed + +- **Prompt self-dismiss**: Approval modal closes immediately on tap without waiting for the next backend event, eliminating UI flicker on slow connections. + ## [0.1.10] — 2026-03-28 ### Fixed diff --git a/README.md b/README.md index 996b470..aad7070 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,9 @@ Start an agent on your laptop, walk away, and check in from your phone or tablet ## Why CloudCode? - **Agent agnostic:** Works with Claude Code, Gemini CLI, OpenAI Codex, GitHub Copilot CLI, or any CLI tool. -- **Task launch:** Start tasks instantly from your mobile dashboard without opening a full terminal. CloudCode defaults to your recent projects and drops you into the live session view. +- **Smart Remote Control:** Detects agent activity and transforms it into a **Task Timeline** of collapsible cards. +- **Instant Approvals:** Automatically pops up native **Approval Modals** when an agent requests permission, saving you from opening the mobile keyboard. +- **Task launch:** Start tasks instantly from your mobile dashboard without opening a full terminal. - **Transcript logs:** Shows the full session output in a scrollable, timestamped transcript view. - **QR code pairing:** Scan a QR code from your terminal to authenticate your phone. No passwords or SSH keys. - **Persistent sessions:** Sessions run inside `tmux`. Your agent keeps working if your laptop sleeps or your connection drops. Reconnect and pick up where you left off. diff --git a/backend/package.json b/backend/package.json index c0fdd42..f536cca 100644 --- a/backend/package.json +++ b/backend/package.json @@ -1,6 +1,6 @@ { "name": "@humans-of-ai/cloudcode", - "version": "0.1.10", + "version": "0.1.11", "description": "CloudCode — Remote control for your local AI coding agents", "license": "MIT", "type": "module", diff --git a/backend/src/terminal/heuristics.test.ts b/backend/src/terminal/heuristics.test.ts new file mode 100644 index 0000000..ccf7b33 --- /dev/null +++ b/backend/src/terminal/heuristics.test.ts @@ -0,0 +1,230 @@ +import { describe, expect, it, beforeEach, afterEach } from 'vitest'; +import { HeuristicsEngine } from './heuristics.js'; + +function encode(text: string): string { + return Buffer.from(text).toString('base64'); +} + +describe('HeuristicsEngine', () => { + let engine: HeuristicsEngine; + + beforeEach(() => { + engine = new HeuristicsEngine(); + }); + + afterEach(() => { + engine.dispose(); + }); + + describe('process', () => { + it('returns empty result for empty chunk', async () => { + const result = await engine.process(''); + expect(result).toEqual({}); + }); + + it('handles null/undefined chunk gracefully', async () => { + // @ts-ignore - testing invalid input + const result = await engine.process(null); + expect(result).toEqual({}); + }); + + it('processes chunk without throwing', async () => { + await expect(engine.process(encode('some terminal output'))).resolves.not.toThrow(); + }); + }); + + describe('prompt detection', () => { + it('detects [Y/n] yesno prompt and extracts context', async () => { + const result = await engine.process(encode('Overwrite file.txt? [Y/n]')); + expect(result.prompt?.isActive).toBe(true); + expect(result.prompt?.type).toBe('yesno'); + expect(result.prompt?.text).toContain('Overwrite file.txt'); + }); + + it('detects (y/N) yesno prompt', async () => { + const result = await engine.process(encode('Do you want to continue? (y/N)')); + expect(result.prompt?.isActive).toBe(true); + expect(result.prompt?.type).toBe('yesno'); + }); + + it('detects "Press Enter to continue" prompt', async () => { + const result = await engine.process(encode('Changes applied.\nPress Enter to continue...')); + expect(result.prompt?.isActive).toBe(true); + expect(result.prompt?.type).toBe('enter'); + }); + + it('detects "Press any key to exit" prompt', async () => { + const result = await engine.process(encode('Press any key to exit')); + expect(result.prompt?.isActive).toBe(true); + expect(result.prompt?.type).toBe('enter'); + }); + + it('does not re-emit unchanged prompt state', async () => { + const result1 = await engine.process(encode('Delete this? [Y/n]')); + expect(result1.prompt?.isActive).toBe(true); + // Empty chunk: state unchanged, should not re-emit + const result2 = await engine.process(encode('')); + expect(result2.prompt).toBeUndefined(); + }); + }); + + describe('action detection — Claude tool-use boxes', () => { + it('detects a Bash tool-use start and returns a running action', async () => { + const toolBox = '┌─ Tool Use: Bash ────────────────────────┐\n│ npm test │\n'; + const result = await engine.process(encode(toolBox)); + expect(result.action).toBeDefined(); + expect(result.action?.type).toBe('bash'); + expect(result.action?.status).toBe('running'); + expect(result.action?.label).toContain('npm test'); + }); + + it('detects a Read tool-use start', async () => { + const toolBox = '┌─ Tool Use: Read ────────────────────────┐\n│ src/index.ts │\n'; + const result = await engine.process(encode(toolBox)); + expect(result.action?.type).toBe('read'); + expect(result.action?.status).toBe('running'); + }); + + it('detects an Edit tool-use start', async () => { + const toolBox = '┌─ Tool Use: Edit ────────────────────────┐\n│ package.json │\n'; + const result = await engine.process(encode(toolBox)); + expect(result.action?.type).toBe('edit'); + expect(result.action?.status).toBe('running'); + }); + + it('detects tool completion and returns completed status', async () => { + const toolStart = '┌─ Tool Use: Bash ────────────────────────┐\n│ ls -la │\n'; + await engine.process(encode(toolStart)); + const toolEnd = 'total 42\ndrwxr-xr-x 8 user group 256 Jan 1 00:00 .\n└──────────────────────────────────────────┘\n'; + const result = await engine.process(encode(toolEnd)); + expect(result.action?.status).toBe('completed'); + }); + + it('does not create duplicate running actions for the same tool call', async () => { + const toolBox = '┌─ Tool Use: Bash ────────────────────────┐\n│ git status │\n'; + const result1 = await engine.process(encode(toolBox)); + expect(result1.action?.status).toBe('running'); + // Empty chunk, same buffer state: should not emit a new action + const result2 = await engine.process(encode('')); + expect(result2.action).toBeUndefined(); + }); + + it('assigns a unique id to each distinct action', async () => { + // Start and complete the first action + const tool1Start = '┌─ Tool Use: Bash ────────────────────────┐\n│ echo hello │\n'; + const r1 = await engine.process(encode(tool1Start)); + const id1 = r1.action?.id; + await engine.process(encode('hello\n└──────────────────────────────────────────┘\n')); + + // Start a second action — must get a different id + const tool2Start = '┌─ Tool Use: Read ────────────────────────┐\n│ README.md │\n'; + const r2 = await engine.process(encode(tool2Start)); + const id2 = r2.action?.id; + + expect(id1).toBeDefined(); + expect(id2).toBeDefined(); + expect(id1).not.toBe(id2); + }); + }); + + describe('action detection — shell command fallback', () => { + it('detects a $ shell command', async () => { + const result = await engine.process(encode('$ git diff HEAD~1')); + expect(result.action?.type).toBe('bash'); + expect(result.action?.label).toBe('git diff HEAD~1'); + expect(result.action?.status).toBe('running'); + }); + + it('detects a ❯ shell command', async () => { + const result = await engine.process(encode('❯ npm run build')); + expect(result.action?.type).toBe('bash'); + expect(result.action?.label).toBe('npm run build'); + }); + + it('ignores bare shell names that are not meaningful commands', async () => { + const result = await engine.process(encode('$ bash')); + expect(result.action).toBeUndefined(); + }); + + it('ignores cd navigation commands', async () => { + const result = await engine.process(encode('$ cd /home/user')); + expect(result.action).toBeUndefined(); + }); + }); + + describe('dispose', () => { + it('cleans up resources without throwing', () => { + expect(() => engine.dispose()).not.toThrow(); + }); + + it('can be called multiple times safely', () => { + expect(() => { + engine.dispose(); + engine.dispose(); + }).not.toThrow(); + }); + + it('returns empty result after dispose', async () => { + engine.dispose(); + const result = await engine.process(encode('test')); + expect(result).toEqual({}); + }); + }); + + describe('UTF-8 handling', () => { + it('handles multi-byte UTF-8 characters without throwing', async () => { + await expect(engine.process(encode('Hello 世界 🌍'))).resolves.not.toThrow(); + }); + + it('handles incomplete UTF-8 sequences across chunk boundaries', async () => { + // '世界' is 6 bytes; split after byte 3 (mid-character) + const fullBytes = Buffer.from('世界'); + const chunk1 = fullBytes.slice(0, 3).toString('base64'); + const chunk2 = fullBytes.slice(3).toString('base64'); + await engine.process(chunk1); + await expect(engine.process(chunk2)).resolves.not.toThrow(); + }); + + it('handles emojis', async () => { + await expect(engine.process(encode('Status: ✅ Error: ❌'))).resolves.not.toThrow(); + }); + }); + + describe('resource management', () => { + it('maintains separate state for multiple instances', async () => { + const engine1 = new HeuristicsEngine(); + const engine2 = new HeuristicsEngine(); + await engine1.process(encode('test')); + await engine2.process(encode('test')); + expect(() => engine1.dispose()).not.toThrow(); + expect(() => engine2.dispose()).not.toThrow(); + }); + }); + + describe('large output handling', () => { + it('handles large chunks without crashing', async () => { + await expect(engine.process(encode('x'.repeat(10000)))).resolves.not.toThrow(); + }); + + it('handles multiple rapid chunks', async () => { + for (let i = 0; i < 10; i++) { + await expect(engine.process(encode(`line ${i}\n`))).resolves.not.toThrow(); + } + }); + }); + + describe('edge cases', () => { + it('handles binary data gracefully', async () => { + const binary = Buffer.from([0x00, 0x01, 0x02, 0xFF, 0xFE, 0xFD]); + await expect(engine.process(binary.toString('base64'))).resolves.not.toThrow(); + }); + + it('handles ANSI escape sequences', async () => { + await expect(engine.process(encode('\x1b[32mGreen\x1b[0m \x07'))).resolves.not.toThrow(); + }); + + it('handles whitespace-only content', async () => { + await expect(engine.process(encode(' \n\n \r\n '))).resolves.not.toThrow(); + }); + }); +}); diff --git a/backend/src/terminal/heuristics.ts b/backend/src/terminal/heuristics.ts new file mode 100644 index 0000000..6d8017d --- /dev/null +++ b/backend/src/terminal/heuristics.ts @@ -0,0 +1,362 @@ +import xtermHeadless from '@xterm/headless'; +import { nanoid } from 'nanoid'; + +// Handle ESM / CJS interop for @xterm/headless +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const Terminal = (xtermHeadless as any).Terminal || (xtermHeadless as any).default?.Terminal; + +export interface PromptState { + isActive: boolean; + type: 'yesno' | 'enter' | null; + text?: string; +} + +export interface TimelineAction { + id: string; + type: 'bash' | 'read' | 'edit' | 'grep' | 'ls' | 'custom'; + label: string; + status: 'running' | 'completed' | 'error'; + content?: string; + startTime: string; + endTime?: string; +} + +export interface HeuristicsResult { + prompt?: PromptState; + action?: TimelineAction; +} + +/** + * Maximum time an action can be 'running' before it's considered stale and marked as error. + * This prevents actions from being stuck in 'running' state forever. + */ +const ACTION_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes + +/** + * Represents a potentially incomplete UTF-8 sequence that needs to be carried over + * to the next chunk to avoid byte-splitting corruption. + */ +interface Utf8IncompleteSequence { + bytes: number[]; + bytesNeeded: number; +} + +export class HeuristicsEngine { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + private term: any; + private lastPromptState: PromptState = { isActive: false, type: null }; + private activeAction: TimelineAction | null = null; + private pendingUtf8: Utf8IncompleteSequence | null = null; + private lastCompletionBufferLine = 0; + private actionStartTime: number | null = null; + private processingQueue: Promise = Promise.resolve({}); + + constructor() { + this.term = new Terminal({ + cols: 1000, + rows: 100, + scrollback: 500, + allowProposedApi: true, + }); + + if (this.term._core?.optionsService?.options) { + this.term._core.optionsService.options.allowProposedApi = true; + } + } + + /** + * Process a chunk of raw PTY data. + * Serializes calls through a queue so concurrent invocations never race on + * shared xterm buffer state. + */ + public process(chunkBase64: string): Promise { + this.processingQueue = this.processingQueue.then(() => this.processChunk(chunkBase64)); + return this.processingQueue; + } + + private async processChunk(chunkBase64: string): Promise { + if (!chunkBase64 || !this.term) return {}; + + // Check for stale actions that should be marked as error + const staleAction = this.checkForStaleAction(); + if (staleAction) { + return staleAction; + } + + const chunk = Buffer.from(chunkBase64, 'base64'); + const text = this.decodeUtf8WithCarryover(chunk); + + if (text.length === 0) return {}; + + // xterm's write() is asynchronous — await the callback so the buffer + // is fully updated before we scan for prompts and actions. + await new Promise(resolve => this.term.write(text, resolve)); + + if (!this.term) return {}; // disposed while awaiting + + return { + prompt: this.detectPrompt() || undefined, + action: this.detectAction() || undefined + }; + } + + /** + * Check if the current active action has been running for too long + * and should be marked as error (stale). + */ + private checkForStaleAction(): HeuristicsResult | null { + if (this.activeAction && this.activeAction.status === 'running' && this.actionStartTime !== null) { + const elapsed = Date.now() - this.actionStartTime; + if (elapsed > ACTION_TIMEOUT_MS) { + this.activeAction.status = 'error'; + this.activeAction.endTime = new Date().toISOString(); + const result = { ...this.activeAction }; + this.activeAction = null; + this.actionStartTime = null; + this.lastCompletionBufferLine = 0; + return { action: result }; + } + } + return null; + } + + /** + * Decodes UTF-8 bytes while handling incomplete sequences at chunk boundaries. + * If a multi-byte UTF-8 sequence is split across chunks, this carries over + * the incomplete bytes and prepends them to the next chunk. + */ + private decodeUtf8WithCarryover(chunk: Buffer): string { + const bytes = Array.from(chunk); + let result: number[] = []; + + // Prepend any pending incomplete sequence from previous chunk + if (this.pendingUtf8) { + result = [...this.pendingUtf8.bytes, ...bytes]; + this.pendingUtf8 = null; + } else { + result = bytes; + } + + // Find and handle incomplete UTF-8 sequences at the end + let finalIndex = result.length - 1; + let bytesNeeded = 0; + + // Check if the last byte is a lead byte that expects more bytes + if (finalIndex >= 0) { + const lastByte = result[finalIndex]; + if ((lastByte & 0x80) === 0x00) { + bytesNeeded = 0; // ASCII + } else if ((lastByte & 0xE0) === 0xC0) { + bytesNeeded = 1; // 2-byte sequence, need 1 more + } else if ((lastByte & 0xF0) === 0xE0) { + bytesNeeded = 2; // 3-byte sequence, need 2 more + } else if ((lastByte & 0xF8) === 0xF0) { + bytesNeeded = 3; // 4-byte sequence, need 3 more + } else if ((lastByte & 0xC0) === 0x80) { + // This is a continuation byte but we have no pending sequence + // Skip it and try from the byte before + bytesNeeded = 0; + for (let i = result.length - 2; i >= 0; i--) { + const b = result[i]; + if ((b & 0x80) === 0x00) { + finalIndex = i; + break; + } + if ((b & 0xE0) === 0xC0) { bytesNeeded = 1; finalIndex = i; break; } + if ((b & 0xF0) === 0xE0) { bytesNeeded = 2; finalIndex = i; break; } + if ((b & 0xF8) === 0xF0) { bytesNeeded = 3; finalIndex = i; break; } + } + } + } + + // If we have an incomplete sequence, carry it over to next chunk + if (bytesNeeded > 0 && finalIndex >= 0) { + const carriedBytes = result.slice(finalIndex); + if (carriedBytes.length < bytesNeeded + 1) { + this.pendingUtf8 = { bytes: carriedBytes, bytesNeeded }; + result = result.slice(0, finalIndex); + } + } + + return Buffer.from(result).toString('utf8'); + } + + /** + * Cleanup resources. + */ + public dispose(): void { + if (this.term) { + this.term.dispose(); + this.term = null; + } + this.pendingUtf8 = null; + this.actionStartTime = null; + } + + private detectPrompt(): PromptState | null { + if (!this.term) return null; + + const activeBuffer = this.term.buffer.active; + const end = activeBuffer.cursorY + activeBuffer.baseY; + const start = Math.max(0, end - 5); + + let text = ''; + for (let i = start; i <= end; i++) { + const line = activeBuffer.getLine(i); + if (line) { + text += line.translateToString(true) + '\n'; + } + } + + const trimmed = text.trim(); + + // Improved regex for prompt detection + // Matches patterns like: "Do you want to continue? [Y/n]" or "Overwrite file? (y/N)" + const yesnoMatch = trimmed.match(/(.*?)(?:\[|\()([yY]\/[nN])(?:\]|\))\s*\??\s*$/s); + + // Matches: "Press Enter to continue..." or "Press [Enter] to exit" + const enterMatch = trimmed.match(/(.*?)Press (?:\[?Enter\]?|any key) to (?:continue|exit)\.*$/is); + + let newState: PromptState = { isActive: false, type: null }; + + if (yesnoMatch) { + const precedingLines = yesnoMatch[1].trim().split('\n'); + const context = precedingLines.slice(-2).join('\n').trim() || 'Permission requested'; + newState = { isActive: true, type: 'yesno', text: context }; + } else if (enterMatch) { + const precedingLines = enterMatch[1].trim().split('\n'); + const context = precedingLines.slice(-1).join('\n').trim(); + newState = { isActive: true, type: 'enter', text: context || 'Action required' }; + } + + // Only return if the prompt state has changed significantly + if (this.lastPromptState.isActive !== newState.isActive || + this.lastPromptState.type !== newState.type || + (newState.isActive && this.lastPromptState.text !== newState.text)) { + this.lastPromptState = newState; + return newState; + } + + return null; + } + + private detectAction(): TimelineAction | null { + if (!this.term) return null; + + const activeBuffer = this.term.buffer.active; + const end = activeBuffer.cursorY + activeBuffer.baseY; + const start = Math.max(0, end - 15); + + let lines: string[] = []; + for (let i = start; i <= end; i++) { + const line = activeBuffer.getLine(i); + if (line) { + lines.push(line.translateToString(true)); + } + } + + const text = lines.join('\n'); + + // 1. Detect Tool Use Start (Claude / Gemini Style boxes) + const toolStartMatch = text.match(/[┌╭]─+ Tool Use: ([\w]+) ─+┐/i); + if (toolStartMatch) { + const toolName = toolStartMatch[1].toLowerCase(); + const type = this.mapToolType(toolName); + + // Look for the specific argument/command line + const startIndex = lines.findIndex(l => l.match(/[┌╭]─+ Tool Use:/i)); + let label = 'Working...'; + if (startIndex !== -1 && lines[startIndex + 1]) { + // Clean up common box characters + label = lines[startIndex + 1].replace(/[│|┃]/g, '').trim(); + } + + // If we already have a running action with the same label, update it instead of creating new + // This prevents orphaned "running" actions when labels change during execution + if (this.activeAction && + this.activeAction.status === 'running' && + this.activeAction.label !== label) { + // Update the existing action's label rather than creating a new one + this.activeAction.label = label; + // Return the updated action to sync with frontend + return { ...this.activeAction }; + } + + // Create new action only if no active action exists or if the previous one was completed + if (!this.activeAction || this.activeAction.status !== 'running') { + this.activeAction = { + id: nanoid(8), + type, + label, + status: 'running', + startTime: new Date().toISOString() + }; + this.actionStartTime = Date.now(); + this.lastCompletionBufferLine = 0; + return this.activeAction; + } + } + + // 2. Detect Tool Completion - scan buffer lines we haven't yet checked. + // Uses absolute buffer line positions so the check range is correct across + // successive process() calls (unlike a local array index which resets to 0-15 + // every call and would cause lines to be re-checked or skipped as the cursor moves). + if (this.activeAction && this.activeAction.status === 'running') { + const scanFrom = Math.max(start, this.lastCompletionBufferLine); + for (let bufLine = scanFrom; bufLine <= end; bufLine++) { + const bufferLine = activeBuffer.getLine(bufLine); + // Looks for └──────────┘ or ╰──────────╯ + if (bufferLine && bufferLine.translateToString(true).match(/[└╰]─+┘/)) { + this.activeAction.status = 'completed'; + this.activeAction.endTime = new Date().toISOString(); + const completedAction = { ...this.activeAction }; + this.activeAction = null; + this.actionStartTime = null; + this.lastCompletionBufferLine = 0; + return completedAction; + } + } + this.lastCompletionBufferLine = end; + } + + // 3. Fallback: Generic Shell Command Detection + // Detects lines starting with common prompt characters: $ # > ❯ λ etc. + if (!this.activeAction) { + for (let i = lines.length - 1; i >= Math.max(0, lines.length - 3); i--) { + const line = lines[i]; + // More inclusive regex: matches $, #, >, ❯, λ prompts + const bashMatch = line.match(/^([\$>#❯λ]\s*)(.+)$/); + if (bashMatch) { + const cmd = bashMatch[2].trim(); + // Filter out common interactive shells/prompts that aren't actions + if (['bash', 'zsh', 'sh', 'node', 'python', 'python3', 'ruby', 'perl'].includes(cmd)) continue; + // Filter out pure path navigation + if (/^cd\s/.test(cmd)) continue; + + this.activeAction = { + id: nanoid(8), + type: 'bash', + label: cmd, + status: 'running', + startTime: new Date().toISOString() + }; + this.actionStartTime = Date.now(); + this.lastCompletionBufferLine = 0; + return this.activeAction; + } + } + } + + return null; + } + + private mapToolType(tool: string): TimelineAction['type'] { + const t = tool.toLowerCase(); + if (t.includes('bash') || t.includes('shell') || t.includes('run')) return 'bash'; + if (t.includes('read') || t.includes('cat')) return 'read'; + if (t.includes('edit') || t.includes('write') || t.includes('patch')) return 'edit'; + if (t.includes('grep') || t.includes('search') || t.includes('find')) return 'grep'; + if (t.includes('ls') || t.includes('list')) return 'ls'; + return 'custom'; + } +} diff --git a/backend/src/terminal/routes.ts b/backend/src/terminal/routes.ts index a43ee19..9f3f726 100644 --- a/backend/src/terminal/routes.ts +++ b/backend/src/terminal/routes.ts @@ -3,6 +3,7 @@ import { getSession, getSessionByPublicId, hasTranscriptRecorder } from '../sess import { validateSession } from '../auth/service.js'; import { sidecarManager, type SidecarStreamHandle } from './sidecar-manager.js'; import * as tmux from '../tmux/adapter.js'; +import { HeuristicsEngine } from './heuristics.js'; import { appendTranscript, appendTranscriptResize, @@ -141,6 +142,7 @@ const terminalRoutes: FastifyPluginAsync = async (fastify) => { let attachedSession: ReturnType | null = null; let attachPromise: Promise | null = null; let lastSize = { cols: 80, rows: 24 }; + const heuristics = new HeuristicsEngine(); // Heartbeat: detect silent/dead connections (e.g. phone sleep, network change). // Server pings every 15s; if no pong arrives before the next ping, the connection @@ -201,6 +203,18 @@ const terminalRoutes: FastifyPluginAsync = async (fastify) => { onData: (dataBase64) => { if (ws.readyState !== 1) return; ws.send(JSON.stringify({ type: 'terminal.output', dataBase64 })); + + void heuristics.process(dataBase64).then(result => { + if (ws.readyState !== 1) return; + if (result.prompt) { + ws.send(JSON.stringify({ type: 'prompt.state', promptState: result.prompt })); + } + if (result.action) { + ws.send(JSON.stringify({ type: 'timeline.action', action: result.action })); + } + }).catch((err: unknown) => { + fastify.log.warn({ err }, 'heuristics.process error'); + }); }, onText: (text) => { if (session && !isMirrorOnly && !hasTranscriptRecorder(session.id)) { @@ -324,6 +338,7 @@ const terminalRoutes: FastifyPluginAsync = async (fastify) => { ws.on('close', () => { cleanupHeartbeat(); + heuristics.dispose(); void ptySession?.close().catch(() => {}); ptySession = null; attachedSession = null; @@ -331,6 +346,7 @@ const terminalRoutes: FastifyPluginAsync = async (fastify) => { ws.on('error', () => { cleanupHeartbeat(); + heuristics.dispose(); void ptySession?.close().catch(() => {}); ptySession = null; attachedSession = null; diff --git a/frontend/package.json b/frontend/package.json index 46194e1..1f3ce75 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "cloudcode-frontend", - "version": "0.1.10", + "version": "0.1.11", "description": "CloudCode frontend — mobile-first React UI for coding agent session management", "license": "MIT", "type": "module", diff --git a/frontend/src/components/ActionTimeline.tsx b/frontend/src/components/ActionTimeline.tsx new file mode 100644 index 0000000..53b0756 --- /dev/null +++ b/frontend/src/components/ActionTimeline.tsx @@ -0,0 +1,112 @@ +import { TimelineAction } from '../hooks/useTerminal' +import { useState } from 'react' + +interface ActionTimelineProps { + actions: TimelineAction[] +} + +export function ActionTimeline({ actions }: ActionTimelineProps) { + const [expandedId, setExpandedId] = useState(null) + + if (actions.length === 0) { + return ( +
+
+ + + +
+

Timeline is empty

+

Agent actions will appear here as they happen.

+
+ ) + } + + return ( +
+ {actions.slice().reverse().map((action) => ( +
+ + + {expandedId === action.id && action.content && ( +
+
+ {action.content} +
+
+ )} +
+ ))} +
+ ) +} + +function ActionIcon({ type }: { type: TimelineAction['type'] }) { + const baseClasses = "w-9 h-9 rounded-xl flex items-center justify-center flex-shrink-0 border" + + const styles = { + bash: "bg-zinc-800/50 border-zinc-700/50 text-zinc-300", + read: "bg-blue-500/10 border-blue-500/20 text-blue-400", + edit: "bg-amber-500/10 border-amber-500/20 text-amber-400", + grep: "bg-purple-500/10 border-purple-500/20 text-purple-400", + ls: "bg-emerald-500/10 border-emerald-500/20 text-emerald-400", + custom: "bg-zinc-800/50 border-zinc-700/50 text-zinc-300", + } + + const iconD = { + bash: "M8 9l3 3-3 3m5 0h3", + read: "M12 6.253v13m0-13C10.832 5.477 9.246 5 7.5 5S4.168 5.477 3 6.253v13C4.168 18.477 5.754 18 7.5 18s3.084.477 4.5 1.253m0-13C13.168 5.477 14.754 5 16.5 5c1.747 0 3.332.477 4.5 1.253v13C19.832 18.477 18.247 18 16.5 18c-1.746 0-3.332.477-4.5 1.253", + edit: "M11 5H6a2 2 0 00-2 2v11a2 2 0 002 2h11a2 2 0 002-2v-5m-1.414-9.414a2 2 0 112.828 2.828L11.828 15H9v-2.828l8.586-8.586z", + grep: "M21 21l-6-6m2-5a7 7 0 11-14 0 7 7 0 0114 0z", + ls: "M3 7v10a2 2 0 002 2h14a2 2 0 002-2V9a2 2 0 00-2-2h-6l-2-2H5a2 2 0 00-2 2z", + custom: "M13 10V3L4 14h7v7l9-11h-7z", + } + + return ( +
+ + + +
+ ) +} diff --git a/frontend/src/components/Terminal.tsx b/frontend/src/components/Terminal.tsx index ea4e5a4..5424377 100644 --- a/frontend/src/components/Terminal.tsx +++ b/frontend/src/components/Terminal.tsx @@ -3,7 +3,7 @@ import { Terminal as XTerm } from '@xterm/xterm' import { FitAddon } from '@xterm/addon-fit' import { WebLinksAddon } from '@xterm/addon-web-links' import { SearchAddon } from '@xterm/addon-search' -import { useTerminal } from '../hooks/useTerminal' +import { TimelineAction, useTerminal } from '../hooks/useTerminal' import { apiFetch } from '../hooks/useApi' import '@xterm/xterm/css/xterm.css' @@ -11,6 +11,7 @@ interface TerminalProps { sessionId: string sessionTitle?: string agentName?: string + onTimelineActions?: (actions: TimelineAction[]) => void } const XTERM_THEME = { @@ -57,13 +58,17 @@ const FONT_SIZE_DEFAULT = window.innerWidth < 768 ? 14 : 13 // True when phone is in landscape (tablets excluded via max-height) const LANDSCAPE_MQ = '(orientation: landscape) and (max-height: 500px)' -export function Terminal({ sessionId, sessionTitle, agentName }: TerminalProps) { +export function Terminal({ sessionId, sessionTitle, agentName, onTimelineActions }: TerminalProps) { const containerRef = useRef(null) const xtermRef = useRef(null) const fitAddonRef = useRef(null) const searchAddonRef = useRef(null) const [terminalInstance, setTerminalInstance] = useState(null) - const { isConnected, bootState, sessionEnded, sendInput, resize } = useTerminal({ sessionId, terminal: terminalInstance }) + const { isConnected, bootState, sessionEnded, promptState, timelineActions, sendInput, dismissPrompt, resize } = useTerminal({ sessionId, terminal: terminalInstance }) + + useEffect(() => { + onTimelineActions?.(timelineActions) + }, [timelineActions, onTimelineActions]) const [ctrlMode, setCtrlMode] = useState(false) const [showSearch, setShowSearch] = useState(false) @@ -589,6 +594,49 @@ export function Terminal({ sessionId, sessionTitle, agentName }: TerminalProps) return (
+ {/* Smart Prompt Overlay */} + {promptState?.isActive && ( +
+
+
+
+ +
+
+

Agent Action Required

+

{promptState.text}

+
+
+ +
+ {promptState.type === 'yesno' ? ( + <> + + + + ) : promptState.type === 'enter' ? ( + + ) : null} +
+
+
+ )} + {/* Search Bar Overlay */} {showSearch && (
diff --git a/frontend/src/hooks/useTerminal.ts b/frontend/src/hooks/useTerminal.ts index 10e2918..25256ea 100644 --- a/frontend/src/hooks/useTerminal.ts +++ b/frontend/src/hooks/useTerminal.ts @@ -7,11 +7,30 @@ export interface UseTerminalOptions { terminal: Terminal | null } +export interface PromptState { + isActive: boolean + type: 'yesno' | 'enter' | null + text?: string +} + +export interface TimelineAction { + id: string + type: 'bash' | 'read' | 'edit' | 'grep' | 'ls' | 'custom' + label: string + status: 'running' | 'completed' | 'error' + content?: string + startTime: string + endTime?: string +} + export interface UseTerminalResult { isConnected: boolean bootState: 'loading-history' | 'connecting' | 'waiting-for-output' | 'ready' sessionEnded: boolean + promptState: PromptState | null + timelineActions: TimelineAction[] sendInput: (data: string) => void + dismissPrompt: () => void resize: (cols: number, rows: number) => void } @@ -46,6 +65,8 @@ export function useTerminal({ sessionId, terminal }: UseTerminalOptions): UseTer const [isConnected, setIsConnected] = useState(false) const [bootState, setBootState] = useState('loading-history') const [sessionEnded, setSessionEnded] = useState(false) + const [promptState, setPromptState] = useState(null) + const [timelineActions, setTimelineActions] = useState([]) const lastSizeRef = useRef<{ cols: number; rows: number } | null>(null) const pendingMessagesRef = useRef([]) const hasRenderedContentRef = useRef(false) @@ -72,6 +93,10 @@ export function useTerminal({ sessionId, terminal }: UseTerminalOptions): UseTer sendMessage({ type: 'terminal.input', data }) }, [sendMessage]) + const dismissPrompt = useCallback(() => { + setPromptState(prev => prev ? { ...prev, isActive: false } : null) + }, []) + const resize = useCallback((cols: number, rows: number) => { lastSizeRef.current = { cols, rows } sendMessage({ type: 'terminal.resize', cols, rows }) @@ -186,6 +211,8 @@ export function useTerminal({ sessionId, terminal }: UseTerminalOptions): UseTer error?: string message?: string timestamp?: number + promptState?: PromptState + action?: TimelineAction } switch (msg.type) { @@ -230,6 +257,26 @@ export function useTerminal({ sessionId, terminal }: UseTerminalOptions): UseTer } } break + case 'prompt.state': + if (msg.promptState) setPromptState(msg.promptState) + break + case 'timeline.action': { + const action = msg.action + if (!action) break + setTimelineActions(prev => { + const exists = prev.findIndex(a => a.id === action.id) + if (exists !== -1) { + const next = [...prev] + next[exists] = action + return next + } + // Cap at 100 actions to prevent memory growth in long sessions + const MAX_TIMELINE_ACTIONS = 100 + const updated = [...prev, action] + return updated.length > MAX_TIMELINE_ACTIONS ? updated.slice(-MAX_TIMELINE_ACTIONS) : updated + }) + break + } default: break } @@ -339,6 +386,8 @@ export function useTerminal({ sessionId, terminal }: UseTerminalOptions): UseTer hasRenderedContentRef.current = false setBootState('loading-history') setSessionEnded(false) + setPromptState(null) + setTimelineActions([]) if (terminal) { terminal.write('\x1bc') void loadBootstrap().finally(() => { @@ -362,5 +411,5 @@ export function useTerminal({ sessionId, terminal }: UseTerminalOptions): UseTer } }, [sessionId, terminal, connect, loadBootstrap]) - return { isConnected, bootState, sessionEnded, sendInput, resize } + return { isConnected, bootState, sessionEnded, promptState, timelineActions, sendInput, dismissPrompt, resize } } diff --git a/frontend/src/pages/SessionDetail.tsx b/frontend/src/pages/SessionDetail.tsx index 2187be4..623ad35 100644 --- a/frontend/src/pages/SessionDetail.tsx +++ b/frontend/src/pages/SessionDetail.tsx @@ -3,8 +3,10 @@ import { useParams, useNavigate, useLocation, useSearchParams } from 'react-rout import { Session } from '../types' import { Terminal } from '../components/Terminal' import { apiFetch } from '../hooks/useApi' +import { ActionTimeline } from '../components/ActionTimeline' +import { TimelineAction } from '../hooks/useTerminal' -type Tab = 'terminal' | 'logs' +type Tab = 'terminal' | 'logs' | 'timeline' interface TranscriptPageItem { index: number @@ -103,11 +105,12 @@ export function SessionDetail() { const isMirrorMode = !!sessionName const navigate = useNavigate() const tabParam = searchParams.get('tab') as Tab | null - const initialTab: Tab = tabParam === 'logs' || tabParam === 'terminal' + const initialTab: Tab = tabParam === 'logs' || tabParam === 'terminal' || tabParam === 'timeline' ? tabParam : (launchState?.activeTab ?? 'terminal') const [activeTab, setActiveTab] = useState(isMirrorMode && initialTab === 'logs' ? 'terminal' : initialTab) + const [timelineActions, setTimelineActions] = useState([]) const [session, setSession] = useState(null) const [loading, setLoading] = useState(true) const [error, setError] = useState(null) @@ -647,8 +650,10 @@ export function SessionDetail() { const tabs: { id: Tab; label: string }[] = isMirrorMode ? [ { id: 'terminal', label: 'Terminal' }, + { id: 'timeline', label: 'Timeline' }, ] : [ { id: 'terminal', label: 'Terminal' }, + { id: 'timeline', label: 'Timeline' }, { id: 'logs', label: 'Logs' }, ] @@ -778,10 +783,21 @@ export function SessionDetail() { sessionId={session.publicId} sessionTitle={session.title} agentName={session.agentProfile?.name ?? 'agent'} + onTimelineActions={setTimelineActions} />
+ {activeTab === 'timeline' && ( +
+
+

Action Timeline

+ {timelineActions.length} events +
+ +
+ )} + {activeTab === 'logs' && (
diff --git a/package.json b/package.json index 033996e..196c51b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "cloudcode", - "version": "0.1.10", + "version": "0.1.11", "description": "Self-hosted, mobile-first web interface for managing local CLI-based coding agents over Tailscale", "license": "MIT", "repository": {