From 18d8c089b156d67a2d01d1589eaba545f266c24f Mon Sep 17 00:00:00 2001 From: Alex Chao Date: Tue, 31 Mar 2026 10:51:20 -0700 Subject: [PATCH 01/10] feat(backend): add HeuristicsEngine for semantic terminal parsing --- backend/src/terminal/heuristics.ts | 210 +++++++++++++++++++++++++++++ 1 file changed, 210 insertions(+) create mode 100644 backend/src/terminal/heuristics.ts diff --git a/backend/src/terminal/heuristics.ts b/backend/src/terminal/heuristics.ts new file mode 100644 index 0000000..3f51550 --- /dev/null +++ b/backend/src/terminal/heuristics.ts @@ -0,0 +1,210 @@ +import xtermHeadless from '@xterm/headless'; +import { nanoid } from 'nanoid'; + +// Handle ESM / CJS interop for @xterm/headless +const Terminal = (xtermHeadless as any).Terminal || (xtermHeadless as any).default?.Terminal; + +export interface PromptState { + isActive: boolean; + type: 'yesno' | 'enter' | null; + text?: string; +} + +export interface TimelineAction { + id: string; + type: 'bash' | 'read' | 'edit' | 'grep' | 'ls' | 'custom'; + label: string; + status: 'running' | 'completed' | 'error'; + content?: string; + startTime: string; + endTime?: string; +} + +export interface HeuristicsResult { + prompt?: PromptState; + action?: TimelineAction; +} + +export class HeuristicsEngine { + private term: any; + private lastPromptState: PromptState = { isActive: false, type: null }; + private activeAction: TimelineAction | null = null; + + constructor() { + this.term = new Terminal({ + cols: 1000, + rows: 100, + scrollback: 500, // Reduced from 100000 to save memory in headless mode + allowProposedApi: true, + }); + + if (this.term._core?.optionsService?.options) { + this.term._core.optionsService.options.allowProposedApi = true; + } + } + + /** + * Process a chunk of raw PTY data. + */ + public process(chunkBase64: string): HeuristicsResult { + if (!chunkBase64) return {}; + + const chunk = Buffer.from(chunkBase64, 'base64').toString('utf8'); + this.term.write(chunk); + + return { + prompt: this.detectPrompt() || undefined, + action: this.detectAction() || undefined + }; + } + + /** + * Cleanup resources. + */ + public dispose(): void { + if (this.term) { + this.term.dispose(); + this.term = null; + } + } + + private detectPrompt(): PromptState | null { + if (!this.term) return null; + + const activeBuffer = this.term.buffer.active; + const end = activeBuffer.cursorY + activeBuffer.baseY; + const start = Math.max(0, end - 5); // Prompts are usually very near the cursor + + let text = ''; + for (let i = start; i <= end; i++) { + const line = activeBuffer.getLine(i); + if (line) { + text += line.translateToString(true) + '\n'; + } + } + + const trimmed = text.trim(); + + // Improved regex for prompt detection + // Matches patterns like: "Do you want to continue? [Y/n]" or "Overwrite file? (y/N)" + const yesnoMatch = trimmed.match(/(.*?)(?:\[|\()([yY]\/[nN])(?:\]|\))\s*\??\s*$/s); + + // Matches: "Press Enter to continue..." or "Press [Enter] to exit" + const enterMatch = trimmed.match(/(.*?)Press (?:\[?Enter\]?|any key) to (?:continue|exit)\.*$/is); + + let newState: PromptState = { isActive: false, type: null }; + + if (yesnoMatch) { + const precedingLines = yesnoMatch[1].trim().split('\n'); + const context = precedingLines.slice(-2).join('\n').trim() || 'Permission requested'; + newState = { isActive: true, type: 'yesno', text: context }; + } else if (enterMatch) { + const precedingLines = enterMatch[1].trim().split('\n'); + const context = precedingLines.slice(-1).join('\n').trim(); + newState = { isActive: true, type: 'enter', text: context || 'Action required' }; + } + + // Only return if the prompt state has changed significantly + if (this.lastPromptState.isActive !== newState.isActive || + this.lastPromptState.type !== newState.type || + (newState.isActive && this.lastPromptState.text !== newState.text)) { + this.lastPromptState = newState; + return newState; + } + + return null; + } + + private detectAction(): TimelineAction | null { + if (!this.term) return null; + + const activeBuffer = this.term.buffer.active; + const end = activeBuffer.cursorY + activeBuffer.baseY; + const start = Math.max(0, end - 15); + + let lines: string[] = []; + for (let i = start; i <= end; i++) { + const line = activeBuffer.getLine(i); + if (line) { + lines.push(line.translateToString(true)); + } + } + + const text = lines.join('\n'); + + // 1. Detect Tool Use Start (Claude / Gemini Style boxes) + const toolStartMatch = text.match(/[┌╭]─+ Tool Use: ([\w]+) ─+┐/i); + if (toolStartMatch) { + const toolName = toolStartMatch[1].toLowerCase(); + const type = this.mapToolType(toolName); + + // Look for the specific argument/command line + const startIndex = lines.findIndex(l => l.match(/[┌╭]─+ Tool Use:/i)); + let label = 'Working...'; + if (startIndex !== -1 && lines[startIndex + 1]) { + // Clean up common box characters + label = lines[startIndex + 1].replace(/[│|┃]/g, '').trim(); + } + + // Avoid creating a new action if the label is exactly the same as the current running one + if (!this.activeAction || this.activeAction.label !== label || this.activeAction.status !== 'running') { + this.activeAction = { + id: nanoid(8), + type, + label, + status: 'running', + startTime: new Date().toISOString() + }; + return this.activeAction; + } + } + + // 2. Detect Tool Completion (Bottom of box) + if (this.activeAction && this.activeAction.status === 'running') { + const lastLine = lines[lines.length - 1] || ''; + // Looks for └──────────┘ or ╰──────────╯ + if (lastLine.match(/[└╰]─+┘/)) { + this.activeAction.status = 'completed'; + this.activeAction.endTime = new Date().toISOString(); + const completedAction = { ...this.activeAction }; + this.activeAction = null; + return completedAction; + } + } + + // 3. Fallback: Generic Shell Command Detection + // Detects lines starting with $ or > followed by a command + if (!this.activeAction) { + for (let i = lines.length - 1; i >= Math.max(0, lines.length - 3); i--) { + const line = lines[i]; + const bashMatch = line.match(/^[\$]\s+([a-zA-Z0-9].+)$/); + if (bashMatch) { + const cmd = bashMatch[1].trim(); + // Filter out common interactive shells/prompts that aren't actions + if (['bash', 'zsh', 'sh', 'node', 'python'].includes(cmd)) continue; + + this.activeAction = { + id: nanoid(8), + type: 'bash', + label: cmd, + status: 'running', + startTime: new Date().toISOString() + }; + return this.activeAction; + } + } + } + + return null; + } + + private mapToolType(tool: string): TimelineAction['type'] { + const t = tool.toLowerCase(); + if (t.includes('bash') || t.includes('shell') || t.includes('run')) return 'bash'; + if (t.includes('read') || t.includes('cat')) return 'read'; + if (t.includes('edit') || t.includes('write') || t.includes('patch')) return 'edit'; + if (t.includes('grep') || t.includes('search') || t.includes('find')) return 'grep'; + if (t.includes('ls') || t.includes('list')) return 'ls'; + return 'custom'; + } +} From 3b1b91f1baf6f05cdd7c703e731bed15a60cd437 Mon Sep 17 00:00:00 2001 From: Alex Chao Date: Tue, 31 Mar 2026 10:51:20 -0700 Subject: [PATCH 02/10] feat(backend): integrate heuristics engine into terminal websocket --- backend/src/terminal/routes.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/backend/src/terminal/routes.ts b/backend/src/terminal/routes.ts index a43ee19..d702901 100644 --- a/backend/src/terminal/routes.ts +++ b/backend/src/terminal/routes.ts @@ -3,6 +3,7 @@ import { getSession, getSessionByPublicId, hasTranscriptRecorder } from '../sess import { validateSession } from '../auth/service.js'; import { sidecarManager, type SidecarStreamHandle } from './sidecar-manager.js'; import * as tmux from '../tmux/adapter.js'; +import { HeuristicsEngine } from './heuristics.js'; import { appendTranscript, appendTranscriptResize, @@ -141,6 +142,7 @@ const terminalRoutes: FastifyPluginAsync = async (fastify) => { let attachedSession: ReturnType | null = null; let attachPromise: Promise | null = null; let lastSize = { cols: 80, rows: 24 }; + const heuristics = new HeuristicsEngine(); // Heartbeat: detect silent/dead connections (e.g. phone sleep, network change). // Server pings every 15s; if no pong arrives before the next ping, the connection @@ -201,6 +203,14 @@ const terminalRoutes: FastifyPluginAsync = async (fastify) => { onData: (dataBase64) => { if (ws.readyState !== 1) return; ws.send(JSON.stringify({ type: 'terminal.output', dataBase64 })); + + const result = heuristics.process(dataBase64); + if (result.prompt) { + ws.send(JSON.stringify({ type: 'prompt.state', promptState: result.prompt })); + } + if (result.action) { + ws.send(JSON.stringify({ type: 'timeline.action', action: result.action })); + } }, onText: (text) => { if (session && !isMirrorOnly && !hasTranscriptRecorder(session.id)) { @@ -324,6 +334,7 @@ const terminalRoutes: FastifyPluginAsync = async (fastify) => { ws.on('close', () => { cleanupHeartbeat(); + heuristics.dispose(); void ptySession?.close().catch(() => {}); ptySession = null; attachedSession = null; @@ -331,6 +342,7 @@ const terminalRoutes: FastifyPluginAsync = async (fastify) => { ws.on('error', () => { cleanupHeartbeat(); + heuristics.dispose(); void ptySession?.close().catch(() => {}); ptySession = null; attachedSession = null; From 3239ec7e0e178669ef93ddbf8d107d6742c26dc0 Mon Sep 17 00:00:00 2001 From: Alex Chao Date: Tue, 31 Mar 2026 10:51:20 -0700 Subject: [PATCH 03/10] feat(frontend): update useTerminal hook for prompt and action state --- frontend/src/hooks/useTerminal.ts | 40 ++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/frontend/src/hooks/useTerminal.ts b/frontend/src/hooks/useTerminal.ts index 10e2918..f052b3a 100644 --- a/frontend/src/hooks/useTerminal.ts +++ b/frontend/src/hooks/useTerminal.ts @@ -7,10 +7,28 @@ export interface UseTerminalOptions { terminal: Terminal | null } +export interface PromptState { + isActive: boolean + type: 'yesno' | 'enter' | null + text?: string +} + +export interface TimelineAction { + id: string + type: 'bash' | 'read' | 'edit' | 'grep' | 'ls' | 'custom' + label: string + status: 'running' | 'completed' | 'error' + content?: string + startTime: string + endTime?: string +} + export interface UseTerminalResult { isConnected: boolean bootState: 'loading-history' | 'connecting' | 'waiting-for-output' | 'ready' sessionEnded: boolean + promptState: PromptState | null + timelineActions: TimelineAction[] sendInput: (data: string) => void resize: (cols: number, rows: number) => void } @@ -46,6 +64,8 @@ export function useTerminal({ sessionId, terminal }: UseTerminalOptions): UseTer const [isConnected, setIsConnected] = useState(false) const [bootState, setBootState] = useState('loading-history') const [sessionEnded, setSessionEnded] = useState(false) + const [promptState, setPromptState] = useState(null) + const [timelineActions, setTimelineActions] = useState([]) const lastSizeRef = useRef<{ cols: number; rows: number } | null>(null) const pendingMessagesRef = useRef([]) const hasRenderedContentRef = useRef(false) @@ -230,6 +250,22 @@ export function useTerminal({ sessionId, terminal }: UseTerminalOptions): UseTer } } break + case 'prompt.state': + setPromptState((msg as any).promptState as PromptState) + break + case 'timeline.action': { + const action = (msg as any).action as TimelineAction + setTimelineActions(prev => { + const exists = prev.findIndex(a => a.id === action.id) + if (exists !== -1) { + const next = [...prev] + next[exists] = action + return next + } + return [...prev, action] + }) + break + } default: break } @@ -339,6 +375,8 @@ export function useTerminal({ sessionId, terminal }: UseTerminalOptions): UseTer hasRenderedContentRef.current = false setBootState('loading-history') setSessionEnded(false) + setPromptState(null) + setTimelineActions([]) if (terminal) { terminal.write('\x1bc') void loadBootstrap().finally(() => { @@ -362,5 +400,5 @@ export function useTerminal({ sessionId, terminal }: UseTerminalOptions): UseTer } }, [sessionId, terminal, connect, loadBootstrap]) - return { isConnected, bootState, sessionEnded, sendInput, resize } + return { isConnected, bootState, sessionEnded, promptState, timelineActions, sendInput, resize } } From 3f52bfa53b8a97196e9c66b46a778fb7abb5946e Mon Sep 17 00:00:00 2001 From: Alex Chao Date: Tue, 31 Mar 2026 10:51:20 -0700 Subject: [PATCH 04/10] feat(frontend): implement smart approval modals and action timeline UI --- frontend/src/components/ActionTimeline.tsx | 112 +++++++++++++++++++++ frontend/src/components/Terminal.tsx | 54 +++++++++- frontend/src/pages/SessionDetail.tsx | 20 +++- 3 files changed, 181 insertions(+), 5 deletions(-) create mode 100644 frontend/src/components/ActionTimeline.tsx diff --git a/frontend/src/components/ActionTimeline.tsx b/frontend/src/components/ActionTimeline.tsx new file mode 100644 index 0000000..c079e3c --- /dev/null +++ b/frontend/src/components/ActionTimeline.tsx @@ -0,0 +1,112 @@ +import { TimelineAction } from '../hooks/useTerminal' +import { useState } from 'react' + +interface ActionTimelineProps { + actions: TimelineAction[] +} + +export function ActionTimeline({ actions }: ActionTimelineProps) { + const [expandedId, setExpandedId] = useState(null) + + if (actions.length === 0) { + return ( +
+
+ + + +
+

Timeline is empty

+

Agent actions will appear here as they happen.

+
+ ) + } + + return ( +
+ {actions.slice().reverse().map((action) => ( +
+ + + {expandedId === action.id && action.content && ( +
+
+ {action.content} +
+
+ )} +
+ ))} +
+ ) +} + +function ActionIcon({ type, status }: { type: TimelineAction['type'], status: TimelineAction['status'] }) { + const baseClasses = "w-9 h-9 rounded-xl flex items-center justify-center flex-shrink-0 border" + + const styles = { + bash: "bg-zinc-800/50 border-zinc-700/50 text-zinc-300", + read: "bg-blue-500/10 border-blue-500/20 text-blue-400", + edit: "bg-amber-500/10 border-amber-500/20 text-amber-400", + grep: "bg-purple-500/10 border-purple-500/20 text-purple-400", + ls: "bg-emerald-500/10 border-emerald-500/20 text-emerald-400", + custom: "bg-zinc-800/50 border-zinc-700/50 text-zinc-300", + } + + const iconD = { + bash: "M8 9l3 3-3 3m5 0h3", + read: "M12 6.253v13m0-13C10.832 5.477 9.246 5 7.5 5S4.168 5.477 3 6.253v13C4.168 18.477 5.754 18 7.5 18s3.084.477 4.5 1.253m0-13C13.168 5.477 14.754 5 16.5 5c1.747 0 3.332.477 4.5 1.253v13C19.832 18.477 18.247 18 16.5 18c-1.746 0-3.332.477-4.5 1.253", + edit: "M11 5H6a2 2 0 00-2 2v11a2 2 0 002 2h11a2 2 0 002-2v-5m-1.414-9.414a2 2 0 112.828 2.828L11.828 15H9v-2.828l8.586-8.586z", + grep: "M21 21l-6-6m2-5a7 7 0 11-14 0 7 7 0 0114 0z", + ls: "M3 7v10a2 2 0 002 2h14a2 2 0 002-2V9a2 2 0 00-2-2h-6l-2-2H5a2 2 0 00-2 2z", + custom: "M13 10V3L4 14h7v7l9-11h-7z", + } + + return ( +
+ + + +
+ ) +} diff --git a/frontend/src/components/Terminal.tsx b/frontend/src/components/Terminal.tsx index ea4e5a4..3384cbe 100644 --- a/frontend/src/components/Terminal.tsx +++ b/frontend/src/components/Terminal.tsx @@ -3,7 +3,7 @@ import { Terminal as XTerm } from '@xterm/xterm' import { FitAddon } from '@xterm/addon-fit' import { WebLinksAddon } from '@xterm/addon-web-links' import { SearchAddon } from '@xterm/addon-search' -import { useTerminal } from '../hooks/useTerminal' +import { TimelineAction, useTerminal } from '../hooks/useTerminal' import { apiFetch } from '../hooks/useApi' import '@xterm/xterm/css/xterm.css' @@ -11,6 +11,7 @@ interface TerminalProps { sessionId: string sessionTitle?: string agentName?: string + onTimelineActions?: (actions: TimelineAction[]) => void } const XTERM_THEME = { @@ -57,13 +58,17 @@ const FONT_SIZE_DEFAULT = window.innerWidth < 768 ? 14 : 13 // True when phone is in landscape (tablets excluded via max-height) const LANDSCAPE_MQ = '(orientation: landscape) and (max-height: 500px)' -export function Terminal({ sessionId, sessionTitle, agentName }: TerminalProps) { +export function Terminal({ sessionId, sessionTitle, agentName, onTimelineActions }: TerminalProps) { const containerRef = useRef(null) const xtermRef = useRef(null) const fitAddonRef = useRef(null) const searchAddonRef = useRef(null) const [terminalInstance, setTerminalInstance] = useState(null) - const { isConnected, bootState, sessionEnded, sendInput, resize } = useTerminal({ sessionId, terminal: terminalInstance }) + const { isConnected, bootState, sessionEnded, promptState, timelineActions, sendInput, resize } = useTerminal({ sessionId, terminal: terminalInstance }) + + useEffect(() => { + onTimelineActions?.(timelineActions) + }, [timelineActions, onTimelineActions]) const [ctrlMode, setCtrlMode] = useState(false) const [showSearch, setShowSearch] = useState(false) @@ -589,6 +594,49 @@ export function Terminal({ sessionId, sessionTitle, agentName }: TerminalProps) return (
+ {/* Smart Prompt Overlay */} + {promptState?.isActive && ( +
+
+
+
+ +
+
+

Agent Action Required

+

{promptState.text}

+
+
+ +
+ {promptState.type === 'yesno' ? ( + <> + + + + ) : promptState.type === 'enter' ? ( + + ) : null} +
+
+
+ )} + {/* Search Bar Overlay */} {showSearch && (
diff --git a/frontend/src/pages/SessionDetail.tsx b/frontend/src/pages/SessionDetail.tsx index 2187be4..623ad35 100644 --- a/frontend/src/pages/SessionDetail.tsx +++ b/frontend/src/pages/SessionDetail.tsx @@ -3,8 +3,10 @@ import { useParams, useNavigate, useLocation, useSearchParams } from 'react-rout import { Session } from '../types' import { Terminal } from '../components/Terminal' import { apiFetch } from '../hooks/useApi' +import { ActionTimeline } from '../components/ActionTimeline' +import { TimelineAction } from '../hooks/useTerminal' -type Tab = 'terminal' | 'logs' +type Tab = 'terminal' | 'logs' | 'timeline' interface TranscriptPageItem { index: number @@ -103,11 +105,12 @@ export function SessionDetail() { const isMirrorMode = !!sessionName const navigate = useNavigate() const tabParam = searchParams.get('tab') as Tab | null - const initialTab: Tab = tabParam === 'logs' || tabParam === 'terminal' + const initialTab: Tab = tabParam === 'logs' || tabParam === 'terminal' || tabParam === 'timeline' ? tabParam : (launchState?.activeTab ?? 'terminal') const [activeTab, setActiveTab] = useState(isMirrorMode && initialTab === 'logs' ? 'terminal' : initialTab) + const [timelineActions, setTimelineActions] = useState([]) const [session, setSession] = useState(null) const [loading, setLoading] = useState(true) const [error, setError] = useState(null) @@ -647,8 +650,10 @@ export function SessionDetail() { const tabs: { id: Tab; label: string }[] = isMirrorMode ? [ { id: 'terminal', label: 'Terminal' }, + { id: 'timeline', label: 'Timeline' }, ] : [ { id: 'terminal', label: 'Terminal' }, + { id: 'timeline', label: 'Timeline' }, { id: 'logs', label: 'Logs' }, ] @@ -778,10 +783,21 @@ export function SessionDetail() { sessionId={session.publicId} sessionTitle={session.title} agentName={session.agentProfile?.name ?? 'agent'} + onTimelineActions={setTimelineActions} />
+ {activeTab === 'timeline' && ( +
+
+

Action Timeline

+ {timelineActions.length} events +
+ +
+ )} + {activeTab === 'logs' && (
From cfb0a389f550f4de1a48547672bbb0db2bfe6e88 Mon Sep 17 00:00:00 2001 From: Alex Chao Date: Tue, 31 Mar 2026 10:51:20 -0700 Subject: [PATCH 05/10] docs: update README with smart remote control features --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 996b470..aad7070 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,9 @@ Start an agent on your laptop, walk away, and check in from your phone or tablet ## Why CloudCode? - **Agent agnostic:** Works with Claude Code, Gemini CLI, OpenAI Codex, GitHub Copilot CLI, or any CLI tool. -- **Task launch:** Start tasks instantly from your mobile dashboard without opening a full terminal. CloudCode defaults to your recent projects and drops you into the live session view. +- **Smart Remote Control:** Detects agent activity and transforms it into a **Task Timeline** of collapsible cards. +- **Instant Approvals:** Automatically pops up native **Approval Modals** when an agent requests permission, saving you from opening the mobile keyboard. +- **Task launch:** Start tasks instantly from your mobile dashboard without opening a full terminal. - **Transcript logs:** Shows the full session output in a scrollable, timestamped transcript view. - **QR code pairing:** Scan a QR code from your terminal to authenticate your phone. No passwords or SSH keys. - **Persistent sessions:** Sessions run inside `tmux`. Your agent keeps working if your laptop sleeps or your connection drops. Reconnect and pick up where you left off. From 77c0d5283a872bac4e09d370edbb69fb7882b9ea Mon Sep 17 00:00:00 2001 From: Alex Chao Date: Mon, 6 Apr 2026 12:35:20 -0700 Subject: [PATCH 06/10] fix: resolve heuristics engine issues identified in code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix UTF-8 byte splitting: add decodeUtf8WithCarryover() to handle incomplete multi-byte sequences at chunk boundaries - Fix fragile completion detection: scan all lines instead of just the last line to detect tool completion markers - Fix orphaned running actions: update existing action label instead of creating duplicate when label changes during execution - Add null check for disposed terminal in process() - Expand shell prompt regex to match $, #, >, ❯, λ prompts - Add stale action timeout: mark actions as 'error' after 5 minutes to prevent stuck 'running' states --- backend/src/terminal/heuristics.ts | 208 ++++++++++++++++++++++++----- 1 file changed, 176 insertions(+), 32 deletions(-) diff --git a/backend/src/terminal/heuristics.ts b/backend/src/terminal/heuristics.ts index 3f51550..9847fb8 100644 --- a/backend/src/terminal/heuristics.ts +++ b/backend/src/terminal/heuristics.ts @@ -2,6 +2,7 @@ import xtermHeadless from '@xterm/headless'; import { nanoid } from 'nanoid'; // Handle ESM / CJS interop for @xterm/headless +// eslint-disable-next-line @typescript-eslint/no-explicit-any const Terminal = (xtermHeadless as any).Terminal || (xtermHeadless as any).default?.Terminal; export interface PromptState { @@ -25,19 +26,38 @@ export interface HeuristicsResult { action?: TimelineAction; } +/** + * Maximum time an action can be 'running' before it's considered stale and marked as error. + * This prevents actions from being stuck in 'running' state forever. + */ +const ACTION_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes + +/** + * Represents a potentially incomplete UTF-8 sequence that needs to be carried over + * to the next chunk to avoid byte-splitting corruption. + */ +interface Utf8IncompleteSequence { + bytes: number[]; + bytesNeeded: number; +} + export class HeuristicsEngine { + // eslint-disable-next-line @typescript-eslint/no-explicit-any private term: any; private lastPromptState: PromptState = { isActive: false, type: null }; private activeAction: TimelineAction | null = null; + private pendingUtf8: Utf8IncompleteSequence | null = null; + private lastCompletionCheckIndex = 0; + private actionStartTime: number | null = null; constructor() { this.term = new Terminal({ cols: 1000, rows: 100, - scrollback: 500, // Reduced from 100000 to save memory in headless mode + scrollback: 500, allowProposedApi: true, }); - + if (this.term._core?.optionsService?.options) { this.term._core.optionsService.options.allowProposedApi = true; } @@ -45,19 +65,111 @@ export class HeuristicsEngine { /** * Process a chunk of raw PTY data. + * Handles UTF-8 byte splitting to prevent corruption of multi-byte characters. */ public process(chunkBase64: string): HeuristicsResult { - if (!chunkBase64) return {}; - - const chunk = Buffer.from(chunkBase64, 'base64').toString('utf8'); - this.term.write(chunk); - + if (!chunkBase64 || !this.term) return {}; + + // Check for stale actions that should be marked as error + const staleAction = this.checkForStaleAction(); + if (staleAction) { + return staleAction; + } + + const chunk = Buffer.from(chunkBase64, 'base64'); + const text = this.decodeUtf8WithCarryover(chunk); + + if (text.length === 0) return {}; + + this.term.write(text); + return { prompt: this.detectPrompt() || undefined, action: this.detectAction() || undefined }; } + /** + * Check if the current active action has been running for too long + * and should be marked as error (stale). + */ + private checkForStaleAction(): HeuristicsResult | null { + if (this.activeAction && this.activeAction.status === 'running' && this.actionStartTime !== null) { + const elapsed = Date.now() - this.actionStartTime; + if (elapsed > ACTION_TIMEOUT_MS) { + this.activeAction.status = 'error'; + this.activeAction.endTime = new Date().toISOString(); + const result = { ...this.activeAction }; + this.activeAction = null; + this.actionStartTime = null; + this.lastCompletionCheckIndex = 0; + return { action: result }; + } + } + return null; + } + + /** + * Decodes UTF-8 bytes while handling incomplete sequences at chunk boundaries. + * If a multi-byte UTF-8 sequence is split across chunks, this carries over + * the incomplete bytes and prepends them to the next chunk. + */ + private decodeUtf8WithCarryover(chunk: Buffer): string { + const bytes = Array.from(chunk); + let result: number[] = []; + + // Prepend any pending incomplete sequence from previous chunk + if (this.pendingUtf8) { + result = [...this.pendingUtf8.bytes, ...bytes]; + this.pendingUtf8 = null; + } else { + result = bytes; + } + + // Find and handle incomplete UTF-8 sequences at the end + let finalIndex = result.length - 1; + let bytesNeeded = 0; + + // Check if the last byte is a lead byte that expects more bytes + if (finalIndex >= 0) { + const lastByte = result[finalIndex]; + if ((lastByte & 0x80) === 0x00) { + bytesNeeded = 0; // ASCII + } else if ((lastByte & 0xE0) === 0xC0) { + bytesNeeded = 1; // 2-byte sequence, need 1 more + } else if ((lastByte & 0xF0) === 0xE0) { + bytesNeeded = 2; // 3-byte sequence, need 2 more + } else if ((lastByte & 0xF8) === 0xF0) { + bytesNeeded = 3; // 4-byte sequence, need 3 more + } else if ((lastByte & 0xC0) === 0x80) { + // This is a continuation byte but we have no pending sequence + // Skip it and try from the byte before + bytesNeeded = 0; + for (let i = result.length - 2; i >= 0; i--) { + const b = result[i]; + if ((b & 0x80) === 0x00) { + finalIndex = i; + break; + } + if ((b & 0xE0) === 0xC0) { bytesNeeded = 1; finalIndex = i; break; } + if ((b & 0xF0) === 0xE0) { bytesNeeded = 2; finalIndex = i; break; } + if ((b & 0xF8) === 0xF0) { bytesNeeded = 3; finalIndex = i; break; } + } + } + } + + // If we have an incomplete sequence, carry it over to next chunk + if (bytesNeeded > 0 && finalIndex >= 0) { + const carriedBytes = result.slice(finalIndex); + if (carriedBytes.length < bytesNeeded + 1) { + this.pendingUtf8 = { bytes: carriedBytes, bytesNeeded }; + result = result.slice(0, finalIndex); + } + } + + return Buffer.from(result).toString('utf8'); + } + /** * Cleanup resources. */ @@ -66,6 +178,8 @@ export class HeuristicsEngine { this.term.dispose(); this.term = null; } + this.pendingUtf8 = null; + this.actionStartTime = null; } private detectPrompt(): PromptState | null { @@ -73,8 +187,8 @@ export class HeuristicsEngine { const activeBuffer = this.term.buffer.active; const end = activeBuffer.cursorY + activeBuffer.baseY; - const start = Math.max(0, end - 5); // Prompts are usually very near the cursor - + const start = Math.max(0, end - 5); + let text = ''; for (let i = start; i <= end; i++) { const line = activeBuffer.getLine(i); @@ -84,16 +198,16 @@ export class HeuristicsEngine { } const trimmed = text.trim(); - + // Improved regex for prompt detection // Matches patterns like: "Do you want to continue? [Y/n]" or "Overwrite file? (y/N)" const yesnoMatch = trimmed.match(/(.*?)(?:\[|\()([yY]\/[nN])(?:\]|\))\s*\??\s*$/s); - + // Matches: "Press Enter to continue..." or "Press [Enter] to exit" const enterMatch = trimmed.match(/(.*?)Press (?:\[?Enter\]?|any key) to (?:continue|exit)\.*$/is); - + let newState: PromptState = { isActive: false, type: null }; - + if (yesnoMatch) { const precedingLines = yesnoMatch[1].trim().split('\n'); const context = precedingLines.slice(-2).join('\n').trim() || 'Permission requested'; @@ -105,13 +219,13 @@ export class HeuristicsEngine { } // Only return if the prompt state has changed significantly - if (this.lastPromptState.isActive !== newState.isActive || + if (this.lastPromptState.isActive !== newState.isActive || this.lastPromptState.type !== newState.type || (newState.isActive && this.lastPromptState.text !== newState.text)) { this.lastPromptState = newState; return newState; } - + return null; } @@ -121,7 +235,7 @@ export class HeuristicsEngine { const activeBuffer = this.term.buffer.active; const end = activeBuffer.cursorY + activeBuffer.baseY; const start = Math.max(0, end - 15); - + let lines: string[] = []; for (let i = start; i <= end; i++) { const line = activeBuffer.getLine(i); @@ -137,7 +251,7 @@ export class HeuristicsEngine { if (toolStartMatch) { const toolName = toolStartMatch[1].toLowerCase(); const type = this.mapToolType(toolName); - + // Look for the specific argument/command line const startIndex = lines.findIndex(l => l.match(/[┌╭]─+ Tool Use:/i)); let label = 'Working...'; @@ -146,8 +260,19 @@ export class HeuristicsEngine { label = lines[startIndex + 1].replace(/[│|┃]/g, '').trim(); } - // Avoid creating a new action if the label is exactly the same as the current running one - if (!this.activeAction || this.activeAction.label !== label || this.activeAction.status !== 'running') { + // If we already have a running action with the same label, update it instead of creating new + // This prevents orphaned "running" actions when labels change during execution + if (this.activeAction && + this.activeAction.status === 'running' && + this.activeAction.label !== label) { + // Update the existing action's label rather than creating a new one + this.activeAction.label = label; + // Return the updated action to sync with frontend + return { ...this.activeAction }; + } + + // Create new action only if no active action exists or if the previous one was completed + if (!this.activeAction || this.activeAction.status !== 'running') { this.activeAction = { id: nanoid(8), type, @@ -155,33 +280,51 @@ export class HeuristicsEngine { status: 'running', startTime: new Date().toISOString() }; + this.actionStartTime = Date.now(); return this.activeAction; } } - // 2. Detect Tool Completion (Bottom of box) + // 2. Detect Tool Completion - scan ALL lines, not just the last one + // This is more robust against wrapped lines, trailing whitespace, etc. if (this.activeAction && this.activeAction.status === 'running') { - const lastLine = lines[lines.length - 1] || ''; - // Looks for └──────────┘ or ╰──────────╯ - if (lastLine.match(/[└╰]─+┘/)) { - this.activeAction.status = 'completed'; - this.activeAction.endTime = new Date().toISOString(); - const completedAction = { ...this.activeAction }; - this.activeAction = null; - return completedAction; + // Reset check position if we've moved to a new part of the buffer + const currentCheckIndex = lines.length - 1; + + // Scan through lines looking for completion marker + // Only check lines we haven't already validated + const checkStart = this.lastCompletionCheckIndex; + for (let i = Math.max(0, checkStart); i <= currentCheckIndex; i++) { + const line = lines[i] || ''; + // Looks for └──────────┘ or ╰──────────╯ + if (line.match(/[└╰]─+┘/)) { + this.activeAction.status = 'completed'; + this.activeAction.endTime = new Date().toISOString(); + const completedAction = { ...this.activeAction }; + this.activeAction = null; + this.actionStartTime = null; + this.lastCompletionCheckIndex = 0; // Reset for next action + return completedAction; + } } + + // Mark that we've checked up to the current index + this.lastCompletionCheckIndex = currentCheckIndex; } // 3. Fallback: Generic Shell Command Detection - // Detects lines starting with $ or > followed by a command + // Detects lines starting with common prompt characters: $ # > ❯ λ etc. if (!this.activeAction) { for (let i = lines.length - 1; i >= Math.max(0, lines.length - 3); i--) { const line = lines[i]; - const bashMatch = line.match(/^[\$]\s+([a-zA-Z0-9].+)$/); + // More inclusive regex: matches $, #, >, ❯, λ prompts + const bashMatch = line.match(/^([\$>#❯λ]\s*)(.+)$/); if (bashMatch) { - const cmd = bashMatch[1].trim(); + const cmd = bashMatch[2].trim(); // Filter out common interactive shells/prompts that aren't actions - if (['bash', 'zsh', 'sh', 'node', 'python'].includes(cmd)) continue; + if (['bash', 'zsh', 'sh', 'node', 'python', 'python3', 'ruby', 'perl'].includes(cmd)) continue; + // Filter out pure path navigation + if (/^cd\s/.test(cmd)) continue; this.activeAction = { id: nanoid(8), @@ -190,6 +333,7 @@ export class HeuristicsEngine { status: 'running', startTime: new Date().toISOString() }; + this.actionStartTime = Date.now(); return this.activeAction; } } From ab323868e6f12bd8d1e60953ea1006847ec21839 Mon Sep 17 00:00:00 2001 From: Alex Chao Date: Mon, 6 Apr 2026 12:35:30 -0700 Subject: [PATCH 07/10] refactor: cap timeline actions, cleanup unused params, add tests --- backend/src/terminal/heuristics.test.ts | 156 +++++++++++++++++++++ frontend/src/components/ActionTimeline.tsx | 4 +- frontend/src/hooks/useTerminal.ts | 5 +- 3 files changed, 162 insertions(+), 3 deletions(-) create mode 100644 backend/src/terminal/heuristics.test.ts diff --git a/backend/src/terminal/heuristics.test.ts b/backend/src/terminal/heuristics.test.ts new file mode 100644 index 0000000..4e66d60 --- /dev/null +++ b/backend/src/terminal/heuristics.test.ts @@ -0,0 +1,156 @@ +import { describe, expect, it, beforeEach, afterEach } from 'vitest'; +import { HeuristicsEngine } from './heuristics.js'; + +describe('HeuristicsEngine', () => { + let engine: HeuristicsEngine; + + beforeEach(() => { + engine = new HeuristicsEngine(); + }); + + afterEach(() => { + engine.dispose(); + }); + + describe('process', () => { + it('returns empty result for empty chunk', () => { + const result = engine.process(''); + expect(result).toEqual({}); + }); + + it('handles null/undefined chunk gracefully', () => { + // @ts-ignore - testing invalid input + const result = engine.process(null); + expect(result).toEqual({}); + }); + + it('processes chunk without throwing', () => { + const chunk = Buffer.from('some terminal output').toString('base64'); + expect(() => engine.process(chunk)).not.toThrow(); + }); + + it('returns a valid HeuristicsResult structure', () => { + const chunk = Buffer.from('test output').toString('base64'); + const result = engine.process(chunk); + expect(result).toHaveProperty('prompt'); + expect(result).toHaveProperty('action'); + }); + }); + + describe('dispose', () => { + it('cleans up resources without throwing', () => { + expect(() => engine.dispose()).not.toThrow(); + }); + + it('can be called multiple times safely', () => { + expect(() => { + engine.dispose(); + engine.dispose(); + }).not.toThrow(); + }); + + it('returns empty result after dispose', () => { + engine.dispose(); + const chunk = Buffer.from('test').toString('base64'); + const result = engine.process(chunk); + expect(result).toEqual({}); + }); + }); + + describe('UTF-8 handling', () => { + it('handles UTF-8 characters correctly', () => { + const chunk = Buffer.from('Hello 世界 🌍').toString('base64'); + expect(() => engine.process(chunk)).not.toThrow(); + }); + + it('handles mixed ASCII and UTF-8', () => { + const chunk = Buffer.from('File: test.py\nContent: 你好\nDone').toString('base64'); + expect(() => engine.process(chunk)).not.toThrow(); + }); + + it('handles multi-byte UTF-8 characters', () => { + const chunk = Buffer.from('日本語テスト').toString('base64'); + expect(() => engine.process(chunk)).not.toThrow(); + }); + + it('handles emojis', () => { + const chunk = Buffer.from('Status: ✅ Error: ❌').toString('base64'); + expect(() => engine.process(chunk)).not.toThrow(); + }); + + it('handles incomplete UTF-8 sequences across chunks', () => { + // Send a chunk that ends with a partial UTF-8 sequence + const fullText = 'Hello 世界'; + const chunk1 = Buffer.from('Hello ').toString('base64'); + const chunk2 = Buffer.from('世界').toString('base64'); + + engine.process(chunk1); + // Second chunk should handle the carryover correctly + expect(() => engine.process(chunk2)).not.toThrow(); + }); + }); + + describe('resource management', () => { + it('maintains separate state for multiple instances', () => { + const engine1 = new HeuristicsEngine(); + const engine2 = new HeuristicsEngine(); + + const chunk = Buffer.from('test').toString('base64'); + engine1.process(chunk); + engine2.process(chunk); + + // Both should work independently + expect(() => engine1.dispose()).not.toThrow(); + expect(() => engine2.dispose()).not.toThrow(); + }); + + it('process method is idempotent per engine instance', () => { + const chunk = Buffer.from('some output').toString('base64'); + const result1 = engine.process(chunk); + const result2 = engine.process(chunk); + // Should not throw on repeated calls + expect(result1).toBeDefined(); + expect(result2).toBeDefined(); + }); + }); + + describe('large output handling', () => { + it('handles large chunks without crashing', () => { + const largeOutput = 'x'.repeat(10000); + const chunk = Buffer.from(largeOutput).toString('base64'); + expect(() => engine.process(chunk)).not.toThrow(); + }); + + it('handles multiple rapid chunks', () => { + const chunk = Buffer.from('line 1\n').toString('base64'); + for (let i = 0; i < 10; i++) { + expect(() => engine.process(chunk)).not.toThrow(); + } + }); + }); + + describe('edge cases', () => { + it('handles binary data gracefully', () => { + // Send some binary-like data + const binary = Buffer.from([0x00, 0x01, 0x02, 0xFF, 0xFE, 0xFD]); + const chunk = binary.toString('base64'); + expect(() => engine.process(chunk)).not.toThrow(); + }); + + it('handles special terminal characters', () => { + const chunk = Buffer.from('\x1b[32mGreen\x1b[0m \x07').toString('base64'); + expect(() => engine.process(chunk)).not.toThrow(); + }); + + it('handles empty buffer', () => { + const chunk = Buffer.from('').toString('base64'); + const result = engine.process(chunk); + expect(result).toEqual({}); + }); + + it('handles whitespace-only content', () => { + const chunk = Buffer.from(' \n\n \r\n ').toString('base64'); + expect(() => engine.process(chunk)).not.toThrow(); + }); + }); +}); diff --git a/frontend/src/components/ActionTimeline.tsx b/frontend/src/components/ActionTimeline.tsx index c079e3c..53b0756 100644 --- a/frontend/src/components/ActionTimeline.tsx +++ b/frontend/src/components/ActionTimeline.tsx @@ -35,7 +35,7 @@ export function ActionTimeline({ actions }: ActionTimelineProps) { onClick={() => setExpandedId(expandedId === action.id ? null : action.id)} className="w-full text-left p-4 flex items-center gap-3" > - +
@@ -81,7 +81,7 @@ export function ActionTimeline({ actions }: ActionTimelineProps) { ) } -function ActionIcon({ type, status }: { type: TimelineAction['type'], status: TimelineAction['status'] }) { +function ActionIcon({ type }: { type: TimelineAction['type'] }) { const baseClasses = "w-9 h-9 rounded-xl flex items-center justify-center flex-shrink-0 border" const styles = { diff --git a/frontend/src/hooks/useTerminal.ts b/frontend/src/hooks/useTerminal.ts index f052b3a..83f76d1 100644 --- a/frontend/src/hooks/useTerminal.ts +++ b/frontend/src/hooks/useTerminal.ts @@ -262,7 +262,10 @@ export function useTerminal({ sessionId, terminal }: UseTerminalOptions): UseTer next[exists] = action return next } - return [...prev, action] + // Cap at 100 actions to prevent memory growth in long sessions + const MAX_TIMELINE_ACTIONS = 100 + const updated = [...prev, action] + return updated.length > MAX_TIMELINE_ACTIONS ? updated.slice(-MAX_TIMELINE_ACTIONS) : updated }) break } From 7d3a0db8170c87e685e3a82c4008743dec429b93 Mon Sep 17 00:00:00 2001 From: Alex Chao Date: Mon, 6 Apr 2026 12:41:27 -0700 Subject: [PATCH 08/10] fix: address code review issues in smart remote control - Fix lastCompletionCheckIndex bug: track absolute buffer line positions instead of local array indices so completion detection is correct across successive process() calls as the cursor advances - Make HeuristicsEngine.process() async so xterm write() completes before buffer is scanned; update routes to await the result - Remove any casts in useTerminal: extend WebSocket message type union with promptState and action fields - Add dismissPrompt() to useTerminal so approval buttons optimistically clear the overlay immediately on tap without waiting for the backend - Replace smoke-only tests with behavioral assertions: prompt detection (yesno, enter), tool-use box start/completion, shell command fallback, deduplication, and unique action IDs --- backend/src/terminal/heuristics.test.ts | 232 ++++++++++++++++-------- backend/src/terminal/heuristics.ts | 39 ++-- backend/src/terminal/routes.ts | 18 +- frontend/src/components/Terminal.tsx | 8 +- frontend/src/hooks/useTerminal.ts | 14 +- 5 files changed, 198 insertions(+), 113 deletions(-) diff --git a/backend/src/terminal/heuristics.test.ts b/backend/src/terminal/heuristics.test.ts index 4e66d60..ccf7b33 100644 --- a/backend/src/terminal/heuristics.test.ts +++ b/backend/src/terminal/heuristics.test.ts @@ -1,6 +1,10 @@ import { describe, expect, it, beforeEach, afterEach } from 'vitest'; import { HeuristicsEngine } from './heuristics.js'; +function encode(text: string): string { + return Buffer.from(text).toString('base64'); +} + describe('HeuristicsEngine', () => { let engine: HeuristicsEngine; @@ -13,27 +17,138 @@ describe('HeuristicsEngine', () => { }); describe('process', () => { - it('returns empty result for empty chunk', () => { - const result = engine.process(''); + it('returns empty result for empty chunk', async () => { + const result = await engine.process(''); expect(result).toEqual({}); }); - it('handles null/undefined chunk gracefully', () => { + it('handles null/undefined chunk gracefully', async () => { // @ts-ignore - testing invalid input - const result = engine.process(null); + const result = await engine.process(null); expect(result).toEqual({}); }); - it('processes chunk without throwing', () => { - const chunk = Buffer.from('some terminal output').toString('base64'); - expect(() => engine.process(chunk)).not.toThrow(); + it('processes chunk without throwing', async () => { + await expect(engine.process(encode('some terminal output'))).resolves.not.toThrow(); + }); + }); + + describe('prompt detection', () => { + it('detects [Y/n] yesno prompt and extracts context', async () => { + const result = await engine.process(encode('Overwrite file.txt? [Y/n]')); + expect(result.prompt?.isActive).toBe(true); + expect(result.prompt?.type).toBe('yesno'); + expect(result.prompt?.text).toContain('Overwrite file.txt'); + }); + + it('detects (y/N) yesno prompt', async () => { + const result = await engine.process(encode('Do you want to continue? (y/N)')); + expect(result.prompt?.isActive).toBe(true); + expect(result.prompt?.type).toBe('yesno'); + }); + + it('detects "Press Enter to continue" prompt', async () => { + const result = await engine.process(encode('Changes applied.\nPress Enter to continue...')); + expect(result.prompt?.isActive).toBe(true); + expect(result.prompt?.type).toBe('enter'); + }); + + it('detects "Press any key to exit" prompt', async () => { + const result = await engine.process(encode('Press any key to exit')); + expect(result.prompt?.isActive).toBe(true); + expect(result.prompt?.type).toBe('enter'); + }); + + it('does not re-emit unchanged prompt state', async () => { + const result1 = await engine.process(encode('Delete this? [Y/n]')); + expect(result1.prompt?.isActive).toBe(true); + // Empty chunk: state unchanged, should not re-emit + const result2 = await engine.process(encode('')); + expect(result2.prompt).toBeUndefined(); + }); + }); + + describe('action detection — Claude tool-use boxes', () => { + it('detects a Bash tool-use start and returns a running action', async () => { + const toolBox = '┌─ Tool Use: Bash ────────────────────────┐\n│ npm test │\n'; + const result = await engine.process(encode(toolBox)); + expect(result.action).toBeDefined(); + expect(result.action?.type).toBe('bash'); + expect(result.action?.status).toBe('running'); + expect(result.action?.label).toContain('npm test'); + }); + + it('detects a Read tool-use start', async () => { + const toolBox = '┌─ Tool Use: Read ────────────────────────┐\n│ src/index.ts │\n'; + const result = await engine.process(encode(toolBox)); + expect(result.action?.type).toBe('read'); + expect(result.action?.status).toBe('running'); + }); + + it('detects an Edit tool-use start', async () => { + const toolBox = '┌─ Tool Use: Edit ────────────────────────┐\n│ package.json │\n'; + const result = await engine.process(encode(toolBox)); + expect(result.action?.type).toBe('edit'); + expect(result.action?.status).toBe('running'); + }); + + it('detects tool completion and returns completed status', async () => { + const toolStart = '┌─ Tool Use: Bash ────────────────────────┐\n│ ls -la │\n'; + await engine.process(encode(toolStart)); + const toolEnd = 'total 42\ndrwxr-xr-x 8 user group 256 Jan 1 00:00 .\n└──────────────────────────────────────────┘\n'; + const result = await engine.process(encode(toolEnd)); + expect(result.action?.status).toBe('completed'); + }); + + it('does not create duplicate running actions for the same tool call', async () => { + const toolBox = '┌─ Tool Use: Bash ────────────────────────┐\n│ git status │\n'; + const result1 = await engine.process(encode(toolBox)); + expect(result1.action?.status).toBe('running'); + // Empty chunk, same buffer state: should not emit a new action + const result2 = await engine.process(encode('')); + expect(result2.action).toBeUndefined(); }); - it('returns a valid HeuristicsResult structure', () => { - const chunk = Buffer.from('test output').toString('base64'); - const result = engine.process(chunk); - expect(result).toHaveProperty('prompt'); - expect(result).toHaveProperty('action'); + it('assigns a unique id to each distinct action', async () => { + // Start and complete the first action + const tool1Start = '┌─ Tool Use: Bash ────────────────────────┐\n│ echo hello │\n'; + const r1 = await engine.process(encode(tool1Start)); + const id1 = r1.action?.id; + await engine.process(encode('hello\n└──────────────────────────────────────────┘\n')); + + // Start a second action — must get a different id + const tool2Start = '┌─ Tool Use: Read ────────────────────────┐\n│ README.md │\n'; + const r2 = await engine.process(encode(tool2Start)); + const id2 = r2.action?.id; + + expect(id1).toBeDefined(); + expect(id2).toBeDefined(); + expect(id1).not.toBe(id2); + }); + }); + + describe('action detection — shell command fallback', () => { + it('detects a $ shell command', async () => { + const result = await engine.process(encode('$ git diff HEAD~1')); + expect(result.action?.type).toBe('bash'); + expect(result.action?.label).toBe('git diff HEAD~1'); + expect(result.action?.status).toBe('running'); + }); + + it('detects a ❯ shell command', async () => { + const result = await engine.process(encode('❯ npm run build')); + expect(result.action?.type).toBe('bash'); + expect(result.action?.label).toBe('npm run build'); + }); + + it('ignores bare shell names that are not meaningful commands', async () => { + const result = await engine.process(encode('$ bash')); + expect(result.action).toBeUndefined(); + }); + + it('ignores cd navigation commands', async () => { + const result = await engine.process(encode('$ cd /home/user')); + expect(result.action).toBeUndefined(); }); }); @@ -49,108 +164,67 @@ describe('HeuristicsEngine', () => { }).not.toThrow(); }); - it('returns empty result after dispose', () => { + it('returns empty result after dispose', async () => { engine.dispose(); - const chunk = Buffer.from('test').toString('base64'); - const result = engine.process(chunk); + const result = await engine.process(encode('test')); expect(result).toEqual({}); }); }); describe('UTF-8 handling', () => { - it('handles UTF-8 characters correctly', () => { - const chunk = Buffer.from('Hello 世界 🌍').toString('base64'); - expect(() => engine.process(chunk)).not.toThrow(); + it('handles multi-byte UTF-8 characters without throwing', async () => { + await expect(engine.process(encode('Hello 世界 🌍'))).resolves.not.toThrow(); }); - it('handles mixed ASCII and UTF-8', () => { - const chunk = Buffer.from('File: test.py\nContent: 你好\nDone').toString('base64'); - expect(() => engine.process(chunk)).not.toThrow(); + it('handles incomplete UTF-8 sequences across chunk boundaries', async () => { + // '世界' is 6 bytes; split after byte 3 (mid-character) + const fullBytes = Buffer.from('世界'); + const chunk1 = fullBytes.slice(0, 3).toString('base64'); + const chunk2 = fullBytes.slice(3).toString('base64'); + await engine.process(chunk1); + await expect(engine.process(chunk2)).resolves.not.toThrow(); }); - it('handles multi-byte UTF-8 characters', () => { - const chunk = Buffer.from('日本語テスト').toString('base64'); - expect(() => engine.process(chunk)).not.toThrow(); - }); - - it('handles emojis', () => { - const chunk = Buffer.from('Status: ✅ Error: ❌').toString('base64'); - expect(() => engine.process(chunk)).not.toThrow(); - }); - - it('handles incomplete UTF-8 sequences across chunks', () => { - // Send a chunk that ends with a partial UTF-8 sequence - const fullText = 'Hello 世界'; - const chunk1 = Buffer.from('Hello ').toString('base64'); - const chunk2 = Buffer.from('世界').toString('base64'); - - engine.process(chunk1); - // Second chunk should handle the carryover correctly - expect(() => engine.process(chunk2)).not.toThrow(); + it('handles emojis', async () => { + await expect(engine.process(encode('Status: ✅ Error: ❌'))).resolves.not.toThrow(); }); }); describe('resource management', () => { - it('maintains separate state for multiple instances', () => { + it('maintains separate state for multiple instances', async () => { const engine1 = new HeuristicsEngine(); const engine2 = new HeuristicsEngine(); - - const chunk = Buffer.from('test').toString('base64'); - engine1.process(chunk); - engine2.process(chunk); - - // Both should work independently + await engine1.process(encode('test')); + await engine2.process(encode('test')); expect(() => engine1.dispose()).not.toThrow(); expect(() => engine2.dispose()).not.toThrow(); }); - - it('process method is idempotent per engine instance', () => { - const chunk = Buffer.from('some output').toString('base64'); - const result1 = engine.process(chunk); - const result2 = engine.process(chunk); - // Should not throw on repeated calls - expect(result1).toBeDefined(); - expect(result2).toBeDefined(); - }); }); describe('large output handling', () => { - it('handles large chunks without crashing', () => { - const largeOutput = 'x'.repeat(10000); - const chunk = Buffer.from(largeOutput).toString('base64'); - expect(() => engine.process(chunk)).not.toThrow(); + it('handles large chunks without crashing', async () => { + await expect(engine.process(encode('x'.repeat(10000)))).resolves.not.toThrow(); }); - it('handles multiple rapid chunks', () => { - const chunk = Buffer.from('line 1\n').toString('base64'); + it('handles multiple rapid chunks', async () => { for (let i = 0; i < 10; i++) { - expect(() => engine.process(chunk)).not.toThrow(); + await expect(engine.process(encode(`line ${i}\n`))).resolves.not.toThrow(); } }); }); describe('edge cases', () => { - it('handles binary data gracefully', () => { - // Send some binary-like data + it('handles binary data gracefully', async () => { const binary = Buffer.from([0x00, 0x01, 0x02, 0xFF, 0xFE, 0xFD]); - const chunk = binary.toString('base64'); - expect(() => engine.process(chunk)).not.toThrow(); + await expect(engine.process(binary.toString('base64'))).resolves.not.toThrow(); }); - it('handles special terminal characters', () => { - const chunk = Buffer.from('\x1b[32mGreen\x1b[0m \x07').toString('base64'); - expect(() => engine.process(chunk)).not.toThrow(); - }); - - it('handles empty buffer', () => { - const chunk = Buffer.from('').toString('base64'); - const result = engine.process(chunk); - expect(result).toEqual({}); + it('handles ANSI escape sequences', async () => { + await expect(engine.process(encode('\x1b[32mGreen\x1b[0m \x07'))).resolves.not.toThrow(); }); - it('handles whitespace-only content', () => { - const chunk = Buffer.from(' \n\n \r\n ').toString('base64'); - expect(() => engine.process(chunk)).not.toThrow(); + it('handles whitespace-only content', async () => { + await expect(engine.process(encode(' \n\n \r\n '))).resolves.not.toThrow(); }); }); }); diff --git a/backend/src/terminal/heuristics.ts b/backend/src/terminal/heuristics.ts index 9847fb8..b100942 100644 --- a/backend/src/terminal/heuristics.ts +++ b/backend/src/terminal/heuristics.ts @@ -47,7 +47,7 @@ export class HeuristicsEngine { private lastPromptState: PromptState = { isActive: false, type: null }; private activeAction: TimelineAction | null = null; private pendingUtf8: Utf8IncompleteSequence | null = null; - private lastCompletionCheckIndex = 0; + private lastCompletionBufferLine = 0; private actionStartTime: number | null = null; constructor() { @@ -67,7 +67,7 @@ export class HeuristicsEngine { * Process a chunk of raw PTY data. * Handles UTF-8 byte splitting to prevent corruption of multi-byte characters. */ - public process(chunkBase64: string): HeuristicsResult { + public async process(chunkBase64: string): Promise { if (!chunkBase64 || !this.term) return {}; // Check for stale actions that should be marked as error @@ -81,7 +81,11 @@ export class HeuristicsEngine { if (text.length === 0) return {}; - this.term.write(text); + // xterm's write() is asynchronous — await the callback so the buffer + // is fully updated before we scan for prompts and actions. + await new Promise(resolve => this.term.write(text, resolve)); + + if (!this.term) return {}; // disposed while awaiting return { prompt: this.detectPrompt() || undefined, @@ -102,7 +106,7 @@ export class HeuristicsEngine { const result = { ...this.activeAction }; this.activeAction = null; this.actionStartTime = null; - this.lastCompletionCheckIndex = 0; + this.lastCompletionBufferLine = 0; return { action: result }; } } @@ -281,35 +285,31 @@ export class HeuristicsEngine { startTime: new Date().toISOString() }; this.actionStartTime = Date.now(); + this.lastCompletionBufferLine = 0; return this.activeAction; } } - // 2. Detect Tool Completion - scan ALL lines, not just the last one - // This is more robust against wrapped lines, trailing whitespace, etc. + // 2. Detect Tool Completion - scan buffer lines we haven't yet checked. + // Uses absolute buffer line positions so the check range is correct across + // successive process() calls (unlike a local array index which resets to 0-15 + // every call and would cause lines to be re-checked or skipped as the cursor moves). if (this.activeAction && this.activeAction.status === 'running') { - // Reset check position if we've moved to a new part of the buffer - const currentCheckIndex = lines.length - 1; - - // Scan through lines looking for completion marker - // Only check lines we haven't already validated - const checkStart = this.lastCompletionCheckIndex; - for (let i = Math.max(0, checkStart); i <= currentCheckIndex; i++) { - const line = lines[i] || ''; + const scanFrom = Math.max(start, this.lastCompletionBufferLine); + for (let bufLine = scanFrom; bufLine <= end; bufLine++) { + const bufferLine = activeBuffer.getLine(bufLine); // Looks for └──────────┘ or ╰──────────╯ - if (line.match(/[└╰]─+┘/)) { + if (bufferLine && bufferLine.translateToString(true).match(/[└╰]─+┘/)) { this.activeAction.status = 'completed'; this.activeAction.endTime = new Date().toISOString(); const completedAction = { ...this.activeAction }; this.activeAction = null; this.actionStartTime = null; - this.lastCompletionCheckIndex = 0; // Reset for next action + this.lastCompletionBufferLine = 0; return completedAction; } } - - // Mark that we've checked up to the current index - this.lastCompletionCheckIndex = currentCheckIndex; + this.lastCompletionBufferLine = end; } // 3. Fallback: Generic Shell Command Detection @@ -334,6 +334,7 @@ export class HeuristicsEngine { startTime: new Date().toISOString() }; this.actionStartTime = Date.now(); + this.lastCompletionBufferLine = 0; return this.activeAction; } } diff --git a/backend/src/terminal/routes.ts b/backend/src/terminal/routes.ts index d702901..ef8d35c 100644 --- a/backend/src/terminal/routes.ts +++ b/backend/src/terminal/routes.ts @@ -203,14 +203,16 @@ const terminalRoutes: FastifyPluginAsync = async (fastify) => { onData: (dataBase64) => { if (ws.readyState !== 1) return; ws.send(JSON.stringify({ type: 'terminal.output', dataBase64 })); - - const result = heuristics.process(dataBase64); - if (result.prompt) { - ws.send(JSON.stringify({ type: 'prompt.state', promptState: result.prompt })); - } - if (result.action) { - ws.send(JSON.stringify({ type: 'timeline.action', action: result.action })); - } + + void heuristics.process(dataBase64).then(result => { + if (ws.readyState !== 1) return; + if (result.prompt) { + ws.send(JSON.stringify({ type: 'prompt.state', promptState: result.prompt })); + } + if (result.action) { + ws.send(JSON.stringify({ type: 'timeline.action', action: result.action })); + } + }); }, onText: (text) => { if (session && !isMirrorOnly && !hasTranscriptRecorder(session.id)) { diff --git a/frontend/src/components/Terminal.tsx b/frontend/src/components/Terminal.tsx index 3384cbe..5424377 100644 --- a/frontend/src/components/Terminal.tsx +++ b/frontend/src/components/Terminal.tsx @@ -64,7 +64,7 @@ export function Terminal({ sessionId, sessionTitle, agentName, onTimelineActions const fitAddonRef = useRef(null) const searchAddonRef = useRef(null) const [terminalInstance, setTerminalInstance] = useState(null) - const { isConnected, bootState, sessionEnded, promptState, timelineActions, sendInput, resize } = useTerminal({ sessionId, terminal: terminalInstance }) + const { isConnected, bootState, sessionEnded, promptState, timelineActions, sendInput, dismissPrompt, resize } = useTerminal({ sessionId, terminal: terminalInstance }) useEffect(() => { onTimelineActions?.(timelineActions) @@ -612,13 +612,13 @@ export function Terminal({ sessionId, sessionTitle, agentName, onTimelineActions {promptState.type === 'yesno' ? ( <>