From 180fa3abd7969626bac9a7292b493a4097c43233 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Tue, 10 Mar 2026 03:14:51 +0000
Subject: [PATCH 1/2] Initial plan


From 3f37e3f4bbcfc75d59f4d1b0b334737e9a55bf1f Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Tue, 10 Mar 2026 03:22:05 +0000
Subject: [PATCH 2/2] Fix Realtime API: correct model, remove beta header,
 improve error surfacing

Co-authored-by: hack-r <7491078+hack-r@users.noreply.github.com>
---
 .env.example                     |  4 ++--
 server/index.ts                  | 19 ++++++++++++++++---
 src/services/providers/openai.ts | 27 +++++++++++++++++++++------
 3 files changed, 39 insertions(+), 11 deletions(-)

diff --git a/.env.example b/.env.example
index ee9c566..1a865e6 100644
--- a/.env.example
+++ b/.env.example
@@ -4,8 +4,8 @@ OPENAI_API_KEY=sk-your-api-key-here
 # Server port (optional, default: 3001)
 PORT=3001
 
-# OpenAI Realtime model (optional, default: gpt-realtime-1.5)
-OPENAI_MODEL=gpt-realtime-1.5
+# OpenAI Realtime model (optional, default: gpt-4o-realtime-preview)
+OPENAI_MODEL=gpt-4o-realtime-preview
 
 # Custom system prompt (optional – overrides the built-in HUD assistant prompt)
 # HUD_SYSTEM_PROMPT=You are a custom HUD assistant...
diff --git a/server/index.ts b/server/index.ts
index 077d7c9..8413823 100644
--- a/server/index.ts
+++ b/server/index.ts
@@ -11,7 +11,7 @@ dotenv.config();
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 const PORT = parseInt(process.env.PORT ?? '3001', 10);
 const OPENAI_API_KEY = process.env.OPENAI_API_KEY ?? '';
-const OPENAI_MODEL = process.env.OPENAI_MODEL ?? 'gpt-realtime-1.5';
+const OPENAI_MODEL = process.env.OPENAI_MODEL ?? 'gpt-4o-realtime-preview';
 const HUD_SYSTEM_PROMPT = process.env.HUD_SYSTEM_PROMPT ?? '';
 const IS_PRODUCTION = process.env.NODE_ENV === 'production';
 
@@ -75,7 +75,6 @@ wss.on('connection', (clientWs) => {
     {
       headers: {
         Authorization: `Bearer ${OPENAI_API_KEY}`,
-        'OpenAI-Beta': 'realtime=v1',
       },
     }
   );
@@ -123,8 +122,22 @@ wss.on('connection', (clientWs) => {
   });
 
   openaiWs.on('close', (code, reason) => {
-    console.log(`[server] OpenAI WebSocket closed: ${code} ${reason.toString()}`);
+    const reasonStr = reason.toString();
+    const isAbnormal = code !== 1000 && code !== 1001;
+    if (isAbnormal || reasonStr) {
+      console.error(`[server] OpenAI WebSocket closed: code=${code} reason=${reasonStr || '(none)'}`);
+    } else {
+      console.log(`[server] OpenAI WebSocket closed: code=${code}`);
+    }
     if (clientWs.readyState === WebSocket.OPEN) {
+      if (isAbnormal) {
+        clientWs.send(
+          JSON.stringify({
+            type: 'error',
+            error: { message: `OpenAI session closed unexpectedly (code ${code})${reasonStr ? ': ' + reasonStr : ''}` },
+          })
+        );
+      }
       clientWs.close();
     }
   });
diff --git a/src/services/providers/openai.ts b/src/services/providers/openai.ts
index 7c07ed0..2a9dbc4 100644
--- a/src/services/providers/openai.ts
+++ b/src/services/providers/openai.ts
@@ -19,7 +19,7 @@ Guidelines:
 - Flag potential errors, warnings, or important details you notice`;
 
 /**
- * OpenAI Realtime API provider using gpt-realtime-1.5.
+ * OpenAI Realtime API provider using gpt-4o-realtime-preview.
  * Communicates via the backend WebSocket proxy at /api/realtime.
  */
 export class OpenAIRealtimeProvider implements AIProvider {
@@ -63,7 +63,7 @@ export class OpenAIRealtimeProvider implements AIProvider {
       this.ws = new WebSocket(wsUrl);
 
       this.ws.onopen = () => {
-        this.configureSession();
+        console.log('[OpenAI Realtime] WebSocket connected, waiting for session.created...');
       };
 
       this.ws.onmessage = (event) => {
@@ -82,8 +82,18 @@ export class OpenAIRealtimeProvider implements AIProvider {
         reject(new Error(msg));
       };
 
-      this.ws.onclose = () => {
-        if (this._state !== 'idle') {
+      this.ws.onclose = (event) => {
+        // Surface any close reason so errors from OpenAI are visible
+        if (event.reason) {
+          console.error('[OpenAI Realtime] WebSocket closed with reason:', event.reason, 'code:', event.code);
+        }
+        if (this._state === 'connecting') {
+          const reason = event.reason ? `: ${event.reason}` : '';
+          const msg = `Connection closed before session was established (code ${event.code})${reason}`;
+          this.setState('error');
+          events.onError(msg);
+          reject(new Error(msg));
+        } else if (this._state !== 'idle') {
           this.setState('idle');
           events.onDisconnected();
         }
@@ -124,7 +134,7 @@ export class OpenAIRealtimeProvider implements AIProvider {
           },
           {
             type: 'input_image',
-            image: `data:image/jpeg;base64,${base64Jpeg}`,
+            image_url: `data:image/jpeg;base64,${base64Jpeg}`,
             detail: 'auto',
           },
         ],
@@ -183,7 +193,12 @@ export class OpenAIRealtimeProvider implements AIProvider {
   private handleEvent(event: RealtimeEvent, resolveConnect?: (value: void) => void): void {
     switch (event.type) {
       case 'session.created':
+        // Initial session from OpenAI – now send our configuration
+        this.configureSession();
+        break;
+
       case 'session.updated':
+        // Our session.update was applied; session is fully ready
         this.sessionConfigured = true;
         this.setState('active');
         this.events?.onConnected();
@@ -218,7 +233,7 @@ export class OpenAIRealtimeProvider implements AIProvider {
 
       case 'error': {
         const errMsg = (event.error as { message?: string })?.message ?? 'Unknown error';
-        console.error('[OpenAI Realtime] Error:', errMsg);
+        console.error('[OpenAI Realtime] Error event:', errMsg, event);
         this.events?.onError(errMsg);
         break;
       }