From 7c072b275fed75b75798f4091e0b0823a7b8292a Mon Sep 17 00:00:00 2001 From: Brian Fioca Date: Wed, 8 Oct 2025 09:25:14 -0700 Subject: [PATCH 1/2] WIP GA migration --- package-lock.json | 403 +++++------------- package.json | 4 +- src/app/App.tsx | 6 +- src/app/agentConfigs/chatSupervisor/index.ts | 2 +- .../customerServiceRetail/authentication.ts | 2 +- .../customerServiceRetail/returns.ts | 2 +- .../customerServiceRetail/sales.ts | 2 +- .../customerServiceRetail/simulatedHuman.ts | 2 +- src/app/agentConfigs/simpleHandoff.ts | 4 +- src/app/api/session/route.ts | 24 +- src/app/hooks/useRealtimeSession.ts | 19 +- src/app/lib/codecUtils.ts | 21 +- 12 files changed, 154 insertions(+), 337 deletions(-) diff --git a/package-lock.json b/package-lock.json index 4ec9db381..db2d9ad86 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,11 +8,11 @@ "name": "realtime-examples", "version": "0.1.0", "dependencies": { - "@openai/agents": "^0.0.5", + "@openai/agents": "^0.1.9", "@radix-ui/react-icons": "^1.3.2", "dotenv": "^16.4.7", "next": "^15.3.1", - "openai": "^4.77.3", + "openai": "^6.2.0", "react": "^19.0.0", "react-dom": "^19.0.0", "react-markdown": "^9.0.3", @@ -764,9 +764,9 @@ } }, "node_modules/@modelcontextprotocol/sdk": { - "version": "1.12.1", - "resolved": "https://registry.npmjs.org/@modelcontextprotocol/sdk/-/sdk-1.12.1.tgz", - "integrity": "sha512-KG1CZhZfWg+u8pxeM/mByJDScJSrjjxLc8fwQqbsS8xCjBmQfMNEBTotYdNanKekepnfRI85GtgQlctLFpcYPw==", + "version": "1.19.1", + "resolved": "https://registry.npmjs.org/@modelcontextprotocol/sdk/-/sdk-1.19.1.tgz", + "integrity": "sha512-3Y2h3MZKjec1eAqSTBclATlX+AbC6n1LgfVzRMJLt3v6w0RCYgwLrjbxPDbhsYHt6Wdqc/aCceNJYgj448ELQQ==", "license": "MIT", "optional": true, "dependencies": { @@ -775,6 +775,7 @@ "cors": "^2.8.5", "cross-spawn": "^7.0.5", "eventsource": "^3.0.2", + "eventsource-parser": "^3.0.0", "express": "^5.0.1", "express-rate-limit": "^7.5.0", "pkce-challenge": "^5.0.0", @@ -979,30 +980,32 @@ } }, "node_modules/@openai/agents": { - "version": "0.0.5", - "resolved": "https://registry.npmjs.org/@openai/agents/-/agents-0.0.5.tgz", - "integrity": "sha512-baR5yTS+0D/mzomxBjvuxMkx9TshedJiY9qVvrMyqngIo1t6sB8d+aUYDnlLx/Pu3Q7TSqbW+xiowXSZD+ycZA==", + "version": "0.1.9", + "resolved": "https://registry.npmjs.org/@openai/agents/-/agents-0.1.9.tgz", + "integrity": "sha512-f0MLi8N9tVqJqxR3oJj7XmXq0roE+lllRaBG3MeMNP0edOC99+Le/24u3lM7HE6/FyqaCdrRuEbv9QEEf0RDkA==", "license": "MIT", "dependencies": { - "@openai/agents-core": "0.0.5", - "@openai/agents-openai": "0.0.5", - "@openai/agents-realtime": "0.0.5", + "@openai/agents-core": "0.1.8", + "@openai/agents-openai": "0.1.9", + "@openai/agents-realtime": "0.1.8", "debug": "^4.4.0", - "openai": "^5.0.1" + "openai": "^5.20.2" + }, + "peerDependencies": { + "zod": "^3.25.40" } }, "node_modules/@openai/agents-core": { - "version": "0.0.5", - "resolved": "https://registry.npmjs.org/@openai/agents-core/-/agents-core-0.0.5.tgz", - "integrity": "sha512-cyxeqN0NkYseAwtKGhaIHa5+h4tENjTLBpA3flqpeAJkeoVy6c38gx7iUNFQBoKZbt1xDIvKItY8dP40F00QkQ==", + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/@openai/agents-core/-/agents-core-0.1.8.tgz", + "integrity": "sha512-6tNFU/oR7EXzYyv6E6xuCVaVFr9ixO3zsPRGb1d+67obGN2V9QftbfXkYN8bobIaVgI7FD5i4d9VQQesL8Odkg==", "license": "MIT", "dependencies": { - "@openai/zod": "npm:zod@^3.25.40", "debug": "^4.4.0", - "openai": "^5.0.1" + "openai": "^5.20.2" }, "optionalDependencies": { - "@modelcontextprotocol/sdk": "^1.12.0" + "@modelcontextprotocol/sdk": "^1.17.2" }, "peerDependencies": { "zod": "^3.25.40" @@ -1014,9 +1017,9 @@ } }, "node_modules/@openai/agents-core/node_modules/openai": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/openai/-/openai-5.2.0.tgz", - "integrity": "sha512-b+Sf2Yk2eApDkhqHr7C4d5hux9gkHUvyqQ7RrdSfLsjrXkCZpJPqkME0u5Py7RPB28Ozz+RkJZpW7YPTOoChew==", + "version": "5.23.2", + "resolved": "https://registry.npmjs.org/openai/-/openai-5.23.2.tgz", + "integrity": "sha512-MQBzmTulj+MM5O8SKEk/gL8a7s5mktS9zUtAkU257WjvobGc9nKcBuVwjyEEcb9SI8a8Y2G/mzn3vm9n1Jlleg==", "license": "Apache-2.0", "bin": { "openai": "bin/cli" @@ -1035,21 +1038,23 @@ } }, "node_modules/@openai/agents-openai": { - "version": "0.0.5", - "resolved": "https://registry.npmjs.org/@openai/agents-openai/-/agents-openai-0.0.5.tgz", - "integrity": "sha512-z++6eI3XpfrYamN2+t2ZwXZzAKkim0fdMiRgCxs4zd4f5DR8Yhe/9wsV4CW41yQmZfMAdaLEk3pHRe/X0IRU+w==", + "version": "0.1.9", + "resolved": "https://registry.npmjs.org/@openai/agents-openai/-/agents-openai-0.1.9.tgz", + "integrity": "sha512-NNca4HSLDQ9J/xVM05+rzS6QvPN4bscZmZTIgPix1V5wFs8buiDd5sR1jFkEXZ3HppFhm2P8cndV2Ab46it2zw==", "license": "MIT", "dependencies": { - "@openai/agents-core": "0.0.5", - "@openai/zod": "npm:zod@^3.25.40", + "@openai/agents-core": "0.1.8", "debug": "^4.4.0", - "openai": "^5.0.1" + "openai": "^5.20.2" + }, + "peerDependencies": { + "zod": "^3.25.40" } }, "node_modules/@openai/agents-openai/node_modules/openai": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/openai/-/openai-5.2.0.tgz", - "integrity": "sha512-b+Sf2Yk2eApDkhqHr7C4d5hux9gkHUvyqQ7RrdSfLsjrXkCZpJPqkME0u5Py7RPB28Ozz+RkJZpW7YPTOoChew==", + "version": "5.23.2", + "resolved": "https://registry.npmjs.org/openai/-/openai-5.23.2.tgz", + "integrity": "sha512-MQBzmTulj+MM5O8SKEk/gL8a7s5mktS9zUtAkU257WjvobGc9nKcBuVwjyEEcb9SI8a8Y2G/mzn3vm9n1Jlleg==", "license": "Apache-2.0", "bin": { "openai": "bin/cli" @@ -1068,22 +1073,24 @@ } }, "node_modules/@openai/agents-realtime": { - "version": "0.0.5", - "resolved": "https://registry.npmjs.org/@openai/agents-realtime/-/agents-realtime-0.0.5.tgz", - "integrity": "sha512-RmID+/VnDCS5T8w/RFCP771Nqm2Ypbk5z/y0oRL3usjr7alVxibn45oWgSmDfBd09k+t168wMaO5K22s62nBGQ==", + "version": "0.1.8", + "resolved": "https://registry.npmjs.org/@openai/agents-realtime/-/agents-realtime-0.1.8.tgz", + "integrity": "sha512-i9+0G3P640l8LmBkDanJ+YvVTfMnQvLvjqj3h7TnVBjNiXBeFi1G5uaCae5e/xH+igciKgvS8VyQ/Mftel0BQg==", "license": "MIT", "dependencies": { - "@openai/agents-core": "0.0.5", - "@openai/zod": "npm:zod@^3.25.40", + "@openai/agents-core": "0.1.8", "@types/ws": "^8.18.1", "debug": "^4.4.0", "ws": "^8.18.1" + }, + "peerDependencies": { + "zod": "^3.25.40" } }, "node_modules/@openai/agents/node_modules/openai": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/openai/-/openai-5.1.1.tgz", - "integrity": "sha512-lgIdLqvpLpz8xPUKcEIV6ml+by74mbSBz8zv/AHHebtLn/WdpH4kdXT3/Q5uUKDHg3vHV/z9+G9wZINRX6rkDg==", + "version": "5.23.2", + "resolved": "https://registry.npmjs.org/openai/-/openai-5.23.2.tgz", + "integrity": "sha512-MQBzmTulj+MM5O8SKEk/gL8a7s5mktS9zUtAkU257WjvobGc9nKcBuVwjyEEcb9SI8a8Y2G/mzn3vm9n1Jlleg==", "license": "Apache-2.0", "bin": { "openai": "bin/cli" @@ -1101,16 +1108,6 @@ } } }, - "node_modules/@openai/zod": { - "name": "zod", - "version": "3.25.57", - "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.57.tgz", - "integrity": "sha512-6tgzLuwVST5oLUxXTmBqoinKMd3JeesgbgseXeFasKKj8Q1FCZrHnbqJOyiEvr4cVAlbug+CgIsmJ8cl/pU5FA==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/colinhacks" - } - }, "node_modules/@pkgjs/parseargs": { "version": "0.11.0", "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", @@ -1225,16 +1222,6 @@ "undici-types": "~6.21.0" } }, - "node_modules/@types/node-fetch": { - "version": "2.6.12", - "resolved": "https://registry.npmjs.org/@types/node-fetch/-/node-fetch-2.6.12.tgz", - "integrity": "sha512-8nneRWKCg3rMtF69nLQJnOYUcbafYeFSjqkw3jCRLsqkWFlHaoQrr5mXmofFGOx3DKn7UfmBMyov8ySvLRVldA==", - "license": "MIT", - "dependencies": { - "@types/node": "*", - "form-data": "^4.0.0" - } - }, "node_modules/@types/react": { "version": "19.1.6", "resolved": "https://registry.npmjs.org/@types/react/-/react-19.1.6.tgz", @@ -1576,18 +1563,6 @@ "darwin" ] }, - "node_modules/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", - "license": "MIT", - "dependencies": { - "event-target-shim": "^5.0.0" - }, - "engines": { - "node": ">=6.5" - } - }, "node_modules/accepts": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/accepts/-/accepts-2.0.0.tgz", @@ -1625,18 +1600,6 @@ "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, - "node_modules/agentkeepalive": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/agentkeepalive/-/agentkeepalive-4.6.0.tgz", - "integrity": "sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==", - "license": "MIT", - "dependencies": { - "humanize-ms": "^1.2.1" - }, - "engines": { - "node": ">= 8.0.0" - } - }, "node_modules/ajv": { "version": "6.12.6", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", @@ -1905,12 +1868,6 @@ "node": ">= 0.4" } }, - "node_modules/asynckit": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", - "license": "MIT" - }, "node_modules/available-typed-arrays": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.7.tgz", @@ -2055,6 +2012,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "devOptional": true, "license": "MIT", "dependencies": { "es-errors": "^1.3.0", @@ -2277,18 +2235,6 @@ "simple-swizzle": "^0.2.2" } }, - "node_modules/combined-stream": { - "version": "1.0.8", - "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", - "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "license": "MIT", - "dependencies": { - "delayed-stream": "~1.0.0" - }, - "engines": { - "node": ">= 0.8" - } - }, "node_modules/comma-separated-tokens": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-2.0.3.tgz", @@ -2541,15 +2487,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/delayed-stream": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", - "license": "MIT", - "engines": { - "node": ">=0.4.0" - } - }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -2635,6 +2572,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "devOptional": true, "license": "MIT", "dependencies": { "call-bind-apply-helpers": "^1.0.1", @@ -2749,6 +2687,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "devOptional": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -2758,6 +2697,7 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "devOptional": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -2795,6 +2735,7 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "devOptional": true, "license": "MIT", "dependencies": { "es-errors": "^1.3.0" @@ -2807,6 +2748,7 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "dev": true, "license": "MIT", "dependencies": { "es-errors": "^1.3.0", @@ -3315,15 +3257,6 @@ "node": ">= 0.6" } }, - "node_modules/event-target-shim": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", - "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, "node_modules/eventsource": { "version": "3.0.7", "resolved": "https://registry.npmjs.org/eventsource/-/eventsource-3.0.7.tgz", @@ -3338,9 +3271,9 @@ } }, "node_modules/eventsource-parser": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.0.2.tgz", - "integrity": "sha512-6RxOBZ/cYgd8usLwsEl+EC09Au/9BcmCKYF2/xbml6DNczf7nv0MQb+7BA2F+li6//I+28VNlQR37XfQtcAJuA==", + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.0.6.tgz", + "integrity": "sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg==", "license": "MIT", "optional": true, "engines": { @@ -3391,9 +3324,9 @@ } }, "node_modules/express-rate-limit": { - "version": "7.5.0", - "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-7.5.0.tgz", - "integrity": "sha512-eB5zbQh5h+VenMPM3fh+nw1YExi5nMr6HUCR62ELSP11huvxm/Uir1H1QEyTkk5QX6A58pX6NmaTMceKZ0Eodg==", + "version": "7.5.1", + "resolved": "https://registry.npmjs.org/express-rate-limit/-/express-rate-limit-7.5.1.tgz", + "integrity": "sha512-7iN8iPMDzOMHPUYllBEsQdWVB6fPDMPqwjBaFrgr4Jgr/+okjvzAy+UHlYYL/Vs0OsOrMkwS6PJDkFlJwoxUnw==", "license": "MIT", "optional": true, "engines": { @@ -3403,7 +3336,7 @@ "url": "https://github.com/sponsors/express-rate-limit" }, "peerDependencies": { - "express": "^4.11 || 5 || ^5.0.0-beta.1" + "express": ">= 4.11" } }, "node_modules/extend": { @@ -3588,62 +3521,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/form-data": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.4.tgz", - "integrity": "sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==", - "license": "MIT", - "dependencies": { - "asynckit": "^0.4.0", - "combined-stream": "^1.0.8", - "es-set-tostringtag": "^2.1.0", - "hasown": "^2.0.2", - "mime-types": "^2.1.12" - }, - "engines": { - "node": ">= 6" - } - }, - "node_modules/form-data-encoder": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/form-data-encoder/-/form-data-encoder-1.7.2.tgz", - "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==", - "license": "MIT" - }, - "node_modules/form-data/node_modules/mime-db": { - "version": "1.52.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", - "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", - "license": "MIT", - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/form-data/node_modules/mime-types": { - "version": "2.1.35", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", - "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", - "license": "MIT", - "dependencies": { - "mime-db": "1.52.0" - }, - "engines": { - "node": ">= 0.6" - } - }, - "node_modules/formdata-node": { - "version": "4.4.1", - "resolved": "https://registry.npmjs.org/formdata-node/-/formdata-node-4.4.1.tgz", - "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==", - "license": "MIT", - "dependencies": { - "node-domexception": "1.0.0", - "web-streams-polyfill": "4.0.0-beta.3" - }, - "engines": { - "node": ">= 12.20" - } - }, "node_modules/forwarded": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", @@ -3668,6 +3545,7 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "devOptional": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/ljharb" @@ -3708,6 +3586,7 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "devOptional": true, "license": "MIT", "dependencies": { "call-bind-apply-helpers": "^1.0.2", @@ -3732,6 +3611,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "devOptional": true, "license": "MIT", "dependencies": { "dunder-proto": "^1.0.1", @@ -3866,6 +3746,7 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "devOptional": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -3937,6 +3818,7 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "devOptional": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -3949,6 +3831,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "dev": true, "license": "MIT", "dependencies": { "has-symbols": "^1.0.3" @@ -3964,6 +3847,7 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "devOptional": true, "license": "MIT", "dependencies": { "function-bind": "^1.1.2" @@ -4049,15 +3933,6 @@ "node": ">= 0.8" } }, - "node_modules/humanize-ms": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz", - "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", - "license": "MIT", - "dependencies": { - "ms": "^2.0.0" - } - }, "node_modules/iconv-lite": { "version": "0.6.3", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", @@ -4889,6 +4764,7 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "devOptional": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -5741,46 +5617,6 @@ "node": "^10 || ^12 || >=14" } }, - "node_modules/node-domexception": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", - "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", - "deprecated": "Use your platform's native DOMException instead", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/jimmywarting" - }, - { - "type": "github", - "url": "https://paypal.me/jimmywarting" - } - ], - "license": "MIT", - "engines": { - "node": ">=10.5.0" - } - }, - "node_modules/node-fetch": { - "version": "2.7.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz", - "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", - "license": "MIT", - "dependencies": { - "whatwg-url": "^5.0.0" - }, - "engines": { - "node": "4.x || >=6.0.0" - }, - "peerDependencies": { - "encoding": "^0.1.0" - }, - "peerDependenciesMeta": { - "encoding": { - "optional": true - } - } - }, "node_modules/normalize-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", @@ -5948,25 +5784,16 @@ } }, "node_modules/openai": { - "version": "4.104.0", - "resolved": "https://registry.npmjs.org/openai/-/openai-4.104.0.tgz", - "integrity": "sha512-p99EFNsA/yX6UhVO93f5kJsDRLAg+CTA2RBqdHK4RtK8u5IJw32Hyb2dTGKbnnFmnuoBv5r7Z2CURI9sGZpSuA==", + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/openai/-/openai-6.2.0.tgz", + "integrity": "sha512-qqjzHls7F5xkXNGy9P1Ei1rorI5LWupUUFWP66zPU8FlZbiITX8SFcHMKNZg/NATJ0LpIZcMUFxSwQmdeQPwSw==", "license": "Apache-2.0", - "dependencies": { - "@types/node": "^18.11.18", - "@types/node-fetch": "^2.6.4", - "abort-controller": "^3.0.0", - "agentkeepalive": "^4.2.1", - "form-data-encoder": "1.7.2", - "formdata-node": "^4.3.2", - "node-fetch": "^2.6.7" - }, "bin": { "openai": "bin/cli" }, "peerDependencies": { "ws": "^8.18.0", - "zod": "^3.23.8" + "zod": "^3.25 || ^4.0" }, "peerDependenciesMeta": { "ws": { @@ -5977,21 +5804,6 @@ } } }, - "node_modules/openai/node_modules/@types/node": { - "version": "18.19.111", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.111.tgz", - "integrity": "sha512-90sGdgA+QLJr1F9X79tQuEut0gEYIfkX9pydI4XGRgvFo9g2JWswefI+WUSUHPYVBHYSEfTEqBxA5hQvAZB3Mw==", - "license": "MIT", - "dependencies": { - "undici-types": "~5.26.4" - } - }, - "node_modules/openai/node_modules/undici-types": { - "version": "5.26.5", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", - "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", - "license": "MIT" - }, "node_modules/optionator": { "version": "0.9.4", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.9.4.tgz", @@ -6160,13 +5972,14 @@ } }, "node_modules/path-to-regexp": { - "version": "8.2.0", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.2.0.tgz", - "integrity": "sha512-TdrF7fW9Rphjq4RjrW0Kp2AW0Ahwu9sRGTkS6bvDi0SCwZlEZYmcfDbEsTz8RVk0EHIS/Vd1bv3JhG+1xZuAyQ==", + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.3.0.tgz", + "integrity": "sha512-7jdwVIRtsP8MYpdXSwOS0YdD0Du+qOoF/AEPIt88PcCFrZCzx41oxku1jD88hZBwbNUIEfpqvuhjFaMAqMTWnA==", "license": "MIT", "optional": true, - "engines": { - "node": ">=16" + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/picocolors": { @@ -6446,19 +6259,36 @@ } }, "node_modules/raw-body": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.0.tgz", - "integrity": "sha512-RmkhL8CAyCRPXCE28MMH0z2PNWQBNk2Q09ZdxM9IOOXwxwZbN+qbWaatPkdkWIKL2ZVDImrN/pK5HTRz2PcS4g==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.1.tgz", + "integrity": "sha512-9G8cA+tuMS75+6G/TzW8OtLzmBDMo8p1JRxN5AZ+LAp8uxGA8V8GZm4GQ4/N5QNQEnLmg6SS7wyuSmbKepiKqA==", "license": "MIT", "optional": true, "dependencies": { "bytes": "3.1.2", "http-errors": "2.0.0", - "iconv-lite": "0.6.3", + "iconv-lite": "0.7.0", "unpipe": "1.0.0" }, "engines": { - "node": ">= 0.8" + "node": ">= 0.10" + } + }, + "node_modules/raw-body/node_modules/iconv-lite": { + "version": "0.7.0", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.0.tgz", + "integrity": "sha512-cf6L2Ds3h57VVmkZe+Pn+5APsT7FpqJtEhhieDCvrE2MK5Qk9MyffgQyuxQTm6BChfeZNtcOLHp9IcWRVcIcBQ==", + "license": "MIT", + "optional": true, + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" } }, "node_modules/react": { @@ -7660,12 +7490,6 @@ "node": ">=0.6" } }, - "node_modules/tr46": { - "version": "0.0.3", - "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", - "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", - "license": "MIT" - }, "node_modules/trim-lines": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/trim-lines/-/trim-lines-3.0.1.tgz", @@ -8068,31 +7892,6 @@ "url": "https://opencollective.com/unified" } }, - "node_modules/web-streams-polyfill": { - "version": "4.0.0-beta.3", - "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz", - "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==", - "license": "MIT", - "engines": { - "node": ">= 14" - } - }, - "node_modules/webidl-conversions": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz", - "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", - "license": "BSD-2-Clause" - }, - "node_modules/whatwg-url": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz", - "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", - "license": "MIT", - "dependencies": { - "tr46": "~0.0.3", - "webidl-conversions": "^3.0.0" - } - }, "node_modules/which": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", @@ -8367,9 +8166,9 @@ } }, "node_modules/zod-to-json-schema": { - "version": "3.24.5", - "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.24.5.tgz", - "integrity": "sha512-/AuWwMP+YqiPbsJx5D6TfgRTc4kTLjsh5SOcd4bLsfUg2RcEXrFMJl1DGgdHy2aCfsIA/cr/1JM0xcB2GZji8g==", + "version": "3.24.6", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.24.6.tgz", + "integrity": "sha512-h/z3PKvcTcTetyjl1fkj79MHNEjm+HpD6NXheWjzOekY7kV+lwDYnHw+ivHkijnCSMz1yJaWBD9vu/Fcmk+vEg==", "license": "ISC", "optional": true, "peerDependencies": { diff --git a/package.json b/package.json index 7e8675313..8dd233354 100644 --- a/package.json +++ b/package.json @@ -9,11 +9,11 @@ "lint": "next lint" }, "dependencies": { - "@openai/agents": "^0.0.5", + "@openai/agents": "^0.1.9", "@radix-ui/react-icons": "^1.3.2", "dotenv": "^16.4.7", "next": "^15.3.1", - "openai": "^4.77.3", + "openai": "^6.2.0", "react": "^19.0.0", "react-dom": "^19.0.0", "react-markdown": "^9.0.3", diff --git a/src/app/App.tsx b/src/app/App.tsx index 66fae55ef..adc87f166 100644 --- a/src/app/App.tsx +++ b/src/app/App.tsx @@ -273,7 +273,11 @@ function App() { sendEvent({ type: 'session.update', session: { - turn_detection: turnDetection, + audio: { + input: { + turn_detection: turnDetection, + }, + }, }, }); diff --git a/src/app/agentConfigs/chatSupervisor/index.ts b/src/app/agentConfigs/chatSupervisor/index.ts index 968a391eb..d92fb4897 100644 --- a/src/app/agentConfigs/chatSupervisor/index.ts +++ b/src/app/agentConfigs/chatSupervisor/index.ts @@ -3,7 +3,7 @@ import { getNextResponseFromSupervisor } from './supervisorAgent'; export const chatAgent = new RealtimeAgent({ name: 'chatAgent', - voice: 'sage', + voice: 'marin', instructions: ` You are a helpful junior customer service agent. Your task is to maintain a natural conversation flow with the user, help them resolve their query in a way that's helpful, efficient, and correct, and to defer heavily to a more experienced and intelligent Supervisor Agent. diff --git a/src/app/agentConfigs/customerServiceRetail/authentication.ts b/src/app/agentConfigs/customerServiceRetail/authentication.ts index e0767d096..b2c4170cb 100644 --- a/src/app/agentConfigs/customerServiceRetail/authentication.ts +++ b/src/app/agentConfigs/customerServiceRetail/authentication.ts @@ -2,7 +2,7 @@ import { RealtimeAgent, tool } from '@openai/agents/realtime'; export const authenticationAgent = new RealtimeAgent({ name: 'authentication', - voice: 'sage', + voice: 'marin', handoffDescription: 'The initial agent that greets the user, does authentication and routes them to the correct downstream agent.', diff --git a/src/app/agentConfigs/customerServiceRetail/returns.ts b/src/app/agentConfigs/customerServiceRetail/returns.ts index 4349265bf..de48c570f 100644 --- a/src/app/agentConfigs/customerServiceRetail/returns.ts +++ b/src/app/agentConfigs/customerServiceRetail/returns.ts @@ -2,7 +2,7 @@ import { RealtimeAgent, tool, RealtimeItem } from '@openai/agents/realtime'; export const returnsAgent = new RealtimeAgent({ name: 'returns', - voice: 'sage', + voice: 'marin', handoffDescription: 'Customer Service Agent specialized in order lookups, policy checks, and return initiations.', diff --git a/src/app/agentConfigs/customerServiceRetail/sales.ts b/src/app/agentConfigs/customerServiceRetail/sales.ts index e5e2b7712..60c6b1227 100644 --- a/src/app/agentConfigs/customerServiceRetail/sales.ts +++ b/src/app/agentConfigs/customerServiceRetail/sales.ts @@ -2,7 +2,7 @@ import { RealtimeAgent, tool } from '@openai/agents/realtime'; export const salesAgent = new RealtimeAgent({ name: 'salesAgent', - voice: 'sage', + voice: 'marin', handoffDescription: "Handles sales-related inquiries, including new product details, recommendations, promotions, and purchase flows. Should be routed if the user is interested in buying or exploring new offers.", diff --git a/src/app/agentConfigs/customerServiceRetail/simulatedHuman.ts b/src/app/agentConfigs/customerServiceRetail/simulatedHuman.ts index 5b0326ca9..1a688aa7a 100644 --- a/src/app/agentConfigs/customerServiceRetail/simulatedHuman.ts +++ b/src/app/agentConfigs/customerServiceRetail/simulatedHuman.ts @@ -2,7 +2,7 @@ import { RealtimeAgent } from '@openai/agents/realtime'; export const simulatedHumanAgent = new RealtimeAgent({ name: 'simulatedHuman', - voice: 'sage', + voice: 'marin', handoffDescription: 'Placeholder, simulated human agent that can provide more advanced help to the user. Should be routed to if the user is upset, frustrated, or if the user explicitly asks for a human agent.', instructions: diff --git a/src/app/agentConfigs/simpleHandoff.ts b/src/app/agentConfigs/simpleHandoff.ts index 3498ca187..355de8ead 100644 --- a/src/app/agentConfigs/simpleHandoff.ts +++ b/src/app/agentConfigs/simpleHandoff.ts @@ -4,7 +4,7 @@ import { export const haikuWriterAgent = new RealtimeAgent({ name: 'haikuWriter', - voice: 'sage', + voice: 'marin', instructions: 'Ask the user for a topic, then reply with a haiku about that topic.', handoffs: [], @@ -14,7 +14,7 @@ export const haikuWriterAgent = new RealtimeAgent({ export const greeterAgent = new RealtimeAgent({ name: 'greeter', - voice: 'sage', + voice: 'marin', instructions: "Please greet the user and ask them if they'd like a Haiku. If yes, hand off to the 'haiku' agent.", handoffs: [haikuWriterAgent], diff --git a/src/app/api/session/route.ts b/src/app/api/session/route.ts index 6e5d15891..1475b3918 100644 --- a/src/app/api/session/route.ts +++ b/src/app/api/session/route.ts @@ -1,21 +1,19 @@ import { NextResponse } from "next/server"; +import OpenAI from "openai"; + +const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY }); export async function GET() { try { - const response = await fetch( - "https://api.openai.com/v1/realtime/sessions", - { - method: "POST", - headers: { - Authorization: `Bearer ${process.env.OPENAI_API_KEY}`, - "Content-Type": "application/json", + const data = await openai.realtime.clientSecrets.create({ + model: "gpt-realtime", + audio: { + output: { + voice: "marin", }, - body: JSON.stringify({ - model: "gpt-4o-realtime-preview-2025-06-03", - }), - } - ); - const data = await response.json(); + }, + }); + return NextResponse.json(data); } catch (error) { console.error("Error in /session:", error); diff --git a/src/app/hooks/useRealtimeSession.ts b/src/app/hooks/useRealtimeSession.ts index 3329f0c61..d30510fbc 100644 --- a/src/app/hooks/useRealtimeSession.ts +++ b/src/app/hooks/useRealtimeSession.ts @@ -50,10 +50,12 @@ export function useRealtimeSession(callbacks: RealtimeSessionCallbacks = {}) { historyHandlers.handleTranscriptionCompleted(event); break; } + case "response.output_audio_transcript.done": case "response.audio_transcript.done": { historyHandlers.handleTranscriptionCompleted(event); break; } + case "response.output_audio_transcript.delta": case "response.audio_transcript.delta": { historyHandlers.handleTranscriptionDelta(event); break; @@ -137,12 +139,19 @@ export function useRealtimeSession(callbacks: RealtimeSessionCallbacks = {}) { return pc; }, }), - model: 'gpt-4o-realtime-preview-2025-06-03', + model: 'gpt-realtime', config: { - inputAudioFormat: audioFormat, - outputAudioFormat: audioFormat, - inputAudioTranscription: { - model: 'gpt-4o-mini-transcribe', + outputModalities: ['audio'], + audio: { + input: { + format: audioFormat, + transcription: { + model: 'gpt-4o-mini-transcribe', + }, + }, + output: { + format: audioFormat, + }, }, }, outputGuardrails: outputGuardrails ?? [], diff --git a/src/app/lib/codecUtils.ts b/src/app/lib/codecUtils.ts index 226a5d4ae..b257019b9 100644 --- a/src/app/lib/codecUtils.ts +++ b/src/app/lib/codecUtils.ts @@ -1,11 +1,18 @@ -export function audioFormatForCodec(codec: string): 'pcm16' | 'g711_ulaw' | 'g711_alaw' { - let audioFormat: 'pcm16' | 'g711_ulaw' | 'g711_alaw' = 'pcm16'; - if (typeof window !== 'undefined') { - const c = codec.toLowerCase(); - if (c === 'pcmu') audioFormat = 'g711_ulaw'; - else if (c === 'pcma') audioFormat = 'g711_alaw'; +import type { RealtimeAudioFormat } from '@openai/agents/realtime'; + +export function audioFormatForCodec(codec: string): RealtimeAudioFormat { + const normalized = typeof codec === 'string' ? codec.toLowerCase() : 'opus'; + + if (normalized === 'pcmu') { + return { type: 'audio/pcmu' }; } - return audioFormat; + + if (normalized === 'pcma') { + return { type: 'audio/pcma' }; + } + + // Default to wideband PCM for Opus or any other codec + return { type: 'audio/pcm', rate: 24000 }; } // Apply preferred codec on a peer connection's audio transceivers. Safe to call multiple times. From 33459a998ee0db60bcee0b5756420556572e80f9 Mon Sep 17 00:00:00 2001 From: Brian Fioca Date: Wed, 8 Oct 2025 09:45:25 -0700 Subject: [PATCH 2/2] Fixing migration issues --- src/app/App.tsx | 34 ++++++++++++++-------------------- src/app/api/session/route.ts | 27 ++++++++++++++++++++++----- 2 files changed, 36 insertions(+), 25 deletions(-) diff --git a/src/app/App.tsx b/src/app/App.tsx index adc87f166..68027c554 100644 --- a/src/app/App.tsx +++ b/src/app/App.tsx @@ -273,6 +273,7 @@ function App() { sendEvent({ type: 'session.update', session: { + type: 'realtime', audio: { input: { turn_detection: turnDetection, @@ -306,6 +307,7 @@ function App() { interrupt(); setIsPTTUserSpeaking(true); + mute(false); sendClientEvent({ type: 'input_audio_buffer.clear' }, 'clear PTT buffer'); // No placeholder; we'll rely on server transcript once ready. @@ -318,6 +320,8 @@ function App() { setIsPTTUserSpeaking(false); sendClientEvent({ type: 'input_audio_buffer.commit' }, 'commit PTT'); sendClientEvent({ type: 'response.create' }, 'trigger response PTT'); + sendClientEvent({ type: 'input_audio_buffer.clear' }, 'reset buffer after PTT'); + mute(true); }; const onToggleConnection = () => { @@ -386,6 +390,16 @@ function App() { ); }, [isAudioPlaybackEnabled]); + useEffect(() => { + if (sessionStatus !== 'CONNECTED') return; + + const shouldMuteMic = isPTTActive + ? !isPTTUserSpeaking + : !isAudioPlaybackEnabled; + + mute(shouldMuteMic); + }, [sessionStatus, isPTTActive, isPTTUserSpeaking, isAudioPlaybackEnabled, mute]); + useEffect(() => { if (audioElementRef.current) { if (isAudioPlaybackEnabled) { @@ -399,28 +413,8 @@ function App() { audioElementRef.current.pause(); } } - - // Toggle server-side audio stream mute so bandwidth is saved when the - // user disables playback. - try { - mute(!isAudioPlaybackEnabled); - } catch (err) { - console.warn('Failed to toggle SDK mute', err); - } }, [isAudioPlaybackEnabled]); - // Ensure mute state is propagated to transport right after we connect or - // whenever the SDK client reference becomes available. - useEffect(() => { - if (sessionStatus === 'CONNECTED') { - try { - mute(!isAudioPlaybackEnabled); - } catch (err) { - console.warn('mute sync after connect failed', err); - } - } - }, [sessionStatus, isAudioPlaybackEnabled]); - useEffect(() => { if (sessionStatus === "CONNECTED" && audioElementRef.current?.srcObject) { // The remote audio stream from the audio element. diff --git a/src/app/api/session/route.ts b/src/app/api/session/route.ts index 1475b3918..b3853dc78 100644 --- a/src/app/api/session/route.ts +++ b/src/app/api/session/route.ts @@ -6,15 +6,32 @@ const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY }); export async function GET() { try { const data = await openai.realtime.clientSecrets.create({ - model: "gpt-realtime", - audio: { - output: { - voice: "marin", + session: { + type: "realtime", + model: "gpt-realtime", + output_modalities: ["audio"], + audio: { + input: { + format: { type: "audio/pcm", rate: 24000 }, + transcription: { + model: "gpt-4o-mini-transcribe", + }, + }, + output: { + format: { type: "audio/pcm", rate: 24000 }, + voice: "marin", + }, }, }, }); - return NextResponse.json(data); + return NextResponse.json({ + client_secret: { + value: data.value, + expires_at: data.expires_at, + }, + session: data.session, + }); } catch (error) { console.error("Error in /session:", error); return NextResponse.json(