Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
91 changes: 91 additions & 0 deletions realtime/app.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
class App {
constructor(prefix, prefs = [], controls = {}, sessionType = "standard") {
this.prefix = prefix;
this.prefs = prefs;
this.controls = controls;
this.sessionType = sessionType;
this.session = null;
}

initState() {
this.prefs.forEach(p => {
const fqid = p.id !== "openai-api-key" ? this.prefix + p.id : p.id;
const v = localStorage.getItem(fqid);
if (v) {
p.value = v;
}
p.addEventListener("change", () => {
localStorage.setItem(fqid, p.value);
});
});
this.updateState(false);
}

updateState(started) {
if (this.controls.statusEl) {
this.controls.statusEl.textContent = "";
}
this.prefs.forEach(p => (p.disabled = started));
if (this.controls.startBtn) this.controls.startBtn.disabled = started;
if (this.controls.stopBtn) this.controls.stopBtn.disabled = !started;
}

getApiKey() {
const el = document.getElementById("openai-api-key");
return el ? el.value : null;
}

async startMicrophone() {
if (!this.getApiKey()) {
window.alert(
"Please enter your OpenAI API Key. You can obtain one from https://platform.openai.com/settings/organization/api-keys"
);
return;
}
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
await this.start(stream);
}

async start(stream) {
this.updateState(true);
this.session = new Session(this.getApiKey());
this.session.onconnectionstatechange = state => this.onConnectionStateChange(state);
this.session.ontrack = e => this.onTrack(e);
this.session.onopen = () => this.onOpen();
this.session.onmessage = msg => this.onMessage(msg);
this.session.onerror = e => this.onError(e);
const config = this.buildSessionConfig();
if (this.sessionType === "transcription") {
await this.session.startTranscription(stream, config);
} else {
await this.session.start(stream, config);
}
}

stop() {
this.updateState(false);
if (this.session) {
this.session.stop();
this.session = null;
}
}

sendMessage(msg) {
this.session?.sendMessage(msg);
}

mute(muted) {
this.session?.mute(muted);
}

// Hooks for subclasses
buildSessionConfig() { return {}; }
onTrack(_e) {}
onOpen() {}
onMessage(_msg) {}
onError(e) { console.error(e); this.stop(); }
onConnectionStateChange(state) {
if (this.controls.statusEl) this.controls.statusEl.textContent = state;
}
}

1 change: 1 addition & 0 deletions realtime/basic/index.html
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@
<p class="github-link"><a href="https://github.com/juberti/demos/tree/main/realtime/basic">GitHub</a></p>
</div>
<script src="../session.js"></script>
<script src="../app.js"></script>
<script src="main.js"></script>
</body>
</html>
114 changes: 36 additions & 78 deletions realtime/basic/main.js
Original file line number Diff line number Diff line change
@@ -1,86 +1,44 @@
const APP_PREFIX = "realtime/basic/";
const $ = document.querySelector.bind(document);
const apiKeyEl = $("#openai-api-key");
const modelEl = $("#model");
const voiceEl = $("#voice");
const instructionsEl = $("#instructions");
const startMicrophoneEl = $("#start-microphone");
const stopEl = $("#stop");
const statusEl = $("#status");
const prefs = [apiKeyEl, modelEl, voiceEl, instructionsEl];

let session = null;

function initState() {
prefs.forEach(p => {
const fqid = p.id != "openai-api-key" ? APP_PREFIX + p.id : p.id;
const v = localStorage.getItem(fqid);
if (v) {
p.value = v;
}
p.addEventListener("change", () => {
localStorage.setItem(fqid, p.value);
});
});
updateState(false);
}

function updateState(started) {
statusEl.textContent = "";
prefs.forEach(p => p.disabled = started);
startMicrophoneEl.disabled = started;
stopEl.disabled = !started;
}

async function startMicrophone() {
if (!apiKeyEl.value) {
window.alert("Please enter your OpenAI API Key. You can obtain one from https://platform.openai.com/settings/organization/api-keys");
return;
class BasicApp extends App {
constructor() {
const $ = document.querySelector.bind(document);
const apiKeyEl = $("#openai-api-key");
const modelEl = $("#model");
const voiceEl = $("#voice");
const instructionsEl = $("#instructions");
const startBtn = $("#start-microphone");
const stopBtn = $("#stop");
const statusEl = $("#status");
const prefs = [apiKeyEl, modelEl, voiceEl, instructionsEl];
super("realtime/basic/", prefs, { startBtn, stopBtn, statusEl });
this.modelEl = modelEl;
this.voiceEl = voiceEl;
this.instructionsEl = instructionsEl;
this.initState();
}
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
start(stream);
}

async function start(stream) {
updateState(true);
session = new Session(apiKeyEl.value);
session.onconnectionstatechange = state => statusEl.textContent = state;
session.ontrack = e => handleTrack(e);
session.onopen = e => handleOpen();
session.onmessage = e => handleMessage(e);
session.onerror = e => handleError(e);
const sessionConfig = {
model: modelEl.value,
voice: voiceEl.value,
instructions: instructionsEl.value || undefined
buildSessionConfig() {
return {
model: this.modelEl.value,
voice: this.voiceEl.value,
instructions: this.instructionsEl.value || undefined,
};
}
await session.start(stream, sessionConfig);
}

function stop() {
updateState(false);
session.stop();
session = null;
}

function handleTrack(e) {
const audio = new Audio();
audio.srcObject = e.streams[0];
audio.play();
}
onTrack(e) {
const audio = new Audio();
audio.srcObject = e.streams[0];
audio.play();
}

function handleOpen() {
const message = { type: "response.create" };
session.sendMessage(message);
}
onOpen() {
this.sendMessage({ type: "response.create" });
}

function handleMessage(message) {
console.log("message", message);
onMessage(message) {
console.log("message", message);
}
}

function handleError(e) {
console.error(e);
stop();
}

initState();
const app = new BasicApp();
window.startMicrophone = () => app.startMicrophone();
window.stop = () => app.stop();
9 changes: 5 additions & 4 deletions realtime/imager/index.html
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,9 @@
<p class="github-link"><a href="https://github.com/juberti/demos/tree/main/realtime/imager">GitHub</a></p>
</div>
</div>
<script src="../session.js"></script>
<script src="../utils.js"></script>
<script src="main.js"></script>
</body>
<script src="../session.js"></script>
<script src="../app.js"></script>
<script src="../utils.js"></script>
<script src="main.js"></script>
</body>
</html>
Loading