From f95efedc47d0447c4fb071b026cd8eb8a395a000 Mon Sep 17 00:00:00 2001 From: john Date: Tue, 10 Dec 2024 23:44:49 +0000 Subject: [PATCH 01/25] Added in some of the changes for momentary buttons. --- src/ButtonModule.js | 113 +++++++++++++++++++++++++++++++++++++- src/events/EventModule.js | 6 ++ 2 files changed, 116 insertions(+), 3 deletions(-) diff --git a/src/ButtonModule.js b/src/ButtonModule.js index 619b8d53fd..9142022992 100644 --- a/src/ButtonModule.js +++ b/src/ButtonModule.js @@ -349,6 +349,36 @@ class ButtonModule { this.callIsActive = isActive; } + updateCallButtonNEW(callButton, svgIcon, label, onClick, isActive = false) { + if (!callButton) { + callButton = document.getElementById("saypi-callButton"); + } + if (callButton) { + this.removeChildNodesFrom(callButton); + this.addIconTo(callButton, svgIcon); + callButton.onmousedown = null; + callButton.onmouseup = null; + callButton.onClick = onClick; + this.toggleActiveState(callButton, isActive); + } + this.callIsActive = isActive; + } + + updateLongClickCallButton(callButton, svgIcon, label, onClick, onLongPressDown, onLongPressUp, isActive = false) { + if (!callButton) { + callButton = document.getElementById("saypi-callButton"); + } + if (callButton) { + this.removeChildNodesFrom(callButton); + this.addIconTo(callButton, svgIcon); + callButton.onclick = ()=> {}; + callButton.setAttribute("aria-label", label); + this.handleLongClick(callButton, onClick, onLongPressDown, onLongPressUp); + this.toggleActiveState(callButton, isActive); + } + this.callIsActive = isActive; + } + callStarting(callButton) { const label = getMessage("callStarting"); this.updateCallButton(callButton, callStartingIconSVG, label, () => @@ -358,15 +388,63 @@ class ButtonModule { callActive(callButton) { const label = getMessage("callInProgress"); - this.updateCallButton( + this.updateLongClickCallButton( callButton, hangupIconSVG, label, - () => this.sayPiActor.send("saypi:hangup"), + () => { + console.log("saypi:hangup sent from callActive()"); + this.sayPiActor.send("saypi:hangup"); }, + () => this.sayPiActor.send("saypi:momentaryListen"), + () => this.sayPiActor.send("saypi:momentaryPause"), true - ); + ); } + callMomentary(callButton) { + // const label = getMessage("callInProgress"); + + const label = "momentary mode listening"; + this.updateLongClickCallButton( + callButton, + momentaryEnableIconSvg, + label, + () => this.sayPiActor.send("saypi:hangup"), + () => this.sayPiActor.send("saypi:momentaryListen"), + () => this.sayPiActor.send("saypi:momentaryPause"), + true + ); + } + + pauseMomentary(callButton) { + // const label = getMessage("callInProgress"); + const label = "momentary paused"; + this.updateLongClickCallButton( + callButton, + momentarySpeakingIconSvg, + label, + () => this.sayPiActor.send("saypi:momentaryStop"), + () => this.sayPiActor.send("saypi:momentaryListen"), + () => this.sayPiActor.send("saypi:momentaryPause"), + true + ); + } + + callInterruptible(callButton) { + const handsFreeInterruptEnabled = + this.userPreferences.getCachedAllowInterruptions(); + if (!handsFreeInterruptEnabled) { + const label = getMessage("callInterruptible"); + this.updateLongClickCallButton( + callButton, + interruptIconSVG, + label, + () => this.sayPiActor.send("saypi:interrupt"), + true + ); + } + } + callInterruptible(callButton) { const handsFreeInterruptEnabled = this.userPreferences.getCachedAllowInterruptions(); @@ -384,6 +462,35 @@ class ButtonModule { } } + handleLongClick(button, onClick, onLongPressDown, onLongPressUp ) { + const longPressMinimumMilliseconds = 500; + var clickStartTime = 0; + var isMouseUpDetected = false; + + if(onLongPressDown) { + button.onmousedown = () => { + isMouseUpDetected = false; + clickStartTime = Date.now(); + window.setTimeout( () => { + if(!isMouseUpDetected) { + onLongPressDown(); + } + }, longPressMinimumMilliseconds); + } + button.onmouseup = () => { + isMouseUpDetected = true; + let isShortClick = (Date.now() - clickStartTime) < longPressMinimumMilliseconds; + if(isShortClick) { + onClick(); + } else { + onLongPressUp(); + } + } + } else if (onClick) { + onClick(); + } + } + callInactive(callButton) { const label = getMessage("callNotStarted", this.chatbot.getName()); this.updateCallButton(callButton, callIconSVG, label, () => diff --git a/src/events/EventModule.js b/src/events/EventModule.js index 77de5a52cc..4217607a51 100644 --- a/src/events/EventModule.js +++ b/src/events/EventModule.js @@ -19,6 +19,9 @@ const AUDIO_DEVICE_RECONNECT = "saypi:audio:reconnect"; const END_CALL = "saypi:hangup"; const SESSION_ASSIGNED = "saypi:session:assigned"; const UI_SHOW_NOTIFICATION = "saypi:ui:show-notification"; +const MOMENTARY_LISTEN = "saypi:momentaryListen"; +const MOMENTARY_PAUSE = "saypi:momentaryPause"; +const MOMENTARY_STOP = "saypi:momentaryStop"; /** * The EventModule translates events sent on the EventBus to StateMachine events, @@ -76,6 +79,9 @@ export default class EventModule { PI_STOPPED_SPEAKING, PI_FINISHED_SPEAKING, END_CALL, + MOMENTARY_LISTEN, + MOMENTARY_PAUSE, + MOMENTARY_STOP, ].forEach((eventName) => { EventBus.on(eventName, () => { actor.send(eventName); From a86760481c7a65ca4090d4ad11f20f77bf8f6e9a Mon Sep 17 00:00:00 2001 From: john Date: Thu, 12 Dec 2024 23:52:01 +0000 Subject: [PATCH 02/25] Fixed button input, but state does not progress to momentary paused --- src/ButtonModule.js | 194 ++++++++++++------------ src/icons/momentary_listening.svg | 10 ++ src/icons/momentary_paused.svg | 10 ++ src/state-machines/AudioInputMachine.ts | 2 +- src/state-machines/SayPiMachine.ts | 164 ++++++++++++++++++++ 5 files changed, 283 insertions(+), 97 deletions(-) create mode 100644 src/icons/momentary_listening.svg create mode 100644 src/icons/momentary_paused.svg diff --git a/src/ButtonModule.js b/src/ButtonModule.js index 9142022992..0a4813d20a 100644 --- a/src/ButtonModule.js +++ b/src/ButtonModule.js @@ -11,6 +11,8 @@ import callIconSVG from "./icons/call.svg"; import callStartingIconSVG from "./icons/call-starting.svg"; import hangupIconSVG from "./icons/hangup.svg"; import interruptIconSVG from "./icons/interrupt.svg"; +import momentaryPausedIconSVG from "./icons/momentary_paused.svg"; +import momentaryListeningIconSVG from "./icons/momentary_listening.svg"; import hangupMincedIconSVG from "./icons/hangup-minced.svg"; import lockIconSVG from "./icons/lock.svg"; import unlockIconSVG from "./icons/unlock.svg"; @@ -45,6 +47,7 @@ class ButtonModule { // track whether a call is active, so that new button instances can be initialized correctly this.callIsActive = false; + this.clickStartTime = 0; } registerOtherEvents() { @@ -189,7 +192,7 @@ class ButtonModule { button.id = id; button.type = "button"; button.className = `saypi-control-button rounded-full bg-cream-550 enabled:hover:bg-cream-650 tooltip mini ${className}`; - button.setAttribute("aria-label", label); + this.setAriaLabelOf(button, label); const svgElement = createSVGElement(icon); button.appendChild(svgElement); @@ -204,7 +207,7 @@ class ButtonModule { createExitButton(container, position = 0) { const button = this.createIconButton({ id: 'saypi-exit-button', - label: getMessage("exitImmersiveModeLong"), + label: "exitImmersiveModeLong", icon: exitIconSVG, onClick: () => ImmersionService.exitImmersiveMode(), className: 'saypi-exit-button' @@ -217,7 +220,7 @@ class ButtonModule { createEnterButton(container, position = 0) { const button = this.createIconButton({ id: 'saypi-enter-button', - label: getMessage("enterImmersiveModeLong"), + label: "enterImmersiveModeLong", icon: maximizeIconSVG, onClick: () => this.immersionService.enterImmersiveMode(), className: 'saypi-enter-button' @@ -234,9 +237,10 @@ class ButtonModule { */ createControlButton(options) { const { shortLabel, longLabel = shortLabel, icon, onClick, className = '' } = options; + const button = createElement("a", { className: `${className} maxi saypi-control-button tooltip flex h-16 w-16 flex-col items-center justify-center rounded-xl text-neutral-900 hover:bg-neutral-50-hover hover:text-neutral-900-hover active:bg-neutral-50-tap active:text-neutral-900-tap gap-0.5`, - ariaLabel: longLabel, + ariaLabel: getMessage(longLabel), onclick: onClick, }); @@ -245,7 +249,7 @@ class ButtonModule { const labelDiv = createElement("div", { className: "t-label", - textContent: shortLabel, + textContent: getMessage(shortLabel), }, ); button.appendChild(labelDiv); @@ -254,8 +258,8 @@ class ButtonModule { createImmersiveModeButton(container, position = 0) { const button = this.createControlButton({ - shortLabel: getMessage("enterImmersiveModeShort"), - longLabel: getMessage("enterImmersiveModeLong"), + shortLabel: "enterImmersiveModeShort", + longLabel: "enterImmersiveModeLong", icon: immersiveIconSVG, onClick: () => this.immersionService.enterImmersiveMode(), className: 'immersive-mode-button' @@ -266,9 +270,8 @@ class ButtonModule { } createSettingsButton(container, position = 0) { - const label = getMessage("extensionSettings"); const button = this.createControlButton({ - shortLabel: label, + shortLabel: "extensionSettings", icon: settingsIconSVG, onClick: () => openSettings(), className: 'settings-button' @@ -281,7 +284,7 @@ class ButtonModule { createMiniSettingsButton(container, position = 0) { const button = this.createIconButton({ id: 'saypi-settingsButton', - label: getMessage("extensionSettings"), + label: "extensionSettings", icon: settingsIconSVG, onClick: () => openSettings(), className: 'settings-button' @@ -327,58 +330,62 @@ class ButtonModule { */ handleAudioFrame(probabilities) { this.glowColorUpdater.updateGlowColor(probabilities.isSpeech); - } - updateCallButton(callButton, svgIcon, label, onClick, isActive = false) { - if (!callButton) { - callButton = document.getElementById("saypi-callButton"); + } + removeChildrenFrom(callButton) { + while (callButton.firstChild) { + callButton.removeChild(callButton.firstChild); } - if (callButton) { - // Remove all existing child nodes - while (callButton.firstChild) { - callButton.removeChild(callButton.firstChild); - } + } - const svgElement = createSVGElement(svgIcon); - callButton.appendChild(svgElement); + addIconTo(callButton, svgIcon) { + const svgElement = createSVGElement(svgIcon); + callButton.appendChild(svgElement); + } - callButton.setAttribute("aria-label", label); - callButton.onclick = onClick; - callButton.classList.toggle("active", isActive); - } - this.callIsActive = isActive; + toggleActiveState(callButton, isActive) { + callButton.classList.toggle("active", isActive); } - updateCallButtonNEW(callButton, svgIcon, label, onClick, isActive = false) { + updateCallButton(callButton, svgIcon, label, onClick, isActive = false) { if (!callButton) { callButton = document.getElementById("saypi-callButton"); } if (callButton) { - this.removeChildNodesFrom(callButton); + this.removeChildrenFrom(callButton); this.addIconTo(callButton, svgIcon); - callButton.onmousedown = null; - callButton.onmouseup = null; - callButton.onClick = onClick; + callButton.setAttribute("aria-label", label); + this.handleLongClick(callButton, onClick); this.toggleActiveState(callButton, isActive); } this.callIsActive = isActive; } - updateLongClickCallButton(callButton, svgIcon, label, onClick, onLongPressDown, onLongPressUp, isActive = false) { + updateLongClickCallButton(callButton, svgIcon, label, clickEventName, longPressEventName, longReleaseEventName, isActive = true) { if (!callButton) { callButton = document.getElementById("saypi-callButton"); } if (callButton) { - this.removeChildNodesFrom(callButton); + this.removeChildrenFrom(callButton); this.addIconTo(callButton, svgIcon); - callButton.onclick = ()=> {}; - callButton.setAttribute("aria-label", label); + callButton.onclick = () => {}; + this.setAriaLabelOf(callButton, label); + let onClick = this.createEvent(clickEventName); + let onLongPressDown = this.createEvent(longPressEventName); + let onLongPressUp = this.createEvent(longReleaseEventName); this.handleLongClick(callButton, onClick, onLongPressDown, onLongPressUp); this.toggleActiveState(callButton, isActive); } this.callIsActive = isActive; } + createEvent(eventName) { + + return () => { + console.log(" ^^^^^^^^^^^ ButtonModule.createEvent() ->>> : " + eventName); + this.sayPiActor.send(eventName); }; + } + callStarting(callButton) { const label = getMessage("callStarting"); this.updateCallButton(callButton, callStartingIconSVG, label, () => @@ -387,64 +394,41 @@ class ButtonModule { } callActive(callButton) { - const label = getMessage("callInProgress"); + console.log("ButtonModule entered callActive()"); this.updateLongClickCallButton( callButton, hangupIconSVG, - label, - () => { - console.log("saypi:hangup sent from callActive()"); - this.sayPiActor.send("saypi:hangup"); }, - () => this.sayPiActor.send("saypi:momentaryListen"), - () => this.sayPiActor.send("saypi:momentaryPause"), - true + "callInProgress", + "saypi:hangup", + "saypi:momentaryListen", + "saypi:momentaryPause", ); } callMomentary(callButton) { - // const label = getMessage("callInProgress"); - - const label = "momentary mode listening"; + console.log("ButtonModule entered callMomentary()"); this.updateLongClickCallButton( callButton, - momentaryEnableIconSvg, - label, - () => this.sayPiActor.send("saypi:hangup"), - () => this.sayPiActor.send("saypi:momentaryListen"), - () => this.sayPiActor.send("saypi:momentaryPause"), - true + momentaryListeningIconSVG, + "callInProgress", + "saypi:momentaryStop", + "saypi:momentaryPause", + "saypi:momentaryPause", ); } pauseMomentary(callButton) { - // const label = getMessage("callInProgress"); - const label = "momentary paused"; + console.log("ButtonModule entered pauseMomentary()"); this.updateLongClickCallButton( callButton, - momentarySpeakingIconSvg, - label, - () => this.sayPiActor.send("saypi:momentaryStop"), - () => this.sayPiActor.send("saypi:momentaryListen"), - () => this.sayPiActor.send("saypi:momentaryPause"), - true + momentaryPausedIconSVG, + "callInProgress", + "saypi:momentaryStop", + "saypi:momentaryListen", + "saypi:momentaryPause", ); } - callInterruptible(callButton) { - const handsFreeInterruptEnabled = - this.userPreferences.getCachedAllowInterruptions(); - if (!handsFreeInterruptEnabled) { - const label = getMessage("callInterruptible"); - this.updateLongClickCallButton( - callButton, - interruptIconSVG, - label, - () => this.sayPiActor.send("saypi:interrupt"), - true - ); - } - } - callInterruptible(callButton) { const handsFreeInterruptEnabled = this.userPreferences.getCachedAllowInterruptions(); @@ -462,32 +446,46 @@ class ButtonModule { } } + setEmptyDefault(runnable) { + return runnable ? runnable : () => {}; + } + + isShortClick(clickStartTime, limit) { + const clickDuration = (Date.now() - clickStartTime); + console.log("ButtonModule.isShortClick() startTime: " + clickStartTime + " duration: " + clickDuration); + return clickStartTime == 0 || clickDuration < limit; + } + + handleLongClick(button, onClick, onLongPressDown, onLongPressUp ) { const longPressMinimumMilliseconds = 500; var clickStartTime = 0; var isMouseUpDetected = false; - if(onLongPressDown) { - button.onmousedown = () => { - isMouseUpDetected = false; - clickStartTime = Date.now(); - window.setTimeout( () => { - if(!isMouseUpDetected) { - onLongPressDown(); - } - }, longPressMinimumMilliseconds); - } - button.onmouseup = () => { - isMouseUpDetected = true; - let isShortClick = (Date.now() - clickStartTime) < longPressMinimumMilliseconds; - if(isShortClick) { - onClick(); - } else { - onLongPressUp(); + onClick = this.setEmptyDefault(onClick); + onLongPressDown = this.setEmptyDefault(onLongPressDown); + onLongPressUp = this.setEmptyDefault(onLongPressUp); + + button.onmousedown = () => { + isMouseUpDetected = false; + this.clickStartTime = Date.now(); + window.setTimeout( () => { + if(!isMouseUpDetected) { + console.log("ButtonModule.handleLongClick(): long press down detected!"); + onLongPressDown(); } + }, longPressMinimumMilliseconds); + } + + button.onmouseup = () => { + isMouseUpDetected = true; + if(this.isShortClick(this.clickStartTime, longPressMinimumMilliseconds)) { + console.log("ButtonModule.handleLongClick(): short click detected!"); + onClick(); + } else { + console.log("ButtonModule.handleLongClick(): long press up detected!"); + onLongPressUp(); } - } else if (onClick) { - onClick(); } } @@ -542,14 +540,18 @@ class ButtonModule { return button; } + setAriaLabelOf(button, labelName) { + const label = getMessage(labelName); + button.setAttribute("aria-label", label); + } + createUnlockButton(container) { - const label = getMessage("unlockButton"); const button = document.createElement("button"); button.id = "saypi-unlockButton"; button.type = "button"; button.className = "lock-button saypi-control-button rounded-full bg-cream-550 enabled:hover:bg-cream-650 tooltip"; - button.setAttribute("aria-label", label); + this.setAriaLabelOf(button, "unlockButton"); button.appendChild(createSVGElement(unlockIconSVG)); if (container) { container.appendChild(button); diff --git a/src/icons/momentary_listening.svg b/src/icons/momentary_listening.svg new file mode 100644 index 0000000000..06cd8f1148 --- /dev/null +++ b/src/icons/momentary_listening.svg @@ -0,0 +1,10 @@ + + + + \ No newline at end of file diff --git a/src/icons/momentary_paused.svg b/src/icons/momentary_paused.svg new file mode 100644 index 0000000000..3f4c78fcf3 --- /dev/null +++ b/src/icons/momentary_paused.svg @@ -0,0 +1,10 @@ + + + + \ No newline at end of file diff --git a/src/state-machines/AudioInputMachine.ts b/src/state-machines/AudioInputMachine.ts index a186eb2b9b..b42cc8271f 100644 --- a/src/state-machines/AudioInputMachine.ts +++ b/src/state-machines/AudioInputMachine.ts @@ -313,7 +313,7 @@ function tearDownRecording(): void { microphone.pause(); listening = false; stream.getTracks().forEach((track) => track.stop()); - microphone.destroy(); //added by JAC + microphone.destroy(); } else { console.log("microphone does not exist!"); } diff --git a/src/state-machines/SayPiMachine.ts b/src/state-machines/SayPiMachine.ts index 56fe266ac1..ecde2cfa6b 100644 --- a/src/state-machines/SayPiMachine.ts +++ b/src/state-machines/SayPiMachine.ts @@ -74,6 +74,9 @@ type SayPiEvent = | { type: "saypi:submit" } | { type: "saypi:promptReady" } | { type: "saypi:call" } + | { type: "saypi:momentaryListen" } + | { type: "saypi:momentaryPause" } + | { type: "saypi:momentaryStop" } | { type: "saypi:callReady" } | { type: "saypi:callFailed" } | { type: "saypi:hangup" } @@ -92,6 +95,7 @@ interface SayPiContext { timeUserStoppedSpeaking: number; defaultPlaceholderText: string; sessionId?: string; + isMomentaryEnabled: boolean; } // Define the state schema @@ -204,6 +208,7 @@ const machine = createMachine( context: { transcriptions: {}, isTranscribing: false, + isMomentaryEnabled: false, lastState: "inactive", userIsSpeaking: false, timeUserStoppedSpeaking: 0, @@ -257,6 +262,7 @@ const machine = createMachine( { type: "callStartingPrompt", }, + assign({ isMomentaryEnabled: false }), ], exit: [ { @@ -443,9 +449,148 @@ const machine = createMachine( description: 'Disable the VAD microphone.\n Aka "call" Pi.\n Stops active listening.', }, + "saypi:momentaryListen": { + actions: [ + assign({ + isMomentaryEnabled: true, + }), + { + type: "momentaryHasStarted", + }, + ], + target: [ + "#sayPi.listening.momentaryRecording", + ], + description: + 'Enable Momentary Mode. Now recording will only stop if the user releases the button.', + }, }, }, + momentaryRecording: { + description: + "Momentary button is depressed, the microphone is on and VAD is actively listening for user speech.", + initial: "notSpeaking", + entry: [ + { + type: "startAnimation", + params: { + animation: "glow", + }, + }, + { + type: "listenPrompt", + }, + { + type: "justForFun" + }, + ], + exit: [ + { + type: "stopAnimation", + params: { + animation: "glow", + }, + }, + { + type: "clearPrompt", + }, + ], + states: { + notSpeaking: { + description: + "Microphone is recording but no speech is detected.", + on: { + "saypi:userSpeaking": { + target: "userSpeaking", + }, + }, + }, + userSpeaking: { + description: + "User is speaking and being recorded by the microphone.\nWaveform animation.", + entry: [ + { + type: "startAnimation", + params: { + animation: "userSpeaking", + }, + }, + assign({ userIsSpeaking: true }), + { + type: "cancelCountdownAnimation", + }, + ], + exit: { + type: "stopAnimation", + params: { + animation: "userSpeaking", + }, + }, + }, + }, + on: { + "saypi:momentaryPause": [ + { + target: [ + "#sayPi.listening.converting.transcribing", + ], + cond: "hasAudio", + actions: [ + assign({ + userIsSpeaking: false, + timeUserStoppedSpeaking: () => new Date().getTime(), + }), + { + type: "transcribeAudio", + }, + ], + }, + { + target: "#sayPi.listening.momentaryPaused", + cond: "hasNoAudio", + description: 'Pause Momentary Mode. Now recording will be ignored.', + actions: [ + { + type: "momentaryHasPaused", + }, + ] + }, + { + actions: [ + { + type: "justForFun2", + }, + ] + }, + ], + }, + }, + + momentaryPaused: { + description: + "In momentary mode and the button has been released, so the microphone is ignoring input", + entry: { + type: "justForFun", + }, + on: { + "saypi:momentaryStop": { + actions: [ + assign({ isMomentaryEnabled: false }), + { + type: "momentaryHasStopped" + }, + ], + target: "#sayPi.listening.recording", + description: 'Returning to the standard recording mode.', + }, + "saypi:momentaryListen": { + target: "#sayPi.listening.momentaryRecording", + description: 'Return to recording in momentary mode.', + }, + }, + }, + converting: { initial: "accumulating", states: { @@ -1028,6 +1173,12 @@ const machine = createMachine( pauseRecording: (context, event) => { EventBus.emit("audio:input:stop"); }, + justForFun : () => { + console.log("Entered just for fun!"); + }, + justForFun2 : () => { + console.log("Entered just for fun 2!"); + }, pauseRecordingIfInterruptionsNotAllowed: (context, event) => { const handsFreeInterrupt = @@ -1153,6 +1304,19 @@ const machine = createMachine( callFailedToStart: () => { buttonModule.callInactive(); audibleNotifications.callFailed(); + }, + momentaryHasStarted: () => { + console.log("SayPiMachine entered momentaryHasStarted()"); + buttonModule.callMomentary(); + // TODO: fix this sayPi.isMomentaryEnabled = true; + }, + momentaryHasPaused: () => { + console.log("SayPiMachine entered momentaryHasPaused()"); + buttonModule.pauseMomentary(); + }, + momentaryHasStopped: () => { + console.log("SayPiMachine entered momentaryHasStopped()"); + buttonModule.callActive(); }, callNotStarted: () => { if (buttonModule) { From d680fda218d01689b31b424a565c1defcb027e2d Mon Sep 17 00:00:00 2001 From: john Date: Thu, 19 Dec 2024 00:08:01 +0000 Subject: [PATCH 03/25] Removed MomentaryRecording state, momentary paused does not resume after interruption, static glow remains on momentary pause, state engine does not wait for button release to send transcription. --- src/state-machines/SayPiMachine.ts | 161 +++++++++-------------------- 1 file changed, 49 insertions(+), 112 deletions(-) diff --git a/src/state-machines/SayPiMachine.ts b/src/state-machines/SayPiMachine.ts index ecde2cfa6b..38954e6560 100644 --- a/src/state-machines/SayPiMachine.ts +++ b/src/state-machines/SayPiMachine.ts @@ -96,6 +96,7 @@ interface SayPiContext { defaultPlaceholderText: string; sessionId?: string; isMomentaryEnabled: boolean; + isMomentaryActive: boolean; } // Define the state schema @@ -204,11 +205,12 @@ function getChatbotDefaultPlaceholder(): string { } const machine = createMachine( { - /** @xstate-layout N4IgpgJg5mDOIC5SwIYE8AKBLAdFgdigMYAuWAbmAMSpoAOWCdATgPYC2dJASmChGgDaABgC6iUHVawsZVvgkgAHogDMAdgAcAThzrt64aoAsAJgBsAVk3HLlgDQg0iAIwvTwnJs0ntmt34uBgC+wY602HiEpBTUtAwIRCgANski4kggUjJyCpkqCNqmjs6Fxrqm1uqm6naVmsLqoeHokQTEZJQ06AkMAMp0fADWBFDpitmyWPKKBcZulnra5tqWRqqW7g5OriaqXqrm6i6ax8KW6hrNIBG4Sal9JCjMZPhQ3fSM98m8-EJiE2kUxm+UQtmMOBcxhs1m8xn0+m2pWhi2MxnMphq2hc5hcwmMh2utxw30ez1e73ijAAFig3gBXOjjTKTXKzRCaYo7BB41Y4NanDaXczmYQuSxE1p3FLJMkvUYfBLfABiKCwyUgzMkQLZoIQaIhUJh3hsCNWJVc2n2llU2jtm3x-ltksw0oeT3lbyoSlgTxIYBwKAAZv7mAAKDzCKMASg+kVJHopWqyOumeVAc0shuhthN8IM5u5Jk8dtUWxqVisErCNylOGSWF9YHwCqpTCwABVqQQRm9k6y0+yEKpDvsEVHrCtzMZzhbhxtzJDvEc3KcjsYXZEG02W162-1Bihe2MASzUyCM2oNmODBPNFOZ0i1O51Dh5mtqqtTD4fJvcNv-V3SkekYchGywAAjDV+3PdNlFcEVNH5Sd-CsTEbFUOdxQsPQdGWdRDEqLY-3rRtANbECEBQekIGmRJ5HwMBSE1U9tRyQc9RxdEcFtdEHytfQsIJTwR3XXiNHRFwSIA5sKM+KiaLo5gmIYpiSBg9iL3gnlp1fNDbHxKMcRMOdr1fQ5UL8Az0WksjZL3SjYDgGR5Co2AZCgRiIA04E4IKLiIXKTE0RXe8DFMtES3FFxDHUGcXDLasWldUid1GHBlKIVhmFohz5NpBkmVYlNNL8q8bSWQxzjC6dZ25SN9lUc46ltW1Kg3GtiRkoCMpUnL0vwVgSAGYY5ISeknOYZUCEbalIBGo9Rh83VL3nCrx2qh86tKTF8RwEVyg2TQs05VRbLSt5eqy-rLsG4bD2PRVGAmsBmAW49lo41btEafbTmhb97xRdQ51MaL9unbwPHMVRMW0c7yMuzLstyqAcBet6HrG57JseVg6EGCB3qW4qBy0gpzG8HA7TtOELD8ZZQcqUwIfKHRp2ncUktrFLuvS5GbrRjHiby8bcZIfHCZFsYXAyNjfKHSmkJpvwbHp9mmeEFmuKtawTsxbmursnqsvwShPTR4giHpdh6WSFAKSoCB5ADAhyFYIYA3YV6YAAeS4LB2DIrAiE+8nXGqt91Csco3DMZnQdQg4lcMMsak0BH7LR03zYpQMiGt237cd162GYHA6GLoNsvYHBveYP2A6D30Q7DsqeXOXQoVsC4THFKNMPq6pGp8FZhAaeFTHhTOTfkXP0qtm27Yd7GEBIZg6VgIhmEgli5ZKhW9QMfYwdOHw4rtI4Qfqi5FitfFoSa0V4TOzq6z5y6c9evPF6LlfRcYOvTe29d6qnVHvQEpUhxgxwraGwVY1h+EHjtWq1MoR2kaGYZYr9kpbmNulL+Ft86F2Xo7NsQD8Bbx3hBSAABRTgJB-j7zJu3GBi44HojsIgnwidhCLnmOULQCUwYiiaG-Xm+DP5z2-gvAuS9i4Kh9H6AMwZQxhlgPSCCzcXL4AACJgHtmgWMRsLrZ2kUQ3+pCSbMNgkOKEMUcCn0uJYWmeIwamQHpCKw6wEpaCaobd+kizFmxkZdDRWjZCOzbkOUSN4qqTmWI+UyMVXzYnKFrYQP1rAzwIeYvOFCqGQVXgUkBNDvKk1sXqG0EInG1BHOffEiccQ8Q0I0Iw95-D4hyVIkJRCSnUOKRvShpSwBgI1OUmxUCqkmEcScAiNpVANOMKDbQaJHE-QSm1XE5wxG4P-EEkkeT0r9KKQAteQzCllPoVwJhkDD6rWqbM04dTFmtOWUWLWi5TA-S1hfTk34AkSNMTgUu2UyQkAmiC5gZdYDel9A7FRIZXphjWDGOM+zgWgren6SFWLYDRL1N80w+wjgFntBcBoyDECVH0G+dBpx8LEsJOIyIylYBSHwKjJ6CACpQEZAS1a8xxRLBWGsEcmxKhYXFEhGcqdMRQgxDFEibKOVcrbMLLGfYKlTMFSOV8QVcyWG-LrJ8w4CT7EyZPUUp1yjKrgKq9KDAuw9lXgeUaWrJn3O0vCLW1MbBFHHqfQ4HiVh6GhEEVZ6JVZ2vZfIVGFdOzdnwI9fcWAADqO8kzaq9XMQifqgqBpOMGosU8WZT05EENwYMCQxodZdN1i0zn9AlgTeamqTyepWtpDEpkoaOIrAYMGhwzAdT2b1WNnLHVYGlty5tks23uplp2r63qFgitWOsCVpq3C-R+qnXM05sTmFrXGqdM7U3TRbLAOaRN20Cu0ka0yVoWbVHmXic4xKswnsnfW6d7buUasXfegoNo+THVcRcY4dhTIWGVugkcYobSbFHTzVl9rT2-ozVMJtf6gPZq7SB1pLSRSBr4YemDJw6XQn0JiDQ2SWW4BVRhtGDAsNkMonO1tEA2PWLuQRxA1Re1lj0BWAk8d8SrO-fGjGABJfAoZmCMnY-JYWLapZ3vwyuwjhhiPWuhuRostpFwdPlR4f1GcGPjrrULSacmFNKdXqp+dt6gOyz41pjkXJSiHGjl4GBpp4T6A2FJs9-62wEHs1wYDiAXHwhwFrTkdh-DWC1qZbZkImrGu8JcEKIXf3nsohF0uSnBBubPDq7tYMXD8hhhiTJ3zbSmo2GsDLCXVY5ZstcQaEA4CKFuO58OCAAC05gsJ8PizOcemwYawhMCRdoMRKADfbnR6rZhK0YlLCkrCNRGo+ZnLif1uzUNullImUYy2hxmDnOt+LlaPxWBxDobpUBLtH1G9yaVr4p5pwIlrWEL2roowu+VnNrg0RYR+i+4SsJJyxcBwLeNd1pZvdWm4BKPF8SUxhtUDw8JQYnBZmk0sdTLj3gR31aTuN22o+0kELz1KFVhrLNHaO5RyjHssx-YJ883i0-8uUUG0IZX+IIscHQdhAeEJ-nIv+FJ+eIGxIsU+lKcu3ypQgb5wrvlRh8DSnwUkucHOl+lcJQcSDy9B-xnk+hjPuGo2iAiOIPs7S-BDNYUJbT6BOJzsd3PDm9PyRc0pIP5bW6CNfV3lRHF4QuFxfQ8MjeYuhWCnF8Arcec1+GniVkNB+OqKa9Cuh3xohOCOm0gOsXgshYNZg7AUgK813UHPGECLn0lUWGKqJ+7j3q0ayvKfsUO1xYP9PYfM-M10BwvP7fC+3YEf84lUa0QD7LtX2AUKYU4BOTQsZkBG80vMrntvGgO8oIx++dYZYhUoZMYjNGVe0+b+yhvoORBaGD4P5BlvBIT8F9Bm3mgmsB4KnCznlq9hnoNmYDoG+EUIasas1onNYI4gDM1HAbUOAQms6smqHgfNbuavqnAXrAgdBp3sJrTERJkkUFfJgQ2seI3ilizEFDoG0rtIWN5t+K+AlPiGTpksdDgidlZsxgmjxnzpAe3IItVgPE1NUKcCAckj9LAf8jDGYFoJ1mOkxj+jZq9HZsVgHGIePlAd4LoJTPiDruKLpKZIstViYC-COO4BgmIqEEAA */ + /** @xstate-layout N4IgpgJg5mDOIC5SwIYE8AKBLAdFgdigMYAuWAbmAMSpoAOWCdATgPYC2dJASmChGgDaABgC6iUHVawsZVvgkgAHogDMAdgAcAThzrt64aoAsAJgBsAVk3HLlgDQg0iAIwvTwnJs0ntmt34uBgC+wY602HiEpBTUtAwIRCgANski4kggUjJyCpkqCNqmjs6Fxrqm1uqm6naVmsLqoeHokQTEZJQ06AkMAMp0fADWBFDpitmyWPKKBcZulnra5tqWRqqW7g5OriaqXqrm6i6ax8KW6hrNIBG4Sal9JCjMZPhQ3fSM98m8-EJiE2kUxm+UQtmMOBcxhs1m8xn0+m2pWhi2MxnMphq2hc5hcwmMh2utxw30ez1e73ijAAFig3gBXOjjTKTXKzRCaYo7BB41Y4NanDaXczmYQuSxE1p3FLJMkvUYfBLfABiKCwyUgzMkQLZoIQaIhUJh3hsCNWJVc2n2llU2jtm3x-ltksw0oeT3lbyoSlgTxIYBwKAAZv7mAAKDzCKMASg+kVJHopWqyOumeVAc0shuhthN8IM5u5Jk8dtUWxqVisErCNylOGSWF9YHwCqpTCwABVqQQRm9k6y0+yEKpDvsEVHrCtzMZzhbhxtzJDvEc3KcjsYXZEG02W162-1Bihe2MASzUyCM2oNmODBPNFOZ0i1O51Dh5mtqqtTD4fJvcNv-V3SkekYchGywAAjDV+3PdNlFcEVNH5Sd-CsTEbFUOdxQsPQdGWdRDEqLY-3rRtANbECEBQekIGmRJ5HwMBSE1U9tRyQc9RxdEcFtdEHytfQsIJTwR3XXiNHRFwSIA5sKM+KiaLo5gmIYpiSBg9iL3gnlp1fNDbHxKMcRMOdr1fQ5UL8Az0WksjZL3SjYDgGR5Co2AZCgRiIA04E4IKLiIXKTE0RXe8DFMtES3FFxDHUGcXDLasWldUid1GHBlKIVhmFohz5NpBkmVYlNNL8q8bSWQxzjC6dZ25SN9lUc46ltW1Kg3GtiRkoCMpUnK5ISdgOGbD00AAGTs-AfN1S8eUsO031MZZ7w0TF8XUOdTEit8mrRHxhFFeFVFstK3l6rL+rO-BWBIAZhgGxh6Sc5hlQIRtqUgO6j1GaaONmkcKvHaqHzq0o1ohEVyg2TQs05Y7Orrbr0sy7LcqgHBrtuw9j0VR7nq+49fq0gptEaHBzFOaFv3vFENvq6LyenbwPHMVRMW0E7yLOlHLvRp6wGYAmHoQfnBZIVg6EGCAhb7YqB2JxAKaQu07ThCw-GWTbKlMRnyh0adp3FJLaxSpHub6tGcFFmXgPk63xclz7sZ+lwMjY3yhyVnAVb8Gx1f1rXhB1rirWsWHMWNrrJvSob2BG540F4C60dxhBY-j5hMGopyibKhBatMoJ9hFNYoVtfQTnMTn7PR9P8FGpPUeFuvRowbOwEEV3AVKz2Z1MtmkKWoyCQLSvq56luE8b3nU8nzO2-5wRTDdkqPb1Aui2hczLCIkeK-vceY+G+up4t9LMZt1Pred2WV-lvP0MXLZNrxTwtrLGpCNhQ+zrnrP+YgLPY+o1HgS1zkOLMxhC5WkZqXPe2ID4I1NtHX+wCE4LycoAtsf8Jo7nAXqSB-d5pLGHuXBBVckFbhQejLK+BKCenRsQIg9J2D0mSCgCkVAIDyADAQcgrAhgBjjswGAAB5LgWB2BkSwEQfBs08TWDfOoKw5Q3BmG1ptVCBwlaGA-loH+ND5D0IpIGIgzDWHsM4QLNgzAcB0EsUGbK7AcDCLERIqRvoZFyO0go3QUJbAXBMOKKMmF6rVEaj4FYwgGjwi2k0Sh-5qEkiMQLExTCWFsI4cLEgzA6SwCIMwSCLE76wSHAYfYpgThaAknaI4dMwYXEWFafE0ImqHUuAY5JdDUnpXSRYrJeUEg5LyQUopqp1TFO7mvWalScK2hsFWNYfhQlg1qt7KEdpGhmGWPDZKVDTqGO6Qw0x5jMmcLbMM-A+TCkQUgAAUU4CQf4JSe56lmYueZ6I7BLJ8Jog6b4NlaASpUkU8S9mJIOV04xvSzEZMsQqH0foAzBlDGGWA9IIIeJcvgAAImAdhaBYxR0hbQ6FZ0+lnJ+nLUpnF5ivkqYKeagQg4uFMiEyEVh1gJWqecTppKelnXRZi2QnDvEFFEjeKqk5liPkLgRdZ5Qg7CFJtYPlKTjmXOuZBbJuSrmjNud5alrzZo2ghAygiNpVA+EMFA+qWiRyxSMPefw+I1VHJMZq-VOqRk3LAOMjUhqXnTO0qanA5ragjmtfiTa2g0RhtJglNquJzhgpNvsrmhyyXo09Tc71erfUQAeVwZ5UyZohpMGGqpEarUaGjUWIOi4h5Bzin4dCkdEZJOsdlMkJAno4C7cwWA3pfQcORSGAWYY1gxjjBCjN-bmA2J7X2gdsAxVqCMjgMUZYNjKtxFoTadgkLRJxFmBKNo7SpuJMpWAUh8ApzbAVKAjI136gWEsFYawAbP25AsJCM5dGYihBiGKJFr23vvZRa+91b6lr+tpAkGg3xFFzDvHQGwnzDgQ5u2N1RRRw3KKBuA4H0oMC7D2YWB5oMniDWWuYhFvY2CKNEhlhw2UrD0NCIIsb0S+0Ize+QltSPdnwDjfcWAADqhSkxGuDXRoODGgrMZOKxosW0dZbU5EENwlSCR8eI2dSj31BmMH6A7KWNsX0YlMszMNFYDCVMOGYDq4Ler8bvSRrAl8xOgMdtLG+Ywu5nmNfBt9+gP3rE2JULCeJXyk10bmac2IKEubAwJjzXnKIMFei2WAH0-NUZfTvUyVodbVAta-HeGxnNptwKl9zBnPP+avvjfzhXbSLBhqrcUBEjamQsMrDZI4xQ2k2NVq9RG0sNYy-JAgoZmCMnUjJ2jiB5rwk3d+So1hxQNC5KUQ4eJIRNW-L7S4IU9OTfRoZ0TlFZvWIW53GjcGCjAZcPyVmGJlVLXa6ZOwngEpBx0N4U7NkEmuf05diTUmKONYK0tp7K3a08UOHhlmiW+snABVvIopgNCqtB3VwTkOpjGfbD5qWknifUdgwrBA1RrNlj0BWAk6j8SxvO-V9GAB3NUFJlTZWwB2VgPnpu9E82Zp2sPHtaT24YJ+R3AeChCo4AoNoj1AsqOzy23PKd8+YALoXDtL6ItHYGcd4Yp1ErrAT9K2vef887AbiWFm4c09V5u9XGHDiKP+8doHJgQcpYmxzq2z0ACS9c7sSJJ-bCW5nWsu7zjaQwSORTMYOmjostpFzOsAx4RjmhNfpVFuHubC3hYx98xZwL7tlsIE5KZI42fZmmnhPoDYoQazXQgHARQtxqd5wALT+Kwv8wyUIjCJYwu2lK7QYiUH70OXHr2zBaYxKWGK9TXA1EaocOKB0TjlEvXWBM5JRgL71GYOcK-N1aY-FYHEOgDHn9mgPzWP7HNhoJERMUAOpKg7NujDzGjM-j4miFhKTKVl-ueisLGkYJ0kAefDdDbCAf5G4I1PiBTKzNUB4PCC-JyAqqWBGpcIgi5gAedE3GdFBkZlACgZaLtogDpouHFGWMosouUOUMljVqlHOn-NPMAUFrJq4A0MIFhEcLoMwZULhniH-qQUkrwWfFdEgf5rQTyDYJoOAexu-C+F-N4J0vIcnEXi1lRioQfuofTNCJ-nojoQXv-nIWgvPO3BACoQPtCP3EEAClOg0EUDvG6lmiYeUJtNCH+ktM6vFloGNh2iSuqmkrCv0hSCodiIsAyg0BoCYI0isgwYkfGlGD4JIT4DIVwWQfysckKlIiQPEQIbXjFMsF4O4FvGiD1iKDGjvLAmKPAmPLYVEe6ulDmtqm8CYeFPVN4WGnhBcFxPoBzJ0XOgOkuvAJUfDggJtkhJcBvktFtO4DUFrBTJupyMqmYA6HFNPumjXPOoun6H2tdMwOwCkCoRYEtHoCwZ+OsZiJvsOOKBCMeuuBYAdCQYUZ2gut2ucbAKcdlHMTXgsd8UwY8WsfMC8YEVGDsUEDjiPDYBEcgpCjMUCSCYOjgL0bcv6pALcSog8aschhsa8ZUEUIiTAVTN+HAVMScZiRwsugCTiVIkQHcqyUSVmCSQRDCeSZtLUK+FtpgacEaCOIXv0fMTTi4WYWDDDIPK0uKKzGgdOJKRDmRiJmftKXnAhq+EFChsduhoXAzqrN-naBYAROqXYjDtQSodYPJkFDoI0EHB4IWHtt+K+P9kdF4TDLslwdbg1hThUeCTTofq9iEk1NUKcB4K8WWKTEhpyBYCYDUAstabbqMLrvrsLsoTqUOCIUWLLodgDidv7pweNm5pbMXhHgumXlKaGbqd4LoBTPiEPMqbgZnv4DxA0SOB-JsvEqEEAA */ context: { transcriptions: {}, isTranscribing: false, isMomentaryEnabled: false, + isMomentaryActive: false, lastState: "inactive", userIsSpeaking: false, timeUserStoppedSpeaking: 0, @@ -453,127 +455,68 @@ const machine = createMachine( actions: [ assign({ isMomentaryEnabled: true, + isMomentaryActive: true, }), { type: "momentaryHasStarted", }, - ], - target: [ - "#sayPi.listening.momentaryRecording", ], description: 'Enable Momentary Mode. Now recording will only stop if the user releases the button.', }, - }, - }, + "saypi:momentaryPause": { - momentaryRecording: { - description: - "Momentary button is depressed, the microphone is on and VAD is actively listening for user speech.", - initial: "notSpeaking", - entry: [ - { - type: "startAnimation", - params: { - animation: "glow", - }, - }, - { - type: "listenPrompt", - }, - { - type: "justForFun" - }, - ], - exit: [ - { - type: "stopAnimation", - params: { - animation: "glow", - }, - }, - { - type: "clearPrompt", - }, - ], - states: { - notSpeaking: { - description: - "Microphone is recording but no speech is detected.", - on: { - "saypi:userSpeaking": { - target: "userSpeaking", + target: [ + "momentaryPaused", + ], + actions: [ + assign({ + isMomentaryActive: false, + }), + { + type: "momentaryHasPaused", }, - }, - }, - userSpeaking: { - description: - "User is speaking and being recorded by the microphone.\nWaveform animation.", - entry: [ { - type: "startAnimation", + type: "stopAnimation", params: { - animation: "userSpeaking", + animation: "glow", }, - }, - assign({ userIsSpeaking: true }), - { - type: "cancelCountdownAnimation", - }, + } ], - exit: { - type: "stopAnimation", - params: { - animation: "userSpeaking", + description: + 'Enable Momentary Mode. Now recording will only stop if the user releases the button.', + }, + "saypi:momentaryStop": { + actions: [ + assign({ + isMomentaryActive: false, + isMomentaryEnabled: false, + }), + { + type: "momentaryHasStopped", }, - }, - }, - }, - on: { - "saypi:momentaryPause": [ - { - target: [ - "#sayPi.listening.converting.transcribing", - ], - cond: "hasAudio", - actions: [ - assign({ - userIsSpeaking: false, - timeUserStoppedSpeaking: () => new Date().getTime(), - }), - { - type: "transcribeAudio", - }, - ], - }, - { - target: "#sayPi.listening.momentaryPaused", - cond: "hasNoAudio", - description: 'Pause Momentary Mode. Now recording will be ignored.', - actions: [ - { - type: "momentaryHasPaused", - }, - ] - }, - { - actions: [ - { - type: "justForFun2", - }, - ] - }, - ], + ], + description: + 'Enable Momentary Mode. Now recording will only stop if the user releases the button.', + }, }, }, - + momentaryPaused: { description: "In momentary mode and the button has been released, so the microphone is ignoring input", - entry: { - type: "justForFun", + + on: { + "saypi:momentaryListen": { + actions: [ + assign({ isMomentaryActive: true }), + { + type: "momentaryHasStarted" + }, + ], + target: "#sayPi.listening.recording", + description: 'Returning to the standard recording mode.', }, - on: { "saypi:momentaryStop": { actions: [ assign({ isMomentaryEnabled: false }), @@ -584,10 +527,7 @@ const machine = createMachine( target: "#sayPi.listening.recording", description: 'Returning to the standard recording mode.', }, - "saypi:momentaryListen": { - target: "#sayPi.listening.momentaryRecording", - description: 'Return to recording in momentary mode.', - }, + }, }, @@ -1173,13 +1113,7 @@ const machine = createMachine( pauseRecording: (context, event) => { EventBus.emit("audio:input:stop"); }, - justForFun : () => { - console.log("Entered just for fun!"); - }, - justForFun2 : () => { - console.log("Entered just for fun 2!"); - }, - + pauseRecordingIfInterruptionsNotAllowed: (context, event) => { const handsFreeInterrupt = userPreferences.getCachedAllowInterruptions(); @@ -1308,15 +1242,18 @@ const machine = createMachine( momentaryHasStarted: () => { console.log("SayPiMachine entered momentaryHasStarted()"); buttonModule.callMomentary(); + EventBus.emit("audio:input:reconnect"); // TODO: fix this sayPi.isMomentaryEnabled = true; }, momentaryHasPaused: () => { console.log("SayPiMachine entered momentaryHasPaused()"); buttonModule.pauseMomentary(); + EventBus.emit("audio:input:stop"); }, momentaryHasStopped: () => { console.log("SayPiMachine entered momentaryHasStopped()"); buttonModule.callActive(); + EventBus.emit("audio:input:reconnect"); }, callNotStarted: () => { if (buttonModule) { From b164c0bdc318132b3724df6e7a2369467852c919 Mon Sep 17 00:00:00 2001 From: john Date: Fri, 20 Dec 2024 23:15:17 +0000 Subject: [PATCH 04/25] Fixed momentary pause button not showing after interrupt button pressed. --- src/state-machines/SayPiMachine.ts | 40 ++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/src/state-machines/SayPiMachine.ts b/src/state-machines/SayPiMachine.ts index 38954e6560..0591d35396 100644 --- a/src/state-machines/SayPiMachine.ts +++ b/src/state-machines/SayPiMachine.ts @@ -506,6 +506,9 @@ const machine = createMachine( description: "In momentary mode and the button has been released, so the microphone is ignoring input", + entry: { + type: "doNothing", + }, on: { "saypi:momentaryListen": { actions: [ @@ -967,15 +970,30 @@ const machine = createMachine( }, waitingForPiToStopSpeaking: { on: { - "saypi:piStoppedSpeaking": { - target: "userInterrupting", - }, - }, + "saypi:piStoppedSpeaking":[ + { + cond: "isMomentaryEnabled", + actions: "momentaryHasPaused", + target: "#sayPi.listening.momentaryPaused", + }, + { + target: "userInterrupting", + cond: "isMomentaryDisabled" + }, + ]}, after: { - 500: { + 500: [ + { + cond: "isMomentaryEnabled", + actions: "momentaryHasPaused", + target: "#sayPi.listening.momentaryPaused", + }, + { target: "userInterrupting", + cond: "isMomentaryDisabled", description: "Fallback transition after 500ms if piStoppedSpeaking event does not fire.", }, + ] }, description: "Interrupt requested. Waiting for Pi to stop speaking before recording.", }, @@ -1248,6 +1266,7 @@ const machine = createMachine( momentaryHasPaused: () => { console.log("SayPiMachine entered momentaryHasPaused()"); buttonModule.pauseMomentary(); + AnimationModule.stopAnimation("glow"); EventBus.emit("audio:input:stop"); }, momentaryHasStopped: () => { @@ -1327,6 +1346,9 @@ const machine = createMachine( }, resumeAudio: () => { EventBus.emit("audio:output:resume"); + }, + doNothing: () => { + console.log("SayPiMachine, Entered doNothing()"); }, }, services: {}, @@ -1371,6 +1393,14 @@ const machine = createMachine( interruptionsNotAllowed: (context: SayPiContext) => { const allowInterrupt = userPreferences.getCachedAllowInterruptions(); return !allowInterrupt; + }, + isMomentaryEnabled: (context: SayPiContext) => { + console.log("Entered isMomentaryEnabled: result: " + context.isMomentaryEnabled); + return context.isMomentaryEnabled; + }, + isMomentaryDisabled: (context: SayPiContext) => { + console.log("SayPiMachine, Entered isMomentaryDiabled: result: " + (!context.isMomentaryEnabled)); + return !context.isMomentaryEnabled; }, }, delays: { From 28d35a097e1e9ae6bc43b6ae510233203e83f49e Mon Sep 17 00:00:00 2001 From: john Date: Sat, 28 Dec 2024 23:53:17 +0000 Subject: [PATCH 05/25] Working on submitting transcriptions from momentary paused mode, and not submitting transcriptions in momentary listening mode. --- src/events/EventModule.js | 2 ++ src/state-machines/SayPiMachine.ts | 29 ++++++++++++++++++++++++++--- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/src/events/EventModule.js b/src/events/EventModule.js index 4217607a51..c05ac926ba 100644 --- a/src/events/EventModule.js +++ b/src/events/EventModule.js @@ -22,6 +22,7 @@ const UI_SHOW_NOTIFICATION = "saypi:ui:show-notification"; const MOMENTARY_LISTEN = "saypi:momentaryListen"; const MOMENTARY_PAUSE = "saypi:momentaryPause"; const MOMENTARY_STOP = "saypi:momentaryStop"; +const MOMENTARY_SUBMIT_TRANSCRIPTIONS = "saypi:momentarySubmitTranscriptions"; /** * The EventModule translates events sent on the EventBus to StateMachine events, @@ -82,6 +83,7 @@ export default class EventModule { MOMENTARY_LISTEN, MOMENTARY_PAUSE, MOMENTARY_STOP, + MOMENTARY_SUBMIT_TRANSCRIPTIONS, ].forEach((eventName) => { EventBus.on(eventName, () => { actor.send(eventName); diff --git a/src/state-machines/SayPiMachine.ts b/src/state-machines/SayPiMachine.ts index 0591d35396..c2a4a4227a 100644 --- a/src/state-machines/SayPiMachine.ts +++ b/src/state-machines/SayPiMachine.ts @@ -77,6 +77,7 @@ type SayPiEvent = | { type: "saypi:momentaryListen" } | { type: "saypi:momentaryPause" } | { type: "saypi:momentaryStop" } + | { type: "saypi:momentarySubmitTranscriptions" } | { type: "saypi:callReady" } | { type: "saypi:callFailed" } | { type: "saypi:hangup" } @@ -507,7 +508,7 @@ const machine = createMachine( "In momentary mode and the button has been released, so the microphone is ignoring input", entry: { - type: "doNothing", + type: "submitTranscriptions" }, on: { "saypi:momentaryListen": { @@ -530,6 +531,10 @@ const machine = createMachine( target: "#sayPi.listening.recording", description: 'Returning to the standard recording mode.', }, + "saypi:momentarySubmitTranscriptions": { + target: "#sayPi.listening.converting.submitting", + description: 'Submit current transcriptions.', + }, }, }, @@ -546,8 +551,13 @@ const machine = createMachine( cond: "submissionConditionsMet", description: "Submit combined transcript to Pi.", }, + 100: { + target: "#sayPi.listening.recording", + cond: "momentaryIsActive", + description: "Will return to listening because momentary mode is active.", + }, }, - entry: { + entry: { type: "draftPrompt", }, invoke: { @@ -1152,6 +1162,12 @@ const machine = createMachine( EventBus.emit("audio:tearDownRecording"); }, + submitTranscriptions: (context: SayPiContext, event) => { + if(readyToSubmitOnAllowedState(true, context)){ + EventBus.emit("saypi:momentarySubmitTranscriptions"); + } + }, + reconnectAudio: (context, event) => { EventBus.emit("audio:input:reconnect"); }, @@ -1378,7 +1394,8 @@ const machine = createMachine( ) => { const { state } = meta; const autoSubmitEnabled = userPreferences.getCachedAutoSubmit(); - return autoSubmitEnabled && readyToSubmit(state, context); + const isMomentaryInactive = !context.isMomentaryEnabled && !context.isMomentaryActive; + return autoSubmitEnabled && readyToSubmit(state, context) && isMomentaryInactive; }, wasListening: (context: SayPiContext) => { return context.lastState === "listening"; @@ -1402,6 +1419,12 @@ const machine = createMachine( console.log("SayPiMachine, Entered isMomentaryDiabled: result: " + (!context.isMomentaryEnabled)); return !context.isMomentaryEnabled; }, + momentaryIsActive: (context: SayPiContext) => { + return context.isMomentaryEnabled && context.isMomentaryActive; + }, + momentaryDisabledOrPaused: (context: SayPiContext) => { + return !context.isMomentaryEnabled || (context.isMomentaryEnabled && !context.isMomentaryActive); + } }, delays: { submissionDelay: (context: SayPiContext, event: SayPiEvent) => { From 73d23a969ba17aa6dbffa19d335b443ec3e02560 Mon Sep 17 00:00:00 2001 From: john Date: Mon, 30 Dec 2024 23:46:16 +0000 Subject: [PATCH 06/25] Fixed momentary mode not engaging after interruption, but momentary mode (paused or engaged) currently doesn't send transcripts. --- src/state-machines/SayPiMachine.ts | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/state-machines/SayPiMachine.ts b/src/state-machines/SayPiMachine.ts index c2a4a4227a..8822c05454 100644 --- a/src/state-machines/SayPiMachine.ts +++ b/src/state-machines/SayPiMachine.ts @@ -1040,6 +1040,20 @@ const machine = createMachine( description: "User has spoken.", }, ], + "saypi:momentaryListen" : [ + { + actions: [ + assign({ + isMomentaryEnabled: true, + isMomentaryActive: true, + }), + { + type: "momentaryHasStarted" + }, + ], + target: "#sayPi.listening.recording", + }, + ], }, entry: [ { From 678919301ec1c0f084846a73bb830456e3641423 Mon Sep 17 00:00:00 2001 From: john Date: Thu, 2 Jan 2025 00:20:38 +0000 Subject: [PATCH 07/25] Momentary paused state submits collected speech, but does not return after response has ended. --- src/state-machines/SayPiMachine.ts | 63 ++++++++++++++++++++++++++---- 1 file changed, 55 insertions(+), 8 deletions(-) diff --git a/src/state-machines/SayPiMachine.ts b/src/state-machines/SayPiMachine.ts index 8822c05454..735a3bf163 100644 --- a/src/state-machines/SayPiMachine.ts +++ b/src/state-machines/SayPiMachine.ts @@ -469,6 +469,7 @@ const machine = createMachine( target: [ "momentaryPaused", + //"#sayPi.momentaryPaused", ], actions: [ assign({ @@ -477,6 +478,9 @@ const machine = createMachine( { type: "momentaryHasPaused", }, + { + type: "submitTranscriptions", + }, { type: "stopAnimation", params: { @@ -502,14 +506,11 @@ const machine = createMachine( }, }, }, - momentaryPaused: { description: "In momentary mode and the button has been released, so the microphone is ignoring input", - entry: { - type: "submitTranscriptions" - }, + on: { "saypi:momentaryListen": { actions: [ @@ -535,10 +536,25 @@ const machine = createMachine( target: "#sayPi.listening.converting.submitting", description: 'Submit current transcriptions.', }, - + "saypi:userStoppedSpeaking": [ + { + target: [ + "#sayPi.listening.converting.transcribing", + ], + cond: "hasAudio", + actions: [ + assign({ + userIsSpeaking: false, + timeUserStoppedSpeaking: () => new Date().getTime(), + }), + { + type: "transcribeAudio", + }, + ], + }, + ], }, }, - converting: { initial: "accumulating", states: { @@ -812,6 +828,9 @@ const machine = createMachine( type: "parallel", }, + + + responding: { initial: "piThinking", on: { @@ -984,6 +1003,7 @@ const machine = createMachine( { cond: "isMomentaryEnabled", actions: "momentaryHasPaused", + //target: "#sayPi.momentaryPaused", target: "#sayPi.listening.momentaryPaused", }, { @@ -996,6 +1016,7 @@ const machine = createMachine( { cond: "isMomentaryEnabled", actions: "momentaryHasPaused", + //target: "#sayPi.momentaryPaused", target: "#sayPi.listening.momentaryPaused", }, { @@ -1177,10 +1198,20 @@ const machine = createMachine( }, submitTranscriptions: (context: SayPiContext, event) => { - if(readyToSubmitOnAllowedState(true, context)){ + let isReadyToSubmit = readyToSubmitOnAllowedState(true, context); + console.log("entered submitTranscriptions() isReadyToSubmit: " + isReadyToSubmit); + if(isReadyToSubmit) { + console.log("calling for momentarySubmitTranscriptions event!"); EventBus.emit("saypi:momentarySubmitTranscriptions"); } }, + test123: (context: SayPiContext, event) => { + console.log("Entered test123() !!!"); + }, + + testAbc: (context: SayPiContext, event) => { + console.log("Entered testAbc() !!!"); + }, reconnectAudio: (context, event) => { EventBus.emit("audio:input:reconnect"); @@ -1364,13 +1395,28 @@ const machine = createMachine( wait_time_ms: submission_delay_ms, }); }, - clearPendingTranscriptionsAction: () => { + clearPendingTranscriptionsAction: (context: SayPiContext) => { // discard in-flight transcriptions. Called after a successful submission + /* + console.log("Entered clearPendingTranscriptionsAction()"); + + if(!context.isMomentaryEnabled){ + console.log("momentary is not enabled, so clearing pending transcripts!!"); + clearPendingTranscriptions(); + } + */ clearPendingTranscriptions(); }, clearTranscriptsAction: assign({ transcriptions: () => ({}), }), + clearTranscripts:(context: SayPiContext) => { + console.log("Entered clearTranscripts()"); + if(!context.isMomentaryEnabled) { + console.log("clearTranscripts() momentary is not enabled, so clearing transcripts!"); + context.transcriptions = {}; + } + }, pauseAudio: () => { EventBus.emit("audio:output:pause"); }, @@ -1500,6 +1546,7 @@ function readyToSubmitOnAllowedState( const empty = Object.keys(context.transcriptions).length === 0; const pending = isTranscriptionPending(); const ready = allowedState && !empty && !pending; + console.log("Entered readyToSubmitOnAllowedState() empty: " + empty + " any pending: " + pending + " is ready: " + ready); return ready; } function provisionallyReadyToSubmit(context: SayPiContext): boolean { From cf1804eef093240332d3a32809e421b88b3f99ae Mon Sep 17 00:00:00 2001 From: john Date: Thu, 9 Jan 2025 00:11:10 +0000 Subject: [PATCH 08/25] momentary paused icon (temp) returns after pi finishes speaking. Using stop-requested, instead of stop when momentary pause is invoked. Still not sending audio, and double transcription is generated. --- src/state-machines/SayPiMachine.ts | 75 ++++++++++++++++++++++++------ 1 file changed, 62 insertions(+), 13 deletions(-) diff --git a/src/state-machines/SayPiMachine.ts b/src/state-machines/SayPiMachine.ts index 735a3bf163..8136445205 100644 --- a/src/state-machines/SayPiMachine.ts +++ b/src/state-machines/SayPiMachine.ts @@ -469,7 +469,6 @@ const machine = createMachine( target: [ "momentaryPaused", - //"#sayPi.momentaryPaused", ], actions: [ assign({ @@ -828,8 +827,41 @@ const machine = createMachine( type: "parallel", }, - - + momentaryPaused2: { + description: + "In momentary mode and the button has been released, so the microphone is ignoring input", + + entry:[ + { + type: "pauseAudio", + }, + { + type: "momentaryReturnsToPaused", + }, + ], + on: { + "saypi:momentaryListen": { + actions: [ + assign({ isMomentaryActive: true }), + { + type: "momentaryHasStarted" + }, + ], + target: "#sayPi.listening.recording", + description: 'Returning to the standard recording mode.', + }, + "saypi:momentaryStop": { + actions: [ + assign({ isMomentaryEnabled: false }), + { + type: "momentaryHasStopped" + }, + ], + target: "#sayPi.listening.recording", + description: 'Returning to the standard recording mode.', + }, + }, + }, responding: { initial: "piThinking", @@ -902,11 +934,19 @@ const machine = createMachine( piSpeaking: { on: { "saypi:piStoppedSpeaking": [ + { + target: "#sayPi.momentaryPaused2", + cond: + { + type: "isMomentaryEnabled", + }, + }, { target: "#sayPi.listening", - cond: { - type: "wasListening", - }, + cond: + { + type: "wasListening", + }, }, { target: "#sayPi.inactive", @@ -915,9 +955,16 @@ const machine = createMachine( }, }, ], - "saypi:piFinishedSpeaking": { - target: "#sayPi.listening", - }, + "saypi:piFinishedSpeaking": [ + { + cond: "isMomentaryEnabled", + target: "#sayPi.momentaryPaused2", + }, + { + target: "#sayPi.listening", + cond: "isMomentaryDisabled" + }, + ], "saypi:userSpeaking": { target: "userInterrupting", cond: { @@ -1003,7 +1050,6 @@ const machine = createMachine( { cond: "isMomentaryEnabled", actions: "momentaryHasPaused", - //target: "#sayPi.momentaryPaused", target: "#sayPi.listening.momentaryPaused", }, { @@ -1016,7 +1062,6 @@ const machine = createMachine( { cond: "isMomentaryEnabled", actions: "momentaryHasPaused", - //target: "#sayPi.momentaryPaused", target: "#sayPi.listening.momentaryPaused", }, { @@ -1328,7 +1373,11 @@ const machine = createMachine( console.log("SayPiMachine entered momentaryHasPaused()"); buttonModule.pauseMomentary(); AnimationModule.stopAnimation("glow"); - EventBus.emit("audio:input:stop"); + EventBus.emit("audio:stopRecording"); //JAC temp message -> soft stop instead of hard stop, see if this will produce an audio blob + }, + momentaryReturnsToPaused: () => { + console.log("SayPiMachine entered momentaryReturnsToPaused()"); + buttonModule.pauseMomentary(); }, momentaryHasStopped: () => { console.log("SayPiMachine entered momentaryHasStopped()"); @@ -1458,7 +1507,7 @@ const machine = createMachine( return autoSubmitEnabled && readyToSubmit(state, context) && isMomentaryInactive; }, wasListening: (context: SayPiContext) => { - return context.lastState === "listening"; + return context.lastState === "listening" && !context.isMomentaryEnabled; }, wasInactive: (context: SayPiContext) => { return context.lastState === "inactive"; From cff3af497d5ced07a519e9ed4acfb294233e7c45 Mon Sep 17 00:00:00 2001 From: john Date: Mon, 13 Jan 2025 23:54:35 +0000 Subject: [PATCH 09/25] Working momentary mode basic operation, but interrupts are currently broken when momentary mode is enabled. --- src/state-machines/SayPiMachine.ts | 140 +++++++++++------------------ 1 file changed, 52 insertions(+), 88 deletions(-) diff --git a/src/state-machines/SayPiMachine.ts b/src/state-machines/SayPiMachine.ts index 8136445205..3b4b80764c 100644 --- a/src/state-machines/SayPiMachine.ts +++ b/src/state-machines/SayPiMachine.ts @@ -465,31 +465,25 @@ const machine = createMachine( description: 'Enable Momentary Mode. Now recording will only stop if the user releases the button.', }, - "saypi:momentaryPause": { - - target: [ - "momentaryPaused", - ], - actions: [ - assign({ - isMomentaryActive: false, - }), - { - type: "momentaryHasPaused", - }, - { - type: "submitTranscriptions", - }, - { - type: "stopAnimation", - params: { - animation: "glow", + "saypi:momentaryPause": + { + target: "#sayPi.listening.converting.submitting", + cond: "readyToSubmitFromMomentary", + actions: [ + assign({ + isMomentaryActive: false, + }), + { + type: "momentaryHasPaused", }, - } - ], - description: - 'Enable Momentary Mode. Now recording will only stop if the user releases the button.', - }, + { + type: "stopAnimation", + params: { + animation: "glow", + }, + }, + ], + }, "saypi:momentaryStop": { actions: [ assign({ @@ -505,55 +499,6 @@ const machine = createMachine( }, }, }, - momentaryPaused: { - description: - "In momentary mode and the button has been released, so the microphone is ignoring input", - - - on: { - "saypi:momentaryListen": { - actions: [ - assign({ isMomentaryActive: true }), - { - type: "momentaryHasStarted" - }, - ], - target: "#sayPi.listening.recording", - description: 'Returning to the standard recording mode.', - }, - "saypi:momentaryStop": { - actions: [ - assign({ isMomentaryEnabled: false }), - { - type: "momentaryHasStopped" - }, - ], - target: "#sayPi.listening.recording", - description: 'Returning to the standard recording mode.', - }, - "saypi:momentarySubmitTranscriptions": { - target: "#sayPi.listening.converting.submitting", - description: 'Submit current transcriptions.', - }, - "saypi:userStoppedSpeaking": [ - { - target: [ - "#sayPi.listening.converting.transcribing", - ], - cond: "hasAudio", - actions: [ - assign({ - userIsSpeaking: false, - timeUserStoppedSpeaking: () => new Date().getTime(), - }), - { - type: "transcribeAudio", - }, - ], - }, - ], - }, - }, converting: { initial: "accumulating", states: { @@ -566,11 +511,12 @@ const machine = createMachine( cond: "submissionConditionsMet", description: "Submit combined transcript to Pi.", }, - 100: { + /* + momentarySubmissionDelay: { target: "#sayPi.listening.recording", cond: "momentaryIsActive", description: "Will return to listening because momentary mode is active.", - }, + }, */ }, entry: { type: "draftPrompt", @@ -701,6 +647,12 @@ const machine = createMachine( }, description: "Successfully transcribed user audio to text.", }, + + "saypi:momentaryPause": { + actions: { + type: "logPauseEvent", + }, + }, "saypi:transcribeFailed": { target: [ "accumulating", @@ -1047,11 +999,13 @@ const machine = createMachine( waitingForPiToStopSpeaking: { on: { "saypi:piStoppedSpeaking":[ + /* { cond: "isMomentaryEnabled", actions: "momentaryHasPaused", target: "#sayPi.listening.momentaryPaused", - }, + }, + */ { target: "userInterrupting", cond: "isMomentaryDisabled" @@ -1059,11 +1013,13 @@ const machine = createMachine( ]}, after: { 500: [ + /* { cond: "isMomentaryEnabled", actions: "momentaryHasPaused", target: "#sayPi.listening.momentaryPaused", }, + */ { target: "userInterrupting", cond: "isMomentaryDisabled", @@ -1200,6 +1156,12 @@ const machine = createMachine( }); } }, + logPauseEvent: ( + SayPiContext, + event: SayPiTranscribedEvent + ) => { + console.log("-------> logPauseEvent() Momentary Pause detected from accumulating state!"); + }, acquireMicrophone: (context, event) => { // warmup the microphone on idle in mobile view, @@ -1241,23 +1203,15 @@ const machine = createMachine( EventBus.emit("audio:stopRecording"); EventBus.emit("audio:tearDownRecording"); }, - +/* submitTranscriptions: (context: SayPiContext, event) => { let isReadyToSubmit = readyToSubmitOnAllowedState(true, context); console.log("entered submitTranscriptions() isReadyToSubmit: " + isReadyToSubmit); if(isReadyToSubmit) { console.log("calling for momentarySubmitTranscriptions event!"); - EventBus.emit("saypi:momentarySubmitTranscriptions"); + EventBus.emit("saypi:submit"); } - }, - test123: (context: SayPiContext, event) => { - console.log("Entered test123() !!!"); - }, - - testAbc: (context: SayPiContext, event) => { - console.log("Entered testAbc() !!!"); - }, - + },*/ reconnectAudio: (context, event) => { EventBus.emit("audio:input:reconnect"); }, @@ -1506,6 +1460,13 @@ const machine = createMachine( const isMomentaryInactive = !context.isMomentaryEnabled && !context.isMomentaryActive; return autoSubmitEnabled && readyToSubmit(state, context) && isMomentaryInactive; }, + readyToSubmitFromMomentary: (context: SayPiContext, event: SayPiEvent) => { + console.log("-----> Entered readyToSubmitFromMomentary() result: " + readyToSubmitOnAllowedState(true, context)); + return readyToSubmitOnAllowedState(true, context); + }, + notReadyToSubmitFromMomentary: (context: SayPiContext, event: SayPiEvent) => { + return !readyToSubmitOnAllowedState(true, context);; + }, wasListening: (context: SayPiContext) => { return context.lastState === "listening" && !context.isMomentaryEnabled; }, @@ -1582,9 +1543,12 @@ const machine = createMachine( // Capture the delay for analytics events lastSubmissionDelay = finalDelay; - + return finalDelay; }, + momentarySubmissionDelay: (context: SayPiContext, event: SayPiEvent) => { + return 100; + }, }, } ); From 50539a18f51eaa775838df3e39a3dea03d8c016d Mon Sep 17 00:00:00 2001 From: john Date: Tue, 14 Jan 2025 23:45:40 +0000 Subject: [PATCH 10/25] Fixed bad state after interrupt button pressed while momentary mode is enabled. --- src/state-machines/SayPiMachine.ts | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/state-machines/SayPiMachine.ts b/src/state-machines/SayPiMachine.ts index 3b4b80764c..18949597e1 100644 --- a/src/state-machines/SayPiMachine.ts +++ b/src/state-machines/SayPiMachine.ts @@ -779,7 +779,7 @@ const machine = createMachine( type: "parallel", }, - momentaryPaused2: { + momentaryPaused: { description: "In momentary mode and the button has been released, so the microphone is ignoring input", @@ -887,7 +887,7 @@ const machine = createMachine( on: { "saypi:piStoppedSpeaking": [ { - target: "#sayPi.momentaryPaused2", + target: "#sayPi.momentaryPaused", cond: { type: "isMomentaryEnabled", @@ -910,7 +910,7 @@ const machine = createMachine( "saypi:piFinishedSpeaking": [ { cond: "isMomentaryEnabled", - target: "#sayPi.momentaryPaused2", + target: "#sayPi.momentaryPaused", }, { target: "#sayPi.listening", @@ -934,6 +934,12 @@ const machine = createMachine( description: `The user has forced an interruption, i.e. tapped to interrupt Pi, during a call.`, actions: "pauseAudio", cond: "wasListening", + }, + { + target: "#sayPi.momentaryPaused", + description: `The user has forced an interruption while momentary mode was enabled, i.e. tapped to interrupt Pi, during a call.`, + actions: "pauseAudio", + cond: "wasListeningWithMomentary", }, { target: "#sayPi.inactive", @@ -999,13 +1005,11 @@ const machine = createMachine( waitingForPiToStopSpeaking: { on: { "saypi:piStoppedSpeaking":[ - /* { cond: "isMomentaryEnabled", actions: "momentaryHasPaused", - target: "#sayPi.listening.momentaryPaused", + target: "#sayPi.momentaryPaused", }, - */ { target: "userInterrupting", cond: "isMomentaryDisabled" @@ -1013,13 +1017,11 @@ const machine = createMachine( ]}, after: { 500: [ - /* { cond: "isMomentaryEnabled", actions: "momentaryHasPaused", - target: "#sayPi.listening.momentaryPaused", + target: "#sayPi.momentaryPaused", }, - */ { target: "userInterrupting", cond: "isMomentaryDisabled", @@ -1470,6 +1472,9 @@ const machine = createMachine( wasListening: (context: SayPiContext) => { return context.lastState === "listening" && !context.isMomentaryEnabled; }, + wasListeningWithMomentary: (context: SayPiContext) => { + return context.lastState === "listening" && context.isMomentaryEnabled; + }, wasInactive: (context: SayPiContext) => { return context.lastState === "inactive"; }, From 4e828a610e0856260d150badd08f7918b88e8685 Mon Sep 17 00:00:00 2001 From: john Date: Thu, 16 Jan 2025 00:34:28 +0000 Subject: [PATCH 11/25] Fixed transcript not submitting if momentary pause is triggered during accumulation or transcription. --- src/state-machines/SayPiMachine.ts | 37 +++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/src/state-machines/SayPiMachine.ts b/src/state-machines/SayPiMachine.ts index 18949597e1..6fec398849 100644 --- a/src/state-machines/SayPiMachine.ts +++ b/src/state-machines/SayPiMachine.ts @@ -603,6 +603,22 @@ const machine = createMachine( description: "Out of sequence empty response from the /transcribe API", }, + "saypi:momentaryPause": { + actions: [ + assign({ + isMomentaryActive: false, + }), + { + type: "momentaryHasPaused", + }, + { + type: "stopAnimation", + params: { + animation: "glow", + }, + }, + ], + }, }, }, submitting: { @@ -647,11 +663,21 @@ const machine = createMachine( }, description: "Successfully transcribed user audio to text.", }, - "saypi:momentaryPause": { - actions: { - type: "logPauseEvent", - }, + actions: [ + assign({ + isMomentaryActive: false, + }), + { + type: "momentaryHasPaused", + }, + { + type: "stopAnimation", + params: { + animation: "glow", + }, + }, + ], }, "saypi:transcribeFailed": { target: [ @@ -1459,8 +1485,7 @@ const machine = createMachine( ) => { const { state } = meta; const autoSubmitEnabled = userPreferences.getCachedAutoSubmit(); - const isMomentaryInactive = !context.isMomentaryEnabled && !context.isMomentaryActive; - return autoSubmitEnabled && readyToSubmit(state, context) && isMomentaryInactive; + return autoSubmitEnabled && readyToSubmit(state, context) && !context.isMomentaryActive; }, readyToSubmitFromMomentary: (context: SayPiContext, event: SayPiEvent) => { console.log("-----> Entered readyToSubmitFromMomentary() result: " + readyToSubmitOnAllowedState(true, context)); From 5f1e688edcb38b18aaa0d18975ee836d150318f7 Mon Sep 17 00:00:00 2001 From: john Date: Fri, 17 Jan 2025 00:35:38 +0000 Subject: [PATCH 12/25] Clean-up of redundant methods. --- src/state-machines/SayPiMachine.ts | 217 +++++++++++++---------------- 1 file changed, 99 insertions(+), 118 deletions(-) diff --git a/src/state-machines/SayPiMachine.ts b/src/state-machines/SayPiMachine.ts index 6fec398849..c4082c1a2e 100644 --- a/src/state-machines/SayPiMachine.ts +++ b/src/state-machines/SayPiMachine.ts @@ -128,6 +128,7 @@ type SayPiStateSchema = { }; }; }; + momentaryPaused: {}; responding: { states: { piThinking: {}; @@ -465,25 +466,24 @@ const machine = createMachine( description: 'Enable Momentary Mode. Now recording will only stop if the user releases the button.', }, - "saypi:momentaryPause": - { - target: "#sayPi.listening.converting.submitting", - cond: "readyToSubmitFromMomentary", - actions: [ - assign({ - isMomentaryActive: false, - }), - { - type: "momentaryHasPaused", - }, - { - type: "stopAnimation", - params: { - animation: "glow", - }, + "saypi:momentaryPause": { + target: "#sayPi.listening.converting.submitting", + cond: "readyToSubmitFromMomentary", + actions: [ + assign({ + isMomentaryActive: false, + }), + { + type: "momentaryHasPaused", + }, + { + type: "stopAnimation", + params: { + animation: "glow", }, - ], - }, + }, + ], + }, "saypi:momentaryStop": { actions: [ assign({ @@ -511,12 +511,6 @@ const machine = createMachine( cond: "submissionConditionsMet", description: "Submit combined transcript to Pi.", }, - /* - momentarySubmissionDelay: { - target: "#sayPi.listening.recording", - cond: "momentaryIsActive", - description: "Will return to listening because momentary mode is active.", - }, */ }, entry: { type: "draftPrompt", @@ -806,17 +800,16 @@ const machine = createMachine( }, momentaryPaused: { - description: - "In momentary mode and the button has been released, so the microphone is ignoring input", - - entry:[ - { - type: "pauseAudio", - }, - { - type: "momentaryReturnsToPaused", - }, - ], + // this state obviates the modification of the user_interrupting state for when momentary mode is enabled + description: "In momentary mode and the button has been released, so the microphone is ignoring input", + entry:[ + { + type: "pauseAudio", + }, + { + type: "momentaryReturnsToPaused", + }, + ], on: { "saypi:momentaryListen": { actions: [ @@ -1049,11 +1042,11 @@ const machine = createMachine( target: "#sayPi.momentaryPaused", }, { - target: "userInterrupting", - cond: "isMomentaryDisabled", - description: "Fallback transition after 500ms if piStoppedSpeaking event does not fire.", - }, - ] + target: "userInterrupting", + cond: "isMomentaryDisabled", + description: "Fallback transition after 500ms if piStoppedSpeaking event does not fire.", + }, + ] }, description: "Interrupt requested. Waiting for Pi to stop speaking before recording.", }, @@ -1184,13 +1177,7 @@ const machine = createMachine( }); } }, - logPauseEvent: ( - SayPiContext, - event: SayPiTranscribedEvent - ) => { - console.log("-------> logPauseEvent() Momentary Pause detected from accumulating state!"); - }, - + acquireMicrophone: (context, event) => { // warmup the microphone on idle in mobile view, // since there's no mouseover event to trigger it @@ -1231,15 +1218,7 @@ const machine = createMachine( EventBus.emit("audio:stopRecording"); EventBus.emit("audio:tearDownRecording"); }, -/* - submitTranscriptions: (context: SayPiContext, event) => { - let isReadyToSubmit = readyToSubmitOnAllowedState(true, context); - console.log("entered submitTranscriptions() isReadyToSubmit: " + isReadyToSubmit); - if(isReadyToSubmit) { - console.log("calling for momentarySubmitTranscriptions event!"); - EventBus.emit("saypi:submit"); - } - },*/ + reconnectAudio: (context, event) => { EventBus.emit("audio:input:reconnect"); }, @@ -1279,6 +1258,7 @@ const machine = createMachine( getPromptOrNull()?.setMessage(message); } }, + callStartingPrompt: () => { const message = getMessage("callStarting"); if (message) { @@ -1287,18 +1267,21 @@ const machine = createMachine( getPromptOrNull()?.setMessage(message); } }, + thinkingPrompt: () => { const message = getMessage("assistantIsThinking", chatbot.getName()); if (message) { getPromptOrNull()?.setMessage(message); } }, + writingPrompt: () => { const message = getMessage("assistantIsWriting", chatbot.getName()); if (message) { getPromptOrNull()?.setMessage(message); } }, + speakingPrompt: (context: SayPiContext) => { const handsFreeInterrupt = userPreferences.getCachedAllowInterruptions(); @@ -1312,6 +1295,7 @@ const machine = createMachine( getPromptOrNull()?.setMessage(message); } }, + interruptingPiPrompt: () => { const message = getMessage( "userStartedInterrupting", @@ -1321,9 +1305,11 @@ const machine = createMachine( getPromptOrNull()?.setMessage(message); } }, + clearPrompt: (context: SayPiContext) => { getPromptOrNull()?.setMessage(context.defaultPlaceholderText); }, + draftPrompt: (context: SayPiContext) => { const text = mergeService .mergeTranscriptsLocal(context.transcriptions) @@ -1341,83 +1327,78 @@ const machine = createMachine( callIsStarting: () => { buttonModule.callStarting(); }, + callFailedToStart: () => { buttonModule.callInactive(); audibleNotifications.callFailed(); }, - momentaryHasStarted: () => { - console.log("SayPiMachine entered momentaryHasStarted()"); - buttonModule.callMomentary(); - EventBus.emit("audio:input:reconnect"); - // TODO: fix this sayPi.isMomentaryEnabled = true; - }, - momentaryHasPaused: () => { - console.log("SayPiMachine entered momentaryHasPaused()"); - buttonModule.pauseMomentary(); - AnimationModule.stopAnimation("glow"); - EventBus.emit("audio:stopRecording"); //JAC temp message -> soft stop instead of hard stop, see if this will produce an audio blob - }, - momentaryReturnsToPaused: () => { - console.log("SayPiMachine entered momentaryReturnsToPaused()"); - buttonModule.pauseMomentary(); - }, - momentaryHasStopped: () => { - console.log("SayPiMachine entered momentaryHasStopped()"); - buttonModule.callActive(); - EventBus.emit("audio:input:reconnect"); - }, + callNotStarted: () => { if (buttonModule) { // buttonModule may not be available on initial load buttonModule.callInactive(); } }, + callHasStarted: () => { buttonModule.callActive(); audibleNotifications.callStarted(); EventBus.emit("session:started"); }, + callInterruptible: () => { buttonModule.callInterruptible(); }, + callInterruptibleIfListening: (context: SayPiContext) => { if (context.lastState === "listening") { buttonModule.callInterruptible(); } }, + callContinues: () => { buttonModule.callActive(); }, + callHasEnded: () => { visualNotifications.listeningStopped(); buttonModule.callInactive(); audibleNotifications.callEnded(); EventBus.emit("session:ended"); }, + callHasErrors: () => { buttonModule.callError(); }, + callHasNoErrors: () => { buttonModule.callActive(); }, + disableCallButton: () => { buttonModule.disableCallButton(); }, + enableCallButton: () => { buttonModule.enableCallButton(); }, + cancelCountdownAnimation: () => { visualNotifications.listeningStopped(); }, + activateAudioOutput: () => { audioControls.activateAudioOutput(true); }, + requestWakeLock: () => { requestWakeLock(); }, + releaseWakeLock: () => { releaseWakeLock(); }, + notifySentMessage: (context: SayPiContext, event: SayPiEvent) => { const delay_ms = Date.now() - context.timeUserStoppedSpeaking; const submission_delay_ms = lastSubmissionDelay; @@ -1426,36 +1407,42 @@ const machine = createMachine( wait_time_ms: submission_delay_ms, }); }, + clearPendingTranscriptionsAction: (context: SayPiContext) => { // discard in-flight transcriptions. Called after a successful submission - /* - console.log("Entered clearPendingTranscriptionsAction()"); - - if(!context.isMomentaryEnabled){ - console.log("momentary is not enabled, so clearing pending transcripts!!"); - clearPendingTranscriptions(); - } - */ clearPendingTranscriptions(); }, + clearTranscriptsAction: assign({ transcriptions: () => ({}), }), - clearTranscripts:(context: SayPiContext) => { - console.log("Entered clearTranscripts()"); - if(!context.isMomentaryEnabled) { - console.log("clearTranscripts() momentary is not enabled, so clearing transcripts!"); - context.transcriptions = {}; - } - }, + pauseAudio: () => { EventBus.emit("audio:output:pause"); }, + resumeAudio: () => { EventBus.emit("audio:output:resume"); }, - doNothing: () => { - console.log("SayPiMachine, Entered doNothing()"); + + momentaryHasStarted: () => { + buttonModule.callMomentary(); + EventBus.emit("audio:input:reconnect"); + }, + + momentaryHasPaused: () => { + buttonModule.pauseMomentary(); + AnimationModule.stopAnimation("glow"); + EventBus.emit("audio:stopRecording"); //JAC temp message -> soft stop instead of hard stop, see if this will produce an audio blob + }, + + momentaryReturnsToPaused: () => { + buttonModule.pauseMomentary(); + }, + + momentaryHasStopped: () => { + buttonModule.callActive(); + EventBus.emit("audio:input:reconnect"); }, }, services: {}, @@ -1487,44 +1474,35 @@ const machine = createMachine( const autoSubmitEnabled = userPreferences.getCachedAutoSubmit(); return autoSubmitEnabled && readyToSubmit(state, context) && !context.isMomentaryActive; }, - readyToSubmitFromMomentary: (context: SayPiContext, event: SayPiEvent) => { - console.log("-----> Entered readyToSubmitFromMomentary() result: " + readyToSubmitOnAllowedState(true, context)); - return readyToSubmitOnAllowedState(true, context); - }, - notReadyToSubmitFromMomentary: (context: SayPiContext, event: SayPiEvent) => { - return !readyToSubmitOnAllowedState(true, context);; - }, + wasListening: (context: SayPiContext) => { return context.lastState === "listening" && !context.isMomentaryEnabled; }, - wasListeningWithMomentary: (context: SayPiContext) => { - return context.lastState === "listening" && context.isMomentaryEnabled; - }, + wasInactive: (context: SayPiContext) => { return context.lastState === "inactive"; }, + interruptionsAllowed: (context: SayPiContext) => { const allowInterrupt = userPreferences.getCachedAllowInterruptions(); return allowInterrupt; - }, - interruptionsNotAllowed: (context: SayPiContext) => { - const allowInterrupt = userPreferences.getCachedAllowInterruptions(); - return !allowInterrupt; - }, + }, + isMomentaryEnabled: (context: SayPiContext) => { - console.log("Entered isMomentaryEnabled: result: " + context.isMomentaryEnabled); return context.isMomentaryEnabled; - }, + }, + isMomentaryDisabled: (context: SayPiContext) => { - console.log("SayPiMachine, Entered isMomentaryDiabled: result: " + (!context.isMomentaryEnabled)); return !context.isMomentaryEnabled; + }, + + readyToSubmitFromMomentary: (context: SayPiContext) => { + return readyToSubmitOnAllowedState(true, context); + }, + + wasListeningWithMomentary: (context: SayPiContext) => { + return context.lastState === "listening" && context.isMomentaryEnabled; }, - momentaryIsActive: (context: SayPiContext) => { - return context.isMomentaryEnabled && context.isMomentaryActive; - }, - momentaryDisabledOrPaused: (context: SayPiContext) => { - return !context.isMomentaryEnabled || (context.isMomentaryEnabled && !context.isMomentaryActive); - } }, delays: { submissionDelay: (context: SayPiContext, event: SayPiEvent) => { @@ -1582,6 +1560,7 @@ const machine = createMachine( }, } ); + function readyToSubmitOnAllowedState( allowedState: boolean, context: SayPiContext @@ -1592,10 +1571,12 @@ function readyToSubmitOnAllowedState( console.log("Entered readyToSubmitOnAllowedState() empty: " + empty + " any pending: " + pending + " is ready: " + ready); return ready; } + function provisionallyReadyToSubmit(context: SayPiContext): boolean { const allowedState = !(context.userIsSpeaking || context.isTranscribing); // we don't have access to the state, so we read from a copy in the context (!DRY) return readyToSubmitOnAllowedState(allowedState, context); } + function readyToSubmit( state: State, context: SayPiContext From c800d59726debcc7d841c4a81608a44c9e3c8d30 Mon Sep 17 00:00:00 2001 From: john Date: Mon, 20 Jan 2025 23:22:43 +0000 Subject: [PATCH 13/25] Cleaned up the button module. --- src/ButtonModule.js | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/src/ButtonModule.js b/src/ButtonModule.js index 0a4813d20a..32c4d1cade 100644 --- a/src/ButtonModule.js +++ b/src/ButtonModule.js @@ -380,21 +380,21 @@ class ButtonModule { } createEvent(eventName) { - return () => { - console.log(" ^^^^^^^^^^^ ButtonModule.createEvent() ->>> : " + eventName); - this.sayPiActor.send(eventName); }; + this.sayPiActor.send(eventName); + }; } callStarting(callButton) { const label = getMessage("callStarting"); - this.updateCallButton(callButton, callStartingIconSVG, label, () => - this.sayPiActor.send("saypi:hangup") + this.updateCallButton(callButton, + callStartingIconSVG, + label, + () => this.sayPiActor.send("saypi:hangup") ); } callActive(callButton) { - console.log("ButtonModule entered callActive()"); this.updateLongClickCallButton( callButton, hangupIconSVG, @@ -406,7 +406,6 @@ class ButtonModule { } callMomentary(callButton) { - console.log("ButtonModule entered callMomentary()"); this.updateLongClickCallButton( callButton, momentaryListeningIconSVG, @@ -418,7 +417,6 @@ class ButtonModule { } pauseMomentary(callButton) { - console.log("ButtonModule entered pauseMomentary()"); this.updateLongClickCallButton( callButton, momentaryPausedIconSVG, @@ -452,14 +450,11 @@ class ButtonModule { isShortClick(clickStartTime, limit) { const clickDuration = (Date.now() - clickStartTime); - console.log("ButtonModule.isShortClick() startTime: " + clickStartTime + " duration: " + clickDuration); return clickStartTime == 0 || clickDuration < limit; } - handleLongClick(button, onClick, onLongPressDown, onLongPressUp ) { const longPressMinimumMilliseconds = 500; - var clickStartTime = 0; var isMouseUpDetected = false; onClick = this.setEmptyDefault(onClick); @@ -471,7 +466,6 @@ class ButtonModule { this.clickStartTime = Date.now(); window.setTimeout( () => { if(!isMouseUpDetected) { - console.log("ButtonModule.handleLongClick(): long press down detected!"); onLongPressDown(); } }, longPressMinimumMilliseconds); @@ -480,10 +474,8 @@ class ButtonModule { button.onmouseup = () => { isMouseUpDetected = true; if(this.isShortClick(this.clickStartTime, longPressMinimumMilliseconds)) { - console.log("ButtonModule.handleLongClick(): short click detected!"); onClick(); } else { - console.log("ButtonModule.handleLongClick(): long press up detected!"); onLongPressUp(); } } From 1fc35d47bc7251a058b01856b68126c014796373 Mon Sep 17 00:00:00 2001 From: john Date: Tue, 21 Jan 2025 00:16:00 +0000 Subject: [PATCH 14/25] Ensured that glow animation is restarted when momentary button is pressed, and added new momentary icons. --- src/icons/momentary_listening.svg | 118 +++++++++++++++++++++++++++--- src/icons/momentary_paused.svg | 118 +++++++++++++++++++++++++++--- 2 files changed, 216 insertions(+), 20 deletions(-) diff --git a/src/icons/momentary_listening.svg b/src/icons/momentary_listening.svg index 06cd8f1148..cc859447a1 100644 --- a/src/icons/momentary_listening.svg +++ b/src/icons/momentary_listening.svg @@ -1,10 +1,108 @@ - - - - \ No newline at end of file + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/icons/momentary_paused.svg b/src/icons/momentary_paused.svg index 3f4c78fcf3..0c8d3118de 100644 --- a/src/icons/momentary_paused.svg +++ b/src/icons/momentary_paused.svg @@ -1,10 +1,108 @@ - - - - \ No newline at end of file + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From d7ecc7b18089eb2abc00e2569b7215c3f238832a Mon Sep 17 00:00:00 2001 From: john Date: Wed, 22 Jan 2025 00:21:04 +0000 Subject: [PATCH 15/25] Making sure animation stops in immersive mode with momentary paused, and ensuring that UI elements don't fade when momentary mode is active in immersive mode. --- src/FullscreenModule.ts | 16 +++++++++++++++- src/state-machines/FocusMachine.ts | 15 +++++++++++++++ src/state-machines/SayPiMachine.ts | 4 ++++ 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/src/FullscreenModule.ts b/src/FullscreenModule.ts index 416edc4640..bce4e16c8f 100644 --- a/src/FullscreenModule.ts +++ b/src/FullscreenModule.ts @@ -8,12 +8,24 @@ import { isMobileDevice } from "./UserAgentModule"; const focusActor = interpret(focusMachine); const tickInterval = 1000; var ticker: NodeJS.Timeout; -const userInputEvents = ["mousemove", "click", "keypress"]; +const userInputEvents = ["mousemove", "keypress"]; function handleUserInput() { focusActor.send({ type: "blur" }); } +function handleMouseDown() { + if(document.fullscreenEnabled) { + focusActor.send({ type: "pause" }); + } +} + +function handleMouseUp() { + if(document.fullscreenEnabled) { + focusActor.send({ type: "resume" }); + } +} + function startFocusModeListener() { focusActor.start(); ticker = setInterval(() => { @@ -23,6 +35,8 @@ function startFocusModeListener() { for (const event of userInputEvents) { document.addEventListener(event, handleUserInput); } + document.addEventListener("mousedown", handleMouseDown); + document.addEventListener("mouseup", handleMouseUp); } function stopFocusModeListener() { diff --git a/src/state-machines/FocusMachine.ts b/src/state-machines/FocusMachine.ts index e01528bf0e..3bb5d254e6 100644 --- a/src/state-machines/FocusMachine.ts +++ b/src/state-machines/FocusMachine.ts @@ -17,6 +17,8 @@ type FocusContext = { }; type TickEvent = { type: "tick"; time_ms: number }; type BlurEvent = { type: "blur" }; +type PauseEvent = { type: "pause" }; +type ResumeEvent = { type: "resume" }; export const machine = createMachine( { @@ -45,6 +47,19 @@ export const machine = createMachine( tick: { actions: "incrementInactivityTime", }, + pause: { + target: "Paused" + } + }, + }, + Paused: { + description: + "The machine is not waiting for user activity", + on: { + resume: { + actions: "resetInactivityTime", + target: "#focusMachine", + }, }, }, Focused: { diff --git a/src/state-machines/SayPiMachine.ts b/src/state-machines/SayPiMachine.ts index c4082c1a2e..8df1f29876 100644 --- a/src/state-machines/SayPiMachine.ts +++ b/src/state-machines/SayPiMachine.ts @@ -1427,21 +1427,25 @@ const machine = createMachine( momentaryHasStarted: () => { buttonModule.callMomentary(); + AnimationModule.startAnimation("glow"); EventBus.emit("audio:input:reconnect"); }, momentaryHasPaused: () => { buttonModule.pauseMomentary(); AnimationModule.stopAnimation("glow"); + AnimationModule.stopAnimation("userSpeaking"); EventBus.emit("audio:stopRecording"); //JAC temp message -> soft stop instead of hard stop, see if this will produce an audio blob }, momentaryReturnsToPaused: () => { + AnimationModule.stopAnimation("glow"); buttonModule.pauseMomentary(); }, momentaryHasStopped: () => { buttonModule.callActive(); + AnimationModule.startAnimation("glow"); EventBus.emit("audio:input:reconnect"); }, }, From 78dc75e896ccdcac1efd2b58c43a9e6434acd3f2 Mon Sep 17 00:00:00 2001 From: john Date: Mon, 27 Jan 2025 00:23:09 +0000 Subject: [PATCH 16/25] Mouse-out from the call button now pauses momentary mode if it was activated. --- src/ButtonModule.js | 70 ++++++++----------- src/buttons/ButtonUpdater.js | 132 +++++++++++++++++++++++++++++++++++ 2 files changed, 160 insertions(+), 42 deletions(-) create mode 100644 src/buttons/ButtonUpdater.js diff --git a/src/ButtonModule.js b/src/ButtonModule.js index 32c4d1cade..2895b786f1 100644 --- a/src/ButtonModule.js +++ b/src/ButtonModule.js @@ -25,6 +25,7 @@ import { IconModule } from "./icons/IconModule.ts"; import { ImmersionStateChecker } from "./ImmersionServiceLite.ts"; import { GlowColorUpdater } from "./buttons/GlowColorUpdater.js"; import { openSettings } from "./popup/popupopener.ts"; +import { ButtonUpdater } from "./buttons/ButtonUpdater.js"; class ButtonModule { /** @@ -48,6 +49,7 @@ class ButtonModule { // track whether a call is active, so that new button instances can be initialized correctly this.callIsActive = false; this.clickStartTime = 0; + this.buttonUpdater = new ButtonUpdater(); } registerOtherEvents() { @@ -361,24 +363,6 @@ class ButtonModule { this.callIsActive = isActive; } - updateLongClickCallButton(callButton, svgIcon, label, clickEventName, longPressEventName, longReleaseEventName, isActive = true) { - if (!callButton) { - callButton = document.getElementById("saypi-callButton"); - } - if (callButton) { - this.removeChildrenFrom(callButton); - this.addIconTo(callButton, svgIcon); - callButton.onclick = () => {}; - this.setAriaLabelOf(callButton, label); - let onClick = this.createEvent(clickEventName); - let onLongPressDown = this.createEvent(longPressEventName); - let onLongPressUp = this.createEvent(longReleaseEventName); - this.handleLongClick(callButton, onClick, onLongPressDown, onLongPressUp); - this.toggleActiveState(callButton, isActive); - } - this.callIsActive = isActive; - } - createEvent(eventName) { return () => { this.sayPiActor.send(eventName); @@ -395,36 +379,38 @@ class ButtonModule { } callActive(callButton) { - this.updateLongClickCallButton( - callButton, - hangupIconSVG, - "callInProgress", - "saypi:hangup", - "saypi:momentaryListen", - "saypi:momentaryPause", - ); + this.buttonUpdater.updateCallButton({ + button:callButton, + icon: hangupIconSVG, + label: "callInProgress", + clickEventName: "saypi:hangup", + longPressEventName: "saypi:momentaryListen", + isCallActive: true, + }); } callMomentary(callButton) { - this.updateLongClickCallButton( - callButton, - momentaryListeningIconSVG, - "callInProgress", - "saypi:momentaryStop", - "saypi:momentaryPause", - "saypi:momentaryPause", - ); + this.callIsActive = true; + this.buttonUpdater.updateCallButton({ + button:callButton, + icon: momentaryListeningIconSVG, + label: "callInProgress", + clickEventName: "saypi:momentaryStop", + longReleaseEventName: "saypi:momentaryPause", + isMouseOutProcessed: true, + isCallActive: true, + }); } pauseMomentary(callButton) { - this.updateLongClickCallButton( - callButton, - momentaryPausedIconSVG, - "callInProgress", - "saypi:momentaryStop", - "saypi:momentaryListen", - "saypi:momentaryPause", - ); + this.buttonUpdater.updateCallButton({ + button:callButton, + icon: momentaryPausedIconSVG, + label: "callInProgress", + clickEventName: "saypi:momentaryStop", + longPressEventName: "saypi:momentaryListen", + isCallActive: true, + }); } callInterruptible(callButton) { diff --git a/src/buttons/ButtonUpdater.js b/src/buttons/ButtonUpdater.js new file mode 100644 index 0000000000..45ebd371a6 --- /dev/null +++ b/src/buttons/ButtonUpdater.js @@ -0,0 +1,132 @@ +import getMessage from "../i18n.ts"; +import StateMachineService from "../StateMachineService.js"; +import { createSVGElement } from "../dom/DOMModule.ts"; + +class ClickHandler{ + + constructor(){ + console.log("^^^ entered ClickHandler constructor, setting initializing clickStartTime to 0"); + this.clickStartTime = 0; + this.isLongClickEngaged = false; + this.isMouseDown = false; + this.sayPiActor = StateMachineService.actor; + } + + removeChildrenFrom(callButton) { + while (callButton.firstChild) { + callButton.removeChild(callButton.firstChild); + } + } + + addIconTo(callButton, svgIcon) { + const svgElement = createSVGElement(svgIcon); + callButton.appendChild(svgElement); + } + + setAriaLabelOf(button, labelName, labelArg) { + const label = labelArg? getMessage(labelName, labelArg) : getMessage(labelName); + button.setAttribute("aria-label", label); + } + + toggleActiveState(callButton, isActive) { + callButton.classList.toggle("active", isActive); + } + + setEmptyDefault(runnable) { + return runnable ? runnable : () => {}; + } + + isShortClick(clickStartTime, limit) { + const clickDuration = (Date.now() - clickStartTime); + return clickStartTime == 0 || clickDuration < limit; + } + + createEvent(eventName) { + return eventName ? () => { + this.sayPiActor.send(eventName); + } : null; + } + + resetListenersOf(button) { + button.onclick = () => {}; + button.addEventListener("mouseout", () => {}); + } + + handleButtonClicks(options) { + let { button, onClick, onLongPressDown, onLongPressUp, isMouseOutHandled = false } = options; + const longPressMinimumMilliseconds = 500; + + onClick = this.setEmptyDefault(onClick); + onLongPressDown = this.setEmptyDefault(onLongPressDown); + onLongPressUp = this.setEmptyDefault(onLongPressUp); + + button.onmousedown = () => { + this.isMouseDown = true; + this.clickStartTime = Date.now(); + console.log("^^^ on mouse down, click start time is: " + this.clickStartTime); + window.setTimeout( () => { + if (this.isMouseDown === true) { + onLongPressDown(); + this.isLongClickEngaged = true; + } + }, longPressMinimumMilliseconds); + } + + button.onmouseup = () => { + this.isMouseDown = false; + if (this.isShortClick(this.clickStartTime, longPressMinimumMilliseconds)) { + console.log("^^^ shortClick onClick() clickStartTime: " + this.clickStartTime); + onClick(); + } else if (this.isLongClickEngaged) { + console.log("^^^ about to run onLongPressUp()"); + onLongPressUp(); + this.isLongClickEngaged = false; + } + } + + if (isMouseOutHandled) { + button.addEventListener("mouseout", () => { + if (this.isLongClickEngaged) { + onLongPressUp(); + this.isLongClickEngaged = false; + } + }); + } + } + + updateCallButton(options) { + let {callButton, icon, label, clickEventName, longPressEventName, longReleaseEventName, labelArgument = null, isCallActive = false, isMouseOutProcessed = false} = options; + if (!callButton) { + callButton = document.getElementById("saypi-callButton"); + } + if (callButton) { + this.removeChildrenFrom(callButton); + this.addIconTo(callButton, icon); + this.resetListenersOf(callButton); + this.setAriaLabelOf(callButton, label, labelArgument); + + this.handleButtonClicks({ + button: callButton, + onClick: this.createEvent(clickEventName), + onLongPressDown: this.createEvent(longPressEventName), + onLongPressUp: this.createEvent(longReleaseEventName), + isMouseOutHandled: isMouseOutProcessed, + }); + this.toggleActiveState(callButton, isCallActive); + } + } +} + +class ButtonUpdater { + constructor() { + this.clickHandler = new ClickHandler(); + } + + updateCallButton(options) { + this.clickHandler.updateCallButton(options); + } +} + + + +export { ButtonUpdater } \ No newline at end of file From 613d2f8f7db2c0732ce6bf7505113e678407b1cf Mon Sep 17 00:00:00 2001 From: john Date: Mon, 27 Jan 2025 20:53:51 +0000 Subject: [PATCH 17/25] Renamed ButtonUpdater to ButtonHelper, have more methods in ButtonModule using ButtonHelper, and removed redundant methods from ButtonHelper. --- src/ButtonModule.js | 156 ++++-------------- .../{ButtonUpdater.js => ButtonHelper.js} | 52 +++--- 2 files changed, 65 insertions(+), 143 deletions(-) rename src/buttons/{ButtonUpdater.js => ButtonHelper.js} (73%) diff --git a/src/ButtonModule.js b/src/ButtonModule.js index 2895b786f1..d73c361102 100644 --- a/src/ButtonModule.js +++ b/src/ButtonModule.js @@ -25,7 +25,7 @@ import { IconModule } from "./icons/IconModule.ts"; import { ImmersionStateChecker } from "./ImmersionServiceLite.ts"; import { GlowColorUpdater } from "./buttons/GlowColorUpdater.js"; import { openSettings } from "./popup/popupopener.ts"; -import { ButtonUpdater } from "./buttons/ButtonUpdater.js"; +import { ButtonHelper } from "./buttons/ButtonHelper.js"; class ButtonModule { /** @@ -34,22 +34,17 @@ class ButtonModule { */ constructor(chatbot) { this.icons = new IconModule(); + this.buttonHelper = new ButtonHelper(); this.userPreferences = UserPreferenceModule.getInstance(); this.chatbot = chatbot; this.immersionService = new ImmersionService(chatbot); this.glowColorUpdater = new GlowColorUpdater(); - this.sayPiActor = StateMachineService.actor; // the Say, Pi state machine this.screenLockActor = StateMachineService.screenLockActor; // Binding methods to the current instance this.registerOtherEvents(); // track the frequency of bug #26 this.submissionsWithoutAnError = 0; - - // track whether a call is active, so that new button instances can be initialized correctly - this.callIsActive = false; - this.clickStartTime = 0; - this.buttonUpdater = new ButtonUpdater(); } registerOtherEvents() { @@ -302,85 +297,40 @@ class ButtonModule { button.type = "button"; button.classList.add("call-button", "saypi-button", "tooltip"); button.classList.add(...this.chatbot.getExtraCallButtonClasses()); - if (this.callIsActive) { + if (this.buttonHelper.isCallActive()) { this.callActive(button); } else { this.callInactive(button); } addChild(container, button, position); - if (this.callIsActive) { + if (this.buttonHelper.isCallActive()) { // if the call is active, start the glow animation once added to the DOM AnimationModule.startAnimation("glow"); } return button; } - updateCallButtonColor(color) { - const callButton = document.getElementById("saypi-callButton"); - // find first path element descendant of the call button's svg element child - const path = callButton?.querySelector("svg path"); - if (path) { - // set the fill color of the path element - path.style.fill = color; - } - } - /** * * @param { isSpeech: number; notSpeech: number } probabilities */ handleAudioFrame(probabilities) { this.glowColorUpdater.updateGlowColor(probabilities.isSpeech); - - } - removeChildrenFrom(callButton) { - while (callButton.firstChild) { - callButton.removeChild(callButton.firstChild); - } - } - - addIconTo(callButton, svgIcon) { - const svgElement = createSVGElement(svgIcon); - callButton.appendChild(svgElement); - } - - toggleActiveState(callButton, isActive) { - callButton.classList.toggle("active", isActive); - } - - updateCallButton(callButton, svgIcon, label, onClick, isActive = false) { - if (!callButton) { - callButton = document.getElementById("saypi-callButton"); - } - if (callButton) { - this.removeChildrenFrom(callButton); - this.addIconTo(callButton, svgIcon); - callButton.setAttribute("aria-label", label); - this.handleLongClick(callButton, onClick); - this.toggleActiveState(callButton, isActive); - } - this.callIsActive = isActive; - } - - createEvent(eventName) { - return () => { - this.sayPiActor.send(eventName); - }; } callStarting(callButton) { - const label = getMessage("callStarting"); - this.updateCallButton(callButton, - callStartingIconSVG, - label, - () => this.sayPiActor.send("saypi:hangup") - ); + this.buttonHelper.updateCallButton({ + button: callButton, + icon: callStartingIconSVG, + label: "callStarting", + clickEventName: "saypi:hangup", + }); } callActive(callButton) { - this.buttonUpdater.updateCallButton({ - button:callButton, + this.buttonHelper.updateCallButton({ + button: callButton, icon: hangupIconSVG, label: "callInProgress", clickEventName: "saypi:hangup", @@ -390,9 +340,8 @@ class ButtonModule { } callMomentary(callButton) { - this.callIsActive = true; - this.buttonUpdater.updateCallButton({ - button:callButton, + this.buttonHelper.updateCallButton({ + button: callButton, icon: momentaryListeningIconSVG, label: "callInProgress", clickEventName: "saypi:momentaryStop", @@ -403,8 +352,8 @@ class ButtonModule { } pauseMomentary(callButton) { - this.buttonUpdater.updateCallButton({ - button:callButton, + this.buttonHelper.updateCallButton({ + button: callButton, icon: momentaryPausedIconSVG, label: "callInProgress", clickEventName: "saypi:momentaryStop", @@ -416,67 +365,34 @@ class ButtonModule { callInterruptible(callButton) { const handsFreeInterruptEnabled = this.userPreferences.getCachedAllowInterruptions(); - if (!handsFreeInterruptEnabled) { - const label = getMessage("callInterruptible"); - this.updateCallButton( - callButton, - interruptIconSVG, - label, - () => { - this.sayPiActor.send("saypi:interrupt"); - }, - true - ); - } - } - - setEmptyDefault(runnable) { - return runnable ? runnable : () => {}; - } - - isShortClick(clickStartTime, limit) { - const clickDuration = (Date.now() - clickStartTime); - return clickStartTime == 0 || clickDuration < limit; - } - handleLongClick(button, onClick, onLongPressDown, onLongPressUp ) { - const longPressMinimumMilliseconds = 500; - var isMouseUpDetected = false; - - onClick = this.setEmptyDefault(onClick); - onLongPressDown = this.setEmptyDefault(onLongPressDown); - onLongPressUp = this.setEmptyDefault(onLongPressUp); - - button.onmousedown = () => { - isMouseUpDetected = false; - this.clickStartTime = Date.now(); - window.setTimeout( () => { - if(!isMouseUpDetected) { - onLongPressDown(); - } - }, longPressMinimumMilliseconds); - } - - button.onmouseup = () => { - isMouseUpDetected = true; - if(this.isShortClick(this.clickStartTime, longPressMinimumMilliseconds)) { - onClick(); - } else { - onLongPressUp(); - } + if (!handsFreeInterruptEnabled) { + this.buttonHelper.updateCallButton({ + button: callButton, + icon: interruptIconSVG, + label: "callInterruptible", + clickEventName: "saypi:interrupt", + isCallActive: true, + }); } } callInactive(callButton) { - const label = getMessage("callNotStarted", this.chatbot.getName()); - this.updateCallButton(callButton, callIconSVG, label, () => - this.sayPiActor.send("saypi:call") - ); + this.buttonHelper.updateCallButton({ + button: callButton, + icon: callIconSVG, + label: "callNotStarted", + labelArgument: this.chatbot.getName(), + clickEventName: "saypi:call", + }); } callError(callButton) { - const label = getMessage("callError"); - this.updateCallButton(callButton, hangupMincedIconSVG, label, null); + this.buttonHelper.updateCallButton({ + button: callButton, + icon: hangupMincedIconSVG, + label: "callError", + }); } disableCallButton() { diff --git a/src/buttons/ButtonUpdater.js b/src/buttons/ButtonHelper.js similarity index 73% rename from src/buttons/ButtonUpdater.js rename to src/buttons/ButtonHelper.js index 45ebd371a6..cfe949081f 100644 --- a/src/buttons/ButtonUpdater.js +++ b/src/buttons/ButtonHelper.js @@ -2,14 +2,17 @@ import getMessage from "../i18n.ts"; import StateMachineService from "../StateMachineService.js"; import { createSVGElement } from "../dom/DOMModule.ts"; -class ClickHandler{ +class ButtonUpdater{ - constructor(){ - console.log("^^^ entered ClickHandler constructor, setting initializing clickStartTime to 0"); + constructor() { this.clickStartTime = 0; this.isLongClickEngaged = false; this.isMouseDown = false; + this.sayPiActor = StateMachineService.actor; + + // track whether a call is active, so that new button instances can be initialized correctly + this.callIsActive = false; } removeChildrenFrom(callButton) { @@ -63,7 +66,6 @@ class ClickHandler{ button.onmousedown = () => { this.isMouseDown = true; this.clickStartTime = Date.now(); - console.log("^^^ on mouse down, click start time is: " + this.clickStartTime); window.setTimeout( () => { if (this.isMouseDown === true) { onLongPressDown(); @@ -75,10 +77,8 @@ class ClickHandler{ button.onmouseup = () => { this.isMouseDown = false; if (this.isShortClick(this.clickStartTime, longPressMinimumMilliseconds)) { - console.log("^^^ shortClick onClick() clickStartTime: " + this.clickStartTime); onClick(); } else if (this.isLongClickEngaged) { - console.log("^^^ about to run onLongPressUp()"); onLongPressUp(); this.isLongClickEngaged = false; } @@ -94,39 +94,45 @@ class ClickHandler{ } } + isCallActive() { + return this.callIsActive; + } + updateCallButton(options) { - let {callButton, icon, label, clickEventName, longPressEventName, longReleaseEventName, labelArgument = null, isCallActive = false, isMouseOutProcessed = false} = options; - if (!callButton) { - callButton = document.getElementById("saypi-callButton"); + let {button, icon, label, labelArgument, clickEventName, longPressEventName, longReleaseEventName, isCallActive = false, isMouseOutProcessed = false} = options; + if (!button) { + button = document.getElementById("saypi-callButton"); } - if (callButton) { - this.removeChildrenFrom(callButton); - this.addIconTo(callButton, icon); - this.resetListenersOf(callButton); - this.setAriaLabelOf(callButton, label, labelArgument); - + if (button) { + this.removeChildrenFrom(button); + this.addIconTo(button, icon); + this.resetListenersOf(button); + this.setAriaLabelOf(button, label, labelArgument); this.handleButtonClicks({ - button: callButton, + button: button, onClick: this.createEvent(clickEventName), onLongPressDown: this.createEvent(longPressEventName), onLongPressUp: this.createEvent(longReleaseEventName), isMouseOutHandled: isMouseOutProcessed, }); - this.toggleActiveState(callButton, isCallActive); + this.toggleActiveState(button, isCallActive); + this.callIsActive = isCallActive; } } } -class ButtonUpdater { +class ButtonHelper { constructor() { - this.clickHandler = new ClickHandler(); + this.buttonUpdater = new ButtonUpdater(); } updateCallButton(options) { - this.clickHandler.updateCallButton(options); + this.buttonUpdater.updateCallButton(options); } -} - + isCallActive() { + return this.buttonUpdater.isCallActive(); + } +} -export { ButtonUpdater } \ No newline at end of file +export { ButtonHelper } \ No newline at end of file From 5cd69d33a9b72d91984fca0496e64f52d57caa87 Mon Sep 17 00:00:00 2001 From: john Date: Tue, 28 Jan 2025 01:48:28 +0000 Subject: [PATCH 18/25] Fixed slow recording stop after momentary pause enabled and no speech detected. --- src/audio/AudioModule.js | 4 +++ src/state-machines/AudioInputMachine.ts | 20 +++++++++-- src/state-machines/SayPiMachine.ts | 48 ++++++++++++++----------- 3 files changed, 49 insertions(+), 23 deletions(-) diff --git a/src/audio/AudioModule.js b/src/audio/AudioModule.js index 0c3bc66bf1..114248eaec 100644 --- a/src/audio/AudioModule.js +++ b/src/audio/AudioModule.js @@ -375,6 +375,10 @@ export default class AudioModule { // soft stop recording inputActor.send("stopRequested"); }); + EventBus.on("audio:quickStopRecording", function (e) { + // soft stop recording but quickly + inputActor.send("quickStopRequested"); + }); // audio input (recording) events (pass media recorder events -> audio input machine actor) EventBus.on("audio:dataavailable", (detail) => { inputActor.send({ type: "dataAvailable", ...detail }); diff --git a/src/state-machines/AudioInputMachine.ts b/src/state-machines/AudioInputMachine.ts index b42cc8271f..d15f460152 100644 --- a/src/state-machines/AudioInputMachine.ts +++ b/src/state-machines/AudioInputMachine.ts @@ -324,6 +324,7 @@ interface AudioInputContext { waitingToStop: boolean; waitingToStart: boolean; recordingStartTime: number; + isQuickStopRequested: boolean; } type AudioInputEvent = @@ -331,6 +332,7 @@ type AudioInputEvent = | { type: "release" } | { type: "start" } | { type: "stopRequested" } + | { type: "quickStopRequested" } | { type: "dataAvailable"; blob: Blob; duration: number } | { type: "stop" } | { type: "error.platform"; data: any }; @@ -347,6 +349,7 @@ export const audioInputMachine = createMachine< waitingToStop: false, waitingToStart: false, recordingStartTime: 0, + isQuickStopRequested: false, }, states: { released: { @@ -412,6 +415,13 @@ export const audioInputMachine = createMachine< stopRequested: { target: "pendingStop", description: "Stop gracefully.", + actions: [ assign({isQuickStopRequested: false}) ], + }, + + quickStopRequested: { + target: "pendingStop", + description: "Stop gracefully and quickly.", + actions: [ assign({isQuickStopRequested: true}) ], }, stop: { @@ -436,7 +446,7 @@ export const audioInputMachine = createMachine< type: "prepareStop", }, after: { - "5000": [ + stopRecordingDelay: [ { target: "#audioInput.acquired.stopped", actions: ["stopIfWaiting"], @@ -459,7 +469,7 @@ export const audioInputMachine = createMachine< }, }, }, - + stopped: { entry: assign({ waitingToStop: false }), always: { @@ -609,7 +619,11 @@ export const audioInputMachine = createMachine< return context.waitingToStart === true; }, }, - delays: {}, + delays: { + stopRecordingDelay: (context) => { + return context.isQuickStopRequested === true ? 1000 : 5000; + } + }, } ); interface OverconstrainedError extends DOMException { diff --git a/src/state-machines/SayPiMachine.ts b/src/state-machines/SayPiMachine.ts index 8df1f29876..918bf7de49 100644 --- a/src/state-machines/SayPiMachine.ts +++ b/src/state-machines/SayPiMachine.ts @@ -98,6 +98,7 @@ interface SayPiContext { sessionId?: string; isMomentaryEnabled: boolean; isMomentaryActive: boolean; + hasUserSpoken: boolean; } // Define the state schema @@ -217,6 +218,7 @@ const machine = createMachine( userIsSpeaking: false, timeUserStoppedSpeaking: 0, defaultPlaceholderText: "", + hasUserSpoken: false, }, id: "sayPi", initial: "inactive", @@ -399,7 +401,10 @@ const machine = createMachine( animation: "userSpeaking", }, }, - assign({ userIsSpeaking: true }), + assign({ + userIsSpeaking: true, + hasUserSpoken: true, + }), { type: "cancelCountdownAnimation", }, @@ -458,6 +463,7 @@ const machine = createMachine( assign({ isMomentaryEnabled: true, isMomentaryActive: true, + hasUserSpoken: false, }), { type: "momentaryHasStarted", @@ -466,24 +472,24 @@ const machine = createMachine( description: 'Enable Momentary Mode. Now recording will only stop if the user releases the button.', }, - "saypi:momentaryPause": { - target: "#sayPi.listening.converting.submitting", - cond: "readyToSubmitFromMomentary", - actions: [ - assign({ - isMomentaryActive: false, - }), - { - type: "momentaryHasPaused", - }, - { - type: "stopAnimation", - params: { - animation: "glow", + "saypi:momentaryPause": [ + { + actions: [ + assign({ + isMomentaryActive: false, + }), + ] + }, + { + target: "#sayPi.listening.converting.submitting", + cond: "readyToSubmitFromMomentary", + actions: [ + { + type: "momentaryHasPaused", }, - }, - ], - }, + ], + }, + ], "saypi:momentaryStop": { actions: [ assign({ @@ -1431,11 +1437,11 @@ const machine = createMachine( EventBus.emit("audio:input:reconnect"); }, - momentaryHasPaused: () => { + momentaryHasPaused: (context: SayPiContext,) => { buttonModule.pauseMomentary(); AnimationModule.stopAnimation("glow"); AnimationModule.stopAnimation("userSpeaking"); - EventBus.emit("audio:stopRecording"); //JAC temp message -> soft stop instead of hard stop, see if this will produce an audio blob + EventBus.emit(context.hasUserSpoken ? "audio:stopRecording" : "audio:quickStopRecording"); }, momentaryReturnsToPaused: () => { @@ -1458,6 +1464,7 @@ const machine = createMachine( } return false; }, + hasNoAudio: (context: SayPiContext, event: SayPiEvent) => { if (event.type === "saypi:userStoppedSpeaking") { event = event as SayPiSpeechStoppedEvent; @@ -1469,6 +1476,7 @@ const machine = createMachine( } return false; }, + submissionConditionsMet: ( context: SayPiContext, event: SayPiEvent, From bf95ec2ae4a55446e4653647380e3e2076211ea0 Mon Sep 17 00:00:00 2001 From: john Date: Tue, 28 Jan 2025 21:14:12 +0000 Subject: [PATCH 19/25] Added prompt and tooltip changes for momentary states. --- _locales/en/messages.json | 18 +++++++++++ src/ButtonModule.js | 4 +-- src/state-machines/SayPiMachine.ts | 52 ++++++++++++++++++++++++++++-- 3 files changed, 70 insertions(+), 4 deletions(-) diff --git a/_locales/en/messages.json b/_locales/en/messages.json index b964be761a..ce5262b2ab 100644 --- a/_locales/en/messages.json +++ b/_locales/en/messages.json @@ -27,6 +27,14 @@ "message": "End hands-free conversation", "description": "The tooltip to display on the call button when a call is in progress." }, + "momentaryCallPaused": { + "message": "Hold to speak, or click to return to hands-free mode", + "description": "The tooltip to display on the call button when a call is in progress and momentary mode is paused." + }, + "momentaryCallListening": { + "message": "Release when finished speaking", + "description": "The tooltip to display on the call button when a call is in progress and momentary mode is active." + }, "callInterruptible": { "message": "Tap to interrupt", "description": "The tooltip to display on the call button when a call is in progress and manual interruption is possible." @@ -198,6 +206,16 @@ "example": "Pi" } } + }, + "microphoneIsMuted": { + "message": "The microphone is muted...", + "description": "Message displayed when the 'momentary' mode is enabled but paused.", + "placeholders": { + "chatbot": { + "content": "$1", + "example": "Pi" + } + } }, "toggleThemeToDarkMode": { "message": "Switch to dark mode", diff --git a/src/ButtonModule.js b/src/ButtonModule.js index d73c361102..7e544c5d86 100644 --- a/src/ButtonModule.js +++ b/src/ButtonModule.js @@ -343,7 +343,7 @@ class ButtonModule { this.buttonHelper.updateCallButton({ button: callButton, icon: momentaryListeningIconSVG, - label: "callInProgress", + label: "momentaryCallListening", clickEventName: "saypi:momentaryStop", longReleaseEventName: "saypi:momentaryPause", isMouseOutProcessed: true, @@ -355,7 +355,7 @@ class ButtonModule { this.buttonHelper.updateCallButton({ button: callButton, icon: momentaryPausedIconSVG, - label: "callInProgress", + label: "momentaryCallPaused", clickEventName: "saypi:momentaryStop", longPressEventName: "saypi:momentaryListen", isCallActive: true, diff --git a/src/state-machines/SayPiMachine.ts b/src/state-machines/SayPiMachine.ts index 918bf7de49..31a0841b44 100644 --- a/src/state-machines/SayPiMachine.ts +++ b/src/state-machines/SayPiMachine.ts @@ -364,7 +364,7 @@ const machine = createMachine( }, }, { - type: "listenPrompt", + type: "listenOrPausedPrompt", }, ], exit: [ @@ -468,6 +468,9 @@ const machine = createMachine( { type: "momentaryHasStarted", }, + { + type: "listenPrompt", + }, ], description: 'Enable Momentary Mode. Now recording will only stop if the user releases the button.', @@ -478,8 +481,12 @@ const machine = createMachine( assign({ isMomentaryActive: false, }), + { + type: "momentaryPausedPrompt", + }, ] }, + { target: "#sayPi.listening.converting.submitting", cond: "readyToSubmitFromMomentary", @@ -499,6 +506,9 @@ const machine = createMachine( { type: "momentaryHasStopped", }, + { + type: "listenPrompt" + }, ], description: 'Enable Momentary Mode. Now recording will only stop if the user releases the button.', @@ -815,10 +825,13 @@ const machine = createMachine( { type: "momentaryReturnsToPaused", }, + { + type: "momentaryPausedPrompt", + }, ], on: { "saypi:momentaryListen": { - actions: [ + actions: [ assign({ isMomentaryActive: true }), { type: "momentaryHasStarted" @@ -833,6 +846,9 @@ const machine = createMachine( { type: "momentaryHasStopped" }, + { + type: "listenPrompt" + }, ], target: "#sayPi.listening.recording", description: 'Returning to the standard recording mode.', @@ -1265,6 +1281,24 @@ const machine = createMachine( } }, + listenOrPausedPrompt: (context: SayPiContext) => { + const isMomentaryPaused = context.isMomentaryEnabled && !context.isMomentaryActive; + const message = isMomentaryPaused + ? getMessage("microphoneIsMuted") + : getMessage("assistantIsListening", chatbot.getName()); + + if (message) { + getPromptOrNull()?.setMessage(message); + } + }, + + momentaryPausedPrompt: () => { + const message = getMessage("microphoneIsMuted"); + if (message) { + getPromptOrNull()?.setMessage(message); + } + }, + callStartingPrompt: () => { const message = getMessage("callStarting"); if (message) { @@ -1312,6 +1346,13 @@ const machine = createMachine( } }, + pushToTalkPrompt: () => { + const message = getMessage("pressAndHoldToSpeak"); + if (message) { + getPromptOrNull()?.setMessage(message); + } + }, + clearPrompt: (context: SayPiContext) => { getPromptOrNull()?.setMessage(context.defaultPlaceholderText); }, @@ -1584,6 +1625,13 @@ function readyToSubmitOnAllowedState( return ready; } +function showMomentaryPausedPrompt (): void { + const message = getMessage("microphoneIsMuted"); + if (message) { + getPromptOrNull()?.setMessage(message); + } +}; + function provisionallyReadyToSubmit(context: SayPiContext): boolean { const allowedState = !(context.userIsSpeaking || context.isTranscribing); // we don't have access to the state, so we read from a copy in the context (!DRY) return readyToSubmitOnAllowedState(allowedState, context); From e52d821368bb11dccf2063e66de09086f678124b Mon Sep 17 00:00:00 2001 From: john Date: Wed, 29 Jan 2025 19:17:45 +0000 Subject: [PATCH 20/25] Added momentary mode interruption when the interrupt button is not displayed. --- src/state-machines/SayPiMachine.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/state-machines/SayPiMachine.ts b/src/state-machines/SayPiMachine.ts index 31a0841b44..3189e45c0b 100644 --- a/src/state-machines/SayPiMachine.ts +++ b/src/state-machines/SayPiMachine.ts @@ -968,6 +968,19 @@ const machine = createMachine( }, description: "The user starting speaking while Pi was speaking.", + }, + "saypi:momentaryListen": { + target: "#sayPi.listening.recording", + actions:[ + { + type: "pauseAudio", + }, + { + type: "momentaryHasStarted", + } + ], + description: + "The user pushed the momentary button while while Pi was speaking.", }, "saypi:interrupt": [ { From 341943b5bd4b7b3bebe0df04ce56514ee6189448 Mon Sep 17 00:00:00 2001 From: john Date: Wed, 29 Jan 2025 23:14:47 +0000 Subject: [PATCH 21/25] Updated prompt to thinking after momentary pause after user has spoken. --- src/state-machines/SayPiMachine.ts | 86 ++++++++++-------------------- 1 file changed, 29 insertions(+), 57 deletions(-) diff --git a/src/state-machines/SayPiMachine.ts b/src/state-machines/SayPiMachine.ts index 3189e45c0b..230856bc15 100644 --- a/src/state-machines/SayPiMachine.ts +++ b/src/state-machines/SayPiMachine.ts @@ -484,17 +484,14 @@ const machine = createMachine( { type: "momentaryPausedPrompt", }, + { + type: "momentaryHasPaused", + }, ] }, - { target: "#sayPi.listening.converting.submitting", cond: "readyToSubmitFromMomentary", - actions: [ - { - type: "momentaryHasPaused", - }, - ], }, ], "saypi:momentaryStop": { @@ -1288,28 +1285,22 @@ const machine = createMachine( }, listenPrompt: () => { - const message = getMessage("assistantIsListening", chatbot.getName()); - if (message) { - getPromptOrNull()?.setMessage(message); - } + setPromptMessage("assistantIsListening", chatbot); }, listenOrPausedPrompt: (context: SayPiContext) => { const isMomentaryPaused = context.isMomentaryEnabled && !context.isMomentaryActive; - const message = isMomentaryPaused - ? getMessage("microphoneIsMuted") - : getMessage("assistantIsListening", chatbot.getName()); - - if (message) { - getPromptOrNull()?.setMessage(message); + if (isMomentaryPaused) { + setPromptMessage("microphoneIsMuted"); + } else { + setPromptMessage("assistantIsListening", chatbot); } }, - momentaryPausedPrompt: () => { - const message = getMessage("microphoneIsMuted"); - if (message) { - getPromptOrNull()?.setMessage(message); - } + momentaryPausedPrompt: (context: SayPiContext) => { + context.hasUserSpoken + ? setPromptMessage("assistantIsThinking", chatbot) + : setPromptMessage("microphoneIsMuted"); }, callStartingPrompt: () => { @@ -1322,48 +1313,28 @@ const machine = createMachine( }, thinkingPrompt: () => { - const message = getMessage("assistantIsThinking", chatbot.getName()); - if (message) { - getPromptOrNull()?.setMessage(message); - } + setPromptMessage("assistantIsThinking", chatbot); }, writingPrompt: () => { - const message = getMessage("assistantIsWriting", chatbot.getName()); - if (message) { - getPromptOrNull()?.setMessage(message); - } + setPromptMessage("assistantIsWriting", chatbot); }, speakingPrompt: (context: SayPiContext) => { const handsFreeInterrupt = userPreferences.getCachedAllowInterruptions(); - const message = handsFreeInterrupt - ? getMessage("assistantIsSpeaking", chatbot.getName()) - : getMessage( - "assistantIsSpeakingWithManualInterrupt", - chatbot.getName() - ); - if (message) { - getPromptOrNull()?.setMessage(message); - } + + handsFreeInterrupt + ? setPromptMessage("assistantIsSpeaking", chatbot) + : setPromptMessage("assistantIsSpeakingWithManualInterrupt", chatbot); }, interruptingPiPrompt: () => { - const message = getMessage( - "userStartedInterrupting", - chatbot.getName() - ); - if (message) { - getPromptOrNull()?.setMessage(message); - } + setPromptMessage("userStartedInterrupting", chatbot); }, pushToTalkPrompt: () => { - const message = getMessage("pressAndHoldToSpeak"); - if (message) { - getPromptOrNull()?.setMessage(message); - } + setPromptMessage("pressAndHoldToSpeak"); }, clearPrompt: (context: SayPiContext) => { @@ -1634,22 +1605,23 @@ function readyToSubmitOnAllowedState( const empty = Object.keys(context.transcriptions).length === 0; const pending = isTranscriptionPending(); const ready = allowedState && !empty && !pending; - console.log("Entered readyToSubmitOnAllowedState() empty: " + empty + " any pending: " + pending + " is ready: " + ready); return ready; } -function showMomentaryPausedPrompt (): void { - const message = getMessage("microphoneIsMuted"); - if (message) { - getPromptOrNull()?.setMessage(message); - } -}; - function provisionallyReadyToSubmit(context: SayPiContext): boolean { const allowedState = !(context.userIsSpeaking || context.isTranscribing); // we don't have access to the state, so we read from a copy in the context (!DRY) return readyToSubmitOnAllowedState(allowedState, context); } +function setPromptMessage(messageName: string, chatbot?: Chatbot) : void { + const message = chatbot + ? getMessage(messageName, chatbot.getName()) + : getMessage(messageName); + if (message) { + getPromptOrNull()?.setMessage(message); + } +} + function readyToSubmit( state: State, context: SayPiContext From 0cfbc8208c137bb3c1945f78d4475f209cb78664 Mon Sep 17 00:00:00 2001 From: john Date: Sat, 1 Feb 2025 03:25:44 +0000 Subject: [PATCH 22/25] Updated ButtonHelper to only configure listeners once, instead of every update. Newly added touch listeners have also been added, but 'touchend' is only detected on every other press. --- src/buttons/ButtonHelper.js | 125 ++++++++++++++++++++++++------------ 1 file changed, 83 insertions(+), 42 deletions(-) diff --git a/src/buttons/ButtonHelper.js b/src/buttons/ButtonHelper.js index cfe949081f..d246247914 100644 --- a/src/buttons/ButtonHelper.js +++ b/src/buttons/ButtonHelper.js @@ -5,6 +5,7 @@ import { createSVGElement } from "../dom/DOMModule.ts"; class ButtonUpdater{ constructor() { + this.longPressMilliseconds = 500; this.clickStartTime = 0; this.isLongClickEngaged = false; this.isMouseDown = false; @@ -13,6 +14,13 @@ class ButtonUpdater{ // track whether a call is active, so that new button instances can be initialized correctly this.callIsActive = false; + + this.currentOnClick = () => {}; + this.currentOnLongClick = () => {}; + this.currentOnLongRelease = () => {}; + + this.isMouseOutHandled = false; + this.areButtonClicksConfigured = false; } removeChildrenFrom(callButton) { @@ -39,59 +47,92 @@ class ButtonUpdater{ return runnable ? runnable : () => {}; } - isShortClick(clickStartTime, limit) { - const clickDuration = (Date.now() - clickStartTime); - return clickStartTime == 0 || clickDuration < limit; + isShortClick() { + const clickDuration = (Date.now() - this.clickStartTime); + return this.clickStartTime == 0 || clickDuration < this.longPressMilliseconds; } createEvent(eventName) { - return eventName ? () => { - this.sayPiActor.send(eventName); - } : null; + return eventName + ? () => { this.sayPiActor.send(eventName); } + : () => {}; } resetListenersOf(button) { button.onclick = () => {}; button.addEventListener("mouseout", () => {}); + button.addEventListener("touchstart", ()=> {}); + button.addEventListener("touchend", ()=> {}); + button.addEventListener("touchcancel", ()=> {}); + } + + onDown() { + this.isMouseDown = true; + this.clickStartTime = Date.now(); + window.setTimeout( () => { + if (this.isMouseDown === true) { + this.currentOnLongClick(); + this.isLongClickEngaged = true; + } + }, this.longPressMilliseconds); } - handleButtonClicks(options) { - let { button, onClick, onLongPressDown, onLongPressUp, isMouseOutHandled = false } = options; - const longPressMinimumMilliseconds = 500; - - onClick = this.setEmptyDefault(onClick); - onLongPressDown = this.setEmptyDefault(onLongPressDown); - onLongPressUp = this.setEmptyDefault(onLongPressUp); - - button.onmousedown = () => { - this.isMouseDown = true; - this.clickStartTime = Date.now(); - window.setTimeout( () => { - if (this.isMouseDown === true) { - onLongPressDown(); - this.isLongClickEngaged = true; - } - }, longPressMinimumMilliseconds); + onUp() { + this.isMouseDown = false; + if (this.isShortClick()) { + this.currentOnClick(); + } else if (this.isLongClickEngaged) { + this.currentOnLongRelease(); + this.isLongClickEngaged = false; } + } - button.onmouseup = () => { - this.isMouseDown = false; - if (this.isShortClick(this.clickStartTime, longPressMinimumMilliseconds)) { - onClick(); - } else if (this.isLongClickEngaged) { - onLongPressUp(); - this.isLongClickEngaged = false; - } + onOut() { + if(this.isMouseOutHandled && this.isLongClickEngaged){ + this.currentOnLongRelease(); + this.isLongClickEngaged = false; } + } - if (isMouseOutHandled) { - button.addEventListener("mouseout", () => { - if (this.isLongClickEngaged) { - onLongPressUp(); - this.isLongClickEngaged = false; - } + setupButtonListeners(button) { + if(typeof(window.ontouchstart) != 'undefined'){ + this.log("setupButtonListeners, onTouchStart is not null, defining touchstart and touchend event listeners"); + button.addEventListener('touchstart',() => { + this.log("ontouchstart!"); + this.onDown(); }); + button.addEventListener('touchend', () => { + this.log("ontouchend!"); + this.onUp(); + }); + button.addEventListener('touchcancel', () => { + this.log("ontouchcancel!!!"); + this.onUp(); + }); + } else { + this.log("ontouchstart could not be found, so defining onmousedown and onmouseup"); + button.onmousedown = () => this.onDown(); + button.onmouseup = () => this.onUp(); + button.addEventListener("mouseout", () => this.onOut()); + } + } + + updateButtonRoles(options) { + let { button, clickEventName, longDownEventName, longUpEventName, isMouseOutHandled = false } = options; + + if(this.areButtonClicksConfigured === false){ + this.setupButtonListeners(button); + this.areButtonClicksConfigured = true; } + + this.currentOnClick = this.createEvent(clickEventName); + this.currentOnLongClick = this.createEvent(longDownEventName); + this.currentOnLongRelease = this.createEvent(longUpEventName); + this.isMouseOutHandled = isMouseOutHandled; + } + + log(msg) { + console.log("^^^ ButtonHelper: " + msg); } isCallActive() { @@ -106,13 +147,13 @@ class ButtonUpdater{ if (button) { this.removeChildrenFrom(button); this.addIconTo(button, icon); - this.resetListenersOf(button); + // this.resetListenersOf(button); this.setAriaLabelOf(button, label, labelArgument); - this.handleButtonClicks({ + this.updateButtonRoles({ button: button, - onClick: this.createEvent(clickEventName), - onLongPressDown: this.createEvent(longPressEventName), - onLongPressUp: this.createEvent(longReleaseEventName), + clickEventName: clickEventName, + longDownEventName: longPressEventName, + longUpEventName: longReleaseEventName, isMouseOutHandled: isMouseOutProcessed, }); this.toggleActiveState(button, isCallActive); From 0914bd5c2d669a8dab995dc5b1906118971107fe Mon Sep 17 00:00:00 2001 From: john Date: Sun, 2 Feb 2025 02:39:46 +0000 Subject: [PATCH 23/25] Working touch events and with cached icon creation, but regression with momentary listening long-click on desktop. --- src/ButtonModule.js | 1 + src/buttons/ButtonHelper.js | 90 +++++++++++++++++++++++++------------ 2 files changed, 62 insertions(+), 29 deletions(-) diff --git a/src/ButtonModule.js b/src/ButtonModule.js index 7e544c5d86..a1a94a3430 100644 --- a/src/ButtonModule.js +++ b/src/ButtonModule.js @@ -348,6 +348,7 @@ class ButtonModule { longReleaseEventName: "saypi:momentaryPause", isMouseOutProcessed: true, isCallActive: true, + overwriteSvg: true, }); } diff --git a/src/buttons/ButtonHelper.js b/src/buttons/ButtonHelper.js index d246247914..c5c64c18a8 100644 --- a/src/buttons/ButtonHelper.js +++ b/src/buttons/ButtonHelper.js @@ -21,19 +21,16 @@ class ButtonUpdater{ this.isMouseOutHandled = false; this.areButtonClicksConfigured = false; + + this.iconSvgCache = new Array(); } - removeChildrenFrom(callButton) { - while (callButton.firstChild) { - callButton.removeChild(callButton.firstChild); + removeChildrenFrom(button) { + while (button.firstChild) { + button.removeChild(button.firstChild); } } - - addIconTo(callButton, svgIcon) { - const svgElement = createSVGElement(svgIcon); - callButton.appendChild(svgElement); - } - + setAriaLabelOf(button, labelName, labelArg) { const label = labelArg? getMessage(labelName, labelArg) : getMessage(labelName); button.setAttribute("aria-label", label); @@ -54,7 +51,7 @@ class ButtonUpdater{ createEvent(eventName) { return eventName - ? () => { this.sayPiActor.send(eventName); } + ? () => { this.log("createEvent() sending event: " + eventName); this.sayPiActor.send(eventName); } : () => {}; } @@ -70,49 +67,54 @@ class ButtonUpdater{ this.isMouseDown = true; this.clickStartTime = Date.now(); window.setTimeout( () => { - if (this.isMouseDown === true) { - this.currentOnLongClick(); - this.isLongClickEngaged = true; - } + if (this.isMouseDown === true) { + this.log("onDown() about to currentOnLongClick()"); + this.currentOnLongClick(); + this.isLongClickEngaged = true; + } }, this.longPressMilliseconds); } onUp() { this.isMouseDown = false; if (this.isShortClick()) { + this.log("onUp() about to currentOnClick()"); this.currentOnClick(); } else if (this.isLongClickEngaged) { + this.log("onUp() about to currentOnLongRelease()"); this.currentOnLongRelease(); this.isLongClickEngaged = false; } } onOut() { - if(this.isMouseOutHandled && this.isLongClickEngaged){ + if (this.isMouseOutHandled && this.isLongClickEngaged) { this.currentOnLongRelease(); this.isLongClickEngaged = false; } } setupButtonListeners(button) { - if(typeof(window.ontouchstart) != 'undefined'){ - this.log("setupButtonListeners, onTouchStart is not null, defining touchstart and touchend event listeners"); + if (typeof(window.ontouchstart) != 'undefined') { button.addEventListener('touchstart',() => { - this.log("ontouchstart!"); this.onDown(); }); - button.addEventListener('touchend', () => { - this.log("ontouchend!"); + button.addEventListener('touchend', ev => { + ev.preventDefault(); this.onUp(); }); button.addEventListener('touchcancel', () => { - this.log("ontouchcancel!!!"); this.onUp(); }); } else { - this.log("ontouchstart could not be found, so defining onmousedown and onmouseup"); - button.onmousedown = () => this.onDown(); - button.onmouseup = () => this.onUp(); + button.onmousedown = () => { + this.log("onmousedown"); + this.onDown(); + } + button.onmouseup = () => { + this.log("onmouseup"); + this.onUp(); + } button.addEventListener("mouseout", () => this.onOut()); } } @@ -120,7 +122,7 @@ class ButtonUpdater{ updateButtonRoles(options) { let { button, clickEventName, longDownEventName, longUpEventName, isMouseOutHandled = false } = options; - if(this.areButtonClicksConfigured === false){ + if (this.areButtonClicksConfigured === false) { this.setupButtonListeners(button); this.areButtonClicksConfigured = true; } @@ -138,17 +140,44 @@ class ButtonUpdater{ isCallActive() { return this.callIsActive; } + + getSvgElement(icon, key) { + if (!this.iconSvgCache.includes(key)) { + this.log("getSvgElement() creating the icon for: " + key); + this.iconSvgCache[key] = createSVGElement(icon); + } + this.log("getting cached icon for: " + key); + return this.iconSvgCache[key]; + } + + changeIcon(button, icon, iconKey, isSvgOverwritten) { + const svgElement = this.getSvgElement(icon, iconKey); + if (!isSvgOverwritten) { + this.log("svgIs not to be overwritten, about to remove children from button"); + this.removeChildrenFrom(button); + this.log("svgIs not to be overwritten, about to append svg child to button"); + button.appendChild(svgElement); + } else if (button.firstChild) { + this.log("button has a first child, appending svg to it"); + button.firstChild.appendChild(svgElement); + } else { + this.log("button has no children, creating an svg child for it"); + button.appendChild(svgElement); + } + this.log("end of changeIcon()"); + } updateCallButton(options) { - let {button, icon, label, labelArgument, clickEventName, longPressEventName, longReleaseEventName, isCallActive = false, isMouseOutProcessed = false} = options; + let {button, icon, label, labelArgument, clickEventName, longPressEventName, longReleaseEventName, isCallActive = false, isMouseOutProcessed = false, overwriteSvg = false} = options; if (!button) { button = document.getElementById("saypi-callButton"); } if (button) { - this.removeChildrenFrom(button); - this.addIconTo(button, icon); - // this.resetListenersOf(button); + const iconKey = label; + this.changeIcon(button, icon, iconKey, overwriteSvg); + this.log("about to setAriaLabelOf button"); this.setAriaLabelOf(button, label, labelArgument); + this.log("about to update button roles"); this.updateButtonRoles({ button: button, clickEventName: clickEventName, @@ -156,8 +185,11 @@ class ButtonUpdater{ longUpEventName: longReleaseEventName, isMouseOutHandled: isMouseOutProcessed, }); + this.log("about to toggleActiveState of button"); this.toggleActiveState(button, isCallActive); + this.log("setting call active"); this.callIsActive = isCallActive; + this.log("exiting updateCallButton"); } } } From 98cd386741a04785e491c21797e98c4b3047d7e4 Mon Sep 17 00:00:00 2001 From: john Date: Mon, 3 Feb 2025 01:03:55 +0000 Subject: [PATCH 24/25] Long-press and long-release button actions are now working as expected with both click events and touch events. --- src/buttons/ButtonHelper.js | 112 ++++++++++++++++-------------------- 1 file changed, 50 insertions(+), 62 deletions(-) diff --git a/src/buttons/ButtonHelper.js b/src/buttons/ButtonHelper.js index c5c64c18a8..b6af7b15c2 100644 --- a/src/buttons/ButtonHelper.js +++ b/src/buttons/ButtonHelper.js @@ -15,14 +15,15 @@ class ButtonUpdater{ // track whether a call is active, so that new button instances can be initialized correctly this.callIsActive = false; - this.currentOnClick = () => {}; - this.currentOnLongClick = () => {}; - this.currentOnLongRelease = () => {}; + this.clickEventName = ""; + this.longDownEventName = ""; + this.longUpEventName = ""; this.isMouseOutHandled = false; this.areButtonClicksConfigured = false; this.iconSvgCache = new Array(); + this.areListenersEnabled = true; } removeChildrenFrom(button) { @@ -40,62 +41,69 @@ class ButtonUpdater{ callButton.classList.toggle("active", isActive); } - setEmptyDefault(runnable) { - return runnable ? runnable : () => {}; - } - isShortClick() { const clickDuration = (Date.now() - this.clickStartTime); return this.clickStartTime == 0 || clickDuration < this.longPressMilliseconds; } - createEvent(eventName) { - return eventName - ? () => { this.log("createEvent() sending event: " + eventName); this.sayPiActor.send(eventName); } - : () => {}; + sendEvent(eventName) { + if (eventName) { + this.sayPiActor.send(eventName); + } + } + + sendClickEvent() { + this.sendEvent(this.clickEventName); + } + + sendLongDownEvent() { + this.sendEvent(this.longDownEventName); } - resetListenersOf(button) { - button.onclick = () => {}; - button.addEventListener("mouseout", () => {}); - button.addEventListener("touchstart", ()=> {}); - button.addEventListener("touchend", ()=> {}); - button.addEventListener("touchcancel", ()=> {}); + sendLongUpEvent() { + this.sendEvent(this.longUpEventName); } onDown() { + if (!this.areListenersEnabled) { + return; + } this.isMouseDown = true; this.clickStartTime = Date.now(); window.setTimeout( () => { if (this.isMouseDown === true) { - this.log("onDown() about to currentOnLongClick()"); - this.currentOnLongClick(); + this.sendLongDownEvent(); this.isLongClickEngaged = true; } }, this.longPressMilliseconds); } onUp() { + if (!this.areListenersEnabled) { + return; + } this.isMouseDown = false; if (this.isShortClick()) { - this.log("onUp() about to currentOnClick()"); - this.currentOnClick(); + this.sendClickEvent(); } else if (this.isLongClickEngaged) { - this.log("onUp() about to currentOnLongRelease()"); - this.currentOnLongRelease(); + this.sendLongUpEvent(); this.isLongClickEngaged = false; } } - onOut() { - if (this.isMouseOutHandled && this.isLongClickEngaged) { - this.currentOnLongRelease(); + onMouseMovedOut() { + if (this.isMouseOutHandled && this.isLongClickEngaged && this.areListenersEnabled) { + this.sendLongUpEvent(); this.isLongClickEngaged = false; } } + isUsingTouchEvents() { + return typeof(window.ontouchstart) != 'undefined'; + } + setupButtonListeners(button) { - if (typeof(window.ontouchstart) != 'undefined') { + if (this.isUsingTouchEvents()) { button.addEventListener('touchstart',() => { this.onDown(); }); @@ -107,64 +115,46 @@ class ButtonUpdater{ this.onUp(); }); } else { - button.onmousedown = () => { - this.log("onmousedown"); - this.onDown(); - } - button.onmouseup = () => { - this.log("onmouseup"); - this.onUp(); - } - button.addEventListener("mouseout", () => this.onOut()); + button.onmousedown = () => this.onDown() ; + button.onmouseup = () => this.onUp(); + button.addEventListener("mouseout", () => this.onMouseMovedOut()); } } updateButtonRoles(options) { - let { button, clickEventName, longDownEventName, longUpEventName, isMouseOutHandled = false } = options; + let { button, clickEventName = "", longDownEventName = "", longUpEventName = "", isMouseOutHandled = false } = options; - if (this.areButtonClicksConfigured === false) { + if (!this.areButtonClicksConfigured) { this.setupButtonListeners(button); this.areButtonClicksConfigured = true; } - - this.currentOnClick = this.createEvent(clickEventName); - this.currentOnLongClick = this.createEvent(longDownEventName); - this.currentOnLongRelease = this.createEvent(longUpEventName); + this.clickEventName = clickEventName + this.longDownEventName = longDownEventName + this.longUpEventName = longUpEventName; this.isMouseOutHandled = isMouseOutHandled; } - log(msg) { - console.log("^^^ ButtonHelper: " + msg); - } - isCallActive() { return this.callIsActive; } getSvgElement(icon, key) { if (!this.iconSvgCache.includes(key)) { - this.log("getSvgElement() creating the icon for: " + key); this.iconSvgCache[key] = createSVGElement(icon); - } - this.log("getting cached icon for: " + key); + } return this.iconSvgCache[key]; } changeIcon(button, icon, iconKey, isSvgOverwritten) { const svgElement = this.getSvgElement(icon, iconKey); - if (!isSvgOverwritten) { - this.log("svgIs not to be overwritten, about to remove children from button"); + if (!isSvgOverwritten || !this.isUsingTouchEvents()) { this.removeChildrenFrom(button); - this.log("svgIs not to be overwritten, about to append svg child to button"); button.appendChild(svgElement); } else if (button.firstChild) { - this.log("button has a first child, appending svg to it"); button.firstChild.appendChild(svgElement); } else { - this.log("button has no children, creating an svg child for it"); button.appendChild(svgElement); } - this.log("end of changeIcon()"); } updateCallButton(options) { @@ -173,11 +163,13 @@ class ButtonUpdater{ button = document.getElementById("saypi-callButton"); } if (button) { + this.areListenersEnabled = false; const iconKey = label; this.changeIcon(button, icon, iconKey, overwriteSvg); - this.log("about to setAriaLabelOf button"); this.setAriaLabelOf(button, label, labelArgument); - this.log("about to update button roles"); + this.toggleActiveState(button, isCallActive); + this.callIsActive = isCallActive; + this.updateButtonRoles({ button: button, clickEventName: clickEventName, @@ -185,11 +177,7 @@ class ButtonUpdater{ longUpEventName: longReleaseEventName, isMouseOutHandled: isMouseOutProcessed, }); - this.log("about to toggleActiveState of button"); - this.toggleActiveState(button, isCallActive); - this.log("setting call active"); - this.callIsActive = isCallActive; - this.log("exiting updateCallButton"); + this.areListenersEnabled = true; } } } From 86849bc088081c85450ea0e69d2877a2576741d6 Mon Sep 17 00:00:00 2001 From: john Date: Tue, 4 Feb 2025 00:36:35 +0000 Subject: [PATCH 25/25] Simplified the updateCallButton method signature. --- src/ButtonModule.js | 3 +-- src/buttons/ButtonHelper.js | 8 +++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/ButtonModule.js b/src/ButtonModule.js index a1a94a3430..c12c681027 100644 --- a/src/ButtonModule.js +++ b/src/ButtonModule.js @@ -346,9 +346,8 @@ class ButtonModule { label: "momentaryCallListening", clickEventName: "saypi:momentaryStop", longReleaseEventName: "saypi:momentaryPause", - isMouseOutProcessed: true, + isCallActive: true, - overwriteSvg: true, }); } diff --git a/src/buttons/ButtonHelper.js b/src/buttons/ButtonHelper.js index b6af7b15c2..70b00f436f 100644 --- a/src/buttons/ButtonHelper.js +++ b/src/buttons/ButtonHelper.js @@ -158,18 +158,20 @@ class ButtonUpdater{ } updateCallButton(options) { - let {button, icon, label, labelArgument, clickEventName, longPressEventName, longReleaseEventName, isCallActive = false, isMouseOutProcessed = false, overwriteSvg = false} = options; + let {button, icon, label, labelArgument, clickEventName, longPressEventName, longReleaseEventName, isCallActive = false} = options; if (!button) { button = document.getElementById("saypi-callButton"); } if (button) { this.areListenersEnabled = false; const iconKey = label; - this.changeIcon(button, icon, iconKey, overwriteSvg); + const isSvgOverwritten = longReleaseEventName? true : false; + const isMouseOutProcessed = isSvgOverwritten; + + this.changeIcon(button, icon, iconKey, isSvgOverwritten); this.setAriaLabelOf(button, label, labelArgument); this.toggleActiveState(button, isCallActive); this.callIsActive = isCallActive; - this.updateButtonRoles({ button: button, clickEventName: clickEventName,