From 2c2afd5876676b81930d049d528219023ee6f6d0 Mon Sep 17 00:00:00 2001 From: mukherja04 Date: Wed, 19 Nov 2025 16:58:48 -0600 Subject: [PATCH 1/2] Added mic input not detected message --- src/App.tsx | 37 +++++++++++++ .../api/scribearServer/scribearRecognizer.tsx | 54 +++++++++++++++++++ .../api/whisper/whisperRecognizer.tsx | 53 ++++++++++++++++++ .../redux/reducers/controlReducers.tsx | 3 ++ .../redux/types/controlStatus.tsx | 2 + src/store.tsx | 2 + 6 files changed, 151 insertions(+) diff --git a/src/App.tsx b/src/App.tsx index 2a1be4e7..8773a415 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -16,6 +16,8 @@ function App() { const scribearStatus = useSelector((state: RootState) => state.APIStatusReducer?.scribearServerStatus as number); const scribearMessage = useSelector((state: RootState) => (state.APIStatusReducer as any)?.scribearServerMessage as string | undefined); + const micNoAudio = useSelector((state: RootState) => (state.ControlReducer as any)?.micNoAudio as boolean | undefined); + const listening = useSelector((state: RootState) => (state.ControlReducer as any)?.listening as boolean | undefined); const [snackbarOpen, setSnackbarOpen] = useState(false); const [snackbarMsg, setSnackbarMsg] = useState(''); @@ -39,6 +41,41 @@ function App() { } }, [scribearStatus]); + useEffect(() => { + // show mic inactivity when mic is on but no audio chunks are received + if (listening && micNoAudio) { + setSnackbarMsg('Microphone is active but no audio detected'); + setSnackbarSeverity('warning'); + setSnackbarOpen(true); + } + + // When listening turns ON, start a one-shot timer that expects at least one ondataavailable + // call within thresholdMs. This avoids firing on normal silent pauses after audio has been + // received previously. We only trigger inactivity if no blob arrives at all after enabling mic. + const thresholdMs = 3000; + try { + if (listening) { + try { (window as any).__hasReceivedAudio = false; } catch (e) {} + if ((window as any).__initialAudioTimer) { try { clearTimeout((window as any).__initialAudioTimer); } catch (e) {} } + (window as any).__initialAudioTimer = setTimeout(() => { + try { + const has = (window as any).__hasReceivedAudio === true; + if (!has) { + try { (window as any).store.dispatch({ type: 'SET_MIC_INACTIVITY', payload: true }); } catch (e) {} + } + } catch (e) {} + }, thresholdMs); + } else { + // listening turned off: clear initial timer and ensure flag reset + try { if ((window as any).__initialAudioTimer) { clearTimeout((window as any).__initialAudioTimer); (window as any).__initialAudioTimer = null; } } catch (e) {} + try { (window as any).store.dispatch({ type: 'SET_MIC_INACTIVITY', payload: false }); } catch (e) {} + } + } catch (e) { + console.warn('Failed to start initial mic monitor', e); + } + // no cleanup needed here because we clear/set timer when listening toggles + }, [listening, micNoAudio]); + const handleClose = (_event?: React.SyntheticEvent | Event, reason?: string) => { if (reason === 'clickaway') return; setSnackbarOpen(false); diff --git a/src/components/api/scribearServer/scribearRecognizer.tsx b/src/components/api/scribearServer/scribearRecognizer.tsx index 931293ce..971d223a 100644 --- a/src/components/api/scribearServer/scribearRecognizer.tsx +++ b/src/components/api/scribearServer/scribearRecognizer.tsx @@ -31,6 +31,10 @@ export class ScribearRecognizer implements Recognizer { private language: string private recorder?: RecordRTC; private kSampleRate = 16000; + // last time we received an audio chunk (ms since epoch) + private lastAudioTimestamp: number | null = null; + // interval id for inactivity checks + private inactivityInterval: any = null; urlParams = new URLSearchParams(window.location.search); mode = this.urlParams.get('mode'); @@ -58,6 +62,18 @@ export class ScribearRecognizer implements Recognizer { desiredSampRate: this.kSampleRate, timeSlice: 50, ondataavailable: async (blob: Blob) => { + // update last audio timestamp and mark that we've received at least one audio chunk + this.lastAudioTimestamp = Date.now(); + try { (window as any).__lastAudioTimestamp = this.lastAudioTimestamp; } catch (e) {} + try { (window as any).__hasReceivedAudio = true; if ((window as any).__initialAudioTimer) { clearTimeout((window as any).__initialAudioTimer); (window as any).__initialAudioTimer = null; } } catch (e) {} + try { + const controlState = (store.getState() as any).ControlReducer; + if (controlState?.micNoAudio === true) { + store.dispatch({ type: 'SET_MIC_INACTIVITY', payload: false }); + } + } catch (e) { + console.warn('Failed to clear mic inactivity', e); + } this.socket?.send(blob); }, recorderType: StereoAudioRecorder, @@ -65,6 +81,34 @@ export class ScribearRecognizer implements Recognizer { }); this.recorder.startRecording(); + + // start inactivity monitor: if mic is on but we haven't received audio for threshold -> set micNoAudio + const thresholdMs = 3000; // consider no audio if no chunks in 3s + if (this.inactivityInterval == null) { + this.inactivityInterval = setInterval(() => { + try { + const state: any = store.getState(); + const listening = state.ControlReducer?.listening === true; + const micNoAudio = state.ControlReducer?.micNoAudio === true; + if (listening) { + if (!this.lastAudioTimestamp || (Date.now() - this.lastAudioTimestamp > thresholdMs)) { + if (!micNoAudio) { + store.dispatch({ type: 'SET_MIC_INACTIVITY', payload: true }); + } + } else { + if (micNoAudio) { + store.dispatch({ type: 'SET_MIC_INACTIVITY', payload: false }); + } + } + } else { + // not listening: ensure flag is cleared + if (micNoAudio) store.dispatch({ type: 'SET_MIC_INACTIVITY', payload: false }); + } + } catch (e) { + console.warn('Error in mic inactivity interval', e); + } + }, 1000); + } } /** @@ -179,6 +223,16 @@ export class ScribearRecognizer implements Recognizer { if (!this.socket) { return; } this.socket.close(); this.socket = null; + // clear inactivity interval and reset mic inactivity flag + if (this.inactivityInterval) { + clearInterval(this.inactivityInterval); + this.inactivityInterval = null; + } + try { + store.dispatch({ type: 'SET_MIC_INACTIVITY', payload: false }); + } catch (e) { + console.warn('Failed to clear mic inactivity on stop', e); + } } /** diff --git a/src/components/api/whisper/whisperRecognizer.tsx b/src/components/api/whisper/whisperRecognizer.tsx index ad61e21f..c33a57d2 100644 --- a/src/components/api/whisper/whisperRecognizer.tsx +++ b/src/components/api/whisper/whisperRecognizer.tsx @@ -48,6 +48,9 @@ export class WhisperRecognizer implements Recognizer { private num_threads: number; private transcribed_callback: ((newFinalBlocks: Array, newInProgressBlock: TranscriptBlock) => void) | null = null; + // mic activity tracking + private lastAudioTimestamp: number | null = null; + private inactivityInterval: any = null; /** * Creates an Whisper recognizer instance that listens to the default microphone @@ -135,6 +138,22 @@ export class WhisperRecognizer implements Recognizer { pcm_data = Float32Concat(last_suffix, pcm_data); last_suffix = pcm_data.slice(-(pcm_data.length % 128)) + // update last audio timestamp and mark that we've received at least one audio chunk + this.lastAudioTimestamp = Date.now(); + try { (window as any).__lastAudioTimestamp = this.lastAudioTimestamp; } catch (e) {} + try { (window as any).__hasReceivedAudio = true; if ((window as any).__initialAudioTimer) { clearTimeout((window as any).__initialAudioTimer); (window as any).__initialAudioTimer = null; } } catch (e) {} + try { + // clear micNoAudio if previously set + // avoid importing store at top; use global store via require to prevent circular import issues + const { store } = require('../../../store'); + const controlState = (store.getState() as any).ControlReducer; + if (controlState?.micNoAudio === true) { + store.dispatch({ type: 'SET_MIC_INACTIVITY', payload: false }); + } + } catch (e) { + console.warn('Failed to clear mic inactivity (whisper)', e); + } + // Feed process_recorder_message audio in 128 sample chunks for (let i = 0; i < pcm_data.length - 127; i+= 128) { const audio_chunk = pcm_data.subarray(i, i + 128) @@ -149,6 +168,29 @@ export class WhisperRecognizer implements Recognizer { this.recorder.startRecording(); console.log("Whisper: Done setting up audio context"); + // start inactivity monitor + const thresholdMs = 3000; + if (this.inactivityInterval == null) { + const { store } = require('../../../store'); + this.inactivityInterval = setInterval(() => { + try { + const state: any = store.getState(); + const listening = state.ControlReducer?.listening === true; + const micNoAudio = state.ControlReducer?.micNoAudio === true; + if (listening) { + if (!this.lastAudioTimestamp || (Date.now() - this.lastAudioTimestamp > thresholdMs)) { + if (!micNoAudio) store.dispatch({ type: 'SET_MIC_INACTIVITY', payload: true }); + } else { + if (micNoAudio) store.dispatch({ type: 'SET_MIC_INACTIVITY', payload: false }); + } + } else { + if (micNoAudio) store.dispatch({ type: 'SET_MIC_INACTIVITY', payload: false }); + } + } catch (e) { + console.warn('Error in whisper mic inactivity interval', e); + } + }, 1000); + } } private async load_model(model: string) { @@ -257,6 +299,17 @@ export class WhisperRecognizer implements Recognizer { this.whisper.set_status("paused"); this.context.suspend(); this.recorder?.stopRecording(); + // clear inactivity interval and reset mic inactivity flag + if (this.inactivityInterval) { + clearInterval(this.inactivityInterval); + this.inactivityInterval = null; + } + try { + const { store } = require('../../../store'); + store.dispatch({ type: 'SET_MIC_INACTIVITY', payload: false }); + } catch (e) { + console.warn('Failed to clear mic inactivity on whisper stop', e); + } } /** diff --git a/src/react-redux&middleware/redux/reducers/controlReducers.tsx b/src/react-redux&middleware/redux/reducers/controlReducers.tsx index f21c2802..847b52b8 100644 --- a/src/react-redux&middleware/redux/reducers/controlReducers.tsx +++ b/src/react-redux&middleware/redux/reducers/controlReducers.tsx @@ -22,6 +22,7 @@ const initialControlState : ControlStatus = { showMFCC: false, showSpeaker: false, showIntent: false, + micNoAudio: false, } export function ControlReducer(state = initialControlState, action) { @@ -41,6 +42,8 @@ export function ControlReducer(state = initialControlState, action) { return { ...state, showIntent: !state.showIntent }; case 'FLIP_RECORDING_PHRASE': return { ...state, listening: action.payload}; + case 'SET_MIC_INACTIVITY': + return { ...state, micNoAudio: action.payload }; case 'SET_SPEECH_LANGUAGE': return { ...state, diff --git a/src/react-redux&middleware/redux/types/controlStatus.tsx b/src/react-redux&middleware/redux/types/controlStatus.tsx index 52197a4e..e736446a 100644 --- a/src/react-redux&middleware/redux/types/controlStatus.tsx +++ b/src/react-redux&middleware/redux/types/controlStatus.tsx @@ -16,4 +16,6 @@ export type ControlStatus = { showMFCC: boolean showSpeaker: boolean showIntent: boolean + // true when microphone is on but no audio chunks have been received for a short time + micNoAudio?: boolean } diff --git a/src/store.tsx b/src/store.tsx index 8b1f8a3c..f90c72f6 100644 --- a/src/store.tsx +++ b/src/store.tsx @@ -37,3 +37,5 @@ export const store = configureStore({ }); export type RootState = ReturnType +// Expose store on window for runtime helpers (used by mic inactivity monitor) +(window as any).store = store; From 639df7bc0345b5435338169edf2cbf82089fd15e Mon Sep 17 00:00:00 2001 From: mukherja04 Date: Sun, 7 Dec 2025 00:52:28 -0600 Subject: [PATCH 2/2] Updated mic no audio detection --- src/App.tsx | 34 ++---------- .../api/scribearServer/scribearRecognizer.tsx | 19 ++++--- .../api/whisper/whisperRecognizer.tsx | 53 ------------------- .../redux/reducers/controlReducers.tsx | 3 -- .../redux/types/controlStatus.tsx | 2 - src/store.tsx | 2 - 6 files changed, 16 insertions(+), 97 deletions(-) diff --git a/src/App.tsx b/src/App.tsx index 8773a415..0db54659 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -16,8 +16,8 @@ function App() { const scribearStatus = useSelector((state: RootState) => state.APIStatusReducer?.scribearServerStatus as number); const scribearMessage = useSelector((state: RootState) => (state.APIStatusReducer as any)?.scribearServerMessage as string | undefined); - const micNoAudio = useSelector((state: RootState) => (state.ControlReducer as any)?.micNoAudio as boolean | undefined); - const listening = useSelector((state: RootState) => (state.ControlReducer as any)?.listening as boolean | undefined); + const listening = useSelector((state: RootState) => (state as any).ControlReducer?.listening === true); + const micNoAudio = useSelector((state: RootState) => (state as any).ControlReducer?.micNoAudio === true); const [snackbarOpen, setSnackbarOpen] = useState(false); const [snackbarMsg, setSnackbarMsg] = useState(''); @@ -42,38 +42,12 @@ function App() { }, [scribearStatus]); useEffect(() => { - // show mic inactivity when mic is on but no audio chunks are received + // Show a snackbar if we expect mic audio but none is coming through if (listening && micNoAudio) { - setSnackbarMsg('Microphone is active but no audio detected'); + setSnackbarMsg('No microphone audio detected. Please check permissions, input device, or mic level.'); setSnackbarSeverity('warning'); setSnackbarOpen(true); } - - // When listening turns ON, start a one-shot timer that expects at least one ondataavailable - // call within thresholdMs. This avoids firing on normal silent pauses after audio has been - // received previously. We only trigger inactivity if no blob arrives at all after enabling mic. - const thresholdMs = 3000; - try { - if (listening) { - try { (window as any).__hasReceivedAudio = false; } catch (e) {} - if ((window as any).__initialAudioTimer) { try { clearTimeout((window as any).__initialAudioTimer); } catch (e) {} } - (window as any).__initialAudioTimer = setTimeout(() => { - try { - const has = (window as any).__hasReceivedAudio === true; - if (!has) { - try { (window as any).store.dispatch({ type: 'SET_MIC_INACTIVITY', payload: true }); } catch (e) {} - } - } catch (e) {} - }, thresholdMs); - } else { - // listening turned off: clear initial timer and ensure flag reset - try { if ((window as any).__initialAudioTimer) { clearTimeout((window as any).__initialAudioTimer); (window as any).__initialAudioTimer = null; } } catch (e) {} - try { (window as any).store.dispatch({ type: 'SET_MIC_INACTIVITY', payload: false }); } catch (e) {} - } - } catch (e) { - console.warn('Failed to start initial mic monitor', e); - } - // no cleanup needed here because we clear/set timer when listening toggles }, [listening, micNoAudio]); const handleClose = (_event?: React.SyntheticEvent | Event, reason?: string) => { diff --git a/src/components/api/scribearServer/scribearRecognizer.tsx b/src/components/api/scribearServer/scribearRecognizer.tsx index 971d223a..a3f65b1d 100644 --- a/src/components/api/scribearServer/scribearRecognizer.tsx +++ b/src/components/api/scribearServer/scribearRecognizer.tsx @@ -31,9 +31,7 @@ export class ScribearRecognizer implements Recognizer { private language: string private recorder?: RecordRTC; private kSampleRate = 16000; - // last time we received an audio chunk (ms since epoch) private lastAudioTimestamp: number | null = null; - // interval id for inactivity checks private inactivityInterval: any = null; urlParams = new URLSearchParams(window.location.search); @@ -54,7 +52,16 @@ export class ScribearRecognizer implements Recognizer { } private async _startRecording() { - let mic_stream = await navigator.mediaDevices.getUserMedia({ audio: true, video: false }); + let mic_stream: MediaStream; + try { + mic_stream = await navigator.mediaDevices.getUserMedia({ audio: true, video: false }); + } catch (e) { + console.error('Failed to access microphone', e); + try { store.dispatch({ type: 'SET_MIC_INACTIVITY', payload: true }); } catch (_) {} + // Surface an API status error to prompt UI; recognizer encapsulates flag setting here + try { store.dispatch({ type: 'CHANGE_API_STATUS', payload: { scribearServerStatus: STATUS.ERROR, scribearServerMessage: 'Microphone permission denied or unavailable' } }); } catch (_) {} + throw e; + } this.recorder = new RecordRTC(mic_stream, { type: 'audio', @@ -82,8 +89,8 @@ export class ScribearRecognizer implements Recognizer { this.recorder.startRecording(); - // start inactivity monitor: if mic is on but we haven't received audio for threshold -> set micNoAudio - const thresholdMs = 3000; // consider no audio if no chunks in 3s + // start inactivity monitor + const thresholdMs = 3000; if (this.inactivityInterval == null) { this.inactivityInterval = setInterval(() => { try { @@ -101,7 +108,6 @@ export class ScribearRecognizer implements Recognizer { } } } else { - // not listening: ensure flag is cleared if (micNoAudio) store.dispatch({ type: 'SET_MIC_INACTIVITY', payload: false }); } } catch (e) { @@ -223,7 +229,6 @@ export class ScribearRecognizer implements Recognizer { if (!this.socket) { return; } this.socket.close(); this.socket = null; - // clear inactivity interval and reset mic inactivity flag if (this.inactivityInterval) { clearInterval(this.inactivityInterval); this.inactivityInterval = null; diff --git a/src/components/api/whisper/whisperRecognizer.tsx b/src/components/api/whisper/whisperRecognizer.tsx index c33a57d2..ad61e21f 100644 --- a/src/components/api/whisper/whisperRecognizer.tsx +++ b/src/components/api/whisper/whisperRecognizer.tsx @@ -48,9 +48,6 @@ export class WhisperRecognizer implements Recognizer { private num_threads: number; private transcribed_callback: ((newFinalBlocks: Array, newInProgressBlock: TranscriptBlock) => void) | null = null; - // mic activity tracking - private lastAudioTimestamp: number | null = null; - private inactivityInterval: any = null; /** * Creates an Whisper recognizer instance that listens to the default microphone @@ -138,22 +135,6 @@ export class WhisperRecognizer implements Recognizer { pcm_data = Float32Concat(last_suffix, pcm_data); last_suffix = pcm_data.slice(-(pcm_data.length % 128)) - // update last audio timestamp and mark that we've received at least one audio chunk - this.lastAudioTimestamp = Date.now(); - try { (window as any).__lastAudioTimestamp = this.lastAudioTimestamp; } catch (e) {} - try { (window as any).__hasReceivedAudio = true; if ((window as any).__initialAudioTimer) { clearTimeout((window as any).__initialAudioTimer); (window as any).__initialAudioTimer = null; } } catch (e) {} - try { - // clear micNoAudio if previously set - // avoid importing store at top; use global store via require to prevent circular import issues - const { store } = require('../../../store'); - const controlState = (store.getState() as any).ControlReducer; - if (controlState?.micNoAudio === true) { - store.dispatch({ type: 'SET_MIC_INACTIVITY', payload: false }); - } - } catch (e) { - console.warn('Failed to clear mic inactivity (whisper)', e); - } - // Feed process_recorder_message audio in 128 sample chunks for (let i = 0; i < pcm_data.length - 127; i+= 128) { const audio_chunk = pcm_data.subarray(i, i + 128) @@ -168,29 +149,6 @@ export class WhisperRecognizer implements Recognizer { this.recorder.startRecording(); console.log("Whisper: Done setting up audio context"); - // start inactivity monitor - const thresholdMs = 3000; - if (this.inactivityInterval == null) { - const { store } = require('../../../store'); - this.inactivityInterval = setInterval(() => { - try { - const state: any = store.getState(); - const listening = state.ControlReducer?.listening === true; - const micNoAudio = state.ControlReducer?.micNoAudio === true; - if (listening) { - if (!this.lastAudioTimestamp || (Date.now() - this.lastAudioTimestamp > thresholdMs)) { - if (!micNoAudio) store.dispatch({ type: 'SET_MIC_INACTIVITY', payload: true }); - } else { - if (micNoAudio) store.dispatch({ type: 'SET_MIC_INACTIVITY', payload: false }); - } - } else { - if (micNoAudio) store.dispatch({ type: 'SET_MIC_INACTIVITY', payload: false }); - } - } catch (e) { - console.warn('Error in whisper mic inactivity interval', e); - } - }, 1000); - } } private async load_model(model: string) { @@ -299,17 +257,6 @@ export class WhisperRecognizer implements Recognizer { this.whisper.set_status("paused"); this.context.suspend(); this.recorder?.stopRecording(); - // clear inactivity interval and reset mic inactivity flag - if (this.inactivityInterval) { - clearInterval(this.inactivityInterval); - this.inactivityInterval = null; - } - try { - const { store } = require('../../../store'); - store.dispatch({ type: 'SET_MIC_INACTIVITY', payload: false }); - } catch (e) { - console.warn('Failed to clear mic inactivity on whisper stop', e); - } } /** diff --git a/src/react-redux&middleware/redux/reducers/controlReducers.tsx b/src/react-redux&middleware/redux/reducers/controlReducers.tsx index 847b52b8..f21c2802 100644 --- a/src/react-redux&middleware/redux/reducers/controlReducers.tsx +++ b/src/react-redux&middleware/redux/reducers/controlReducers.tsx @@ -22,7 +22,6 @@ const initialControlState : ControlStatus = { showMFCC: false, showSpeaker: false, showIntent: false, - micNoAudio: false, } export function ControlReducer(state = initialControlState, action) { @@ -42,8 +41,6 @@ export function ControlReducer(state = initialControlState, action) { return { ...state, showIntent: !state.showIntent }; case 'FLIP_RECORDING_PHRASE': return { ...state, listening: action.payload}; - case 'SET_MIC_INACTIVITY': - return { ...state, micNoAudio: action.payload }; case 'SET_SPEECH_LANGUAGE': return { ...state, diff --git a/src/react-redux&middleware/redux/types/controlStatus.tsx b/src/react-redux&middleware/redux/types/controlStatus.tsx index e736446a..52197a4e 100644 --- a/src/react-redux&middleware/redux/types/controlStatus.tsx +++ b/src/react-redux&middleware/redux/types/controlStatus.tsx @@ -16,6 +16,4 @@ export type ControlStatus = { showMFCC: boolean showSpeaker: boolean showIntent: boolean - // true when microphone is on but no audio chunks have been received for a short time - micNoAudio?: boolean } diff --git a/src/store.tsx b/src/store.tsx index f90c72f6..8b1f8a3c 100644 --- a/src/store.tsx +++ b/src/store.tsx @@ -37,5 +37,3 @@ export const store = configureStore({ }); export type RootState = ReturnType -// Expose store on window for runtime helpers (used by mic inactivity monitor) -(window as any).store = store;