diff --git a/src/App.tsx b/src/App.tsx index 2a1be4e..0db5465 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -16,6 +16,8 @@ function App() { const scribearStatus = useSelector((state: RootState) => state.APIStatusReducer?.scribearServerStatus as number); const scribearMessage = useSelector((state: RootState) => (state.APIStatusReducer as any)?.scribearServerMessage as string | undefined); + const listening = useSelector((state: RootState) => (state as any).ControlReducer?.listening === true); + const micNoAudio = useSelector((state: RootState) => (state as any).ControlReducer?.micNoAudio === true); const [snackbarOpen, setSnackbarOpen] = useState(false); const [snackbarMsg, setSnackbarMsg] = useState(''); @@ -39,6 +41,15 @@ function App() { } }, [scribearStatus]); + useEffect(() => { + // Show a snackbar if we expect mic audio but none is coming through + if (listening && micNoAudio) { + setSnackbarMsg('No microphone audio detected. Please check permissions, input device, or mic level.'); + setSnackbarSeverity('warning'); + setSnackbarOpen(true); + } + }, [listening, micNoAudio]); + const handleClose = (_event?: React.SyntheticEvent | Event, reason?: string) => { if (reason === 'clickaway') return; setSnackbarOpen(false); diff --git a/src/components/api/scribearServer/scribearRecognizer.tsx b/src/components/api/scribearServer/scribearRecognizer.tsx index 931293c..a3f65b1 100644 --- a/src/components/api/scribearServer/scribearRecognizer.tsx +++ b/src/components/api/scribearServer/scribearRecognizer.tsx @@ -31,6 +31,8 @@ export class ScribearRecognizer implements Recognizer { private language: string private recorder?: RecordRTC; private kSampleRate = 16000; + private lastAudioTimestamp: number | null = null; + private inactivityInterval: any = null; urlParams = new URLSearchParams(window.location.search); mode = this.urlParams.get('mode'); @@ -50,7 +52,16 @@ export class ScribearRecognizer implements Recognizer { } private async _startRecording() { - let mic_stream = await navigator.mediaDevices.getUserMedia({ audio: true, video: false }); + let mic_stream: MediaStream; + try { + mic_stream = await navigator.mediaDevices.getUserMedia({ audio: true, video: false }); + } catch (e) { + console.error('Failed to access microphone', e); + try { store.dispatch({ type: 'SET_MIC_INACTIVITY', payload: true }); } catch (_) {} + // Surface an API status error to prompt UI; recognizer encapsulates flag setting here + try { store.dispatch({ type: 'CHANGE_API_STATUS', payload: { scribearServerStatus: STATUS.ERROR, scribearServerMessage: 'Microphone permission denied or unavailable' } }); } catch (_) {} + throw e; + } this.recorder = new RecordRTC(mic_stream, { type: 'audio', @@ -58,6 +69,18 @@ export class ScribearRecognizer implements Recognizer { desiredSampRate: this.kSampleRate, timeSlice: 50, ondataavailable: async (blob: Blob) => { + // update last audio timestamp and mark that we've received at least one audio chunk + this.lastAudioTimestamp = Date.now(); + try { (window as any).__lastAudioTimestamp = this.lastAudioTimestamp; } catch (e) {} + try { (window as any).__hasReceivedAudio = true; if ((window as any).__initialAudioTimer) { clearTimeout((window as any).__initialAudioTimer); (window as any).__initialAudioTimer = null; } } catch (e) {} + try { + const controlState = (store.getState() as any).ControlReducer; + if (controlState?.micNoAudio === true) { + store.dispatch({ type: 'SET_MIC_INACTIVITY', payload: false }); + } + } catch (e) { + console.warn('Failed to clear mic inactivity', e); + } this.socket?.send(blob); }, recorderType: StereoAudioRecorder, @@ -65,6 +88,33 @@ export class ScribearRecognizer implements Recognizer { }); this.recorder.startRecording(); + + // start inactivity monitor + const thresholdMs = 3000; + if (this.inactivityInterval == null) { + this.inactivityInterval = setInterval(() => { + try { + const state: any = store.getState(); + const listening = state.ControlReducer?.listening === true; + const micNoAudio = state.ControlReducer?.micNoAudio === true; + if (listening) { + if (!this.lastAudioTimestamp || (Date.now() - this.lastAudioTimestamp > thresholdMs)) { + if (!micNoAudio) { + store.dispatch({ type: 'SET_MIC_INACTIVITY', payload: true }); + } + } else { + if (micNoAudio) { + store.dispatch({ type: 'SET_MIC_INACTIVITY', payload: false }); + } + } + } else { + if (micNoAudio) store.dispatch({ type: 'SET_MIC_INACTIVITY', payload: false }); + } + } catch (e) { + console.warn('Error in mic inactivity interval', e); + } + }, 1000); + } } /** @@ -179,6 +229,15 @@ export class ScribearRecognizer implements Recognizer { if (!this.socket) { return; } this.socket.close(); this.socket = null; + if (this.inactivityInterval) { + clearInterval(this.inactivityInterval); + this.inactivityInterval = null; + } + try { + store.dispatch({ type: 'SET_MIC_INACTIVITY', payload: false }); + } catch (e) { + console.warn('Failed to clear mic inactivity on stop', e); + } } /**