Skip to content
This repository was archived by the owner on May 6, 2022. It is now read-only.

Commit 73e9c6e

Browse files
timmywiltimmywil
committed
fix(mic): allow mic errors to propagate; show them in demo
1 parent 5028a61 commit 73e9c6e

File tree

4 files changed

+75
-41
lines changed

4 files changed

+75
-41
lines changed

‎examples/with-next/pages/index.tsx‎

Lines changed: 62 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@ interface State {
3333
results: Repo[]
3434
searching: boolean
3535
status: string
36-
streaming: boolean
3736
term: string
3837
total: number
3938
wakeword: CommandDemo
@@ -51,7 +50,6 @@ export default class Index extends PureComponent {
5150
results: [],
5251
searching: false,
5352
status: 'Idle',
54-
streaming: false,
5553
term: '',
5654
total: 0,
5755
wakeword: { error: '', status: 'Idle', result: false }
@@ -96,7 +94,7 @@ export default class Index extends PureComponent {
9694
this.setState({ wakeword: { error: '', result: true } })
9795
break
9896
case SpeechEventType.Timeout:
99-
this.setState({ wakeword: { error: 'timeout' } })
97+
this.setState({ wakeword: { error: 'Detected speech, but did not match' } })
10098
break
10199
case SpeechEventType.Error:
102100
console.error(event.error)
@@ -107,6 +105,13 @@ export default class Index extends PureComponent {
107105
})
108106
this.setState({ wakeword: { status: 'Listening...', result: '' } })
109107
} catch (e) {
108+
this.setState({
109+
wakeword: {
110+
error:
111+
'This browser does not support wake word detection. Please try a Blink browser, such as Chrome, Edge, Opera, Vivaldi, or Brave.'
112+
},
113+
result: false
114+
})
110115
console.error(e)
111116
this.stopRecording()
112117
}
@@ -151,6 +156,13 @@ export default class Index extends PureComponent {
151156
})
152157
this.setState({ keyword: { status: 'Listening...', result: '' } })
153158
} catch (e) {
159+
this.setState({
160+
keyword: {
161+
error:
162+
'This browser does not support keyword detection. Please try a Blink browser, such as Chrome, Edge, Opera, Vivaldi, or Brave.',
163+
result: ''
164+
}
165+
})
154166
console.error(e)
155167
this.stopRecording()
156168
}
@@ -159,11 +171,12 @@ export default class Index extends PureComponent {
159171

160172
stopRecording = () => {
161173
stopPipeline()
174+
const { keyword, wakeword } = this.state
162175
this.setState({
163176
activeDemo: null,
164177
status: 'Idle',
165-
keyword: { status: 'Idle' },
166-
wakeword: { status: 'Idle' }
178+
keyword: { ...keyword, status: 'Idle' },
179+
wakeword: { ...wakeword, status: 'Idle' }
167180
})
168181
}
169182

@@ -179,7 +192,7 @@ export default class Index extends PureComponent {
179192
this.audio.addEventListener('pause', this.pause)
180193
this.audio.addEventListener('error', () => {
181194
this.setState({ status: 'There was an error loading the audio.' })
182-
if (this.state.streaming) {
195+
if (this.state.activeDemo === 'searchStream') {
183196
this.toggleRecordStream()
184197
}
185198
})
@@ -188,7 +201,7 @@ export default class Index extends PureComponent {
188201

189202
pause = () => {
190203
this.playing = false
191-
this.setState({ status: this.state.streaming ? 'Recording...' : 'Idle' })
204+
this.setState({ status: this.state.activeDemo === 'searchStream' ? 'Recording...' : 'Idle' })
192205
}
193206

194207
getPrompt(response: { total_count: number; items: { name: string }[] }) {
@@ -207,7 +220,7 @@ export default class Index extends PureComponent {
207220

208221
search = async () => {
209222
const { term } = this.state
210-
console.log(`Searching with term: ${term}`)
223+
// console.log(`Searching with term: ${term}`)
211224
const result = search(term)
212225
if (!result) {
213226
this.setState({
@@ -220,7 +233,7 @@ export default class Index extends PureComponent {
220233
this.setState({ searching: true, status: 'Searching...' })
221234
result
222235
.then(async (response) => {
223-
console.log(`Got response for term: ${term}`)
236+
// console.log(`Got response for term: ${term}`)
224237
this.setState({
225238
results: response.items || [],
226239
total: response.total_count
@@ -257,14 +270,24 @@ export default class Index extends PureComponent {
257270
}
258271

259272
record3Seconds = async () => {
273+
if (this.state.activeDemo || this.playing) {
274+
return
275+
}
260276
this.initialize()
261277
this.setState({ activeDemo: 'search' })
262-
const buffer = await record({
263-
time: 3,
264-
onProgress: (remaining) => {
265-
this.setState({ status: `Recording..${remaining}` })
266-
}
267-
})
278+
let buffer: AudioBuffer
279+
try {
280+
buffer = await record({
281+
time: 3,
282+
onProgress: (remaining) => {
283+
this.setState({ status: `Recording..${remaining}` })
284+
}
285+
})
286+
} catch (e) {
287+
console.error(e)
288+
this.setState({ activeDemo: null, error: e.message })
289+
return
290+
}
268291
this.setState({ activeDemo: null })
269292
upload(buffer)
270293
.then(({ text, message }) => {
@@ -293,37 +316,45 @@ export default class Index extends PureComponent {
293316
}
294317

295318
toggleRecordStream = async () => {
296-
const { streaming } = this.state
297-
if (streaming) {
319+
const { activeDemo } = this.state
320+
if (activeDemo !== null && activeDemo !== 'searchStream') {
321+
return
322+
}
323+
if (activeDemo === 'searchStream') {
298324
stopStream()
299-
this.setState({ activeDemo: null, streaming: false })
300-
} else {
325+
this.setState({ activeDemo: null })
326+
} else if (!this.playing) {
301327
this.initialize()
302328
try {
303-
const [ws] = await startStream({ isPlaying: () => this.playing })
329+
let ws: WebSocket
330+
try {
331+
;[ws] = await startStream({ isPlaying: () => this.playing })
332+
} catch (e) {
333+
console.error(e)
334+
this.setState({ activeDemo: null, error: e.message })
335+
return
336+
}
304337
ws.addEventListener('open', () =>
305-
this.setState({ activeDemo: 'searchStream', status: 'Recording...', streaming: true })
338+
this.setState({ activeDemo: 'searchStream', status: 'Recording...' })
306339
)
307340
ws.addEventListener('close', (event) => {
308341
this.setState({
309342
activeDemo: null,
310343
status:
311344
event.code === 1002
312-
? 'There was a problem starting the record stream. Please refresh and try again.'
313-
: 'Idle',
314-
streaming: false
345+
? event.reason ||
346+
'There was a problem starting the record stream. Please refresh and try again.'
347+
: 'Idle'
315348
})
316349
})
317350
ws.addEventListener('error', (event) => {
318351
console.error(event)
319352
this.setState({
320353
activeDemo: null,
321-
status: 'There was a problem starting the record stream. Please refresh and try again.',
322-
streaming: false
354+
status: 'There was a problem starting the record stream. Please refresh and try again.'
323355
})
324356
})
325357
ws.addEventListener('message', (e) => {
326-
console.log(e)
327358
this.updateTerm(e.data)
328359
})
329360
} catch (e) {
@@ -344,7 +375,6 @@ export default class Index extends PureComponent {
344375
results,
345376
searching,
346377
status,
347-
streaming,
348378
term,
349379
total,
350380
wakeword
@@ -354,6 +384,7 @@ export default class Index extends PureComponent {
354384
<Layout>
355385
<h1>Test a wakeword model</h1>
356386
<p>Press record and say, "Spokestack"</p>
387+
{wakeword.error && <p className="error">{wakeword.error}</p>}
357388
<div className="buttons">
358389
<button
359390
disabled={isActive && activeDemo !== 'wakeword'}
@@ -365,11 +396,11 @@ export default class Index extends PureComponent {
365396
<h4>
366397
Status: <span id="wakeword-status">{wakeword.status}</span>
367398
</h4>
368-
{wakeword.error && <p className="error">{wakeword.error}</p>}
369399
{wakeword.result && <p className="wrapper">Detected!</p>}
370400
<hr />
371401
<h1>Test a keyword model</h1>
372402
<p>Press record and say a number between 0 and 9.</p>
403+
{keyword.error && <p className="error">{keyword.error}</p>}
373404
<div className="buttons">
374405
<button
375406
disabled={isActive && activeDemo !== 'keyword'}
@@ -381,10 +412,10 @@ export default class Index extends PureComponent {
381412
<h4>
382413
Status: <span id="keyword-status">{keyword.status}</span>
383414
</h4>
384-
{keyword.error && <p className="error">{keyword.error}</p>}
385415
{keyword.result && <p className="wrapper">Detected: {keyword.result}</p>}
386416
<hr />
387417
<h1>Search GitHub for repositories using your voice</h1>
418+
{error && <p className="error">{error}</p>}
388419
<div className="buttons">
389420
<button
390421
disabled={isActive && activeDemo !== 'search'}
@@ -396,14 +427,13 @@ export default class Index extends PureComponent {
396427
disabled={isActive && activeDemo !== 'searchStream'}
397428
className="btn btn-primary"
398429
onClick={this.toggleRecordStream}>
399-
{streaming ? 'Stop' : 'Start'} streaming
430+
{activeDemo === 'searchStream' ? 'Stop' : 'Start'} streaming
400431
</button>
401432
</div>
402433
<h4>
403434
Status: <span id="status">{status}</span>
404435
</h4>
405436
<hr />
406-
{error && <p className="error">{error}</p>}
407437
{term && <p className="wrapper">Search term: {term}</p>}
408438
{searching ? (
409439
<p className="wrapper">Searching...</p>

‎src/client/SpeechPipeline.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -84,8 +84,11 @@ export default class SpeechPipeline {
8484
*/
8585
async start(): Promise<SpeechPipeline> {
8686
this.worker = await this.initWorker()
87-
const stream = await startRecord()
88-
if (!stream) {
87+
let stream: MediaStream
88+
try {
89+
stream = await startRecord()
90+
} catch (e) {
91+
console.error(e)
8992
throw new Error(
9093
`There was a problem starting the microphone. ${
9194
navigator.__polyfilledMediaDevices

‎src/client/mic.ts‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,17 +12,15 @@ declare global {
1212

1313
let stream: MediaStream | undefined
1414

15-
export function startRecord(): Promise<MediaStream | void> {
15+
export async function startRecord(): Promise<MediaStream> {
1616
const constraints = {
1717
autoGainControl: true,
1818
channelCount: 1,
1919
echoCancellation: true,
2020
noiseSuppression: true
2121
}
22-
return navigator.mediaDevices
23-
.getUserMedia({ audio: constraints })
24-
.then((s) => (stream = s))
25-
.catch(console.error.bind(console))
22+
const s = await navigator.mediaDevices.getUserMedia({ audio: constraints })
23+
return (stream = s)
2624
}
2725

2826
export function stopRecord() {

‎src/client/processor.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,11 @@ export interface ProcessorReturnValue {
2121
*/
2222
export async function startProcessor(): Promise<[Error] | [null, ProcessorReturnValue]> {
2323
stopProcessor()
24-
const stream = await startRecord()
25-
if (!stream) {
24+
let stream: MediaStream
25+
try {
26+
stream = await startRecord()
27+
} catch (e) {
28+
console.error(e)
2629
return [
2730
new Error(
2831
`There was a problem starting the microphone. ${

0 commit comments

Comments
 (0)