From d51634432ad72ef516a3b8b279203f8969df0ec4 Mon Sep 17 00:00:00 2001 From: Dilawar Singh Date: Mon, 24 Mar 2025 14:12:14 +0530 Subject: [PATCH 1/7] Can record audio now --- leptos/Cargo.toml | 5 +++++ leptos/src/components/audio.rs | 21 +++++++++++++++++---- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/leptos/Cargo.toml b/leptos/Cargo.toml index e3455a4..c68beb4 100644 --- a/leptos/Cargo.toml +++ b/leptos/Cargo.toml @@ -22,3 +22,8 @@ thaw = { version = "0.4.4", features = ["csr"] } tracing.workspace = true tracing-subscriber.workspace = true tracing-subscriber-wasm = "0.1.0" +web-sys = { version = "0.3.77", features = [ + "BlobEvent", + "MediaRecorder", + "AudioNode", +] } diff --git a/leptos/src/components/audio.rs b/leptos/src/components/audio.rs index 18aa5d8..f2db245 100644 --- a/leptos/src/components/audio.rs +++ b/leptos/src/components/audio.rs @@ -3,6 +3,8 @@ use leptos::prelude::*; use leptos_use::{use_user_media_with_options, UseUserMediaOptions, UseUserMediaReturn}; use thaw::*; +use web_sys::wasm_bindgen::{closure::Closure, JsCast, JsValue}; +use web_sys::MediaRecorder; use crate::css::styles; @@ -23,7 +25,15 @@ pub fn AudioStream() -> impl IntoView { .. } = use_user_media_with_options(options); - // start/stop recording + let on_data_available = Closure::wrap(Box::new(move |data: JsValue| { + if let Ok(blob) = data.dyn_into::() { + tracing::info!(" Data available on stream: {:?}", &blob); + } else { + tracing::info!(" bad data"); + } + }) as Box); + + // start/stop recording let _effect = Effect::watch( move || start_rec.get(), move |val, _prev, _| { @@ -35,16 +45,19 @@ pub fn AudioStream() -> impl IntoView { start(); } }, - true, /* Trigger it as soon as possible */ + false, /* Trigger it as soon as possible */ ); Effect::new(move |_| { tracing::debug!("State of the recording: {}.", start_rec.get_untracked()); node.get().map(|v| match stream.get() { Some(Ok(s)) => { - tracing::debug!("Setting stream {s:?} to src..."); + tracing::info!("Setting stream {s:?} to src..."); v.set_src_object(Some(&s)); - }, + let recorder = MediaRecorder::new_with_media_stream(&s).unwrap(); + recorder.set_ondataavailable(Some(on_data_available.as_ref().unchecked_ref())); + recorder.start_with_time_slice(1000).unwrap(); + } Some(Err(e)) => tracing::error!("Failed to get media stream: {e:?}"), None => tracing::debug!("No stream yet"), }); From a1a1dca6f8b44548db2fc843a2511da41b792dac Mon Sep 17 00:00:00 2001 From: Dilawar Singh Date: Mon, 24 Mar 2025 15:24:14 +0530 Subject: [PATCH 2/7] Show pcm data in console --- leptos/Cargo.toml | 2 ++ leptos/src/components/audio.rs | 42 ++++++++++++++++++++-------------- leptos/src/components/qr.rs | 4 ++-- 3 files changed, 29 insertions(+), 19 deletions(-) diff --git a/leptos/Cargo.toml b/leptos/Cargo.toml index c68beb4..d5abff2 100644 --- a/leptos/Cargo.toml +++ b/leptos/Cargo.toml @@ -6,6 +6,8 @@ edition = "2024" [dependencies] codee = { version = "0.3.0", features = ["json_serde"] } console_error_panic_hook = "0.1.7" +futures = "0.3.31" +gloo-file = { version = "0.3.0", features = ["futures"] } leptos = { version = "0.7.7", features = ["csr", "tracing"] } leptos-qr-scanner = { git = "https://github.com/dilawar/leptos-qr-scanner" } leptos-use = { version = "0.15.7", features = [ diff --git a/leptos/src/components/audio.rs b/leptos/src/components/audio.rs index f2db245..444f354 100644 --- a/leptos/src/components/audio.rs +++ b/leptos/src/components/audio.rs @@ -8,6 +8,10 @@ use web_sys::MediaRecorder; use crate::css::styles; +// Record audio +// +// Thanks you and chatgpt sucked +// lemons. #[component] pub fn AudioStream() -> impl IntoView { let node = NodeRef::::new(); @@ -27,12 +31,31 @@ pub fn AudioStream() -> impl IntoView { let on_data_available = Closure::wrap(Box::new(move |data: JsValue| { if let Ok(blob) = data.dyn_into::() { - tracing::info!(" Data available on stream: {:?}", &blob); + if let Some(data) = blob.data() { + let blob = gloo_file::Blob::from(data); + tracing::info!(" Data available on stream: {:?}", &blob); + let data = futures::executor::block_on(gloo_file::futures::read_as_bytes(&blob)).unwrap(); + tracing::info!("{} bytes", data.len()); + } } else { tracing::info!(" bad data"); } }) as Box); + Effect::new(move |_| { + node.get().map(|v| match stream.get() { + Some(Ok(s)) => { + tracing::info!("Setting stream {s:?} to src..."); + v.set_src_object(Some(&s)); + let recorder = MediaRecorder::new_with_media_stream(&s).unwrap(); + recorder.set_ondataavailable(Some(on_data_available.as_ref().unchecked_ref())); + recorder.start_with_time_slice(5000).unwrap(); + } + Some(Err(e)) => tracing::error!("Failed to get media stream: {e:?}"), + None => tracing::debug!("No stream yet"), + }); + }); + // start/stop recording let _effect = Effect::watch( move || start_rec.get(), @@ -48,26 +71,11 @@ pub fn AudioStream() -> impl IntoView { false, /* Trigger it as soon as possible */ ); - Effect::new(move |_| { - tracing::debug!("State of the recording: {}.", start_rec.get_untracked()); - node.get().map(|v| match stream.get() { - Some(Ok(s)) => { - tracing::info!("Setting stream {s:?} to src..."); - v.set_src_object(Some(&s)); - let recorder = MediaRecorder::new_with_media_stream(&s).unwrap(); - recorder.set_ondataavailable(Some(on_data_available.as_ref().unchecked_ref())); - recorder.start_with_time_slice(1000).unwrap(); - } - Some(Err(e)) => tracing::error!("Failed to get media stream: {e:?}"), - None => tracing::debug!("No stream yet"), - }); - }); - view! { // Eventually I was to draw something related to audio stream here. - } diff --git a/leptos/src/components/qr.rs b/leptos/src/components/qr.rs index 0dece52..a23176f 100644 --- a/leptos/src/components/qr.rs +++ b/leptos/src/components/qr.rs @@ -20,11 +20,11 @@ pub fn QrScanner() -> impl IntoView { active=scan on_scan=move |a| { tracing::info!("Found: {}", &a); - if ! multiple.get_untracked() { + if !multiple.get_untracked() { result.set(vec![a]); } else { let vals = result.read_untracked(); - if ! vals.contains(&a) { + if !vals.contains(&a) { result.write().push(a) } } From c858d2eaeb6e7194bd750c2078bb2be32710028c Mon Sep 17 00:00:00 2001 From: Dilawar Singh Date: Mon, 24 Mar 2025 21:48:00 +0530 Subject: [PATCH 3/7] Got the data. I am so cool! --- leptos/src/components/audio.rs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/leptos/src/components/audio.rs b/leptos/src/components/audio.rs index 444f354..0c3f47d 100644 --- a/leptos/src/components/audio.rs +++ b/leptos/src/components/audio.rs @@ -17,7 +17,7 @@ pub fn AudioStream() -> impl IntoView { let node = NodeRef::::new(); let start_rec = RwSignal::new(true); - let result = RwSignal::new_local("".to_string()); + let async_blob = RwSignal::new(None); // Create options to fetch only audio stream. let options = UseUserMediaOptions::default().video(false).audio(true); @@ -32,10 +32,11 @@ pub fn AudioStream() -> impl IntoView { let on_data_available = Closure::wrap(Box::new(move |data: JsValue| { if let Ok(blob) = data.dyn_into::() { if let Some(data) = blob.data() { + // convert to gloo_file Blog. let blob = gloo_file::Blob::from(data); - tracing::info!(" Data available on stream: {:?}", &blob); - let data = futures::executor::block_on(gloo_file::futures::read_as_bytes(&blob)).unwrap(); - tracing::info!("{} bytes", data.len()); + *async_blob.write() = Some(LocalResource::new(move || { + gloo_file::futures::read_as_bytes(&blob) + })); } } else { tracing::info!(" bad data"); @@ -71,6 +72,13 @@ pub fn AudioStream() -> impl IntoView { false, /* Trigger it as soon as possible */ ); + // Watch async_blob + Effect::new(move |_| { + if let Some(blob_resource) = async_blob.get() { + tracing::info!("Got blob of buffer {:?}", blob_resource.read().as_deref()); + } + }); + view! { // Eventually I was to draw something related to audio stream here. From 45a46a5bd2d45ba824e98084351b5459e8c38907 Mon Sep 17 00:00:00 2001 From: Dilawar Singh Date: Mon, 24 Mar 2025 21:59:52 +0530 Subject: [PATCH 4/7] Now plot it on canvas --- leptos/src/components/audio.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/leptos/src/components/audio.rs b/leptos/src/components/audio.rs index 0c3f47d..042851d 100644 --- a/leptos/src/components/audio.rs +++ b/leptos/src/components/audio.rs @@ -69,13 +69,15 @@ pub fn AudioStream() -> impl IntoView { start(); } }, - false, /* Trigger it as soon as possible */ + true, /* Trigger it as soon as possible */ ); // Watch async_blob Effect::new(move |_| { if let Some(blob_resource) = async_blob.get() { - tracing::info!("Got blob of buffer {:?}", blob_resource.read().as_deref()); + if let Some(Ok(data)) = blob_resource.read().as_deref() { + tracing::info!("Got blob of buffer {data:?}"); + } } }); From 1a29a45fceb6fbf419323648e1c3347a3eb0105d Mon Sep 17 00:00:00 2001 From: Dilawar Singh Date: Mon, 24 Mar 2025 22:31:40 +0530 Subject: [PATCH 5/7] Plot raw data --- leptos/Cargo.toml | 1 + leptos/app.css | 8 ++++++++ leptos/src/_app.module.scss | 8 ++++++++ leptos/src/components/audio.rs | 28 +++++++++++++++++++++++++--- 4 files changed, 42 insertions(+), 3 deletions(-) diff --git a/leptos/Cargo.toml b/leptos/Cargo.toml index d5abff2..d317212 100644 --- a/leptos/Cargo.toml +++ b/leptos/Cargo.toml @@ -26,6 +26,7 @@ tracing-subscriber.workspace = true tracing-subscriber-wasm = "0.1.0" web-sys = { version = "0.3.77", features = [ "BlobEvent", + "CanvasRenderingContext2d", "MediaRecorder", "AudioNode", ] } diff --git a/leptos/app.css b/leptos/app.css index fa98181..9d3b3dd 100644 --- a/leptos/app.css +++ b/leptos/app.css @@ -38,3 +38,11 @@ main { min-height: 400px; border: 1px dotted gray; } + +.canvas-372e0a1 { + border: 1px dotted blue; + border-radius: 10px; + width: 100%; + max-height: 100px; + background-color: ivory; +} diff --git a/leptos/src/_app.module.scss b/leptos/src/_app.module.scss index c5815ba..9a1ddc6 100644 --- a/leptos/src/_app.module.scss +++ b/leptos/src/_app.module.scss @@ -38,3 +38,11 @@ main { min-height: 400px; border: 1px dotted gray; } + +.canvas { + border: 1px dotted blue; + border-radius: 10px; + width: 100%; + max-height: 100px; + background-color: ivory; +} diff --git a/leptos/src/components/audio.rs b/leptos/src/components/audio.rs index 042851d..f41413e 100644 --- a/leptos/src/components/audio.rs +++ b/leptos/src/components/audio.rs @@ -15,8 +15,9 @@ use crate::css::styles; #[component] pub fn AudioStream() -> impl IntoView { let node = NodeRef::::new(); + let canvas_node = NodeRef::::new(); - let start_rec = RwSignal::new(true); + let start_rec = RwSignal::new(false); let async_blob = RwSignal::new(None); // Create options to fetch only audio stream. @@ -50,7 +51,7 @@ pub fn AudioStream() -> impl IntoView { v.set_src_object(Some(&s)); let recorder = MediaRecorder::new_with_media_stream(&s).unwrap(); recorder.set_ondataavailable(Some(on_data_available.as_ref().unchecked_ref())); - recorder.start_with_time_slice(5000).unwrap(); + recorder.start_with_time_slice(500).unwrap(); } Some(Err(e)) => tracing::error!("Failed to get media stream: {e:?}"), None => tracing::debug!("No stream yet"), @@ -74,9 +75,27 @@ pub fn AudioStream() -> impl IntoView { // Watch async_blob Effect::new(move |_| { + let ctx = canvas_node + .get() + .unwrap() + .get_context("2d") + .unwrap() + .unwrap() + .dyn_into::() + .unwrap(); + if let Some(blob_resource) = async_blob.get() { if let Some(Ok(data)) = blob_resource.read().as_deref() { tracing::info!("Got blob of buffer {data:?}"); + ctx.reset(); + ctx.begin_path(); + for (i, v) in data.iter().enumerate() { + ctx.line_to( + i as f64, + *v as f64 / 10.0 + 50.0, /* 50 is half of height of canvas */ + ) + } + ctx.stroke(); // render } } }); @@ -84,9 +103,12 @@ pub fn AudioStream() -> impl IntoView { view! { // Eventually I was to draw something related to audio stream here. - + } } From 2a6f18f8256f44fc03bba7d75579ad4803edb492 Mon Sep 17 00:00:00 2001 From: Dilawar Singh Date: Mon, 24 Mar 2025 22:34:23 +0530 Subject: [PATCH 6/7] Increase the amplitude --- leptos/src/components/audio.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/leptos/src/components/audio.rs b/leptos/src/components/audio.rs index f41413e..aad895c 100644 --- a/leptos/src/components/audio.rs +++ b/leptos/src/components/audio.rs @@ -92,7 +92,7 @@ pub fn AudioStream() -> impl IntoView { for (i, v) in data.iter().enumerate() { ctx.line_to( i as f64, - *v as f64 / 10.0 + 50.0, /* 50 is half of height of canvas */ + *v as f64 / 5.0 + 50.0, /* 50 is half of height of canvas */ ) } ctx.stroke(); // render From 55b627f8e7a824bee6a78edc2a4837543c1e0f49 Mon Sep 17 00:00:00 2001 From: Dilawar Singh Date: Mon, 24 Mar 2025 22:40:49 +0530 Subject: [PATCH 7/7] Adds screenshot --- leptos/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/leptos/README.md b/leptos/README.md index 2030146..f1f6b07 100644 --- a/leptos/README.md +++ b/leptos/README.md @@ -2,5 +2,7 @@ A simple SPA using leptos. +![](https://file.notion.so/f/f/5f1a3a94-a29f-407d-80af-617105cb793d/8938aed8-802c-4a23-9d47-10cd593ef1ca/image.png?table=block&id=1c0e579b-ff89-80df-abb6-c7766ae48094&spaceId=5f1a3a94-a29f-407d-80af-617105cb793d&expirationTimestamp=1742860800000&signature=FwK8fJ19MquzJKhRT0vRtm7JaOl7yjWOgEYxAz95gtE&downloadName=image.png) + - A QR Scanner - A Audio Recorder