Plot audio data on canvas (#5)

* Can record audio now

* Show pcm data in console

* Got the data.

I am so cool!

* Now plot it on canvas

* Plot raw data

* Increase the amplitude

* Adds screenshot

---------

Co-authored-by: Dilawar Singh <dilawar@dognosis.tech>

Author: dilawar

leptos/Cargo.toml

@@ -6,6 +6,8 @@ edition = "2024"
 [dependencies]
 codee = { version = "0.3.0", features = ["json_serde"] }
 console_error_panic_hook = "0.1.7"
+futures = "0.3.31"
+gloo-file = { version = "0.3.0", features = ["futures"] }
 leptos = { version = "0.7.7", features = ["csr", "tracing"] }
 leptos-qr-scanner = { git = "https://github.com/dilawar/leptos-qr-scanner" }
 leptos-use = { version = "0.15.7", features = [
@@ -22,3 +24,9 @@ thaw = { version = "0.4.4", features = ["csr"] }
 tracing.workspace = true
 tracing-subscriber.workspace = true
 tracing-subscriber-wasm = "0.1.0"
+web-sys = { version = "0.3.77", features = [
+  "BlobEvent",
+  "CanvasRenderingContext2d",
+  "MediaRecorder",
+  "AudioNode",
+] }

leptos/README.md

@@ -2,5 +2,7 @@
 
 A simple SPA using leptos. 
 
+![](https://file.notion.so/f/f/5f1a3a94-a29f-407d-80af-617105cb793d/8938aed8-802c-4a23-9d47-10cd593ef1ca/image.png?table=block&id=1c0e579b-ff89-80df-abb6-c7766ae48094&spaceId=5f1a3a94-a29f-407d-80af-617105cb793d&expirationTimestamp=1742860800000&signature=FwK8fJ19MquzJKhRT0vRtm7JaOl7yjWOgEYxAz95gtE&downloadName=image.png)
+
 - A QR Scanner
 - A Audio Recorder

leptos/app.css

@@ -38,3 +38,11 @@ main {
   min-height: 400px;
   border: 1px dotted gray;
 }
+
+.canvas-372e0a1 {
+  border: 1px dotted blue;
+  border-radius: 10px;
+  width: 100%;
+  max-height: 100px;
+  background-color: ivory;
+}

leptos/src/_app.module.scss

@@ -38,3 +38,11 @@ main {
   min-height: 400px;
   border: 1px dotted gray;
 }
+
+.canvas {
+  border: 1px dotted blue;
+  border-radius: 10px;
+  width: 100%;
+  max-height: 100px;
+  background-color: ivory;
+}

leptos/src/components/audio.rs

@@ -3,15 +3,22 @@
 use leptos::prelude::*;
 use leptos_use::{use_user_media_with_options, UseUserMediaOptions, UseUserMediaReturn};
 use thaw::*;
+use web_sys::wasm_bindgen::{closure::Closure, JsCast, JsValue};
+use web_sys::MediaRecorder;
 
 use crate::css::styles;
 
+// Record audio
+//
+// Thanks you <https://github.com/wayeast/mediarecorder/blob/master/src/lib.rs> and chatgpt sucked
+// lemons.
 #[component]
 pub fn AudioStream() -> impl IntoView {
     let node = NodeRef::<leptos::html::Audio>::new();
+    let canvas_node = NodeRef::<leptos::html::Canvas>::new();
 
-    let start_rec = RwSignal::new(true);
-    let result = RwSignal::new_local("".to_string());
+    let start_rec = RwSignal::new(false);
+    let async_blob = RwSignal::new(None);
 
     // Create options to fetch only audio stream.
     let options = UseUserMediaOptions::default().video(false).audio(true);
@@ -23,7 +30,35 @@ pub fn AudioStream() -> impl IntoView {
         ..
     } = use_user_media_with_options(options);
 
-    // start/stop recording 
+    let on_data_available = Closure::wrap(Box::new(move |data: JsValue| {
+        if let Ok(blob) = data.dyn_into::<web_sys::BlobEvent>() {
+            if let Some(data) = blob.data() {
+                // convert to gloo_file Blog.
+                let blob = gloo_file::Blob::from(data);
+                *async_blob.write() = Some(LocalResource::new(move || {
+                    gloo_file::futures::read_as_bytes(&blob)
+                }));
+            }
+        } else {
+            tracing::info!(" bad data");
+        }
+    }) as Box<dyn FnMut(JsValue)>);
+
+    Effect::new(move |_| {
+        node.get().map(|v| match stream.get() {
+            Some(Ok(s)) => {
+                tracing::info!("Setting stream {s:?} to src...");
+                v.set_src_object(Some(&s));
+                let recorder = MediaRecorder::new_with_media_stream(&s).unwrap();
+                recorder.set_ondataavailable(Some(on_data_available.as_ref().unchecked_ref()));
+                recorder.start_with_time_slice(500).unwrap();
+            }
+            Some(Err(e)) => tracing::error!("Failed to get media stream: {e:?}"),
+            None => tracing::debug!("No stream yet"),
+        });
+    });
+
+    // start/stop recording
     let _effect = Effect::watch(
         move || start_rec.get(),
         move |val, _prev, _| {
@@ -38,24 +73,42 @@ pub fn AudioStream() -> impl IntoView {
         true, /* Trigger it as soon as possible */
     );
 
+    // Watch async_blob
     Effect::new(move |_| {
-        tracing::debug!("State of the recording: {}.", start_rec.get_untracked());
-        node.get().map(|v| match stream.get() {
-            Some(Ok(s)) => {
-                tracing::debug!("Setting stream {s:?} to src...");
-                v.set_src_object(Some(&s));
-            },
-            Some(Err(e)) => tracing::error!("Failed to get media stream: {e:?}"),
-            None => tracing::debug!("No stream yet"),
-        });
+        let ctx = canvas_node
+            .get()
+            .unwrap()
+            .get_context("2d")
+            .unwrap()
+            .unwrap()
+            .dyn_into::<web_sys::CanvasRenderingContext2d>()
+            .unwrap();
+
+        if let Some(blob_resource) = async_blob.get() {
+            if let Some(Ok(data)) = blob_resource.read().as_deref() {
+                tracing::info!("Got blob of buffer {data:?}");
+                ctx.reset();
+                ctx.begin_path();
+                for (i, v) in data.iter().enumerate() {
+                    ctx.line_to(
+                        i as f64,
+                        *v as f64 / 5.0 + 50.0, /* 50 is half of height of canvas */
+                    )
+                }
+                ctx.stroke(); // render
+            }
+        }
     });
 
     view! {
         <Space vertical=true>
             // Eventually I was to draw something related to audio stream here.
-            <canvas />
-            <audio node_ref=node controls  />
+            <canvas node_ref=canvas_node class=styles::canvas  />
+            <audio node_ref=node controls />
             <Switch checked=start_rec label="Start Record" />
+            <div>
+                "Record and plot every half second of data"
+            </div>
         </Space>
     }
 }

leptos/src/components/qr.rs

@@ -20,11 +20,11 @@ pub fn QrScanner() -> impl IntoView {
             active=scan
             on_scan=move |a| {
                 tracing::info!("Found: {}", &a);
-                if ! multiple.get_untracked() {
+                if !multiple.get_untracked() {
                     result.set(vec![a]);
                 } else {
                     let vals = result.read_untracked();
-                    if ! vals.contains(&a) {
+                    if !vals.contains(&a) {
                         result.write().push(a)
                     }
                 }