From 18103ae14006e4559943ab36b97c99f05f318454 Mon Sep 17 00:00:00 2001 From: Kyujin Cho Date: Sun, 30 Jun 2024 17:50:59 +0900 Subject: [PATCH 01/13] implement automatic driver radio transcription --- dash/next.config.mjs | 10 + dash/package.json | 1 + dash/src/app/(nav)/settings/page.tsx | 60 ++- dash/src/asr-worker.js | 125 ++++++ dash/src/components/TeamRadioMessage.tsx | 28 +- dash/src/components/TeamRadios.tsx | 107 ++++- dash/src/context/ModeContext.tsx | 4 + dash/src/lib/constants.ts | 27 ++ dash/yarn.lock | 475 ++++++++++++++++++++++- live-backend/src/server.rs | 4 +- live-backend/src/server/audio.rs | 30 ++ 11 files changed, 863 insertions(+), 8 deletions(-) create mode 100644 dash/src/asr-worker.js create mode 100644 dash/src/lib/constants.ts create mode 100644 live-backend/src/server/audio.rs diff --git a/dash/next.config.mjs b/dash/next.config.mjs index e46afabe..3e91a718 100644 --- a/dash/next.config.mjs +++ b/dash/next.config.mjs @@ -7,5 +7,15 @@ await import("./src/env.mjs"); /** @type {import("next").NextConfig} */ const config = { reactStrictMode: false, + webpack: ( + config, + ) => { + config.resolve.alias = { + ...config.resolve.alias, + "sharp$": false, + "onnxruntime-node$": false, + } + return config + }, }; export default config; diff --git a/dash/package.json b/dash/package.json index 0947a840..a98e04a2 100644 --- a/dash/package.json +++ b/dash/package.json @@ -11,6 +11,7 @@ }, "dependencies": { "@headlessui/react": "2.0.3", + "@xenova/transformers": "2.7.0", "clsx": "2.1.1", "framer-motion": "11.2.12", "geist": "1.3.0", diff --git a/dash/src/app/(nav)/settings/page.tsx b/dash/src/app/(nav)/settings/page.tsx index d486d3d4..4965751f 100644 --- a/dash/src/app/(nav)/settings/page.tsx +++ b/dash/src/app/(nav)/settings/page.tsx @@ -3,7 +3,7 @@ import { useEffect, useState } from "react"; import { useRouter } from "next/navigation"; -import { modes, type UiElements } from "@/context/ModeContext"; +import { modes, type UiElements, type TranscriptionSettings } from "@/context/ModeContext"; import DelayInput from "@/components/DelayInput"; import Button from "@/components/Button"; @@ -20,6 +20,9 @@ export default function SettingsPage() { const [delay, setDelay] = useState(0); + const [enableTranscription, setEnableTranscription] = useState(false); + const [transcriptionModel, setTranscriptionModel] = useState(""); + useEffect(() => { if (typeof window != undefined) { const delayStorage = localStorage.getItem("delay"); @@ -33,6 +36,12 @@ export default function SettingsPage() { setTableHeaders(customSettings.tableHeaders); setSectorFastest(customSettings.sectorFastest); setCarMetrics(customSettings.carMetrics); + + const transcriptionStorage = localStorage.getItem("transcription"); + const transcriptionSettings: TranscriptionSettings = transcriptionStorage ? JSON.parse(transcriptionStorage) : { enableTranscription: false, whisperModel: "" }; + + setEnableTranscription(transcriptionSettings.enableTranscription); + setTranscriptionModel(transcriptionSettings.whisperModel); } }, []); @@ -60,6 +69,26 @@ export default function SettingsPage() { } }; + const handleTranscriptionSettingUpdate = (type: "transcription" | "model", newValue: string | boolean) => { + if (typeof window != undefined) { + const transcriptionStorage = localStorage.getItem("transcription"); + const transcriptionSettings: TranscriptionSettings = transcriptionStorage ? JSON.parse(transcriptionStorage) : modes.custom; + + switch (type) { + case "transcription": { + transcriptionSettings.enableTranscription = newValue as boolean; + break; + } + case "model": { + transcriptionSettings.whisperModel = newValue as string; + break; + } + } + + localStorage.setItem("transcription", JSON.stringify(transcriptionSettings)); + } + } + const updateDelay = (newDelay: number) => { setDelay(newDelay); @@ -137,6 +166,35 @@ export default function SettingsPage() { Reset delay +

Enable Radio Transcription

+ +

+ Only available when the corresponding feature is enabled from server. +

+ +
+ { + setEnableTranscription(v); + handleTranscriptionSettingUpdate("transcription", v); + }} + /> +

Enable Radio Transcription

+
+ +
+ +

Transcription Mode

+
+ {/*

Walkthrough

diff --git a/dash/src/asr-worker.js b/dash/src/asr-worker.js new file mode 100644 index 00000000..f6f4b0f8 --- /dev/null +++ b/dash/src/asr-worker.js @@ -0,0 +1,125 @@ +/* eslint-disable camelcase */ +// from https://github.com/xenova/whisper-web/blob/main/src/worker.js +import { pipeline, env } from "@xenova/transformers"; + +// Disable local models +env.allowLocalModels = false; + +// Define model factories +// Ensures only one model is created of each type +class PipelineFactory { + static task = null; + static model = null; + static quantized = null; + static instance = null; + + constructor(model, quantized) { + this.model = model; + this.quantized = quantized; + } + + static async getInstance(progress_callback = null) { + if (this.instance === null) { + this.instance = pipeline(this.task, this.model, { + quantized: this.quantized, + progress_callback, + // For medium models, we need to load the `no_attentions` revision to avoid running out of memory + revision: this.model.includes("/whisper-medium") ? "no_attentions" : "main" + }); + } + + return this.instance; + } +} + +self.addEventListener("message", async (event) => { + const message = event.data; + + // Do some work... + // TODO use message data + let transcript + try { + transcript = await transcribe( + message.audio, + message.model, + message.multilingual, + message.quantized, + message.subtask, + message.language, + ); + } catch (e) { + console.warn("Error while transcribing: " + e); + transcript = { + text: "", + chunks: [], + } + } + + // Send the result back to the main thread + self.postMessage({ + status: "complete", + task: "automatic-speech-recognition", + key: message.key, + data: transcript, + }); +}); + +class AutomaticSpeechRecognitionPipelineFactory extends PipelineFactory { + static task = "automatic-speech-recognition"; + static model = null; + static quantized = null; +} + +const transcribe = async ( + audio, + model, + multilingual, + quantized, + subtask, + language, +) => { + const isDistilWhisper = model.startsWith("distil-whisper/"); + + let modelName = model; + if (!isDistilWhisper && !multilingual) { + modelName += ".en" + } + + const p = AutomaticSpeechRecognitionPipelineFactory; + if (p.model !== modelName || p.quantized !== quantized) { + // Invalidate model if different + p.model = modelName; + p.quantized = quantized; + + if (p.instance !== null) { + (await p.getInstance()).dispose(); + p.instance = null; + } + } + + // Load transcriber model + let transcriber = await p.getInstance((data) => { + self.postMessage(data); + }); + + // Actually run transcription + let output = await transcriber(audio, { + // Greedy + top_k: 0, + do_sample: false, + + // Sliding window + chunk_length_s: isDistilWhisper ? 20 : 30, + stride_length_s: isDistilWhisper ? 3 : 5, + + // Language and task + language: language, + task: subtask, + + // Return timestamps + return_timestamps: true, + force_full_sequences: false, + }); + + return output; +}; \ No newline at end of file diff --git a/dash/src/components/TeamRadioMessage.tsx b/dash/src/components/TeamRadioMessage.tsx index dc543f1f..9f2dcbfd 100644 --- a/dash/src/components/TeamRadioMessage.tsx +++ b/dash/src/components/TeamRadioMessage.tsx @@ -1,6 +1,7 @@ -import { useRef, useState } from "react"; +import { useMemo, useRef, useState } from "react"; import { motion } from "framer-motion"; import { utc } from "moment"; +import clsx from "clsx"; import DriverTag from "./driver/DriverTag"; import PlayControls from "./PlayControls"; @@ -12,9 +13,10 @@ type Props = { driver: Driver; capture: RadioCapture; basePath: string; + transcription?: string; }; -export default function TeamRadioMessage({ driver, capture, basePath }: Props) { +export default function TeamRadioMessage({ driver, capture, basePath, transcription }: Props) { const audioRef = useRef(null); const intervalRef = useRef(null); @@ -22,6 +24,19 @@ export default function TeamRadioMessage({ driver, capture, basePath }: Props) { const [duration, setDuration] = useState(10); const [progress, setProgress] = useState(0); + const transcriptionElement = useMemo(() => { + if (transcription === undefined) { + return <>; + } + else if (transcription === "") { + return ; + } else { + return ( +

{transcription}

+ ); + } + }, [transcription]); + const loadMeta = () => { if (!audioRef.current) return; setDuration(audioRef.current.duration); @@ -97,6 +112,15 @@ export default function TeamRadioMessage({ driver, capture, basePath }: Props) { /> +
+ {transcriptionElement} +
); } + +const SkeletonTranscription = () => { + const animateClass = "h-6 animate-pulse rounded-md bg-zinc-800"; + + return
; +} \ No newline at end of file diff --git a/dash/src/components/TeamRadios.tsx b/dash/src/components/TeamRadios.tsx index 4e4e05d5..c6dfb010 100644 --- a/dash/src/components/TeamRadios.tsx +++ b/dash/src/components/TeamRadios.tsx @@ -1,12 +1,16 @@ +import { useEffect, useRef, useState } from "react"; import { AnimatePresence } from "framer-motion"; import { utc } from "moment"; import clsx from "clsx"; import { sortUtc } from "@/lib/sorting/sortUtc"; -import { DriverList, TeamRadio } from "@/types/state.type"; +import { DriverList, RadioCapture, TeamRadio } from "@/types/state.type"; import TeamRadioMessage from "@/components/TeamRadioMessage"; +import { TranscriptionSettings } from "@/context/ModeContext"; +import Constants from '@/lib/constants'; +import { env } from "@/env.mjs"; type Props = { sessionPath: string | undefined; @@ -14,10 +18,110 @@ type Props = { teamRadios: TeamRadio | undefined; }; +type TranscriberCompleteData = { + key: string; + data: { + text: string; + chunks: { text: string; timestamp: [number, number | null] }[]; + }; +} + +const loadAudioFromRadioCapture = async (audioContext: AudioContext, path: string, retries = 0): Promise => { + const queryString = new URLSearchParams({path}).toString(); + try { + const response = await fetch(`${env.NEXT_PUBLIC_LIVE_SOCKET_URL}/api/audio?${queryString}`); + if (response.status === 429) throw new Error("met rate limiter") ; + const abuf = await response.arrayBuffer(); + const audioData = await audioContext.decodeAudioData(abuf); + return audioData.getChannelData(0); + } catch (e) { + console.warn(e); + await new Promise((res) => setTimeout(res, 500 * Math.pow(2, (retries + 1)))); // to deal with possible rate limit violation + return await loadAudioFromRadioCapture(audioContext, path, retries + 1); + } +} + export default function TeamRadios({ sessionPath, drivers, teamRadios }: Props) { const basePath = `https://livetiming.formula1.com/static/${sessionPath}`; // TODO add notice that we only show 20 + const workerRef = useRef(null); + + const [enableTranscription, setEnableTranscription] = useState(false); + const [transcriptions, setTranscriptions] = useState<{[key: string]: string}>({}); + const [transcriptionModel, setTranscriptionModel] = useState(""); + + const workerEventHandler = (event: MessageEvent) => { + const message = event.data; + // Update the state with the result + switch (message.status) { + case "complete": + const completeMessage = message as TranscriberCompleteData; + const path = message.key; + if (path) { + setTranscriptions((oldTranscription) => ( + {...oldTranscription, [path]: completeMessage.data.chunks.map((c) => c.text).join("\n")} + )) + } + console.log("Transcription completed: " + path); + break; + } + }; + + const beginTranscripting = async (teamRadios: RadioCapture[]) => { + const audioContext = new (window.AudioContext || window.webkitAudioContext)({ + sampleRate: Constants.SAMPLING_RATE, + }); + + for (const teamRadio of teamRadios) { + const audio = await loadAudioFromRadioCapture(audioContext, `${sessionPath}${teamRadio.path}`); + console.log("Requesting worker to transcript " + `${sessionPath}${teamRadio.path}`); + workerRef.current?.postMessage({ + key: teamRadio.path, + audio, + model: transcriptionModel, + multilingual: false, + quantized: Constants.DEFAULT_QUANTIZED, + subtask: null, + language: null, + }); + await new Promise((res) => setTimeout(res, 1000)); // To avoid rate limit + } + }; + + useEffect(() => { + if (typeof window != undefined) { + const worker = new Worker(new URL("../asr-worker.js", import.meta.url), { + type: "module", + }); + // Listen for messages from the Web Worker + worker.addEventListener("message", workerEventHandler); + workerRef.current = worker; + + const transcriptionStorage = localStorage.getItem("transcription"); + const transcriptionSettings: TranscriptionSettings = transcriptionStorage ? JSON.parse(transcriptionStorage) : { enableTranscription: false, whisperModel: "" }; + + setEnableTranscription(transcriptionSettings.enableTranscription); + setTranscriptionModel(transcriptionSettings.whisperModel); + } + }, []); + + useEffect(() => { + if (teamRadios && drivers && teamRadios.captures && enableTranscription && transcriptionModel) { + const targetRadios = teamRadios.captures + .sort(sortUtc) + .slice(0, 20); + + setTranscriptions((oldTranscriptions) => { + const newRadios = targetRadios.filter((c) => oldTranscriptions[c.path] === undefined); + const newTranscriptions = {...oldTranscriptions}; + newRadios.forEach((c) => newTranscriptions[c.path] = ""); + + beginTranscripting(newRadios); + return newTranscriptions; + }); + } + }, [teamRadios, drivers, enableTranscription, transcriptionModel]); return (
    @@ -34,6 +138,7 @@ export default function TeamRadios({ sessionPath, drivers, teamRadios }: Props) driver={drivers[teamRadio.racingNumber]} capture={teamRadio} basePath={basePath} + transcription={transcriptions[teamRadio.path]} /> ))} diff --git a/dash/src/context/ModeContext.tsx b/dash/src/context/ModeContext.tsx index 6bd3f988..ca7645fa 100644 --- a/dash/src/context/ModeContext.tsx +++ b/dash/src/context/ModeContext.tsx @@ -15,6 +15,10 @@ export type UiElements = { sectorFastest: boolean; carMetrics: boolean; }; +export type TranscriptionSettings = { + enableTranscription: boolean; + whisperModel: string; +} type Mode = "simple" | "advanced" | "expert" | "custom"; diff --git a/dash/src/lib/constants.ts b/dash/src/lib/constants.ts new file mode 100644 index 00000000..f1bd9f81 --- /dev/null +++ b/dash/src/lib/constants.ts @@ -0,0 +1,27 @@ +function mobileTabletCheck() { + // https://stackoverflow.com/questions/11381673/detecting-a-mobile-browser + let check = false; + (function (a: string) { + if ( + /(android|bb\d+|meego).+mobile|avantgo|bada\/|blackberry|blazer|compal|elaine|fennec|hiptop|iemobile|ip(hone|od)|iris|kindle|lge |maemo|midp|mmp|mobile.+firefox|netfront|opera m(ob|in)i|palm( os)?|phone|p(ixi|re)\/|plucker|pocket|psp|series(4|6)0|symbian|treo|up\.(browser|link)|vodafone|wap|windows ce|xda|xiino|android|ipad|playbook|silk/i.test( + a, + ) || + /1207|6310|6590|3gso|4thp|50[1-6]i|770s|802s|a wa|abac|ac(er|oo|s\-)|ai(ko|rn)|al(av|ca|co)|amoi|an(ex|ny|yw)|aptu|ar(ch|go)|as(te|us)|attw|au(di|\-m|r |s )|avan|be(ck|ll|nq)|bi(lb|rd)|bl(ac|az)|br(e|v)w|bumb|bw\-(n|u)|c55\/|capi|ccwa|cdm\-|cell|chtm|cldc|cmd\-|co(mp|nd)|craw|da(it|ll|ng)|dbte|dc\-s|devi|dica|dmob|do(c|p)o|ds(12|\-d)|el(49|ai)|em(l2|ul)|er(ic|k0)|esl8|ez([4-7]0|os|wa|ze)|fetc|fly(\-|_)|g1 u|g560|gene|gf\-5|g\-mo|go(\.w|od)|gr(ad|un)|haie|hcit|hd\-(m|p|t)|hei\-|hi(pt|ta)|hp( i|ip)|hs\-c|ht(c(\-| |_|a|g|p|s|t)|tp)|hu(aw|tc)|i\-(20|go|ma)|i230|iac( |\-|\/)|ibro|idea|ig01|ikom|im1k|inno|ipaq|iris|ja(t|v)a|jbro|jemu|jigs|kddi|keji|kgt( |\/)|klon|kpt |kwc\-|kyo(c|k)|le(no|xi)|lg( g|\/(k|l|u)|50|54|\-[a-w])|libw|lynx|m1\-w|m3ga|m50\/|ma(te|ui|xo)|mc(01|21|ca)|m\-cr|me(rc|ri)|mi(o8|oa|ts)|mmef|mo(01|02|bi|de|do|t(\-| |o|v)|zz)|mt(50|p1|v )|mwbp|mywa|n10[0-2]|n20[2-3]|n30(0|2)|n50(0|2|5)|n7(0(0|1)|10)|ne((c|m)\-|on|tf|wf|wg|wt)|nok(6|i)|nzph|o2im|op(ti|wv)|oran|owg1|p800|pan(a|d|t)|pdxg|pg(13|\-([1-8]|c))|phil|pire|pl(ay|uc)|pn\-2|po(ck|rt|se)|prox|psio|pt\-g|qa\-a|qc(07|12|21|32|60|\-[2-7]|i\-)|qtek|r380|r600|raks|rim9|ro(ve|zo)|s55\/|sa(ge|ma|mm|ms|ny|va)|sc(01|h\-|oo|p\-)|sdk\/|se(c(\-|0|1)|47|mc|nd|ri)|sgh\-|shar|sie(\-|m)|sk\-0|sl(45|id)|sm(al|ar|b3|it|t5)|so(ft|ny)|sp(01|h\-|v\-|v )|sy(01|mb)|t2(18|50)|t6(00|10|18)|ta(gt|lk)|tcl\-|tdg\-|tel(i|m)|tim\-|t\-mo|to(pl|sh)|ts(70|m\-|m3|m5)|tx\-9|up(\.b|g1|si)|utst|v400|v750|veri|vi(rg|te)|vk(40|5[0-3]|\-v)|vm40|voda|vulc|vx(52|53|60|61|70|80|81|83|85|98)|w3c(\-| )|webc|whit|wi(g |nc|nw)|wmlb|wonu|x700|yas\-|your|zeto|zte\-/i.test( + a.substr(0, 4), + ) + ) + check = true; + })( + navigator.userAgent || + navigator.vendor || + ("opera" in window && typeof window.opera === "string" + ? window.opera + : ""), + ); + return check; +} +const isMobileOrTablet = mobileTabletCheck(); +export default { + SAMPLING_RATE: 16000, + DEFAULT_QUANTIZED: isMobileOrTablet, +}; \ No newline at end of file diff --git a/dash/yarn.lock b/dash/yarn.lock index 969184f7..92905240 100644 --- a/dash/yarn.lock +++ b/dash/yarn.lock @@ -293,6 +293,59 @@ resolved "https://registry.yarnpkg.com/@pkgjs/parseargs/-/parseargs-0.11.0.tgz#a77ea742fab25775145434eb1d2328cf5013ac33" integrity sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg== +"@protobufjs/aspromise@^1.1.1", "@protobufjs/aspromise@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@protobufjs/aspromise/-/aspromise-1.1.2.tgz#9b8b0cc663d669a7d8f6f5d0893a14d348f30fbf" + integrity sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ== + +"@protobufjs/base64@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@protobufjs/base64/-/base64-1.1.2.tgz#4c85730e59b9a1f1f349047dbf24296034bb2735" + integrity sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg== + +"@protobufjs/codegen@^2.0.4": + version "2.0.4" + resolved "https://registry.yarnpkg.com/@protobufjs/codegen/-/codegen-2.0.4.tgz#7ef37f0d010fb028ad1ad59722e506d9262815cb" + integrity sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg== + +"@protobufjs/eventemitter@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz#355cbc98bafad5978f9ed095f397621f1d066b70" + integrity sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q== + +"@protobufjs/fetch@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/fetch/-/fetch-1.1.0.tgz#ba99fb598614af65700c1619ff06d454b0d84c45" + integrity sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ== + dependencies: + "@protobufjs/aspromise" "^1.1.1" + "@protobufjs/inquire" "^1.1.0" + +"@protobufjs/float@^1.0.2": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@protobufjs/float/-/float-1.0.2.tgz#5e9e1abdcb73fc0a7cb8b291df78c8cbd97b87d1" + integrity sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ== + +"@protobufjs/inquire@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/inquire/-/inquire-1.1.0.tgz#ff200e3e7cf2429e2dcafc1140828e8cc638f089" + integrity sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q== + +"@protobufjs/path@^1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@protobufjs/path/-/path-1.1.2.tgz#6cc2b20c5c9ad6ad0dccfd21ca7673d8d7fbf68d" + integrity sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA== + +"@protobufjs/pool@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/pool/-/pool-1.1.0.tgz#09fd15f2d6d3abfa9b65bc366506d6ad7846ff54" + integrity sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw== + +"@protobufjs/utf8@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@protobufjs/utf8/-/utf8-1.1.0.tgz#a777360b5b39a1a2e5106f8e858f2fd2d060c570" + integrity sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw== + "@react-aria/focus@^3.16.2": version "3.17.1" resolved "https://registry.yarnpkg.com/@react-aria/focus/-/focus-3.17.1.tgz#c796a188120421e2fedf438cadacdf463c77ad29" @@ -381,6 +434,11 @@ resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.17.6.tgz#193ced6a40c8006cfc1ca3f4553444fb38f0e543" integrity sha512-OpXEVoCKSS3lQqjx9GGGOapBeuW5eUboYHRlHP9urXPX25IKZ6AnP5ZRxtVf63iieUbsHxLn8NQ5Nlftc6yzAA== +"@types/long@^4.0.1": + version "4.0.2" + resolved "https://registry.yarnpkg.com/@types/long/-/long-4.0.2.tgz#b74129719fc8d11c01868010082d483b7545591a" + integrity sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA== + "@types/node@20.11.10": version "20.11.10" resolved "https://registry.yarnpkg.com/@types/node/-/node-20.11.10.tgz#6c3de8974d65c362f82ee29db6b5adf4205462f9" @@ -388,6 +446,13 @@ dependencies: undici-types "~5.26.4" +"@types/node@>=13.7.0": + version "20.14.9" + resolved "https://registry.yarnpkg.com/@types/node/-/node-20.14.9.tgz#12e8e765ab27f8c421a1820c99f5f313a933b420" + integrity sha512-06OCtnTXtWOZBJlRApleWndH4JsRVs1pDCc8dLSQp+7PpUpX3ePdHyeNSFTeSe7FtKyQkrlPvHwJOW3SLd8Oyg== + dependencies: + undici-types "~5.26.4" + "@types/pako@2.0.3": version "2.0.3" resolved "https://registry.yarnpkg.com/@types/pako/-/pako-2.0.3.tgz#b6993334f3af27c158f3fe0dfeeba987c578afb1" @@ -413,6 +478,16 @@ "@types/prop-types" "*" csstype "^3.0.2" +"@xenova/transformers@2.7.0": + version "2.7.0" + resolved "https://registry.yarnpkg.com/@xenova/transformers/-/transformers-2.7.0.tgz#0aabc8700d32ed8e28f6aa61abf8653f62fb1678" + integrity sha512-py5RqZt9lL/FFUT5X6St+TOSBoVaEmDETI98lK9ApEOvlWeX4bTS2nMQDFe3nFMpv24+wllhmPw2Www/f/ubJA== + dependencies: + onnxruntime-web "1.14.0" + sharp "^0.32.0" + optionalDependencies: + onnxruntime-node "1.14.0" + ansi-regex@^5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304" @@ -465,16 +540,68 @@ autoprefixer@10.4.19: picocolors "^1.0.0" postcss-value-parser "^4.2.0" +b4a@^1.6.4: + version "1.6.6" + resolved "https://registry.yarnpkg.com/b4a/-/b4a-1.6.6.tgz#a4cc349a3851987c3c4ac2d7785c18744f6da9ba" + integrity sha512-5Tk1HLk6b6ctmjIkAcU/Ujv/1WqiDl0F0JdRCR80VsOcUlHcu7pWeWRlOqQLHfDEsVx9YH/aif5AG4ehoCtTmg== + balanced-match@^1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee" integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw== +bare-events@^2.0.0, bare-events@^2.2.0: + version "2.4.2" + resolved "https://registry.yarnpkg.com/bare-events/-/bare-events-2.4.2.tgz#3140cca7a0e11d49b3edc5041ab560659fd8e1f8" + integrity sha512-qMKFd2qG/36aA4GwvKq8MxnPgCQAmBWmSyLWsJcbn8v03wvIPQ/hG1Ms8bPzndZxMDoHpxez5VOS+gC9Yi24/Q== + +bare-fs@^2.1.1: + version "2.3.1" + resolved "https://registry.yarnpkg.com/bare-fs/-/bare-fs-2.3.1.tgz#cdbd63dac7a552dfb2b87d18c822298d1efd213d" + integrity sha512-W/Hfxc/6VehXlsgFtbB5B4xFcsCl+pAh30cYhoFyXErf6oGrwjh8SwiPAdHgpmWonKuYpZgGywN0SXt7dgsADA== + dependencies: + bare-events "^2.0.0" + bare-path "^2.0.0" + bare-stream "^2.0.0" + +bare-os@^2.1.0: + version "2.4.0" + resolved "https://registry.yarnpkg.com/bare-os/-/bare-os-2.4.0.tgz#5de5e3ba7704f459c9656629edca7cc736e06608" + integrity sha512-v8DTT08AS/G0F9xrhyLtepoo9EJBJ85FRSMbu1pQUlAf6A8T0tEEQGMVObWeqpjhSPXsE0VGlluFBJu2fdoTNg== + +bare-path@^2.0.0, bare-path@^2.1.0: + version "2.1.3" + resolved "https://registry.yarnpkg.com/bare-path/-/bare-path-2.1.3.tgz#594104c829ef660e43b5589ec8daef7df6cedb3e" + integrity sha512-lh/eITfU8hrj9Ru5quUp0Io1kJWIk1bTjzo7JH1P5dWmQ2EL4hFUlfI8FonAhSlgIfhn63p84CDY/x+PisgcXA== + dependencies: + bare-os "^2.1.0" + +bare-stream@^2.0.0: + version "2.1.3" + resolved "https://registry.yarnpkg.com/bare-stream/-/bare-stream-2.1.3.tgz#070b69919963a437cc9e20554ede079ce0a129b2" + integrity sha512-tiDAH9H/kP+tvNO5sczyn9ZAA7utrSMobyDchsnyyXBuUe2FSQWbxhtuHB8jwpHYYevVo2UJpcmvvjrbHboUUQ== + dependencies: + streamx "^2.18.0" + +base64-js@^1.3.1: + version "1.5.1" + resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" + integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== + binary-extensions@^2.0.0: version "2.3.0" resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.3.0.tgz#f6e14a97858d327252200242d4ccfe522c445522" integrity sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw== +bl@^4.0.3: + version "4.1.0" + resolved "https://registry.yarnpkg.com/bl/-/bl-4.1.0.tgz#451535264182bec2fbbc83a62ab98cf11d9f7b3a" + integrity sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w== + dependencies: + buffer "^5.5.0" + inherits "^2.0.4" + readable-stream "^3.4.0" + brace-expansion@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-2.0.1.tgz#1edc459e0f0c548486ecf9fc99f2221364b9a0ae" @@ -499,6 +626,14 @@ browserslist@^4.23.0: node-releases "^2.0.14" update-browserslist-db "^1.0.16" +buffer@^5.5.0: + version "5.7.1" + resolved "https://registry.yarnpkg.com/buffer/-/buffer-5.7.1.tgz#ba62e7c13133053582197160851a8f648e99eed0" + integrity sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ== + dependencies: + base64-js "^1.3.1" + ieee754 "^1.1.13" + busboy@1.6.0: version "1.6.0" resolved "https://registry.yarnpkg.com/busboy/-/busboy-1.6.0.tgz#966ea36a9502e43cdb9146962523b92f531f6893" @@ -531,6 +666,11 @@ chokidar@^3.5.3: optionalDependencies: fsevents "~2.3.2" +chownr@^1.1.1: + version "1.1.4" + resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.4.tgz#6fc9d7b42d32a583596337666e7d08084da2cc6b" + integrity sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg== + client-only@0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/client-only/-/client-only-0.0.1.tgz#38bba5d403c41ab150bff64a95c85013cf73bca1" @@ -593,7 +733,19 @@ csstype@^3.0.2: resolved "https://registry.yarnpkg.com/csstype/-/csstype-3.1.3.tgz#d80ff294d114fb0e6ac500fbf85b60137d7eff81" integrity sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw== -detect-libc@^2.0.3: +decompress-response@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/decompress-response/-/decompress-response-6.0.0.tgz#ca387612ddb7e104bd16d85aab00d5ecf09c66fc" + integrity sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ== + dependencies: + mimic-response "^3.1.0" + +deep-extend@^0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/deep-extend/-/deep-extend-0.6.0.tgz#c4fa7c95404a17a9c3e8ca7e1537312b736330ac" + integrity sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA== + +detect-libc@^2.0.0, detect-libc@^2.0.2, detect-libc@^2.0.3: version "2.0.3" resolved "https://registry.yarnpkg.com/detect-libc/-/detect-libc-2.0.3.tgz#f0cd503b40f9939b894697d19ad50895e30cf700" integrity sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw== @@ -628,11 +780,28 @@ emoji-regex@^9.2.2: resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-9.2.2.tgz#840c8803b0d8047f4ff0cf963176b32d4ef3ed72" integrity sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg== +end-of-stream@^1.1.0, end-of-stream@^1.4.1: + version "1.4.4" + resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0" + integrity sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q== + dependencies: + once "^1.4.0" + escalade@^3.1.2: version "3.1.2" resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.2.tgz#54076e9ab29ea5bf3d8f1ed62acffbb88272df27" integrity sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA== +expand-template@^2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/expand-template/-/expand-template-2.0.3.tgz#6e14b3fcee0f3a6340ecb57d2e8918692052a47c" + integrity sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg== + +fast-fifo@^1.2.0, fast-fifo@^1.3.2: + version "1.3.2" + resolved "https://registry.yarnpkg.com/fast-fifo/-/fast-fifo-1.3.2.tgz#286e31de96eb96d38a97899815740ba2a4f3640c" + integrity sha512-/d9sfos4yxzpwkDkuN7k2SqFKtYNmCTzgfEpz82x34IM9/zc8KGxQoXg1liNC/izpRM/MBdt44Nmx41ZWqk+FQ== + fast-glob@^3.3.0: version "3.3.2" resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.3.2.tgz#a904501e57cfdd2ffcded45e99a54fef55e46129" @@ -658,6 +827,11 @@ fill-range@^7.1.1: dependencies: to-regex-range "^5.0.1" +flatbuffers@^1.12.0: + version "1.12.0" + resolved "https://registry.yarnpkg.com/flatbuffers/-/flatbuffers-1.12.0.tgz#72e87d1726cb1b216e839ef02658aa87dcef68aa" + integrity sha512-c7CZADjRcl6j0PlvFy0ZqXQ67qSEZfrVPynmnL+2zPc+NtMvrF8Y0QceMo7QqnSPc7+uWjUIAbvCQ5WIKlMVdQ== + foreground-child@^3.1.0: version "3.2.1" resolved "https://registry.yarnpkg.com/foreground-child/-/foreground-child-3.2.1.tgz#767004ccf3a5b30df39bed90718bab43fe0a59f7" @@ -678,6 +852,11 @@ framer-motion@11.2.12: dependencies: tslib "^2.4.0" +fs-constants@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad" + integrity sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow== + fsevents@~2.3.2: version "2.3.3" resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.3.tgz#cac6407785d03675a2a5e1a5305c697b347d90d6" @@ -693,6 +872,11 @@ geist@1.3.0: resolved "https://registry.yarnpkg.com/geist/-/geist-1.3.0.tgz#e22a87478d64ab452c03e4766f8929d76058f93f" integrity sha512-IoGBfcqVEYB4bEwsfHd35jF4+X9LHRPYZymHL4YOltHSs9LJa24DYs1Z7rEMQ/lsEvaAIc61Y9aUxgcJaQ8lrg== +github-from-package@0.0.0: + version "0.0.0" + resolved "https://registry.yarnpkg.com/github-from-package/-/github-from-package-0.0.0.tgz#97fb5d96bfde8973313f20e8288ef9a167fa64ce" + integrity sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw== + glob-parent@^5.1.2, glob-parent@~5.1.2: version "5.1.2" resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.2.tgz#869832c58034fe68a4093c17dc15e8340d8401c4" @@ -724,6 +908,11 @@ graceful-fs@^4.2.11: resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.11.tgz#4183e4e8bf08bb6e05bbb2f7d2e0c8f712ca40e3" integrity sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ== +guid-typescript@^1.0.9: + version "1.0.9" + resolved "https://registry.yarnpkg.com/guid-typescript/-/guid-typescript-1.0.9.tgz#e35f77003535b0297ea08548f5ace6adb1480ddc" + integrity sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ== + hasown@^2.0.2: version "2.0.2" resolved "https://registry.yarnpkg.com/hasown/-/hasown-2.0.2.tgz#003eaf91be7adc372e84ec59dc37252cedb80003" @@ -731,6 +920,21 @@ hasown@^2.0.2: dependencies: function-bind "^1.1.2" +ieee754@^1.1.13: + version "1.2.1" + resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352" + integrity sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA== + +inherits@^2.0.3, inherits@^2.0.4: + version "2.0.4" + resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" + integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== + +ini@~1.3.0: + version "1.3.8" + resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.8.tgz#a29da425b48806f34767a4efce397269af28432c" + integrity sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew== + is-arrayish@^0.3.1: version "0.3.2" resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.3.2.tgz#4574a2ae56f7ab206896fb431eaeed066fdf8f03" @@ -811,6 +1015,11 @@ lines-and-columns@^1.1.6: resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.2.4.tgz#eca284f75d2965079309dc0ad9255abb2ebc1632" integrity sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg== +long@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/long/-/long-4.0.0.tgz#9a7b71cfb7d361a194ea555241c92f7468d5bf28" + integrity sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA== + loose-envify@^1.1.0: version "1.4.0" resolved "https://registry.yarnpkg.com/loose-envify/-/loose-envify-1.4.0.tgz#71ee51fa7be4caec1a63839f7e682d8132d30caf" @@ -836,6 +1045,11 @@ micromatch@^4.0.4, micromatch@^4.0.5: braces "^3.0.3" picomatch "^2.3.1" +mimic-response@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-3.1.0.tgz#2d1d59af9c1b129815accc2c46a022a5ce1fa3c9" + integrity sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ== + minimatch@^9.0.4: version "9.0.5" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-9.0.5.tgz#d74f9dd6b57d83d8e98cfb82133b03978bc929e5" @@ -843,11 +1057,21 @@ minimatch@^9.0.4: dependencies: brace-expansion "^2.0.1" +minimist@^1.2.0, minimist@^1.2.3: + version "1.2.8" + resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c" + integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA== + "minipass@^5.0.0 || ^6.0.2 || ^7.0.0", minipass@^7.1.2: version "7.1.2" resolved "https://registry.yarnpkg.com/minipass/-/minipass-7.1.2.tgz#93a9626ce5e5e66bd4db86849e7515e92340a707" integrity sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw== +mkdirp-classic@^0.5.2, mkdirp-classic@^0.5.3: + version "0.5.3" + resolved "https://registry.yarnpkg.com/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz#fa10c9115cc6d8865be221ba47ee9bed78601113" + integrity sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A== + moment@2.30.1: version "2.30.1" resolved "https://registry.yarnpkg.com/moment/-/moment-2.30.1.tgz#f8c91c07b7a786e30c59926df530b4eac96974ae" @@ -867,6 +1091,11 @@ nanoid@^3.3.6, nanoid@^3.3.7: resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.7.tgz#d0c301a691bc8d54efa0a2226ccf3fe2fd656bd8" integrity sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g== +napi-build-utils@^1.0.1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/napi-build-utils/-/napi-build-utils-1.0.2.tgz#b1fddc0b2c46e380a0b7a76f984dd47c41a13806" + integrity sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg== + next@14.2.4: version "14.2.4" resolved "https://registry.yarnpkg.com/next/-/next-14.2.4.tgz#ef66c39c71e2d8ad0a3caa0383c8933f4663e4d1" @@ -890,6 +1119,18 @@ next@14.2.4: "@next/swc-win32-ia32-msvc" "14.2.4" "@next/swc-win32-x64-msvc" "14.2.4" +node-abi@^3.3.0: + version "3.65.0" + resolved "https://registry.yarnpkg.com/node-abi/-/node-abi-3.65.0.tgz#ca92d559388e1e9cab1680a18c1a18757cdac9d3" + integrity sha512-ThjYBfoDNr08AWx6hGaRbfPwxKV9kVzAzOzlLKbk2CuqXE2xnCh+cbAGnwM3t8Lq4v9rUB7VfondlkBckcJrVA== + dependencies: + semver "^7.3.5" + +node-addon-api@^6.1.0: + version "6.1.0" + resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-6.1.0.tgz#ac8470034e58e67d0c6f1204a18ae6995d9c0d76" + integrity sha512-+eawOlIgy680F0kBzPUNFhMZGtJ1YmqM6l4+Crf4IkImjYrO/mqPwRMh352g23uIaQKFItcQ64I7KMaJxHgAVA== + node-releases@^2.0.14: version "2.0.14" resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.14.tgz#2ffb053bceb8b2be8495ece1ab6ce600c4461b0b" @@ -915,6 +1156,44 @@ object-hash@^3.0.0: resolved "https://registry.yarnpkg.com/object-hash/-/object-hash-3.0.0.tgz#73f97f753e7baffc0e2cc9d6e079079744ac82e9" integrity sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw== +once@^1.3.1, once@^1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" + integrity sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w== + dependencies: + wrappy "1" + +onnx-proto@^4.0.4: + version "4.0.4" + resolved "https://registry.yarnpkg.com/onnx-proto/-/onnx-proto-4.0.4.tgz#2431a25bee25148e915906dda0687aafe3b9e044" + integrity sha512-aldMOB3HRoo6q/phyB6QRQxSt895HNNw82BNyZ2CMh4bjeKv7g/c+VpAFtJuEMVfYLMbRx61hbuqnKceLeDcDA== + dependencies: + protobufjs "^6.8.8" + +onnxruntime-common@~1.14.0: + version "1.14.0" + resolved "https://registry.yarnpkg.com/onnxruntime-common/-/onnxruntime-common-1.14.0.tgz#2bb5dac5261269779aa5fb6536ca379657de8bf6" + integrity sha512-3LJpegM2iMNRX2wUmtYfeX/ytfOzNwAWKSq1HbRrKc9+uqG/FsEA0bbKZl1btQeZaXhC26l44NWpNUeXPII7Ew== + +onnxruntime-node@1.14.0: + version "1.14.0" + resolved "https://registry.yarnpkg.com/onnxruntime-node/-/onnxruntime-node-1.14.0.tgz#c4ae6c355cfae7d83abaf36dd39a905c4a010217" + integrity sha512-5ba7TWomIV/9b6NH/1x/8QEeowsb+jBEvFzU6z0T4mNsFwdPqXeFUM7uxC6QeSRkEbWu3qEB0VMjrvzN/0S9+w== + dependencies: + onnxruntime-common "~1.14.0" + +onnxruntime-web@1.14.0: + version "1.14.0" + resolved "https://registry.yarnpkg.com/onnxruntime-web/-/onnxruntime-web-1.14.0.tgz#c8cee538781b1d4c1c6b043934f4a3e6ddf1466e" + integrity sha512-Kcqf43UMfW8mCydVGcX9OMXI2VN17c0p6XvR7IPSZzBf/6lteBzXHvcEVWDPmCKuGombl997HgLqj91F11DzXw== + dependencies: + flatbuffers "^1.12.0" + guid-typescript "^1.0.9" + long "^4.0.0" + onnx-proto "^4.0.4" + onnxruntime-common "~1.14.0" + platform "^1.3.6" + package-json-from-dist@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/package-json-from-dist/-/package-json-from-dist-1.0.0.tgz#e501cd3094b278495eb4258d4c9f6d5ac3019f00" @@ -963,6 +1242,11 @@ pirates@^4.0.1: resolved "https://registry.yarnpkg.com/pirates/-/pirates-4.0.6.tgz#3018ae32ecfcff6c29ba2267cbf21166ac1f36b9" integrity sha512-saLsH7WeYYPiD25LDuLRRY/i+6HaPYr6G1OUlN39otzkSTxKnubR9RTxS3/Kk50s1g2JTgFwWQDQyplC5/SHZg== +platform@^1.3.6: + version "1.3.6" + resolved "https://registry.yarnpkg.com/platform/-/platform-1.3.6.tgz#48b4ce983164b209c2d45a107adb31f473a6e7a7" + integrity sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg== + postcss-import@^15.1.0: version "15.1.0" resolved "https://registry.yarnpkg.com/postcss-import/-/postcss-import-15.1.0.tgz#41c64ed8cc0e23735a9698b3249ffdbf704adc70" @@ -1025,6 +1309,24 @@ postcss@8.4.38, postcss@^8.4.23: picocolors "^1.0.0" source-map-js "^1.2.0" +prebuild-install@^7.1.1: + version "7.1.2" + resolved "https://registry.yarnpkg.com/prebuild-install/-/prebuild-install-7.1.2.tgz#a5fd9986f5a6251fbc47e1e5c65de71e68c0a056" + integrity sha512-UnNke3IQb6sgarcZIDU3gbMeTp/9SSU1DAIkil7PrqG1vZlBtY5msYccSKSHDqa3hNg436IXK+SNImReuA1wEQ== + dependencies: + detect-libc "^2.0.0" + expand-template "^2.0.3" + github-from-package "0.0.0" + minimist "^1.2.3" + mkdirp-classic "^0.5.3" + napi-build-utils "^1.0.1" + node-abi "^3.3.0" + pump "^3.0.0" + rc "^1.2.7" + simple-get "^4.0.0" + tar-fs "^2.0.0" + tunnel-agent "^0.6.0" + prettier-plugin-tailwindcss@0.6.5: version "0.6.5" resolved "https://registry.yarnpkg.com/prettier-plugin-tailwindcss/-/prettier-plugin-tailwindcss-0.6.5.tgz#e05202784a3f41889711ae38c75c5b8cad72f368" @@ -1035,11 +1337,53 @@ prettier@3.3.2: resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.3.2.tgz#03ff86dc7c835f2d2559ee76876a3914cec4a90a" integrity sha512-rAVeHYMcv8ATV5d508CFdn+8/pHPpXeIid1DdrPwXnaAdH7cqjVbpJaT5eq4yRAFU/lsbwYwSF/n5iNrdJHPQA== +protobufjs@^6.8.8: + version "6.11.4" + resolved "https://registry.yarnpkg.com/protobufjs/-/protobufjs-6.11.4.tgz#29a412c38bf70d89e537b6d02d904a6f448173aa" + integrity sha512-5kQWPaJHi1WoCpjTGszzQ32PG2F4+wRY6BmAT4Vfw56Q2FZ4YZzK20xUYQH4YkfehY1e6QSICrJquM6xXZNcrw== + dependencies: + "@protobufjs/aspromise" "^1.1.2" + "@protobufjs/base64" "^1.1.2" + "@protobufjs/codegen" "^2.0.4" + "@protobufjs/eventemitter" "^1.1.0" + "@protobufjs/fetch" "^1.1.0" + "@protobufjs/float" "^1.0.2" + "@protobufjs/inquire" "^1.1.0" + "@protobufjs/path" "^1.1.2" + "@protobufjs/pool" "^1.1.0" + "@protobufjs/utf8" "^1.1.0" + "@types/long" "^4.0.1" + "@types/node" ">=13.7.0" + long "^4.0.0" + +pump@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/pump/-/pump-3.0.0.tgz#b4a2116815bde2f4e1ea602354e8c75565107a64" + integrity sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww== + dependencies: + end-of-stream "^1.1.0" + once "^1.3.1" + queue-microtask@^1.2.2: version "1.2.3" resolved "https://registry.yarnpkg.com/queue-microtask/-/queue-microtask-1.2.3.tgz#4929228bbc724dfac43e0efb058caf7b6cfb6243" integrity sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A== +queue-tick@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/queue-tick/-/queue-tick-1.0.1.tgz#f6f07ac82c1fd60f82e098b417a80e52f1f4c142" + integrity sha512-kJt5qhMxoszgU/62PLP1CJytzd2NKetjSRnyuj31fDd3Rlcz3fzlFdFLD1SItunPwyqEOkca6GbV612BWfaBag== + +rc@^1.2.7: + version "1.2.8" + resolved "https://registry.yarnpkg.com/rc/-/rc-1.2.8.tgz#cd924bf5200a075b83c188cd6b9e211b7fc0d3ed" + integrity sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw== + dependencies: + deep-extend "^0.6.0" + ini "~1.3.0" + minimist "^1.2.0" + strip-json-comments "~2.0.1" + react-dom@18.3.1: version "18.3.1" resolved "https://registry.yarnpkg.com/react-dom/-/react-dom-18.3.1.tgz#c2265d79511b57d479b3dd3fdfa51536494c5cb4" @@ -1062,6 +1406,15 @@ read-cache@^1.0.0: dependencies: pify "^2.3.0" +readable-stream@^3.1.1, readable-stream@^3.4.0: + version "3.6.2" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.2.tgz#56a9b36ea965c00c5a93ef31eb111a0f11056967" + integrity sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA== + dependencies: + inherits "^2.0.3" + string_decoder "^1.1.1" + util-deprecate "^1.0.1" + readdirp@~3.6.0: version "3.6.0" resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-3.6.0.tgz#74a370bd857116e245b29cc97340cd431a02a6c7" @@ -1090,6 +1443,11 @@ run-parallel@^1.1.9: dependencies: queue-microtask "^1.2.2" +safe-buffer@^5.0.1, safe-buffer@~5.2.0: + version "5.2.1" + resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6" + integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ== + scheduler@^0.23.2: version "0.23.2" resolved "https://registry.yarnpkg.com/scheduler/-/scheduler-0.23.2.tgz#414ba64a3b282892e944cf2108ecc078d115cdc3" @@ -1097,7 +1455,7 @@ scheduler@^0.23.2: dependencies: loose-envify "^1.1.0" -semver@^7.6.0: +semver@^7.3.5, semver@^7.5.4, semver@^7.6.0: version "7.6.2" resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.2.tgz#1e3b34759f896e8f14d6134732ce798aeb0c6e13" integrity sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w== @@ -1131,6 +1489,20 @@ sharp@0.33.4: "@img/sharp-win32-ia32" "0.33.4" "@img/sharp-win32-x64" "0.33.4" +sharp@^0.32.0: + version "0.32.6" + resolved "https://registry.yarnpkg.com/sharp/-/sharp-0.32.6.tgz#6ad30c0b7cd910df65d5f355f774aa4fce45732a" + integrity sha512-KyLTWwgcR9Oe4d9HwCwNM2l7+J0dUQwn/yf7S0EnTtb0eVS4RxO0eUSvxPtzT4F3SY+C4K6fqdv/DO27sJ/v/w== + dependencies: + color "^4.2.3" + detect-libc "^2.0.2" + node-addon-api "^6.1.0" + prebuild-install "^7.1.1" + semver "^7.5.4" + simple-get "^4.0.1" + tar-fs "^3.0.4" + tunnel-agent "^0.6.0" + shebang-command@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/shebang-command/-/shebang-command-2.0.0.tgz#ccd0af4f8835fbdc265b82461aaf0c36663f34ea" @@ -1148,6 +1520,20 @@ signal-exit@^4.0.1: resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-4.1.0.tgz#952188c1cbd546070e2dd20d0f41c0ae0530cb04" integrity sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw== +simple-concat@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/simple-concat/-/simple-concat-1.0.1.tgz#f46976082ba35c2263f1c8ab5edfe26c41c9552f" + integrity sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q== + +simple-get@^4.0.0, simple-get@^4.0.1: + version "4.0.1" + resolved "https://registry.yarnpkg.com/simple-get/-/simple-get-4.0.1.tgz#4a39db549287c979d352112fa03fd99fd6bc3543" + integrity sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA== + dependencies: + decompress-response "^6.0.0" + once "^1.3.1" + simple-concat "^1.0.0" + simple-swizzle@^0.2.2: version "0.2.2" resolved "https://registry.yarnpkg.com/simple-swizzle/-/simple-swizzle-0.2.2.tgz#a4da6b635ffcccca33f70d17cb92592de95e557a" @@ -1165,6 +1551,17 @@ streamsearch@^1.1.0: resolved "https://registry.yarnpkg.com/streamsearch/-/streamsearch-1.1.0.tgz#404dd1e2247ca94af554e841a8ef0eaa238da764" integrity sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg== +streamx@^2.15.0, streamx@^2.18.0: + version "2.18.0" + resolved "https://registry.yarnpkg.com/streamx/-/streamx-2.18.0.tgz#5bc1a51eb412a667ebfdcd4e6cf6a6fc65721ac7" + integrity sha512-LLUC1TWdjVdn1weXGcSxyTR3T4+acB6tVGXT95y0nGbca4t4o/ng1wKAGTljm9VicuCVLvRlqFYXYy5GwgM7sQ== + dependencies: + fast-fifo "^1.3.2" + queue-tick "^1.0.1" + text-decoder "^1.1.0" + optionalDependencies: + bare-events "^2.2.0" + "string-width-cjs@npm:string-width@^4.2.0", string-width@^4.1.0: name string-width-cjs version "4.2.3" @@ -1184,6 +1581,13 @@ string-width@^5.0.1, string-width@^5.1.2: emoji-regex "^9.2.2" strip-ansi "^7.0.1" +string_decoder@^1.1.1: + version "1.3.0" + resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.3.0.tgz#42f114594a46cf1a8e30b0a84f56c78c3edac21e" + integrity sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA== + dependencies: + safe-buffer "~5.2.0" + "strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: name strip-ansi-cjs version "6.0.1" @@ -1199,6 +1603,11 @@ strip-ansi@^7.0.1: dependencies: ansi-regex "^6.0.1" +strip-json-comments@~2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-2.0.1.tgz#3c531942e908c2697c0ec344858c286c7ca0a60a" + integrity sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ== + styled-jsx@5.1.1: version "5.1.1" resolved "https://registry.yarnpkg.com/styled-jsx/-/styled-jsx-5.1.1.tgz#839a1c3aaacc4e735fed0781b8619ea5d0009d1f" @@ -1257,6 +1666,54 @@ tailwindcss@3.4.4: resolve "^1.22.2" sucrase "^3.32.0" +tar-fs@^2.0.0: + version "2.1.1" + resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-2.1.1.tgz#489a15ab85f1f0befabb370b7de4f9eb5cbe8784" + integrity sha512-V0r2Y9scmbDRLCNex/+hYzvp/zyYjvFbHPNgVTKfQvVrb6guiE/fxP+XblDNR011utopbkex2nM4dHNV6GDsng== + dependencies: + chownr "^1.1.1" + mkdirp-classic "^0.5.2" + pump "^3.0.0" + tar-stream "^2.1.4" + +tar-fs@^3.0.4: + version "3.0.6" + resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-3.0.6.tgz#eaccd3a67d5672f09ca8e8f9c3d2b89fa173f217" + integrity sha512-iokBDQQkUyeXhgPYaZxmczGPhnhXZ0CmrqI+MOb/WFGS9DW5wnfrLgtjUJBvz50vQ3qfRwJ62QVoCFu8mPVu5w== + dependencies: + pump "^3.0.0" + tar-stream "^3.1.5" + optionalDependencies: + bare-fs "^2.1.1" + bare-path "^2.1.0" + +tar-stream@^2.1.4: + version "2.2.0" + resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-2.2.0.tgz#acad84c284136b060dc3faa64474aa9aebd77287" + integrity sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ== + dependencies: + bl "^4.0.3" + end-of-stream "^1.4.1" + fs-constants "^1.0.0" + inherits "^2.0.3" + readable-stream "^3.1.1" + +tar-stream@^3.1.5: + version "3.1.7" + resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-3.1.7.tgz#24b3fb5eabada19fe7338ed6d26e5f7c482e792b" + integrity sha512-qJj60CXt7IU1Ffyc3NJMjh6EkuCFej46zUqJ4J7pqYlThyd9bO0XBTmcOIhSzZJVWfsLks0+nle/j538YAW9RQ== + dependencies: + b4a "^1.6.4" + fast-fifo "^1.2.0" + streamx "^2.15.0" + +text-decoder@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/text-decoder/-/text-decoder-1.1.0.tgz#3379e728fcf4d3893ec1aea35e8c2cac215ef190" + integrity sha512-TmLJNj6UgX8xcUZo4UDStGQtDiTzF7BzWlzn9g7UWrjkpHr5uJTK1ld16wZ3LXb2vb6jH8qU89dW5whuMdXYdw== + dependencies: + b4a "^1.6.4" + thenify-all@^1.0.0: version "1.6.0" resolved "https://registry.yarnpkg.com/thenify-all/-/thenify-all-1.6.0.tgz#1a1918d402d8fc3f98fbf234db0bcc8cc10e9726" @@ -1288,6 +1745,13 @@ tslib@^2.4.0: resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.3.tgz#0438f810ad7a9edcde7a241c3d80db693c8cbfe0" integrity sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ== +tunnel-agent@^0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz#27a5dea06b36b04a0a9966774b290868f0fc40fd" + integrity sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w== + dependencies: + safe-buffer "^5.0.1" + typescript@5.5.2: version "5.5.2" resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.5.2.tgz#c26f023cb0054e657ce04f72583ea2d85f8d0507" @@ -1306,7 +1770,7 @@ update-browserslist-db@^1.0.16: escalade "^3.1.2" picocolors "^1.0.1" -util-deprecate@^1.0.2: +util-deprecate@^1.0.1, util-deprecate@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" integrity sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw== @@ -1336,6 +1800,11 @@ wrap-ansi@^8.1.0: string-width "^5.0.1" strip-ansi "^7.0.1" +wrappy@1: + version "1.0.2" + resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" + integrity sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ== + yaml@^2.3.4: version "2.4.5" resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.4.5.tgz#60630b206dd6d84df97003d33fc1ddf6296cca5e" diff --git a/live-backend/src/server.rs b/live-backend/src/server.rs index a6049f2a..6a3afd91 100644 --- a/live-backend/src/server.rs +++ b/live-backend/src/server.rs @@ -11,6 +11,7 @@ use crate::LiveState; mod cors; mod health; +mod audio; pub mod live; pub struct AppState { @@ -59,8 +60,9 @@ pub async fn init( let app = Router::new() .route("/api/sse", get(live::sse_handler)) .route("/api/health", get(health::check)) - .layer(cors) + .route("/api/audio", get(audio::get_audio)) .layer(governor) + .layer(cors) .with_state(app_state) .into_make_service_with_connect_info::(); diff --git a/live-backend/src/server/audio.rs b/live-backend/src/server/audio.rs new file mode 100644 index 00000000..4b805a98 --- /dev/null +++ b/live-backend/src/server/audio.rs @@ -0,0 +1,30 @@ +use axum::{extract::Query, response::IntoResponse}; +use serde::Deserialize; +use tracing::error; + +#[derive(Deserialize)] + +pub struct Params { + path: String +} + +pub async fn get_audio(Query(params): Query) -> Result { + let Ok(_) = std::env::var("ENABLE_AUDIO_FETCH") else { return Err(axum::http::StatusCode::NOT_FOUND) }; + + let audio_url = format!("https://livetiming.formula1.com/static/{}", params.path); + match reqwest::get(&audio_url).await { + Ok(response) => { + match response.bytes().await { + Ok(bytes) => Ok(bytes.as_ref().to_vec()), + Err(_) => { + error!("Failed to decode response from {}", audio_url); + Err(axum::http::StatusCode::INTERNAL_SERVER_ERROR) + } + } + } + Err(_) => { + error!("Failed to retrieve audio data from {}", audio_url); + Err(axum::http::StatusCode::INTERNAL_SERVER_ERROR) + } + } +} From 78108537493dd790adb0407c2568c0c0822baf04 Mon Sep 17 00:00:00 2001 From: Kyujin Cho Date: Sun, 30 Jun 2024 18:11:58 +0900 Subject: [PATCH 02/13] fix indent --- dash/next.config.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dash/next.config.mjs b/dash/next.config.mjs index 3e91a718..76013449 100644 --- a/dash/next.config.mjs +++ b/dash/next.config.mjs @@ -16,6 +16,6 @@ const config = { "onnxruntime-node$": false, } return config - }, + }, }; export default config; From 9bf0ed2b1684a4a655f8f25cbaf59eb366e639d9 Mon Sep 17 00:00:00 2001 From: Kyujin Cho Date: Sat, 6 Jul 2024 23:20:45 +0900 Subject: [PATCH 03/13] Update dash/src/components/TeamRadioMessage.tsx Co-authored-by: slowlydev --- dash/src/components/TeamRadioMessage.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dash/src/components/TeamRadioMessage.tsx b/dash/src/components/TeamRadioMessage.tsx index 9f2dcbfd..4fedb3f1 100644 --- a/dash/src/components/TeamRadioMessage.tsx +++ b/dash/src/components/TeamRadioMessage.tsx @@ -13,7 +13,7 @@ type Props = { driver: Driver; capture: RadioCapture; basePath: string; - transcription?: string; + transcription?: string; }; export default function TeamRadioMessage({ driver, capture, basePath, transcription }: Props) { From 6698a9ce01f66604fe406d6b592f378b864b892d Mon Sep 17 00:00:00 2001 From: Kyujin Cho Date: Sat, 6 Jul 2024 23:25:57 +0900 Subject: [PATCH 04/13] fix lint --- dash/src/app/(nav)/info/page.tsx | 2 +- dash/src/app/(nav)/page.tsx | 2 +- dash/src/app/(nav)/settings/page.tsx | 27 ++- dash/src/app/(socket)/dashboard/page.tsx | 2 +- dash/src/asr-worker.js | 207 +++++++++--------- dash/src/components/ConnectionStatus.tsx | 2 +- dash/src/components/TeamRadioMessage.tsx | 15 +- dash/src/components/TeamRadios.tsx | 73 +++--- dash/src/components/Timeline.tsx | 2 +- dash/src/components/Windows.tsx | 2 +- .../components/complications/WindSpeed.tsx | 2 +- .../components/driver/DriverHistoryTires.tsx | 2 +- dash/src/context/ModeContext.tsx | 2 +- dash/src/lib/constants.ts | 44 ++-- dash/src/lib/data/features.tsx | 2 +- 15 files changed, 193 insertions(+), 193 deletions(-) diff --git a/dash/src/app/(nav)/info/page.tsx b/dash/src/app/(nav)/info/page.tsx index c9a875fc..ada007ac 100644 --- a/dash/src/app/(nav)/info/page.tsx +++ b/dash/src/app/(nav)/info/page.tsx @@ -1,6 +1,6 @@ export default function Page() { return ( -
    +

    Infos about issues and v2 beta

    diff --git a/dash/src/app/(nav)/page.tsx b/dash/src/app/(nav)/page.tsx index c18f5a55..c039bcad 100644 --- a/dash/src/app/(nav)/page.tsx +++ b/dash/src/app/(nav)/page.tsx @@ -49,7 +49,7 @@ export default function Home() {

    What are our Features?

    -
    +
    {features.map((feature, i) => ( ))} diff --git a/dash/src/app/(nav)/settings/page.tsx b/dash/src/app/(nav)/settings/page.tsx index 4965751f..4fedb708 100644 --- a/dash/src/app/(nav)/settings/page.tsx +++ b/dash/src/app/(nav)/settings/page.tsx @@ -38,10 +38,12 @@ export default function SettingsPage() { setCarMetrics(customSettings.carMetrics); const transcriptionStorage = localStorage.getItem("transcription"); - const transcriptionSettings: TranscriptionSettings = transcriptionStorage ? JSON.parse(transcriptionStorage) : { enableTranscription: false, whisperModel: "" }; + const transcriptionSettings: TranscriptionSettings = transcriptionStorage + ? JSON.parse(transcriptionStorage) + : { enableTranscription: false, whisperModel: "" }; setEnableTranscription(transcriptionSettings.enableTranscription); - setTranscriptionModel(transcriptionSettings.whisperModel); + setTranscriptionModel(transcriptionSettings.whisperModel); } }, []); @@ -72,7 +74,9 @@ export default function SettingsPage() { const handleTranscriptionSettingUpdate = (type: "transcription" | "model", newValue: string | boolean) => { if (typeof window != undefined) { const transcriptionStorage = localStorage.getItem("transcription"); - const transcriptionSettings: TranscriptionSettings = transcriptionStorage ? JSON.parse(transcriptionStorage) : modes.custom; + const transcriptionSettings: TranscriptionSettings = transcriptionStorage + ? JSON.parse(transcriptionStorage) + : modes.custom; switch (type) { case "transcription": { @@ -87,7 +91,7 @@ export default function SettingsPage() { localStorage.setItem("transcription", JSON.stringify(transcriptionSettings)); } - } + }; const updateDelay = (newDelay: number) => { setDelay(newDelay); @@ -168,9 +172,7 @@ export default function SettingsPage() {

    Enable Radio Transcription

    -

    - Only available when the corresponding feature is enabled from server. -

    +

    Only available when the corresponding feature is enabled from server.

    - { + setTranscriptionModel(s.target.value); + handleTranscriptionSettingUpdate("model", s.target.value); + }} + > diff --git a/dash/src/app/(socket)/dashboard/page.tsx b/dash/src/app/(socket)/dashboard/page.tsx index 2ce677a0..cd73c341 100644 --- a/dash/src/app/(socket)/dashboard/page.tsx +++ b/dash/src/app/(socket)/dashboard/page.tsx @@ -51,7 +51,7 @@ export default function Page() {
    -
    +
    { - const message = event.data; - - // Do some work... - // TODO use message data - let transcript - try { - transcript = await transcribe( - message.audio, - message.model, - message.multilingual, - message.quantized, - message.subtask, - message.language, - ); - } catch (e) { - console.warn("Error while transcribing: " + e); - transcript = { - text: "", - chunks: [], - } - } - - // Send the result back to the main thread - self.postMessage({ - status: "complete", - task: "automatic-speech-recognition", - key: message.key, - data: transcript, - }); + const message = event.data; + + // Do some work... + // TODO use message data + let transcript; + try { + transcript = await transcribe( + message.audio, + message.model, + message.multilingual, + message.quantized, + message.subtask, + message.language, + ); + } catch (e) { + console.warn("Error while transcribing: " + e); + transcript = { + text: "", + chunks: [], + }; + } + + // Send the result back to the main thread + self.postMessage({ + status: "complete", + task: "automatic-speech-recognition", + key: message.key, + data: transcript, + }); }); class AutomaticSpeechRecognitionPipelineFactory extends PipelineFactory { - static task = "automatic-speech-recognition"; - static model = null; - static quantized = null; + static task = "automatic-speech-recognition"; + static model = null; + static quantized = null; } -const transcribe = async ( - audio, - model, - multilingual, - quantized, - subtask, - language, -) => { - const isDistilWhisper = model.startsWith("distil-whisper/"); - - let modelName = model; - if (!isDistilWhisper && !multilingual) { - modelName += ".en" - } - - const p = AutomaticSpeechRecognitionPipelineFactory; - if (p.model !== modelName || p.quantized !== quantized) { - // Invalidate model if different - p.model = modelName; - p.quantized = quantized; - - if (p.instance !== null) { - (await p.getInstance()).dispose(); - p.instance = null; - } - } - - // Load transcriber model - let transcriber = await p.getInstance((data) => { - self.postMessage(data); - }); - - // Actually run transcription - let output = await transcriber(audio, { - // Greedy - top_k: 0, - do_sample: false, - - // Sliding window - chunk_length_s: isDistilWhisper ? 20 : 30, - stride_length_s: isDistilWhisper ? 3 : 5, - - // Language and task - language: language, - task: subtask, - - // Return timestamps - return_timestamps: true, - force_full_sequences: false, - }); - - return output; -}; \ No newline at end of file +const transcribe = async (audio, model, multilingual, quantized, subtask, language) => { + const isDistilWhisper = model.startsWith("distil-whisper/"); + + let modelName = model; + if (!isDistilWhisper && !multilingual) { + modelName += ".en"; + } + + const p = AutomaticSpeechRecognitionPipelineFactory; + if (p.model !== modelName || p.quantized !== quantized) { + // Invalidate model if different + p.model = modelName; + p.quantized = quantized; + + if (p.instance !== null) { + (await p.getInstance()).dispose(); + p.instance = null; + } + } + + // Load transcriber model + let transcriber = await p.getInstance((data) => { + self.postMessage(data); + }); + + // Actually run transcription + let output = await transcriber(audio, { + // Greedy + top_k: 0, + do_sample: false, + + // Sliding window + chunk_length_s: isDistilWhisper ? 20 : 30, + stride_length_s: isDistilWhisper ? 3 : 5, + + // Language and task + language: language, + task: subtask, + + // Return timestamps + return_timestamps: true, + force_full_sequences: false, + }); + + return output; +}; diff --git a/dash/src/components/ConnectionStatus.tsx b/dash/src/components/ConnectionStatus.tsx index e70c8ce5..c329004a 100644 --- a/dash/src/components/ConnectionStatus.tsx +++ b/dash/src/components/ConnectionStatus.tsx @@ -6,5 +6,5 @@ import { useSocket } from "@/context/SocketContext"; export default function ConnectionStatus() { const { connected } = useSocket(); - return
    ; + return
    ; } diff --git a/dash/src/components/TeamRadioMessage.tsx b/dash/src/components/TeamRadioMessage.tsx index 4fedb3f1..6af5678f 100644 --- a/dash/src/components/TeamRadioMessage.tsx +++ b/dash/src/components/TeamRadioMessage.tsx @@ -13,7 +13,7 @@ type Props = { driver: Driver; capture: RadioCapture; basePath: string; - transcription?: string; + transcription?: string; }; export default function TeamRadioMessage({ driver, capture, basePath, transcription }: Props) { @@ -27,12 +27,13 @@ export default function TeamRadioMessage({ driver, capture, basePath, transcript const transcriptionElement = useMemo(() => { if (transcription === undefined) { return <>; - } - else if (transcription === "") { + } else if (transcription === "") { return ; } else { return ( -

    {transcription}

    +

    + {transcription} +

    ); } }, [transcription]); @@ -112,9 +113,7 @@ export default function TeamRadioMessage({ driver, capture, basePath, transcript />
    -
    - {transcriptionElement} -
    +
    {transcriptionElement}
    ); } @@ -123,4 +122,4 @@ const SkeletonTranscription = () => { const animateClass = "h-6 animate-pulse rounded-md bg-zinc-800"; return
    ; -} \ No newline at end of file +}; diff --git a/dash/src/components/TeamRadios.tsx b/dash/src/components/TeamRadios.tsx index c6dfb010..4a28b25c 100644 --- a/dash/src/components/TeamRadios.tsx +++ b/dash/src/components/TeamRadios.tsx @@ -9,7 +9,7 @@ import { DriverList, RadioCapture, TeamRadio } from "@/types/state.type"; import TeamRadioMessage from "@/components/TeamRadioMessage"; import { TranscriptionSettings } from "@/context/ModeContext"; -import Constants from '@/lib/constants'; +import Constants from "@/lib/constants"; import { env } from "@/env.mjs"; type Props = { @@ -19,27 +19,31 @@ type Props = { }; type TranscriberCompleteData = { - key: string; - data: { - text: string; - chunks: { text: string; timestamp: [number, number | null] }[]; - }; -} + key: string; + data: { + text: string; + chunks: { text: string; timestamp: [number, number | null] }[]; + }; +}; -const loadAudioFromRadioCapture = async (audioContext: AudioContext, path: string, retries = 0): Promise => { - const queryString = new URLSearchParams({path}).toString(); - try { - const response = await fetch(`${env.NEXT_PUBLIC_LIVE_SOCKET_URL}/api/audio?${queryString}`); - if (response.status === 429) throw new Error("met rate limiter") ; - const abuf = await response.arrayBuffer(); - const audioData = await audioContext.decodeAudioData(abuf); - return audioData.getChannelData(0); - } catch (e) { - console.warn(e); - await new Promise((res) => setTimeout(res, 500 * Math.pow(2, (retries + 1)))); // to deal with possible rate limit violation - return await loadAudioFromRadioCapture(audioContext, path, retries + 1); - } -} +const loadAudioFromRadioCapture = async ( + audioContext: AudioContext, + path: string, + retries = 0, +): Promise => { + const queryString = new URLSearchParams({ path }).toString(); + try { + const response = await fetch(`${env.NEXT_PUBLIC_LIVE_SOCKET_URL}/api/audio?${queryString}`); + if (response.status === 429) throw new Error("met rate limiter"); + const abuf = await response.arrayBuffer(); + const audioData = await audioContext.decodeAudioData(abuf); + return audioData.getChannelData(0); + } catch (e) { + console.warn(e); + await new Promise((res) => setTimeout(res, 500 * Math.pow(2, retries + 1))); // to deal with possible rate limit violation + return await loadAudioFromRadioCapture(audioContext, path, retries + 1); + } +}; export default function TeamRadios({ sessionPath, drivers, teamRadios }: Props) { const basePath = `https://livetiming.formula1.com/static/${sessionPath}`; @@ -48,7 +52,7 @@ export default function TeamRadios({ sessionPath, drivers, teamRadios }: Props) const workerRef = useRef(null); const [enableTranscription, setEnableTranscription] = useState(false); - const [transcriptions, setTranscriptions] = useState<{[key: string]: string}>({}); + const [transcriptions, setTranscriptions] = useState<{ [key: string]: string }>({}); const [transcriptionModel, setTranscriptionModel] = useState(""); const workerEventHandler = (event: MessageEvent) => { @@ -59,9 +63,10 @@ export default function TeamRadios({ sessionPath, drivers, teamRadios }: Props) const completeMessage = message as TranscriberCompleteData; const path = message.key; if (path) { - setTranscriptions((oldTranscription) => ( - {...oldTranscription, [path]: completeMessage.data.chunks.map((c) => c.text).join("\n")} - )) + setTranscriptions((oldTranscription) => ({ + ...oldTranscription, + [path]: completeMessage.data.chunks.map((c) => c.text).join("\n"), + })); } console.log("Transcription completed: " + path); break; @@ -85,11 +90,11 @@ export default function TeamRadios({ sessionPath, drivers, teamRadios }: Props) subtask: null, language: null, }); - await new Promise((res) => setTimeout(res, 1000)); // To avoid rate limit + await new Promise((res) => setTimeout(res, 1000)); // To avoid rate limit } }; - useEffect(() => { + useEffect(() => { if (typeof window != undefined) { const worker = new Worker(new URL("../asr-worker.js", import.meta.url), { type: "module", @@ -97,9 +102,11 @@ export default function TeamRadios({ sessionPath, drivers, teamRadios }: Props) // Listen for messages from the Web Worker worker.addEventListener("message", workerEventHandler); workerRef.current = worker; - + const transcriptionStorage = localStorage.getItem("transcription"); - const transcriptionSettings: TranscriptionSettings = transcriptionStorage ? JSON.parse(transcriptionStorage) : { enableTranscription: false, whisperModel: "" }; + const transcriptionSettings: TranscriptionSettings = transcriptionStorage + ? JSON.parse(transcriptionStorage) + : { enableTranscription: false, whisperModel: "" }; setEnableTranscription(transcriptionSettings.enableTranscription); setTranscriptionModel(transcriptionSettings.whisperModel); @@ -108,14 +115,12 @@ export default function TeamRadios({ sessionPath, drivers, teamRadios }: Props) useEffect(() => { if (teamRadios && drivers && teamRadios.captures && enableTranscription && transcriptionModel) { - const targetRadios = teamRadios.captures - .sort(sortUtc) - .slice(0, 20); + const targetRadios = teamRadios.captures.sort(sortUtc).slice(0, 20); setTranscriptions((oldTranscriptions) => { const newRadios = targetRadios.filter((c) => oldTranscriptions[c.path] === undefined); - const newTranscriptions = {...oldTranscriptions}; - newRadios.forEach((c) => newTranscriptions[c.path] = ""); + const newTranscriptions = { ...oldTranscriptions }; + newRadios.forEach((c) => (newTranscriptions[c.path] = "")); beginTranscripting(newRadios); return newTranscriptions; diff --git a/dash/src/components/Timeline.tsx b/dash/src/components/Timeline.tsx index 7caeeb34..e426f9d6 100644 --- a/dash/src/components/Timeline.tsx +++ b/dash/src/components/Timeline.tsx @@ -122,7 +122,7 @@ export default function Timeline({ playing, maxDelay, time, setTime }: Props) {
    */} -
    +

    {windowOption.label}

    diff --git a/dash/src/components/complications/WindSpeed.tsx b/dash/src/components/complications/WindSpeed.tsx index 5ea04322..2d62809b 100644 --- a/dash/src/components/complications/WindSpeed.tsx +++ b/dash/src/components/complications/WindSpeed.tsx @@ -13,7 +13,7 @@ export default function WindSpeedComplication({ speed, directionDeg }: Props) { {getWindDirection(directionDeg)}

    -

    {speed}

    +

    {speed}

    m/s

    diff --git a/dash/src/components/driver/DriverHistoryTires.tsx b/dash/src/components/driver/DriverHistoryTires.tsx index 7bcce47f..05c9663e 100644 --- a/dash/src/components/driver/DriverHistoryTires.tsx +++ b/dash/src/components/driver/DriverHistoryTires.tsx @@ -44,7 +44,7 @@ function LoadingTire() { return (
    -
    +
    ); } diff --git a/dash/src/context/ModeContext.tsx b/dash/src/context/ModeContext.tsx index ca7645fa..484046e5 100644 --- a/dash/src/context/ModeContext.tsx +++ b/dash/src/context/ModeContext.tsx @@ -18,7 +18,7 @@ export type UiElements = { export type TranscriptionSettings = { enableTranscription: boolean; whisperModel: string; -} +}; type Mode = "simple" | "advanced" | "expert" | "custom"; diff --git a/dash/src/lib/constants.ts b/dash/src/lib/constants.ts index f1bd9f81..9799d0b0 100644 --- a/dash/src/lib/constants.ts +++ b/dash/src/lib/constants.ts @@ -1,27 +1,25 @@ function mobileTabletCheck() { - // https://stackoverflow.com/questions/11381673/detecting-a-mobile-browser - let check = false; - (function (a: string) { - if ( - /(android|bb\d+|meego).+mobile|avantgo|bada\/|blackberry|blazer|compal|elaine|fennec|hiptop|iemobile|ip(hone|od)|iris|kindle|lge |maemo|midp|mmp|mobile.+firefox|netfront|opera m(ob|in)i|palm( os)?|phone|p(ixi|re)\/|plucker|pocket|psp|series(4|6)0|symbian|treo|up\.(browser|link)|vodafone|wap|windows ce|xda|xiino|android|ipad|playbook|silk/i.test( - a, - ) || - /1207|6310|6590|3gso|4thp|50[1-6]i|770s|802s|a wa|abac|ac(er|oo|s\-)|ai(ko|rn)|al(av|ca|co)|amoi|an(ex|ny|yw)|aptu|ar(ch|go)|as(te|us)|attw|au(di|\-m|r |s )|avan|be(ck|ll|nq)|bi(lb|rd)|bl(ac|az)|br(e|v)w|bumb|bw\-(n|u)|c55\/|capi|ccwa|cdm\-|cell|chtm|cldc|cmd\-|co(mp|nd)|craw|da(it|ll|ng)|dbte|dc\-s|devi|dica|dmob|do(c|p)o|ds(12|\-d)|el(49|ai)|em(l2|ul)|er(ic|k0)|esl8|ez([4-7]0|os|wa|ze)|fetc|fly(\-|_)|g1 u|g560|gene|gf\-5|g\-mo|go(\.w|od)|gr(ad|un)|haie|hcit|hd\-(m|p|t)|hei\-|hi(pt|ta)|hp( i|ip)|hs\-c|ht(c(\-| |_|a|g|p|s|t)|tp)|hu(aw|tc)|i\-(20|go|ma)|i230|iac( |\-|\/)|ibro|idea|ig01|ikom|im1k|inno|ipaq|iris|ja(t|v)a|jbro|jemu|jigs|kddi|keji|kgt( |\/)|klon|kpt |kwc\-|kyo(c|k)|le(no|xi)|lg( g|\/(k|l|u)|50|54|\-[a-w])|libw|lynx|m1\-w|m3ga|m50\/|ma(te|ui|xo)|mc(01|21|ca)|m\-cr|me(rc|ri)|mi(o8|oa|ts)|mmef|mo(01|02|bi|de|do|t(\-| |o|v)|zz)|mt(50|p1|v )|mwbp|mywa|n10[0-2]|n20[2-3]|n30(0|2)|n50(0|2|5)|n7(0(0|1)|10)|ne((c|m)\-|on|tf|wf|wg|wt)|nok(6|i)|nzph|o2im|op(ti|wv)|oran|owg1|p800|pan(a|d|t)|pdxg|pg(13|\-([1-8]|c))|phil|pire|pl(ay|uc)|pn\-2|po(ck|rt|se)|prox|psio|pt\-g|qa\-a|qc(07|12|21|32|60|\-[2-7]|i\-)|qtek|r380|r600|raks|rim9|ro(ve|zo)|s55\/|sa(ge|ma|mm|ms|ny|va)|sc(01|h\-|oo|p\-)|sdk\/|se(c(\-|0|1)|47|mc|nd|ri)|sgh\-|shar|sie(\-|m)|sk\-0|sl(45|id)|sm(al|ar|b3|it|t5)|so(ft|ny)|sp(01|h\-|v\-|v )|sy(01|mb)|t2(18|50)|t6(00|10|18)|ta(gt|lk)|tcl\-|tdg\-|tel(i|m)|tim\-|t\-mo|to(pl|sh)|ts(70|m\-|m3|m5)|tx\-9|up(\.b|g1|si)|utst|v400|v750|veri|vi(rg|te)|vk(40|5[0-3]|\-v)|vm40|voda|vulc|vx(52|53|60|61|70|80|81|83|85|98)|w3c(\-| )|webc|whit|wi(g |nc|nw)|wmlb|wonu|x700|yas\-|your|zeto|zte\-/i.test( - a.substr(0, 4), - ) - ) - check = true; - })( - navigator.userAgent || - navigator.vendor || - ("opera" in window && typeof window.opera === "string" - ? window.opera - : ""), - ); - return check; + // https://stackoverflow.com/questions/11381673/detecting-a-mobile-browser + let check = false; + (function (a: string) { + if ( + /(android|bb\d+|meego).+mobile|avantgo|bada\/|blackberry|blazer|compal|elaine|fennec|hiptop|iemobile|ip(hone|od)|iris|kindle|lge |maemo|midp|mmp|mobile.+firefox|netfront|opera m(ob|in)i|palm( os)?|phone|p(ixi|re)\/|plucker|pocket|psp|series(4|6)0|symbian|treo|up\.(browser|link)|vodafone|wap|windows ce|xda|xiino|android|ipad|playbook|silk/i.test( + a, + ) || + /1207|6310|6590|3gso|4thp|50[1-6]i|770s|802s|a wa|abac|ac(er|oo|s\-)|ai(ko|rn)|al(av|ca|co)|amoi|an(ex|ny|yw)|aptu|ar(ch|go)|as(te|us)|attw|au(di|\-m|r |s )|avan|be(ck|ll|nq)|bi(lb|rd)|bl(ac|az)|br(e|v)w|bumb|bw\-(n|u)|c55\/|capi|ccwa|cdm\-|cell|chtm|cldc|cmd\-|co(mp|nd)|craw|da(it|ll|ng)|dbte|dc\-s|devi|dica|dmob|do(c|p)o|ds(12|\-d)|el(49|ai)|em(l2|ul)|er(ic|k0)|esl8|ez([4-7]0|os|wa|ze)|fetc|fly(\-|_)|g1 u|g560|gene|gf\-5|g\-mo|go(\.w|od)|gr(ad|un)|haie|hcit|hd\-(m|p|t)|hei\-|hi(pt|ta)|hp( i|ip)|hs\-c|ht(c(\-| |_|a|g|p|s|t)|tp)|hu(aw|tc)|i\-(20|go|ma)|i230|iac( |\-|\/)|ibro|idea|ig01|ikom|im1k|inno|ipaq|iris|ja(t|v)a|jbro|jemu|jigs|kddi|keji|kgt( |\/)|klon|kpt |kwc\-|kyo(c|k)|le(no|xi)|lg( g|\/(k|l|u)|50|54|\-[a-w])|libw|lynx|m1\-w|m3ga|m50\/|ma(te|ui|xo)|mc(01|21|ca)|m\-cr|me(rc|ri)|mi(o8|oa|ts)|mmef|mo(01|02|bi|de|do|t(\-| |o|v)|zz)|mt(50|p1|v )|mwbp|mywa|n10[0-2]|n20[2-3]|n30(0|2)|n50(0|2|5)|n7(0(0|1)|10)|ne((c|m)\-|on|tf|wf|wg|wt)|nok(6|i)|nzph|o2im|op(ti|wv)|oran|owg1|p800|pan(a|d|t)|pdxg|pg(13|\-([1-8]|c))|phil|pire|pl(ay|uc)|pn\-2|po(ck|rt|se)|prox|psio|pt\-g|qa\-a|qc(07|12|21|32|60|\-[2-7]|i\-)|qtek|r380|r600|raks|rim9|ro(ve|zo)|s55\/|sa(ge|ma|mm|ms|ny|va)|sc(01|h\-|oo|p\-)|sdk\/|se(c(\-|0|1)|47|mc|nd|ri)|sgh\-|shar|sie(\-|m)|sk\-0|sl(45|id)|sm(al|ar|b3|it|t5)|so(ft|ny)|sp(01|h\-|v\-|v )|sy(01|mb)|t2(18|50)|t6(00|10|18)|ta(gt|lk)|tcl\-|tdg\-|tel(i|m)|tim\-|t\-mo|to(pl|sh)|ts(70|m\-|m3|m5)|tx\-9|up(\.b|g1|si)|utst|v400|v750|veri|vi(rg|te)|vk(40|5[0-3]|\-v)|vm40|voda|vulc|vx(52|53|60|61|70|80|81|83|85|98)|w3c(\-| )|webc|whit|wi(g |nc|nw)|wmlb|wonu|x700|yas\-|your|zeto|zte\-/i.test( + a.substr(0, 4), + ) + ) + check = true; + })( + navigator.userAgent || + navigator.vendor || + ("opera" in window && typeof window.opera === "string" ? window.opera : ""), + ); + return check; } const isMobileOrTablet = mobileTabletCheck(); export default { - SAMPLING_RATE: 16000, - DEFAULT_QUANTIZED: isMobileOrTablet, -}; \ No newline at end of file + SAMPLING_RATE: 16000, + DEFAULT_QUANTIZED: isMobileOrTablet, +}; diff --git a/dash/src/lib/data/features.tsx b/dash/src/lib/data/features.tsx index 3202cb59..51c0ce91 100644 --- a/dash/src/lib/data/features.tsx +++ b/dash/src/lib/data/features.tsx @@ -51,7 +51,7 @@ export const features = [ "Monitor tire usage and pit stop strategies throughout the race. Keep track of when drivers make their crucial pit stops, switch tire compounds, and manage their tire wear to optimize race performance.", children: (
    -
    +
    soft medium hard From 0f52157046d0353667692a5ce2e585da597fc29f Mon Sep 17 00:00:00 2001 From: Kyujin Cho Date: Sat, 6 Jul 2024 23:27:13 +0900 Subject: [PATCH 05/13] remove console.log --- dash/src/components/TeamRadios.tsx | 2 -- 1 file changed, 2 deletions(-) diff --git a/dash/src/components/TeamRadios.tsx b/dash/src/components/TeamRadios.tsx index 4a28b25c..3fe6a6c3 100644 --- a/dash/src/components/TeamRadios.tsx +++ b/dash/src/components/TeamRadios.tsx @@ -68,7 +68,6 @@ export default function TeamRadios({ sessionPath, drivers, teamRadios }: Props) [path]: completeMessage.data.chunks.map((c) => c.text).join("\n"), })); } - console.log("Transcription completed: " + path); break; } }; @@ -80,7 +79,6 @@ export default function TeamRadios({ sessionPath, drivers, teamRadios }: Props) for (const teamRadio of teamRadios) { const audio = await loadAudioFromRadioCapture(audioContext, `${sessionPath}${teamRadio.path}`); - console.log("Requesting worker to transcript " + `${sessionPath}${teamRadio.path}`); workerRef.current?.postMessage({ key: teamRadio.path, audio, From 74e9585ee10804c212e1928d24cb74257fbe9400 Mon Sep 17 00:00:00 2001 From: Kyujin Cho Date: Sat, 6 Jul 2024 23:29:45 +0900 Subject: [PATCH 06/13] remove reference to webkitAudioContext --- dash/src/components/TeamRadios.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dash/src/components/TeamRadios.tsx b/dash/src/components/TeamRadios.tsx index 3fe6a6c3..f330edcb 100644 --- a/dash/src/components/TeamRadios.tsx +++ b/dash/src/components/TeamRadios.tsx @@ -73,7 +73,7 @@ export default function TeamRadios({ sessionPath, drivers, teamRadios }: Props) }; const beginTranscripting = async (teamRadios: RadioCapture[]) => { - const audioContext = new (window.AudioContext || window.webkitAudioContext)({ + const audioContext = new window.AudioContext({ sampleRate: Constants.SAMPLING_RATE, }); From d7a4f6dd1d2bf485cbb31a93a4395195e84b40f4 Mon Sep 17 00:00:00 2001 From: Kyujin Cho Date: Sat, 6 Jul 2024 23:31:40 +0900 Subject: [PATCH 07/13] import constant directly --- dash/src/components/TeamRadios.tsx | 6 +++--- dash/src/lib/constants.ts | 6 ++---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/dash/src/components/TeamRadios.tsx b/dash/src/components/TeamRadios.tsx index f330edcb..02a61032 100644 --- a/dash/src/components/TeamRadios.tsx +++ b/dash/src/components/TeamRadios.tsx @@ -9,7 +9,7 @@ import { DriverList, RadioCapture, TeamRadio } from "@/types/state.type"; import TeamRadioMessage from "@/components/TeamRadioMessage"; import { TranscriptionSettings } from "@/context/ModeContext"; -import Constants from "@/lib/constants"; +import { SAMPLING_RATE, DEFAULT_QUANTIZED } from "@/lib/constants"; import { env } from "@/env.mjs"; type Props = { @@ -74,7 +74,7 @@ export default function TeamRadios({ sessionPath, drivers, teamRadios }: Props) const beginTranscripting = async (teamRadios: RadioCapture[]) => { const audioContext = new window.AudioContext({ - sampleRate: Constants.SAMPLING_RATE, + sampleRate: SAMPLING_RATE, }); for (const teamRadio of teamRadios) { @@ -84,7 +84,7 @@ export default function TeamRadios({ sessionPath, drivers, teamRadios }: Props) audio, model: transcriptionModel, multilingual: false, - quantized: Constants.DEFAULT_QUANTIZED, + quantized: DEFAULT_QUANTIZED, subtask: null, language: null, }); diff --git a/dash/src/lib/constants.ts b/dash/src/lib/constants.ts index 9799d0b0..1b67bead 100644 --- a/dash/src/lib/constants.ts +++ b/dash/src/lib/constants.ts @@ -19,7 +19,5 @@ function mobileTabletCheck() { return check; } const isMobileOrTablet = mobileTabletCheck(); -export default { - SAMPLING_RATE: 16000, - DEFAULT_QUANTIZED: isMobileOrTablet, -}; +export const SAMPLING_RATE = 16000; +export const DEFAULT_QUANTIZED = isMobileOrTablet; From d7d4716a5f8f5656ae746239f8715cdd34c14c3f Mon Sep 17 00:00:00 2001 From: Kyujin Cho Date: Sat, 6 Jul 2024 23:43:13 +0900 Subject: [PATCH 08/13] use headless' select API --- dash/src/app/(nav)/settings/page.tsx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dash/src/app/(nav)/settings/page.tsx b/dash/src/app/(nav)/settings/page.tsx index 4fedb708..59b722a3 100644 --- a/dash/src/app/(nav)/settings/page.tsx +++ b/dash/src/app/(nav)/settings/page.tsx @@ -2,6 +2,7 @@ import { useEffect, useState } from "react"; import { useRouter } from "next/navigation"; +import { Select } from "@headlessui/react"; import { modes, type UiElements, type TranscriptionSettings } from "@/context/ModeContext"; @@ -186,7 +187,7 @@ export default function SettingsPage() {
    - +

    Transcription Mode

    From 33f99557d39b6236bbe2b23ab7afdb06127e3e23 Mon Sep 17 00:00:00 2001 From: Kyujin Cho Date: Sun, 7 Jul 2024 00:01:33 +0900 Subject: [PATCH 09/13] update response code --- live-backend/src/server/audio.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/live-backend/src/server/audio.rs b/live-backend/src/server/audio.rs index 4b805a98..20f7f98f 100644 --- a/live-backend/src/server/audio.rs +++ b/live-backend/src/server/audio.rs @@ -9,7 +9,7 @@ pub struct Params { } pub async fn get_audio(Query(params): Query) -> Result { - let Ok(_) = std::env::var("ENABLE_AUDIO_FETCH") else { return Err(axum::http::StatusCode::NOT_FOUND) }; + let Ok(_) = std::env::var("ENABLE_AUDIO_FETCH") else { return Err(axum::http::StatusCode::NOT_IMPLEMENTED) }; let audio_url = format!("https://livetiming.formula1.com/static/{}", params.path); match reqwest::get(&audio_url).await { From 00649d828101b80f415fc7129c20f47856679821 Mon Sep 17 00:00:00 2001 From: Slowlydev Date: Sun, 20 Apr 2025 22:55:15 +0200 Subject: [PATCH 10/13] chore: merge leftovers --- dash/package.json | 2 +- dash/src/stores/useTranscriptionStore.ts | 50 ++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 dash/src/stores/useTranscriptionStore.ts diff --git a/dash/package.json b/dash/package.json index 3e2d7bd7..5002fd51 100644 --- a/dash/package.json +++ b/dash/package.json @@ -10,9 +10,9 @@ "prettier": "prettier --write src" }, "dependencies": { - "@xenova/transformers": "2.7.0", "@fireworks-js/react": "2.10.8", "@headlessui/react": "2.2.0", + "@xenova/transformers": "2.7.0", "clsx": "2.1.1", "framer-motion": "11.11.17", "geist": "1.3.1", diff --git a/dash/src/stores/useTranscriptionStore.ts b/dash/src/stores/useTranscriptionStore.ts new file mode 100644 index 00000000..9dd16b53 --- /dev/null +++ b/dash/src/stores/useTranscriptionStore.ts @@ -0,0 +1,50 @@ +import { persist, createJSONStorage, subscribeWithSelector } from "zustand/middleware"; +import { create } from "zustand"; + +type Models = { + value: Model; + label: string; +}[] + +export const models: Models = [ + { + value: "Xenova/whisper-tiny", + label: "Low Latency", + }, + { + value: "Xenova/whisper-base", + label: "Balanced", + }, + { + value: "distil-whisper/distil-small.en", + label: "High Quality", + }, +] + +type Model = "distil-whisper/distil-small.en" | "Xenova/whisper-base" | "Xenova/whisper-tiny"; + +type TranscriptionStore = { + enabled: boolean; + setEnabled: (enabled: boolean) => void; + + model: Model; + setModel: (model: Model) => void; +}; + +export const useTranscriptionStore = create( + subscribeWithSelector( + persist( + (set) => ({ + enabled: false, + setEnabled: (enabled) => set(() => ({ enabled })), + + model: "Xenova/whisper-tiny", + setModel: (model) => set(() => ({ model })), + }), + { + name: "transcription-storage", + storage: createJSONStorage(() => localStorage), + }, + ), + ), +); From a0afa0fae8a944d481f0a1360787c6e20d739c34 Mon Sep 17 00:00:00 2001 From: Slowlydev Date: Sun, 20 Apr 2025 23:00:04 +0200 Subject: [PATCH 11/13] fix: double clsx import --- dash/src/components/TeamRadioMessage.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/dash/src/components/TeamRadioMessage.tsx b/dash/src/components/TeamRadioMessage.tsx index 13172d71..c2b92ce3 100644 --- a/dash/src/components/TeamRadioMessage.tsx +++ b/dash/src/components/TeamRadioMessage.tsx @@ -10,7 +10,6 @@ import PlayControls from "./PlayControls"; import AudioProgress from "./AudioProgress"; import { Driver, RadioCapture } from "@/types/state.type"; -import clsx from "clsx"; type Props = { driver: Driver; From 98ff2b38371497d8e8ae3ac0d9c540b8ab4c8dea Mon Sep 17 00:00:00 2001 From: Slowlydev Date: Sun, 20 Apr 2025 23:00:15 +0200 Subject: [PATCH 12/13] refactor: use the transcription store --- dash/src/components/TeamRadios.tsx | 46 +++++++++++++++--------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/dash/src/components/TeamRadios.tsx b/dash/src/components/TeamRadios.tsx index 43ea995a..acbc58bb 100644 --- a/dash/src/components/TeamRadios.tsx +++ b/dash/src/components/TeamRadios.tsx @@ -3,15 +3,17 @@ import { AnimatePresence } from "framer-motion"; import { utc } from "moment"; import clsx from "clsx"; +import { env } from "@/env.mjs"; + +import { RadioCapture } from "@/types/state.type"; + +import { useTranscriptionStore } from "@/stores/useTranscriptionStore"; import { useDataStore } from "@/stores/useDataStore"; +import { SAMPLING_RATE, DEFAULT_QUANTIZED } from "@/lib/constants"; import { sortUtc } from "@/lib/sorting"; import TeamRadioMessage from "@/components/TeamRadioMessage"; -import { TranscriptionSettings } from "@/context/ModeContext"; -import { SAMPLING_RATE, DEFAULT_QUANTIZED } from "@/lib/constants"; -import { env } from "@/env.mjs"; -import { RadioCapture } from "@/types/state.type"; type TranscriberCompleteData = { key: string; @@ -45,13 +47,13 @@ export default function TeamRadios() { const teamRadios = useDataStore((state) => state.teamRadio); const sessionPath = useDataStore((state) => state.sessionInfo?.path); + const transcription = useTranscriptionStore(); + const basePath = `https://livetiming.formula1.com/static/${sessionPath}`; const workerRef = useRef(null); - const [enableTranscription, setEnableTranscription] = useState(false); const [transcriptions, setTranscriptions] = useState<{ [key: string]: string }>({}); - const [transcriptionModel, setTranscriptionModel] = useState(""); const workerEventHandler = (event: MessageEvent) => { const message = event.data; @@ -77,40 +79,38 @@ export default function TeamRadios() { for (const teamRadio of teamRadios) { const audio = await loadAudioFromRadioCapture(audioContext, `${sessionPath}${teamRadio.path}`); + workerRef.current?.postMessage({ key: teamRadio.path, audio, - model: transcriptionModel, + model: transcription.model, multilingual: false, quantized: DEFAULT_QUANTIZED, subtask: null, language: null, }); + await new Promise((res) => setTimeout(res, 1000)); // To avoid rate limit } }; useEffect(() => { - if (typeof window != undefined) { - const worker = new Worker(new URL("../asr-worker.js", import.meta.url), { - type: "module", - }); - // Listen for messages from the Web Worker - worker.addEventListener("message", workerEventHandler); - workerRef.current = worker; + const worker = new Worker(new URL("../asr-worker.js", import.meta.url), { + type: "module", + }); - const transcriptionStorage = localStorage.getItem("transcription"); - const transcriptionSettings: TranscriptionSettings = transcriptionStorage - ? JSON.parse(transcriptionStorage) - : { enableTranscription: false, whisperModel: "" }; + worker.addEventListener("message", workerEventHandler); - setEnableTranscription(transcriptionSettings.enableTranscription); - setTranscriptionModel(transcriptionSettings.whisperModel); - } + workerRef.current = worker; + + return () => { + worker.terminate(); + workerRef.current = null; + }; }, []); useEffect(() => { - if (teamRadios && drivers && teamRadios.captures && enableTranscription && transcriptionModel) { + if (teamRadios && drivers && teamRadios.captures && transcription.enabled) { const targetRadios = teamRadios.captures.sort(sortUtc).slice(0, 20); setTranscriptions((oldTranscriptions) => { @@ -122,7 +122,7 @@ export default function TeamRadios() { return newTranscriptions; }); } - }, [teamRadios, drivers, enableTranscription, transcriptionModel]); + }, [teamRadios, drivers, transcription.enabled]); return (
      From 6dcd8a0ba97e283215635abe063116e3f9739372 Mon Sep 17 00:00:00 2001 From: Slowlydev Date: Sun, 20 Apr 2025 23:20:27 +0200 Subject: [PATCH 13/13] refactor(live): audio endpoint --- crates/live/src/server/audio.rs | 39 ++++++++++++++++----------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/crates/live/src/server/audio.rs b/crates/live/src/server/audio.rs index 20f7f98f..d86e1e0f 100644 --- a/crates/live/src/server/audio.rs +++ b/crates/live/src/server/audio.rs @@ -1,30 +1,29 @@ -use axum::{extract::Query, response::IntoResponse}; +use axum::{extract::Query, http::StatusCode, response::IntoResponse}; use serde::Deserialize; +use std::env; use tracing::error; #[derive(Deserialize)] - pub struct Params { - path: String + path: String, } -pub async fn get_audio(Query(params): Query) -> Result { - let Ok(_) = std::env::var("ENABLE_AUDIO_FETCH") else { return Err(axum::http::StatusCode::NOT_IMPLEMENTED) }; +pub async fn get_audio(Query(params): Query) -> Result { + let Ok(_) = env::var("ENABLE_AUDIO_FETCH") else { + return Err(StatusCode::NOT_IMPLEMENTED); + }; let audio_url = format!("https://livetiming.formula1.com/static/{}", params.path); - match reqwest::get(&audio_url).await { - Ok(response) => { - match response.bytes().await { - Ok(bytes) => Ok(bytes.as_ref().to_vec()), - Err(_) => { - error!("Failed to decode response from {}", audio_url); - Err(axum::http::StatusCode::INTERNAL_SERVER_ERROR) - } - } - } - Err(_) => { - error!("Failed to retrieve audio data from {}", audio_url); - Err(axum::http::StatusCode::INTERNAL_SERVER_ERROR) - } - } + + let Ok(response) = reqwest::get(&audio_url).await else { + error!("Failed to retrieve audio data from {}", audio_url); + return Err(StatusCode::INTERNAL_SERVER_ERROR); + }; + + let Ok(bytes) = response.bytes().await else { + error!("Failed to decode response from {}", audio_url); + return Err(StatusCode::INTERNAL_SERVER_ERROR); + }; + + Ok(bytes.as_ref().to_vec()) }