From fa10bfb1ca022e7be75b24d1c6ea242a06b85db5 Mon Sep 17 00:00:00 2001 From: Segergren Date: Tue, 17 Mar 2026 22:12:26 +0100 Subject: [PATCH 1/5] feat: add multi-track audio playback and per-selection track control --- Backend/Api/ContentServer.cs | 115 ++++++++++- Backend/App/MessageService.cs | 1 + Backend/App/Program.cs | 1 + Backend/Media/ClipService.cs | 67 ++++++- Backend/Media/FFmpegService.cs | 9 + Backend/Shared/FolderNames.cs | 10 + Frontend/src/Components/SelectionCard.tsx | 64 +++++- Frontend/src/Hooks/useAudioTracks.ts | 227 ++++++++++++++++++++++ Frontend/src/Models/types.ts | 4 + Frontend/src/Pages/video.tsx | 120 ++++++++++-- 10 files changed, 595 insertions(+), 23 deletions(-) create mode 100644 Frontend/src/Hooks/useAudioTracks.ts diff --git a/Backend/Api/ContentServer.cs b/Backend/Api/ContentServer.cs index 41d9fa92..29f25b7a 100644 --- a/Backend/Api/ContentServer.cs +++ b/Backend/Api/ContentServer.cs @@ -1,7 +1,10 @@ using Serilog; using System.Net; +using System.Text.Json; using System.Web; +using Segra.Backend.Core.Models; using Segra.Backend.Media; +using Segra.Backend.Shared; namespace Segra.Backend.Api { @@ -63,6 +66,10 @@ private static async Task ProcessRequestAsync(HttpListenerContext context) { await HandleThumbnailRequest(context); } + else if (rawUrl.StartsWith("/api/audiotracks")) + { + await HandleAudioTracksRequest(context); + } else if (rawUrl.StartsWith("/api/content")) { await HandleContentRequest(context); @@ -217,7 +224,8 @@ private static async Task HandleContentRequest(HttpListenerContext context) return; } - if (fileName.EndsWith(".mp4", StringComparison.OrdinalIgnoreCase)) + if (fileName.EndsWith(".mp4", StringComparison.OrdinalIgnoreCase) || + fileName.EndsWith(".m4a", StringComparison.OrdinalIgnoreCase)) { await StreamVideoFile(fileName, context); } @@ -236,6 +244,109 @@ private static async Task HandleContentRequest(HttpListenerContext context) } } + private static async Task HandleAudioTracksRequest(HttpListenerContext context) + { + var query = HttpUtility.ParseQueryString(context.Request?.Url?.Query ?? ""); + string input = query["input"] ?? ""; + string typeStr = query["type"] ?? ""; + var response = context.Response; + + response.AddHeader("Access-Control-Allow-Origin", "*"); + + if (!File.Exists(input)) + { + response.StatusCode = (int)HttpStatusCode.NotFound; + response.ContentType = "text/plain"; + using (var writer = new StreamWriter(response.OutputStream)) + { + await writer.WriteAsync("File not found."); + } + return; + } + + try + { + if (!Enum.TryParse(typeStr, true, out var contentType)) + { + contentType = FolderNames.GetContentTypeFromPath(input) ?? Content.ContentType.Session; + } + + // Read metadata to get audio track names + string fileName = Path.GetFileNameWithoutExtension(input); + string metadataFolderPath = FolderNames.GetMetadataFolderPath(contentType); + string metadataFilePath = Path.Combine(metadataFolderPath, $"{fileName}.json"); + + List? trackNames = null; + if (File.Exists(metadataFilePath)) + { + var metadataContent = await File.ReadAllTextAsync(metadataFilePath); + var metadata = JsonSerializer.Deserialize(metadataContent); + trackNames = metadata?.AudioTrackNames; + } + + if (trackNames == null || trackNames.Count <= 1) + { + response.StatusCode = (int)HttpStatusCode.OK; + response.ContentType = "application/json"; + using (var writer = new StreamWriter(response.OutputStream)) + { + await writer.WriteAsync("[]"); + } + return; + } + + string audioTracksDir = FolderNames.GetAudioTracksFolderPath(contentType); + Directory.CreateDirectory(audioTracksDir); + + var result = new List(); + + // Skip track 0 (Full Mix) - it duplicates the individual tracks + for (int i = 1; i < trackNames.Count; i++) + { + string trackFileName = $"{fileName}_track{i}.m4a"; + string trackFilePath = Path.Combine(audioTracksDir, trackFileName); + + if (!File.Exists(trackFilePath)) + { + try + { + await FFmpegService.ExtractAudioTrack(input, trackFilePath, i); + } + catch (Exception ex) + { + Log.Warning($"Failed to extract audio track {i} from {input}: {ex.Message}"); + continue; + } + } + + result.Add(new + { + index = i, + name = trackNames[i], + url = $"/api/content?input={Uri.EscapeDataString(trackFilePath)}" + }); + } + + response.StatusCode = (int)HttpStatusCode.OK; + response.ContentType = "application/json"; + string json = JsonSerializer.Serialize(result); + using (var writer = new StreamWriter(response.OutputStream)) + { + await writer.WriteAsync(json); + } + } + catch (Exception ex) + { + Log.Error(ex, "Error handling audio tracks request"); + response.StatusCode = (int)HttpStatusCode.InternalServerError; + response.ContentType = "text/plain"; + using (var writer = new StreamWriter(response.OutputStream)) + { + await writer.WriteAsync("Error extracting audio tracks."); + } + } + } + private static async Task StreamVideoFile(string fileName, HttpListenerContext context) { var response = context.Response; @@ -273,7 +384,7 @@ private static async Task StreamVideoFile(string fileName, HttpListenerContext c long contentLength = end - start + 1; response.StatusCode = string.IsNullOrEmpty(rangeHeader) ? (int)HttpStatusCode.OK : (int)HttpStatusCode.PartialContent; - response.ContentType = "video/mp4"; + response.ContentType = fileName.EndsWith(".m4a", StringComparison.OrdinalIgnoreCase) ? "audio/mp4" : "video/mp4"; response.AddHeader("Accept-Ranges", "bytes"); if (!string.IsNullOrEmpty(rangeHeader)) diff --git a/Backend/App/MessageService.cs b/Backend/App/MessageService.cs index 4694eba1..9e701d9c 100644 --- a/Backend/App/MessageService.cs +++ b/Backend/App/MessageService.cs @@ -25,6 +25,7 @@ public class Selection public required string Game { get; set; } public string Title { get; set; } = string.Empty; public int? IgdbId { get; set; } + public List? MutedAudioTracks { get; set; } } public static class MessageService diff --git a/Backend/App/Program.cs b/Backend/App/Program.cs index eabcd3d4..473510ea 100644 --- a/Backend/App/Program.cs +++ b/Backend/App/Program.cs @@ -445,6 +445,7 @@ private static void LoadFrontend() Log.Information("Loading frontend, app url is " + appUrl); // Initialize the PhotinoWindow Window = new PhotinoWindow() + .SetBrowserControlInitParameters("--enable-blink-features=AudioVideoTracks") .SetNotificationsEnabled(false) // Disabled due to it creating a second start menu entry with incorrect start path. See https://github.com/tryphotino/photino.NET/issues/85 .SetUseOsDefaultSize(false) .SetIconFile(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "icon.ico")) diff --git a/Backend/Media/ClipService.cs b/Backend/Media/ClipService.cs index d727773b..3926d3a8 100644 --- a/Backend/Media/ClipService.cs +++ b/Backend/Media/ClipService.cs @@ -58,9 +58,10 @@ public static async Task CreateClips(List selections) return; } - // Read source audio track names for embedding in clip metadata + // Read source audio track names when keeping separate tracks or when any selection has muted tracks List? sourceAudioTrackNames = null; - if (Settings.Instance.ClipKeepSeparateAudioTracks && firstSelection != null) + bool anySelectionHasMutedTracks = selections.Any(s => s.MutedAudioTracks != null && s.MutedAudioTracks.Count > 0); + if ((Settings.Instance.ClipKeepSeparateAudioTracks || anySelectionHasMutedTracks) && firstSelection != null) { sourceAudioTrackNames = GetSourceAudioTrackNames(firstSelection); } @@ -90,7 +91,7 @@ public static async Task CreateClips(List selections) string tempFileName = Path.Combine(Path.GetTempPath(), $"clip{Guid.NewGuid()}.mp4"); double clipDuration = selection.EndTime - selection.StartTime; - await ExtractClip(id, inputFilePath, tempFileName, selection.StartTime, selection.EndTime, sourceAudioTrackNames, progress => + await ExtractClip(id, inputFilePath, tempFileName, selection.StartTime, selection.EndTime, sourceAudioTrackNames, selection.MutedAudioTracks, progress => { double clampedProgress = Math.Min(progress, 1.0); double currentProgress = (processedDuration + (clampedProgress * clipDuration)) / totalDuration * 95; @@ -205,7 +206,7 @@ await FFmpegService.RunWithProgress(id, } private static async Task ExtractClip(int clipId, string inputFilePath, string outputFilePath, double startTime, double endTime, - List? audioTrackNames, Action progressCallback) + List? audioTrackNames, List? mutedAudioTracks, Action progressCallback) { double duration = endTime - startTime; var settings = Settings.Instance; @@ -288,12 +289,62 @@ private static async Task ExtractClip(int clipId, string inputFilePath, string o } string fpsArg = settings.ClipFps > 0 ? $"-r {settings.ClipFps}" : ""; - string mapArgs = settings.ClipKeepSeparateAudioTracks ? "-map 0:v:0 -map 0:a " : ""; - // Set audio track title metadata so editing software shows the correct names + // Build audio mapping, filter, and metadata based on per-selection muted tracks + string mapArgs = ""; string metadataArgs = ""; - if (settings.ClipKeepSeparateAudioTracks && audioTrackNames != null) + string filterArgs = ""; + + // Determine which individual tracks (index > 0) are not muted + bool hasMutedTracks = mutedAudioTracks != null && mutedAudioTracks.Count > 0 && audioTrackNames != null && audioTrackNames.Count > 1; + + if (hasMutedTracks) + { + // Build list of non-muted individual track indices (skip track 0 = original Full Mix) + var enabledTracks = new List(); + for (int i = 1; i < audioTrackNames!.Count; i++) + { + if (!mutedAudioTracks!.Contains(i)) + enabledTracks.Add(i); + } + + if (enabledTracks.Count > 0) + { + // Create a new Full Mix from only the non-muted individual tracks + if (enabledTracks.Count == 1) + { + // Single track: use it directly as the mix, no filter needed + filterArgs = ""; + mapArgs = $"-map 0:v:0 -map 0:a:{enabledTracks[0]} "; + } + else + { + // Multiple tracks: use amix filter to create a new Full Mix + string filterInputs = string.Join("", enabledTracks.Select(i => $"[0:a:{i}]")); + filterArgs = $"-filter_complex \"{filterInputs}amix=inputs={enabledTracks.Count}:duration=longest[mix]\" "; + mapArgs = "-map 0:v:0 -map \"[mix]\" "; + } + + int outputTrackIndex = 0; + metadataArgs = $"-metadata:s:a:{outputTrackIndex} title=\"Full Mix\" "; + outputTrackIndex++; + + // When keeping separate tracks, also map each individual non-muted track + if (settings.ClipKeepSeparateAudioTracks) + { + foreach (int i in enabledTracks) + { + mapArgs += $"-map 0:a:{i} "; + metadataArgs += $"-metadata:s:a:{outputTrackIndex} title=\"{audioTrackNames[i]}\" "; + outputTrackIndex++; + } + } + } + } + else if (settings.ClipKeepSeparateAudioTracks && audioTrackNames != null) { + // No muted tracks, keep all audio tracks as-is + mapArgs = "-map 0:v:0 -map 0:a "; for (int i = 0; i < audioTrackNames.Count; i++) { metadataArgs += $"-metadata:s:a:{i} title=\"{audioTrackNames[i]}\" "; @@ -301,7 +352,7 @@ private static async Task ExtractClip(int clipId, string inputFilePath, string o } string arguments = $"-y -ss {startTime.ToString(CultureInfo.InvariantCulture)} -t {duration.ToString(CultureInfo.InvariantCulture)} " + - $"-i \"{inputFilePath}\" {mapArgs}-c:v {videoCodec} {presetArgs} {qualityArgs} {fpsArg} " + + $"-i \"{inputFilePath}\" {filterArgs}{mapArgs}-c:v {videoCodec} {presetArgs} {qualityArgs} {fpsArg} " + $"-c:a aac -b:a {settings.ClipAudioQuality} {metadataArgs}-movflags +faststart \"{outputFilePath}\""; Log.Information("Extracting clip"); Log.Information($"FFmpeg arguments: {arguments}"); diff --git a/Backend/Media/FFmpegService.cs b/Backend/Media/FFmpegService.cs index 08962da3..91f15874 100644 --- a/Backend/Media/FFmpegService.cs +++ b/Backend/Media/FFmpegService.cs @@ -347,6 +347,15 @@ public static async Task CreateThumbnailFile(string inputFilePath, string output await RunSimple(arguments); } + /// + /// Extracts a single audio track from a video file to an M4A file (copy, no re-encoding) + /// + public static async Task ExtractAudioTrack(string inputFilePath, string outputM4aPath, int audioStreamIndex) + { + string arguments = $"-y -i \"{inputFilePath}\" -map 0:a:{audioStreamIndex} -c copy \"{outputM4aPath}\""; + await RunSimple(arguments); + } + /// /// Extracts audio as PCM data for waveform generation /// diff --git a/Backend/Shared/FolderNames.cs b/Backend/Shared/FolderNames.cs index 0ff372e8..d6a208a5 100644 --- a/Backend/Shared/FolderNames.cs +++ b/Backend/Shared/FolderNames.cs @@ -24,6 +24,7 @@ public static class FolderNames public const string Metadata = "metadata"; public const string Thumbnails = "thumbnails"; public const string Waveforms = "waveforms"; + public const string AudioTracks = "audiotracks"; // Legacy metadata folder names (with dot prefix, for migration) public const string LegacyMetadata = ".metadata"; @@ -118,6 +119,15 @@ public static string GetWaveformsFolderPath(Content.ContentType type) return Path.Combine(CacheFolder, Waveforms, GetMetadataSubfolderName(type)); } + /// + /// Gets the full path to the audio tracks folder for a content type. + /// Audio tracks are stored in AppData/Roaming/Segra/audiotracks/{ContentType} + /// + public static string GetAudioTracksFolderPath(Content.ContentType type) + { + return Path.Combine(CacheFolder, AudioTracks, GetMetadataSubfolderName(type)); + } + /// /// Checks if a path contains a specific content type folder. /// Useful for determining content type from a file path. diff --git a/Frontend/src/Components/SelectionCard.tsx b/Frontend/src/Components/SelectionCard.tsx index 938d0aba..5865f14b 100644 --- a/Frontend/src/Components/SelectionCard.tsx +++ b/Frontend/src/Components/SelectionCard.tsx @@ -1,6 +1,7 @@ -import React from 'react'; +import React, { useState } from 'react'; import { SelectionCardProps } from '../Models/types'; import { useDrag, useDrop } from 'react-dnd'; +import { TbHeadphones } from 'react-icons/tb'; const DRAG_TYPE = 'SELECTION_CARD'; @@ -13,7 +14,11 @@ const SelectionCard: React.FC = React.memo( isHovered, setHoveredSelectionId, removeSelection, + audioTrackNames, + onMutedAudioTracksChange, }) => { + const [showAudioMenu, setShowAudioMenu] = useState(false); + const [{ isDragging }, dragRef] = useDrag( () => ({ type: DRAG_TYPE, @@ -44,6 +49,20 @@ const SelectionCard: React.FC = React.memo( }; const { startTime, endTime, thumbnailDataUrl, isLoading } = selection; + const hasAudioTracks = + audioTrackNames && audioTrackNames.length > 1 && onMutedAudioTracksChange; + const mutedTracks = selection.mutedAudioTracks ?? []; + + const toggleTrack = (trackIndex: number) => { + if (!onMutedAudioTracksChange) return; + const isMuted = mutedTracks.includes(trackIndex); + const newMuted = isMuted + ? mutedTracks.filter((t) => t !== trackIndex) + : [...mutedTracks, trackIndex]; + onMutedAudioTracksChange(selection.id, newMuted); + }; + + const hasMutedTracks = mutedTracks.length > 0; return (
= React.memo( className={`mb-2 cursor-move w-full relative rounded-xl transition-all duration-200 !outline !outline-1 ${isHovered ? '!outline-primary' : '!outline-base-400'}`} style={{ opacity: isDragging ? 0.3 : 1 }} onMouseEnter={() => setHoveredSelectionId(selection.id)} - onMouseLeave={() => setHoveredSelectionId(null)} + onMouseLeave={() => { + setHoveredSelectionId(null); + setShowAudioMenu(false); + }} onContextMenu={(e) => { e.preventDefault(); removeSelection(selection.id); @@ -76,6 +98,44 @@ const SelectionCard: React.FC = React.memo( No thumbnail
)} + + {hasAudioTracks && ( +
+ + {showAudioMenu && ( +
e.stopPropagation()} + > + {audioTrackNames.map((name, i) => { + // Skip track 0 (Full Mix) + if (i === 0) return null; + const isMuted = mutedTracks.includes(i); + return ( + + ); + })} +
+ )} +
+ )} ); }, diff --git a/Frontend/src/Hooks/useAudioTracks.ts b/Frontend/src/Hooks/useAudioTracks.ts new file mode 100644 index 00000000..8576271a --- /dev/null +++ b/Frontend/src/Hooks/useAudioTracks.ts @@ -0,0 +1,227 @@ +import { useEffect, useRef, useState, useCallback } from 'react'; +import { Content } from '../Models/types'; + +export interface AudioTrackInfo { + index: number; + name: string; + url: string; +} + +export interface AudioTrackState { + tracks: AudioTrackInfo[]; + volumes: Record; + mutedTracks: Set; + soloTrack: number | null; + setTrackVolume: (index: number, volume: number) => void; + toggleTrackMute: (index: number) => void; + setMutedTracks: (muted: Set) => void; + toggleSolo: (index: number) => void; + isMultiTrack: boolean; +} + +const SYNC_THRESHOLD = 0.15; +const SYNC_CHECK_INTERVAL = 500; + +function cleanupAudioElements(elements: Map) { + for (const audio of elements.values()) { + audio.pause(); + audio.src = ''; + audio.load(); + } + elements.clear(); +} + +export function useAudioTracks( + videoRef: React.RefObject, + video: Content, +): AudioTrackState { + const audioElementsRef = useRef>(new Map()); + const [tracks, setTracks] = useState([]); + const [volumes, setVolumes] = useState>({}); + const [mutedTracks, setMutedTracks] = useState>(new Set()); + const [soloTrack, setSoloTrack] = useState(null); + const isMultiTrack = tracks.length > 1; + + // Fetch extracted audio track files from backend + useEffect(() => { + if (!video.audioTrackNames || video.audioTrackNames.length <= 1) { + setTracks([]); + return; + } + + let cancelled = false; + + const fetchTracks = async () => { + try { + const url = `http://localhost:2222/api/audiotracks?input=${encodeURIComponent(video.filePath)}&type=${video.type}`; + const res = await fetch(url); + const data: AudioTrackInfo[] = await res.json(); + + // Exclude track 0 (Full Mix) since it doubles individual tracks + const individualTracks = data.filter((t) => t.index > 0); + if (cancelled || individualTracks.length === 0) return; + + setTracks(individualTracks); + + const initialVolumes: Record = {}; + for (const t of individualTracks) { + initialVolumes[t.index] = 1; + } + setVolumes(initialVolumes); + setMutedTracks(new Set()); + setSoloTrack(null); + + // Clean up existing elements before creating new ones + cleanupAudioElements(audioElementsRef.current); + + const elements = new Map(); + for (const track of individualTracks) { + const audio = new Audio(); + audio.src = `http://localhost:2222${track.url}`; + audio.preload = 'auto'; + audio.volume = initialVolumes[track.index] ?? 1; + elements.set(track.index, audio); + } + audioElementsRef.current = elements; + } catch (err) { + console.error('Failed to load audio tracks:', err); + } + }; + + fetchTracks(); + + return () => { + cancelled = true; + cleanupAudioElements(audioElementsRef.current); + setTracks([]); + }; + }, [video.filePath, video.audioTrackNames?.length]); + + // Sync audio elements with video: play/pause/seek/rate + mute video + useEffect(() => { + const vid = videoRef.current; + if (!vid || !isMultiTrack) return; + + // Mute the video element here (not earlier) so that video.tsx's + // initialization effect has already seen isMultiTrack=true and + // skipped its own vid.muted = isMuted assignment. + vid.muted = true; + + const syncTime = () => { + const t = vid.currentTime; + for (const audio of audioElementsRef.current.values()) { + audio.currentTime = t; + } + }; + + const onPlay = () => { + syncTime(); + for (const audio of audioElementsRef.current.values()) { + audio.play().catch(() => {}); + } + }; + + const onPause = () => { + for (const audio of audioElementsRef.current.values()) { + audio.pause(); + } + }; + + const onSeeked = () => { + syncTime(); + }; + + const onRateChange = () => { + const rate = vid.playbackRate; + for (const audio of audioElementsRef.current.values()) { + audio.playbackRate = rate; + } + }; + + vid.addEventListener('play', onPlay); + vid.addEventListener('pause', onPause); + vid.addEventListener('seeked', onSeeked); + vid.addEventListener('ratechange', onRateChange); + + // Initial sync if already playing + if (!vid.paused) { + for (const audio of audioElementsRef.current.values()) { + audio.playbackRate = vid.playbackRate; + audio.currentTime = vid.currentTime; + audio.play().catch(() => {}); + } + } + + // Periodic drift correction + const intervalId = setInterval(() => { + if (vid.paused) return; + const vidTime = vid.currentTime; + for (const audio of audioElementsRef.current.values()) { + const drift = Math.abs(audio.currentTime - vidTime); + if (drift > SYNC_THRESHOLD) { + audio.currentTime = vidTime; + } + } + }, SYNC_CHECK_INTERVAL); + + return () => { + vid.removeEventListener('play', onPlay); + vid.removeEventListener('pause', onPause); + vid.removeEventListener('seeked', onSeeked); + vid.removeEventListener('ratechange', onRateChange); + clearInterval(intervalId); + + // Restore video audio when multi-track is deactivated + vid.muted = localStorage.getItem('segra-muted') === 'true'; + }; + }, [videoRef, isMultiTrack]); + + // Apply volume/mute/solo to audio elements + useEffect(() => { + for (const [index, audio] of audioElementsRef.current.entries()) { + if (soloTrack !== null) { + audio.muted = index !== soloTrack; + } else { + audio.muted = mutedTracks.has(index); + } + audio.volume = volumes[index] ?? 1; + } + }, [volumes, mutedTracks, soloTrack, tracks]); + + const setTrackVolume = useCallback((index: number, volume: number) => { + const clamped = Math.max(0, Math.min(1, volume)); + setVolumes((prev) => ({ ...prev, [index]: clamped })); + }, []); + + const toggleTrackMute = useCallback((index: number) => { + setMutedTracks((prev) => { + const next = new Set(prev); + if (next.has(index)) { + next.delete(index); + } else { + next.add(index); + } + return next; + }); + }, []); + + const replaceMutedTracks = useCallback((muted: Set) => { + setMutedTracks(muted); + }, []); + + const toggleSolo = useCallback((index: number) => { + setSoloTrack((prev) => (prev === index ? null : index)); + }, []); + + return { + tracks, + volumes, + mutedTracks, + soloTrack, + setTrackVolume, + toggleTrackMute, + setMutedTracks: replaceMutedTracks, + toggleSolo, + isMultiTrack, + }; +} diff --git a/Frontend/src/Models/types.ts b/Frontend/src/Models/types.ts index fa2042d5..efc4cc59 100644 --- a/Frontend/src/Models/types.ts +++ b/Frontend/src/Models/types.ts @@ -20,6 +20,7 @@ export interface Content { uploadId?: string; igdbId?: number; isImported: boolean; + audioTrackNames?: string[]; } export interface OBSVersion { @@ -349,6 +350,7 @@ export interface Selection { game?: string; title?: string; igdbId?: number; + mutedAudioTracks?: number[]; } export interface SelectionCardProps { @@ -359,6 +361,8 @@ export interface SelectionCardProps { isHovered: boolean; setHoveredSelectionId: (id: number | null) => void; removeSelection: (id: number) => void; + audioTrackNames?: string[]; + onMutedAudioTracksChange?: (id: number, mutedTracks: number[]) => void; } export interface AiProgress { diff --git a/Frontend/src/Pages/video.tsx b/Frontend/src/Pages/video.tsx index 17c8e81f..51311814 100644 --- a/Frontend/src/Pages/video.tsx +++ b/Frontend/src/Pages/video.tsx @@ -35,7 +35,8 @@ import { import { IoSkull, IoAdd, IoRemove } from 'react-icons/io5'; import SelectionCard from '../Components/SelectionCard'; -import { TbZoomIn, TbZoomOut } from 'react-icons/tb'; +import { TbZoomIn, TbZoomOut, TbHeadphones } from 'react-icons/tb'; +import { useAudioTracks } from '../Hooks/useAudioTracks'; import { IoIosFootball } from 'react-icons/io'; import { AnimatePresence, motion } from 'framer-motion'; import Button from '../Components/Button'; @@ -211,6 +212,10 @@ export default function VideoComponent({ video }: { video: Content }) { const waveformStateRef = useRef({ pixelsPerSecond: 0, duration: 0 }); const waveformBufferRef = useRef({ regionLeft: 0, regionRight: 0 }); + // Audio tracks + const audioTracks = useAudioTracks(videoRef, video); + const [showAudioTracks, setShowAudioTracks] = useState(false); + // Video state const [currentTime, setCurrentTime] = useState(0); const [duration, setDuration] = useState(0); @@ -455,8 +460,11 @@ export default function VideoComponent({ video }: { video: Content }) { if (!vid) return; // Apply saved volume and muted state on load - vid.volume = volume; - vid.muted = isMuted; + // When multi-track is active, the hook controls muting + if (!audioTracks.isMultiTrack) { + vid.volume = volume; + vid.muted = isMuted; + } // Apply saved playback rate vid.playbackRate = playbackRate; @@ -469,6 +477,9 @@ export default function VideoComponent({ video }: { video: Content }) { const onPause = () => setIsPlaying(false); const onVolumeChange = () => { if (vid) { + // When multi-track audio is active, the video is muted by the hook + if (audioTracks.isMultiTrack) return; + setVolume(vid.volume); setIsMuted(vid.muted); @@ -569,7 +580,7 @@ export default function VideoComponent({ video }: { video: Content }) { vid.removeEventListener('ratechange', onRateChange); window.removeEventListener('keydown', handleKeyDown, keyOptions as any); }; - }, [volume, isMuted, isFullscreen]); + }, [volume, isMuted, isFullscreen, audioTracks.isMultiTrack]); // Handle video playback time updates using requestAnimationFrame for smooth updates useEffect(() => { @@ -598,6 +609,45 @@ export default function VideoComponent({ video }: { video: Content }) { }; }, []); + // Apply per-selection audio track muting during playback + const activeSelectionIdRef = useRef(null); + const savedMuteStateRef = useRef | null>(null); + useEffect(() => { + if (!audioTracks.isMultiTrack) return; + + const checkInterval = setInterval(() => { + const vid = videoRef.current; + if (!vid || vid.paused) return; + + const t = vid.currentTime; + const activeSelection = selections.find( + (s) => + s.mutedAudioTracks && s.mutedAudioTracks.length > 0 && t >= s.startTime && t <= s.endTime, + ); + + const activeId = activeSelection?.id ?? null; + if (activeId === activeSelectionIdRef.current) return; + + if (activeId !== null && activeSelectionIdRef.current === null) { + // Entering a selection with muted tracks -- save current state + savedMuteStateRef.current = new Set(audioTracks.mutedTracks); + } + + activeSelectionIdRef.current = activeId; + + if (activeSelection?.mutedAudioTracks) { + // Apply this selection's muted tracks + audioTracks.setMutedTracks(new Set(activeSelection.mutedAudioTracks)); + } else if (savedMuteStateRef.current !== null) { + // Exiting selection -- restore saved state + audioTracks.setMutedTracks(savedMuteStateRef.current); + savedMuteStateRef.current = null; + } + }, 100); + + return () => clearInterval(checkInterval); + }, [audioTracks.isMultiTrack, selections]); + // Update container width on window resize useEffect(() => { if (scrollContainerRef.current) { @@ -794,13 +844,16 @@ export default function VideoComponent({ video }: { video: Content }) { setVolume(target); return; } - el.volume = target; - if (target === 0) { - el.muted = true; - setIsMuted(true); - } else if (el.muted) { - el.muted = false; - setIsMuted(false); + // When multi-track is active, don't touch the video element's volume/muted + if (!audioTracks.isMultiTrack) { + el.volume = target; + if (target === 0) { + el.muted = true; + setIsMuted(true); + } else if (el.muted) { + el.muted = false; + setIsMuted(false); + } } setVolume(target); localStorage.setItem('segra-volume', target.toString()); @@ -985,6 +1038,7 @@ export default function VideoComponent({ video }: { video: Content }) { game: video.game, title: video.title, igdbId: video.igdbId, + mutedAudioTracks: audioTracks.isMultiTrack ? [...audioTracks.mutedTracks] : undefined, }; addSelection(newSelection); // Kick off thumbnail generation; uses latest state and guards against stale overwrites @@ -1010,6 +1064,7 @@ export default function VideoComponent({ video }: { video: Content }) { startTime: s.startTime, endTime: s.endTime, igdbId: s.igdbId, + mutedAudioTracks: s.mutedAudioTracks, })), }; sendMessageToBackend('CreateClip', params); @@ -1417,6 +1472,9 @@ export default function VideoComponent({ video }: { video: Content }) { // Toggle mute state const toggleMute = () => { if (videoRef.current) { + // When multi-track is active, don't toggle the video element's muted state + if (audioTracks.isMultiTrack) return; + const newMutedState = !videoRef.current.muted; videoRef.current.muted = newMutedState; setIsMuted(newMutedState); @@ -1547,6 +1605,39 @@ export default function VideoComponent({ video }: { video: Content }) { /> + {audioTracks.isMultiTrack && ( +
+ + {showAudioTracks && ( +
+ {audioTracks.tracks.map((track) => { + const isMuted = audioTracks.mutedTracks.has(track.index); + return ( + + ); + })} +
+ )} +
+ )} +
From 9e206b7dd1de1af185f367ac45063f19d13139df Mon Sep 17 00:00:00 2001 From: Segergren Date: Wed, 18 Mar 2026 19:45:17 +0100 Subject: [PATCH 2/5] feat: add per-selection audio track control for clip creation - Add MutedAudioTracks to Selection model (backend + frontend) - Build union audio layout across selections so all temp clips have identical stream layout for concat - Use lavfi inputs for silence to avoid FFmpeg filtergraph deadlock when mixing real and synthetic streams - Add headphones icon button per SelectionCard with per-track checkbox dropdown - Capture current mute state when adding a selection - Fix concat step dropping extra audio streams (add -map 0 and metadata args) - Fix selection card drag reorder instability by using refs in useDrop --- Backend/App/MessageService.cs | 9 +- Backend/App/Program.cs | 11 ++ Backend/Media/ClipService.cs | 228 +++++++++++++++++----- Frontend/src/Components/SelectionCard.tsx | 17 +- Frontend/src/Pages/video.tsx | 90 ++++++++- 5 files changed, 301 insertions(+), 54 deletions(-) diff --git a/Backend/App/MessageService.cs b/Backend/App/MessageService.cs index 9e701d9c..d93cb6d7 100644 --- a/Backend/App/MessageService.cs +++ b/Backend/App/MessageService.cs @@ -358,6 +358,12 @@ private static async Task HandleCreateClip(JsonElement message) string? filePath = selectionElement.TryGetProperty("filePath", out JsonElement filePathElement) ? filePathElement.GetString() : null; + List? mutedAudioTracks = null; + if (selectionElement.TryGetProperty("mutedAudioTracks", out JsonElement mutedEl) + && mutedEl.ValueKind == JsonValueKind.Array) + { + mutedAudioTracks = mutedEl.EnumerateArray().Select(e => e.GetInt32()).ToList(); + } // Create a new Selection instance with all required properties. selections.Add(new Selection @@ -370,7 +376,8 @@ private static async Task HandleCreateClip(JsonElement message) FilePath = filePath, Game = game, Title = title, - IgdbId = igdbId + IgdbId = igdbId, + MutedAudioTracks = mutedAudioTracks }); } } diff --git a/Backend/App/Program.cs b/Backend/App/Program.cs index 473510ea..33acb217 100644 --- a/Backend/App/Program.cs +++ b/Backend/App/Program.cs @@ -217,6 +217,17 @@ static void Main(string[] args) Directory.CreateDirectory(Settings.Instance.ContentFolder); } + // Clean up cached audio track files from previous session + Task.Run(() => + { + string audioTracksCacheFolder = Path.Combine(Settings.Instance.CacheFolder, FolderNames.AudioTracks); + if (Directory.Exists(audioTracksCacheFolder)) + { + try { Directory.Delete(audioTracksCacheFolder, true); } + catch (Exception ex) { Log.Warning($"Failed to clean up audio tracks cache: {ex.Message}"); } + } + }); + // Run data migrations Task.Run(MigrationService.RunMigrations); diff --git a/Backend/Media/ClipService.cs b/Backend/Media/ClipService.cs index 3926d3a8..477d78e5 100644 --- a/Backend/Media/ClipService.cs +++ b/Backend/Media/ClipService.cs @@ -58,15 +58,26 @@ public static async Task CreateClips(List selections) return; } - // Read source audio track names when keeping separate tracks or when any selection has muted tracks - List? sourceAudioTrackNames = null; + // Read per-selection audio track names and build union layout bool anySelectionHasMutedTracks = selections.Any(s => s.MutedAudioTracks != null && s.MutedAudioTracks.Count > 0); - if ((Settings.Instance.ClipKeepSeparateAudioTracks || anySelectionHasMutedTracks) && firstSelection != null) + var perSelectionTrackNames = new List?>(); + if (Settings.Instance.ClipKeepSeparateAudioTracks || anySelectionHasMutedTracks) { - sourceAudioTrackNames = GetSourceAudioTrackNames(firstSelection); + foreach (var sel in selections) + perSelectionTrackNames.Add(GetSourceAudioTrackNames(sel)); } + else + { + perSelectionTrackNames.AddRange(Enumerable.Repeat?>(null, selections.Count)); + } + + // Union of all track names across sources -- used to normalise every temp clip to the same stream layout + List? unionAudioLayout = Settings.Instance.ClipKeepSeparateAudioTracks + ? BuildUnionAudioLayout(perSelectionTrackNames) + : null; double processedDuration = 0; + int selectionIndex = 0; foreach (var selection in selections) { // Use the actual file path from metadata when available, fall back to reconstructed path @@ -85,13 +96,17 @@ public static async Task CreateClips(List selections) if (!File.Exists(inputFilePath)) { Log.Information($"Input video file not found: {inputFilePath}"); + selectionIndex++; continue; } string tempFileName = Path.Combine(Path.GetTempPath(), $"clip{Guid.NewGuid()}.mp4"); double clipDuration = selection.EndTime - selection.StartTime; - await ExtractClip(id, inputFilePath, tempFileName, selection.StartTime, selection.EndTime, sourceAudioTrackNames, selection.MutedAudioTracks, progress => + List? selectionTrackNames = perSelectionTrackNames[selectionIndex]; + List? targetLayout = Settings.Instance.ClipKeepSeparateAudioTracks ? unionAudioLayout : null; + + await ExtractClip(id, inputFilePath, tempFileName, selection.StartTime, selection.EndTime, selectionTrackNames, selection.MutedAudioTracks, targetLayout, progress => { double clampedProgress = Math.Min(progress, 1.0); double currentProgress = (processedDuration + (clampedProgress * clipDuration)) / totalDuration * 95; @@ -107,6 +122,7 @@ await ExtractClip(id, inputFilePath, tempFileName, selection.StartTime, selectio processedDuration += clipDuration; tempClipFiles.Add(tempFileName); + selectionIndex++; } if (!tempClipFiles.Any()) @@ -133,8 +149,14 @@ await ExtractClip(id, inputFilePath, tempFileName, selection.StartTime, selectio try { + string mapAllArg = unionAudioLayout != null ? "-map 0 " : ""; + string concatMetadataArgs = ""; + if (unionAudioLayout != null) + { + concatMetadataArgs = string.Join(" ", unionAudioLayout.Select((name, i) => $"-metadata:s:a:{i} title=\"{name}\"")) + " "; + } await FFmpegService.RunWithProgress(id, - $"-y -f concat -safe 0 -i \"{concatFilePath}\" -c copy -movflags +faststart \"{outputFilePath}\"", + $"-y -f concat -safe 0 -i \"{concatFilePath}\" {mapAllArg}-c copy {concatMetadataArgs}-movflags +faststart \"{outputFilePath}\"", totalDuration, progress => { }, process => @@ -171,7 +193,7 @@ await FFmpegService.RunWithProgress(id, _ = MessageService.SendFrontendMessage("ClipProgress", new { id, progress = 98, selections }); - await ContentService.CreateMetadataFile(outputFilePath, Content.ContentType.Clip, firstSelection?.Game!, null, firstSelection?.Title, igdbId: firstSelection?.IgdbId, audioTrackNames: sourceAudioTrackNames); + await ContentService.CreateMetadataFile(outputFilePath, Content.ContentType.Clip, firstSelection?.Game!, null, firstSelection?.Title, igdbId: firstSelection?.IgdbId, audioTrackNames: unionAudioLayout); await ContentService.CreateThumbnail(outputFilePath, Content.ContentType.Clip); await ContentService.CreateWaveformFile(outputFilePath, Content.ContentType.Clip); @@ -206,7 +228,7 @@ await FFmpegService.RunWithProgress(id, } private static async Task ExtractClip(int clipId, string inputFilePath, string outputFilePath, double startTime, double endTime, - List? audioTrackNames, List? mutedAudioTracks, Action progressCallback) + List? audioTrackNames, List? mutedAudioTracks, List? targetAudioLayout, Action progressCallback) { double duration = endTime - startTime; var settings = Settings.Instance; @@ -294,66 +316,165 @@ private static async Task ExtractClip(int clipId, string inputFilePath, string o string mapArgs = ""; string metadataArgs = ""; string filterArgs = ""; + string extraInputArgs = ""; - // Determine which individual tracks (index > 0) are not muted - bool hasMutedTracks = mutedAudioTracks != null && mutedAudioTracks.Count > 0 && audioTrackNames != null && audioTrackNames.Count > 1; - - if (hasMutedTracks) + if (targetAudioLayout != null && targetAudioLayout.Count > 0) { - // Build list of non-muted individual track indices (skip track 0 = original Full Mix) - var enabledTracks = new List(); - for (int i = 1; i < audioTrackNames!.Count; i++) + // Normalise output to the union layout so every temp clip has identical stream layout. + // Silent (muted/missing) tracks use separate -f lavfi inputs so they are never mixed + // with real decoded streams inside filter_complex -- mixing synthetic sources and real + // streams in the same filtergraph causes a scheduler deadlock in FFmpeg. + var filterParts = new List(); + var mapParts = new List { "-map 0:v:0" }; + var metaParts = new List(); + var extraInputParts = new List(); + int silenceInputIdx = 1; // lavfi inputs start at 1 (0 is the main file) + + bool sourceHasIndividualTracks = audioTrackNames != null && audioTrackNames.Count > 1; + + // Enabled individual source tracks (index > 0, not muted) + var enabledSourceTracks = new List(); + if (sourceHasIndividualTracks) + { + for (int i = 1; i < audioTrackNames!.Count; i++) + { + if (mutedAudioTracks == null || !mutedAudioTracks.Contains(i)) + enabledSourceTracks.Add(i); + } + } + + // Pre-compute which individual track positions map to an enabled source stream + var indivSourceMap = new Dictionary(); // layout position j -> source audio index + if (sourceHasIndividualTracks) + { + for (int j = 1; j < targetAudioLayout.Count; j++) + { + int srcIdx = audioTrackNames!.FindIndex(n => string.Equals(n, targetAudioLayout[j], StringComparison.OrdinalIgnoreCase)); + if (srcIdx > 0 && (mutedAudioTracks == null || !mutedAudioTracks.Contains(srcIdx))) + indivSourceMap[j] = srcIdx; + } + } + + // Count how many times each source audio stream index is referenced + var refCount = new Dictionary(); + foreach (int i in enabledSourceTracks) + { + refCount.TryGetValue(i, out int c); + refCount[i] = c + 1; + } + foreach (int srcIdx in indivSourceMap.Values) + { + refCount.TryGetValue(srcIdx, out int c); + refCount[srcIdx] = c + 1; + } + + // Build asplit filters for streams referenced more than once + var available = new Dictionary>(); + foreach (var (srcIdx, count) in refCount) + { + if (count <= 1) + { + available[srcIdx] = new Queue(new[] { $"[0:a:{srcIdx}]" }); + } + else + { + var labels = Enumerable.Range(0, count).Select(k => $"[split_{srcIdx}_{k}]").ToList(); + filterParts.Add($"[0:a:{srcIdx}]asplit={count}{string.Join("", labels)}"); + available[srcIdx] = new Queue(labels); + } + } + + string durationStr = duration.ToString(CultureInfo.InvariantCulture); + string silenceInput = $"-f lavfi -t {durationStr} -i \"anullsrc=cl=stereo:r=44100\""; + + // Position 0: Full Mix + if (!sourceHasIndividualTracks) + { + // No individual track metadata -- pass through the source's default audio + filterParts.Add("[0:a:0]acopy[out_a0]"); + mapParts.Add("-map \"[out_a0]\""); + } + else if (enabledSourceTracks.Count == 0) { - if (!mutedAudioTracks!.Contains(i)) - enabledTracks.Add(i); + extraInputParts.Add(silenceInput); + mapParts.Add($"-map {silenceInputIdx++}:a:0"); } + else if (enabledSourceTracks.Count == 1) + { + filterParts.Add($"{available[enabledSourceTracks[0]].Dequeue()}acopy[out_a0]"); + mapParts.Add("-map \"[out_a0]\""); + } + else + { + string inputs = string.Join("", enabledSourceTracks.Select(i => available[i].Dequeue())); + filterParts.Add($"{inputs}amix=inputs={enabledSourceTracks.Count}:duration=longest[out_a0]"); + mapParts.Add("-map \"[out_a0]\""); + } + metaParts.Add($"-metadata:s:a:0 title=\"{targetAudioLayout[0]}\""); - if (enabledTracks.Count > 0) + // Positions 1+: individual tracks aligned by name + for (int j = 1; j < targetAudioLayout.Count; j++) { - // Create a new Full Mix from only the non-muted individual tracks - if (enabledTracks.Count == 1) + if (indivSourceMap.TryGetValue(j, out int srcIdx)) { - // Single track: use it directly as the mix, no filter needed - filterArgs = ""; - mapArgs = $"-map 0:v:0 -map 0:a:{enabledTracks[0]} "; + filterParts.Add($"{available[srcIdx].Dequeue()}acopy[out_a{j}]"); + mapParts.Add($"-map \"[out_a{j}]\""); } else { - // Multiple tracks: use amix filter to create a new Full Mix - string filterInputs = string.Join("", enabledTracks.Select(i => $"[0:a:{i}]")); - filterArgs = $"-filter_complex \"{filterInputs}amix=inputs={enabledTracks.Count}:duration=longest[mix]\" "; - mapArgs = "-map 0:v:0 -map \"[mix]\" "; + extraInputParts.Add(silenceInput); + mapParts.Add($"-map {silenceInputIdx++}:a:0"); } + metaParts.Add($"-metadata:s:a:{j} title=\"{targetAudioLayout[j]}\""); + } - int outputTrackIndex = 0; - metadataArgs = $"-metadata:s:a:{outputTrackIndex} title=\"Full Mix\" "; - outputTrackIndex++; + filterArgs = filterParts.Count > 0 ? $"-filter_complex \"{string.Join(";", filterParts)}\" " : ""; + extraInputArgs = extraInputParts.Count > 0 ? string.Join(" ", extraInputParts) + " " : ""; + mapArgs = string.Join(" ", mapParts) + " "; + metadataArgs = string.Join(" ", metaParts) + " "; + } + else + { + // Legacy paths (keepSeparate=false or no track metadata) + bool hasMutedTracks = mutedAudioTracks != null && mutedAudioTracks.Count > 0 && audioTrackNames != null && audioTrackNames.Count > 1; + + if (hasMutedTracks) + { + var enabledTracks = new List(); + for (int i = 1; i < audioTrackNames!.Count; i++) + { + if (!mutedAudioTracks!.Contains(i)) + enabledTracks.Add(i); + } - // When keeping separate tracks, also map each individual non-muted track - if (settings.ClipKeepSeparateAudioTracks) + if (enabledTracks.Count > 0) { - foreach (int i in enabledTracks) + if (enabledTracks.Count == 1) + { + mapArgs = $"-map 0:v:0 -map 0:a:{enabledTracks[0]} "; + } + else { - mapArgs += $"-map 0:a:{i} "; - metadataArgs += $"-metadata:s:a:{outputTrackIndex} title=\"{audioTrackNames[i]}\" "; - outputTrackIndex++; + string filterInputs = string.Join("", enabledTracks.Select(i => $"[0:a:{i}]")); + filterArgs = $"-filter_complex \"{filterInputs}amix=inputs={enabledTracks.Count}:duration=longest[mix]\" "; + mapArgs = "-map 0:v:0 -map \"[mix]\" "; } + + metadataArgs = "-metadata:s:a:0 title=\"Full Mix\" "; } } - } - else if (settings.ClipKeepSeparateAudioTracks && audioTrackNames != null) - { - // No muted tracks, keep all audio tracks as-is - mapArgs = "-map 0:v:0 -map 0:a "; - for (int i = 0; i < audioTrackNames.Count; i++) + else if (settings.ClipKeepSeparateAudioTracks && audioTrackNames != null) { - metadataArgs += $"-metadata:s:a:{i} title=\"{audioTrackNames[i]}\" "; + mapArgs = "-map 0:v:0 -map 0:a "; + for (int i = 0; i < audioTrackNames.Count; i++) + metadataArgs += $"-metadata:s:a:{i} title=\"{audioTrackNames[i]}\" "; } } - string arguments = $"-y -ss {startTime.ToString(CultureInfo.InvariantCulture)} -t {duration.ToString(CultureInfo.InvariantCulture)} " + - $"-i \"{inputFilePath}\" {filterArgs}{mapArgs}-c:v {videoCodec} {presetArgs} {qualityArgs} {fpsArg} " + - $"-c:a aac -b:a {settings.ClipAudioQuality} {metadataArgs}-movflags +faststart \"{outputFilePath}\""; + string verboseFlag = filterArgs.Length > 0 || extraInputArgs.Length > 0 ? "-v verbose " : ""; + string arguments = $"-y {verboseFlag}-ss {startTime.ToString(CultureInfo.InvariantCulture)} -t {duration.ToString(CultureInfo.InvariantCulture)} " + + $"-i \"{inputFilePath}\" {extraInputArgs}{filterArgs}{mapArgs}-c:v {videoCodec} {presetArgs} {qualityArgs} {fpsArg} " + + $"-c:a aac -b:a {settings.ClipAudioQuality} {metadataArgs}-t {duration.ToString(CultureInfo.InvariantCulture)} -movflags +faststart \"{outputFilePath}\""; Log.Information("Extracting clip"); Log.Information($"FFmpeg arguments: {arguments}"); @@ -436,6 +557,21 @@ public static async void CancelClip(int clipId) } } + private static List? BuildUnionAudioLayout(List?> perSelectionTrackNames) + { + var union = new List { "Full Mix" }; + foreach (var trackNames in perSelectionTrackNames) + { + if (trackNames == null) continue; + foreach (var name in trackNames.Skip(1)) + { + if (!union.Any(u => string.Equals(u, name, StringComparison.OrdinalIgnoreCase))) + union.Add(name); + } + } + return union.Count > 1 ? union : null; + } + private static List? GetSourceAudioTrackNames(Selection selection) { try diff --git a/Frontend/src/Components/SelectionCard.tsx b/Frontend/src/Components/SelectionCard.tsx index 5865f14b..7d106cd0 100644 --- a/Frontend/src/Components/SelectionCard.tsx +++ b/Frontend/src/Components/SelectionCard.tsx @@ -1,4 +1,4 @@ -import React, { useState } from 'react'; +import React, { useLayoutEffect, useRef, useState } from 'react'; import { SelectionCardProps } from '../Models/types'; import { useDrag, useDrop } from 'react-dnd'; import { TbHeadphones } from 'react-icons/tb'; @@ -19,6 +19,13 @@ const SelectionCard: React.FC = React.memo( }) => { const [showAudioMenu, setShowAudioMenu] = useState(false); + const indexRef = useRef(index); + const moveCardRef = useRef(moveCard); + useLayoutEffect(() => { + indexRef.current = index; + moveCardRef.current = moveCard; + }); + const [{ isDragging }, dragRef] = useDrag( () => ({ type: DRAG_TYPE, @@ -34,13 +41,13 @@ const SelectionCard: React.FC = React.memo( () => ({ accept: DRAG_TYPE, hover: (item: { index: number }) => { - if (item.index !== index) { - moveCard(item.index, index); - item.index = index; + if (item.index !== indexRef.current) { + moveCardRef.current(item.index, indexRef.current); + item.index = indexRef.current; } }, }), - [index, moveCard], + [], ); const dragDropRef = (node: HTMLDivElement | null) => { diff --git a/Frontend/src/Pages/video.tsx b/Frontend/src/Pages/video.tsx index 51311814..56a10ea8 100644 --- a/Frontend/src/Pages/video.tsx +++ b/Frontend/src/Pages/video.tsx @@ -215,6 +215,11 @@ export default function VideoComponent({ video }: { video: Content }) { // Audio tracks const audioTracks = useAudioTracks(videoRef, video); const [showAudioTracks, setShowAudioTracks] = useState(false); + const [timelineAudioMenu, setTimelineAudioMenu] = useState<{ + selId: number; + x: number; + y: number; + } | null>(null); // Video state const [currentTime, setCurrentTime] = useState(0); @@ -261,6 +266,14 @@ export default function VideoComponent({ video }: { video: Content }) { }; }, [showSpeedMenu]); + // Close timeline audio menu when clicking outside + useEffect(() => { + if (!timelineAudioMenu) return; + const handleClickOutside = () => setTimelineAudioMenu(null); + document.addEventListener('mousedown', handleClickOutside); + return () => document.removeEventListener('mousedown', handleClickOutside); + }, [timelineAudioMenu]); + // Clamp translation so the video remains at least partially visible const clampTranslate = (t: { x: number; y: number }) => { const el = playerContainerRef.current; @@ -615,6 +628,15 @@ export default function VideoComponent({ video }: { video: Content }) { useEffect(() => { if (!audioTracks.isMultiTrack) return; + // If already inside a selection when selections change (e.g. user toggled a track), + // re-apply immediately without waiting for the next interval tick. + if (activeSelectionIdRef.current !== null) { + const currentSel = selections.find((s) => s.id === activeSelectionIdRef.current); + if (currentSel) { + audioTracks.setMutedTracks(new Set(currentSel.mutedAudioTracks ?? [])); + } + } + const checkInterval = setInterval(() => { const vid = videoRef.current; if (!vid || vid.paused) return; @@ -643,7 +665,7 @@ export default function VideoComponent({ video }: { video: Content }) { audioTracks.setMutedTracks(savedMuteStateRef.current); savedMuteStateRef.current = null; } - }, 100); + }, 50); return () => clearInterval(checkInterval); }, [audioTracks.isMultiTrack, selections]); @@ -1038,7 +1060,12 @@ export default function VideoComponent({ video }: { video: Content }) { game: video.game, title: video.title, igdbId: video.igdbId, - mutedAudioTracks: audioTracks.isMultiTrack ? [...audioTracks.mutedTracks] : undefined, + mutedAudioTracks: + selections.length > 0 + ? selections[selections.length - 1].mutedAudioTracks + : audioTracks.isMultiTrack + ? [...audioTracks.mutedTracks] + : undefined, }; addSelection(newSelection); // Kick off thumbnail generation; uses latest state and guards against stale overwrites @@ -1861,6 +1888,30 @@ export default function VideoComponent({ video }: { video: Content }) {
+ {audioTracks.isMultiTrack && + video.audioTrackNames && + video.audioTrackNames.length > 1 && ( + + )} +
handleResizeMouseDown(e, sel.id, 'start')} @@ -1884,6 +1935,41 @@ export default function VideoComponent({ video }: { video: Content }) { )}
+ {timelineAudioMenu && + (() => { + const menuSel = selections.find((s) => s.id === timelineAudioMenu.selId); + if (!menuSel || !video.audioTrackNames) return null; + const mutedTracks = menuSel.mutedAudioTracks ?? []; + return ( +
e.stopPropagation()} + onClick={(e) => e.stopPropagation()} + > + {video.audioTrackNames.map((name, i) => { + if (i === 0) return null; + const isMuted = mutedTracks.includes(i); + return ( + + ); + })} +
+ ); + })()}
From 97ad93834b9855f0c5b923d7a4eb66bafb619026 Mon Sep 17 00:00:00 2001 From: Segergren Date: Wed, 18 Mar 2026 22:09:27 +0100 Subject: [PATCH 3/5] fix: decouple default and per-selection audio muting during playback --- Frontend/src/Hooks/useAudioTracks.ts | 44 ++++++++++++++++++---- Frontend/src/Pages/video.tsx | 56 +++++++++++----------------- 2 files changed, 58 insertions(+), 42 deletions(-) diff --git a/Frontend/src/Hooks/useAudioTracks.ts b/Frontend/src/Hooks/useAudioTracks.ts index 8576271a..f6976e2b 100644 --- a/Frontend/src/Hooks/useAudioTracks.ts +++ b/Frontend/src/Hooks/useAudioTracks.ts @@ -1,4 +1,4 @@ -import { useEffect, useRef, useState, useCallback } from 'react'; +import { useEffect, useLayoutEffect, useRef, useState, useCallback } from 'react'; import { Content } from '../Models/types'; export interface AudioTrackInfo { @@ -16,6 +16,7 @@ export interface AudioTrackState { toggleTrackMute: (index: number) => void; setMutedTracks: (muted: Set) => void; toggleSolo: (index: number) => void; + setMuteOverride: (mutedIndices: number[] | null) => void; isMultiTrack: boolean; } @@ -176,17 +177,33 @@ export function useAudioTracks( }; }, [videoRef, isMultiTrack]); - // Apply volume/mute/solo to audio elements - useEffect(() => { + // Override for per-selection muting -- stored as ref so changes don't cause re-renders. + // When set, audio elements use this instead of the default mutedTracks state. + const muteOverrideRef = useRef | null>(null); + + // Keep latest state in refs so the stable setMuteOverride can read them + const latestRef = useRef({ mutedTracks, soloTrack, volumes }); + useLayoutEffect(() => { + latestRef.current = { mutedTracks, soloTrack, volumes }; + }); + + const applyMuting = useCallback(() => { + const { mutedTracks: defaultMuted, soloTrack: solo, volumes: vols } = latestRef.current; + const effectiveMuted = muteOverrideRef.current ?? defaultMuted; for (const [index, audio] of audioElementsRef.current.entries()) { - if (soloTrack !== null) { - audio.muted = index !== soloTrack; + if (solo !== null) { + audio.muted = index !== solo; } else { - audio.muted = mutedTracks.has(index); + audio.muted = effectiveMuted.has(index); } - audio.volume = volumes[index] ?? 1; + audio.volume = vols[index] ?? 1; } - }, [volumes, mutedTracks, soloTrack, tracks]); + }, []); + + // Apply when React state changes + useEffect(() => { + applyMuting(); + }, [volumes, mutedTracks, soloTrack, tracks, applyMuting]); const setTrackVolume = useCallback((index: number, volume: number) => { const clamped = Math.max(0, Math.min(1, volume)); @@ -213,6 +230,16 @@ export function useAudioTracks( setSoloTrack((prev) => (prev === index ? null : index)); }, []); + // Stable function to set/clear a per-selection mute override. + // Directly applies to audio elements without touching React state. + const setMuteOverride = useCallback( + (mutedIndices: number[] | null) => { + muteOverrideRef.current = mutedIndices ? new Set(mutedIndices) : null; + applyMuting(); + }, + [applyMuting], + ); + return { tracks, volumes, @@ -222,6 +249,7 @@ export function useAudioTracks( toggleTrackMute, setMutedTracks: replaceMutedTracks, toggleSolo, + setMuteOverride, isMultiTrack, }; } diff --git a/Frontend/src/Pages/video.tsx b/Frontend/src/Pages/video.tsx index 56a10ea8..511d77db 100644 --- a/Frontend/src/Pages/video.tsx +++ b/Frontend/src/Pages/video.tsx @@ -383,6 +383,7 @@ export default function VideoComponent({ video }: { video: Content }) { useEffect(() => { if (!controlsVisible) { setShowSpeedMenu(false); + setShowAudioTracks(false); speedButtonRef.current?.blur(); } }, [controlsVisible]); @@ -622,53 +623,40 @@ export default function VideoComponent({ video }: { video: Content }) { }; }, []); - // Apply per-selection audio track muting during playback + // Apply per-selection audio track muting during playback. + // The default mute state (mixer panel) is never touched -- we use a ref-based + // override so entering/leaving selections can't desync or bug out. const activeSelectionIdRef = useRef(null); - const savedMuteStateRef = useRef | null>(null); useEffect(() => { if (!audioTracks.isMultiTrack) return; - // If already inside a selection when selections change (e.g. user toggled a track), - // re-apply immediately without waiting for the next interval tick. - if (activeSelectionIdRef.current !== null) { - const currentSel = selections.find((s) => s.id === activeSelectionIdRef.current); - if (currentSel) { - audioTracks.setMutedTracks(new Set(currentSel.mutedAudioTracks ?? [])); - } - } - - const checkInterval = setInterval(() => { + const check = () => { const vid = videoRef.current; - if (!vid || vid.paused) return; + if (!vid) return; const t = vid.currentTime; - const activeSelection = selections.find( - (s) => - s.mutedAudioTracks && s.mutedAudioTracks.length > 0 && t >= s.startTime && t <= s.endTime, - ); - + const activeSelection = selections.find((s) => t >= s.startTime && t <= s.endTime); const activeId = activeSelection?.id ?? null; - if (activeId === activeSelectionIdRef.current) return; - - if (activeId !== null && activeSelectionIdRef.current === null) { - // Entering a selection with muted tracks -- save current state - savedMuteStateRef.current = new Set(audioTracks.mutedTracks); - } + if (activeId === activeSelectionIdRef.current) return; activeSelectionIdRef.current = activeId; - if (activeSelection?.mutedAudioTracks) { - // Apply this selection's muted tracks - audioTracks.setMutedTracks(new Set(activeSelection.mutedAudioTracks)); - } else if (savedMuteStateRef.current !== null) { - // Exiting selection -- restore saved state - audioTracks.setMutedTracks(savedMuteStateRef.current); - savedMuteStateRef.current = null; + if (activeSelection) { + audioTracks.setMuteOverride(activeSelection.mutedAudioTracks ?? []); + } else { + audioTracks.setMuteOverride(null); } - }, 50); + }; - return () => clearInterval(checkInterval); - }, [audioTracks.isMultiTrack, selections]); + activeSelectionIdRef.current = null; // force re-apply after selections change + check(); + const intervalId = setInterval(check, 50); + + return () => { + clearInterval(intervalId); + audioTracks.setMuteOverride(null); + }; + }, [audioTracks.isMultiTrack, audioTracks.setMuteOverride, selections]); // Update container width on window resize useEffect(() => { From da7279c6d263ff02914cf94f4fc912457c8bb242 Mon Sep 17 00:00:00 2001 From: Segergren Date: Wed, 8 Apr 2026 18:32:27 +0200 Subject: [PATCH 4/5] feat: add per-selection audio volumes --- Backend/Api/ContentServer.cs | 8 +- Backend/App/MessageService.cs | 15 +- Backend/Media/ClipService.cs | 129 ++++++++++-- Frontend/src/Components/SelectionCard.tsx | 68 ++++-- Frontend/src/Hooks/useAudioTracks.ts | 56 ++++- Frontend/src/Models/types.ts | 2 + Frontend/src/Pages/video.tsx | 246 +++++++++++++++------- 7 files changed, 404 insertions(+), 120 deletions(-) diff --git a/Backend/Api/ContentServer.cs b/Backend/Api/ContentServer.cs index 29f25b7a..e0127ff4 100644 --- a/Backend/Api/ContentServer.cs +++ b/Backend/Api/ContentServer.cs @@ -12,6 +12,7 @@ internal class ContentServer { private static readonly HttpListener _httpListener = new(); private static CancellationTokenSource? _cancellationTokenSource; + private static readonly System.Collections.Concurrent.ConcurrentDictionary _extractionTasks = new(); public static void StartServer(string prefix) { @@ -310,13 +311,18 @@ private static async Task HandleAudioTracksRequest(HttpListenerContext context) { try { - await FFmpegService.ExtractAudioTrack(input, trackFilePath, i); + var task = _extractionTasks.GetOrAdd(trackFilePath, _ => FFmpegService.ExtractAudioTrack(input, trackFilePath, i)); + await task; } catch (Exception ex) { Log.Warning($"Failed to extract audio track {i} from {input}: {ex.Message}"); continue; } + finally + { + _extractionTasks.TryRemove(trackFilePath, out _); + } } result.Add(new diff --git a/Backend/App/MessageService.cs b/Backend/App/MessageService.cs index d93cb6d7..8c013ba3 100644 --- a/Backend/App/MessageService.cs +++ b/Backend/App/MessageService.cs @@ -26,6 +26,7 @@ public class Selection public string Title { get; set; } = string.Empty; public int? IgdbId { get; set; } public List? MutedAudioTracks { get; set; } + public Dictionary? AudioTrackVolumes { get; set; } } public static class MessageService @@ -364,6 +365,17 @@ private static async Task HandleCreateClip(JsonElement message) { mutedAudioTracks = mutedEl.EnumerateArray().Select(e => e.GetInt32()).ToList(); } + Dictionary? audioTrackVolumes = null; + if (selectionElement.TryGetProperty("audioTrackVolumes", out JsonElement volEl) + && volEl.ValueKind == JsonValueKind.Object) + { + audioTrackVolumes = new Dictionary(); + foreach (var prop in volEl.EnumerateObject()) + { + if (int.TryParse(prop.Name, out int trackIdx) && prop.Value.TryGetDouble(out double vol)) + audioTrackVolumes[trackIdx] = vol; + } + } // Create a new Selection instance with all required properties. selections.Add(new Selection @@ -377,7 +389,8 @@ private static async Task HandleCreateClip(JsonElement message) Game = game, Title = title, IgdbId = igdbId, - MutedAudioTracks = mutedAudioTracks + MutedAudioTracks = mutedAudioTracks, + AudioTrackVolumes = audioTrackVolumes }); } } diff --git a/Backend/Media/ClipService.cs b/Backend/Media/ClipService.cs index 477d78e5..a364c1ab 100644 --- a/Backend/Media/ClipService.cs +++ b/Backend/Media/ClipService.cs @@ -106,7 +106,7 @@ public static async Task CreateClips(List selections) List? selectionTrackNames = perSelectionTrackNames[selectionIndex]; List? targetLayout = Settings.Instance.ClipKeepSeparateAudioTracks ? unionAudioLayout : null; - await ExtractClip(id, inputFilePath, tempFileName, selection.StartTime, selection.EndTime, selectionTrackNames, selection.MutedAudioTracks, targetLayout, progress => + await ExtractClip(id, inputFilePath, tempFileName, selection.StartTime, selection.EndTime, selectionTrackNames, selection.MutedAudioTracks, selection.AudioTrackVolumes, targetLayout, progress => { double clampedProgress = Math.Min(progress, 1.0); double currentProgress = (processedDuration + (clampedProgress * clipDuration)) / totalDuration * 95; @@ -151,12 +151,19 @@ await ExtractClip(id, inputFilePath, tempFileName, selection.StartTime, selectio { string mapAllArg = unionAudioLayout != null ? "-map 0 " : ""; string concatMetadataArgs = ""; + // When multi-track is active, re-encode audio during concat to fix + // DTS misalignment between streams (different seek offsets cause + // slightly different AAC frame counts per stream). + // Video is always stream-copied. + string codecArg = unionAudioLayout != null + ? $"-c:v copy -c:a aac -b:a {Settings.Instance.ClipAudioQuality} " + : "-c copy "; if (unionAudioLayout != null) { concatMetadataArgs = string.Join(" ", unionAudioLayout.Select((name, i) => $"-metadata:s:a:{i} title=\"{name}\"")) + " "; } await FFmpegService.RunWithProgress(id, - $"-y -f concat -safe 0 -i \"{concatFilePath}\" {mapAllArg}-c copy {concatMetadataArgs}-movflags +faststart \"{outputFilePath}\"", + $"-y -f concat -safe 0 -i \"{concatFilePath}\" {mapAllArg}{codecArg}{concatMetadataArgs}-avoid_negative_ts make_zero -movflags +faststart \"{outputFilePath}\"", totalDuration, progress => { }, process => @@ -228,7 +235,7 @@ await FFmpegService.RunWithProgress(id, } private static async Task ExtractClip(int clipId, string inputFilePath, string outputFilePath, double startTime, double endTime, - List? audioTrackNames, List? mutedAudioTracks, List? targetAudioLayout, Action progressCallback) + List? audioTrackNames, List? mutedAudioTracks, Dictionary? audioTrackVolumes, List? targetAudioLayout, Action progressCallback) { double duration = endTime - startTime; var settings = Settings.Instance; @@ -385,29 +392,51 @@ private static async Task ExtractClip(int clipId, string inputFilePath, string o } string durationStr = duration.ToString(CultureInfo.InvariantCulture); - string silenceInput = $"-f lavfi -t {durationStr} -i \"anullsrc=cl=stereo:r=44100\""; + string silenceInput = $"-f lavfi -t {durationStr} -i \"anullsrc=cl=stereo:r=48000\""; + string atrim = $"atrim=end={durationStr},asetpts=PTS-STARTPTS"; // Position 0: Full Mix if (!sourceHasIndividualTracks) { // No individual track metadata -- pass through the source's default audio - filterParts.Add("[0:a:0]acopy[out_a0]"); + filterParts.Add($"[0:a:0]{atrim}[out_a0]"); mapParts.Add("-map \"[out_a0]\""); } else if (enabledSourceTracks.Count == 0) { extraInputParts.Add(silenceInput); - mapParts.Add($"-map {silenceInputIdx++}:a:0"); + filterParts.Add($"[{silenceInputIdx}:a:0]{atrim}[out_a0]"); + mapParts.Add("-map \"[out_a0]\""); + silenceInputIdx++; } else if (enabledSourceTracks.Count == 1) { - filterParts.Add($"{available[enabledSourceTracks[0]].Dequeue()}acopy[out_a0]"); + int trackIdx = enabledSourceTracks[0]; + string volFilter = WithAtrim(GetVolumeFilter(audioTrackVolumes, trackIdx), durationStr); + filterParts.Add($"{available[trackIdx].Dequeue()}{volFilter}[out_a0]"); mapParts.Add("-map \"[out_a0]\""); } else { - string inputs = string.Join("", enabledSourceTracks.Select(i => available[i].Dequeue())); - filterParts.Add($"{inputs}amix=inputs={enabledSourceTracks.Count}:duration=longest[out_a0]"); + // Apply per-track volume before mixing + var mixInputLabels = new List(); + foreach (int i in enabledSourceTracks) + { + double vol = GetTrackVolume(audioTrackVolumes, i); + string srcLabel = available[i].Dequeue(); + if (Math.Abs(vol - 1.0) > 0.001) + { + string volLabel = $"[vol_mix_{i}]"; + filterParts.Add($"{srcLabel}volume={vol.ToString(CultureInfo.InvariantCulture)}{volLabel}"); + mixInputLabels.Add(volLabel); + } + else + { + mixInputLabels.Add(srcLabel); + } + } + string inputs = string.Join("", mixInputLabels); + filterParts.Add($"{inputs}amix=inputs={enabledSourceTracks.Count}:duration=longest,{atrim}[out_a0]"); mapParts.Add("-map \"[out_a0]\""); } metaParts.Add($"-metadata:s:a:0 title=\"{targetAudioLayout[0]}\""); @@ -417,13 +446,16 @@ private static async Task ExtractClip(int clipId, string inputFilePath, string o { if (indivSourceMap.TryGetValue(j, out int srcIdx)) { - filterParts.Add($"{available[srcIdx].Dequeue()}acopy[out_a{j}]"); + string volFilter = WithAtrim(GetVolumeFilter(audioTrackVolumes, srcIdx), durationStr); + filterParts.Add($"{available[srcIdx].Dequeue()}{volFilter}[out_a{j}]"); mapParts.Add($"-map \"[out_a{j}]\""); } else { extraInputParts.Add(silenceInput); - mapParts.Add($"-map {silenceInputIdx++}:a:0"); + filterParts.Add($"[{silenceInputIdx}:a:0]{atrim}[out_a{j}]"); + mapParts.Add($"-map \"[out_a{j}]\""); + silenceInputIdx++; } metaParts.Add($"-metadata:s:a:{j} title=\"{targetAudioLayout[j]}\""); } @@ -437,27 +469,58 @@ private static async Task ExtractClip(int clipId, string inputFilePath, string o { // Legacy paths (keepSeparate=false or no track metadata) bool hasMutedTracks = mutedAudioTracks != null && mutedAudioTracks.Count > 0 && audioTrackNames != null && audioTrackNames.Count > 1; + bool hasVolumeChanges = audioTrackVolumes != null && audioTrackVolumes.Any(kv => Math.Abs(kv.Value - 1.0) > 0.001); + bool needsAudioProcessing = hasMutedTracks || (hasVolumeChanges && audioTrackNames != null && audioTrackNames.Count > 1); - if (hasMutedTracks) + if (needsAudioProcessing && audioTrackNames != null) { var enabledTracks = new List(); - for (int i = 1; i < audioTrackNames!.Count; i++) + for (int i = 1; i < audioTrackNames.Count; i++) { - if (!mutedAudioTracks!.Contains(i)) + if (mutedAudioTracks == null || !mutedAudioTracks.Contains(i)) enabledTracks.Add(i); } if (enabledTracks.Count > 0) { - if (enabledTracks.Count == 1) + bool anyVolChange = enabledTracks.Any(i => Math.Abs(GetTrackVolume(audioTrackVolumes, i) - 1.0) > 0.001); + + if (enabledTracks.Count == 1 && !anyVolChange) { mapArgs = $"-map 0:v:0 -map 0:a:{enabledTracks[0]} "; } else { - string filterInputs = string.Join("", enabledTracks.Select(i => $"[0:a:{i}]")); - filterArgs = $"-filter_complex \"{filterInputs}amix=inputs={enabledTracks.Count}:duration=longest[mix]\" "; - mapArgs = "-map 0:v:0 -map \"[mix]\" "; + var filterPartsList = new List(); + var mixInputLabels = new List(); + + foreach (int i in enabledTracks) + { + double vol = GetTrackVolume(audioTrackVolumes, i); + if (Math.Abs(vol - 1.0) > 0.001) + { + filterPartsList.Add($"[0:a:{i}]volume={vol.ToString(CultureInfo.InvariantCulture)}[vol_{i}]"); + mixInputLabels.Add($"[vol_{i}]"); + } + else + { + mixInputLabels.Add($"[0:a:{i}]"); + } + } + + if (enabledTracks.Count == 1) + { + // Single track with volume change + filterArgs = $"-filter_complex \"{string.Join(";", filterPartsList)}\" "; + mapArgs = $"-map 0:v:0 -map \"{mixInputLabels[0]}\" "; + } + else + { + string allInputs = string.Join("", mixInputLabels); + filterPartsList.Add($"{allInputs}amix=inputs={enabledTracks.Count}:duration=longest[mix]"); + filterArgs = $"-filter_complex \"{string.Join(";", filterPartsList)}\" "; + mapArgs = "-map 0:v:0 -map \"[mix]\" "; + } } metadataArgs = "-metadata:s:a:0 title=\"Full Mix\" "; @@ -472,9 +535,15 @@ private static async Task ExtractClip(int clipId, string inputFilePath, string o } string verboseFlag = filterArgs.Length > 0 || extraInputArgs.Length > 0 ? "-v verbose " : ""; + // When using the union layout, force a consistent sample rate on every output audio stream. + // Source tracks may be 44.1 kHz while the anullsrc silence inputs are 48 kHz, so without this + // each temp clip's per-slot sample rate depends on whether the slot got real audio or silence. + // The concat demuxer then uses the first clip's stream params for subsequent clips, playing + // mismatched samples at the wrong rate (the reported "shrunken audio"). + string audioRateArg = targetAudioLayout != null ? "-ar 48000 " : ""; string arguments = $"-y {verboseFlag}-ss {startTime.ToString(CultureInfo.InvariantCulture)} -t {duration.ToString(CultureInfo.InvariantCulture)} " + $"-i \"{inputFilePath}\" {extraInputArgs}{filterArgs}{mapArgs}-c:v {videoCodec} {presetArgs} {qualityArgs} {fpsArg} " + - $"-c:a aac -b:a {settings.ClipAudioQuality} {metadataArgs}-t {duration.ToString(CultureInfo.InvariantCulture)} -movflags +faststart \"{outputFilePath}\""; + $"-c:a aac -b:a {settings.ClipAudioQuality} {audioRateArg}{metadataArgs}-t {duration.ToString(CultureInfo.InvariantCulture)} -movflags +faststart \"{outputFilePath}\""; Log.Information("Extracting clip"); Log.Information($"FFmpeg arguments: {arguments}"); @@ -603,5 +672,27 @@ private static void SafeDelete(string path) try { File.Delete(path); } catch (Exception ex) { Log.Information($"Error deleting file {path}: {ex.Message}"); } } + + private static string WithAtrim(string filterChain, string durationStr) + { + if (filterChain == "acopy") + return $"atrim=end={durationStr},asetpts=PTS-STARTPTS"; + return $"{filterChain},atrim=end={durationStr},asetpts=PTS-STARTPTS"; + } + + private static double GetTrackVolume(Dictionary? audioTrackVolumes, int trackIndex) + { + if (audioTrackVolumes != null && audioTrackVolumes.TryGetValue(trackIndex, out double vol)) + return Math.Max(0, Math.Min(1, vol)); + return 1.0; + } + + private static string GetVolumeFilter(Dictionary? audioTrackVolumes, int trackIndex) + { + double vol = GetTrackVolume(audioTrackVolumes, trackIndex); + if (Math.Abs(vol - 1.0) > 0.001) + return $"volume={vol.ToString(CultureInfo.InvariantCulture)}"; + return "acopy"; + } } } diff --git a/Frontend/src/Components/SelectionCard.tsx b/Frontend/src/Components/SelectionCard.tsx index 7d106cd0..0c5ddceb 100644 --- a/Frontend/src/Components/SelectionCard.tsx +++ b/Frontend/src/Components/SelectionCard.tsx @@ -16,8 +16,9 @@ const SelectionCard: React.FC = React.memo( removeSelection, audioTrackNames, onMutedAudioTracksChange, + onAudioTrackVolumesChange, }) => { - const [showAudioMenu, setShowAudioMenu] = useState(false); + const [audioMenuPos, setAudioMenuPos] = useState<{ x: number; y: number } | null>(null); const indexRef = useRef(index); const moveCardRef = useRef(moveCard); @@ -59,6 +60,7 @@ const SelectionCard: React.FC = React.memo( const hasAudioTracks = audioTrackNames && audioTrackNames.length > 1 && onMutedAudioTracksChange; const mutedTracks = selection.mutedAudioTracks ?? []; + const trackVolumes = selection.audioTrackVolumes ?? {}; const toggleTrack = (trackIndex: number) => { if (!onMutedAudioTracksChange) return; @@ -69,8 +71,6 @@ const SelectionCard: React.FC = React.memo( onMutedAudioTracksChange(selection.id, newMuted); }; - const hasMutedTracks = mutedTracks.length > 0; - return (
= React.memo( onMouseEnter={() => setHoveredSelectionId(selection.id)} onMouseLeave={() => { setHoveredSelectionId(null); - setShowAudioMenu(false); + setAudioMenuPos(null); }} onContextMenu={(e) => { e.preventDefault(); @@ -107,36 +107,68 @@ const SelectionCard: React.FC = React.memo( )} {hasAudioTracks && ( -
+
- {showAudioMenu && ( + {audioMenuPos && (
e.stopPropagation()} > {audioTrackNames.map((name, i) => { // Skip track 0 (Full Mix) if (i === 0) return null; const isMuted = mutedTracks.includes(i); + const vol = trackVolumes[i] ?? 1; return ( - +
+
+ toggleTrack(i)} + className="checkbox checkbox-primary checkbox-xs shrink-0" + /> + + {name.replace(' (Default)', '')} + +
+
+ { + if (!onAudioTrackVolumesChange) return; + const newVolumes = { ...trackVolumes, [i]: parseFloat(e.target.value) }; + onAudioTrackVolumesChange(selection.id, newVolumes); + }} + className="w-16 h-1 bg-gray-600 rounded-lg appearance-none cursor-pointer accent-accent" + /> + + {Math.round(vol * 100)}% + +
+
); })}
diff --git a/Frontend/src/Hooks/useAudioTracks.ts b/Frontend/src/Hooks/useAudioTracks.ts index f6976e2b..a3a34845 100644 --- a/Frontend/src/Hooks/useAudioTracks.ts +++ b/Frontend/src/Hooks/useAudioTracks.ts @@ -12,11 +12,16 @@ export interface AudioTrackState { volumes: Record; mutedTracks: Set; soloTrack: number | null; + masterMuted: boolean; + masterVolume: number; setTrackVolume: (index: number, volume: number) => void; toggleTrackMute: (index: number) => void; setMutedTracks: (muted: Set) => void; toggleSolo: (index: number) => void; + setMasterMuted: (muted: boolean) => void; + setMasterVolume: (vol: number) => void; setMuteOverride: (mutedIndices: number[] | null) => void; + setVolumeOverride: (volumes: Record | null) => void; isMultiTrack: boolean; } @@ -177,9 +182,16 @@ export function useAudioTracks( }; }, [videoRef, isMultiTrack]); - // Override for per-selection muting -- stored as ref so changes don't cause re-renders. - // When set, audio elements use this instead of the default mutedTracks state. + // Master mute/volume -- playback-only controls, don't affect per-track state or clip output. + const masterMutedRef = useRef(false); + const [masterMuted, setMasterMutedState] = useState(false); + const masterVolumeRef = useRef(1); + const [masterVolume, setMasterVolumeState] = useState(1); + + // Overrides for per-selection muting/volumes -- stored as refs so changes don't cause re-renders. + // When set, audio elements use these instead of the default state. const muteOverrideRef = useRef | null>(null); + const volumeOverrideRef = useRef | null>(null); // Keep latest state in refs so the stable setMuteOverride can read them const latestRef = useRef({ mutedTracks, soloTrack, volumes }); @@ -190,13 +202,16 @@ export function useAudioTracks( const applyMuting = useCallback(() => { const { mutedTracks: defaultMuted, soloTrack: solo, volumes: vols } = latestRef.current; const effectiveMuted = muteOverrideRef.current ?? defaultMuted; + const effectiveVolumes = volumeOverrideRef.current ?? vols; for (const [index, audio] of audioElementsRef.current.entries()) { - if (solo !== null) { + if (masterMutedRef.current) { + audio.muted = true; + } else if (solo !== null) { audio.muted = index !== solo; } else { audio.muted = effectiveMuted.has(index); } - audio.volume = vols[index] ?? 1; + audio.volume = (effectiveVolumes[index] ?? 1) * masterVolumeRef.current; } }, []); @@ -230,6 +245,25 @@ export function useAudioTracks( setSoloTrack((prev) => (prev === index ? null : index)); }, []); + const setMasterMuted = useCallback( + (muted: boolean) => { + masterMutedRef.current = muted; + setMasterMutedState(muted); + applyMuting(); + }, + [applyMuting], + ); + + const setMasterVolume = useCallback( + (vol: number) => { + const clamped = Math.max(0, Math.min(1, vol)); + masterVolumeRef.current = clamped; + setMasterVolumeState(clamped); + applyMuting(); + }, + [applyMuting], + ); + // Stable function to set/clear a per-selection mute override. // Directly applies to audio elements without touching React state. const setMuteOverride = useCallback( @@ -240,16 +274,30 @@ export function useAudioTracks( [applyMuting], ); + // Stable function to set/clear a per-selection volume override. + const setVolumeOverride = useCallback( + (vols: Record | null) => { + volumeOverrideRef.current = vols; + applyMuting(); + }, + [applyMuting], + ); + return { tracks, volumes, mutedTracks, soloTrack, + masterMuted, + masterVolume, setTrackVolume, toggleTrackMute, setMutedTracks: replaceMutedTracks, toggleSolo, + setMasterMuted, + setMasterVolume, setMuteOverride, + setVolumeOverride, isMultiTrack, }; } diff --git a/Frontend/src/Models/types.ts b/Frontend/src/Models/types.ts index efc4cc59..9fd9c269 100644 --- a/Frontend/src/Models/types.ts +++ b/Frontend/src/Models/types.ts @@ -351,6 +351,7 @@ export interface Selection { title?: string; igdbId?: number; mutedAudioTracks?: number[]; + audioTrackVolumes?: Record; } export interface SelectionCardProps { @@ -363,6 +364,7 @@ export interface SelectionCardProps { removeSelection: (id: number) => void; audioTrackNames?: string[]; onMutedAudioTracksChange?: (id: number, mutedTracks: number[]) => void; + onAudioTrackVolumesChange?: (id: number, volumes: Record) => void; } export interface AiProgress { diff --git a/Frontend/src/Pages/video.tsx b/Frontend/src/Pages/video.tsx index 511d77db..eec34623 100644 --- a/Frontend/src/Pages/video.tsx +++ b/Frontend/src/Pages/video.tsx @@ -596,19 +596,71 @@ export default function VideoComponent({ video }: { video: Content }) { }; }, [volume, isMuted, isFullscreen, audioTracks.isMultiTrack]); - // Handle video playback time updates using requestAnimationFrame for smooth updates + // Per-selection audio override state, kept in refs for the rAF loop below. + const activeSelectionIdRef = useRef(null); + const audioTracksRef = useRef(audioTracks); + useLayoutEffect(() => { + audioTracksRef.current = audioTracks; + }); + + // Reset active selection when selections change so the override is re-applied + useEffect(() => { + activeSelectionIdRef.current = null; + }, [selections]); + + // Clean up overrides when multi-track is deactivated + useEffect(() => { + if (audioTracks.isMultiTrack) { + // Sync master mute/volume from saved state when entering multi-track + audioTracks.setMasterMuted(localStorage.getItem('segra-muted') === 'true'); + const savedVol = localStorage.getItem('segra-volume'); + audioTracks.setMasterVolume(savedVol ? parseFloat(savedVol) : 1); + } else { + audioTracks.setMuteOverride(null); + audioTracks.setVolumeOverride(null); + } + }, [ + audioTracks.isMultiTrack, + audioTracks.setMasterMuted, + audioTracks.setMuteOverride, + audioTracks.setVolumeOverride, + ]); + + // Handle video playback time updates using requestAnimationFrame for smooth updates. + // Also checks per-selection audio overrides each frame (cheap: one find + early return). useEffect(() => { const vid = videoRef.current; if (!vid) return; let rafId = 0; - const updateCurrentTime = () => { + const tick = () => { setCurrentTime(vid.currentTime); + + // Per-selection audio mute/volume override + const at = audioTracksRef.current; + if (at.isMultiTrack) { + const t = vid.currentTime; + const sels = selectionsRef.current; + const activeSel = sels.find((s) => t >= s.startTime && t <= s.endTime); + const activeId = activeSel?.id ?? null; + + if (activeId !== activeSelectionIdRef.current) { + activeSelectionIdRef.current = activeId; + if (activeSel) { + at.setMuteOverride(activeSel.mutedAudioTracks ?? []); + at.setVolumeOverride(activeSel.audioTrackVolumes ?? null); + } else { + at.setMuteOverride(null); + at.setVolumeOverride(null); + } + } + } + if (!vid.paused && !vid.ended) { - rafId = requestAnimationFrame(updateCurrentTime); + rafId = requestAnimationFrame(tick); } }; const onPlay = () => { - rafId = requestAnimationFrame(updateCurrentTime); + rafId = requestAnimationFrame(tick); }; const onPause = () => { cancelAnimationFrame(rafId); @@ -623,41 +675,6 @@ export default function VideoComponent({ video }: { video: Content }) { }; }, []); - // Apply per-selection audio track muting during playback. - // The default mute state (mixer panel) is never touched -- we use a ref-based - // override so entering/leaving selections can't desync or bug out. - const activeSelectionIdRef = useRef(null); - useEffect(() => { - if (!audioTracks.isMultiTrack) return; - - const check = () => { - const vid = videoRef.current; - if (!vid) return; - - const t = vid.currentTime; - const activeSelection = selections.find((s) => t >= s.startTime && t <= s.endTime); - const activeId = activeSelection?.id ?? null; - - if (activeId === activeSelectionIdRef.current) return; - activeSelectionIdRef.current = activeId; - - if (activeSelection) { - audioTracks.setMuteOverride(activeSelection.mutedAudioTracks ?? []); - } else { - audioTracks.setMuteOverride(null); - } - }; - - activeSelectionIdRef.current = null; // force re-apply after selections change - check(); - const intervalId = setInterval(check, 50); - - return () => { - clearInterval(intervalId); - audioTracks.setMuteOverride(null); - }; - }, [audioTracks.isMultiTrack, audioTracks.setMuteOverride, selections]); - // Update container width on window resize useEffect(() => { if (scrollContainerRef.current) { @@ -854,8 +871,17 @@ export default function VideoComponent({ video }: { video: Content }) { setVolume(target); return; } - // When multi-track is active, don't touch the video element's volume/muted - if (!audioTracks.isMultiTrack) { + if (audioTracks.isMultiTrack) { + // Route to master volume -- purely a preview control + audioTracks.setMasterVolume(target); + if (target === 0) { + audioTracks.setMasterMuted(true); + setIsMuted(true); + } else if (audioTracks.masterMuted) { + audioTracks.setMasterMuted(false); + setIsMuted(false); + } + } else { el.volume = target; if (target === 0) { el.muted = true; @@ -867,7 +893,10 @@ export default function VideoComponent({ video }: { video: Content }) { } setVolume(target); localStorage.setItem('segra-volume', target.toString()); - localStorage.setItem('segra-muted', el.muted.toString()); + localStorage.setItem( + 'segra-muted', + (audioTracks.isMultiTrack ? audioTracks.masterMuted : el.muted).toString(), + ); }; // Pointer handlers for panning the video when zoomed @@ -1080,6 +1109,7 @@ export default function VideoComponent({ video }: { video: Content }) { endTime: s.endTime, igdbId: s.igdbId, mutedAudioTracks: s.mutedAudioTracks, + audioTrackVolumes: s.audioTrackVolumes, })), }; sendMessageToBackend('CreateClip', params); @@ -1487,14 +1517,18 @@ export default function VideoComponent({ video }: { video: Content }) { // Toggle mute state const toggleMute = () => { if (videoRef.current) { - // When multi-track is active, don't toggle the video element's muted state - if (audioTracks.isMultiTrack) return; + if (audioTracks.isMultiTrack) { + // Master mute for multi-track: silences all audio elements + const newMuted = !audioTracks.masterMuted; + audioTracks.setMasterMuted(newMuted); + setIsMuted(newMuted); + localStorage.setItem('segra-muted', newMuted.toString()); + return; + } const newMutedState = !videoRef.current.muted; videoRef.current.muted = newMutedState; setIsMuted(newMutedState); - - // Save to localStorage localStorage.setItem('segra-muted', newMutedState.toString()); } }; @@ -1630,22 +1664,49 @@ export default function VideoComponent({ video }: { video: Content }) { {showAudioTracks && ( -
+
{audioTracks.tracks.map((track) => { const isMuted = audioTracks.mutedTracks.has(track.index); + const vol = audioTracks.volumes[track.index] ?? 1; return ( - +
+ audioTracks.toggleTrackMute(track.index)} + className="checkbox checkbox-primary checkbox-xs shrink-0" + /> + + {track.name.replace(' (Default)', '')} + +
+
+ + audioTracks.setTrackVolume( + track.index, + parseFloat(e.target.value), + ) + } + className="w-16 h-1 bg-gray-600 rounded-lg appearance-none cursor-pointer accent-accent" + /> + + {Math.round(vol * 100)}% + +
+
); })}
@@ -1880,11 +1941,7 @@ export default function VideoComponent({ video }: { video: Content }) { video.audioTrackNames && video.audioTrackNames.length > 1 && (
From ca77b7281a389e69a55f4dfa7bc743a2bf01279d Mon Sep 17 00:00:00 2001 From: Segergren Date: Wed, 8 Apr 2026 18:59:40 +0200 Subject: [PATCH 5/5] refactor: match game integrations by IGDB ID instead of name --- Backend/Recorder/OBSService.cs | 2 +- Backend/Services/GameIntegrationService.cs | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Backend/Recorder/OBSService.cs b/Backend/Recorder/OBSService.cs index 613c5ece..e9d36173 100644 --- a/Backend/Recorder/OBSService.cs +++ b/Backend/Recorder/OBSService.cs @@ -939,7 +939,7 @@ public static bool StartRecording(string name = "Manual Recording", string exePa GeneralUtils.SetProcessPriority(ProcessPriorityClass.High); if (!isReplayBufferMode) { - _ = GameIntegrationService.Start(name); + _ = GameIntegrationService.Start(GameUtils.GetIgdbIdFromExePath(exePath)); } Task.Run(KeybindCaptureService.Start); return true; diff --git a/Backend/Services/GameIntegrationService.cs b/Backend/Services/GameIntegrationService.cs index bed80751..99de6a39 100644 --- a/Backend/Services/GameIntegrationService.cs +++ b/Backend/Services/GameIntegrationService.cs @@ -10,14 +10,14 @@ namespace Segra.Backend.Services { public static class GameIntegrationService { - private const string PUBG = "PLAYERUNKNOWN'S BATTLEGROUNDS"; - private const string LOL = "League of Legends"; - private const string CS2 = "Counter-Strike 2"; - private const string ROCKET_LEAGUE = "Rocket League"; + private const int PUBG_IGDB_ID = 27789; + private const int LOL_IGDB_ID = 115; + private const int CS2_IGDB_ID = 242408; + private const int ROCKET_LEAGUE_IGDB_ID = 11198; private static Integration? _gameIntegration; public static Integration? GameIntegration => _gameIntegration; - public static async Task Start(string gameName) + public static async Task Start(int? igdbId) { if (_gameIntegration != null) { @@ -25,12 +25,12 @@ public static async Task Start(string gameName) await _gameIntegration.Shutdown(); } - _gameIntegration = gameName switch + _gameIntegration = igdbId switch { - PUBG => Settings.Instance.GameIntegrations.Pubg.Enabled ? new PubgIntegration() : null, - LOL => Settings.Instance.GameIntegrations.LeagueOfLegends.Enabled ? new LeagueOfLegendsIntegration() : null, - CS2 => Settings.Instance.GameIntegrations.CounterStrike2.Enabled ? new CounterStrike2Integration() : null, - ROCKET_LEAGUE => Settings.Instance.GameIntegrations.RocketLeague.Enabled ? new RocketLeagueIntegration() : null, + PUBG_IGDB_ID => Settings.Instance.GameIntegrations.Pubg.Enabled ? new PubgIntegration() : null, + LOL_IGDB_ID => Settings.Instance.GameIntegrations.LeagueOfLegends.Enabled ? new LeagueOfLegendsIntegration() : null, + CS2_IGDB_ID => Settings.Instance.GameIntegrations.CounterStrike2.Enabled ? new CounterStrike2Integration() : null, + ROCKET_LEAGUE_IGDB_ID => Settings.Instance.GameIntegrations.RocketLeague.Enabled ? new RocketLeagueIntegration() : null, _ => null, }; @@ -39,7 +39,7 @@ public static async Task Start(string gameName) return; } - Log.Information($"Starting game integration for: {gameName}"); + Log.Information($"Starting game integration for IGDB ID: {igdbId}"); _ = _gameIntegration.Start(); }