diff --git a/Backend/Api/ContentServer.cs b/Backend/Api/ContentServer.cs index 41d9fa92..e0127ff4 100644 --- a/Backend/Api/ContentServer.cs +++ b/Backend/Api/ContentServer.cs @@ -1,7 +1,10 @@ using Serilog; using System.Net; +using System.Text.Json; using System.Web; +using Segra.Backend.Core.Models; using Segra.Backend.Media; +using Segra.Backend.Shared; namespace Segra.Backend.Api { @@ -9,6 +12,7 @@ internal class ContentServer { private static readonly HttpListener _httpListener = new(); private static CancellationTokenSource? _cancellationTokenSource; + private static readonly System.Collections.Concurrent.ConcurrentDictionary _extractionTasks = new(); public static void StartServer(string prefix) { @@ -63,6 +67,10 @@ private static async Task ProcessRequestAsync(HttpListenerContext context) { await HandleThumbnailRequest(context); } + else if (rawUrl.StartsWith("/api/audiotracks")) + { + await HandleAudioTracksRequest(context); + } else if (rawUrl.StartsWith("/api/content")) { await HandleContentRequest(context); @@ -217,7 +225,8 @@ private static async Task HandleContentRequest(HttpListenerContext context) return; } - if (fileName.EndsWith(".mp4", StringComparison.OrdinalIgnoreCase)) + if (fileName.EndsWith(".mp4", StringComparison.OrdinalIgnoreCase) || + fileName.EndsWith(".m4a", StringComparison.OrdinalIgnoreCase)) { await StreamVideoFile(fileName, context); } @@ -236,6 +245,114 @@ private static async Task HandleContentRequest(HttpListenerContext context) } } + private static async Task HandleAudioTracksRequest(HttpListenerContext context) + { + var query = HttpUtility.ParseQueryString(context.Request?.Url?.Query ?? ""); + string input = query["input"] ?? ""; + string typeStr = query["type"] ?? ""; + var response = context.Response; + + response.AddHeader("Access-Control-Allow-Origin", "*"); + + if (!File.Exists(input)) + { + response.StatusCode = (int)HttpStatusCode.NotFound; + response.ContentType = "text/plain"; + using (var writer = new StreamWriter(response.OutputStream)) + { + await writer.WriteAsync("File not found."); + } + return; + } + + try + { + if (!Enum.TryParse(typeStr, true, out var contentType)) + { + contentType = FolderNames.GetContentTypeFromPath(input) ?? Content.ContentType.Session; + } + + // Read metadata to get audio track names + string fileName = Path.GetFileNameWithoutExtension(input); + string metadataFolderPath = FolderNames.GetMetadataFolderPath(contentType); + string metadataFilePath = Path.Combine(metadataFolderPath, $"{fileName}.json"); + + List? trackNames = null; + if (File.Exists(metadataFilePath)) + { + var metadataContent = await File.ReadAllTextAsync(metadataFilePath); + var metadata = JsonSerializer.Deserialize(metadataContent); + trackNames = metadata?.AudioTrackNames; + } + + if (trackNames == null || trackNames.Count <= 1) + { + response.StatusCode = (int)HttpStatusCode.OK; + response.ContentType = "application/json"; + using (var writer = new StreamWriter(response.OutputStream)) + { + await writer.WriteAsync("[]"); + } + return; + } + + string audioTracksDir = FolderNames.GetAudioTracksFolderPath(contentType); + Directory.CreateDirectory(audioTracksDir); + + var result = new List(); + + // Skip track 0 (Full Mix) - it duplicates the individual tracks + for (int i = 1; i < trackNames.Count; i++) + { + string trackFileName = $"{fileName}_track{i}.m4a"; + string trackFilePath = Path.Combine(audioTracksDir, trackFileName); + + if (!File.Exists(trackFilePath)) + { + try + { + var task = _extractionTasks.GetOrAdd(trackFilePath, _ => FFmpegService.ExtractAudioTrack(input, trackFilePath, i)); + await task; + } + catch (Exception ex) + { + Log.Warning($"Failed to extract audio track {i} from {input}: {ex.Message}"); + continue; + } + finally + { + _extractionTasks.TryRemove(trackFilePath, out _); + } + } + + result.Add(new + { + index = i, + name = trackNames[i], + url = $"/api/content?input={Uri.EscapeDataString(trackFilePath)}" + }); + } + + response.StatusCode = (int)HttpStatusCode.OK; + response.ContentType = "application/json"; + string json = JsonSerializer.Serialize(result); + using (var writer = new StreamWriter(response.OutputStream)) + { + await writer.WriteAsync(json); + } + } + catch (Exception ex) + { + Log.Error(ex, "Error handling audio tracks request"); + response.StatusCode = (int)HttpStatusCode.InternalServerError; + response.ContentType = "text/plain"; + using (var writer = new StreamWriter(response.OutputStream)) + { + await writer.WriteAsync("Error extracting audio tracks."); + } + } + } + private static async Task StreamVideoFile(string fileName, HttpListenerContext context) { var response = context.Response; @@ -273,7 +390,7 @@ private static async Task StreamVideoFile(string fileName, HttpListenerContext c long contentLength = end - start + 1; response.StatusCode = string.IsNullOrEmpty(rangeHeader) ? (int)HttpStatusCode.OK : (int)HttpStatusCode.PartialContent; - response.ContentType = "video/mp4"; + response.ContentType = fileName.EndsWith(".m4a", StringComparison.OrdinalIgnoreCase) ? "audio/mp4" : "video/mp4"; response.AddHeader("Accept-Ranges", "bytes"); if (!string.IsNullOrEmpty(rangeHeader)) diff --git a/Backend/App/MessageService.cs b/Backend/App/MessageService.cs index 4694eba1..8c013ba3 100644 --- a/Backend/App/MessageService.cs +++ b/Backend/App/MessageService.cs @@ -25,6 +25,8 @@ public class Selection public required string Game { get; set; } public string Title { get; set; } = string.Empty; public int? IgdbId { get; set; } + public List? MutedAudioTracks { get; set; } + public Dictionary? AudioTrackVolumes { get; set; } } public static class MessageService @@ -357,6 +359,23 @@ private static async Task HandleCreateClip(JsonElement message) string? filePath = selectionElement.TryGetProperty("filePath", out JsonElement filePathElement) ? filePathElement.GetString() : null; + List? mutedAudioTracks = null; + if (selectionElement.TryGetProperty("mutedAudioTracks", out JsonElement mutedEl) + && mutedEl.ValueKind == JsonValueKind.Array) + { + mutedAudioTracks = mutedEl.EnumerateArray().Select(e => e.GetInt32()).ToList(); + } + Dictionary? audioTrackVolumes = null; + if (selectionElement.TryGetProperty("audioTrackVolumes", out JsonElement volEl) + && volEl.ValueKind == JsonValueKind.Object) + { + audioTrackVolumes = new Dictionary(); + foreach (var prop in volEl.EnumerateObject()) + { + if (int.TryParse(prop.Name, out int trackIdx) && prop.Value.TryGetDouble(out double vol)) + audioTrackVolumes[trackIdx] = vol; + } + } // Create a new Selection instance with all required properties. selections.Add(new Selection @@ -369,7 +388,9 @@ private static async Task HandleCreateClip(JsonElement message) FilePath = filePath, Game = game, Title = title, - IgdbId = igdbId + IgdbId = igdbId, + MutedAudioTracks = mutedAudioTracks, + AudioTrackVolumes = audioTrackVolumes }); } } diff --git a/Backend/App/Program.cs b/Backend/App/Program.cs index eabcd3d4..33acb217 100644 --- a/Backend/App/Program.cs +++ b/Backend/App/Program.cs @@ -217,6 +217,17 @@ static void Main(string[] args) Directory.CreateDirectory(Settings.Instance.ContentFolder); } + // Clean up cached audio track files from previous session + Task.Run(() => + { + string audioTracksCacheFolder = Path.Combine(Settings.Instance.CacheFolder, FolderNames.AudioTracks); + if (Directory.Exists(audioTracksCacheFolder)) + { + try { Directory.Delete(audioTracksCacheFolder, true); } + catch (Exception ex) { Log.Warning($"Failed to clean up audio tracks cache: {ex.Message}"); } + } + }); + // Run data migrations Task.Run(MigrationService.RunMigrations); @@ -445,6 +456,7 @@ private static void LoadFrontend() Log.Information("Loading frontend, app url is " + appUrl); // Initialize the PhotinoWindow Window = new PhotinoWindow() + .SetBrowserControlInitParameters("--enable-blink-features=AudioVideoTracks") .SetNotificationsEnabled(false) // Disabled due to it creating a second start menu entry with incorrect start path. See https://github.com/tryphotino/photino.NET/issues/85 .SetUseOsDefaultSize(false) .SetIconFile(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "icon.ico")) diff --git a/Backend/Media/ClipService.cs b/Backend/Media/ClipService.cs index d727773b..a364c1ab 100644 --- a/Backend/Media/ClipService.cs +++ b/Backend/Media/ClipService.cs @@ -58,14 +58,26 @@ public static async Task CreateClips(List selections) return; } - // Read source audio track names for embedding in clip metadata - List? sourceAudioTrackNames = null; - if (Settings.Instance.ClipKeepSeparateAudioTracks && firstSelection != null) + // Read per-selection audio track names and build union layout + bool anySelectionHasMutedTracks = selections.Any(s => s.MutedAudioTracks != null && s.MutedAudioTracks.Count > 0); + var perSelectionTrackNames = new List?>(); + if (Settings.Instance.ClipKeepSeparateAudioTracks || anySelectionHasMutedTracks) { - sourceAudioTrackNames = GetSourceAudioTrackNames(firstSelection); + foreach (var sel in selections) + perSelectionTrackNames.Add(GetSourceAudioTrackNames(sel)); } + else + { + perSelectionTrackNames.AddRange(Enumerable.Repeat?>(null, selections.Count)); + } + + // Union of all track names across sources -- used to normalise every temp clip to the same stream layout + List? unionAudioLayout = Settings.Instance.ClipKeepSeparateAudioTracks + ? BuildUnionAudioLayout(perSelectionTrackNames) + : null; double processedDuration = 0; + int selectionIndex = 0; foreach (var selection in selections) { // Use the actual file path from metadata when available, fall back to reconstructed path @@ -84,13 +96,17 @@ public static async Task CreateClips(List selections) if (!File.Exists(inputFilePath)) { Log.Information($"Input video file not found: {inputFilePath}"); + selectionIndex++; continue; } string tempFileName = Path.Combine(Path.GetTempPath(), $"clip{Guid.NewGuid()}.mp4"); double clipDuration = selection.EndTime - selection.StartTime; - await ExtractClip(id, inputFilePath, tempFileName, selection.StartTime, selection.EndTime, sourceAudioTrackNames, progress => + List? selectionTrackNames = perSelectionTrackNames[selectionIndex]; + List? targetLayout = Settings.Instance.ClipKeepSeparateAudioTracks ? unionAudioLayout : null; + + await ExtractClip(id, inputFilePath, tempFileName, selection.StartTime, selection.EndTime, selectionTrackNames, selection.MutedAudioTracks, selection.AudioTrackVolumes, targetLayout, progress => { double clampedProgress = Math.Min(progress, 1.0); double currentProgress = (processedDuration + (clampedProgress * clipDuration)) / totalDuration * 95; @@ -106,6 +122,7 @@ await ExtractClip(id, inputFilePath, tempFileName, selection.StartTime, selectio processedDuration += clipDuration; tempClipFiles.Add(tempFileName); + selectionIndex++; } if (!tempClipFiles.Any()) @@ -132,8 +149,21 @@ await ExtractClip(id, inputFilePath, tempFileName, selection.StartTime, selectio try { + string mapAllArg = unionAudioLayout != null ? "-map 0 " : ""; + string concatMetadataArgs = ""; + // When multi-track is active, re-encode audio during concat to fix + // DTS misalignment between streams (different seek offsets cause + // slightly different AAC frame counts per stream). + // Video is always stream-copied. + string codecArg = unionAudioLayout != null + ? $"-c:v copy -c:a aac -b:a {Settings.Instance.ClipAudioQuality} " + : "-c copy "; + if (unionAudioLayout != null) + { + concatMetadataArgs = string.Join(" ", unionAudioLayout.Select((name, i) => $"-metadata:s:a:{i} title=\"{name}\"")) + " "; + } await FFmpegService.RunWithProgress(id, - $"-y -f concat -safe 0 -i \"{concatFilePath}\" -c copy -movflags +faststart \"{outputFilePath}\"", + $"-y -f concat -safe 0 -i \"{concatFilePath}\" {mapAllArg}{codecArg}{concatMetadataArgs}-avoid_negative_ts make_zero -movflags +faststart \"{outputFilePath}\"", totalDuration, progress => { }, process => @@ -170,7 +200,7 @@ await FFmpegService.RunWithProgress(id, _ = MessageService.SendFrontendMessage("ClipProgress", new { id, progress = 98, selections }); - await ContentService.CreateMetadataFile(outputFilePath, Content.ContentType.Clip, firstSelection?.Game!, null, firstSelection?.Title, igdbId: firstSelection?.IgdbId, audioTrackNames: sourceAudioTrackNames); + await ContentService.CreateMetadataFile(outputFilePath, Content.ContentType.Clip, firstSelection?.Game!, null, firstSelection?.Title, igdbId: firstSelection?.IgdbId, audioTrackNames: unionAudioLayout); await ContentService.CreateThumbnail(outputFilePath, Content.ContentType.Clip); await ContentService.CreateWaveformFile(outputFilePath, Content.ContentType.Clip); @@ -205,7 +235,7 @@ await FFmpegService.RunWithProgress(id, } private static async Task ExtractClip(int clipId, string inputFilePath, string outputFilePath, double startTime, double endTime, - List? audioTrackNames, Action progressCallback) + List? audioTrackNames, List? mutedAudioTracks, Dictionary? audioTrackVolumes, List? targetAudioLayout, Action progressCallback) { double duration = endTime - startTime; var settings = Settings.Instance; @@ -288,21 +318,232 @@ private static async Task ExtractClip(int clipId, string inputFilePath, string o } string fpsArg = settings.ClipFps > 0 ? $"-r {settings.ClipFps}" : ""; - string mapArgs = settings.ClipKeepSeparateAudioTracks ? "-map 0:v:0 -map 0:a " : ""; - // Set audio track title metadata so editing software shows the correct names + // Build audio mapping, filter, and metadata based on per-selection muted tracks + string mapArgs = ""; string metadataArgs = ""; - if (settings.ClipKeepSeparateAudioTracks && audioTrackNames != null) + string filterArgs = ""; + string extraInputArgs = ""; + + if (targetAudioLayout != null && targetAudioLayout.Count > 0) + { + // Normalise output to the union layout so every temp clip has identical stream layout. + // Silent (muted/missing) tracks use separate -f lavfi inputs so they are never mixed + // with real decoded streams inside filter_complex -- mixing synthetic sources and real + // streams in the same filtergraph causes a scheduler deadlock in FFmpeg. + var filterParts = new List(); + var mapParts = new List { "-map 0:v:0" }; + var metaParts = new List(); + var extraInputParts = new List(); + int silenceInputIdx = 1; // lavfi inputs start at 1 (0 is the main file) + + bool sourceHasIndividualTracks = audioTrackNames != null && audioTrackNames.Count > 1; + + // Enabled individual source tracks (index > 0, not muted) + var enabledSourceTracks = new List(); + if (sourceHasIndividualTracks) + { + for (int i = 1; i < audioTrackNames!.Count; i++) + { + if (mutedAudioTracks == null || !mutedAudioTracks.Contains(i)) + enabledSourceTracks.Add(i); + } + } + + // Pre-compute which individual track positions map to an enabled source stream + var indivSourceMap = new Dictionary(); // layout position j -> source audio index + if (sourceHasIndividualTracks) + { + for (int j = 1; j < targetAudioLayout.Count; j++) + { + int srcIdx = audioTrackNames!.FindIndex(n => string.Equals(n, targetAudioLayout[j], StringComparison.OrdinalIgnoreCase)); + if (srcIdx > 0 && (mutedAudioTracks == null || !mutedAudioTracks.Contains(srcIdx))) + indivSourceMap[j] = srcIdx; + } + } + + // Count how many times each source audio stream index is referenced + var refCount = new Dictionary(); + foreach (int i in enabledSourceTracks) + { + refCount.TryGetValue(i, out int c); + refCount[i] = c + 1; + } + foreach (int srcIdx in indivSourceMap.Values) + { + refCount.TryGetValue(srcIdx, out int c); + refCount[srcIdx] = c + 1; + } + + // Build asplit filters for streams referenced more than once + var available = new Dictionary>(); + foreach (var (srcIdx, count) in refCount) + { + if (count <= 1) + { + available[srcIdx] = new Queue(new[] { $"[0:a:{srcIdx}]" }); + } + else + { + var labels = Enumerable.Range(0, count).Select(k => $"[split_{srcIdx}_{k}]").ToList(); + filterParts.Add($"[0:a:{srcIdx}]asplit={count}{string.Join("", labels)}"); + available[srcIdx] = new Queue(labels); + } + } + + string durationStr = duration.ToString(CultureInfo.InvariantCulture); + string silenceInput = $"-f lavfi -t {durationStr} -i \"anullsrc=cl=stereo:r=48000\""; + string atrim = $"atrim=end={durationStr},asetpts=PTS-STARTPTS"; + + // Position 0: Full Mix + if (!sourceHasIndividualTracks) + { + // No individual track metadata -- pass through the source's default audio + filterParts.Add($"[0:a:0]{atrim}[out_a0]"); + mapParts.Add("-map \"[out_a0]\""); + } + else if (enabledSourceTracks.Count == 0) + { + extraInputParts.Add(silenceInput); + filterParts.Add($"[{silenceInputIdx}:a:0]{atrim}[out_a0]"); + mapParts.Add("-map \"[out_a0]\""); + silenceInputIdx++; + } + else if (enabledSourceTracks.Count == 1) + { + int trackIdx = enabledSourceTracks[0]; + string volFilter = WithAtrim(GetVolumeFilter(audioTrackVolumes, trackIdx), durationStr); + filterParts.Add($"{available[trackIdx].Dequeue()}{volFilter}[out_a0]"); + mapParts.Add("-map \"[out_a0]\""); + } + else + { + // Apply per-track volume before mixing + var mixInputLabels = new List(); + foreach (int i in enabledSourceTracks) + { + double vol = GetTrackVolume(audioTrackVolumes, i); + string srcLabel = available[i].Dequeue(); + if (Math.Abs(vol - 1.0) > 0.001) + { + string volLabel = $"[vol_mix_{i}]"; + filterParts.Add($"{srcLabel}volume={vol.ToString(CultureInfo.InvariantCulture)}{volLabel}"); + mixInputLabels.Add(volLabel); + } + else + { + mixInputLabels.Add(srcLabel); + } + } + string inputs = string.Join("", mixInputLabels); + filterParts.Add($"{inputs}amix=inputs={enabledSourceTracks.Count}:duration=longest,{atrim}[out_a0]"); + mapParts.Add("-map \"[out_a0]\""); + } + metaParts.Add($"-metadata:s:a:0 title=\"{targetAudioLayout[0]}\""); + + // Positions 1+: individual tracks aligned by name + for (int j = 1; j < targetAudioLayout.Count; j++) + { + if (indivSourceMap.TryGetValue(j, out int srcIdx)) + { + string volFilter = WithAtrim(GetVolumeFilter(audioTrackVolumes, srcIdx), durationStr); + filterParts.Add($"{available[srcIdx].Dequeue()}{volFilter}[out_a{j}]"); + mapParts.Add($"-map \"[out_a{j}]\""); + } + else + { + extraInputParts.Add(silenceInput); + filterParts.Add($"[{silenceInputIdx}:a:0]{atrim}[out_a{j}]"); + mapParts.Add($"-map \"[out_a{j}]\""); + silenceInputIdx++; + } + metaParts.Add($"-metadata:s:a:{j} title=\"{targetAudioLayout[j]}\""); + } + + filterArgs = filterParts.Count > 0 ? $"-filter_complex \"{string.Join(";", filterParts)}\" " : ""; + extraInputArgs = extraInputParts.Count > 0 ? string.Join(" ", extraInputParts) + " " : ""; + mapArgs = string.Join(" ", mapParts) + " "; + metadataArgs = string.Join(" ", metaParts) + " "; + } + else { - for (int i = 0; i < audioTrackNames.Count; i++) + // Legacy paths (keepSeparate=false or no track metadata) + bool hasMutedTracks = mutedAudioTracks != null && mutedAudioTracks.Count > 0 && audioTrackNames != null && audioTrackNames.Count > 1; + bool hasVolumeChanges = audioTrackVolumes != null && audioTrackVolumes.Any(kv => Math.Abs(kv.Value - 1.0) > 0.001); + bool needsAudioProcessing = hasMutedTracks || (hasVolumeChanges && audioTrackNames != null && audioTrackNames.Count > 1); + + if (needsAudioProcessing && audioTrackNames != null) + { + var enabledTracks = new List(); + for (int i = 1; i < audioTrackNames.Count; i++) + { + if (mutedAudioTracks == null || !mutedAudioTracks.Contains(i)) + enabledTracks.Add(i); + } + + if (enabledTracks.Count > 0) + { + bool anyVolChange = enabledTracks.Any(i => Math.Abs(GetTrackVolume(audioTrackVolumes, i) - 1.0) > 0.001); + + if (enabledTracks.Count == 1 && !anyVolChange) + { + mapArgs = $"-map 0:v:0 -map 0:a:{enabledTracks[0]} "; + } + else + { + var filterPartsList = new List(); + var mixInputLabels = new List(); + + foreach (int i in enabledTracks) + { + double vol = GetTrackVolume(audioTrackVolumes, i); + if (Math.Abs(vol - 1.0) > 0.001) + { + filterPartsList.Add($"[0:a:{i}]volume={vol.ToString(CultureInfo.InvariantCulture)}[vol_{i}]"); + mixInputLabels.Add($"[vol_{i}]"); + } + else + { + mixInputLabels.Add($"[0:a:{i}]"); + } + } + + if (enabledTracks.Count == 1) + { + // Single track with volume change + filterArgs = $"-filter_complex \"{string.Join(";", filterPartsList)}\" "; + mapArgs = $"-map 0:v:0 -map \"{mixInputLabels[0]}\" "; + } + else + { + string allInputs = string.Join("", mixInputLabels); + filterPartsList.Add($"{allInputs}amix=inputs={enabledTracks.Count}:duration=longest[mix]"); + filterArgs = $"-filter_complex \"{string.Join(";", filterPartsList)}\" "; + mapArgs = "-map 0:v:0 -map \"[mix]\" "; + } + } + + metadataArgs = "-metadata:s:a:0 title=\"Full Mix\" "; + } + } + else if (settings.ClipKeepSeparateAudioTracks && audioTrackNames != null) { - metadataArgs += $"-metadata:s:a:{i} title=\"{audioTrackNames[i]}\" "; + mapArgs = "-map 0:v:0 -map 0:a "; + for (int i = 0; i < audioTrackNames.Count; i++) + metadataArgs += $"-metadata:s:a:{i} title=\"{audioTrackNames[i]}\" "; } } - string arguments = $"-y -ss {startTime.ToString(CultureInfo.InvariantCulture)} -t {duration.ToString(CultureInfo.InvariantCulture)} " + - $"-i \"{inputFilePath}\" {mapArgs}-c:v {videoCodec} {presetArgs} {qualityArgs} {fpsArg} " + - $"-c:a aac -b:a {settings.ClipAudioQuality} {metadataArgs}-movflags +faststart \"{outputFilePath}\""; + string verboseFlag = filterArgs.Length > 0 || extraInputArgs.Length > 0 ? "-v verbose " : ""; + // When using the union layout, force a consistent sample rate on every output audio stream. + // Source tracks may be 44.1 kHz while the anullsrc silence inputs are 48 kHz, so without this + // each temp clip's per-slot sample rate depends on whether the slot got real audio or silence. + // The concat demuxer then uses the first clip's stream params for subsequent clips, playing + // mismatched samples at the wrong rate (the reported "shrunken audio"). + string audioRateArg = targetAudioLayout != null ? "-ar 48000 " : ""; + string arguments = $"-y {verboseFlag}-ss {startTime.ToString(CultureInfo.InvariantCulture)} -t {duration.ToString(CultureInfo.InvariantCulture)} " + + $"-i \"{inputFilePath}\" {extraInputArgs}{filterArgs}{mapArgs}-c:v {videoCodec} {presetArgs} {qualityArgs} {fpsArg} " + + $"-c:a aac -b:a {settings.ClipAudioQuality} {audioRateArg}{metadataArgs}-t {duration.ToString(CultureInfo.InvariantCulture)} -movflags +faststart \"{outputFilePath}\""; Log.Information("Extracting clip"); Log.Information($"FFmpeg arguments: {arguments}"); @@ -385,6 +626,21 @@ public static async void CancelClip(int clipId) } } + private static List? BuildUnionAudioLayout(List?> perSelectionTrackNames) + { + var union = new List { "Full Mix" }; + foreach (var trackNames in perSelectionTrackNames) + { + if (trackNames == null) continue; + foreach (var name in trackNames.Skip(1)) + { + if (!union.Any(u => string.Equals(u, name, StringComparison.OrdinalIgnoreCase))) + union.Add(name); + } + } + return union.Count > 1 ? union : null; + } + private static List? GetSourceAudioTrackNames(Selection selection) { try @@ -416,5 +672,27 @@ private static void SafeDelete(string path) try { File.Delete(path); } catch (Exception ex) { Log.Information($"Error deleting file {path}: {ex.Message}"); } } + + private static string WithAtrim(string filterChain, string durationStr) + { + if (filterChain == "acopy") + return $"atrim=end={durationStr},asetpts=PTS-STARTPTS"; + return $"{filterChain},atrim=end={durationStr},asetpts=PTS-STARTPTS"; + } + + private static double GetTrackVolume(Dictionary? audioTrackVolumes, int trackIndex) + { + if (audioTrackVolumes != null && audioTrackVolumes.TryGetValue(trackIndex, out double vol)) + return Math.Max(0, Math.Min(1, vol)); + return 1.0; + } + + private static string GetVolumeFilter(Dictionary? audioTrackVolumes, int trackIndex) + { + double vol = GetTrackVolume(audioTrackVolumes, trackIndex); + if (Math.Abs(vol - 1.0) > 0.001) + return $"volume={vol.ToString(CultureInfo.InvariantCulture)}"; + return "acopy"; + } } } diff --git a/Backend/Media/FFmpegService.cs b/Backend/Media/FFmpegService.cs index 08962da3..91f15874 100644 --- a/Backend/Media/FFmpegService.cs +++ b/Backend/Media/FFmpegService.cs @@ -347,6 +347,15 @@ public static async Task CreateThumbnailFile(string inputFilePath, string output await RunSimple(arguments); } + /// + /// Extracts a single audio track from a video file to an M4A file (copy, no re-encoding) + /// + public static async Task ExtractAudioTrack(string inputFilePath, string outputM4aPath, int audioStreamIndex) + { + string arguments = $"-y -i \"{inputFilePath}\" -map 0:a:{audioStreamIndex} -c copy \"{outputM4aPath}\""; + await RunSimple(arguments); + } + /// /// Extracts audio as PCM data for waveform generation /// diff --git a/Backend/Recorder/OBSService.cs b/Backend/Recorder/OBSService.cs index 613c5ece..e9d36173 100644 --- a/Backend/Recorder/OBSService.cs +++ b/Backend/Recorder/OBSService.cs @@ -939,7 +939,7 @@ public static bool StartRecording(string name = "Manual Recording", string exePa GeneralUtils.SetProcessPriority(ProcessPriorityClass.High); if (!isReplayBufferMode) { - _ = GameIntegrationService.Start(name); + _ = GameIntegrationService.Start(GameUtils.GetIgdbIdFromExePath(exePath)); } Task.Run(KeybindCaptureService.Start); return true; diff --git a/Backend/Services/GameIntegrationService.cs b/Backend/Services/GameIntegrationService.cs index bed80751..99de6a39 100644 --- a/Backend/Services/GameIntegrationService.cs +++ b/Backend/Services/GameIntegrationService.cs @@ -10,14 +10,14 @@ namespace Segra.Backend.Services { public static class GameIntegrationService { - private const string PUBG = "PLAYERUNKNOWN'S BATTLEGROUNDS"; - private const string LOL = "League of Legends"; - private const string CS2 = "Counter-Strike 2"; - private const string ROCKET_LEAGUE = "Rocket League"; + private const int PUBG_IGDB_ID = 27789; + private const int LOL_IGDB_ID = 115; + private const int CS2_IGDB_ID = 242408; + private const int ROCKET_LEAGUE_IGDB_ID = 11198; private static Integration? _gameIntegration; public static Integration? GameIntegration => _gameIntegration; - public static async Task Start(string gameName) + public static async Task Start(int? igdbId) { if (_gameIntegration != null) { @@ -25,12 +25,12 @@ public static async Task Start(string gameName) await _gameIntegration.Shutdown(); } - _gameIntegration = gameName switch + _gameIntegration = igdbId switch { - PUBG => Settings.Instance.GameIntegrations.Pubg.Enabled ? new PubgIntegration() : null, - LOL => Settings.Instance.GameIntegrations.LeagueOfLegends.Enabled ? new LeagueOfLegendsIntegration() : null, - CS2 => Settings.Instance.GameIntegrations.CounterStrike2.Enabled ? new CounterStrike2Integration() : null, - ROCKET_LEAGUE => Settings.Instance.GameIntegrations.RocketLeague.Enabled ? new RocketLeagueIntegration() : null, + PUBG_IGDB_ID => Settings.Instance.GameIntegrations.Pubg.Enabled ? new PubgIntegration() : null, + LOL_IGDB_ID => Settings.Instance.GameIntegrations.LeagueOfLegends.Enabled ? new LeagueOfLegendsIntegration() : null, + CS2_IGDB_ID => Settings.Instance.GameIntegrations.CounterStrike2.Enabled ? new CounterStrike2Integration() : null, + ROCKET_LEAGUE_IGDB_ID => Settings.Instance.GameIntegrations.RocketLeague.Enabled ? new RocketLeagueIntegration() : null, _ => null, }; @@ -39,7 +39,7 @@ public static async Task Start(string gameName) return; } - Log.Information($"Starting game integration for: {gameName}"); + Log.Information($"Starting game integration for IGDB ID: {igdbId}"); _ = _gameIntegration.Start(); } diff --git a/Backend/Shared/FolderNames.cs b/Backend/Shared/FolderNames.cs index 0ff372e8..d6a208a5 100644 --- a/Backend/Shared/FolderNames.cs +++ b/Backend/Shared/FolderNames.cs @@ -24,6 +24,7 @@ public static class FolderNames public const string Metadata = "metadata"; public const string Thumbnails = "thumbnails"; public const string Waveforms = "waveforms"; + public const string AudioTracks = "audiotracks"; // Legacy metadata folder names (with dot prefix, for migration) public const string LegacyMetadata = ".metadata"; @@ -118,6 +119,15 @@ public static string GetWaveformsFolderPath(Content.ContentType type) return Path.Combine(CacheFolder, Waveforms, GetMetadataSubfolderName(type)); } + /// + /// Gets the full path to the audio tracks folder for a content type. + /// Audio tracks are stored in AppData/Roaming/Segra/audiotracks/{ContentType} + /// + public static string GetAudioTracksFolderPath(Content.ContentType type) + { + return Path.Combine(CacheFolder, AudioTracks, GetMetadataSubfolderName(type)); + } + /// /// Checks if a path contains a specific content type folder. /// Useful for determining content type from a file path. diff --git a/Frontend/src/Components/SelectionCard.tsx b/Frontend/src/Components/SelectionCard.tsx index 938d0aba..0c5ddceb 100644 --- a/Frontend/src/Components/SelectionCard.tsx +++ b/Frontend/src/Components/SelectionCard.tsx @@ -1,6 +1,7 @@ -import React from 'react'; +import React, { useLayoutEffect, useRef, useState } from 'react'; import { SelectionCardProps } from '../Models/types'; import { useDrag, useDrop } from 'react-dnd'; +import { TbHeadphones } from 'react-icons/tb'; const DRAG_TYPE = 'SELECTION_CARD'; @@ -13,7 +14,19 @@ const SelectionCard: React.FC = React.memo( isHovered, setHoveredSelectionId, removeSelection, + audioTrackNames, + onMutedAudioTracksChange, + onAudioTrackVolumesChange, }) => { + const [audioMenuPos, setAudioMenuPos] = useState<{ x: number; y: number } | null>(null); + + const indexRef = useRef(index); + const moveCardRef = useRef(moveCard); + useLayoutEffect(() => { + indexRef.current = index; + moveCardRef.current = moveCard; + }); + const [{ isDragging }, dragRef] = useDrag( () => ({ type: DRAG_TYPE, @@ -29,13 +42,13 @@ const SelectionCard: React.FC = React.memo( () => ({ accept: DRAG_TYPE, hover: (item: { index: number }) => { - if (item.index !== index) { - moveCard(item.index, index); - item.index = index; + if (item.index !== indexRef.current) { + moveCardRef.current(item.index, indexRef.current); + item.index = indexRef.current; } }, }), - [index, moveCard], + [], ); const dragDropRef = (node: HTMLDivElement | null) => { @@ -44,6 +57,19 @@ const SelectionCard: React.FC = React.memo( }; const { startTime, endTime, thumbnailDataUrl, isLoading } = selection; + const hasAudioTracks = + audioTrackNames && audioTrackNames.length > 1 && onMutedAudioTracksChange; + const mutedTracks = selection.mutedAudioTracks ?? []; + const trackVolumes = selection.audioTrackVolumes ?? {}; + + const toggleTrack = (trackIndex: number) => { + if (!onMutedAudioTracksChange) return; + const isMuted = mutedTracks.includes(trackIndex); + const newMuted = isMuted + ? mutedTracks.filter((t) => t !== trackIndex) + : [...mutedTracks, trackIndex]; + onMutedAudioTracksChange(selection.id, newMuted); + }; return (
= React.memo( className={`mb-2 cursor-move w-full relative rounded-xl transition-all duration-200 !outline !outline-1 ${isHovered ? '!outline-primary' : '!outline-base-400'}`} style={{ opacity: isDragging ? 0.3 : 1 }} onMouseEnter={() => setHoveredSelectionId(selection.id)} - onMouseLeave={() => setHoveredSelectionId(null)} + onMouseLeave={() => { + setHoveredSelectionId(null); + setAudioMenuPos(null); + }} onContextMenu={(e) => { e.preventDefault(); removeSelection(selection.id); @@ -76,6 +105,76 @@ const SelectionCard: React.FC = React.memo( No thumbnail
)} + + {hasAudioTracks && ( +
+ + {audioMenuPos && ( +
e.stopPropagation()} + > + {audioTrackNames.map((name, i) => { + // Skip track 0 (Full Mix) + if (i === 0) return null; + const isMuted = mutedTracks.includes(i); + const vol = trackVolumes[i] ?? 1; + return ( +
+
+ toggleTrack(i)} + className="checkbox checkbox-primary checkbox-xs shrink-0" + /> + + {name.replace(' (Default)', '')} + +
+
+ { + if (!onAudioTrackVolumesChange) return; + const newVolumes = { ...trackVolumes, [i]: parseFloat(e.target.value) }; + onAudioTrackVolumesChange(selection.id, newVolumes); + }} + className="w-16 h-1 bg-gray-600 rounded-lg appearance-none cursor-pointer accent-accent" + /> + + {Math.round(vol * 100)}% + +
+
+ ); + })} +
+ )} +
+ )} ); }, diff --git a/Frontend/src/Hooks/useAudioTracks.ts b/Frontend/src/Hooks/useAudioTracks.ts new file mode 100644 index 00000000..a3a34845 --- /dev/null +++ b/Frontend/src/Hooks/useAudioTracks.ts @@ -0,0 +1,303 @@ +import { useEffect, useLayoutEffect, useRef, useState, useCallback } from 'react'; +import { Content } from '../Models/types'; + +export interface AudioTrackInfo { + index: number; + name: string; + url: string; +} + +export interface AudioTrackState { + tracks: AudioTrackInfo[]; + volumes: Record; + mutedTracks: Set; + soloTrack: number | null; + masterMuted: boolean; + masterVolume: number; + setTrackVolume: (index: number, volume: number) => void; + toggleTrackMute: (index: number) => void; + setMutedTracks: (muted: Set) => void; + toggleSolo: (index: number) => void; + setMasterMuted: (muted: boolean) => void; + setMasterVolume: (vol: number) => void; + setMuteOverride: (mutedIndices: number[] | null) => void; + setVolumeOverride: (volumes: Record | null) => void; + isMultiTrack: boolean; +} + +const SYNC_THRESHOLD = 0.15; +const SYNC_CHECK_INTERVAL = 500; + +function cleanupAudioElements(elements: Map) { + for (const audio of elements.values()) { + audio.pause(); + audio.src = ''; + audio.load(); + } + elements.clear(); +} + +export function useAudioTracks( + videoRef: React.RefObject, + video: Content, +): AudioTrackState { + const audioElementsRef = useRef>(new Map()); + const [tracks, setTracks] = useState([]); + const [volumes, setVolumes] = useState>({}); + const [mutedTracks, setMutedTracks] = useState>(new Set()); + const [soloTrack, setSoloTrack] = useState(null); + const isMultiTrack = tracks.length > 1; + + // Fetch extracted audio track files from backend + useEffect(() => { + if (!video.audioTrackNames || video.audioTrackNames.length <= 1) { + setTracks([]); + return; + } + + let cancelled = false; + + const fetchTracks = async () => { + try { + const url = `http://localhost:2222/api/audiotracks?input=${encodeURIComponent(video.filePath)}&type=${video.type}`; + const res = await fetch(url); + const data: AudioTrackInfo[] = await res.json(); + + // Exclude track 0 (Full Mix) since it doubles individual tracks + const individualTracks = data.filter((t) => t.index > 0); + if (cancelled || individualTracks.length === 0) return; + + setTracks(individualTracks); + + const initialVolumes: Record = {}; + for (const t of individualTracks) { + initialVolumes[t.index] = 1; + } + setVolumes(initialVolumes); + setMutedTracks(new Set()); + setSoloTrack(null); + + // Clean up existing elements before creating new ones + cleanupAudioElements(audioElementsRef.current); + + const elements = new Map(); + for (const track of individualTracks) { + const audio = new Audio(); + audio.src = `http://localhost:2222${track.url}`; + audio.preload = 'auto'; + audio.volume = initialVolumes[track.index] ?? 1; + elements.set(track.index, audio); + } + audioElementsRef.current = elements; + } catch (err) { + console.error('Failed to load audio tracks:', err); + } + }; + + fetchTracks(); + + return () => { + cancelled = true; + cleanupAudioElements(audioElementsRef.current); + setTracks([]); + }; + }, [video.filePath, video.audioTrackNames?.length]); + + // Sync audio elements with video: play/pause/seek/rate + mute video + useEffect(() => { + const vid = videoRef.current; + if (!vid || !isMultiTrack) return; + + // Mute the video element here (not earlier) so that video.tsx's + // initialization effect has already seen isMultiTrack=true and + // skipped its own vid.muted = isMuted assignment. + vid.muted = true; + + const syncTime = () => { + const t = vid.currentTime; + for (const audio of audioElementsRef.current.values()) { + audio.currentTime = t; + } + }; + + const onPlay = () => { + syncTime(); + for (const audio of audioElementsRef.current.values()) { + audio.play().catch(() => {}); + } + }; + + const onPause = () => { + for (const audio of audioElementsRef.current.values()) { + audio.pause(); + } + }; + + const onSeeked = () => { + syncTime(); + }; + + const onRateChange = () => { + const rate = vid.playbackRate; + for (const audio of audioElementsRef.current.values()) { + audio.playbackRate = rate; + } + }; + + vid.addEventListener('play', onPlay); + vid.addEventListener('pause', onPause); + vid.addEventListener('seeked', onSeeked); + vid.addEventListener('ratechange', onRateChange); + + // Initial sync if already playing + if (!vid.paused) { + for (const audio of audioElementsRef.current.values()) { + audio.playbackRate = vid.playbackRate; + audio.currentTime = vid.currentTime; + audio.play().catch(() => {}); + } + } + + // Periodic drift correction + const intervalId = setInterval(() => { + if (vid.paused) return; + const vidTime = vid.currentTime; + for (const audio of audioElementsRef.current.values()) { + const drift = Math.abs(audio.currentTime - vidTime); + if (drift > SYNC_THRESHOLD) { + audio.currentTime = vidTime; + } + } + }, SYNC_CHECK_INTERVAL); + + return () => { + vid.removeEventListener('play', onPlay); + vid.removeEventListener('pause', onPause); + vid.removeEventListener('seeked', onSeeked); + vid.removeEventListener('ratechange', onRateChange); + clearInterval(intervalId); + + // Restore video audio when multi-track is deactivated + vid.muted = localStorage.getItem('segra-muted') === 'true'; + }; + }, [videoRef, isMultiTrack]); + + // Master mute/volume -- playback-only controls, don't affect per-track state or clip output. + const masterMutedRef = useRef(false); + const [masterMuted, setMasterMutedState] = useState(false); + const masterVolumeRef = useRef(1); + const [masterVolume, setMasterVolumeState] = useState(1); + + // Overrides for per-selection muting/volumes -- stored as refs so changes don't cause re-renders. + // When set, audio elements use these instead of the default state. + const muteOverrideRef = useRef | null>(null); + const volumeOverrideRef = useRef | null>(null); + + // Keep latest state in refs so the stable setMuteOverride can read them + const latestRef = useRef({ mutedTracks, soloTrack, volumes }); + useLayoutEffect(() => { + latestRef.current = { mutedTracks, soloTrack, volumes }; + }); + + const applyMuting = useCallback(() => { + const { mutedTracks: defaultMuted, soloTrack: solo, volumes: vols } = latestRef.current; + const effectiveMuted = muteOverrideRef.current ?? defaultMuted; + const effectiveVolumes = volumeOverrideRef.current ?? vols; + for (const [index, audio] of audioElementsRef.current.entries()) { + if (masterMutedRef.current) { + audio.muted = true; + } else if (solo !== null) { + audio.muted = index !== solo; + } else { + audio.muted = effectiveMuted.has(index); + } + audio.volume = (effectiveVolumes[index] ?? 1) * masterVolumeRef.current; + } + }, []); + + // Apply when React state changes + useEffect(() => { + applyMuting(); + }, [volumes, mutedTracks, soloTrack, tracks, applyMuting]); + + const setTrackVolume = useCallback((index: number, volume: number) => { + const clamped = Math.max(0, Math.min(1, volume)); + setVolumes((prev) => ({ ...prev, [index]: clamped })); + }, []); + + const toggleTrackMute = useCallback((index: number) => { + setMutedTracks((prev) => { + const next = new Set(prev); + if (next.has(index)) { + next.delete(index); + } else { + next.add(index); + } + return next; + }); + }, []); + + const replaceMutedTracks = useCallback((muted: Set) => { + setMutedTracks(muted); + }, []); + + const toggleSolo = useCallback((index: number) => { + setSoloTrack((prev) => (prev === index ? null : index)); + }, []); + + const setMasterMuted = useCallback( + (muted: boolean) => { + masterMutedRef.current = muted; + setMasterMutedState(muted); + applyMuting(); + }, + [applyMuting], + ); + + const setMasterVolume = useCallback( + (vol: number) => { + const clamped = Math.max(0, Math.min(1, vol)); + masterVolumeRef.current = clamped; + setMasterVolumeState(clamped); + applyMuting(); + }, + [applyMuting], + ); + + // Stable function to set/clear a per-selection mute override. + // Directly applies to audio elements without touching React state. + const setMuteOverride = useCallback( + (mutedIndices: number[] | null) => { + muteOverrideRef.current = mutedIndices ? new Set(mutedIndices) : null; + applyMuting(); + }, + [applyMuting], + ); + + // Stable function to set/clear a per-selection volume override. + const setVolumeOverride = useCallback( + (vols: Record | null) => { + volumeOverrideRef.current = vols; + applyMuting(); + }, + [applyMuting], + ); + + return { + tracks, + volumes, + mutedTracks, + soloTrack, + masterMuted, + masterVolume, + setTrackVolume, + toggleTrackMute, + setMutedTracks: replaceMutedTracks, + toggleSolo, + setMasterMuted, + setMasterVolume, + setMuteOverride, + setVolumeOverride, + isMultiTrack, + }; +} diff --git a/Frontend/src/Models/types.ts b/Frontend/src/Models/types.ts index fa2042d5..9fd9c269 100644 --- a/Frontend/src/Models/types.ts +++ b/Frontend/src/Models/types.ts @@ -20,6 +20,7 @@ export interface Content { uploadId?: string; igdbId?: number; isImported: boolean; + audioTrackNames?: string[]; } export interface OBSVersion { @@ -349,6 +350,8 @@ export interface Selection { game?: string; title?: string; igdbId?: number; + mutedAudioTracks?: number[]; + audioTrackVolumes?: Record; } export interface SelectionCardProps { @@ -359,6 +362,9 @@ export interface SelectionCardProps { isHovered: boolean; setHoveredSelectionId: (id: number | null) => void; removeSelection: (id: number) => void; + audioTrackNames?: string[]; + onMutedAudioTracksChange?: (id: number, mutedTracks: number[]) => void; + onAudioTrackVolumesChange?: (id: number, volumes: Record) => void; } export interface AiProgress { diff --git a/Frontend/src/Pages/video.tsx b/Frontend/src/Pages/video.tsx index 17c8e81f..eec34623 100644 --- a/Frontend/src/Pages/video.tsx +++ b/Frontend/src/Pages/video.tsx @@ -35,7 +35,8 @@ import { import { IoSkull, IoAdd, IoRemove } from 'react-icons/io5'; import SelectionCard from '../Components/SelectionCard'; -import { TbZoomIn, TbZoomOut } from 'react-icons/tb'; +import { TbZoomIn, TbZoomOut, TbHeadphones } from 'react-icons/tb'; +import { useAudioTracks } from '../Hooks/useAudioTracks'; import { IoIosFootball } from 'react-icons/io'; import { AnimatePresence, motion } from 'framer-motion'; import Button from '../Components/Button'; @@ -211,6 +212,15 @@ export default function VideoComponent({ video }: { video: Content }) { const waveformStateRef = useRef({ pixelsPerSecond: 0, duration: 0 }); const waveformBufferRef = useRef({ regionLeft: 0, regionRight: 0 }); + // Audio tracks + const audioTracks = useAudioTracks(videoRef, video); + const [showAudioTracks, setShowAudioTracks] = useState(false); + const [timelineAudioMenu, setTimelineAudioMenu] = useState<{ + selId: number; + x: number; + y: number; + } | null>(null); + // Video state const [currentTime, setCurrentTime] = useState(0); const [duration, setDuration] = useState(0); @@ -256,6 +266,14 @@ export default function VideoComponent({ video }: { video: Content }) { }; }, [showSpeedMenu]); + // Close timeline audio menu when clicking outside + useEffect(() => { + if (!timelineAudioMenu) return; + const handleClickOutside = () => setTimelineAudioMenu(null); + document.addEventListener('mousedown', handleClickOutside); + return () => document.removeEventListener('mousedown', handleClickOutside); + }, [timelineAudioMenu]); + // Clamp translation so the video remains at least partially visible const clampTranslate = (t: { x: number; y: number }) => { const el = playerContainerRef.current; @@ -365,6 +383,7 @@ export default function VideoComponent({ video }: { video: Content }) { useEffect(() => { if (!controlsVisible) { setShowSpeedMenu(false); + setShowAudioTracks(false); speedButtonRef.current?.blur(); } }, [controlsVisible]); @@ -455,8 +474,11 @@ export default function VideoComponent({ video }: { video: Content }) { if (!vid) return; // Apply saved volume and muted state on load - vid.volume = volume; - vid.muted = isMuted; + // When multi-track is active, the hook controls muting + if (!audioTracks.isMultiTrack) { + vid.volume = volume; + vid.muted = isMuted; + } // Apply saved playback rate vid.playbackRate = playbackRate; @@ -469,6 +491,9 @@ export default function VideoComponent({ video }: { video: Content }) { const onPause = () => setIsPlaying(false); const onVolumeChange = () => { if (vid) { + // When multi-track audio is active, the video is muted by the hook + if (audioTracks.isMultiTrack) return; + setVolume(vid.volume); setIsMuted(vid.muted); @@ -569,21 +594,73 @@ export default function VideoComponent({ video }: { video: Content }) { vid.removeEventListener('ratechange', onRateChange); window.removeEventListener('keydown', handleKeyDown, keyOptions as any); }; - }, [volume, isMuted, isFullscreen]); + }, [volume, isMuted, isFullscreen, audioTracks.isMultiTrack]); + + // Per-selection audio override state, kept in refs for the rAF loop below. + const activeSelectionIdRef = useRef(null); + const audioTracksRef = useRef(audioTracks); + useLayoutEffect(() => { + audioTracksRef.current = audioTracks; + }); + + // Reset active selection when selections change so the override is re-applied + useEffect(() => { + activeSelectionIdRef.current = null; + }, [selections]); - // Handle video playback time updates using requestAnimationFrame for smooth updates + // Clean up overrides when multi-track is deactivated + useEffect(() => { + if (audioTracks.isMultiTrack) { + // Sync master mute/volume from saved state when entering multi-track + audioTracks.setMasterMuted(localStorage.getItem('segra-muted') === 'true'); + const savedVol = localStorage.getItem('segra-volume'); + audioTracks.setMasterVolume(savedVol ? parseFloat(savedVol) : 1); + } else { + audioTracks.setMuteOverride(null); + audioTracks.setVolumeOverride(null); + } + }, [ + audioTracks.isMultiTrack, + audioTracks.setMasterMuted, + audioTracks.setMuteOverride, + audioTracks.setVolumeOverride, + ]); + + // Handle video playback time updates using requestAnimationFrame for smooth updates. + // Also checks per-selection audio overrides each frame (cheap: one find + early return). useEffect(() => { const vid = videoRef.current; if (!vid) return; let rafId = 0; - const updateCurrentTime = () => { + const tick = () => { setCurrentTime(vid.currentTime); + + // Per-selection audio mute/volume override + const at = audioTracksRef.current; + if (at.isMultiTrack) { + const t = vid.currentTime; + const sels = selectionsRef.current; + const activeSel = sels.find((s) => t >= s.startTime && t <= s.endTime); + const activeId = activeSel?.id ?? null; + + if (activeId !== activeSelectionIdRef.current) { + activeSelectionIdRef.current = activeId; + if (activeSel) { + at.setMuteOverride(activeSel.mutedAudioTracks ?? []); + at.setVolumeOverride(activeSel.audioTrackVolumes ?? null); + } else { + at.setMuteOverride(null); + at.setVolumeOverride(null); + } + } + } + if (!vid.paused && !vid.ended) { - rafId = requestAnimationFrame(updateCurrentTime); + rafId = requestAnimationFrame(tick); } }; const onPlay = () => { - rafId = requestAnimationFrame(updateCurrentTime); + rafId = requestAnimationFrame(tick); }; const onPause = () => { cancelAnimationFrame(rafId); @@ -794,17 +871,32 @@ export default function VideoComponent({ video }: { video: Content }) { setVolume(target); return; } - el.volume = target; - if (target === 0) { - el.muted = true; - setIsMuted(true); - } else if (el.muted) { - el.muted = false; - setIsMuted(false); + if (audioTracks.isMultiTrack) { + // Route to master volume -- purely a preview control + audioTracks.setMasterVolume(target); + if (target === 0) { + audioTracks.setMasterMuted(true); + setIsMuted(true); + } else if (audioTracks.masterMuted) { + audioTracks.setMasterMuted(false); + setIsMuted(false); + } + } else { + el.volume = target; + if (target === 0) { + el.muted = true; + setIsMuted(true); + } else if (el.muted) { + el.muted = false; + setIsMuted(false); + } } setVolume(target); localStorage.setItem('segra-volume', target.toString()); - localStorage.setItem('segra-muted', el.muted.toString()); + localStorage.setItem( + 'segra-muted', + (audioTracks.isMultiTrack ? audioTracks.masterMuted : el.muted).toString(), + ); }; // Pointer handlers for panning the video when zoomed @@ -985,6 +1077,12 @@ export default function VideoComponent({ video }: { video: Content }) { game: video.game, title: video.title, igdbId: video.igdbId, + mutedAudioTracks: + selections.length > 0 + ? selections[selections.length - 1].mutedAudioTracks + : audioTracks.isMultiTrack + ? [...audioTracks.mutedTracks] + : undefined, }; addSelection(newSelection); // Kick off thumbnail generation; uses latest state and guards against stale overwrites @@ -1010,6 +1108,8 @@ export default function VideoComponent({ video }: { video: Content }) { startTime: s.startTime, endTime: s.endTime, igdbId: s.igdbId, + mutedAudioTracks: s.mutedAudioTracks, + audioTrackVolumes: s.audioTrackVolumes, })), }; sendMessageToBackend('CreateClip', params); @@ -1417,11 +1517,18 @@ export default function VideoComponent({ video }: { video: Content }) { // Toggle mute state const toggleMute = () => { if (videoRef.current) { + if (audioTracks.isMultiTrack) { + // Master mute for multi-track: silences all audio elements + const newMuted = !audioTracks.masterMuted; + audioTracks.setMasterMuted(newMuted); + setIsMuted(newMuted); + localStorage.setItem('segra-muted', newMuted.toString()); + return; + } + const newMutedState = !videoRef.current.muted; videoRef.current.muted = newMutedState; setIsMuted(newMutedState); - - // Save to localStorage localStorage.setItem('segra-muted', newMutedState.toString()); } }; @@ -1547,6 +1654,66 @@ export default function VideoComponent({ video }: { video: Content }) { /> + {audioTracks.isMultiTrack && ( +
+ + {showAudioTracks && ( +
+ {audioTracks.tracks.map((track) => { + const isMuted = audioTracks.mutedTracks.has(track.index); + const vol = audioTracks.volumes[track.index] ?? 1; + return ( +
+
+ audioTracks.toggleTrackMute(track.index)} + className="checkbox checkbox-primary checkbox-xs shrink-0" + /> + + {track.name.replace(' (Default)', '')} + +
+
+ + audioTracks.setTrackVolume( + track.index, + parseFloat(e.target.value), + ) + } + className="w-16 h-1 bg-gray-600 rounded-lg appearance-none cursor-pointer accent-accent" + /> + + {Math.round(vol * 100)}% + +
+
+ ); + })} +
+ )} +
+ )} +
+ )} +
handleResizeMouseDown(e, sel.id, 'start')} @@ -1793,6 +1980,70 @@ export default function VideoComponent({ video }: { video: Content }) { )}
+ {timelineAudioMenu && + (() => { + const menuSel = selections.find((s) => s.id === timelineAudioMenu.selId); + if (!menuSel || !video.audioTrackNames) return null; + const mutedTracks = menuSel.mutedAudioTracks ?? []; + const trackVolumes = menuSel.audioTrackVolumes ?? {}; + return ( +
e.stopPropagation()} + onClick={(e) => e.stopPropagation()} + > + {video.audioTrackNames.map((name, i) => { + if (i === 0) return null; + const isMuted = mutedTracks.includes(i); + const vol = trackVolumes[i] ?? 1; + return ( +
+
+ { + const newMuted = isMuted + ? mutedTracks.filter((t) => t !== i) + : [...mutedTracks, i]; + updateSelection({ ...menuSel, mutedAudioTracks: newMuted }); + }} + className="checkbox checkbox-primary checkbox-xs shrink-0" + /> + + {name.replace(' (Default)', '')} + +
+
+ { + const newVolumes = { + ...trackVolumes, + [i]: parseFloat(e.target.value), + }; + updateSelection({ ...menuSel, audioTrackVolumes: newVolumes }); + }} + className="w-16 h-1 bg-gray-600 rounded-lg appearance-none cursor-pointer accent-accent" + /> + + {Math.round(vol * 100)}% + +
+
+ ); + })} +
+ ); + })()}
@@ -1931,6 +2182,19 @@ export default function VideoComponent({ video }: { video: Content }) { isHovered={hoveredSelectionId === sel.id} setHoveredSelectionId={setHoveredSelectionId} removeSelection={removeSelection} + audioTrackNames={video.audioTrackNames} + onMutedAudioTracksChange={(id, mutedTracks) => + updateSelection({ + ...selections.find((s) => s.id === id)!, + mutedAudioTracks: mutedTracks, + }) + } + onAudioTrackVolumesChange={(id, volumes) => + updateSelection({ + ...selections.find((s) => s.id === id)!, + audioTrackVolumes: volumes, + }) + } /> ))}