Feat/separate audio tracks

Backend/Api/ContentServer.cs

@@ -1,14 +1,18 @@
 using Serilog;
 using System.Net;
+using System.Text.Json;
 using System.Web;
+using Segra.Backend.Core.Models;
 using Segra.Backend.Media;
+using Segra.Backend.Shared;
 
 namespace Segra.Backend.Api
 {
     internal class ContentServer
     {
         private static readonly HttpListener _httpListener = new();
         private static CancellationTokenSource? _cancellationTokenSource;
+        private static readonly System.Collections.Concurrent.ConcurrentDictionary<string, Task> _extractionTasks = new();
 
         public static void StartServer(string prefix)
         {
@@ -63,6 +67,10 @@ private static async Task ProcessRequestAsync(HttpListenerContext context)
                 {
                     await HandleThumbnailRequest(context);
                 }
+                else if (rawUrl.StartsWith("/api/audiotracks"))
+                {
+                    await HandleAudioTracksRequest(context);
+                }
                 else if (rawUrl.StartsWith("/api/content"))
                 {
                     await HandleContentRequest(context);
@@ -217,7 +225,8 @@ private static async Task HandleContentRequest(HttpListenerContext context)
                 return;
             }
 
-            if (fileName.EndsWith(".mp4", StringComparison.OrdinalIgnoreCase))
+            if (fileName.EndsWith(".mp4", StringComparison.OrdinalIgnoreCase) ||
+                fileName.EndsWith(".m4a", StringComparison.OrdinalIgnoreCase))
             {
                 await StreamVideoFile(fileName, context);
             }
@@ -236,6 +245,114 @@ private static async Task HandleContentRequest(HttpListenerContext context)
             }
         }
 
+        private static async Task HandleAudioTracksRequest(HttpListenerContext context)
+        {
+            var query = HttpUtility.ParseQueryString(context.Request?.Url?.Query ?? "");
+            string input = query["input"] ?? "";
+            string typeStr = query["type"] ?? "";
+            var response = context.Response;
+
+            response.AddHeader("Access-Control-Allow-Origin", "*");
+
+            if (!File.Exists(input))
+            {
+                response.StatusCode = (int)HttpStatusCode.NotFound;
+                response.ContentType = "text/plain";
+                using (var writer = new StreamWriter(response.OutputStream))
+                {
+                    await writer.WriteAsync("File not found.");
+                }
+                return;
+            }
+
+            try
+            {
+                if (!Enum.TryParse<Content.ContentType>(typeStr, true, out var contentType))
+                {
+                    contentType = FolderNames.GetContentTypeFromPath(input) ?? Content.ContentType.Session;
+                }
+
+                // Read metadata to get audio track names
+                string fileName = Path.GetFileNameWithoutExtension(input);
+                string metadataFolderPath = FolderNames.GetMetadataFolderPath(contentType);
+                string metadataFilePath = Path.Combine(metadataFolderPath, $"{fileName}.json");
+
+                List<string>? trackNames = null;
+                if (File.Exists(metadataFilePath))
+                {
+                    var metadataContent = await File.ReadAllTextAsync(metadataFilePath);
+                    var metadata = JsonSerializer.Deserialize<Content>(metadataContent);
+                    trackNames = metadata?.AudioTrackNames;
+                }
+
+                if (trackNames == null || trackNames.Count <= 1)
+                {
+                    response.StatusCode = (int)HttpStatusCode.OK;
+                    response.ContentType = "application/json";
+                    using (var writer = new StreamWriter(response.OutputStream))
+                    {
+                        await writer.WriteAsync("[]");
+                    }
+                    return;
+                }
+
+                string audioTracksDir = FolderNames.GetAudioTracksFolderPath(contentType);
+                Directory.CreateDirectory(audioTracksDir);
+
+                var result = new List<object>();
+
+                // Skip track 0 (Full Mix) - it duplicates the individual tracks
+                for (int i = 1; i < trackNames.Count; i++)
+                {
+                    string trackFileName = $"{fileName}_track{i}.m4a";
+                    string trackFilePath = Path.Combine(audioTracksDir, trackFileName);
+
+                    if (!File.Exists(trackFilePath))
+                    {
+                        try
+                        {
+                            var task = _extractionTasks.GetOrAdd(trackFilePath, _ => FFmpegService.ExtractAudioTrack(input, trackFilePath, i));
+                            await task;
+                        }
+                        catch (Exception ex)
+                        {
+                            Log.Warning($"Failed to extract audio track {i} from {input}: {ex.Message}");
+                            continue;
+                        }
+                        finally
+                        {
+                            _extractionTasks.TryRemove(trackFilePath, out _);
+                        }
+                    }
+
+                    result.Add(new
+                    {
+                        index = i,
+                        name = trackNames[i],
+                        url = $"/api/content?input={Uri.EscapeDataString(trackFilePath)}"
+                    });
+                }
+
+                response.StatusCode = (int)HttpStatusCode.OK;
+                response.ContentType = "application/json";
+                string json = JsonSerializer.Serialize(result);
+                using (var writer = new StreamWriter(response.OutputStream))
+                {
+                    await writer.WriteAsync(json);
+                }
+            }
+            catch (Exception ex)
+            {
+                Log.Error(ex, "Error handling audio tracks request");
+                response.StatusCode = (int)HttpStatusCode.InternalServerError;
+                response.ContentType = "text/plain";
+                using (var writer = new StreamWriter(response.OutputStream))
+                {
+                    await writer.WriteAsync("Error extracting audio tracks.");
+                }
+            }
+        }
+
         private static async Task StreamVideoFile(string fileName, HttpListenerContext context)
         {
             var response = context.Response;
@@ -273,7 +390,7 @@ private static async Task StreamVideoFile(string fileName, HttpListenerContext c
                 long contentLength = end - start + 1;
 
                 response.StatusCode = string.IsNullOrEmpty(rangeHeader) ? (int)HttpStatusCode.OK : (int)HttpStatusCode.PartialContent;
-                response.ContentType = "video/mp4";
+                response.ContentType = fileName.EndsWith(".m4a", StringComparison.OrdinalIgnoreCase) ? "audio/mp4" : "video/mp4";
                 response.AddHeader("Accept-Ranges", "bytes");
 
                 if (!string.IsNullOrEmpty(rangeHeader))

Backend/App/MessageService.cs

@@ -25,6 +25,8 @@ public class Selection
         public required string Game { get; set; }
         public string Title { get; set; } = string.Empty;
         public int? IgdbId { get; set; }
+        public List<int>? MutedAudioTracks { get; set; }
+        public Dictionary<int, double>? AudioTrackVolumes { get; set; }
     }
 
     public static class MessageService
@@ -357,6 +359,23 @@ private static async Task HandleCreateClip(JsonElement message)
                         string? filePath = selectionElement.TryGetProperty("filePath", out JsonElement filePathElement)
                             ? filePathElement.GetString()
                             : null;
+                        List<int>? mutedAudioTracks = null;
+                        if (selectionElement.TryGetProperty("mutedAudioTracks", out JsonElement mutedEl)
+                            && mutedEl.ValueKind == JsonValueKind.Array)
+                        {
+                            mutedAudioTracks = mutedEl.EnumerateArray().Select(e => e.GetInt32()).ToList();
+                        }
+                        Dictionary<int, double>? audioTrackVolumes = null;
+                        if (selectionElement.TryGetProperty("audioTrackVolumes", out JsonElement volEl)
+                            && volEl.ValueKind == JsonValueKind.Object)
+                        {
+                            audioTrackVolumes = new Dictionary<int, double>();
+                            foreach (var prop in volEl.EnumerateObject())
+                            {
+                                if (int.TryParse(prop.Name, out int trackIdx) && prop.Value.TryGetDouble(out double vol))
+                                    audioTrackVolumes[trackIdx] = vol;
+                            }
+                        }
 
                         // Create a new Selection instance with all required properties.
                         selections.Add(new Selection
@@ -369,7 +388,9 @@ private static async Task HandleCreateClip(JsonElement message)
                             FilePath = filePath,
                             Game = game,
                             Title = title,
-                            IgdbId = igdbId
+                            IgdbId = igdbId,
+                            MutedAudioTracks = mutedAudioTracks,
+                            AudioTrackVolumes = audioTrackVolumes
                         });
                     }
                 }

Backend/App/Program.cs

@@ -217,6 +217,17 @@ static void Main(string[] args)
                     Directory.CreateDirectory(Settings.Instance.ContentFolder);
                 }
 
+                // Clean up cached audio track files from previous session
+                Task.Run(() =>
+                {
+                    string audioTracksCacheFolder = Path.Combine(Settings.Instance.CacheFolder, FolderNames.AudioTracks);
+                    if (Directory.Exists(audioTracksCacheFolder))
+                    {
+                        try { Directory.Delete(audioTracksCacheFolder, true); }
+                        catch (Exception ex) { Log.Warning($"Failed to clean up audio tracks cache: {ex.Message}"); }
+                    }
+                });
+
                 // Run data migrations
                 Task.Run(MigrationService.RunMigrations);
 
@@ -445,6 +456,7 @@ private static void LoadFrontend()
             Log.Information("Loading frontend, app url is " + appUrl);
             // Initialize the PhotinoWindow
             Window = new PhotinoWindow()
+                .SetBrowserControlInitParameters("--enable-blink-features=AudioVideoTracks")
                 .SetNotificationsEnabled(false) // Disabled due to it creating a second start menu entry with incorrect start path. See https://github.com/tryphotino/photino.NET/issues/85
                 .SetUseOsDefaultSize(false)
                 .SetIconFile(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "icon.ico"))