Merge branch 'TextDemo'

Author: saddam213

Examples/TensorStack.Example.TextGeneration/Common/TextModel.cs

@@ -20,6 +20,8 @@ public class TextModel : BaseModel
     public enum TextModelType
     {
         Summary = 0,
-        Phi3 = 1
+        Phi3 = 1,
+        Whisper = 2,
+        Supertonic = 3
     }
 }

Examples/TensorStack.Example.TextGeneration/MainWindow.xaml

@@ -15,15 +15,17 @@
 
         <!--Main Menu-->
         <Grid DockPanel.Dock="Top" WindowChrome.IsHitTestVisibleInChrome="True">
-            <UniformGrid Columns="3" Height="30" Margin="2">
+            <UniformGrid Columns="5" Height="30" Margin="2">
 
                 <!--Logo-->
                 <Grid IsHitTestVisible="False">
                     <Image Source="{StaticResource ImageTensorstackText}" Height="32" HorizontalAlignment="Left" Margin="4,2,50,0" />
                 </Grid>
 
                 <!--Views-->
-                <Button Command="{Binding NavigateCommand}" CommandParameter="{x:Static Views:View.TextSummary}" Content="Text Summary" />
+                <Button Command="{Binding NavigateCommand}" CommandParameter="{x:Static Views:View.Summary}" Content="Summary" />
+                <Button Command="{Binding NavigateCommand}" CommandParameter="{x:Static Views:View.Whisper}" Content="Whisper" />
+                <Button Command="{Binding NavigateCommand}" CommandParameter="{x:Static Views:View.Supertonic}" Content="Supertonic" />
 
                 <!--Window Options-->
                 <StackPanel Orientation="Horizontal" HorizontalAlignment="Right">

Examples/TensorStack.Example.TextGeneration/MainWindow.xaml.cs

@@ -17,7 +17,7 @@ public MainWindow(Settings configuration, NavigationService navigation)
             NavigateCommand = new AsyncRelayCommand<View>(NavigateAsync, CanNavigate);
             InitializeComponent();
 
-            NavigateCommand.Execute(View.TextSummary);
+            NavigateCommand.Execute(View.Whisper);
         }
 
         public NavigationService Navigation { get; }

Examples/TensorStack.Example.TextGeneration/Services/TextService.cs

@@ -8,14 +8,16 @@
 using TensorStack.TextGeneration.Pipelines.Other;
 using TensorStack.TextGeneration.Pipelines.Phi;
 using TensorStack.Providers;
+using TensorStack.TextGeneration.Pipelines.Whisper;
+using TensorStack.Common.Tensor;
+using TensorStack.TextGeneration.Pipelines.Supertonic;
 
 namespace TensorStack.Example.Services
 {
     public class TextService : ServiceBase, ITextService
     {
         private readonly Settings _settings;
-        private IPipeline<GenerateResult, GenerateOptions, GenerateProgress> _greedyPipeline;
-        private IPipeline<GenerateResult[], SearchOptions, GenerateProgress> _beamSearchPipeline;
+        private IPipeline _currentPipeline;
         private CancellationTokenSource _cancellationTokenSource;
         private bool _isLoaded;
         private bool _isLoading;
@@ -77,34 +79,41 @@ public async Task LoadAsync(TextModel model, Device device)
                 using (_cancellationTokenSource = new CancellationTokenSource())
                 {
                     var cancellationToken = _cancellationTokenSource.Token;
-                    if (_greedyPipeline != null)
-                        await _greedyPipeline.UnloadAsync(cancellationToken);
+                    if (_currentPipeline != null)
+                        await _currentPipeline.UnloadAsync(cancellationToken);
 
                     var provider = device.GetProvider();
                     var providerCPU = Provider.GetProvider(DeviceType.CPU); // TODO: DirectML not working with decoder
                     if (model.Type == TextModelType.Phi3)
                     {
                         if (!Enum.TryParse<PhiType>(model.Version, true, out var phiType))
-                            throw new ArgumentException("Invalid PhiType Version");
+                            throw new ArgumentException("Invalid Phi Version");
 
-                        var pipeline = Phi3Pipeline.Create(providerCPU, model.Path, phiType);
-                        _greedyPipeline = pipeline;
-                        _beamSearchPipeline = pipeline;
+                        _currentPipeline = Phi3Pipeline.Create(providerCPU, model.Path, phiType);
                     }
                     else if (model.Type == TextModelType.Summary)
                     {
-                        var pipeline = SummaryPipeline.Create(provider, providerCPU, model.Path);
-                        _greedyPipeline = pipeline;
-                        _beamSearchPipeline = pipeline;
+                        _currentPipeline = SummaryPipeline.Create(provider, providerCPU, model.Path);
                     }
-                    await Task.Run(() => _greedyPipeline.LoadAsync(cancellationToken), cancellationToken);
+                    else if (model.Type == TextModelType.Whisper)
+                    {
+                        if (!Enum.TryParse<WhisperType>(model.Version, true, out var whisperType))
+                            throw new ArgumentException("Invalid Whisper Version");
+
+                        _currentPipeline = WhisperPipeline.Create(provider, providerCPU, model.Path, whisperType);
+                    }
+                    else if (model.Type == TextModelType.Supertonic)
+                    {
+                        _currentPipeline = SupertonicPipeline.Create(model.Path, provider);
+                    }
+                    await Task.Run(() => _currentPipeline.LoadAsync(cancellationToken), cancellationToken);
 
                 }
             }
             catch (OperationCanceledException)
             {
-                _greedyPipeline?.Dispose();
-                _greedyPipeline = null;
+                _currentPipeline?.Dispose();
+                _currentPipeline = null;
                 _currentConfig = null;
                 throw;
             }
@@ -148,11 +157,63 @@ public async Task<GenerateResult[]> ExecuteAsync(TextRequest options)
                         if (options.Beams == 0)
                         {
                             // Greedy Search
-                            return [await _greedyPipeline.RunAsync(pipelineOptions, cancellationToken: _cancellationTokenSource.Token)];
+                            var greedyPipeline = _currentPipeline as IPipeline<GenerateResult, GenerateOptions, GenerateProgress>;
+                            return [await greedyPipeline.RunAsync(pipelineOptions, cancellationToken: _cancellationTokenSource.Token)];
+                        }
+
+                        // Beam Search
+                        var beamSearchPipeline = _currentPipeline as IPipeline<GenerateResult[], SearchOptions, GenerateProgress>;
+                        return await beamSearchPipeline.RunAsync(new SearchOptions(pipelineOptions), cancellationToken: _cancellationTokenSource.Token);
+                    });
+
+                    return pipelineResult;
+                }
+            }
+            finally
+            {
+                IsExecuting = false;
+            }
+        }
+
+
+        public async Task<GenerateResult[]> ExecuteAsync(WhisperRequest options)
+        {
+            try
+            {
+                IsExecuting = true;
+                using (_cancellationTokenSource = new CancellationTokenSource())
+                {
+                    var pipelineOptions = new WhisperOptions
+                    {
+                        Prompt = options.Prompt,
+                        Seed = options.Seed,
+                        Beams = options.Beams,
+                        TopK = options.TopK,
+                        TopP = options.TopP,
+                        Temperature = options.Temperature,
+                        MaxLength = options.MaxLength,
+                        MinLength = options.MinLength,
+                        NoRepeatNgramSize = options.NoRepeatNgramSize,
+                        LengthPenalty = options.LengthPenalty,
+                        DiversityLength = options.DiversityLength,
+                        EarlyStopping = options.EarlyStopping,
+                        AudioInput = options.AudioInput,
+                        Language = options.Language,
+                        Task = options.Task
+                    };
+
+                    var pipelineResult = await Task.Run(async () =>
+                    {
+                        if (options.Beams == 0)
+                        {
+                            // Greedy Search
+                            var greedyPipeline = _currentPipeline as IPipeline<GenerateResult, WhisperOptions, GenerateProgress>;
+                            return [await greedyPipeline.RunAsync(pipelineOptions, cancellationToken: _cancellationTokenSource.Token)];
                         }
 
                         // Beam Search
-                        return await _beamSearchPipeline.RunAsync(new SearchOptions(pipelineOptions), cancellationToken: _cancellationTokenSource.Token);
+                        var beamSearchPipeline = _currentPipeline as IPipeline<GenerateResult[], WhisperSearchOptions, GenerateProgress>;
+                        return await beamSearchPipeline.RunAsync(new WhisperSearchOptions(pipelineOptions), cancellationToken: _cancellationTokenSource.Token);
                     });
 
                     return pipelineResult;
@@ -165,6 +226,34 @@ public async Task<GenerateResult[]> ExecuteAsync(TextRequest options)
         }
 
 
+        public async Task<AudioTensor> ExecuteAsync(SupertonicRequest options)
+        {
+            try
+            {
+                IsExecuting = true;
+                using (_cancellationTokenSource = new CancellationTokenSource())
+                {
+                    var pipeline = _currentPipeline as IPipeline<AudioTensor, SupertonicOptions, GenerateProgress>;
+                    var pipelineOptions = new SupertonicOptions
+                    {
+                        TextInput = options.InputText,
+                        VoiceStyle = options.VoiceStyle,
+                        Steps = options.Steps,
+                        Speed = options.Speed,
+                        SilenceDuration = options.SilenceDuration,
+                        Seed = options.Seed,
+                    };
+
+                    return await pipeline.RunAsync(pipelineOptions, cancellationToken: _cancellationTokenSource.Token);
+                }
+            }
+            finally
+            {
+                IsExecuting = false;
+            }
+        }
+
+
         /// <summary>
         /// Cancel the running task (Load or Execute)
         /// </summary>
@@ -179,12 +268,12 @@ public async Task CancelAsync()
         /// </summary>
         public async Task UnloadAsync()
         {
-            if (_greedyPipeline != null)
+            if (_currentPipeline != null)
             {
                 await _cancellationTokenSource.SafeCancelAsync();
-                await _greedyPipeline.UnloadAsync();
-                _greedyPipeline.Dispose();
-                _greedyPipeline = null;
+                await _currentPipeline.UnloadAsync();
+                _currentPipeline.Dispose();
+                _currentPipeline = null;
                 _currentConfig = null;
             }
 
@@ -205,6 +294,8 @@ public interface ITextService
         Task UnloadAsync();
         Task CancelAsync();
         Task<GenerateResult[]> ExecuteAsync(TextRequest options);
+        Task<GenerateResult[]> ExecuteAsync(WhisperRequest options);
+        Task<AudioTensor> ExecuteAsync(SupertonicRequest options);
     }
 
 
@@ -224,4 +315,22 @@ public record TextRequest : ITransformerRequest
         public int DiversityLength { get; set; } = 5;
     }
 
+
+    public record WhisperRequest : TextRequest
+    {
+        public AudioTensor AudioInput { get; set; }
+        public LanguageType Language { get; set; } = LanguageType.EN;
+        public TaskType Task { get; set; } = TaskType.Transcribe;
+    }
+
+
+    public record SupertonicRequest
+    {
+        public string InputText { get; set; }
+        public string VoiceStyle { get; set; }
+        public int Steps { get; set; } = 5;
+        public float Speed { get; set; } = 1f;
+        public float SilenceDuration { get; set; } = 0.3f;
+        public int Seed { get; set; }
+    }
 }

Examples/TensorStack.Example.TextGeneration/Settings.cs

@@ -19,7 +19,9 @@ public class Settings : IUIConfiguration
         public string VideoCodec { get; set; } = "mp4v";
         public string DirectoryTemp { get; set; }
         public IReadOnlyList<Device> Devices { get; set; }
-        public ObservableCollection<TextModel> TextModels { get; set; }
+        public ObservableCollection<TextModel> TextToTextModels { get; set; }
+        public ObservableCollection<TextModel> TextToAudioModels { get; set; }
+        public ObservableCollection<TextModel> AudioToTextModels { get; set; }
 
 
         public void Initialize()