diff --git a/Assets/uCosyVoice/Runtime/Inference/FlowRunner.cs b/Assets/uCosyVoice/Runtime/Inference/FlowRunner.cs index b0c2038..1eef58d 100644 --- a/Assets/uCosyVoice/Runtime/Inference/FlowRunner.cs +++ b/Assets/uCosyVoice/Runtime/Inference/FlowRunner.cs @@ -189,11 +189,15 @@ public Tensor ProcessWithPrompt( _estimatorWorker.SetInput("cond", condsBatch); _estimatorWorker.Schedule(); - var velocity = _estimatorWorker.PeekOutput() as Tensor; - velocity.ReadbackAndClone(); + // Must dispose velocity tensor after copying data + float[] vData; + using (var velocity = _estimatorWorker.PeekOutput() as Tensor) + { + velocity.ReadbackAndClone(); + vData = velocity.DownloadToArray(); + } - // Download velocity and update x on CPU - var vData = velocity.DownloadToArray(); + // Download x and update on CPU var xData = xBatch.DownloadToArray(); // Euler step: x = x + dt * v diff --git a/Assets/uCosyVoice/Runtime/Inference/LLMRunner.cs b/Assets/uCosyVoice/Runtime/Inference/LLMRunner.cs index 8ae6ca1..de6a502 100644 --- a/Assets/uCosyVoice/Runtime/Inference/LLMRunner.cs +++ b/Assets/uCosyVoice/Runtime/Inference/LLMRunner.cs @@ -165,11 +165,15 @@ public Tensor GenerateWithPrompt( var hiddenStates = _backboneInitialWorker.PeekOutput("hidden_states") as Tensor; hiddenStates.ReadbackAndClone(); - // Get KV cache - var kvCache = _backboneInitialWorker.PeekOutput("past_key_values") as Tensor; - kvCache.ReadbackAndClone(); - var kvCacheData = kvCache.DownloadToArray(); - var kvCacheShape = kvCache.shape; + // Get KV cache (must dispose after copying data) + float[] kvCacheData; + TensorShape kvCacheShape; + using (var kvCache = _backboneInitialWorker.PeekOutput("past_key_values") as Tensor) + { + kvCache.ReadbackAndClone(); + kvCacheData = kvCache.DownloadToArray(); + kvCacheShape = kvCache.shape; + } lmInput.Dispose(); @@ -215,11 +219,13 @@ public Tensor GenerateWithPrompt( var newHiddenStates = _backboneDecodeWorker.PeekOutput("hidden_states") as Tensor; newHiddenStates.ReadbackAndClone(); - // Update KV cache (decode model outputs "new_past_key_values") - var newKvCache = _backboneDecodeWorker.PeekOutput("new_past_key_values") as Tensor; - newKvCache.ReadbackAndClone(); - kvCacheData = newKvCache.DownloadToArray(); - kvCacheShape = newKvCache.shape; + // Update KV cache (decode model outputs "new_past_key_values") - must dispose after copying data + using (var newKvCache = _backboneDecodeWorker.PeekOutput("new_past_key_values") as Tensor) + { + newKvCache.ReadbackAndClone(); + kvCacheData = newKvCache.DownloadToArray(); + kvCacheShape = newKvCache.shape; + } // Get logits logits = GetLogits(newHiddenStates);