Fix T5Tokenizer regression

saddam213 · saddam213 · commit 63522651a6ba · 2025-11-25T14:07:33.000+13:00
diff --git a/TensorStack.TextGeneration/TensorStack.TextGeneration.csproj b/TensorStack.TextGeneration/TensorStack.TextGeneration.csproj
@@ -18,7 +18,7 @@
   <!--Common Packages-->
   <ItemGroup>
     <PackageReference Include="MathNet.Numerics" Version="5.0.0" />
-    <PackageReference Include="Microsoft.ML.Tokenizers" Version="2.0.0" />
+    <PackageReference Include="Microsoft.ML.Tokenizers" Version="2.0.0-preview.1.25125.4" />
     <PackageReference Include="System.Linq.Async" Version="6.0.3" />
   </ItemGroup>
 
diff --git a/TensorStack.TextGeneration/Tokenizers/T5Tokenizer.cs b/TensorStack.TextGeneration/Tokenizers/T5Tokenizer.cs
@@ -143,7 +143,7 @@ private SentencePieceTokenizer CreateTokenizer()
             var specialTokens = GetSpecialTokens(_configuration.Path);
             using (var fileStream = File.OpenRead(_configuration.Path))
             {
-                return SentencePieceTokenizer.Create(fileStream, addBeginningOfSentence: false, addEndOfSentence: true, specialTokens);
+                return SentencePieceTokenizer.Create(fileStream, addBeginOfSentence: false, addEndOfSentence: true, specialTokens);
             }
         }
 

Original file line number	Diff line number	Diff line change
`@@ -143,7 +143,7 @@ private SentencePieceTokenizer CreateTokenizer()`
`143`	`143`	`var specialTokens = GetSpecialTokens(_configuration.Path);`
`144`	`144`	`using (var fileStream = File.OpenRead(_configuration.Path))`
`145`	`145`	`{`
`146`		`- return SentencePieceTokenizer.Create(fileStream, addBeginningOfSentence: false, addEndOfSentence: true, specialTokens);`
	`146`	`+ return SentencePieceTokenizer.Create(fileStream, addBeginOfSentence: false, addEndOfSentence: true, specialTokens);`
`147`	`147`	`}`
`148`	`148`	`}`
`149`	`149`