From fd292bba0c751f7d2ad348070f7ab5cd27542667 Mon Sep 17 00:00:00 2001 From: ishitab02 Date: Fri, 7 Nov 2025 07:24:34 +0000 Subject: [PATCH] fix: used normalize_str in BaseTokenizer.normalize --- .../python/py_src/tokenizers/implementations/base_tokenizer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bindings/python/py_src/tokenizers/implementations/base_tokenizer.py b/bindings/python/py_src/tokenizers/implementations/base_tokenizer.py index 432a4754c..c2e7effb4 100644 --- a/bindings/python/py_src/tokenizers/implementations/base_tokenizer.py +++ b/bindings/python/py_src/tokenizers/implementations/base_tokenizer.py @@ -187,7 +187,7 @@ def normalize(self, sequence: str) -> str: Returns: The normalized string """ - return self._tokenizer.normalize(sequence) + return self._tokenizer.normalizer.normalize_str(sequence) def encode( self,