diff --git a/mlx_lm/tokenizer_utils.py b/mlx_lm/tokenizer_utils.py index 4e11e811d..d33f70e67 100644 --- a/mlx_lm/tokenizer_utils.py +++ b/mlx_lm/tokenizer_utils.py @@ -532,7 +532,7 @@ def load( chat_template = None tokenizer = AutoTokenizer.from_pretrained( - model_path, **(tokenizer_config_extra or {}) + model_path, fix_mistral_regex=True, **(tokenizer_config_extra or {}) ) tokenizer_config = tokenizer.init_kwargs