Merge pull request #205 from codelion/fix-mlx-model-id

Update MLX model patterns and reduce max_tokens in eval script

Author: codelion

optillm/__init__.py

@@ -2,7 +2,7 @@
 import os
 
 # Version information
-__version__ = "0.1.17"
+__version__ = "0.1.18"
 
 # Get the path to the root optillm.py
 spec = util.spec_from_file_location(

optillm/inference.py

@@ -189,7 +189,8 @@ def should_use_mlx(model_id: str) -> bool:
     # Models that should use MLX
     mlx_patterns = [
         "mlx-community/",
-        "mlx-"
+        "mlx-",
+        "-mlx-"
     ]
 
     # Known problematic models that should prefer MLX on Apple Silicon

scripts/eval_math500_benchmark.py

@@ -692,7 +692,7 @@ def get_llm_response(problem: str, model: str) -> str:
             messages=[
                 {"role": "user", "content": SYSTEM_PROMPT + "\n" + problem}
             ],
-            max_tokens=32768, # for thinking models, we need to use a lot more tokens
+            max_tokens=8192, # for thinking models, we need to use a lot more tokens
             # extra_body = {
             #     "decoding" : "thinkdeeper",
             # }

setup.py

@@ -3,7 +3,7 @@
 
 setup(
     name="optillm",
-    version="0.1.17",
+    version="0.1.18",
     packages=find_packages(include=['optillm', 'optillm.*']),  # This ensures all subpackages are included
     py_modules=['optillm'],
     package_data={