log model config (#627)

* log model config

* log model config

* fix tests

* make tests fail at first failure

* make tests fail at first failure

* fix tests

* fix tests

Author: NathanHB

.github/workflows/tests.yaml

@@ -38,7 +38,7 @@ jobs:
         HF_HOME: "cache/models"
         HF_DATASETS_CACHE: "cache/datasets"
        run: | # PYTHONPATH="${PYTHONPATH}:src" HF_DATASETS_CACHE="cache/datasets" HF_HOME="cache/models"
-        python -m pytest --disable-pytest-warnings
+        python -m pytest -x --disable-pytest-warnings
      - name: Write cache
        uses: actions/cache@v4
        with:

src/lighteval/logging/info_loggers.py

@@ -90,6 +90,8 @@ class GeneralConfigLogger:
     model_dtype: str = None
     model_size: str = None
 
+    generation_parameters: dict | None = None
+
     # Nanotron config
     config: "Config" = None
 
@@ -133,14 +135,16 @@ def log_args_info(
         self.job_id = job_id
         self.config = config
 
-    def log_model_info(self, model_info: ModelInfo) -> None:
+    def log_model_info(self, generation_parameters: dict, model_info: ModelInfo) -> None:
         """
         Logs the model information.
 
         Args:
+            model_config: the model config used to initalize the model.
             model_info (ModelInfo): Model information to be logged.
 
         """
+        self.generation_parameters = generation_parameters
         self.model_name = model_info.model_name
         self.model_sha = model_info.model_sha
         self.model_dtype = model_info.model_dtype

src/lighteval/pipeline.py

@@ -27,7 +27,7 @@
 import re
 import shutil
 from contextlib import nullcontext
-from dataclasses import dataclass, field
+from dataclasses import asdict, dataclass, field
 from datetime import timedelta
 from enum import Enum, auto
 
@@ -156,7 +156,9 @@ def __init__(
         self.accelerator, self.parallel_context = self._init_parallelism_manager()
         self.model = self._init_model(model_config, model)
 
-        self.evaluation_tracker.general_config_logger.log_model_info(self.model.model_info)
+        generation_parameters = asdict(model_config.generation_parameters) if model_config else {}
+
+        self.evaluation_tracker.general_config_logger.log_model_info(generation_parameters, self.model.model_info)
         self._init_tasks_and_requests(tasks=tasks)
         self._init_random_seeds()
         # Final results

tests/models/test_base_model.py

@@ -26,7 +26,9 @@
 
 
 def test_empty_requests():
-    model_config = TransformersModelConfig("hf-internal-testing/tiny-random-LlamaForCausalLM")
+    model_config = TransformersModelConfig(
+        "hf-internal-testing/tiny-random-LlamaForCausalLM", model_parallel=False, revision="main"
+    )
     model: TransformersModel = load_model(config=model_config, env_config=EnvConfig(cache_dir="."))
 
     assert model.loglikelihood([]) == []