From ba17deaa49d5998981706e60cb3fc0300f601780 Mon Sep 17 00:00:00 2001 From: Nathan Habib Date: Tue, 23 Sep 2025 08:55:29 +0000 Subject: [PATCH 01/13] run slow tests aginst vllm and transformers main --- .github/workflows/slow_tests.yaml | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/.github/workflows/slow_tests.yaml b/.github/workflows/slow_tests.yaml index a1f68f9d4..25e339017 100644 --- a/.github/workflows/slow_tests.yaml +++ b/.github/workflows/slow_tests.yaml @@ -1,18 +1,16 @@ -name: Slow end to end tests +name: Slow end to end tests, against stable and main versions of dependencies on: - push: - branches: - - main - - v*-release - pull_request: - branches: - - main + schedule: + - cron: "0 */48 * * *" # every 48 hours jobs: run_tests: - name: Run tests + name: Run tests (${{ matrix.dependency }}) runs-on: 'aws-g4dn-2xlarge-use1-public-80' + strategy: + matrix: + dependency: [stable, vllm-transformers-main] steps: - name: Install Git LFS run: | @@ -37,9 +35,15 @@ jobs: - name: Install the project run: uv sync --extra dev + - name: Override dependencies + if: matrix.dependency == 'vllm-transformers-main' + run: | + uv pip install --upgrade git+https://github.com/vllm-project/vllm.git@main + uv pip install --upgrade git+https://github.com/huggingface/transformers.git@main - name: run nvidia-smi run: nvidia-smi - name: Run tests run: uv run pytest --disable-pytest-warnings --runslow tests/slow_tests/ + From aee7264627232f3d81cc4dfcc46fc732f098d925 Mon Sep 17 00:00:00 2001 From: Nathan Habib Date: Tue, 23 Sep 2025 09:01:16 +0000 Subject: [PATCH 02/13] run only when merged on main and every week --- .github/workflows/slow_tests.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/slow_tests.yaml b/.github/workflows/slow_tests.yaml index 25e339017..03ea9ab32 100644 --- a/.github/workflows/slow_tests.yaml +++ b/.github/workflows/slow_tests.yaml @@ -1,8 +1,12 @@ name: Slow end to end tests, against stable and main versions of dependencies on: + push: + branches: + - main + - v*-release schedule: - - cron: "0 */48 * * *" # every 48 hours + - cron: "0 0 * * 6" # Every Saturday at midnight jobs: run_tests: From b462a1a5c142d56d8fcc52bc40315925b9d335ee Mon Sep 17 00:00:00 2001 From: Nathan Habib Date: Tue, 23 Sep 2025 09:06:40 +0000 Subject: [PATCH 03/13] run only when merged on main and every week --- .github/workflows/slow_tests.yaml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/slow_tests.yaml b/.github/workflows/slow_tests.yaml index 03ea9ab32..3da63ebe2 100644 --- a/.github/workflows/slow_tests.yaml +++ b/.github/workflows/slow_tests.yaml @@ -1,12 +1,18 @@ name: Slow end to end tests, against stable and main versions of dependencies on: + # Run automatically every Saturday at 00:00 UTC + schedule: + - cron: "0 0 * * 6" + + # Allow manual triggering via GitHub UI + workflow_dispatch: + + # Optional: run on pushes to main or release branches push: branches: - main - v*-release - schedule: - - cron: "0 0 * * 6" # Every Saturday at midnight jobs: run_tests: From 6516b15e515a1dc910c1550af581191ecb2cc4a6 Mon Sep 17 00:00:00 2001 From: Nathan Habib Date: Tue, 23 Sep 2025 09:08:53 +0000 Subject: [PATCH 04/13] run only when merged on main and every week --- .github/workflows/slow_tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/slow_tests.yaml b/.github/workflows/slow_tests.yaml index 3da63ebe2..564fefe78 100644 --- a/.github/workflows/slow_tests.yaml +++ b/.github/workflows/slow_tests.yaml @@ -1,4 +1,4 @@ -name: Slow end to end tests, against stable and main versions of dependencies +name: Slow end to end tests on: # Run automatically every Saturday at 00:00 UTC From da51c9457cd61e38aad8897b73c34565007378c7 Mon Sep 17 00:00:00 2001 From: Nathan Habib Date: Tue, 23 Sep 2025 09:11:22 +0000 Subject: [PATCH 05/13] dnsio --- .github/workflows/slow_tests.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/slow_tests.yaml b/.github/workflows/slow_tests.yaml index 564fefe78..81a62d2a3 100644 --- a/.github/workflows/slow_tests.yaml +++ b/.github/workflows/slow_tests.yaml @@ -14,6 +14,10 @@ on: - main - v*-release + pull_request: + branches: + - main + jobs: run_tests: name: Run tests (${{ matrix.dependency }}) From ac4052d5e7d880fa66f34da99e97d3c3c85a01fa Mon Sep 17 00:00:00 2001 From: Nathan Habib Date: Tue, 23 Sep 2025 10:41:03 +0000 Subject: [PATCH 06/13] dnsio --- .github/workflows/slow_tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/slow_tests.yaml b/.github/workflows/slow_tests.yaml index 81a62d2a3..603598b05 100644 --- a/.github/workflows/slow_tests.yaml +++ b/.github/workflows/slow_tests.yaml @@ -52,7 +52,7 @@ jobs: - name: Override dependencies if: matrix.dependency == 'vllm-transformers-main' run: | - uv pip install --upgrade git+https://github.com/vllm-project/vllm.git@main + VLLM_USE_PRECOMPILED=1 uv pip install --upgrade git+https://github.com/vllm-project/vllm.git@main uv pip install --upgrade git+https://github.com/huggingface/transformers.git@main - name: run nvidia-smi From 1231367710557245f665cd4f4d541649e44e4bfb Mon Sep 17 00:00:00 2001 From: Nathan Habib Date: Wed, 24 Sep 2025 14:07:31 +0000 Subject: [PATCH 07/13] fix vllm model with main modifs from vllm --- src/lighteval/models/vllm/vllm_model.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/lighteval/models/vllm/vllm_model.py b/src/lighteval/models/vllm/vllm_model.py index 969caf8fa..d92d5f223 100644 --- a/src/lighteval/models/vllm/vllm_model.py +++ b/src/lighteval/models/vllm/vllm_model.py @@ -48,6 +48,7 @@ import ray from more_itertools import distribute from vllm import LLM, RequestOutput, SamplingParams + from vllm.inputs.data import TokensPrompt from vllm.distributed.parallel_state import ( destroy_distributed_environment, destroy_model_parallel, @@ -291,7 +292,7 @@ def _create_auto_model(self, config: VLLMModelConfig) -> Optional[LLM]: # Inferring from the tokenizer will cause vllm to bug for models with mismatches between model # config and tk config, like mistralai/Mistral-7B-v0.1 if self._max_length is None: - self._max_length = model.llm_engine.model_config.max_seq_len_to_capture + self._max_length = model.llm_engine.model_config.max_model_len return model @@ -455,7 +456,7 @@ def run_inference_one_model(model_args: dict, sampling_params: SamplingParams, r ] else: outputs = self.model.generate( - prompt_token_ids=inputs, + prompts=[TokensPrompt(prompt_token_ids=input) for input in inputs], sampling_params=sampling_params, use_tqdm=True, ) From fe100f9bc1dc338972e4670269df9e50c1731b9e Mon Sep 17 00:00:00 2001 From: Nathan Habib Date: Thu, 25 Sep 2025 09:55:35 +0000 Subject: [PATCH 08/13] use another file for latest tests --- .github/workflows/latest_tests.yaml | 52 +++++++++++++++++++++++++++++ .github/workflows/slow_tests.yaml | 20 +---------- 2 files changed, 53 insertions(+), 19 deletions(-) create mode 100644 .github/workflows/latest_tests.yaml diff --git a/.github/workflows/latest_tests.yaml b/.github/workflows/latest_tests.yaml new file mode 100644 index 000000000..72c1e1232 --- /dev/null +++ b/.github/workflows/latest_tests.yaml @@ -0,0 +1,52 @@ +name: Tests on dev branch of vllm and transformers + +on: + # Run automatically every Saturday at 00:00 UTC + schedule: + - cron: "0 0 * * 6" + + # Allow manual triggering via GitHub UI + workflow_dispatch: + + # Optional: run on pushes to main or release branches + push: + branches: + - main + - v*-release + + pull_request: + branches: + - main + +jobs: + run_tests: + name: Run tests on dev branch of vllm and transformers + runs-on: 'aws-g4dn-2xlarge-use1-public-80' + steps: + - name: Install Git LFS + run: | + sudo apt-get update && sudo apt-get install -y git-lfs + git lfs install + + - name: Checkout repository + uses: actions/checkout@v4 + with: + lfs: true + + - name: Install uv + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + + - name: Install the project + run: | + uv sync --extra dev + VLLM_USE_PRECOMPILED=1 uv pip install --upgrade git+https://github.com/vllm-project/vllm.git@main + uv pip install --upgrade git+https://github.com/huggingface/transformers.git@main + + - name: run nvidia-smi + run: nvidia-smi + + - name: Run tests + run: uv run pytest --disable-pytest-warnings --runslow tests/slow_tests/ + diff --git a/.github/workflows/slow_tests.yaml b/.github/workflows/slow_tests.yaml index 603598b05..a1f68f9d4 100644 --- a/.github/workflows/slow_tests.yaml +++ b/.github/workflows/slow_tests.yaml @@ -1,30 +1,18 @@ name: Slow end to end tests on: - # Run automatically every Saturday at 00:00 UTC - schedule: - - cron: "0 0 * * 6" - - # Allow manual triggering via GitHub UI - workflow_dispatch: - - # Optional: run on pushes to main or release branches push: branches: - main - v*-release - pull_request: branches: - main jobs: run_tests: - name: Run tests (${{ matrix.dependency }}) + name: Run tests runs-on: 'aws-g4dn-2xlarge-use1-public-80' - strategy: - matrix: - dependency: [stable, vllm-transformers-main] steps: - name: Install Git LFS run: | @@ -49,15 +37,9 @@ jobs: - name: Install the project run: uv sync --extra dev - - name: Override dependencies - if: matrix.dependency == 'vllm-transformers-main' - run: | - VLLM_USE_PRECOMPILED=1 uv pip install --upgrade git+https://github.com/vllm-project/vllm.git@main - uv pip install --upgrade git+https://github.com/huggingface/transformers.git@main - name: run nvidia-smi run: nvidia-smi - name: Run tests run: uv run pytest --disable-pytest-warnings --runslow tests/slow_tests/ - From cd67bdd28b539106b74ef45297d94c63d61aa923 Mon Sep 17 00:00:00 2001 From: Nathan Habib Date: Thu, 25 Sep 2025 11:25:46 +0000 Subject: [PATCH 09/13] use another file for latest tests --- .github/workflows/latest_tests.yaml | 3 +++ pyproject.toml | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/latest_tests.yaml b/.github/workflows/latest_tests.yaml index 72c1e1232..5d1250ffc 100644 --- a/.github/workflows/latest_tests.yaml +++ b/.github/workflows/latest_tests.yaml @@ -47,6 +47,9 @@ jobs: - name: run nvidia-smi run: nvidia-smi + - name: Pip freeze + run: uv pip freeze + - name: Run tests run: uv run pytest --disable-pytest-warnings --runslow tests/slow_tests/ diff --git a/pyproject.toml b/pyproject.toml index 411a7b898..744b49645 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,7 +63,7 @@ dependencies = [ "GitPython>=3.1.41", # for logging "datasets>=4.0.0", "pydantic", - "numpy>=2", # pinned to avoid incompatibilities + "numpy>=2,<2.3", # pinned to avoid incompatibilities "hf-xet>=1.1.8", # pinned to avoid failing test suite # Prettiness "typer", @@ -98,7 +98,7 @@ nanotron = [ "tensorboardX" ] tensorboardX = ["tensorboardX"] -vllm = ["vllm>=0.10.0,<0.10.2", "ray", "more_itertools"] +vllm = ["vllm", "ray", "more_itertools"] sglang = ["sglang"] quality = ["ruff>=v0.11.0","pre-commit"] tests = ["pytest>=7.4.0","deepdiff","pip>=25.2"] From 9ec50d0860e7b6b0d8448b5409403d01367978b1 Mon Sep 17 00:00:00 2001 From: Nathan Habib Date: Thu, 25 Sep 2025 11:59:51 +0000 Subject: [PATCH 10/13] run vllm first --- .github/workflows/latest_tests.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/latest_tests.yaml b/.github/workflows/latest_tests.yaml index 5d1250ffc..012a19536 100644 --- a/.github/workflows/latest_tests.yaml +++ b/.github/workflows/latest_tests.yaml @@ -51,5 +51,8 @@ jobs: run: uv pip freeze - name: Run tests - run: uv run pytest --disable-pytest-warnings --runslow tests/slow_tests/ + run: | + uv run pytest --disable-pytest-warnings --runslow tests/slow_tests/test_vllm_model.py + uv run pytest --disable-pytest-warnings --runslow tests/slow_tests/test_accelerate_model.py + From 90a9d9a78a7d43d619bd4de0d1f8f20717ed748c Mon Sep 17 00:00:00 2001 From: Nathan Habib Date: Thu, 25 Sep 2025 12:07:25 +0000 Subject: [PATCH 11/13] fix vllm --- .github/workflows/latest_tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/latest_tests.yaml b/.github/workflows/latest_tests.yaml index 012a19536..a17d253ed 100644 --- a/.github/workflows/latest_tests.yaml +++ b/.github/workflows/latest_tests.yaml @@ -52,7 +52,7 @@ jobs: - name: Run tests run: | - uv run pytest --disable-pytest-warnings --runslow tests/slow_tests/test_vllm_model.py + VLLM_WORKER_MULTIPROC_METHOD=spawn uv run pytest --disable-pytest-warnings --runslow tests/slow_tests/test_vllm_model.py uv run pytest --disable-pytest-warnings --runslow tests/slow_tests/test_accelerate_model.py From b40fc032ba8e7682b202d15a5a0734667e5db8f0 Mon Sep 17 00:00:00 2001 From: Nathan Habib Date: Thu, 25 Sep 2025 12:37:17 +0000 Subject: [PATCH 12/13] fix vllm --- .github/workflows/latest_tests.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/latest_tests.yaml b/.github/workflows/latest_tests.yaml index a17d253ed..582f8bd98 100644 --- a/.github/workflows/latest_tests.yaml +++ b/.github/workflows/latest_tests.yaml @@ -28,6 +28,9 @@ jobs: sudo apt-get update && sudo apt-get install -y git-lfs git lfs install + - name: Install Python development headers + run: sudo apt-get update && sudo apt-get install -y python3.10-dev + - name: Checkout repository uses: actions/checkout@v4 with: @@ -54,5 +57,8 @@ jobs: run: | VLLM_WORKER_MULTIPROC_METHOD=spawn uv run pytest --disable-pytest-warnings --runslow tests/slow_tests/test_vllm_model.py uv run pytest --disable-pytest-warnings --runslow tests/slow_tests/test_accelerate_model.py + env: + VLLM_ATTENTION_BACKEND: "FLASHINFER" + VLLM_USE_FLASH_ATTN: "0" From a8fa3c552770b03ad89ab54927ac77b1674b5d96 Mon Sep 17 00:00:00 2001 From: Nathan Habib Date: Thu, 25 Sep 2025 12:52:23 +0000 Subject: [PATCH 13/13] fix vllm --- .github/workflows/latest_tests.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/latest_tests.yaml b/.github/workflows/latest_tests.yaml index 582f8bd98..cd0029e39 100644 --- a/.github/workflows/latest_tests.yaml +++ b/.github/workflows/latest_tests.yaml @@ -57,8 +57,3 @@ jobs: run: | VLLM_WORKER_MULTIPROC_METHOD=spawn uv run pytest --disable-pytest-warnings --runslow tests/slow_tests/test_vllm_model.py uv run pytest --disable-pytest-warnings --runslow tests/slow_tests/test_accelerate_model.py - env: - VLLM_ATTENTION_BACKEND: "FLASHINFER" - VLLM_USE_FLASH_ATTN: "0" - -