Merge branch 'main' into dev

Author: pisceskkk

.buildkite/test-pipeline.yaml

@@ -546,8 +546,11 @@ steps:
 
 - label: Model Executor Test # 23min
   timeout_in_minutes: 35
+  torch_nightly: true
   mirror_hardwares: [amdexperimental]
   source_file_dependencies:
+  - vllm/engine/arg_utils.py
+  - vllm/config/model.py
   - vllm/model_executor
   - tests/model_executor
   - tests/entrypoints/openai/test_tensorizer_entrypoint.py

tests/entrypoints/openai/test_translation_validation.py

@@ -79,7 +79,7 @@ async def test_basic_audio_with_lora(mary_had_lamb):
             temperature=0.0,
         )
     out = json.loads(translation)["text"].strip().lower()
-    assert "mary tenía un pequeño cordero" in out
+    assert "pequeño" in out.split(" ")
 
 
 # NOTE: (NickLucche) the large-v3-turbo model was not trained on translation!

tests/model_executor/model_loader/runai_model_streamer/test_runai_model_streamer_loader.py

@@ -1,12 +1,16 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 
+import pytest
+
 from vllm import SamplingParams
 from vllm.config.load import LoadConfig
 from vllm.model_executor.model_loader import get_model_loader
 
 load_format = "runai_streamer"
 test_model = "openai-community/gpt2"
+# TODO(amacaskill): Replace with a GKE owned GCS bucket.
+test_gcs_model = "gs://vertex-model-garden-public-us/codegemma/codegemma-2b/"
 
 prompts = [
     "Hello, my name is",
@@ -32,3 +36,16 @@ def test_runai_model_loader_download_files(vllm_runner):
     with vllm_runner(test_model, load_format=load_format) as llm:
         deserialized_outputs = llm.generate(prompts, sampling_params)
         assert deserialized_outputs
+
+
+def test_runai_model_loader_download_files_gcs(
+    vllm_runner, monkeypatch: pytest.MonkeyPatch
+):
+    monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "fake-project")
+    monkeypatch.setenv("RUNAI_STREAMER_GCS_USE_ANONYMOUS_CREDENTIALS", "true")
+    monkeypatch.setenv(
+        "CLOUD_STORAGE_EMULATOR_ENDPOINT", "https://storage.googleapis.com"
+    )
+    with vllm_runner(test_gcs_model, load_format=load_format) as llm:
+        deserialized_outputs = llm.generate(prompts, sampling_params)
+        assert deserialized_outputs

vllm/config/compilation.py

@@ -251,13 +251,6 @@ class CompilationConfig:
     disabled when running with Inductor: mode>=VLLM_COMPILE and use_inductor=True.
     Inductor generates (fused) Triton kernels for disabled custom ops."""
     splitting_ops: list[str] | None = None
-
-    """
-    Provide control over whether to compile the multimodal encoder 
-    such as Qwen2_5_vl 
-    """
-    compile_mm_encoder: bool = True
-
     """A list of ops to exclude from cudagraphs, used in piecewise compilation.
 
     The behavior depends on use_inductor_graph_partition:
@@ -275,6 +268,9 @@ class CompilationConfig:
 
     If None, defaults to attention ops for piecewise cudagraphs.
     If empty list [], no ops are excluded (suitable for full cudagraphs)."""
+    compile_mm_encoder: bool = True
+    """Whether or not to compile the multimodal encoder.
+    Currently, this only works for `Qwen2_5_vl`."""
 
     # Inductor capture
     use_inductor: bool | None = None

vllm/model_executor/layers/quantization/mxfp4.py

@@ -43,6 +43,7 @@
 from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
     _can_support_mxfp4,
     _swizzle_mxfp4,
+    get_padding_alignment,
 )
 from vllm.model_executor.layers.quantization.utils.quant_utils import is_layer_skipped
 from vllm.model_executor.utils import set_weight_attrs
@@ -282,10 +283,11 @@ def create_weights(
             )
             hidden_size = round_up(hidden_size, 128)
         elif current_platform.is_rocm():
+            pad_align = get_padding_alignment()
             intermediate_size_per_partition_after_pad = round_up(
-                intermediate_size_per_partition, 256
+                intermediate_size_per_partition, pad_align
             )
-            hidden_size = round_up(hidden_size, 256)
+            hidden_size = round_up(hidden_size, pad_align)
         else:
             intermediate_size_per_partition_after_pad = round_up(
                 intermediate_size_per_partition, 64

vllm/model_executor/layers/quantization/utils/mxfp4_utils.py

@@ -7,6 +7,7 @@
 
 from vllm.logger import init_logger
 from vllm.platforms import current_platform
+from vllm.triton_utils import triton
 from vllm.utils.torch_utils import direct_register_custom_op, is_torch_equal_or_newer
 
 logger = init_logger(__name__)
@@ -99,6 +100,14 @@ def _can_support_mxfp4(
     )
 
 
+def get_padding_alignment():
+    return (
+        256
+        if triton.runtime.driver.active.get_current_target().arch in ("gfx950",)
+        else 128
+    )
+
+
 def _dequant_mxfp4(
     x: torch.Tensor, scale: torch.Tensor, float_dtype: torch.dtype
 ) -> torch.Tensor:

vllm/model_executor/models/qwen2_5_vl.py

@@ -67,9 +67,7 @@
 from vllm.model_executor.layers.quantization import QuantizationConfig
 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
 from vllm.model_executor.models.module_mapping import MultiModelKeys
-from vllm.model_executor.models.transformers.utils import (
-    should_torch_compile_mm_vit,
-)
+from vllm.model_executor.models.vision import should_torch_compile_mm_vit
 from vllm.multimodal import MULTIMODAL_REGISTRY
 from vllm.multimodal.evs import (
     compute_mrope_for_media,

vllm/model_executor/models/transformers/utils.py

@@ -205,14 +205,3 @@ def can_enable_torch_compile(vllm_config: "VllmConfig") -> bool:
     # Dynamic rope scaling is not compatible with torch.compile
     rope_scaling: dict = getattr(text_config, "rope_scaling", None) or {}
     return rope_scaling.get("rope_type") != "dynamic"
-
-
-def should_torch_compile_mm_vit(vllm_config: "VllmConfig") -> bool:
-    """
-    Callable to be passed to `@support_torch_compile`'s `enable_if` argument.
-
-    Defaults to `True` but is disabled in the following situations:
-
-    - The model uses dynamic rope scaling.
-    """
-    return vllm_config.compilation_config.compile_mm_encoder

vllm/model_executor/models/vision.py

@@ -11,6 +11,7 @@
 from transformers import PretrainedConfig
 
 from vllm.attention.backends.registry import _Backend
+from vllm.config import VllmConfig
 from vllm.distributed import (
     get_tensor_model_parallel_rank,
     get_tensor_model_parallel_world_size,
@@ -100,6 +101,11 @@ def get_vit_attn_backend(
     return current_platform.get_vit_attn_backend(head_size, dtype)
 
 
+def should_torch_compile_mm_vit(vllm_config: VllmConfig) -> bool:
+    """Callable to be passed to `@support_torch_compile`'s `enable_if` argument."""
+    return vllm_config.compilation_config.compile_mm_encoder
+
+
 VisionFeatureSelectStrategyStr = Literal["class", "default", "full"]
 
 VisionFeatureSelectStrategy: TypeAlias = (