feat: DynamoPlanner profiler to use hf_id for AIConfigurator 0.4.0 (#4167)

Signed-off-by: Jason Zhou <jasonzho@jasonzho-mlt.client.nvidia.com>
Signed-off-by: Jason Zhou <jasonzho@nvidia.com>
Co-authored-by: Jason Zhou <jasonzho@jasonzho-mlt.client.nvidia.com>
Signed-off-by: Daiyaan <darfeen@nvidia.com>

Author: 2 people

ATTRIBUTIONS-Python.md

@@ -441,7 +441,7 @@ License: `Apache`
   - `Homepage`: https://github.com/huggingface/accelerate
 
 
-## aiconfigurator (0.2.0)
+## aiconfigurator (0.4.0)
 
 ### Licenses
 License: `Apache-2.0`

benchmarks/profiler/deploy/profile_sla_aic_dgdr.yaml

@@ -19,7 +19,7 @@ spec:
         # AI Configurator mode (fast simulation-based profiling)
         use_ai_configurator: true
         aic_system: h200_sxm
-        aic_model_name: QWEN3_32B
+        aic_hf_id: Qwen/Qwen3-32B
         aic_backend_version: "0.20.0"
 
       # SLA targets for profiling

benchmarks/profiler/profile_sla.py

@@ -143,9 +143,6 @@ async def run_profile(args):
             assert args.backend in [
                 "sglang"
             ], "MoE model support is only available for SGLang"
-            assert (
-                not args.use_ai_configurator
-            ), "MoE model is not supported in ai-configurator"
         else:
             logger.info(
                 "Dense model profiling, sweeping TP size for prefill and decode"
@@ -204,26 +201,30 @@ async def run_profile(args):
                 raise ValueError(
                     "Must provide --aic-system when using --use-ai-configurator."
                 )
-            if not args.aic_model_name:
-                raise ValueError(
-                    "Must provide --aic-model-name when using --use-ai-configurator."
-                )
-            if not args.aic_backend_version:
-                raise ValueError(
-                    "Must provide --aic-backend-version when using --use-ai-configurator."
-                )
+
+            # Fallback to args.model if aic_hf_id is not provided
+            if not args.aic_hf_id:
+                if args.model:
+                    logger.info(
+                        f"--aic-hf-id not provided, using --model ({args.model}) as HuggingFace ID for AI configurator"
+                    )
+                    args.aic_hf_id = args.model
+                else:
+                    raise ValueError(
+                        "Must provide --aic-hf-id or --model when using --use-ai-configurator."
+                    )
 
             logger.info("Using aiconfigurator to estimate performance...")
             ai_configurator_perf_estimator = AIConfiguratorPerfEstimator(
-                args.aic_model_name,
+                args.aic_hf_id,
                 args.aic_system.lower(),
                 args.aic_backend,
                 args.aic_backend_version,
             )
         else:
-            if args.aic_system or args.aic_model_name or args.aic_backend_version:
+            if args.aic_system or args.aic_hf_id or args.aic_backend_version:
                 logger.warning(
-                    "Ignoring --aic-system, --aic-model-name, and/or --backend-version "
+                    "Ignoring --aic-system, --aic-hf-id, and/or --backend-version "
                     "when not using --use-ai-configurator."
                 )
 

benchmarks/profiler/utils/estimate_perf.py

@@ -36,14 +36,19 @@ class AIConfiguratorPerfEstimator:
 
     def __init__(
         self,
-        model_name: str,  # e.g. "QWEN3_32B"
+        hf_id: str,  # e.g. "Qwen/Qwen3-32B"
         system: str,  # e.g. "h200_sxm"
         backend: str,  # e.g. "trtllm"
         version: str,  # e.g. "0.20.0"
     ):
         aiconfigurator = _try_import_aiconfigurator()
 
         logger.info("Loading aiconfigurator database. This might take a few seconds...")
+        if not version:
+            version = aiconfigurator.sdk.perf_database.get_latest_database_version(
+                system,
+                backend,
+            )
         self.database = aiconfigurator.sdk.perf_database.get_database(
             system=system,
             backend=backend,
@@ -56,18 +61,15 @@ def __init__(
         logger.info("aiconfigurator database loaded.")
 
         self.backend = aiconfigurator.sdk.backends.factory.get_backend(backend)
-
-        # This is the aiconfigurator model name (such as QWEN3_32B or DEEPSEEK_V3)
-        # rather than the HF model name.
-        self.model_name = model_name
+        self.hf_id = hf_id
 
     def _get_model(self, **model_config_kwargs):
         aiconfigurator = _try_import_aiconfigurator()
 
         # NOTE: MOE models error out unless moe_tp_size and moe_ep_size are provided.
         model_config = aiconfigurator.sdk.config.ModelConfig(**model_config_kwargs)
         model = aiconfigurator.sdk.models.get_model(
-            self.model_name, model_config, self.backend
+            self.hf_id, model_config, self.backend
         )
         return model
 

benchmarks/profiler/utils/profiler_argparse.py

@@ -80,7 +80,7 @@ def create_profiler_parser() -> argparse.Namespace:
             decode_interpolation_granularity: Int (how many samples to benchmark to interpolate ITL under different active kv cache size and decode context length, default: 6)
             use_ai_configurator: Boolean (use ai-configurator to estimate benchmarking results instead of running actual deployment, default: False)
             aic_system: String (target system for use with aiconfigurator, default: None)
-            aic_model_name: String (aiconfigurator name of the target model, default: None)
+            aic_hf_id: String (aiconfigurator huggingface id of the target model, default: None)
             aic_backend: String (aiconfigurator backend of the target model, if not provided, will use args.backend, default: "")
             aic_backend_version: String (specify backend version when using aiconfigurator to estimate perf, default: None)
             dry_run: Boolean (dry run the profile job, default: False)
@@ -260,10 +260,10 @@ def create_profiler_parser() -> argparse.Namespace:
         help="Target system for use with aiconfigurator (e.g. h100_sxm, h200_sxm)",
     )
     parser.add_argument(
-        "--aic-model-name",
+        "--aic-hf-id",
         type=str,
-        default=config.get("sweep", {}).get("aic_model_name"),
-        help="aiconfigurator name of the target model (e.g. QWEN3_32B, DEEPSEEK_V3)",
+        default=config.get("sweep", {}).get("aic_hf_id"),
+        help="aiconfigurator name of the target model (e.g. Qwen/Qwen3-32B, meta-llama/Llama-3.1-405B)",
     )
     parser.add_argument(
         "--aic-backend",

benchmarks/pyproject.toml

@@ -40,7 +40,7 @@ classifiers = [
 ]
 
 dependencies = [
-    "aiconfigurator @ git+https://github.com/ai-dynamo/aiconfigurator.git@11b6d821f1fbb34300bb0ed4945f647e89fb411a",
+    "aiconfigurator @ git+https://github.com/ai-dynamo/aiconfigurator.git@5554d2eb8206738c66048bf2d72183e9bcd85759",
     "networkx",
     "pandas",
     "pydantic>=2",

container/deps/requirements.txt

@@ -2,7 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 accelerate==1.6.0
-aiconfigurator @ git+https://github.com/ai-dynamo/aiconfigurator.git@11b6d821f1fbb34300bb0ed4945f647e89fb411a
+aiconfigurator @ git+https://github.com/ai-dynamo/aiconfigurator.git@5554d2eb8206738c66048bf2d72183e9bcd85759
 aiofiles
 aiperf @ git+https://github.com/ai-dynamo/aiperf.git@16dad7c02fcd959ba96823d7bfe7e681e5d5b41d
 av==15.0.0

deploy/cloud/operator/config/samples/nvidia.com_v1alpha1_dynamographdeploymentrequest.yaml

@@ -53,7 +53,7 @@ spec:
         # AI Configurator mode (fast simulation-based profiling, 20-30 seconds)
         use_ai_configurator: false  # Set to false for online profiling (2-4 hours)
         aic_system: h200_sxm  # Target GPU system for AI Configurator
-        aic_model_name: QWEN3_0.6B  # Model name for AI Configurator
+        aic_hf_id: Qwen/Qwen3-0.6B  # HuggingFace model ID for AI Configurator
         aic_backend_version: "0.20.0"  # Backend version for AI Configurator
 
       # SLA targets for profiling

deploy/cloud/operator/internal/controller/dynamographdeploymentrequest_controller_test.go

@@ -348,7 +348,7 @@ var _ = Describe("DynamoGraphDeploymentRequest Controller", func() {
 							"sweep": map[string]interface{}{
 								"use_ai_configurator": true,
 								"aic_system":          "h200_sxm",
-								"aic_model_name":      "QWEN3_32B",
+								"aic_hf_id":           "Qwen/Qwen3-32B",
 								"aic_backend_version": "0.20.0",
 							},
 						}),
@@ -1058,7 +1058,7 @@ var _ = Describe("DGDR Profiler Arguments", func() {
 							"sweep": map[string]interface{}{
 								"use_ai_configurator": true,
 								"aic_system":          "h200_sxm",
-								"aic_model_name":      "QWEN3_32B",
+								"aic_hf_id":           "Qwen/Qwen3-32B",
 								"aic_backend_version": "0.20.0",
 							},
 						}),

docs/benchmarks/sla_driven_profiling.md

@@ -303,17 +303,12 @@ profilingConfig:
     sweep:
       use_ai_configurator: true
       aic_system: h200_sxm              # GPU system: h100_sxm, h200_sxm, b200_sxm, gb200_sxm, a100_sxm
-      aic_model_name: QWEN3_32B         # AIC model identifier (see supported list)
-      aic_backend_version: "0.20.0"     # TensorRT-LLM version: 0.20.0, 1.0.0rc3, 1.0.0rc6
+      aic_hf_id: Qwen/Qwen3-32B         # Huggingface model id
+      aic_backend_version: "0.20.0"     # TensorRT-LLM version: 0.20.0, 1.0.0rc3
 ```
 
 **Supported configurations:** See [AI Configurator documentation](https://github.com/ai-dynamo/aiconfigurator#supported-features)
 
-**Model name mapping examples:**
-- `Qwen/Qwen3-32B` → `QWEN3_32B`
-- `meta-llama/Llama-3.1-70B` → `LLAMA3.1_70B`
-- `deepseek-ai/DeepSeek-V3` → `DEEPSEEK_V3`
-
 ### Planner Configuration (Optional)
 
 Pass arguments to the SLA planner: