[CI][ROCm] Fix deterministic AMD L3 blockers (#6884)

Signed-off-by: andyluo7 <andy.luo@amd.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>

.buildkite/amd/test-amd-merge.yml

@@ -238,13 +238,10 @@ steps:
       grade: Blocking
       timeout_in_minutes: 50
       commands:
-        - |
-          timeout 40m bash -c '
-            export VLLM_LOGGING_LEVEL=DEBUG
-            export VLLM_WORKER_MULTIPROC_METHOD=spawn
-            export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
-            pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m 'advanced_model and cuda' --run-level 'advanced_model'
-          '
+        - export VLLM_LOGGING_LEVEL=DEBUG
+        - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+        - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
+        - timeout 40m pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m "advanced_model and cuda" --run-level "advanced_model"
 
 # TODO: Bagel test on ROCm is very unstable. @tjtanaa
 # Need to debug before reneable numerical changes across large PRs

tests/buildkite/test_amd_pipeline.py

@@ -0,0 +1,42 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+from pathlib import Path
+from shlex import split
+
+import pytest
+import yaml
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
+AMD_MERGE_PIPELINE = Path(".buildkite/amd/test-amd-merge.yml")
+
+
+def _find_step(label: str) -> dict:
+    pipeline = yaml.safe_load(AMD_MERGE_PIPELINE.read_text(encoding="utf-8"))
+
+    def walk(steps: list[dict]) -> dict | None:
+        for step in steps:
+            if step.get("label") == label:
+                return step
+            if nested := walk(step.get("steps", [])):
+                return nested
+        return None
+
+    step = walk(pipeline.get("steps", []))
+    assert step is not None, f"missing AMD pipeline step: {label}"
+    return step
+
+
+def test_qwen3_tts_base_preserves_advanced_model_arguments() -> None:
+    step = _find_step("Qwen3-TTS Base E2E Test")
+    commands = step["commands"]
+
+    assert all("bash -c" not in command for command in commands)
+    pytest_command = next(command for command in commands if "pytest" in command)
+    argv = split(pytest_command)
+
+    marker_index = argv.index("-m")
+    run_level_index = argv.index("--run-level")
+    assert argv[marker_index + 1] == "advanced_model and cuda"
+    assert argv[run_level_index + 1] == "advanced_model"

tests/config/test_config_factory.py

@@ -2399,6 +2399,21 @@ class TestBaseConfigInheritance:
         # CI overrides max_tokens
         assert s0["max_tokens"] == 150
 
+    def test_qwen3_omni_colocate_async_bounds_only_rocm_kv_cache(self):
+        ci_path = Path(get_deploy_config_path("ci/qwen3_omni_moe_colocate_async.yaml"))
+        pipeline = resolve_pipeline_config("qwen3_omni_moe_thinker_only")
+        assert isinstance(pipeline, PipelineConfig)
+
+        cuda = _apply_platform_overrides(load_deploy_config(ci_path), platform="cuda")
+        cuda_stage = merge_pipeline_deploy(pipeline, cuda)[0]
+        assert cuda_stage.yaml_engine_args["gpu_memory_utilization"] == 0.9
+        assert "kv_cache_memory_bytes" not in cuda_stage.yaml_engine_args
+
+        rocm = _apply_platform_overrides(load_deploy_config(ci_path), platform="rocm")
+        rocm_stage = merge_pipeline_deploy(pipeline, rocm)[0]
+        assert "gpu_memory_utilization" not in rocm_stage.yaml_engine_args
+        assert rocm_stage.yaml_engine_args["kv_cache_memory_bytes"] == 2 * 1024**3
+
     def test_pure_inheritance_overlay(self, tmp_path):
         """An overlay with only ``base_config`` inherits everything."""
         base = Path(get_deploy_config_path("qwen3_omni_moe.yaml"))

tests/e2e/offline_inference/test_qwen3_omni_colocate_async.py

@@ -22,7 +22,7 @@ from vllm.inputs import TokensPrompt
 
 from tests.helpers.clean import wait_for_gpu_memory_to_clear
 from tests.helpers.mark import hardware_test
-from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
+from tests.helpers.stage_config import get_deploy_config_path
 from vllm_omni.entrypoints.async_omni import AsyncOmni
 from vllm_omni.platforms import current_omni_platform
 
@@ -52,26 +52,7 @@ def _colocate_async_deploy() -> str:
     CI yaml plus CuMem on every stage fails engine-core startup in the L3
     merge job: Stage 1 resolves ``devices: "1"`` while only GPU 0 is visible.
     """
-    return modify_stage_config(
-        get_deploy_config_path("qwen3_omni_moe_thinking.yaml"),
-        updates={
-            "stages": {
-                0: {
-                    "enable_sleep_mode": True,
-                    "enforce_eager": True,
-                    "trust_remote_code": True,
-                    "tensor_parallel_size": 1,
-                    "devices": "0",
-                    "gpu_memory_utilization": 0.9,
-                    "max_num_seqs": 1,
-                    "max_model_len": 8192,
-                    "max_num_batched_tokens": 8192,
-                    "enable_prefix_caching": False,
-                    "skip_mm_profiling": True,
-                },
-            },
-        },
-    )
+    return get_deploy_config_path("ci/qwen3_omni_moe_colocate_async.yaml")
 
 
 @pytest.mark.advanced_model

tests/helpers/stage_config.py

@@ -381,6 +381,40 @@ _CI_OVERLAYS: dict[str, dict[str, Any]] = {
             },
         },
     },
+    "qwen3_omni_moe_colocate_async": {
+        "base_config": "qwen3_omni_moe_thinking.yaml",
+        "stages": [
+            {
+                "stage_id": 0,
+                "enable_sleep_mode": True,
+                "enforce_eager": True,
+                "trust_remote_code": True,
+                "tensor_parallel_size": 1,
+                "devices": "0",
+                "max_num_seqs": 1,
+                "max_model_len": 8192,
+                "max_num_batched_tokens": 8192,
+                "enable_prefix_caching": False,
+                "skip_mm_profiling": True,
+            },
+        ],
+        "platforms": {
+            "rocm": {
+                "stages": [
+                    {
+                        "stage_id": 0,
+                        # This control-plane test needs only 8K tokens. On MI300,
+                        # percentage sizing attempted a ~125 GiB KV allocation
+                        # after the 64.5 GiB model load and then failed while
+                        # allocating the cache. Keep CUDA's existing sizing and
+                        # give only ROCm a bounded cache with ample headroom.
+                        "gpu_memory_utilization": None,
+                        "kv_cache_memory_bytes": 2 * 1024**3,
+                    },
+                ],
+            },
+        },
+    },
     "qwen3_omni_moe_multi_replicas_4gpu": {
         "base_config": "qwen3_omni_moe.yaml",
         "async_chunk": True,