Merge origin/main into dev/rebase-v0.16.0.
Resolve arg_utils merge conflict and align CI base image tag to v0.16.0 for the vLLM rebase track. Co-authored-by: Cursor <cursoragent@cursor.com> Signed-off-by: Taichang Zhou <tzhouam@connect.ust.hk>
This commit is contained in:
commit
e2ced80d59
64 files changed
+5186
-1228
No files matched your search
@@ -17,13 +17,15 @@ if str(REPO_ROOT) not in sys.path:
|
||||
from vllm_omni import Omni
|
||||
|
||||
# Models to test and expected saved memory in MB, correspondingly
|
||||
MODELS_SAVED_MEMORY_MB = {"riverclouds/qwen_image_random": 4500}
|
||||
MODELS_SAVED_MEMORY_MB = {
|
||||
"riverclouds/qwen_image_random": 4500,
|
||||
# "Wan-AI/Wan2.2-T2V-A14B-Diffusers": 45000,
|
||||
}
|
||||
|
||||
|
||||
def run_inference(
|
||||
model_name: str,
|
||||
layerwise_offload: bool = False,
|
||||
num_gpu_layers: int = 1,
|
||||
num_inference_steps: int = 3,
|
||||
) -> float:
|
||||
# For now, only support on GPU, so apply torch.cuda operations here
|
||||
@@ -36,7 +38,6 @@ def run_inference(
|
||||
m = Omni(
|
||||
model=model_name,
|
||||
enable_layerwise_offload=layerwise_offload,
|
||||
layerwise_num_gpu_layers=num_gpu_layers,
|
||||
boundary_ratio=0.875,
|
||||
flow_shift=5.0,
|
||||
)
|
||||
@@ -83,16 +84,12 @@ def test_layerwise_offload_diffusion_model(model_name: str):
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
# Run with layerwise offloading (1 layer on device)
|
||||
layerwise_offload_peak_memory = run_inference(model_name, layerwise_offload=True, num_gpu_layers=1)
|
||||
layerwise_offload_peak_memory = run_inference(model_name, layerwise_offload=True)
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
# Run with 2 layers on device
|
||||
layerwise_offload_two_layers_peak = run_inference(model_name, layerwise_offload=True, num_gpu_layers=2)
|
||||
except Exception:
|
||||
pytest.fail("Inference failed")
|
||||
|
||||
print(f"Layerwise offload peak memory (1 GPU layer): {layerwise_offload_peak_memory} MB")
|
||||
print(f"Layerwise offload peak memory (2 GPU layers): {layerwise_offload_two_layers_peak} MB")
|
||||
print(f"No offload peak memory: {no_offload_peak_memory} MB")
|
||||
|
||||
# Verify that layerwise offloading significantly reduces memory usage
|
||||
@@ -101,10 +98,3 @@ def test_layerwise_offload_diffusion_model(model_name: str):
|
||||
f"Layerwise offload peak memory {layerwise_offload_peak_memory} MB "
|
||||
f"should be significantly less than no offload peak memory {no_offload_peak_memory} MB"
|
||||
)
|
||||
|
||||
# Verify that 2 GPU layers uses more memory than 1 GPU layer
|
||||
# But not excessively more (should be a reasonable increase)
|
||||
assert layerwise_offload_peak_memory < layerwise_offload_two_layers_peak, (
|
||||
f"1 GPU layer peak {layerwise_offload_peak_memory} MB should be < "
|
||||
f"2 GPU layers peak {layerwise_offload_two_layers_peak} MB"
|
||||
)
|
||||
@@ -34,10 +34,17 @@ elif current_omni_platform.is_rocm():
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 1, "rocm": 2})
|
||||
@pytest.mark.parametrize("model_name", models)
|
||||
def test_diffusion_model(model_name: str):
|
||||
def test_diffusion_model(model_name: str, run_level):
|
||||
if run_level == "core_model" and model_name != "riverclouds/qwen_image_random":
|
||||
pytest.skip()
|
||||
|
||||
if run_level == "advanced_model" and model_name == "riverclouds/qwen_image_random":
|
||||
pytest.skip()
|
||||
|
||||
m = None
|
||||
try:
|
||||
m = Omni(model=model_name)
|
||||
|
||||
@@ -154,7 +154,7 @@ def _run_zimage_generate(
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
@pytest.mark.parallel
|
||||
@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards={"cuda": 4, "rocm": 2})
|
||||
|
||||
@@ -145,7 +145,7 @@ def _basic_payload() -> dict:
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.diffusion
|
||||
@hardware_test(res={"cuda": "L4", "rocm": "MI325"})
|
||||
def test_images_generations_per_request_lora_switching(omni_server: OmniServer, tmp_path: Path) -> None:
|
||||
|
||||
@@ -10,7 +10,7 @@ stage_args:
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
gpu_memory_utilization: 0.6
|
||||
enforce_eager: true
|
||||
enforce_eager: false
|
||||
trust_remote_code: true
|
||||
engine_output_type: latent # Output hidden states for talker
|
||||
distributed_executor_backend: "mp"
|
||||
|
||||
Reference in new issue
Block a user