[Test] Cleanup for helpers (#6523)
Signed-off-by: wangyu <410167048@qq.com> Signed-off-by: [Your Name] <your.email@example.com> Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
1 parent
8360346a66
commit
4af0ddc1c2
140 files changed
+4121
-4194
No files matched your search
@@ -1,3 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""
|
||||
Wan2.2 reliability integration tests.
|
||||
"""
|
||||
@@ -183,7 +186,7 @@ def _assert_post_fault_health_terminal(host: str, port: int, *, scenario: str) -
|
||||
@pytest.mark.slow
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server_function", DIFFUSION_VIDEO_PARAMS, indirect=True)
|
||||
def test_reliability_fault_gpu_oom_video_large_request_failure(omni_server_function, openai_client_function) -> None:
|
||||
def test_reliability_fault_gpu_oom_video_large_request_failure(omni_server_function, online_client_function) -> None:
|
||||
stage_config_path = getattr(omni_server_function, "stage_config_path", None)
|
||||
device_spec = resolve_oom_device_spec(OOM_INJECTION_CONFIG, stage_config_path)
|
||||
handle = inject_gpu_oom(
|
||||
@@ -211,7 +214,7 @@ def test_reliability_fault_gpu_oom_video_large_request_failure(omni_server_funct
|
||||
"stream": False,
|
||||
}
|
||||
try:
|
||||
openai_client_function.send_video_diffusion_request(request_config, request_num=1)
|
||||
online_client_function.send_video_diffusion_request(request_config, request_num=1)
|
||||
except Exception as exc:
|
||||
assert_fault_exception(exc, FAULT_ERROR_KEYWORDS)
|
||||
else:
|
||||
@@ -230,11 +233,11 @@ def test_reliability_fault_gpu_oom_video_large_request_failure(omni_server_funct
|
||||
@pytest.mark.parametrize("omni_server_function", DIFFUSION_VIDEO_PARAMS, indirect=True)
|
||||
def test_reliability_fault_process_kill_request_failure(
|
||||
omni_server_after_fault_function,
|
||||
openai_client_function,
|
||||
online_client_function,
|
||||
) -> None:
|
||||
request_config = _video_request_config()
|
||||
try:
|
||||
openai_client_function.send_video_diffusion_request(request_config, request_num=1)
|
||||
online_client_function.send_video_diffusion_request(request_config, request_num=1)
|
||||
except Exception as exc:
|
||||
assert_fault_exception(exc, PROCESS_KILL_ERROR_KEYWORDS)
|
||||
else:
|
||||
@@ -251,7 +254,7 @@ def test_reliability_fault_process_kill_request_failure(
|
||||
@pytest.mark.parametrize("omni_server_function", DIFFUSION_VIDEO_PARAMS, indirect=True)
|
||||
def test_reliability_fault_process_kill_health_fast_fail_and_concurrent(
|
||||
omni_server_after_fault_function,
|
||||
openai_client_function,
|
||||
online_client_function,
|
||||
) -> None:
|
||||
"""Black-box: after process kill, /v1/videos fails fast and concurrent calls don't hang.
|
||||
|
||||
@@ -308,7 +311,7 @@ def test_reliability_fault_process_kill_health_fast_fail_and_concurrent(
|
||||
start = time.monotonic()
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=3) as executor:
|
||||
futures = [
|
||||
executor.submit(openai_client_function.send_video_diffusion_request, request_config, 1) for _ in range(3)
|
||||
executor.submit(online_client_function.send_video_diffusion_request, request_config, 1) for _ in range(3)
|
||||
]
|
||||
done, pending = concurrent.futures.wait(
|
||||
futures,
|
||||
@@ -367,13 +370,13 @@ def test_reliability_fault_process_kill_health_fast_fail_and_concurrent(
|
||||
@pytest.mark.parametrize("omni_server_function", DIFFUSION_VIDEO_PARAMS, indirect=True)
|
||||
def test_reliability_fault_process_kill_worker_with_load_request_failure(
|
||||
omni_server_function,
|
||||
openai_client_function,
|
||||
online_client_function,
|
||||
fault_injector: FaultInjector,
|
||||
) -> None:
|
||||
request_config = _video_request_config()
|
||||
scenario = "kill_worker_with_load"
|
||||
load_result = run_fault_injection_with_rate_load(
|
||||
submit_request=lambda: openai_client_function.send_video_diffusion_request(request_config, request_num=1),
|
||||
submit_request=lambda: online_client_function.send_video_diffusion_request(request_config, request_num=1),
|
||||
inject_fault=lambda: fault_injector(omni_server_function),
|
||||
num_requests=INFLIGHT_INJECTION_REQUEST_COUNT,
|
||||
request_rate=INFLIGHT_INJECTION_REQUEST_RATE,
|
||||
@@ -516,7 +519,7 @@ def test_reliability_video_oom_recovers_after_fault_removed(
|
||||
@pytest.mark.parametrize("omni_server_function", DIFFUSION_VIDEO_PARAMS, indirect=True)
|
||||
def test_reliability_fault_process_kill_serve_root_with_load_fast_fail_and_cleanup(
|
||||
omni_server_function,
|
||||
openai_client_function,
|
||||
online_client_function,
|
||||
signal_name: str,
|
||||
) -> None:
|
||||
"""Black-box: during in-flight video requests, kill serve root and verify post-fault behavior/cleanup."""
|
||||
@@ -524,7 +527,7 @@ def test_reliability_fault_process_kill_serve_root_with_load_fast_fail_and_clean
|
||||
scenario = f"kill_serve_root_with_load_{signal_name.lower()}"
|
||||
injector = make_server_root_kill_fault_injector(signal_name=signal_name, post_kill_wait_seconds=2.0)
|
||||
load_result = run_fault_injection_with_rate_load(
|
||||
submit_request=lambda: openai_client_function.send_video_diffusion_request(request_config, request_num=1),
|
||||
submit_request=lambda: online_client_function.send_video_diffusion_request(request_config, request_num=1),
|
||||
inject_fault=lambda: injector(omni_server_function),
|
||||
num_requests=INFLIGHT_INJECTION_REQUEST_COUNT,
|
||||
request_rate=INFLIGHT_INJECTION_REQUEST_RATE,
|
||||
@@ -601,7 +604,7 @@ def test_reliability_fault_process_kill_tree_no_load_fast_fail_and_cleanup(
|
||||
@pytest.mark.parametrize("omni_server_function", DIFFUSION_VIDEO_PARAMS, indirect=True)
|
||||
def test_reliability_fault_process_kill_tree_with_load_fast_fail_and_cleanup(
|
||||
omni_server_function,
|
||||
openai_client_function,
|
||||
online_client_function,
|
||||
signal_name: str,
|
||||
) -> None:
|
||||
"""Black-box: during in-flight video requests, kill server process tree and verify post-fault behavior/cleanup."""
|
||||
@@ -613,7 +616,7 @@ def test_reliability_fault_process_kill_tree_with_load_fast_fail_and_cleanup(
|
||||
inter_kill_wait_seconds=0.1,
|
||||
)
|
||||
load_result = run_fault_injection_with_rate_load(
|
||||
submit_request=lambda: openai_client_function.send_video_diffusion_request(request_config, request_num=1),
|
||||
submit_request=lambda: online_client_function.send_video_diffusion_request(request_config, request_num=1),
|
||||
inject_fault=lambda: injector(omni_server_function),
|
||||
num_requests=INFLIGHT_INJECTION_REQUEST_COUNT,
|
||||
request_rate=INFLIGHT_INJECTION_REQUEST_RATE,
|
||||
|
||||
Reference in new issue
Block a user