Increase timeout for Diffusion Model Test and Diffusion Images API LoRA E2E in Buildkite configurations to 30 minutes. Also, introduce a new timeout constant for diffusion initialization in relevant test files to enhance stability during testing. Update OmniServer initialization to utilize the new timeout settings.
This commit is contained in:
1 parent
004c60a031
commit
8a31e7865d
6 files changed
+24
-7
No files matched your search
@@ -1,7 +1,7 @@
|
||||
steps:
|
||||
|
||||
- label: "Diffusion Model Test"
|
||||
timeout_in_minutes: 20
|
||||
timeout_in_minutes: 30
|
||||
agent_pool: mi325_2
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
@@ -11,7 +11,7 @@ steps:
|
||||
- pytest -s -v tests/e2e/offline_inference/test_t2i_model.py
|
||||
|
||||
- label: "Diffusion Images API LoRA E2E"
|
||||
timeout_in_minutes: 20
|
||||
timeout_in_minutes: 30
|
||||
agent_pool: mi325_1
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
|
||||
@@ -16,7 +16,7 @@ steps:
|
||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||
|
||||
- label: "Diffusion Model Test"
|
||||
timeout_in_minutes: 20
|
||||
timeout_in_minutes: 30
|
||||
depends_on: upload-merge-pipeline
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/offline_inference/test_t2i_model.py -m "advanced_model and diffusion" --run-level "advanced_model"
|
||||
@@ -33,7 +33,7 @@ steps:
|
||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||
|
||||
- label: "Diffusion Images API LoRA E2E"
|
||||
timeout_in_minutes: 20
|
||||
timeout_in_minutes: 30
|
||||
depends_on: upload-merge-pipeline
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/online_serving/test_images_generations_lora.py
|
||||
|
||||
@@ -18,7 +18,7 @@ steps:
|
||||
- label: "Diffusion Model Test"
|
||||
depends_on: upload-ready-pipeline
|
||||
commands:
|
||||
- timeout 20m pytest -s -v tests/e2e/offline_inference/test_t2i_model.py -m "core_model and diffusion" --run-level "core_model"
|
||||
- timeout 30m pytest -s -v tests/e2e/offline_inference/test_t2i_model.py -m "core_model and diffusion" --run-level "core_model"
|
||||
agents:
|
||||
queue: "gpu_1_queue"
|
||||
plugins:
|
||||
|
||||
@@ -24,6 +24,7 @@ os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "1"
|
||||
# This test is specific to Z-Image LoRA behavior. Keep it focused on a single
|
||||
# model to reduce runtime and avoid extra downloads.
|
||||
models = ["Tongyi-MAI/Z-Image-Turbo"]
|
||||
DIFFUSION_INIT_TIMEOUT_S = 600
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name", models)
|
||||
@@ -76,7 +77,11 @@ def test_diffusion_model(model_name: str, tmp_path: Path):
|
||||
)
|
||||
return str(adapter_dir)
|
||||
|
||||
m = Omni(model=model_name)
|
||||
m = Omni(
|
||||
model=model_name,
|
||||
stage_init_timeout=DIFFUSION_INIT_TIMEOUT_S,
|
||||
init_timeout=DIFFUSION_INIT_TIMEOUT_S,
|
||||
)
|
||||
try:
|
||||
# high resolution may cause OOM on L4
|
||||
height = 256
|
||||
|
||||
@@ -28,6 +28,7 @@ from tests.utils import hardware_test
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
MODEL = "Tongyi-MAI/Z-Image-Turbo"
|
||||
DIFFUSION_INIT_TIMEOUT_S = 600
|
||||
|
||||
|
||||
PROMPT = "a photo of a cat sitting on a laptop keyboard"
|
||||
@@ -37,7 +38,17 @@ SEED = 42
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def omni_server():
|
||||
with OmniServer(MODEL, ["--num-gpus", "1"]) as server:
|
||||
with OmniServer(
|
||||
MODEL,
|
||||
[
|
||||
"--num-gpus",
|
||||
"1",
|
||||
"--stage-init-timeout",
|
||||
str(DIFFUSION_INIT_TIMEOUT_S),
|
||||
"--init-timeout",
|
||||
str(DIFFUSION_INIT_TIMEOUT_S),
|
||||
],
|
||||
) as server:
|
||||
yield server
|
||||
|
||||
|
||||
|
||||
@@ -161,6 +161,7 @@ class Qwen3Omni_VisionTransformer(_Qwen3Omni_VisionTransformer):
|
||||
cu_seqlens,
|
||||
self.hidden_size,
|
||||
self.tp_size,
|
||||
self.device,
|
||||
)
|
||||
cu_seqlens = torch.from_numpy(cu_seqlens).to(self.device, non_blocking=True)
|
||||
|
||||
|
||||
Reference in new issue
Block a user