Merge origin/main into dev/vllm-align

Resolve conflicts:
- docker/Dockerfile.ci: Keep wheel install section, update hash to 6042e66cd5
- vllm_omni/platforms/npu/worker/npu_model_runner.py: Accept main's refactoring

Made-with: Cursor

Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
tzhouam committed 2026-02-26 08:19:53 +00:00
commit 8d0c46487e
58 files changed
+4358 -1530

No files matched your search

+33 -32
View File
@@ -848,41 +848,42 @@ def modify_stage_config(
del config[key]
# Apply updates
for key, value in updates.items():
if key == "stage_args":
if value and isinstance(value, dict):
stage_args = config.get("stage_args", [])
if not stage_args:
raise ValueError("stage_args does not exist in config")
if updates:
for key, value in updates.items():
if key == "stage_args":
if value and isinstance(value, dict):
stage_args = config.get("stage_args", [])
if not stage_args:
raise ValueError("stage_args does not exist in config")
for stage_id, stage_updates in value.items():
# Find stage by ID
target_stage = None
for stage in stage_args:
if stage.get("stage_id") == int(stage_id):
target_stage = stage
break
for stage_id, stage_updates in value.items():
# Find stage by ID
target_stage = None
for stage in stage_args:
if stage.get("stage_id") == int(stage_id):
target_stage = stage
break
if target_stage is None:
available_ids = [s.get("stage_id") for s in stage_args if "stage_id" in s]
raise KeyError(f"Stage ID {stage_id} not found, available: {available_ids}")
if target_stage is None:
available_ids = [s.get("stage_id") for s in stage_args if "stage_id" in s]
raise KeyError(f"Stage ID {stage_id} not found, available: {available_ids}")
# Apply updates to this stage
for path, val in stage_updates.items():
# Check if this is a simple key (not dot-separated)
# Example: 'engine_input_source' vs 'engine_args.max_model_len'
if "." not in path:
# Direct key assignment (e.g., updating a list value)
target_stage[path] = val
else:
# Dot-separated path (e.g., nested dict access)
apply_update(target_stage, path, val)
elif "." in key:
# Apply using dot-separated path
apply_update(config, key, value)
else:
# Direct top-level key
config[key] = value
# Apply updates to this stage
for path, val in stage_updates.items():
# Check if this is a simple key (not dot-separated)
# Example: 'engine_input_source' vs 'engine_args.max_model_len'
if "." not in path:
# Direct key assignment (e.g., updating a list value)
target_stage[path] = val
else:
# Dot-separated path (e.g., nested dict access)
apply_update(target_stage, path, val)
elif "." in key:
# Apply using dot-separated path
apply_update(config, key, value)
else:
# Direct top-level key
config[key] = value
# Save to new file with timestamp
timestamp = int(time.time())
@@ -0,0 +1,217 @@
from types import SimpleNamespace
import pytest
import torch
from PIL import Image
import vllm_omni.diffusion.models.nextstep_1_1.pipeline_nextstep_1_1 as nextstep_pipeline_module
from vllm_omni.diffusion.models.nextstep_1_1.modeling_nextstep_heads import FlowMatchingHead
from vllm_omni.diffusion.models.nextstep_1_1.pipeline_nextstep_1_1 import NextStep11Pipeline
class _DummyImageHead:
def __init__(self, token_dim: int):
self.token_dim = token_dim
self.calls = []
def sample(
self,
c: torch.Tensor,
cfg: float,
cfg_img: float,
cfg_mult: int,
timesteps_shift: float,
num_sampling_steps: int,
noise_repeat: int,
) -> torch.Tensor:
self.calls.append(
{
"batch": c.shape[0],
"cfg": cfg,
"cfg_img": cfg_img,
"cfg_mult": cfg_mult,
"noise_repeat": noise_repeat,
}
)
batch_per_prompt = c.shape[0] // cfg_mult
return torch.ones(batch_per_prompt, self.token_dim, dtype=c.dtype, device=c.device)
class _DummyModel:
def __init__(self, hidden_dim: int, token_dim: int):
self.hidden_dim = hidden_dim
self.image_head = _DummyImageHead(token_dim)
self.forward_batches = []
def image_out_projector(self, c: torch.Tensor) -> torch.Tensor:
return c
def image_in_projector(self, sampled_tokens: torch.Tensor) -> torch.Tensor:
bsz = sampled_tokens.shape[0]
return torch.zeros(bsz, 1, self.hidden_dim, dtype=sampled_tokens.dtype, device=sampled_tokens.device)
def forward_model(self, inputs_embeds: torch.Tensor, attention_mask, past_key_values, use_cache: bool):
del attention_mask, use_cache
self.forward_batches.append(inputs_embeds.shape[0])
return SimpleNamespace(
last_hidden_state=torch.zeros(
inputs_embeds.shape[0],
1,
self.hidden_dim,
dtype=inputs_embeds.dtype,
device=inputs_embeds.device,
),
past_key_values=past_key_values,
)
def _make_minimal_pipeline_for_decoding(hidden_dim: int = 8, token_dim: int = 4):
pipeline = object.__new__(NextStep11Pipeline)
pipeline.config = SimpleNamespace(latent_channels=token_dim, latent_patch_size=1, use_gen_pos_embed=False)
pipeline.model = _DummyModel(hidden_dim=hidden_dim, token_dim=token_dim)
return pipeline
@pytest.mark.parametrize(
("cfg", "cfg_img", "has_image_conditions", "expected_cfg_mult", "expected_cfg_img"),
[
(1.0, 1.0, False, 1, 1.0),
(7.5, 1.0, False, 2, 1.0),
(7.5, 8.0, False, 2, 1.0),
(7.5, 1.5, True, 3, 1.5),
],
)
def test_resolve_cfg_layout(cfg, cfg_img, has_image_conditions, expected_cfg_mult, expected_cfg_img):
cfg_mult, effective_cfg_img = NextStep11Pipeline._resolve_cfg_layout(cfg, cfg_img, has_image_conditions)
assert cfg_mult == expected_cfg_mult
assert effective_cfg_img == expected_cfg_img
def test_build_captions_ignores_image_cfg_without_image_conditions():
pipeline = object.__new__(NextStep11Pipeline)
pipeline._image_str = lambda hw: f"<image:{hw}>"
captions, images, cfg_mult, effective_cfg_img = pipeline._build_captions(
captions=["a prompt"],
images=None,
num_images_per_caption=1,
positive_prompt=None,
negative_prompt="bad quality",
cfg=7.5,
cfg_img=8.0,
)
assert cfg_mult == 2
assert effective_cfg_img == 1.0
assert images is None
assert captions == ["a prompt", "bad quality"]
def test_build_captions_enables_three_way_cfg_when_image_conditions_exist():
pipeline = object.__new__(NextStep11Pipeline)
pipeline._image_str = lambda hw: f"<image:{hw}>"
image = Image.new("RGB", (64, 32))
captions, images, cfg_mult, effective_cfg_img = pipeline._build_captions(
captions=["a prompt"],
images=[image],
num_images_per_caption=1,
positive_prompt=None,
negative_prompt="bad quality",
cfg=7.5,
cfg_img=1.5,
)
assert cfg_mult == 3
assert effective_cfg_img == 1.5
assert len(captions) == 3
assert captions[1].startswith("<image:")
assert captions[2] == "bad quality"
assert len(images) == 2
def test_decoding_non_parallel_uses_cfg_mult_for_sampling_and_duplication(monkeypatch):
pipeline = _make_minimal_pipeline_for_decoding()
monkeypatch.setattr(nextstep_pipeline_module, "get_classifier_free_guidance_world_size", lambda: 1)
c = torch.zeros(2, 1, 8)
attention_mask = torch.ones(2, 3, dtype=torch.long)
tokens = pipeline.decoding(
c=c,
attention_mask=attention_mask,
past_key_values=None,
max_new_len=1,
num_images_per_caption=1,
cfg=7.5,
cfg_img=1.0,
cfg_mult=2,
progress=False,
)
assert tokens.shape == (1, 1, 4)
assert pipeline.model.image_head.calls[0]["cfg_mult"] == 2
assert pipeline.model.forward_batches == [2]
def test_decoding_cfg_parallel_mismatch_falls_back_to_non_parallel(monkeypatch):
pipeline = _make_minimal_pipeline_for_decoding()
monkeypatch.setattr(nextstep_pipeline_module, "get_classifier_free_guidance_world_size", lambda: 2)
def _unexpected(*args, **kwargs):
del args, kwargs
raise AssertionError("CFG rank/group should not be queried on mismatch fallback.")
monkeypatch.setattr(nextstep_pipeline_module, "get_classifier_free_guidance_rank", _unexpected)
monkeypatch.setattr(nextstep_pipeline_module, "get_cfg_group", _unexpected)
c = torch.zeros(3, 1, 8)
attention_mask = torch.ones(3, 3, dtype=torch.long)
tokens = pipeline.decoding(
c=c,
attention_mask=attention_mask,
past_key_values=None,
max_new_len=1,
num_images_per_caption=1,
cfg=7.5,
cfg_img=1.5,
cfg_mult=3,
progress=False,
)
assert tokens.shape == (1, 1, 4)
assert pipeline.model.image_head.calls[0]["cfg_mult"] == 3
assert pipeline.model.forward_batches == [3]
def test_decoding_rejects_incompatible_batch_and_cfg_mult(monkeypatch):
pipeline = _make_minimal_pipeline_for_decoding()
monkeypatch.setattr(nextstep_pipeline_module, "get_classifier_free_guidance_world_size", lambda: 1)
with pytest.raises(ValueError, match="not divisible"):
pipeline.decoding(
c=torch.zeros(5, 1, 8),
attention_mask=torch.ones(5, 3, dtype=torch.long),
past_key_values=None,
max_new_len=1,
num_images_per_caption=1,
cfg=7.5,
cfg_img=1.0,
cfg_mult=2,
progress=False,
)
def test_flow_matching_head_sample_validates_cfg_mult_divisibility():
head = FlowMatchingHead(input_dim=4, cond_dim=8, dim=8, layers=1)
with pytest.raises(ValueError, match="not divisible"):
head.sample(
c=torch.zeros(5, 8),
cfg=2.0,
cfg_img=1.0,
cfg_mult=2,
num_sampling_steps=1,
)
@@ -0,0 +1,161 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from contextlib import contextmanager
from types import SimpleNamespace
import pytest
import torch
import vllm_omni.diffusion.worker.diffusion_model_runner as model_runner_module
from vllm_omni.diffusion.worker.diffusion_model_runner import DiffusionModelRunner
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
@contextmanager
def _noop_forward_context(*args, **kwargs):
del args, kwargs
yield
class _DummyPipeline:
def __init__(self, output):
self._output = output
self.forward_calls = 0
def forward(self, req):
del req
self.forward_calls += 1
return self._output
def _make_request(skip_cache_refresh: bool = True):
sampling_params = SimpleNamespace(
generator=None,
seed=None,
generator_device=None,
num_inference_steps=4,
)
return SimpleNamespace(
prompts=["a prompt"],
sampling_params=sampling_params,
skip_cache_refresh=skip_cache_refresh,
)
def _make_runner(cache_backend, cache_backend_name: str, enable_cache_dit_summary: bool = True):
runner = object.__new__(DiffusionModelRunner)
runner.vllm_config = object()
runner.device = torch.device("cpu")
runner.pipeline = _DummyPipeline(output=SimpleNamespace(output="ok"))
runner.cache_backend = cache_backend
runner.offload_backend = None
runner.od_config = SimpleNamespace(
cache_backend=cache_backend_name,
enable_cache_dit_summary=enable_cache_dit_summary,
)
runner.kv_transfer_manager = SimpleNamespace(receive_kv_cache=lambda req, target_device: None)
return runner
def test_execute_model_skips_cache_summary_without_active_cache_backend(monkeypatch):
"""Guard cache diagnostics with runtime backend state to avoid stale-config crashes."""
runner = _make_runner(cache_backend=None, cache_backend_name="cache_dit")
req = _make_request(skip_cache_refresh=True)
cache_summary_calls = []
monkeypatch.setattr(model_runner_module, "set_forward_context", _noop_forward_context)
monkeypatch.setattr(
model_runner_module,
"cache_summary",
lambda pipeline, details: cache_summary_calls.append((pipeline, details)),
)
output = DiffusionModelRunner.execute_model(runner, req)
assert output.output == "ok"
assert cache_summary_calls == []
def test_execute_model_emits_cache_summary_with_active_cache_dit_backend(monkeypatch):
class _EnabledCacheBackend:
def is_enabled(self):
return True
runner = _make_runner(cache_backend=_EnabledCacheBackend(), cache_backend_name="cache_dit")
req = _make_request(skip_cache_refresh=True)
cache_summary_calls = []
monkeypatch.setattr(model_runner_module, "set_forward_context", _noop_forward_context)
monkeypatch.setattr(
model_runner_module,
"cache_summary",
lambda pipeline, details: cache_summary_calls.append((pipeline, details)),
)
output = DiffusionModelRunner.execute_model(runner, req)
assert output.output == "ok"
assert cache_summary_calls == [(runner.pipeline, True)]
def test_load_model_clears_cache_backend_for_unsupported_pipeline(monkeypatch):
class _DummyLoader:
def __init__(self, load_config):
del load_config
def load_model(self, **kwargs):
del kwargs
return SimpleNamespace(transformer=torch.nn.Identity())
class _DummyMemoryProfiler:
consumed_memory = 0
def __enter__(self):
return self
def __exit__(self, exc_type, exc, tb):
del exc_type, exc, tb
return False
class _DummyCacheBackend:
def __init__(self):
self.enabled = False
def enable(self, pipeline):
del pipeline
self.enabled = True
dummy_cache_backend = _DummyCacheBackend()
runner = object.__new__(DiffusionModelRunner)
runner.vllm_config = object()
runner.device = torch.device("cpu")
runner.pipeline = None
runner.cache_backend = None
runner.offload_backend = None
runner.od_config = SimpleNamespace(
enable_cpu_offload=False,
enable_layerwise_offload=False,
cache_backend="cache_dit",
cache_config={},
model_class_name="NextStep11Pipeline",
enforce_eager=True,
)
monkeypatch.setattr(model_runner_module, "LoadConfig", lambda: object())
monkeypatch.setattr(model_runner_module, "DiffusersPipelineLoader", _DummyLoader)
monkeypatch.setattr(model_runner_module, "DeviceMemoryProfiler", _DummyMemoryProfiler)
monkeypatch.setattr(model_runner_module, "get_offload_backend", lambda od_config, device: None)
monkeypatch.setattr(
model_runner_module, "get_cache_backend", lambda cache_backend, cache_config: dummy_cache_backend
)
DiffusionModelRunner.load_model(runner)
assert runner.cache_backend is None
assert runner.od_config.cache_backend is None
assert dummy_cache_backend.enabled is False
@@ -105,6 +105,98 @@ def test_manager_extraction(kv_config, mock_connector, common_constants):
assert data["layer_blocks"]["key_cache"][0].shape == expected_shape
def test_manager_extraction_tuple_layout(kv_config, mock_connector, common_constants):
"""Test extraction with tuple layout."""
num_layers = common_constants["num_layers"]
block_size = common_constants["block_size"]
num_heads = common_constants["num_heads"]
head_dim = common_constants["head_dim"]
seq_len = common_constants["seq_len"]
req_id = common_constants["req_id"]
num_blocks = 10
kv_caches = []
for _ in range(num_layers):
k_cache = torch.randn(num_blocks, block_size, num_heads, head_dim)
v_cache = torch.randn(num_blocks, block_size, num_heads, head_dim)
kv_caches.append((k_cache, v_cache))
block_ids = [1, 3, 5]
finished_reqs = {req_id: {"block_ids": block_ids, "seq_len": seq_len}}
manager = OmniKVTransferManager(kv_config)
manager._connector = mock_connector
processed = manager.handle_finished_requests_kv_transfer(finished_reqs, kv_caches, block_size, "float32")
assert req_id in processed
full_request_id = f"omni_stage1_to_stage2_kv_cache_{req_id}"
expected_key = f"stage1->stage2:{full_request_id}"
assert expected_key in mock_connector.store
data = mock_connector.store[expected_key]
expected_shape = (seq_len, num_heads, head_dim)
for idx in range(len(kv_caches)):
assert data["layer_blocks"]["key_cache"][idx].shape == expected_shape
assert data["layer_blocks"]["value_cache"][idx].shape == expected_shape
def test_manager_extraction_mismatched_kv_block_counts(kv_config, mock_connector, common_constants):
"""Mismatched key/value block counts should not crash extraction."""
block_size = common_constants["block_size"]
num_heads = common_constants["num_heads"]
head_dim = common_constants["head_dim"]
req_id = common_constants["req_id"]
key_blocks = torch.randn(3, block_size, num_heads, head_dim)
value_blocks = torch.randn(2, block_size, num_heads, head_dim)
kv_caches = [(key_blocks, value_blocks)]
finished_reqs = {req_id: {"block_ids": [0, 1, 2], "seq_len": 32}}
manager = OmniKVTransferManager(kv_config)
manager._connector = mock_connector
processed = manager.handle_finished_requests_kv_transfer(finished_reqs, kv_caches, block_size, "float32")
assert req_id in processed
full_request_id = f"omni_stage1_to_stage2_kv_cache_{req_id}"
expected_key = f"stage1->stage2:{full_request_id}"
assert expected_key in mock_connector.store
data = mock_connector.store[expected_key]
expected_shape = (2 * block_size, num_heads, head_dim)
assert data["layer_blocks"]["key_cache"][0].shape == expected_shape
assert data["layer_blocks"]["value_cache"][0].shape == expected_shape
@pytest.mark.parametrize(
"invalid_case",
["invalid_stacked_shape", "invalid_tuple_length", "non_tensor_entries"],
)
def test_normalize_layer_kv_rejects_invalid_inputs(kv_config, common_constants, invalid_case):
"""_normalize_layer_kv should reject malformed KV representations."""
block_size = common_constants["block_size"]
num_heads = common_constants["num_heads"]
head_dim = common_constants["head_dim"]
req_id = common_constants["req_id"]
if invalid_case == "invalid_stacked_shape":
layer_kv = torch.randn(3, block_size, num_heads, head_dim)
elif invalid_case == "invalid_tuple_length":
layer_kv = (
torch.randn(2, block_size, num_heads, head_dim),
torch.randn(2, block_size, num_heads, head_dim),
torch.randn(2, block_size, num_heads, head_dim),
)
else:
layer_kv = (torch.randn(2, block_size, num_heads, head_dim), "not-a-tensor")
manager = OmniKVTransferManager(kv_config)
normalized = manager._normalize_layer_kv(layer_kv, req_id=req_id, layer_idx=0)
assert normalized is None
def test_manager_reception(kv_config, mock_connector, common_constants):
"""Test reception and injection logic in OmniKVTransferManager."""
num_layers = common_constants["num_layers"]
+1 -1
View File
@@ -114,7 +114,7 @@ def test_mix_to_text_audio_001(omni_server, openai_client) -> None:
"messages": messages,
"stream": True,
"key_words": {
"audio": ["water", "chirping", "crackling"],
"audio": ["water", "chirping", "crackling", "rain"],
"image": ["square", "quadrate"],
},
}
@@ -22,7 +22,7 @@ from tests.utils import hardware_test
models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"]
AUDIO_KEY = ["water", "chirping", "crackling"]
AUDIO_KEY = ["water", "chirping", "crackling", "rain"]
IMAGE_KEY = ["square", "quadrate"]
VIDEO_KEY = ["sphere", "globe", "circle", "round", "ball"]
@@ -267,6 +267,7 @@ def test_text_image_to_text_audio_001(omni_server, openai_client) -> None:
openai_client.send_request(request_config)
@pytest.mark.skip(reason="There is a known issue with oom error.")
@pytest.mark.advanced_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
@@ -287,6 +288,7 @@ def test_text_video_to_text_audio_001(omni_server, openai_client) -> None:
openai_client.send_request(request_config, request_num=get_max_batch_size())
@pytest.mark.skip(reason="There is a known issue with shape mismatch error.")
@pytest.mark.advanced_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
@@ -291,6 +291,7 @@ class TestTTSMethods:
mock_engine_client = MagicMock()
mock_engine_client.errored = False
mock_engine_client.stage_list = None
mock_engine_client.tts_max_instructions_length = None
mock_models = MagicMock()
mock_models.is_base_model.return_value = True
return OmniOpenAIServingSpeech(
@@ -299,16 +300,35 @@ class TestTTSMethods:
request_logger=MagicMock(),
)
def test_is_tts_model(self, speech_server):
"""Test TTS model detection."""
# No stage_list -> False
assert speech_server._is_tts_model() is False
def test_is_tts_detection_no_stage(self, speech_server):
"""Test TTS model detection when no TTS stage exists."""
# Fixture creates server with stage_list = None -> _is_tts should be False
assert speech_server._is_tts is False
assert speech_server._tts_stage is None
# With qwen3_tts stage -> True
def test_is_tts_detection_with_tts_stage(self):
"""Test TTS model detection when TTS stage exists."""
mock_engine_client = MagicMock()
mock_engine_client.errored = False
mock_engine_client.tts_max_instructions_length = None
# Create a TTS stage
mock_stage = MagicMock()
mock_stage.model_stage = "qwen3_tts"
speech_server.engine_client.stage_list = [mock_stage]
assert speech_server._is_tts_model() is True
mock_stage.tts_args = {}
mock_engine_client.stage_list = [mock_stage]
mock_models = MagicMock()
mock_models.is_base_model.return_value = True
server = OmniOpenAIServingSpeech(
engine_client=mock_engine_client,
models=mock_models,
request_logger=MagicMock(),
)
assert server._is_tts is True
assert server._tts_stage is mock_stage
def test_estimate_prompt_len_fallback(self, speech_server):
"""Test prompt length estimation falls back to 2048 when model is unavailable."""
@@ -381,3 +401,108 @@ class TestTTSMethods:
# Verify speakers are normalized to lowercase
assert server.supported_speakers == {"ryan", "vivian", "aiden"}
def test_max_instructions_length_default(self, speech_server):
"""Test default max instructions length (500) when no config provided."""
# Fixture creates server with no CLI override and no TTS stage
assert speech_server._max_instructions_length == 500
def test_max_instructions_length_cli_override(self):
"""Test CLI override (stored in engine_client) takes highest priority."""
mock_engine_client = MagicMock()
mock_engine_client.errored = False
mock_engine_client.stage_list = None
# CLI override is stored in engine_client
mock_engine_client.tts_max_instructions_length = 1000
mock_models = MagicMock()
mock_models.is_base_model.return_value = True
server = OmniOpenAIServingSpeech(
engine_client=mock_engine_client,
models=mock_models,
request_logger=MagicMock(),
)
# Value is cached during __init__
assert server._max_instructions_length == 1000
def test_max_instructions_length_stage_config(self):
"""Test stage config value is used when no CLI override."""
mock_engine_client = MagicMock()
mock_engine_client.errored = False
mock_engine_client.tts_max_instructions_length = None # No CLI override
mock_models = MagicMock()
mock_models.is_base_model.return_value = True
# Mock stage with tts_args
mock_stage = MagicMock()
mock_stage.model_stage = "qwen3_tts"
mock_stage.tts_args = {"max_instructions_length": 750}
mock_engine_client.stage_list = [mock_stage]
server = OmniOpenAIServingSpeech(
engine_client=mock_engine_client,
models=mock_models,
request_logger=MagicMock(),
)
# Value is cached during __init__
assert server._max_instructions_length == 750
def test_max_instructions_length_cli_overrides_stage_config(self):
"""Test CLI override (in engine_client) takes precedence over stage config."""
mock_engine_client = MagicMock()
mock_engine_client.errored = False
# CLI override stored in engine_client
mock_engine_client.tts_max_instructions_length = 2000
mock_models = MagicMock()
mock_models.is_base_model.return_value = True
# Mock stage with tts_args
mock_stage = MagicMock()
mock_stage.model_stage = "qwen3_tts"
mock_stage.tts_args = {"max_instructions_length": 750}
mock_engine_client.stage_list = [mock_stage]
server = OmniOpenAIServingSpeech(
engine_client=mock_engine_client,
models=mock_models,
request_logger=MagicMock(),
)
# CLI value (2000) should override stage config (750)
assert server._max_instructions_length == 2000
def test_validate_instructions_length_uses_cached_value(self):
"""Test instructions length validation uses cached _max_instructions_length."""
mock_engine_client = MagicMock()
mock_engine_client.errored = False
mock_engine_client.stage_list = None
# CLI override with max length of 10 characters
mock_engine_client.tts_max_instructions_length = 10
mock_models = MagicMock()
mock_models.is_base_model.return_value = True
server = OmniOpenAIServingSpeech(
engine_client=mock_engine_client,
models=mock_models,
request_logger=MagicMock(),
)
# Verify cached value
assert server._max_instructions_length == 10
# Instructions within limit should pass
req = OpenAICreateSpeechRequest(
input="Hello",
task_type="VoiceDesign",
instructions="short",
)
assert server._validate_tts_request(req) is None
# Instructions exceeding limit should fail
req = OpenAICreateSpeechRequest(
input="Hello",
task_type="VoiceDesign",
instructions="this is too long",
)
error = server._validate_tts_request(req)
assert error is not None
assert "max 10 characters" in error
-114
View File
@@ -6,28 +6,6 @@
"stage_config_name": "qwen3_omni.yaml"
},
"benchmark_params": [
{
"dataset_name": "random",
"num_prompts": [
10,
40,
100
],
"request_rate": [
0.1,
0.5,
1
],
"random_input_len": 2500,
"random_output_len": 900,
"ignore_eos": true,
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": 100000,
"mean_audio_ttfp_ms": 100000,
"mean_audio_rtf": 100000
}
},
{
"dataset_name": "random",
"num_prompts": [
@@ -84,41 +62,6 @@
"mean_audio_ttfp_ms": 100000,
"mean_audio_rtf": 100000
}
},
{
"dataset_name": "random-mm",
"num_prompts": [
10,
40,
100
],
"request_rate": [
0.1,
0.5,
1
],
"random_input_len": 100,
"random_output_len": 100,
"random_range_ratio": 0.0,
"ignore_eos": true,
"random_mm_base_items_per_request": 3,
"random_mm_num_mm_items_range_ratio": 0,
"random_mm_limit_mm_per_prompt": {
"image": 1,
"video": 1,
"audio": 1
},
"random_mm_bucket_config": {
"(32, 32, 1)": 0.5,
"(0, 1, 1)": 0.1,
"(32, 32, 2)": 0.4
},
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": 100000,
"mean_audio_ttfp_ms": 100000,
"mean_audio_rtf": 100000
}
}
]
},
@@ -147,28 +90,6 @@
}
},
"benchmark_params": [
{
"dataset_name": "random",
"num_prompts": [
10,
40,
100
],
"request_rate": [
0.1,
0.5,
1
],
"random_input_len": 2500,
"random_output_len": 900,
"ignore_eos": true,
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": 100000,
"mean_audio_ttfp_ms": 100000,
"mean_audio_rtf": 100000
}
},
{
"dataset_name": "random",
"num_prompts": [
@@ -225,41 +146,6 @@
"mean_audio_ttfp_ms": 100000,
"mean_audio_rtf": 100000
}
},
{
"dataset_name": "random-mm",
"num_prompts": [
10,
40,
100
],
"request_rate": [
0.1,
0.5,
1
],
"random_input_len": 100,
"random_output_len": 100,
"random_range_ratio": 0.0,
"ignore_eos": true,
"random_mm_base_items_per_request": 3,
"random_mm_num_mm_items_range_ratio": 0,
"random_mm_limit_mm_per_prompt": {
"image": 1,
"video": 1,
"audio": 1
},
"random_mm_bucket_config": {
"(32, 32, 1)": 0.5,
"(0, 1, 1)": 0.1,
"(32, 32, 2)": 0.4
},
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": 100000,
"mean_audio_ttfp_ms": 100000,
"mean_audio_rtf": 100000
}
}
]
}