Merge origin/main into dev/vllm-align
Resolve conflicts: - docker/Dockerfile.ci: Keep wheel install section, update hash to 6042e66cd5 - vllm_omni/platforms/npu/worker/npu_model_runner.py: Accept main's refactoring Made-with: Cursor Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
commit
8d0c46487e
58 files changed
+4358
-1530
No files matched your search
+33
-32
@@ -848,41 +848,42 @@ def modify_stage_config(
|
||||
del config[key]
|
||||
|
||||
# Apply updates
|
||||
for key, value in updates.items():
|
||||
if key == "stage_args":
|
||||
if value and isinstance(value, dict):
|
||||
stage_args = config.get("stage_args", [])
|
||||
if not stage_args:
|
||||
raise ValueError("stage_args does not exist in config")
|
||||
if updates:
|
||||
for key, value in updates.items():
|
||||
if key == "stage_args":
|
||||
if value and isinstance(value, dict):
|
||||
stage_args = config.get("stage_args", [])
|
||||
if not stage_args:
|
||||
raise ValueError("stage_args does not exist in config")
|
||||
|
||||
for stage_id, stage_updates in value.items():
|
||||
# Find stage by ID
|
||||
target_stage = None
|
||||
for stage in stage_args:
|
||||
if stage.get("stage_id") == int(stage_id):
|
||||
target_stage = stage
|
||||
break
|
||||
for stage_id, stage_updates in value.items():
|
||||
# Find stage by ID
|
||||
target_stage = None
|
||||
for stage in stage_args:
|
||||
if stage.get("stage_id") == int(stage_id):
|
||||
target_stage = stage
|
||||
break
|
||||
|
||||
if target_stage is None:
|
||||
available_ids = [s.get("stage_id") for s in stage_args if "stage_id" in s]
|
||||
raise KeyError(f"Stage ID {stage_id} not found, available: {available_ids}")
|
||||
if target_stage is None:
|
||||
available_ids = [s.get("stage_id") for s in stage_args if "stage_id" in s]
|
||||
raise KeyError(f"Stage ID {stage_id} not found, available: {available_ids}")
|
||||
|
||||
# Apply updates to this stage
|
||||
for path, val in stage_updates.items():
|
||||
# Check if this is a simple key (not dot-separated)
|
||||
# Example: 'engine_input_source' vs 'engine_args.max_model_len'
|
||||
if "." not in path:
|
||||
# Direct key assignment (e.g., updating a list value)
|
||||
target_stage[path] = val
|
||||
else:
|
||||
# Dot-separated path (e.g., nested dict access)
|
||||
apply_update(target_stage, path, val)
|
||||
elif "." in key:
|
||||
# Apply using dot-separated path
|
||||
apply_update(config, key, value)
|
||||
else:
|
||||
# Direct top-level key
|
||||
config[key] = value
|
||||
# Apply updates to this stage
|
||||
for path, val in stage_updates.items():
|
||||
# Check if this is a simple key (not dot-separated)
|
||||
# Example: 'engine_input_source' vs 'engine_args.max_model_len'
|
||||
if "." not in path:
|
||||
# Direct key assignment (e.g., updating a list value)
|
||||
target_stage[path] = val
|
||||
else:
|
||||
# Dot-separated path (e.g., nested dict access)
|
||||
apply_update(target_stage, path, val)
|
||||
elif "." in key:
|
||||
# Apply using dot-separated path
|
||||
apply_update(config, key, value)
|
||||
else:
|
||||
# Direct top-level key
|
||||
config[key] = value
|
||||
|
||||
# Save to new file with timestamp
|
||||
timestamp = int(time.time())
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
import vllm_omni.diffusion.models.nextstep_1_1.pipeline_nextstep_1_1 as nextstep_pipeline_module
|
||||
from vllm_omni.diffusion.models.nextstep_1_1.modeling_nextstep_heads import FlowMatchingHead
|
||||
from vllm_omni.diffusion.models.nextstep_1_1.pipeline_nextstep_1_1 import NextStep11Pipeline
|
||||
|
||||
|
||||
class _DummyImageHead:
|
||||
def __init__(self, token_dim: int):
|
||||
self.token_dim = token_dim
|
||||
self.calls = []
|
||||
|
||||
def sample(
|
||||
self,
|
||||
c: torch.Tensor,
|
||||
cfg: float,
|
||||
cfg_img: float,
|
||||
cfg_mult: int,
|
||||
timesteps_shift: float,
|
||||
num_sampling_steps: int,
|
||||
noise_repeat: int,
|
||||
) -> torch.Tensor:
|
||||
self.calls.append(
|
||||
{
|
||||
"batch": c.shape[0],
|
||||
"cfg": cfg,
|
||||
"cfg_img": cfg_img,
|
||||
"cfg_mult": cfg_mult,
|
||||
"noise_repeat": noise_repeat,
|
||||
}
|
||||
)
|
||||
batch_per_prompt = c.shape[0] // cfg_mult
|
||||
return torch.ones(batch_per_prompt, self.token_dim, dtype=c.dtype, device=c.device)
|
||||
|
||||
|
||||
class _DummyModel:
|
||||
def __init__(self, hidden_dim: int, token_dim: int):
|
||||
self.hidden_dim = hidden_dim
|
||||
self.image_head = _DummyImageHead(token_dim)
|
||||
self.forward_batches = []
|
||||
|
||||
def image_out_projector(self, c: torch.Tensor) -> torch.Tensor:
|
||||
return c
|
||||
|
||||
def image_in_projector(self, sampled_tokens: torch.Tensor) -> torch.Tensor:
|
||||
bsz = sampled_tokens.shape[0]
|
||||
return torch.zeros(bsz, 1, self.hidden_dim, dtype=sampled_tokens.dtype, device=sampled_tokens.device)
|
||||
|
||||
def forward_model(self, inputs_embeds: torch.Tensor, attention_mask, past_key_values, use_cache: bool):
|
||||
del attention_mask, use_cache
|
||||
self.forward_batches.append(inputs_embeds.shape[0])
|
||||
return SimpleNamespace(
|
||||
last_hidden_state=torch.zeros(
|
||||
inputs_embeds.shape[0],
|
||||
1,
|
||||
self.hidden_dim,
|
||||
dtype=inputs_embeds.dtype,
|
||||
device=inputs_embeds.device,
|
||||
),
|
||||
past_key_values=past_key_values,
|
||||
)
|
||||
|
||||
|
||||
def _make_minimal_pipeline_for_decoding(hidden_dim: int = 8, token_dim: int = 4):
|
||||
pipeline = object.__new__(NextStep11Pipeline)
|
||||
pipeline.config = SimpleNamespace(latent_channels=token_dim, latent_patch_size=1, use_gen_pos_embed=False)
|
||||
pipeline.model = _DummyModel(hidden_dim=hidden_dim, token_dim=token_dim)
|
||||
return pipeline
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("cfg", "cfg_img", "has_image_conditions", "expected_cfg_mult", "expected_cfg_img"),
|
||||
[
|
||||
(1.0, 1.0, False, 1, 1.0),
|
||||
(7.5, 1.0, False, 2, 1.0),
|
||||
(7.5, 8.0, False, 2, 1.0),
|
||||
(7.5, 1.5, True, 3, 1.5),
|
||||
],
|
||||
)
|
||||
def test_resolve_cfg_layout(cfg, cfg_img, has_image_conditions, expected_cfg_mult, expected_cfg_img):
|
||||
cfg_mult, effective_cfg_img = NextStep11Pipeline._resolve_cfg_layout(cfg, cfg_img, has_image_conditions)
|
||||
assert cfg_mult == expected_cfg_mult
|
||||
assert effective_cfg_img == expected_cfg_img
|
||||
|
||||
|
||||
def test_build_captions_ignores_image_cfg_without_image_conditions():
|
||||
pipeline = object.__new__(NextStep11Pipeline)
|
||||
pipeline._image_str = lambda hw: f"<image:{hw}>"
|
||||
|
||||
captions, images, cfg_mult, effective_cfg_img = pipeline._build_captions(
|
||||
captions=["a prompt"],
|
||||
images=None,
|
||||
num_images_per_caption=1,
|
||||
positive_prompt=None,
|
||||
negative_prompt="bad quality",
|
||||
cfg=7.5,
|
||||
cfg_img=8.0,
|
||||
)
|
||||
|
||||
assert cfg_mult == 2
|
||||
assert effective_cfg_img == 1.0
|
||||
assert images is None
|
||||
assert captions == ["a prompt", "bad quality"]
|
||||
|
||||
|
||||
def test_build_captions_enables_three_way_cfg_when_image_conditions_exist():
|
||||
pipeline = object.__new__(NextStep11Pipeline)
|
||||
pipeline._image_str = lambda hw: f"<image:{hw}>"
|
||||
|
||||
image = Image.new("RGB", (64, 32))
|
||||
captions, images, cfg_mult, effective_cfg_img = pipeline._build_captions(
|
||||
captions=["a prompt"],
|
||||
images=[image],
|
||||
num_images_per_caption=1,
|
||||
positive_prompt=None,
|
||||
negative_prompt="bad quality",
|
||||
cfg=7.5,
|
||||
cfg_img=1.5,
|
||||
)
|
||||
|
||||
assert cfg_mult == 3
|
||||
assert effective_cfg_img == 1.5
|
||||
assert len(captions) == 3
|
||||
assert captions[1].startswith("<image:")
|
||||
assert captions[2] == "bad quality"
|
||||
assert len(images) == 2
|
||||
|
||||
|
||||
def test_decoding_non_parallel_uses_cfg_mult_for_sampling_and_duplication(monkeypatch):
|
||||
pipeline = _make_minimal_pipeline_for_decoding()
|
||||
monkeypatch.setattr(nextstep_pipeline_module, "get_classifier_free_guidance_world_size", lambda: 1)
|
||||
|
||||
c = torch.zeros(2, 1, 8)
|
||||
attention_mask = torch.ones(2, 3, dtype=torch.long)
|
||||
|
||||
tokens = pipeline.decoding(
|
||||
c=c,
|
||||
attention_mask=attention_mask,
|
||||
past_key_values=None,
|
||||
max_new_len=1,
|
||||
num_images_per_caption=1,
|
||||
cfg=7.5,
|
||||
cfg_img=1.0,
|
||||
cfg_mult=2,
|
||||
progress=False,
|
||||
)
|
||||
|
||||
assert tokens.shape == (1, 1, 4)
|
||||
assert pipeline.model.image_head.calls[0]["cfg_mult"] == 2
|
||||
assert pipeline.model.forward_batches == [2]
|
||||
|
||||
|
||||
def test_decoding_cfg_parallel_mismatch_falls_back_to_non_parallel(monkeypatch):
|
||||
pipeline = _make_minimal_pipeline_for_decoding()
|
||||
|
||||
monkeypatch.setattr(nextstep_pipeline_module, "get_classifier_free_guidance_world_size", lambda: 2)
|
||||
|
||||
def _unexpected(*args, **kwargs):
|
||||
del args, kwargs
|
||||
raise AssertionError("CFG rank/group should not be queried on mismatch fallback.")
|
||||
|
||||
monkeypatch.setattr(nextstep_pipeline_module, "get_classifier_free_guidance_rank", _unexpected)
|
||||
monkeypatch.setattr(nextstep_pipeline_module, "get_cfg_group", _unexpected)
|
||||
|
||||
c = torch.zeros(3, 1, 8)
|
||||
attention_mask = torch.ones(3, 3, dtype=torch.long)
|
||||
|
||||
tokens = pipeline.decoding(
|
||||
c=c,
|
||||
attention_mask=attention_mask,
|
||||
past_key_values=None,
|
||||
max_new_len=1,
|
||||
num_images_per_caption=1,
|
||||
cfg=7.5,
|
||||
cfg_img=1.5,
|
||||
cfg_mult=3,
|
||||
progress=False,
|
||||
)
|
||||
|
||||
assert tokens.shape == (1, 1, 4)
|
||||
assert pipeline.model.image_head.calls[0]["cfg_mult"] == 3
|
||||
assert pipeline.model.forward_batches == [3]
|
||||
|
||||
|
||||
def test_decoding_rejects_incompatible_batch_and_cfg_mult(monkeypatch):
|
||||
pipeline = _make_minimal_pipeline_for_decoding()
|
||||
monkeypatch.setattr(nextstep_pipeline_module, "get_classifier_free_guidance_world_size", lambda: 1)
|
||||
|
||||
with pytest.raises(ValueError, match="not divisible"):
|
||||
pipeline.decoding(
|
||||
c=torch.zeros(5, 1, 8),
|
||||
attention_mask=torch.ones(5, 3, dtype=torch.long),
|
||||
past_key_values=None,
|
||||
max_new_len=1,
|
||||
num_images_per_caption=1,
|
||||
cfg=7.5,
|
||||
cfg_img=1.0,
|
||||
cfg_mult=2,
|
||||
progress=False,
|
||||
)
|
||||
|
||||
|
||||
def test_flow_matching_head_sample_validates_cfg_mult_divisibility():
|
||||
head = FlowMatchingHead(input_dim=4, cond_dim=8, dim=8, layers=1)
|
||||
|
||||
with pytest.raises(ValueError, match="not divisible"):
|
||||
head.sample(
|
||||
c=torch.zeros(5, 8),
|
||||
cfg=2.0,
|
||||
cfg_img=1.0,
|
||||
cfg_mult=2,
|
||||
num_sampling_steps=1,
|
||||
)
|
||||
@@ -0,0 +1,161 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from contextlib import contextmanager
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm_omni.diffusion.worker.diffusion_model_runner as model_runner_module
|
||||
from vllm_omni.diffusion.worker.diffusion_model_runner import DiffusionModelRunner
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _noop_forward_context(*args, **kwargs):
|
||||
del args, kwargs
|
||||
yield
|
||||
|
||||
|
||||
class _DummyPipeline:
|
||||
def __init__(self, output):
|
||||
self._output = output
|
||||
self.forward_calls = 0
|
||||
|
||||
def forward(self, req):
|
||||
del req
|
||||
self.forward_calls += 1
|
||||
return self._output
|
||||
|
||||
|
||||
def _make_request(skip_cache_refresh: bool = True):
|
||||
sampling_params = SimpleNamespace(
|
||||
generator=None,
|
||||
seed=None,
|
||||
generator_device=None,
|
||||
num_inference_steps=4,
|
||||
)
|
||||
return SimpleNamespace(
|
||||
prompts=["a prompt"],
|
||||
sampling_params=sampling_params,
|
||||
skip_cache_refresh=skip_cache_refresh,
|
||||
)
|
||||
|
||||
|
||||
def _make_runner(cache_backend, cache_backend_name: str, enable_cache_dit_summary: bool = True):
|
||||
runner = object.__new__(DiffusionModelRunner)
|
||||
runner.vllm_config = object()
|
||||
runner.device = torch.device("cpu")
|
||||
runner.pipeline = _DummyPipeline(output=SimpleNamespace(output="ok"))
|
||||
runner.cache_backend = cache_backend
|
||||
runner.offload_backend = None
|
||||
runner.od_config = SimpleNamespace(
|
||||
cache_backend=cache_backend_name,
|
||||
enable_cache_dit_summary=enable_cache_dit_summary,
|
||||
)
|
||||
runner.kv_transfer_manager = SimpleNamespace(receive_kv_cache=lambda req, target_device: None)
|
||||
return runner
|
||||
|
||||
|
||||
def test_execute_model_skips_cache_summary_without_active_cache_backend(monkeypatch):
|
||||
"""Guard cache diagnostics with runtime backend state to avoid stale-config crashes."""
|
||||
runner = _make_runner(cache_backend=None, cache_backend_name="cache_dit")
|
||||
req = _make_request(skip_cache_refresh=True)
|
||||
|
||||
cache_summary_calls = []
|
||||
|
||||
monkeypatch.setattr(model_runner_module, "set_forward_context", _noop_forward_context)
|
||||
monkeypatch.setattr(
|
||||
model_runner_module,
|
||||
"cache_summary",
|
||||
lambda pipeline, details: cache_summary_calls.append((pipeline, details)),
|
||||
)
|
||||
|
||||
output = DiffusionModelRunner.execute_model(runner, req)
|
||||
|
||||
assert output.output == "ok"
|
||||
assert cache_summary_calls == []
|
||||
|
||||
|
||||
def test_execute_model_emits_cache_summary_with_active_cache_dit_backend(monkeypatch):
|
||||
class _EnabledCacheBackend:
|
||||
def is_enabled(self):
|
||||
return True
|
||||
|
||||
runner = _make_runner(cache_backend=_EnabledCacheBackend(), cache_backend_name="cache_dit")
|
||||
req = _make_request(skip_cache_refresh=True)
|
||||
|
||||
cache_summary_calls = []
|
||||
|
||||
monkeypatch.setattr(model_runner_module, "set_forward_context", _noop_forward_context)
|
||||
monkeypatch.setattr(
|
||||
model_runner_module,
|
||||
"cache_summary",
|
||||
lambda pipeline, details: cache_summary_calls.append((pipeline, details)),
|
||||
)
|
||||
|
||||
output = DiffusionModelRunner.execute_model(runner, req)
|
||||
|
||||
assert output.output == "ok"
|
||||
assert cache_summary_calls == [(runner.pipeline, True)]
|
||||
|
||||
|
||||
def test_load_model_clears_cache_backend_for_unsupported_pipeline(monkeypatch):
|
||||
class _DummyLoader:
|
||||
def __init__(self, load_config):
|
||||
del load_config
|
||||
|
||||
def load_model(self, **kwargs):
|
||||
del kwargs
|
||||
return SimpleNamespace(transformer=torch.nn.Identity())
|
||||
|
||||
class _DummyMemoryProfiler:
|
||||
consumed_memory = 0
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
del exc_type, exc, tb
|
||||
return False
|
||||
|
||||
class _DummyCacheBackend:
|
||||
def __init__(self):
|
||||
self.enabled = False
|
||||
|
||||
def enable(self, pipeline):
|
||||
del pipeline
|
||||
self.enabled = True
|
||||
|
||||
dummy_cache_backend = _DummyCacheBackend()
|
||||
|
||||
runner = object.__new__(DiffusionModelRunner)
|
||||
runner.vllm_config = object()
|
||||
runner.device = torch.device("cpu")
|
||||
runner.pipeline = None
|
||||
runner.cache_backend = None
|
||||
runner.offload_backend = None
|
||||
runner.od_config = SimpleNamespace(
|
||||
enable_cpu_offload=False,
|
||||
enable_layerwise_offload=False,
|
||||
cache_backend="cache_dit",
|
||||
cache_config={},
|
||||
model_class_name="NextStep11Pipeline",
|
||||
enforce_eager=True,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(model_runner_module, "LoadConfig", lambda: object())
|
||||
monkeypatch.setattr(model_runner_module, "DiffusersPipelineLoader", _DummyLoader)
|
||||
monkeypatch.setattr(model_runner_module, "DeviceMemoryProfiler", _DummyMemoryProfiler)
|
||||
monkeypatch.setattr(model_runner_module, "get_offload_backend", lambda od_config, device: None)
|
||||
monkeypatch.setattr(
|
||||
model_runner_module, "get_cache_backend", lambda cache_backend, cache_config: dummy_cache_backend
|
||||
)
|
||||
|
||||
DiffusionModelRunner.load_model(runner)
|
||||
|
||||
assert runner.cache_backend is None
|
||||
assert runner.od_config.cache_backend is None
|
||||
assert dummy_cache_backend.enabled is False
|
||||
@@ -105,6 +105,98 @@ def test_manager_extraction(kv_config, mock_connector, common_constants):
|
||||
assert data["layer_blocks"]["key_cache"][0].shape == expected_shape
|
||||
|
||||
|
||||
def test_manager_extraction_tuple_layout(kv_config, mock_connector, common_constants):
|
||||
"""Test extraction with tuple layout."""
|
||||
num_layers = common_constants["num_layers"]
|
||||
block_size = common_constants["block_size"]
|
||||
num_heads = common_constants["num_heads"]
|
||||
head_dim = common_constants["head_dim"]
|
||||
seq_len = common_constants["seq_len"]
|
||||
req_id = common_constants["req_id"]
|
||||
|
||||
num_blocks = 10
|
||||
kv_caches = []
|
||||
for _ in range(num_layers):
|
||||
k_cache = torch.randn(num_blocks, block_size, num_heads, head_dim)
|
||||
v_cache = torch.randn(num_blocks, block_size, num_heads, head_dim)
|
||||
kv_caches.append((k_cache, v_cache))
|
||||
|
||||
block_ids = [1, 3, 5]
|
||||
finished_reqs = {req_id: {"block_ids": block_ids, "seq_len": seq_len}}
|
||||
|
||||
manager = OmniKVTransferManager(kv_config)
|
||||
manager._connector = mock_connector
|
||||
|
||||
processed = manager.handle_finished_requests_kv_transfer(finished_reqs, kv_caches, block_size, "float32")
|
||||
assert req_id in processed
|
||||
|
||||
full_request_id = f"omni_stage1_to_stage2_kv_cache_{req_id}"
|
||||
expected_key = f"stage1->stage2:{full_request_id}"
|
||||
assert expected_key in mock_connector.store
|
||||
|
||||
data = mock_connector.store[expected_key]
|
||||
expected_shape = (seq_len, num_heads, head_dim)
|
||||
for idx in range(len(kv_caches)):
|
||||
assert data["layer_blocks"]["key_cache"][idx].shape == expected_shape
|
||||
assert data["layer_blocks"]["value_cache"][idx].shape == expected_shape
|
||||
|
||||
|
||||
def test_manager_extraction_mismatched_kv_block_counts(kv_config, mock_connector, common_constants):
|
||||
"""Mismatched key/value block counts should not crash extraction."""
|
||||
block_size = common_constants["block_size"]
|
||||
num_heads = common_constants["num_heads"]
|
||||
head_dim = common_constants["head_dim"]
|
||||
req_id = common_constants["req_id"]
|
||||
|
||||
key_blocks = torch.randn(3, block_size, num_heads, head_dim)
|
||||
value_blocks = torch.randn(2, block_size, num_heads, head_dim)
|
||||
kv_caches = [(key_blocks, value_blocks)]
|
||||
|
||||
finished_reqs = {req_id: {"block_ids": [0, 1, 2], "seq_len": 32}}
|
||||
|
||||
manager = OmniKVTransferManager(kv_config)
|
||||
manager._connector = mock_connector
|
||||
|
||||
processed = manager.handle_finished_requests_kv_transfer(finished_reqs, kv_caches, block_size, "float32")
|
||||
assert req_id in processed
|
||||
|
||||
full_request_id = f"omni_stage1_to_stage2_kv_cache_{req_id}"
|
||||
expected_key = f"stage1->stage2:{full_request_id}"
|
||||
assert expected_key in mock_connector.store
|
||||
|
||||
data = mock_connector.store[expected_key]
|
||||
expected_shape = (2 * block_size, num_heads, head_dim)
|
||||
assert data["layer_blocks"]["key_cache"][0].shape == expected_shape
|
||||
assert data["layer_blocks"]["value_cache"][0].shape == expected_shape
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"invalid_case",
|
||||
["invalid_stacked_shape", "invalid_tuple_length", "non_tensor_entries"],
|
||||
)
|
||||
def test_normalize_layer_kv_rejects_invalid_inputs(kv_config, common_constants, invalid_case):
|
||||
"""_normalize_layer_kv should reject malformed KV representations."""
|
||||
block_size = common_constants["block_size"]
|
||||
num_heads = common_constants["num_heads"]
|
||||
head_dim = common_constants["head_dim"]
|
||||
req_id = common_constants["req_id"]
|
||||
|
||||
if invalid_case == "invalid_stacked_shape":
|
||||
layer_kv = torch.randn(3, block_size, num_heads, head_dim)
|
||||
elif invalid_case == "invalid_tuple_length":
|
||||
layer_kv = (
|
||||
torch.randn(2, block_size, num_heads, head_dim),
|
||||
torch.randn(2, block_size, num_heads, head_dim),
|
||||
torch.randn(2, block_size, num_heads, head_dim),
|
||||
)
|
||||
else:
|
||||
layer_kv = (torch.randn(2, block_size, num_heads, head_dim), "not-a-tensor")
|
||||
|
||||
manager = OmniKVTransferManager(kv_config)
|
||||
normalized = manager._normalize_layer_kv(layer_kv, req_id=req_id, layer_idx=0)
|
||||
assert normalized is None
|
||||
|
||||
|
||||
def test_manager_reception(kv_config, mock_connector, common_constants):
|
||||
"""Test reception and injection logic in OmniKVTransferManager."""
|
||||
num_layers = common_constants["num_layers"]
|
||||
|
||||
@@ -114,7 +114,7 @@ def test_mix_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
"messages": messages,
|
||||
"stream": True,
|
||||
"key_words": {
|
||||
"audio": ["water", "chirping", "crackling"],
|
||||
"audio": ["water", "chirping", "crackling", "rain"],
|
||||
"image": ["square", "quadrate"],
|
||||
},
|
||||
}
|
||||
|
||||
@@ -22,7 +22,7 @@ from tests.utils import hardware_test
|
||||
|
||||
models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"]
|
||||
|
||||
AUDIO_KEY = ["water", "chirping", "crackling"]
|
||||
AUDIO_KEY = ["water", "chirping", "crackling", "rain"]
|
||||
IMAGE_KEY = ["square", "quadrate"]
|
||||
VIDEO_KEY = ["sphere", "globe", "circle", "round", "ball"]
|
||||
|
||||
@@ -267,6 +267,7 @@ def test_text_image_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
openai_client.send_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="There is a known issue with oom error.")
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||
@@ -287,6 +288,7 @@ def test_text_video_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
openai_client.send_request(request_config, request_num=get_max_batch_size())
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="There is a known issue with shape mismatch error.")
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||
|
||||
@@ -291,6 +291,7 @@ class TestTTSMethods:
|
||||
mock_engine_client = MagicMock()
|
||||
mock_engine_client.errored = False
|
||||
mock_engine_client.stage_list = None
|
||||
mock_engine_client.tts_max_instructions_length = None
|
||||
mock_models = MagicMock()
|
||||
mock_models.is_base_model.return_value = True
|
||||
return OmniOpenAIServingSpeech(
|
||||
@@ -299,16 +300,35 @@ class TestTTSMethods:
|
||||
request_logger=MagicMock(),
|
||||
)
|
||||
|
||||
def test_is_tts_model(self, speech_server):
|
||||
"""Test TTS model detection."""
|
||||
# No stage_list -> False
|
||||
assert speech_server._is_tts_model() is False
|
||||
def test_is_tts_detection_no_stage(self, speech_server):
|
||||
"""Test TTS model detection when no TTS stage exists."""
|
||||
# Fixture creates server with stage_list = None -> _is_tts should be False
|
||||
assert speech_server._is_tts is False
|
||||
assert speech_server._tts_stage is None
|
||||
|
||||
# With qwen3_tts stage -> True
|
||||
def test_is_tts_detection_with_tts_stage(self):
|
||||
"""Test TTS model detection when TTS stage exists."""
|
||||
mock_engine_client = MagicMock()
|
||||
mock_engine_client.errored = False
|
||||
mock_engine_client.tts_max_instructions_length = None
|
||||
|
||||
# Create a TTS stage
|
||||
mock_stage = MagicMock()
|
||||
mock_stage.model_stage = "qwen3_tts"
|
||||
speech_server.engine_client.stage_list = [mock_stage]
|
||||
assert speech_server._is_tts_model() is True
|
||||
mock_stage.tts_args = {}
|
||||
mock_engine_client.stage_list = [mock_stage]
|
||||
|
||||
mock_models = MagicMock()
|
||||
mock_models.is_base_model.return_value = True
|
||||
|
||||
server = OmniOpenAIServingSpeech(
|
||||
engine_client=mock_engine_client,
|
||||
models=mock_models,
|
||||
request_logger=MagicMock(),
|
||||
)
|
||||
|
||||
assert server._is_tts is True
|
||||
assert server._tts_stage is mock_stage
|
||||
|
||||
def test_estimate_prompt_len_fallback(self, speech_server):
|
||||
"""Test prompt length estimation falls back to 2048 when model is unavailable."""
|
||||
@@ -381,3 +401,108 @@ class TestTTSMethods:
|
||||
|
||||
# Verify speakers are normalized to lowercase
|
||||
assert server.supported_speakers == {"ryan", "vivian", "aiden"}
|
||||
|
||||
def test_max_instructions_length_default(self, speech_server):
|
||||
"""Test default max instructions length (500) when no config provided."""
|
||||
# Fixture creates server with no CLI override and no TTS stage
|
||||
assert speech_server._max_instructions_length == 500
|
||||
|
||||
def test_max_instructions_length_cli_override(self):
|
||||
"""Test CLI override (stored in engine_client) takes highest priority."""
|
||||
mock_engine_client = MagicMock()
|
||||
mock_engine_client.errored = False
|
||||
mock_engine_client.stage_list = None
|
||||
# CLI override is stored in engine_client
|
||||
mock_engine_client.tts_max_instructions_length = 1000
|
||||
mock_models = MagicMock()
|
||||
mock_models.is_base_model.return_value = True
|
||||
|
||||
server = OmniOpenAIServingSpeech(
|
||||
engine_client=mock_engine_client,
|
||||
models=mock_models,
|
||||
request_logger=MagicMock(),
|
||||
)
|
||||
# Value is cached during __init__
|
||||
assert server._max_instructions_length == 1000
|
||||
|
||||
def test_max_instructions_length_stage_config(self):
|
||||
"""Test stage config value is used when no CLI override."""
|
||||
mock_engine_client = MagicMock()
|
||||
mock_engine_client.errored = False
|
||||
mock_engine_client.tts_max_instructions_length = None # No CLI override
|
||||
mock_models = MagicMock()
|
||||
mock_models.is_base_model.return_value = True
|
||||
|
||||
# Mock stage with tts_args
|
||||
mock_stage = MagicMock()
|
||||
mock_stage.model_stage = "qwen3_tts"
|
||||
mock_stage.tts_args = {"max_instructions_length": 750}
|
||||
mock_engine_client.stage_list = [mock_stage]
|
||||
|
||||
server = OmniOpenAIServingSpeech(
|
||||
engine_client=mock_engine_client,
|
||||
models=mock_models,
|
||||
request_logger=MagicMock(),
|
||||
)
|
||||
# Value is cached during __init__
|
||||
assert server._max_instructions_length == 750
|
||||
|
||||
def test_max_instructions_length_cli_overrides_stage_config(self):
|
||||
"""Test CLI override (in engine_client) takes precedence over stage config."""
|
||||
mock_engine_client = MagicMock()
|
||||
mock_engine_client.errored = False
|
||||
# CLI override stored in engine_client
|
||||
mock_engine_client.tts_max_instructions_length = 2000
|
||||
mock_models = MagicMock()
|
||||
mock_models.is_base_model.return_value = True
|
||||
|
||||
# Mock stage with tts_args
|
||||
mock_stage = MagicMock()
|
||||
mock_stage.model_stage = "qwen3_tts"
|
||||
mock_stage.tts_args = {"max_instructions_length": 750}
|
||||
mock_engine_client.stage_list = [mock_stage]
|
||||
|
||||
server = OmniOpenAIServingSpeech(
|
||||
engine_client=mock_engine_client,
|
||||
models=mock_models,
|
||||
request_logger=MagicMock(),
|
||||
)
|
||||
# CLI value (2000) should override stage config (750)
|
||||
assert server._max_instructions_length == 2000
|
||||
|
||||
def test_validate_instructions_length_uses_cached_value(self):
|
||||
"""Test instructions length validation uses cached _max_instructions_length."""
|
||||
mock_engine_client = MagicMock()
|
||||
mock_engine_client.errored = False
|
||||
mock_engine_client.stage_list = None
|
||||
# CLI override with max length of 10 characters
|
||||
mock_engine_client.tts_max_instructions_length = 10
|
||||
mock_models = MagicMock()
|
||||
mock_models.is_base_model.return_value = True
|
||||
|
||||
server = OmniOpenAIServingSpeech(
|
||||
engine_client=mock_engine_client,
|
||||
models=mock_models,
|
||||
request_logger=MagicMock(),
|
||||
)
|
||||
|
||||
# Verify cached value
|
||||
assert server._max_instructions_length == 10
|
||||
|
||||
# Instructions within limit should pass
|
||||
req = OpenAICreateSpeechRequest(
|
||||
input="Hello",
|
||||
task_type="VoiceDesign",
|
||||
instructions="short",
|
||||
)
|
||||
assert server._validate_tts_request(req) is None
|
||||
|
||||
# Instructions exceeding limit should fail
|
||||
req = OpenAICreateSpeechRequest(
|
||||
input="Hello",
|
||||
task_type="VoiceDesign",
|
||||
instructions="this is too long",
|
||||
)
|
||||
error = server._validate_tts_request(req)
|
||||
assert error is not None
|
||||
assert "max 10 characters" in error
|
||||
@@ -6,28 +6,6 @@
|
||||
"stage_config_name": "qwen3_omni.yaml"
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"dataset_name": "random",
|
||||
"num_prompts": [
|
||||
10,
|
||||
40,
|
||||
100
|
||||
],
|
||||
"request_rate": [
|
||||
0.1,
|
||||
0.5,
|
||||
1
|
||||
],
|
||||
"random_input_len": 2500,
|
||||
"random_output_len": 900,
|
||||
"ignore_eos": true,
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": 100000,
|
||||
"mean_audio_ttfp_ms": 100000,
|
||||
"mean_audio_rtf": 100000
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "random",
|
||||
"num_prompts": [
|
||||
@@ -84,41 +62,6 @@
|
||||
"mean_audio_ttfp_ms": 100000,
|
||||
"mean_audio_rtf": 100000
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "random-mm",
|
||||
"num_prompts": [
|
||||
10,
|
||||
40,
|
||||
100
|
||||
],
|
||||
"request_rate": [
|
||||
0.1,
|
||||
0.5,
|
||||
1
|
||||
],
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
"ignore_eos": true,
|
||||
"random_mm_base_items_per_request": 3,
|
||||
"random_mm_num_mm_items_range_ratio": 0,
|
||||
"random_mm_limit_mm_per_prompt": {
|
||||
"image": 1,
|
||||
"video": 1,
|
||||
"audio": 1
|
||||
},
|
||||
"random_mm_bucket_config": {
|
||||
"(32, 32, 1)": 0.5,
|
||||
"(0, 1, 1)": 0.1,
|
||||
"(32, 32, 2)": 0.4
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": 100000,
|
||||
"mean_audio_ttfp_ms": 100000,
|
||||
"mean_audio_rtf": 100000
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -147,28 +90,6 @@
|
||||
}
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"dataset_name": "random",
|
||||
"num_prompts": [
|
||||
10,
|
||||
40,
|
||||
100
|
||||
],
|
||||
"request_rate": [
|
||||
0.1,
|
||||
0.5,
|
||||
1
|
||||
],
|
||||
"random_input_len": 2500,
|
||||
"random_output_len": 900,
|
||||
"ignore_eos": true,
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": 100000,
|
||||
"mean_audio_ttfp_ms": 100000,
|
||||
"mean_audio_rtf": 100000
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "random",
|
||||
"num_prompts": [
|
||||
@@ -225,41 +146,6 @@
|
||||
"mean_audio_ttfp_ms": 100000,
|
||||
"mean_audio_rtf": 100000
|
||||
}
|
||||
},
|
||||
{
|
||||
"dataset_name": "random-mm",
|
||||
"num_prompts": [
|
||||
10,
|
||||
40,
|
||||
100
|
||||
],
|
||||
"request_rate": [
|
||||
0.1,
|
||||
0.5,
|
||||
1
|
||||
],
|
||||
"random_input_len": 100,
|
||||
"random_output_len": 100,
|
||||
"random_range_ratio": 0.0,
|
||||
"ignore_eos": true,
|
||||
"random_mm_base_items_per_request": 3,
|
||||
"random_mm_num_mm_items_range_ratio": 0,
|
||||
"random_mm_limit_mm_per_prompt": {
|
||||
"image": 1,
|
||||
"video": 1,
|
||||
"audio": 1
|
||||
},
|
||||
"random_mm_bucket_config": {
|
||||
"(32, 32, 1)": 0.5,
|
||||
"(0, 1, 1)": 0.1,
|
||||
"(32, 32, 2)": 0.4
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"mean_ttft_ms": 100000,
|
||||
"mean_audio_ttfp_ms": 100000,
|
||||
"mean_audio_rtf": 100000
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user