[Feature] Support text encoder Disaggregation for H3 (#5885)
Signed-off-by: Canlin Guo <canlinguosdu@gmail.com> Signed-off-by: lishunyang12 <lishunyang12@163.com> Signed-off-by: yuanwu <yuan.wu@intel.com> Co-authored-by: lishunyang12 <lishunyang12@163.com> Co-authored-by: yuanwu <yuan.wu@intel.com>
This commit is contained in:
61 files changed
+2952
-366
No files matched your search
@@ -67,6 +67,39 @@ def test_build_add_request_message_preserves_additional_information(mocker: Mock
|
||||
output_processor.add_request.assert_not_called()
|
||||
|
||||
|
||||
def test_build_add_request_message_injects_global_id_before_prompt_transform(mocker: MockerFixture):
|
||||
engine = object.__new__(AsyncOmniEngine)
|
||||
params = SamplingParams(max_tokens=8)
|
||||
engine.default_sampling_params_list = [params]
|
||||
engine.stage_metadata = [StageRuntimeInfo(final_output=False, final_output_type=None, stage_type="llm")]
|
||||
engine.supported_tasks = ("speech",)
|
||||
|
||||
input_processor = mocker.Mock()
|
||||
input_processor.process_inputs.return_value = _make_engine_core_request()
|
||||
engine.input_processor = input_processor
|
||||
engine.output_processors = [mocker.Mock()]
|
||||
|
||||
seen_global_ids: list[list[str]] = []
|
||||
|
||||
def transform(prompt, _sampling_params):
|
||||
seen_global_ids.append(prompt["additional_information"]["global_request_id"])
|
||||
return prompt
|
||||
|
||||
engine.prompt_transform_func = transform
|
||||
prompt = {"prompt_token_ids": [1, 1, 1]}
|
||||
|
||||
message = engine._build_add_request_message(
|
||||
request_id="req-1",
|
||||
prompt=prompt,
|
||||
sampling_params_list=[params],
|
||||
final_stage_id=0,
|
||||
arrival_time=0.0,
|
||||
)
|
||||
|
||||
assert seen_global_ids == [["req-1"]]
|
||||
assert message.original_prompt["additional_information"]["global_request_id"] == ["req-1"]
|
||||
|
||||
|
||||
def test_build_add_request_message_preserves_model_intermediate_buffer(mocker: MockerFixture):
|
||||
engine = object.__new__(AsyncOmniEngine)
|
||||
params = SamplingParams(max_tokens=8)
|
||||
|
||||
@@ -433,11 +433,35 @@ def test_initialize_local_diffusion_replica_passes_stage_init_timeout_and_inline
|
||||
"stage_id": 0,
|
||||
"stage_init_timeout": 302,
|
||||
"batch_size": 4,
|
||||
"use_inline": True,
|
||||
"use_inline": False,
|
||||
"omni_master_server": None,
|
||||
}
|
||||
|
||||
|
||||
def test_initialize_local_diffusion_replica_uses_explicit_inline_opt_in(monkeypatch):
|
||||
import vllm_omni.engine.stage_runtime as runtime_mod
|
||||
from vllm_omni.engine.stage_engine_startup import StageReplicaResources
|
||||
|
||||
runtime = _make_stage_runtime()
|
||||
plan = _make_diffusion_plan(0, stage_id=1).replicas[0]
|
||||
plan.stage_cfg.engine_args["inline_diffusion"] = True
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
monkeypatch.setattr(runtime_mod, "inject_kv_stage_info", lambda *_: None)
|
||||
monkeypatch.setattr(
|
||||
runtime_mod,
|
||||
"launch_diffusion_stage_replica",
|
||||
lambda **kwargs: (
|
||||
captured.update(use_inline=kwargs["use_inline"]),
|
||||
(types.SimpleNamespace(), StageReplicaResources()),
|
||||
)[1],
|
||||
)
|
||||
|
||||
runtime._initialize_local_diffusion_replica(plan, stage_init_timeout=1)
|
||||
|
||||
assert captured["use_inline"] is True
|
||||
|
||||
|
||||
def test_initialize_local_llm_replica_scopes_runtime_env(monkeypatch):
|
||||
import vllm_omni.engine.stage_runtime as runtime_mod
|
||||
|
||||
@@ -1140,6 +1164,21 @@ def test_build_engine_args_keeps_stage_owned_tokenizer_subdir(tmp_path):
|
||||
assert engine_args["tokenizer"] == os.path.join(str(tmp_path), "tokenizer")
|
||||
|
||||
|
||||
def test_model_path_resolver_is_generic_and_model_owned(tmp_path):
|
||||
from vllm_omni.engine.stage_init_utils import _resolve_model_path
|
||||
|
||||
engine_args = {
|
||||
"model_path_resolver": ("vllm_omni.model_executor.models.minimax_h3.checkpoint.resolve_minimax_h3_model_root"),
|
||||
"revision": None,
|
||||
"task_type": "ref2va",
|
||||
}
|
||||
|
||||
resolved = _resolve_model_path(str(tmp_path), engine_args)
|
||||
|
||||
assert resolved == str(tmp_path / "Ref2VA" / "text_encoder")
|
||||
assert "model_path_resolver" not in engine_args
|
||||
|
||||
|
||||
def test_build_stage0_input_processor_uses_omni_input_preprocessor(monkeypatch):
|
||||
import vllm_omni.engine.stage_init_utils as init_mod
|
||||
|
||||
|
||||
@@ -74,6 +74,25 @@ class TestOmniPayloadStruct:
|
||||
for top, sub in original.items():
|
||||
assert sorted(d[top].keys()) == sorted(sub.keys())
|
||||
|
||||
def test_text_conditioning_uses_shared_payload_fields(self):
|
||||
hidden = torch.zeros(4, 5120)
|
||||
token_role_ids = torch.tensor([[1], [1], [0], [0]])
|
||||
|
||||
payload = to_dict(
|
||||
to_struct(
|
||||
{
|
||||
"hidden_states": {"output": hidden},
|
||||
"meta": {"token_role_ids": token_role_ids},
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
assert torch.equal(payload["hidden_states"]["output"], hidden)
|
||||
assert torch.equal(payload["meta"]["token_role_ids"], token_role_ids)
|
||||
for legacy_key in ("encoder_hidden_states", "token_tags"):
|
||||
with pytest.raises(msgspec.ValidationError, match="unknown field"):
|
||||
to_struct({legacy_key: torch.zeros(1)})
|
||||
|
||||
def test_to_dict_drops_unset_fields(self):
|
||||
s = OmniPayloadStruct(meta=MetaStruct(left_context_size=10))
|
||||
d = to_dict(s)
|
||||
@@ -397,3 +416,12 @@ class TestSerializeDeserializePayload:
|
||||
original: OmniPayload = {"meta": {"finished": None}}
|
||||
wire = serialize_payload(original)
|
||||
assert wire is None
|
||||
|
||||
def test_minimax_h3_prepared_reference_video_descriptor_round_trip(self):
|
||||
descriptor = '{"artifact_dir":"/tmp/h3","videos":[]}'
|
||||
|
||||
wire = serialize_payload({"meta": {"minimax_h3_prepared_reference_videos": descriptor}})
|
||||
assert wire is not None
|
||||
restored = deserialize_payload(wire)
|
||||
|
||||
assert restored["meta"]["minimax_h3_prepared_reference_videos"] == descriptor
|
||||
@@ -27,7 +27,11 @@ from vllm_omni.engine.messages import (
|
||||
ShutdownRequestMessage,
|
||||
StageSubmissionMessage,
|
||||
)
|
||||
from vllm_omni.engine.orchestrator import Orchestrator, OrchestratorRequestState
|
||||
from vllm_omni.engine.orchestrator import (
|
||||
Orchestrator,
|
||||
OrchestratorRequestState,
|
||||
cleanup_request_artifact_dirs,
|
||||
)
|
||||
from vllm_omni.engine.stage_pool import StageUnavailableError
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
@@ -44,6 +48,18 @@ from .test_orchestrator import (
|
||||
_wait_for,
|
||||
)
|
||||
|
||||
|
||||
def test_request_artifact_cleanup_is_idempotent(tmp_path):
|
||||
artifact_dir = tmp_path / "request-artifacts"
|
||||
artifact_dir.mkdir()
|
||||
(artifact_dir / "prepared.mp4").touch()
|
||||
|
||||
cleanup_request_artifact_dirs([str(artifact_dir)])
|
||||
cleanup_request_artifact_dirs([str(artifact_dir)])
|
||||
|
||||
assert not artifact_dir.exists()
|
||||
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
|
||||
@@ -1207,6 +1207,48 @@ class TestSingleStageReplicaInitialization:
|
||||
)
|
||||
assert mock_from_addresses.call_args.kwargs["proc_manager"].proc is proc
|
||||
|
||||
def test_initialize_local_custom_diffusion_replica_uses_inline_client(self, mocker: MockerFixture):
|
||||
import vllm_omni.engine.stage_runtime as runtime_mod
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
runtime = DistStageRuntime(
|
||||
stage_configs=[],
|
||||
model="fake-model",
|
||||
config_path="/fake/stages.yaml",
|
||||
stage_init_timeout=60,
|
||||
diffusion_batch_size=4,
|
||||
async_chunk=False,
|
||||
single_stage_id_filter=None,
|
||||
omni_master_address="127.0.0.1",
|
||||
omni_master_port=26000,
|
||||
)
|
||||
runtime._omni_master_server = mocker.Mock(spec=OmniMasterServer)
|
||||
runtime._coordinator_runtime = None
|
||||
plan = _make_diffusion_plan(0, stage_id=0, launch_mode="local").replicas[0]
|
||||
plan.stage_cfg.engine_args = {"custom_pipeline_args": {"pipeline_class": "test.CustomPipeline"}}
|
||||
sentinel_client = SimpleNamespace()
|
||||
mock_launch = mocker.patch.object(
|
||||
runtime_mod,
|
||||
"launch_diffusion_stage_replica",
|
||||
return_value=(sentinel_client, StageReplicaResources()),
|
||||
)
|
||||
mocker.patch.object(runtime_mod, "inject_kv_stage_info")
|
||||
|
||||
device_env_var = current_omni_platform.device_control_env_var
|
||||
prev_device_env = os.environ.get(device_env_var)
|
||||
os.environ[device_env_var] = "0"
|
||||
runtime._init_visible_devices_baseline = "0"
|
||||
try:
|
||||
result = runtime._initialize_local_diffusion_replica(plan, stage_init_timeout=60)
|
||||
finally:
|
||||
if prev_device_env is None:
|
||||
os.environ.pop(device_env_var, None)
|
||||
else:
|
||||
os.environ[device_env_var] = prev_device_env
|
||||
|
||||
assert result is sentinel_client
|
||||
assert mock_launch.call_args.kwargs["use_inline"] is True
|
||||
|
||||
def test_initialize_local_diffusion_replica_failure_terminates_proc(self, mocker: MockerFixture):
|
||||
import vllm_omni.engine.stage_runtime as runtime_mod
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
@@ -72,6 +72,7 @@ def _effective_backend_values(config_cls: type, engine_args: dict) -> dict[str,
|
||||
|
||||
_LLM_BACKEND_FIELDS = frozenset(field.name for field in fields(OmniEngineArgs))
|
||||
_DIFFUSION_BACKEND_FIELDS = frozenset(field.name for field in fields(OmniDiffusionConfig))
|
||||
_TOPOLOGY_ONLY_ENGINE_ARGS = frozenset({"inline_diffusion"})
|
||||
_OMNI_ONLY_LLM_STAGE_ENGINE_FIELDS = frozenset(
|
||||
{
|
||||
"active_stream_window",
|
||||
@@ -368,7 +369,9 @@ def test_typed_llm_engine_args_preserve_legacy_adapter_behavior(tmp_path):
|
||||
)
|
||||
typed_args_by_stage[stage_id] = typed_args
|
||||
|
||||
assert {name: typed_args[name] for name in legacy_args} == legacy_args
|
||||
expected_args = {name: value for name, value in legacy_args.items() if name not in _TOPOLOGY_ONLY_ENGINE_ARGS}
|
||||
assert {name: typed_args[name] for name in expected_args} == expected_args
|
||||
assert _TOPOLOGY_ONLY_ENGINE_ARGS.isdisjoint(typed_args)
|
||||
|
||||
thinker_args = typed_args_by_stage[0]
|
||||
inherited_vllm_fields = {
|
||||
@@ -658,6 +661,7 @@ def test_typed_engine_args_match_current_registry_backend_semantics(model_type):
|
||||
backend_fields = _LLM_BACKEND_FIELDS
|
||||
backend_config_cls = OmniEngineArgs
|
||||
|
||||
backend_fields -= _TOPOLOGY_ONLY_ENGINE_ARGS
|
||||
missing_fields = (legacy_args.keys() & backend_fields) - typed_args.keys()
|
||||
assert not missing_fields, f"{model_type} stage {stage_id} lost backend fields: {sorted(missing_fields)}"
|
||||
|
||||
|
||||
Reference in new issue
Block a user