[Feature] Support text encoder Disaggregation for H3 (#5885)

Signed-off-by: Canlin Guo <canlinguosdu@gmail.com>
Signed-off-by: lishunyang12 <lishunyang12@163.com>
Signed-off-by: yuanwu <yuan.wu@intel.com>
Co-authored-by: lishunyang12 <lishunyang12@163.com>
Co-authored-by: yuanwu <yuan.wu@intel.com>
This commit is contained in:
authored and GitHub committed 2026-08-28 14:27:05 +08:00
1 parent ed837e350e
commit b6a4baa659
61 files changed
+2952 -366

No files matched your search

@@ -67,6 +67,39 @@ def test_build_add_request_message_preserves_additional_information(mocker: Mock
output_processor.add_request.assert_not_called()
def test_build_add_request_message_injects_global_id_before_prompt_transform(mocker: MockerFixture):
engine = object.__new__(AsyncOmniEngine)
params = SamplingParams(max_tokens=8)
engine.default_sampling_params_list = [params]
engine.stage_metadata = [StageRuntimeInfo(final_output=False, final_output_type=None, stage_type="llm")]
engine.supported_tasks = ("speech",)
input_processor = mocker.Mock()
input_processor.process_inputs.return_value = _make_engine_core_request()
engine.input_processor = input_processor
engine.output_processors = [mocker.Mock()]
seen_global_ids: list[list[str]] = []
def transform(prompt, _sampling_params):
seen_global_ids.append(prompt["additional_information"]["global_request_id"])
return prompt
engine.prompt_transform_func = transform
prompt = {"prompt_token_ids": [1, 1, 1]}
message = engine._build_add_request_message(
request_id="req-1",
prompt=prompt,
sampling_params_list=[params],
final_stage_id=0,
arrival_time=0.0,
)
assert seen_global_ids == [["req-1"]]
assert message.original_prompt["additional_information"]["global_request_id"] == ["req-1"]
def test_build_add_request_message_preserves_model_intermediate_buffer(mocker: MockerFixture):
engine = object.__new__(AsyncOmniEngine)
params = SamplingParams(max_tokens=8)
@@ -433,11 +433,35 @@ def test_initialize_local_diffusion_replica_passes_stage_init_timeout_and_inline
"stage_id": 0,
"stage_init_timeout": 302,
"batch_size": 4,
"use_inline": True,
"use_inline": False,
"omni_master_server": None,
}
def test_initialize_local_diffusion_replica_uses_explicit_inline_opt_in(monkeypatch):
import vllm_omni.engine.stage_runtime as runtime_mod
from vllm_omni.engine.stage_engine_startup import StageReplicaResources
runtime = _make_stage_runtime()
plan = _make_diffusion_plan(0, stage_id=1).replicas[0]
plan.stage_cfg.engine_args["inline_diffusion"] = True
captured: dict[str, object] = {}
monkeypatch.setattr(runtime_mod, "inject_kv_stage_info", lambda *_: None)
monkeypatch.setattr(
runtime_mod,
"launch_diffusion_stage_replica",
lambda **kwargs: (
captured.update(use_inline=kwargs["use_inline"]),
(types.SimpleNamespace(), StageReplicaResources()),
)[1],
)
runtime._initialize_local_diffusion_replica(plan, stage_init_timeout=1)
assert captured["use_inline"] is True
def test_initialize_local_llm_replica_scopes_runtime_env(monkeypatch):
import vllm_omni.engine.stage_runtime as runtime_mod
@@ -1140,6 +1164,21 @@ def test_build_engine_args_keeps_stage_owned_tokenizer_subdir(tmp_path):
assert engine_args["tokenizer"] == os.path.join(str(tmp_path), "tokenizer")
def test_model_path_resolver_is_generic_and_model_owned(tmp_path):
from vllm_omni.engine.stage_init_utils import _resolve_model_path
engine_args = {
"model_path_resolver": ("vllm_omni.model_executor.models.minimax_h3.checkpoint.resolve_minimax_h3_model_root"),
"revision": None,
"task_type": "ref2va",
}
resolved = _resolve_model_path(str(tmp_path), engine_args)
assert resolved == str(tmp_path / "Ref2VA" / "text_encoder")
assert "model_path_resolver" not in engine_args
def test_build_stage0_input_processor_uses_omni_input_preprocessor(monkeypatch):
import vllm_omni.engine.stage_init_utils as init_mod
+28
View File
@@ -74,6 +74,25 @@ class TestOmniPayloadStruct:
for top, sub in original.items():
assert sorted(d[top].keys()) == sorted(sub.keys())
def test_text_conditioning_uses_shared_payload_fields(self):
hidden = torch.zeros(4, 5120)
token_role_ids = torch.tensor([[1], [1], [0], [0]])
payload = to_dict(
to_struct(
{
"hidden_states": {"output": hidden},
"meta": {"token_role_ids": token_role_ids},
}
)
)
assert torch.equal(payload["hidden_states"]["output"], hidden)
assert torch.equal(payload["meta"]["token_role_ids"], token_role_ids)
for legacy_key in ("encoder_hidden_states", "token_tags"):
with pytest.raises(msgspec.ValidationError, match="unknown field"):
to_struct({legacy_key: torch.zeros(1)})
def test_to_dict_drops_unset_fields(self):
s = OmniPayloadStruct(meta=MetaStruct(left_context_size=10))
d = to_dict(s)
@@ -397,3 +416,12 @@ class TestSerializeDeserializePayload:
original: OmniPayload = {"meta": {"finished": None}}
wire = serialize_payload(original)
assert wire is None
def test_minimax_h3_prepared_reference_video_descriptor_round_trip(self):
descriptor = '{"artifact_dir":"/tmp/h3","videos":[]}'
wire = serialize_payload({"meta": {"minimax_h3_prepared_reference_videos": descriptor}})
assert wire is not None
restored = deserialize_payload(wire)
assert restored["meta"]["minimax_h3_prepared_reference_videos"] == descriptor
@@ -27,7 +27,11 @@ from vllm_omni.engine.messages import (
ShutdownRequestMessage,
StageSubmissionMessage,
)
from vllm_omni.engine.orchestrator import Orchestrator, OrchestratorRequestState
from vllm_omni.engine.orchestrator import (
Orchestrator,
OrchestratorRequestState,
cleanup_request_artifact_dirs,
)
from vllm_omni.engine.stage_pool import StageUnavailableError
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.outputs import OmniRequestOutput
@@ -44,6 +48,18 @@ from .test_orchestrator import (
_wait_for,
)
def test_request_artifact_cleanup_is_idempotent(tmp_path):
artifact_dir = tmp_path / "request-artifacts"
artifact_dir.mkdir()
(artifact_dir / "prepared.mp4").touch()
cleanup_request_artifact_dirs([str(artifact_dir)])
cleanup_request_artifact_dirs([str(artifact_dir)])
assert not artifact_dir.exists()
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+42
View File
@@ -1207,6 +1207,48 @@ class TestSingleStageReplicaInitialization:
)
assert mock_from_addresses.call_args.kwargs["proc_manager"].proc is proc
def test_initialize_local_custom_diffusion_replica_uses_inline_client(self, mocker: MockerFixture):
import vllm_omni.engine.stage_runtime as runtime_mod
from vllm_omni.platforms import current_omni_platform
runtime = DistStageRuntime(
stage_configs=[],
model="fake-model",
config_path="/fake/stages.yaml",
stage_init_timeout=60,
diffusion_batch_size=4,
async_chunk=False,
single_stage_id_filter=None,
omni_master_address="127.0.0.1",
omni_master_port=26000,
)
runtime._omni_master_server = mocker.Mock(spec=OmniMasterServer)
runtime._coordinator_runtime = None
plan = _make_diffusion_plan(0, stage_id=0, launch_mode="local").replicas[0]
plan.stage_cfg.engine_args = {"custom_pipeline_args": {"pipeline_class": "test.CustomPipeline"}}
sentinel_client = SimpleNamespace()
mock_launch = mocker.patch.object(
runtime_mod,
"launch_diffusion_stage_replica",
return_value=(sentinel_client, StageReplicaResources()),
)
mocker.patch.object(runtime_mod, "inject_kv_stage_info")
device_env_var = current_omni_platform.device_control_env_var
prev_device_env = os.environ.get(device_env_var)
os.environ[device_env_var] = "0"
runtime._init_visible_devices_baseline = "0"
try:
result = runtime._initialize_local_diffusion_replica(plan, stage_init_timeout=60)
finally:
if prev_device_env is None:
os.environ.pop(device_env_var, None)
else:
os.environ[device_env_var] = prev_device_env
assert result is sentinel_client
assert mock_launch.call_args.kwargs["use_inline"] is True
def test_initialize_local_diffusion_replica_failure_terminates_proc(self, mocker: MockerFixture):
import vllm_omni.engine.stage_runtime as runtime_mod
from vllm_omni.platforms import current_omni_platform
+5 -1
View File
@@ -72,6 +72,7 @@ def _effective_backend_values(config_cls: type, engine_args: dict) -> dict[str,
_LLM_BACKEND_FIELDS = frozenset(field.name for field in fields(OmniEngineArgs))
_DIFFUSION_BACKEND_FIELDS = frozenset(field.name for field in fields(OmniDiffusionConfig))
_TOPOLOGY_ONLY_ENGINE_ARGS = frozenset({"inline_diffusion"})
_OMNI_ONLY_LLM_STAGE_ENGINE_FIELDS = frozenset(
{
"active_stream_window",
@@ -368,7 +369,9 @@ def test_typed_llm_engine_args_preserve_legacy_adapter_behavior(tmp_path):
)
typed_args_by_stage[stage_id] = typed_args
assert {name: typed_args[name] for name in legacy_args} == legacy_args
expected_args = {name: value for name, value in legacy_args.items() if name not in _TOPOLOGY_ONLY_ENGINE_ARGS}
assert {name: typed_args[name] for name in expected_args} == expected_args
assert _TOPOLOGY_ONLY_ENGINE_ARGS.isdisjoint(typed_args)
thinker_args = typed_args_by_stage[0]
inherited_vllm_fields = {
@@ -658,6 +661,7 @@ def test_typed_engine_args_match_current_registry_backend_semantics(model_type):
backend_fields = _LLM_BACKEND_FIELDS
backend_config_cls = OmniEngineArgs
backend_fields -= _TOPOLOGY_ONLY_ENGINE_ARGS
missing_fields = (legacy_args.keys() & backend_fields) - typed_args.keys()
assert not missing_fields, f"{model_type} stage {stage_id} lost backend fields: {sorted(missing_fields)}"