From 5356357cf0bf95a7818ba2452b2248b5a0e59af2 Mon Sep 17 00:00:00 2001 From: Zhou Taichang Date: Mon, 21 Sep 2026 19:00:46 +0800 Subject: [PATCH 1/5] [LingBot World] Shard the streaming VAE decode across the Ulysses ranks (#7651) Signed-off-by: tzhouam Co-authored-by: Claude Fable 5.1 --- .../distributed/test_wan_spatial_shard.py | 177 ++++++++++++++++++ .../test_pipeline_lingbot_world.py | 96 +++++++++- .../deploy/lingbot_world_v2_stepwise.yaml | 3 + .../autoencoders/wan_spatial_shard.py | 35 +++- .../models/lingbot_world/pipeline.py | 78 +++++++- 5 files changed, 378 insertions(+), 11 deletions(-) diff --git a/tests/diffusion/distributed/test_wan_spatial_shard.py b/tests/diffusion/distributed/test_wan_spatial_shard.py index 23c29c938..27177745e 100644 --- a/tests/diffusion/distributed/test_wan_spatial_shard.py +++ b/tests/diffusion/distributed/test_wan_spatial_shard.py @@ -493,6 +493,84 @@ def test_dist_zero_pad_only_applies_global_width_edges(monkeypatch: pytest.Monke assert last.shape == (1, 1, 2, 3) +@pytest.mark.core_model +@pytest.mark.cpu +class _PlainDecoder(torch.nn.Module): + """A decoder with nothing to patch, so the install only wraps ``forward``.""" + + def __init__(self): + super().__init__() + self.calls: list[dict] = [] + + def forward(self, x, feat_cache=None, feat_idx=None, first_chunk=False): + self.calls.append({"feat_cache": feat_cache, "feat_idx": feat_idx, "first_chunk": first_chunk}) + return x + + +@pytest.mark.core_model +@pytest.mark.cpu +@pytest.mark.parametrize("world_size, dst", [(4, -1), (4, 4), (4, 5), (1, 1), (4, 1.5)]) +def test_install_rejects_invalid_destination_before_mutating_decoder(monkeypatch, world_size, dst): + monkeypatch.setattr(wan_spatial_shard, "_rank_world", lambda group: (0, world_size)) + vae = SimpleNamespace(decoder=torch.nn.Identity()) + original_forward = vae.decoder.forward + + def unexpected_patch(*args, **kwargs): + pytest.fail("invalid destination must fail before patching modules") + + monkeypatch.setattr(wan_spatial_shard, "_patch_decoder_modules", unexpected_patch) + with pytest.raises(ValueError, match="dst must be None or an integer"): + wan_spatial_shard.install_wan_spatial_shard_decode(vae, object(), dst=dst) + assert vae.decoder.forward == original_forward + assert not getattr(vae, "_vllm_omni_wan_spatial_shard_installed", False) + + +def _install_on_plain_decoder(monkeypatch: pytest.MonkeyPatch, *, dst): + monkeypatch.setattr(wan_spatial_shard, "_rank_world", lambda group: (1, 2)) + monkeypatch.setattr( + wan_spatial_shard, + "split_for_parallel_decode", + lambda x, *, upsample_count, split_dim, group: (x, x.shape[wan_spatial_shard._spatial_dim(split_dim)]), + ) + gathers: list[dict] = [] + + def gather(x, *, expected_extent, split_dim, group, dst=None): + gathers.append({"expected_extent": expected_extent, "split_dim": split_dim, "dst": dst}) + return x + + monkeypatch.setattr(wan_spatial_shard, "gather_and_trim_extent", gather) + vae = SimpleNamespace(decoder=_PlainDecoder()) + wan_spatial_shard.install_wan_spatial_shard_decode(vae, object(), split_dim="width", dst=dst) + return vae, gathers + + +@pytest.mark.core_model +@pytest.mark.cpu +@pytest.mark.parametrize("dst", [0, 1, None]) +def test_install_passes_the_assembling_rank_through_to_the_gather(monkeypatch: pytest.MonkeyPatch, dst): + vae, gathers = _install_on_plain_decoder(monkeypatch, dst=dst) + cache = [None] + + out = vae.decoder(torch.zeros(1, 3, 1, 4, 6), feat_cache=cache, feat_idx=[0], first_chunk=True) + + assert out.shape == (1, 3, 1, 4, 6) + assert gathers == [{"expected_extent": 6, "split_dim": "width", "dst": dst}] + # The streaming arguments reach the wrapped decoder untouched. + assert vae.decoder.calls == [{"feat_cache": cache, "feat_idx": [0], "first_chunk": True}] + assert vae._vllm_omni_wan_spatial_shard_dst == dst + + +@pytest.mark.core_model +@pytest.mark.cpu +def test_install_refuses_to_change_the_assembling_rank(monkeypatch: pytest.MonkeyPatch): + vae, _ = _install_on_plain_decoder(monkeypatch, dst=None) + + # Same settings again: a no-op, as before. + wan_spatial_shard.install_wan_spatial_shard_decode(vae, object(), split_dim="width", dst=None) + with pytest.raises(ValueError, match="already patched to assemble on None"): + wan_spatial_shard.install_wan_spatial_shard_decode(vae, object(), split_dim="width", dst=0) + + @pytest.mark.core_model @pytest.mark.cpu def test_spatial_shard_height_gate_falls_back_for_partial_group(monkeypatch: pytest.MonkeyPatch): @@ -626,6 +704,105 @@ def _spatial_shard_decode_worker(rank: int, split_dim: str, return_dict, master_ dist.destroy_process_group() +_STREAMING_SHARD_CHUNKS = 3 + + +def _streaming_shard_worker(rank: int, return_dict, master_port: str) -> None: + """Sharded streaming decode with every rank keeping the frame, against an unsharded streaming reference. + + Two sessions are interleaved chunk by chunk, so the per-session temporal cache + and the halo exchange are exercised across successive calls, then one session + is reset and must reproduce its opening chunk. Every rank checks every chunk. + """ + from vllm_omni.diffusion.distributed.parallel_state import ( + destroy_model_parallel, + get_sp_group, + init_distributed_environment, + initialize_model_parallel, + ) + from vllm_omni.experimental.ar_diffusion.streaming_decode import WanStreamingDecoder + + os.environ["MASTER_ADDR"] = "127.0.0.1" + os.environ["MASTER_PORT"] = master_port + device = current_omni_platform.get_torch_device(rank) + current_omni_platform.set_device(device) + dtype = torch.float32 + backend = current_omni_platform.dist_backend + init_distributed_environment(world_size=_SPATIAL_SHARD_WORLD_SIZE, rank=rank, local_rank=rank, backend=backend) + initialize_model_parallel( + sequence_parallel_size=_SPATIAL_SHARD_WORLD_SIZE, ulysses_degree=_SPATIAL_SHARD_WORLD_SIZE, backend=backend + ) + # VLLM_OMNI_TEST_WAN_VAE may point at a local diffusers checkpoint (with or + # without a ``vae`` subfolder) for machines without hub access. + model = os.environ.get("VLLM_OMNI_TEST_WAN_VAE", _SPATIAL_SHARD_MODEL) + local_vae_only = os.path.isdir(model) and not os.path.isdir(os.path.join(model, "vae")) + load = {"torch_dtype": dtype} if local_vae_only else {"subfolder": _SPATIAL_SHARD_SUBFOLDER, "torch_dtype": dtype} + try: + reference_vae = DistributedAutoencoderKLWan.from_pretrained(model, **load).to(device=device, dtype=dtype).eval() + sharded_vae = DistributedAutoencoderKLWan.from_pretrained(model, **load).to(device=device, dtype=dtype).eval() + wan_spatial_shard.install_wan_spatial_shard_decode( + sharded_vae, get_sp_group().device_group, split_dim="width", dst=None + ) + reference = WanStreamingDecoder(reference_vae, bytes_per_pixel_fp32=1.0) + sharded = WanStreamingDecoder(sharded_vae, bytes_per_pixel_fp32=1.0) + + generator = torch.Generator(device=device).manual_seed(0) + # Width 104 latents = 832 px: not divisible by two ranks' halo-padded shards without padding. + shape = (1, reference_vae.config.z_dim, 1, _SPATIAL_SHARD_LATENT_HEIGHT, _SPATIAL_SHARD_LATENT_WIDTH) + chunks = { + name: [ + torch.randn(shape, generator=generator, device=device, dtype=dtype) + for _ in range(_STREAMING_SHARD_CHUNKS) + ] + for name in ("a", "b") + } + max_diff = 0.0 + with torch.inference_mode(): + ref_states = {name: reference.new_decode_state(name) for name in chunks} + shard_states = {name: sharded.new_decode_state(name) for name in chunks} + for index in range(_STREAMING_SHARD_CHUNKS): + for name in ("a", "b"): + expected = reference.decode_chunk(chunks[name][index], ref_states[name]).float() + got = sharded.decode_chunk(chunks[name][index], shard_states[name]).float() + if got.shape != expected.shape: + raise AssertionError( + f"rank {rank} {name}[{index}]: {tuple(got.shape)} != {tuple(expected.shape)}" + ) + max_diff = max(max_diff, (got - expected).abs().max().item()) + # Reset session a: a fresh state must reproduce its opening chunk, on this rank too. + shard_states["a"].release() + ref_states["a"].release() + shard_states["a"] = sharded.new_decode_state("a") + ref_states["a"] = reference.new_decode_state("a") + expected = reference.decode_chunk(chunks["a"][0], ref_states["a"]).float() + got = sharded.decode_chunk(chunks["a"][0], shard_states["a"]).float() + max_diff = max(max_diff, (got - expected).abs().max().item()) + return_dict[f"max_abs_diff_rank{rank}"] = max_diff + return_dict[f"shape_rank{rank}"] = tuple(got.shape) + finally: + destroy_model_parallel() + if dist.is_initialized(): + dist.destroy_process_group() + + +@pytest.mark.full_model +@pytest.mark.diffusion +@pytest.mark.parallel +@hardware_test(res={"cuda": ["H100", "B200"]}, num_cards=_SPATIAL_SHARD_WORLD_SIZE) +def test_streaming_shard_decode_matches_unsharded_streaming_on_every_rank(): + manager = mp.get_context("spawn").Manager() + return_dict = manager.dict() + mp.spawn(_streaming_shard_worker, args=(return_dict, "29510"), nprocs=_SPATIAL_SHARD_WORLD_SIZE, join=True) + + for rank in range(_SPATIAL_SHARD_WORLD_SIZE): + assert f"max_abs_diff_rank{rank}" in return_dict, f"rank {rank} did not report" + diff = return_dict[f"max_abs_diff_rank{rank}"] + shape = return_dict[f"shape_rank{rank}"] + print(f"streaming shard rank {rank}: max_abs_diff={diff:.6e} shape={shape}") + assert shape[-1] == _SPATIAL_SHARD_LATENT_WIDTH * 8 and shape[-2] == _SPATIAL_SHARD_LATENT_HEIGHT * 8 + assert diff <= _SPATIAL_SHARD_TOLERANCE, f"rank {rank}: {diff} exceeds {_SPATIAL_SHARD_TOLERANCE}" + + @pytest.mark.full_model @pytest.mark.diffusion @pytest.mark.parallel diff --git a/tests/diffusion/models/lingbot_world/test_pipeline_lingbot_world.py b/tests/diffusion/models/lingbot_world/test_pipeline_lingbot_world.py index 1ccc7aadc..b5a57ce75 100644 --- a/tests/diffusion/models/lingbot_world/test_pipeline_lingbot_world.py +++ b/tests/diffusion/models/lingbot_world/test_pipeline_lingbot_world.py @@ -737,6 +737,99 @@ def test_component_discovery_uses_official_checkpoint_contract() -> None: assert module._loader_state.prefetch_calls == [("checkpoint", ("tokenizer", "text_encoder", "vae"), False)] +def _ulysses_config(size: int): + parallel_config = _od_config().parallel_config + parallel_config.sequence_parallel_size = size + parallel_config.ulysses_degree = size + return _od_config(parallel_config=parallel_config) + + +def _record_shard_install(monkeypatch, module, *, world_size: int): + """Stand in for the Ulysses group and the decoder patch; return what the install was asked for.""" + installs: list[dict] = [] + group = object() + + def install(vae, group_arg, split_dim, *, dst): + installs.append({"vae": vae, "group": group_arg, "split_dim": split_dim, "dst": dst}) + + monkeypatch.setattr(module, "install_wan_spatial_shard_decode", install) + monkeypatch.setattr(module.LingBotWorldCausalDMDPipeline, "_vae_shard_group", lambda self: (group, world_size)) + return installs, group + + +def test_a_multi_rank_deployment_shards_the_decoder_across_the_ulysses_ranks(monkeypatch) -> None: + """Multi-rank deployments shard decode by default.""" + module = _load_pipeline_module() + installs, group = _record_shard_install(monkeypatch, module, world_size=2) + + pipeline = module.LingBotWorldCausalDMDPipeline(od_config=_ulysses_config(2)) + + # Along the width, and assembled on every rank: each rank's post_decode consumes the frame. + assert installs == [{"vae": pipeline.vae, "group": group, "split_dim": "width", "dst": None}] + assert pipeline._vae_shard_split_dim == "width" + + +def test_a_single_rank_deployment_leaves_the_decoder_alone(monkeypatch) -> None: + module = _load_pipeline_module() + installs, _ = _record_shard_install(monkeypatch, module, world_size=1) + + pipeline = module.LingBotWorldCausalDMDPipeline(od_config=_ulysses_config(1)) + + assert installs == [] and pipeline._vae_shard_split_dim is None + + +@pytest.mark.parametrize("enabled", [True, False]) +@pytest.mark.parametrize("width, local_width", [(832, 208), (840, 216)]) +def test_streaming_decode_reservation_uses_padded_shards(monkeypatch, enabled, width, local_width): + module = _load_pipeline_module() + installs, _ = _record_shard_install(monkeypatch, module, world_size=4) + config = _ulysses_config(4) + config.model_config.update(lingbot_vae_spatial_sharding=enabled, ar_diffusion_height=480, ar_diffusion_width=width) + pipeline = module.LingBotWorldCausalDMDPipeline(od_config=config) + from vllm_omni.experimental.ar_diffusion.streaming_decode import WanStreamingDecoder + + # Exercise the real byte estimator without constructing a checkpoint VAE. + decoder = object.__new__(WanStreamingDecoder) + decoder._bytes_per_pixel_fp32 = 16.0 + monkeypatch.setattr(pipeline, "_streaming_decoder", lambda: decoder) + expected_width = local_width if enabled else width + assert pipeline._streaming_decode_bytes_per_session() == 16 * 480 * expected_width + assert len(installs) == int(enabled) + assert pipeline._vae_shard_world_size == (4 if enabled else 1) + assert config.parallel_config.sequence_parallel_size == config.parallel_config.ulysses_degree == 4 + + +def test_disabling_vae_sharding_does_not_require_a_shard_group(monkeypatch): + module = _load_pipeline_module() + config = _ulysses_config(4) + config.model_config["lingbot_vae_spatial_sharding"] = False + + def unexpected_group(self): + pytest.fail("disabled VAE sharding must not access its process group") + + monkeypatch.setattr(module.LingBotWorldCausalDMDPipeline, "_vae_shard_group", unexpected_group) + pipeline = module.LingBotWorldCausalDMDPipeline(od_config=config) + assert pipeline._vae_shard_split_dim is None + + +@pytest.mark.parametrize("value", ["false", 0, None]) +def test_vae_sharding_switch_requires_a_boolean(value): + module = _load_pipeline_module() + config = _ulysses_config(4) + config.model_config["lingbot_vae_spatial_sharding"] = value + with pytest.raises(ValueError, match="lingbot_vae_spatial_sharding must be a boolean"): + module.LingBotWorldCausalDMDPipeline(od_config=config) + + +def test_vae_shard_refuses_a_group_of_the_wrong_size(monkeypatch) -> None: + module = _load_pipeline_module() + installs, _ = _record_shard_install(monkeypatch, module, world_size=4) + + with pytest.raises(RuntimeError, match="sequence_parallel_size=2 but the Ulysses group has 4 ranks"): + module.LingBotWorldCausalDMDPipeline(od_config=_ulysses_config(2)) + assert installs == [] + + @pytest.mark.parametrize( ("field", "value", "feature"), [ @@ -758,8 +851,9 @@ def test_unsupported_parallel_modes_fail_before_component_loading(field: str, va assert module._loader_state.prefetch_calls == [] -def test_pure_ulysses_parallel_config_is_supported() -> None: +def test_pure_ulysses_parallel_config_is_supported(monkeypatch) -> None: module = _load_pipeline_module() + _record_shard_install(monkeypatch, module, world_size=2) parallel_config = _od_config().parallel_config parallel_config.sequence_parallel_size = 2 parallel_config.ulysses_degree = 2 diff --git a/vllm_omni/deploy/lingbot_world_v2_stepwise.yaml b/vllm_omni/deploy/lingbot_world_v2_stepwise.yaml index 3fc950e4c..9a00b4d6c 100644 --- a/vllm_omni/deploy/lingbot_world_v2_stepwise.yaml +++ b/vllm_omni/deploy/lingbot_world_v2_stepwise.yaml @@ -34,6 +34,9 @@ stages: extra_args: flow_shift: 5.0 model_config: + # Multi-rank width sharding changes floating-point numerics at shard borders. + # Set false for full-frame VAE decode while keeping DiT sequence parallelism. + lingbot_vae_spatial_sharding: true ar_diffusion_height: 480 ar_diffusion_width: 832 ar_diffusion_kv_config: diff --git a/vllm_omni/diffusion/distributed/autoencoders/wan_spatial_shard.py b/vllm_omni/diffusion/distributed/autoencoders/wan_spatial_shard.py index 16fdd13a4..e3f0ef5c3 100644 --- a/vllm_omni/diffusion/distributed/autoencoders/wan_spatial_shard.py +++ b/vllm_omni/diffusion/distributed/autoencoders/wan_spatial_shard.py @@ -781,22 +781,34 @@ def _decoder_upsample_count(decoder: nn.Module) -> int: return count -def install_wan_spatial_shard_decode(vae: Any, group: dist.ProcessGroup, split_dim: str = "height") -> None: +def install_wan_spatial_shard_decode( + vae: Any, + group: dist.ProcessGroup, + split_dim: str = "height", + *, + dst: int | None = 0, +) -> None: """Patch ``vae.decoder`` once for spatially-sharded decode. This mutates the already-loaded decoder in place by swapping its spatial convolutions/padding for halo-exchanging variants and wrapping ``decoder.forward``. The patch is permanent for the lifetime of the VAE instance and is applied only once (subsequent calls are no-ops). A given - instance is bound to a single ``split_dim``; switching between - ``"height"`` and ``"width"`` requires a fresh VAE instance and raises here - otherwise. + instance is bound to a single ``split_dim`` and ``dst``; switching either + requires a fresh VAE instance and raises here otherwise. - Only group-relative rank 0 assembles the final decoded frame, mirroring the - distributed tiled-decode ``broadcast_result=False`` contract; the other ranks - take part in the collectives but return an empty placeholder. + With ``dst=0`` (the default) only group-relative rank 0 assembles the + final decoded frame, mirroring the distributed tiled-decode + ``broadcast_result=False`` contract; the other ranks take part in the + collectives but return an empty placeholder. ``dst=None`` lets every rank + keep the assembled frame, for a caller whose every rank goes on to + consume the decoded pixels; the gather is an all-gather either way, so + this costs no extra communication. """ _spatial_dim(split_dim) + _, world_size = _rank_world(group) + if dst is not None and (not isinstance(dst, int) or not 0 <= dst < world_size): + raise ValueError(f"Wan spatial-shard dst must be None or an integer in [0, {world_size}), got {dst!r}.") if getattr(vae, "_vllm_omni_wan_spatial_shard_installed", False): installed_split_dim = getattr(vae, "_vllm_omni_wan_spatial_shard_split_dim", "height") if installed_split_dim != split_dim: @@ -804,6 +816,12 @@ def install_wan_spatial_shard_decode(vae: Any, group: dist.ProcessGroup, split_d "Wan spatial-shard VAE decoder was already patched for " f"{installed_split_dim!r} split; create a fresh VAE instance to use {split_dim!r} split." ) + installed_dst = getattr(vae, "_vllm_omni_wan_spatial_shard_dst", 0) + if installed_dst != dst: + raise ValueError( + "Wan spatial-shard VAE decoder was already patched to assemble on " + f"{installed_dst!r}; create a fresh VAE instance to assemble on {dst!r}." + ) return decoder = getattr(vae, "decoder", None) if decoder is None: @@ -844,11 +862,12 @@ def install_wan_spatial_shard_decode(vae: Any, group: dist.ProcessGroup, split_d out = orig_forward(x, feat_cache=feat_cache, feat_idx=feat_idx, first_chunk=first_chunk) finally: _SPATIAL_SHARD_CONTEXT.reset(token) - return gather_and_trim_extent(out, expected_extent=expected_extent, split_dim=split_dim, group=group, dst=0) + return gather_and_trim_extent(out, expected_extent=expected_extent, split_dim=split_dim, group=group, dst=dst) decoder.forward = MethodType(_forward, decoder) vae._vllm_omni_wan_spatial_shard_installed = True vae._vllm_omni_wan_spatial_shard_split_dim = split_dim + vae._vllm_omni_wan_spatial_shard_dst = dst logger.info("Installed Wan VAE %s-sharded decode.", split_dim) diff --git a/vllm_omni/diffusion/models/lingbot_world/pipeline.py b/vllm_omni/diffusion/models/lingbot_world/pipeline.py index ae24c0138..8bb83ef80 100644 --- a/vllm_omni/diffusion/models/lingbot_world/pipeline.py +++ b/vllm_omni/diffusion/models/lingbot_world/pipeline.py @@ -26,6 +26,7 @@ from vllm.model_executor.models.utils import AutoWeightsLoader from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_wan import DistributedAutoencoderKLWan +from vllm_omni.diffusion.distributed.autoencoders.wan_spatial_shard import install_wan_spatial_shard_decode from vllm_omni.diffusion.distributed.utils import get_local_device from vllm_omni.diffusion.interaction.mixin import InteractionMixin from vllm_omni.diffusion.interaction.modality_handlers.camera import CameraSession @@ -92,6 +93,10 @@ logger = init_logger(__name__) # measured to hold. _STREAMING_DECODE_BYTES_PER_PIXEL_FP32 = 37832 * 1024 / (64 * 64) +# The frame dimension the decoder is sharded along when the pipeline runs on +# more than one Ulysses rank; see ``_install_sharded_vae_decode``. +_VAE_SHARD_SPLIT_DIM = "width" + if TYPE_CHECKING: from tqdm.std import tqdm as TqdmProgressBar @@ -580,6 +585,7 @@ class LingBotWorldCausalDMDPipeline( super().__init__() del prefix _validate_parallel_config(od_config) + parallel_config = getattr(od_config, "parallel_config", None) self.od_config = od_config model_config = getattr(od_config, "model_config", None) or {} reuse_last_step_kv = model_config.get("lingbot_reuse_last_step_kv", False) @@ -683,6 +689,14 @@ class LingBotWorldCausalDMDPipeline( self._streaming_decode_states: dict[str, StreamingDecodeState] = {} self._cached_streaming_decoder: WanStreamingDecoder | None = None self._streaming_decode_unsupported = False + self._vae_shard_split_dim: str | None = None + self._vae_shard_world_size = 1 + vae_sharding = model_config.get("lingbot_vae_spatial_sharding", True) + if not isinstance(vae_sharding, bool): + raise ValueError("lingbot_vae_spatial_sharding must be a boolean.") + sequence_parallel_size = int(getattr(parallel_config, "sequence_parallel_size", 1) or 1) + if sequence_parallel_size > 1 and vae_sharding: + self._install_sharded_vae_decode(sequence_parallel_size) self.setup_diffusion_pipeline_profiler( profiler_targets=[ "vae.encode", @@ -803,9 +817,24 @@ class LingBotWorldCausalDMDPipeline( decoder = self._streaming_decoder() if decoder is None: return 0 + height, width = self._ar_height, self._ar_width + if self._vae_shard_split_dim is not None: + # The decoder pads the latent extent to a multiple of the group + # size before splitting. Account for that padding on every rank, + # including ranks whose final pixels are trimmed after gathering. + scale = self.vae_scale_factor_spatial + extent = width if self._vae_shard_split_dim == "width" else height + latent_extent = (extent + scale - 1) // scale + local_extent = (latent_extent + self._vae_shard_world_size - 1) // self._vae_shard_world_size + if self._vae_shard_split_dim == "width": + width = local_extent * scale + else: + height = local_extent * scale + # Temporal caches retain local inputs, before spatial halo exchange. + # Halo receive buffers belong to the decoder, not to each session. return decoder.declared_state_bytes( - height=self._ar_height, - width=self._ar_width, + height=height, + width=width, dtype=self.vae.dtype, ) @@ -1659,6 +1688,51 @@ class LingBotWorldCausalDMDPipeline( tile_latent_min_width = int(getattr(self.vae, "tile_sample_min_width", None) or 0) // compression return bool(latents.shape[-2] > tile_latent_min_height or latents.shape[-1] > tile_latent_min_width) + def _vae_shard_group(self) -> tuple[Any, int]: + """The Ulysses ranks the decoder is sharded across, and how many there are.""" + from vllm_omni.diffusion.distributed.parallel_state import get_sp_group + + group = get_sp_group().device_group + return group, torch.distributed.get_world_size(group=group) + + def _install_sharded_vae_decode(self, sequence_parallel_size: int) -> None: + """Shard the Wan decoder across the Ulysses ranks along the width. + + The DiT runs on ``sequence_parallel_size`` ranks, and the decode of + one AR block used to run on every one of them in full: each rank + decoded the whole frame so that its own ``post_decode`` could carry + on. Splitting the frame across those same ranks with halo exchange at + the convolution borders makes each decode a quarter of the work at + four ranks; every rank still ends the block with the assembled frame + because the gather is an all-gather, so nothing downstream changes. + The streaming decoder is unaffected: ``feat_cache``, ``feat_idx`` and + ``first_chunk`` pass straight through to the patched decoder, and the + per-session temporal cache shrinks by the same factor. + + This is the default for multi-rank deployments; setting + ``model_config.lingbot_vae_spatial_sharding=False`` keeps full-frame + decode on each rank without changing DiT parallelism. + ``vae_patch_parallel_size`` stays rejected because it would make the + registry set up the tiled distributed executor and force + tiling on a decoder that is already sharded. Width rather than height + because at 480x832 it is the longer side, so each rank keeps a + 208-pixel shard against the same halo, and it is the split this was + measured on: 269.9 -> 98.7 ms per block on 4xH200. Numerically it is + not bit-exact against a single-rank decode: the halo-exchanged + convolutions accumulate in a different order at the shard borders + (max abs diff 0.0195 on [-1, 1] at 480x832). + """ + group, world_size = self._vae_shard_group() + if world_size != sequence_parallel_size: + raise RuntimeError( + f"LingBot World sequence_parallel_size={sequence_parallel_size} but the Ulysses group has " + f"{world_size} ranks." + ) + install_wan_spatial_shard_decode(self.vae, group, split_dim=_VAE_SHARD_SPLIT_DIM, dst=None) + self._vae_shard_split_dim = _VAE_SHARD_SPLIT_DIM + self._vae_shard_world_size = world_size + logger.info("LingBot World VAE decode sharded along %s across %d ranks.", _VAE_SHARD_SPLIT_DIM, world_size) + def _streaming_decoder(self) -> WanStreamingDecoder | None: """The session-owned streaming decoder, or ``None`` on a VAE that cannot stream. From 5e28163ba5b329f5bed36272aca2f9f65e53ef88 Mon Sep 17 00:00:00 2001 From: zhumingjue138 Date: Mon, 21 Sep 2026 19:03:18 +0800 Subject: [PATCH 2/5] [Test] add stability test case for High-priority model (#7571) Signed-off-by: zhumingjue --- .../references/production-validation.md | 2 +- docs/contributing/ci/test_execution_guide.md | 4 +- docs/contributing/ci/test_writing_guide.md | 4 +- ...mage.py => run_stability_hunyuan_image.py} | 6 +- .../scripts/run_stability_hunyuanvideo15.py | 70 ++++++++++++++ .../scripts/run_stability_minicpmo_4_5.py | 71 +++++++++++++++ .../scripts/run_stability_minimax_h3.py | 91 +++++++++++++++++++ ...n3_omni.py => run_stability_qwen3_omni.py} | 6 +- ...wen3_tts.py => run_stability_qwen3_tts.py} | 6 +- ...n_image.py => run_stability_qwen_image.py} | 6 +- ...bility_wan22.py => run_stability_wan22.py} | 6 +- .../stability/tests/test_hunyuan_image.json | 51 ++++++++++- .../stability/tests/test_hunyuanvideo15.json | 57 ++++++++++++ .../stability/tests/test_minicpmo_4_5.json | 53 +++++++++++ .../dfx/stability/tests/test_minimax_h3.json | 62 +++++++++++++ .../dfx/stability/tests/test_qwen3_omni.json | 4 +- tests/dfx/stability/tests/test_qwen3_tts.json | 1 + .../dfx/stability/tests/test_qwen_image.json | 40 ++++++++ tests/dfx/stability/tests/test_wan22.json | 2 + tools/nightly/run_nightly_jobs.sh | 38 ++++++-- 20 files changed, 557 insertions(+), 23 deletions(-) rename tests/dfx/stability/scripts/{test_stability_hunyuan_image.py => run_stability_hunyuan_image.py} (92%) create mode 100644 tests/dfx/stability/scripts/run_stability_hunyuanvideo15.py create mode 100644 tests/dfx/stability/scripts/run_stability_minicpmo_4_5.py create mode 100644 tests/dfx/stability/scripts/run_stability_minimax_h3.py rename tests/dfx/stability/scripts/{test_stability_qwen3_omni.py => run_stability_qwen3_omni.py} (92%) rename tests/dfx/stability/scripts/{test_stability_qwen3_tts.py => run_stability_qwen3_tts.py} (92%) rename tests/dfx/stability/scripts/{test_stability_qwen_image.py => run_stability_qwen_image.py} (92%) rename tests/dfx/stability/scripts/{test_stability_wan22.py => run_stability_wan22.py} (92%) create mode 100644 tests/dfx/stability/tests/test_hunyuanvideo15.json create mode 100644 tests/dfx/stability/tests/test_minicpmo_4_5.json create mode 100644 tests/dfx/stability/tests/test_minimax_h3.json diff --git a/.claude/skills/production-add-diffusion-model/references/production-validation.md b/.claude/skills/production-add-diffusion-model/references/production-validation.md index eed568415..3b2f505fa 100644 --- a/.claude/skills/production-add-diffusion-model/references/production-validation.md +++ b/.claude/skills/production-add-diffusion-model/references/production-validation.md @@ -340,7 +340,7 @@ On supported GPU hosts, wrap a test or soak with the repository monitor: ```bash bash tests/dfx/stability/scripts/resource_monitor.sh run --backend gpu -- \ - pytest -s -v tests/dfx/stability/scripts/test_stability_.py -m slow + pytest -s -v tests/dfx/stability/scripts/run_stability_.py -m slow ``` Check the target revision's monitor backends. Do not claim CPU/NPU monitoring diff --git a/docs/contributing/ci/test_execution_guide.md b/docs/contributing/ci/test_execution_guide.md index 7597f72e4..d3b02d27f 100644 --- a/docs/contributing/ci/test_execution_guide.md +++ b/docs/contributing/ci/test_execution_guide.md @@ -177,10 +177,10 @@ Failed jobs: 1/2 cd tests # Stability: Qwen3-Omni - pytest -s -v dfx/stability/scripts/test_stability_qwen3_omni.py -m slow + pytest -s -v dfx/stability/scripts/run_stability_qwen3_omni.py -m slow # Stability: Wan2.2 (v1/videos diffusion benchmark loop) - pytest -s -v dfx/stability/scripts/test_stability_wan22.py -m slow + pytest -s -v dfx/stability/scripts/run_stability_wan22.py -m slow # Reliability: Qwen3-Omni (H100 × 2) pytest -s -v dfx/reliability/test_reliability_qwen3_omni.py -m slow diff --git a/docs/contributing/ci/test_writing_guide.md b/docs/contributing/ci/test_writing_guide.md index 762863bb3..47a629112 100644 --- a/docs/contributing/ci/test_writing_guide.md +++ b/docs/contributing/ci/test_writing_guide.md @@ -555,7 +555,7 @@ L5 level testing focuses on the performance of model services under ***long-runn - ***Trigger Timing***: **`Weekly`** (weekly) or **`Days before Release`** (several days before a major release). Due to long execution times, the frequency is lower. - ***Run Command***: - - ***Stability***: `pytest -s -v tests/dfx/stability/scripts/test_stability_qwen3_omni.py` or `pytest -s -v tests/dfx/stability/scripts/test_stability_wan22.py` (or add `test_stability_.py` alongside a matching JSON config) + - ***Stability***: `pytest -s -v tests/dfx/stability/scripts/run_stability_qwen3_omni.py` or `pytest -s -v tests/dfx/stability/scripts/run_stability_wan22.py` (or add `run_stability_.py` alongside a matching JSON config) - ***Reliability***: `pytest -s -v tests/dfx/reliability/test_reliability_.py -m slow` (current suites: `qwen3_omni`, `wan22`, `hunyuan_image`). - ***Script Example***: @@ -564,7 +564,7 @@ L5 level testing focuses on the performance of model services under ***long-runn ##### Stability -When you want to add L5-level stability test cases, add or extend the appropriate JSON file under `tests/dfx/stability/tests/` (for example `test_qwen3_omni.json` for Omni bench traffic, or `test_wan22.json` for diffusion `/v1/videos` workloads). Pair the JSON with `tests/dfx/stability/scripts/test_stability_.py`. The following illustrates the Qwen3-Omni shape: +When you want to add L5-level stability test cases, add or extend the appropriate JSON file under `tests/dfx/stability/tests/` (for example `test_qwen3_omni.json` for Omni bench traffic, or `test_wan22.json` for diffusion `/v1/videos` workloads). Pair the JSON with `tests/dfx/stability/scripts/run_stability_.py`. The following illustrates the Qwen3-Omni shape: ```json { diff --git a/tests/dfx/stability/scripts/test_stability_hunyuan_image.py b/tests/dfx/stability/scripts/run_stability_hunyuan_image.py similarity index 92% rename from tests/dfx/stability/scripts/test_stability_hunyuan_image.py rename to tests/dfx/stability/scripts/run_stability_hunyuan_image.py index be2aec805..d9db77205 100644 --- a/tests/dfx/stability/scripts/test_stability_hunyuan_image.py +++ b/tests/dfx/stability/scripts/run_stability_hunyuan_image.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + """ Hunyuan-Image stability: OmniServer (diffusion) + ``diffusion_benchmark_serving.py``. @@ -34,8 +37,7 @@ server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) i benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping) -@pytest.mark.slow -@pytest.mark.diffusion +# Pytest marks (hardware, local_model, slow, diffusion) come from test_hunyuan_image.json. @pytest.mark.parametrize("omni_server", test_params, indirect=True) @pytest.mark.parametrize("stability_benchmark_params", benchmark_indices, indirect=True) def test_stability_hunyuan_image(omni_server, stability_benchmark_params): diff --git a/tests/dfx/stability/scripts/run_stability_hunyuanvideo15.py b/tests/dfx/stability/scripts/run_stability_hunyuanvideo15.py new file mode 100644 index 000000000..a0e7dfce8 --- /dev/null +++ b/tests/dfx/stability/scripts/run_stability_hunyuanvideo15.py @@ -0,0 +1,70 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +""" +HunyuanVideo-1.5 I2V stability: OmniServer (diffusion) + ``diffusion_benchmark_serving.py`` / ``v1/videos``. + +Configuration: ``tests/dfx/stability/tests/test_hunyuanvideo15.json``. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.dfx.conftest import ( + create_benchmark_indices, + create_test_parameter_mapping, + create_unique_server_pytest_params, + load_configs, +) +from tests.dfx.stability.helpers import _run_one_diffusion_batch, run_stability_benchmark_loop + +STABILITY_DIR = Path(__file__).resolve().parent.parent +DEPLOY_CONFIGS_DIR = STABILITY_DIR / "deploy" +CONFIG_FILE_PATH = str(STABILITY_DIR / "tests" / "test_hunyuanvideo15.json") +DEFAULT_NUM_PROMPTS_PER_BATCH = 50 +STABILITY_SERVER_TIMEOUT_ARGS = ["--stage-init-timeout", "600", "--init-timeout", "900"] + +try: + BENCHMARK_CONFIGS = load_configs(CONFIG_FILE_PATH) +except FileNotFoundError: + BENCHMARK_CONFIGS = [] + +test_params = create_unique_server_pytest_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR) if BENCHMARK_CONFIGS else [] +server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) if BENCHMARK_CONFIGS else {} +benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping) + + +# Pytest marks (hardware, local_model, slow, diffusion) come from test_hunyuanvideo15.json. +@pytest.mark.parametrize("omni_server", test_params, indirect=True) +@pytest.mark.parametrize("stability_benchmark_params", benchmark_indices, indirect=True) +def test_stability_hunyuanvideo15(omni_server, stability_benchmark_params): + test_name = stability_benchmark_params["test_name"] + params = stability_benchmark_params["params"] + duration_sec = params.get("duration_sec", 300) + num_prompts_per_batch = params.get("num_prompts_per_batch", DEFAULT_NUM_PROMPTS_PER_BATCH) + request_rate = params.get("request_rate") + max_concurrency = params.get("max_concurrency") + + bench_params = { + k: v + for k, v in params.items() + if k not in ("duration_sec", "request_rate", "max_concurrency", "num_prompts_per_batch") + } + + result = run_stability_benchmark_loop( + host=omni_server.host, + port=omni_server.port, + model=omni_server.model, + duration_sec=duration_sec, + params=bench_params, + request_rate=request_rate, + max_concurrency=max_concurrency, + result_dir=str(STABILITY_DIR), + num_prompts_per_batch=num_prompts_per_batch, + run_one_batch=_run_one_diffusion_batch, + ) + + assert result.get("failed", 0) == 0, f"[{test_name}] Failed requests detected: {result.get('errors', [])}" + assert result.get("completed", 0) > 0, f"[{test_name}] No requests completed" diff --git a/tests/dfx/stability/scripts/run_stability_minicpmo_4_5.py b/tests/dfx/stability/scripts/run_stability_minicpmo_4_5.py new file mode 100644 index 000000000..fcb0590ce --- /dev/null +++ b/tests/dfx/stability/scripts/run_stability_minicpmo_4_5.py @@ -0,0 +1,71 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +""" +MiniCPM-o-4.5 stability: OmniServer + ``vllm bench serve --omni`` for a fixed duration. + +Configuration: ``tests/dfx/stability/tests/test_minicpmo_4_5.json``. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.dfx.conftest import ( + create_benchmark_indices, + create_test_parameter_mapping, + create_unique_server_pytest_params, + load_configs, +) +from tests.dfx.stability.helpers import _run_one_vllm_bench_batch, run_stability_benchmark_loop + +STABILITY_DIR = Path(__file__).resolve().parent.parent +DEPLOY_CONFIGS_DIR = STABILITY_DIR / "deploy" +CONFIG_FILE_PATH = str(STABILITY_DIR / "tests" / "test_minicpmo_4_5.json") +DEFAULT_NUM_PROMPTS_PER_BATCH = 20 +STABILITY_SERVER_TIMEOUT_ARGS = ["--stage-init-timeout", "600"] + +try: + BENCHMARK_CONFIGS = load_configs(CONFIG_FILE_PATH) +except FileNotFoundError: + BENCHMARK_CONFIGS = [] + +test_params = create_unique_server_pytest_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR) if BENCHMARK_CONFIGS else [] +server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) if BENCHMARK_CONFIGS else {} +benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping) + + +# Pytest marks (hardware, local_model, slow, omni) come from test_minicpmo_4_5.json. +@pytest.mark.parametrize("omni_server", test_params, indirect=True) +@pytest.mark.parametrize("stability_benchmark_params", benchmark_indices, indirect=True) +def test_stability_minicpmo_4_5(omni_server, stability_benchmark_params): + test_name = stability_benchmark_params["test_name"] + params = stability_benchmark_params["params"] + duration_sec = params.get("duration_sec", 300) + num_prompts_per_batch = params.get("num_prompts_per_batch", DEFAULT_NUM_PROMPTS_PER_BATCH) + request_rate = params.get("request_rate") + max_concurrency = params.get("max_concurrency") + + bench_params = { + k: v + for k, v in params.items() + if k not in ("duration_sec", "request_rate", "max_concurrency", "num_prompts_per_batch") + } + + result = run_stability_benchmark_loop( + host=omni_server.host, + port=omni_server.port, + model=omni_server.model, + duration_sec=duration_sec, + params=bench_params, + request_rate=request_rate, + max_concurrency=max_concurrency, + result_dir=str(STABILITY_DIR), + num_prompts_per_batch=num_prompts_per_batch, + run_one_batch=_run_one_vllm_bench_batch, + ) + + assert result.get("failed", 0) == 0, f"[{test_name}] Failed requests detected: {result.get('errors', [])}" + assert result.get("completed", 0) > 0, f"[{test_name}] No requests completed" diff --git a/tests/dfx/stability/scripts/run_stability_minimax_h3.py b/tests/dfx/stability/scripts/run_stability_minimax_h3.py new file mode 100644 index 000000000..252d830d5 --- /dev/null +++ b/tests/dfx/stability/scripts/run_stability_minimax_h3.py @@ -0,0 +1,91 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +""" +MiniMax-H3 T2V stability: OmniServer (diffusion) + ``diffusion_benchmark_serving.py`` / ``v1/videos``. + +Configuration: ``tests/dfx/stability/tests/test_minimax_h3.json``. + +``--lora-path`` is resolved like e2e FastH3 / Buildkite ``test-merge.yml``: +set ``VLLM_TEST_MINIMAX_H3_FASTH3_LORA`` to the adapter file, or populate the local +HF cache for ``FastVideo/FastVideo-FastH3-4-step-Preview-v1-LoRA``. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.dfx.conftest import ( + create_benchmark_indices, + create_test_parameter_mapping, + create_unique_server_pytest_params, + load_configs, +) +from tests.dfx.stability.helpers import _run_one_diffusion_batch, run_stability_benchmark_loop +from tests.e2e.online_serving.minimax_h3._common import FASTH3_LORA + +STABILITY_DIR = Path(__file__).resolve().parent.parent +DEPLOY_CONFIGS_DIR = STABILITY_DIR / "deploy" +CONFIG_FILE_PATH = str(STABILITY_DIR / "tests" / "test_minimax_h3.json") +DEFAULT_NUM_PROMPTS_PER_BATCH = 20 +STABILITY_SERVER_TIMEOUT_ARGS = ["--stage-init-timeout", "1800", "--init-timeout", "1800"] + + +def _inject_fasth3_lora_path(configs: list[dict]) -> list[dict]: + """Fill serve_args.lora-path from VLLM_TEST_MINIMAX_H3_FASTH3_LORA / HF cache.""" + if not FASTH3_LORA: + return configs + for config in configs: + serve_args = config.setdefault("server_params", {}).setdefault("serve_args", {}) + serve_args["lora-path"] = FASTH3_LORA + return configs + + +try: + BENCHMARK_CONFIGS = _inject_fasth3_lora_path(load_configs(CONFIG_FILE_PATH)) +except FileNotFoundError: + BENCHMARK_CONFIGS = [] + +test_params = create_unique_server_pytest_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR) if BENCHMARK_CONFIGS else [] +server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) if BENCHMARK_CONFIGS else {} +benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping) + + +# Pytest marks (hardware, local_model, slow, diffusion) come from test_minimax_h3.json. +@pytest.mark.skipif( + FASTH3_LORA is None, + reason="set VLLM_TEST_MINIMAX_H3_FASTH3_LORA or populate the local HF cache", +) +@pytest.mark.parametrize("omni_server", test_params, indirect=True) +@pytest.mark.parametrize("stability_benchmark_params", benchmark_indices, indirect=True) +def test_stability_minimax_h3(omni_server, stability_benchmark_params): + test_name = stability_benchmark_params["test_name"] + params = stability_benchmark_params["params"] + duration_sec = params.get("duration_sec", 300) + num_prompts_per_batch = params.get("num_prompts_per_batch", DEFAULT_NUM_PROMPTS_PER_BATCH) + request_rate = params.get("request_rate") + max_concurrency = params.get("max_concurrency") + + bench_params = { + k: v + for k, v in params.items() + if k not in ("duration_sec", "request_rate", "max_concurrency", "num_prompts_per_batch") + } + + result = run_stability_benchmark_loop( + host=omni_server.host, + port=omni_server.port, + model=omni_server.model, + duration_sec=duration_sec, + params=bench_params, + request_rate=request_rate, + max_concurrency=max_concurrency, + result_dir=str(STABILITY_DIR), + num_prompts_per_batch=num_prompts_per_batch, + run_one_batch=_run_one_diffusion_batch, + ) + + assert result.get("failed", 0) == 0, f"[{test_name}] Failed requests detected: {result.get('errors', [])}" + assert result.get("completed", 0) > 0, f"[{test_name}] No requests completed" diff --git a/tests/dfx/stability/scripts/test_stability_qwen3_omni.py b/tests/dfx/stability/scripts/run_stability_qwen3_omni.py similarity index 92% rename from tests/dfx/stability/scripts/test_stability_qwen3_omni.py rename to tests/dfx/stability/scripts/run_stability_qwen3_omni.py index 98ab5ee89..ddb3339d1 100644 --- a/tests/dfx/stability/scripts/test_stability_qwen3_omni.py +++ b/tests/dfx/stability/scripts/run_stability_qwen3_omni.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + """ Qwen3-Omni stability: OmniServer + ``vllm bench serve --omni`` for a fixed duration. @@ -34,8 +37,7 @@ server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) i benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping) -@pytest.mark.slow -@pytest.mark.omni +# Pytest marks (hardware, local_model, slow, omni) come from test_qwen3_omni.json. @pytest.mark.parametrize("omni_server", test_params, indirect=True) @pytest.mark.parametrize("stability_benchmark_params", benchmark_indices, indirect=True) def test_stability_qwen3_omni(omni_server, stability_benchmark_params): diff --git a/tests/dfx/stability/scripts/test_stability_qwen3_tts.py b/tests/dfx/stability/scripts/run_stability_qwen3_tts.py similarity index 92% rename from tests/dfx/stability/scripts/test_stability_qwen3_tts.py rename to tests/dfx/stability/scripts/run_stability_qwen3_tts.py index 293b723ca..dde525b36 100644 --- a/tests/dfx/stability/scripts/test_stability_qwen3_tts.py +++ b/tests/dfx/stability/scripts/run_stability_qwen3_tts.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + """ Qwen3-TTS stability: OmniServer + ``vllm bench serve --omni`` for a fixed duration. @@ -34,8 +37,7 @@ server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) i benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping) -@pytest.mark.slow -@pytest.mark.tts +# Pytest marks (hardware, local_model, slow, tts) come from test_qwen3_tts.json. @pytest.mark.parametrize("omni_server", test_params, indirect=True) @pytest.mark.parametrize("stability_benchmark_params", benchmark_indices, indirect=True) def test_stability_qwen3_tts(omni_server, stability_benchmark_params): diff --git a/tests/dfx/stability/scripts/test_stability_qwen_image.py b/tests/dfx/stability/scripts/run_stability_qwen_image.py similarity index 92% rename from tests/dfx/stability/scripts/test_stability_qwen_image.py rename to tests/dfx/stability/scripts/run_stability_qwen_image.py index 62b0cd86c..84e15fccb 100644 --- a/tests/dfx/stability/scripts/test_stability_qwen_image.py +++ b/tests/dfx/stability/scripts/run_stability_qwen_image.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + """ Qwen-Image stability: OmniServer (diffusion) + ``diffusion_benchmark_serving.py``. @@ -34,8 +37,7 @@ server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) i benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping) -@pytest.mark.slow -@pytest.mark.diffusion +# Pytest marks (hardware, local_model, slow, diffusion) come from test_qwen_image.json. @pytest.mark.parametrize("omni_server", test_params, indirect=True) @pytest.mark.parametrize("stability_benchmark_params", benchmark_indices, indirect=True) def test_stability_qwen_image(omni_server, stability_benchmark_params): diff --git a/tests/dfx/stability/scripts/test_stability_wan22.py b/tests/dfx/stability/scripts/run_stability_wan22.py similarity index 92% rename from tests/dfx/stability/scripts/test_stability_wan22.py rename to tests/dfx/stability/scripts/run_stability_wan22.py index aae0ae429..08c711d85 100644 --- a/tests/dfx/stability/scripts/test_stability_wan22.py +++ b/tests/dfx/stability/scripts/run_stability_wan22.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + """ Wan2.2 T2V stability: OmniServer (diffusion) + ``diffusion_benchmark_serving.py`` / ``v1/videos``. @@ -34,8 +37,7 @@ server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS) i benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping) -@pytest.mark.slow -@pytest.mark.diffusion +# Pytest marks (hardware, local_model, slow, diffusion) come from test_wan22.json. @pytest.mark.parametrize("omni_server", test_params, indirect=True) @pytest.mark.parametrize("stability_benchmark_params", benchmark_indices, indirect=True) def test_stability_wan22(omni_server, stability_benchmark_params): diff --git a/tests/dfx/stability/tests/test_hunyuan_image.json b/tests/dfx/stability/tests/test_hunyuan_image.json index b468ceec3..43c92d1e2 100644 --- a/tests/dfx/stability/tests/test_hunyuan_image.json +++ b/tests/dfx/stability/tests/test_hunyuan_image.json @@ -7,10 +7,11 @@ "res": { "cuda": "H800" }, - "num_cards": 2 + "num_cards": 4 } }, "local_model", + "slow", "diffusion" ], "server_params": { @@ -44,5 +45,53 @@ "enable_negative_prompt": true } ] + }, + { + "test_name": "test_hunyuan_image_stability_tp2_sp2_a3", + "description": "A3: same as perf test_hunyuan_image_tp2_sp2 — DiT-only, TP=2 USP=2, 4 dies, no fp8", + "mark": [ + { + "hardware_marks": { + "res": { + "npu": "A3" + }, + "num_cards": 4 + } + }, + "local_model", + "slow", + "diffusion" + ], + "server_params": { + "model": "tencent/HunyuanImage-3.0-Instruct", + "serve_args": { + "deploy-config": "vllm_omni/deploy/hunyuan_image3_dit.yaml", + "tensor-parallel-size": 2, + "ulysses-degree": 2, + "vae-patch-parallel-size": 4, + "vae-use-tiling": true, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "dataset": "random", + "task": "t2i", + "backend": "vllm-omni", + "duration_sec": 86400, + "max_concurrency": 1, + "num_prompts_per_batch": 50, + "width": { + "min": 512, + "max": 1024 + }, + "height": { + "min": 512, + "max": 1024 + }, + "num_inference_steps": 50, + "enable_negative_prompt": true + } + ] } ] diff --git a/tests/dfx/stability/tests/test_hunyuanvideo15.json b/tests/dfx/stability/tests/test_hunyuanvideo15.json new file mode 100644 index 000000000..1da2feec3 --- /dev/null +++ b/tests/dfx/stability/tests/test_hunyuanvideo15.json @@ -0,0 +1,57 @@ +[ + { + "test_name": "test_hunyuanvideo15_i2v_cachedit_tp2_vae_patch2_stability", + "description": "HunyuanVideo-1.5 I2V 24h: CacheDiT + TP=2 + VAE patch parallel=2 + VAE tiling", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": ["H800"] + }, + "num_cards": 2 + } + }, + "local_model", + "slow", + "diffusion" + ], + "server_params": { + "model": "hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v", + "serve_args": { + "flow-shift": 5.0, + "cache-backend": "cache_dit", + "tensor-parallel-size": 2, + "vae-patch-parallel-size": 2, + "vae-use-tiling": true, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "dataset": "random", + "task": "i2v", + "backend": "v1/videos", + "duration_sec": 86400, + "max_concurrency": 1, + "num_prompts_per_batch": 50, + "num_input_images": 1, + "seed": 42, + "enable_negative_prompt": true, + "random_request_config": [ + { + "width": 832, + "height": 480, + "num_inference_steps": 4, + "num_frames": 33, + "fps": 24, + "weight": 1 + } + ], + "extra_body": { + "guidance_scale": 6.0, + "flow_shift": 5.0 + } + } + ] + } +] diff --git a/tests/dfx/stability/tests/test_minicpmo_4_5.json b/tests/dfx/stability/tests/test_minicpmo_4_5.json new file mode 100644 index 000000000..eaf38ec1d --- /dev/null +++ b/tests/dfx/stability/tests/test_minicpmo_4_5.json @@ -0,0 +1,53 @@ +[ + { + "test_name": "test_minicpmo_4_5_stability_ready", + "description": "MiniCPM-o-4.5 ready deploy + seed-tts chat/audio stability (from perf test_minicpmo_4_5_ready)", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H800" + }, + "num_cards": 1 + } + }, + "local_model", + "slow", + "omni" + ], + "server_params": { + "model": "openbmb/MiniCPM-o-4_5", + "extra_cli_args": [ + "--deploy-config", + "vllm_omni/deploy/minicpmo_4_5.yaml", + "--trust-remote-code" + ] + }, + "benchmark_params": [ + { + "dataset_name": "seed-tts", + "dataset_path": "zhaochenyang20/seed-tts-eval", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "duration_sec": 86400, + "max_concurrency": 1, + "num_prompts_per_batch": 50, + "no_oversample": true, + "disable_shuffle": true, + "trust_remote_code": true, + "seed_tts_locale": "en", + "extra_body": { + "modalities": [ + "text", + "audio" + ], + "chat_template_kwargs": { + "enable_thinking": false, + "use_tts_template": true + } + }, + "percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration" + } + ] + } +] diff --git a/tests/dfx/stability/tests/test_minimax_h3.json b/tests/dfx/stability/tests/test_minimax_h3.json new file mode 100644 index 000000000..d671c48fd --- /dev/null +++ b/tests/dfx/stability/tests/test_minimax_h3.json @@ -0,0 +1,62 @@ +[ + { + "test_name": "test_minimax_h3_t2v_stability_4gpu", + "description": "MiniMax-H3 FastH3 T2VA on 4 GPUs (USP4 + HSDP4, encoder-TP4, VAE-PP4 tile); fused lora-path from VLLM_TEST_MINIMAX_H3_FASTH3_LORA; requests must use num_inference_steps=4", + "mark": [ + { + "hardware_marks": { + "res": { + "cuda": "H800" + }, + "num_cards": 4 + } + }, + "local_model", + "slow", + "diffusion" + ], + "server_params": { + "model": "MiniMaxAI/MiniMax-H3", + "serve_args": { + "trust-remote-code": true, + "task-type": "fl2va", + "num-gpus": 4, + "usp": 4, + "ring": 1, + "use-hsdp": true, + "hsdp-shard-size": 4, + "text-encoder-tp-size": 4, + "vae-patch-parallel-size": 4, + "vae-parallel-mode": "tile", + "vae-use-tiling": true, + "stage-init-timeout": 1800, + "init-timeout": 1800 + } + }, + "benchmark_params": [ + { + "dataset": "random", + "task": "t2v", + "backend": "v1/videos", + "duration_sec": 86400, + "max_concurrency": 1, + "num_prompts_per_batch": 50, + "seed": 42, + "random_request_config": [ + { + "width": 1344, + "height": 768, + "prompt": "Cinematic drone shot pushing through misty mountain ridges at sunrise, golden light catching the clouds, slow and smooth.", + "num_inference_steps": 4, + "num_frames": 209, + "fps": 24, + "weight": 1 + } + ], + "extra_body": { + "extra_params": "{\"task\":\"t2va\",\"duration\":8.7,\"aspect_ratio\":\"16:9\"}" + } + } + ] + } +] diff --git a/tests/dfx/stability/tests/test_qwen3_omni.json b/tests/dfx/stability/tests/test_qwen3_omni.json index 816e981b6..bc2f7dacd 100644 --- a/tests/dfx/stability/tests/test_qwen3_omni.json +++ b/tests/dfx/stability/tests/test_qwen3_omni.json @@ -5,12 +5,14 @@ { "hardware_marks": { "res": { - "cuda": "H800" + "cuda": "H800", + "npu": "A3" }, "num_cards": 2 } }, "local_model", + "slow", "omni" ], "server_params": { diff --git a/tests/dfx/stability/tests/test_qwen3_tts.json b/tests/dfx/stability/tests/test_qwen3_tts.json index 1c9356574..37b7cca52 100644 --- a/tests/dfx/stability/tests/test_qwen3_tts.json +++ b/tests/dfx/stability/tests/test_qwen3_tts.json @@ -12,6 +12,7 @@ } }, "local_model", + "slow", "tts" ], "server_params": { diff --git a/tests/dfx/stability/tests/test_qwen_image.json b/tests/dfx/stability/tests/test_qwen_image.json index 0c21d0ac1..9756f1299 100644 --- a/tests/dfx/stability/tests/test_qwen_image.json +++ b/tests/dfx/stability/tests/test_qwen_image.json @@ -11,6 +11,7 @@ } }, "local_model", + "slow", "diffusion" ], "server_params": { @@ -36,5 +37,44 @@ "enable_negative_prompt": true } ] + }, + { + "test_name": "test_qwen_image_stability_a3", + "mark": [ + { + "hardware_marks": { + "res": { + "npu": "A3" + }, + "num_cards": 1 + } + }, + "local_model", + "slow", + "diffusion" + ], + "server_params": { + "model": "Qwen/Qwen-Image" + }, + "benchmark_params": [ + { + "dataset": "random", + "task": "t2i", + "backend": "vllm-omni", + "duration_sec": 86400, + "max_concurrency": 1, + "num_prompts_per_batch": 50, + "width": { + "min": 512, + "max": 1000 + }, + "height": { + "min": 512, + "max": 1000 + }, + "num_inference_steps": 50, + "enable_negative_prompt": true + } + ] } ] diff --git a/tests/dfx/stability/tests/test_wan22.json b/tests/dfx/stability/tests/test_wan22.json index e6ab0b5cf..de6627de4 100644 --- a/tests/dfx/stability/tests/test_wan22.json +++ b/tests/dfx/stability/tests/test_wan22.json @@ -12,6 +12,7 @@ } }, "local_model", + "slow", "diffusion" ], "server_params": { @@ -54,6 +55,7 @@ } }, "local_model", + "slow", "diffusion" ], "server_params": { diff --git a/tools/nightly/run_nightly_jobs.sh b/tools/nightly/run_nightly_jobs.sh index c1cca73e2..8a0809726 100644 --- a/tools/nightly/run_nightly_jobs.sh +++ b/tools/nightly/run_nightly_jobs.sh @@ -47,7 +47,10 @@ # From repo root: pytest -s -v --run-level full_model -m "" tests/dfx/stability/scripts/... # -m comes from --pytest-mark / PYTEST_MARK when set (e.g. H800 or A3). # Otherwise -m defaults to H800. PYTEST_MARK is rejected unless stability is enabled. -# model_type: omni → qwen3_omni; tts → qwen3_tts; diffusion → qwen_image + wan22 + hunyuan_image; all → all five +# model_type: omni → qwen3_omni + minicpmo_4_5; tts → qwen3_tts; +# diffusion → qwen_image + wan22 + hunyuan_image + minimax_h3 + hunyuanvideo15; all → all eight +# stability_minimax_h3: download FastH3 LoRA only when VLLM_TEST_MINIMAX_H3_FASTH3_LORA is unset, +# then export before pytest (collection skipif needs the path). # LABEL_SUBSTR: if set, script path / job key / filename must contain it # # Requirements: bash, python3, PyYAML (pip install pyyaml) @@ -596,11 +599,14 @@ def _write_job_timeouts_manifest(jobs_dir: Path, job_timeouts: dict[str, int]) - # Fixed stability scripts (when stability is selected); MODEL_TYPES narrows which run. STABILITY_CASES: list[tuple[str, str, tuple[str, ...]]] = [ - ("stability_qwen3_omni", "tests/dfx/stability/scripts/test_stability_qwen3_omni.py", ("omni",)), - ("stability_qwen3_tts", "tests/dfx/stability/scripts/test_stability_qwen3_tts.py", ("tts",)), - ("stability_qwen_image", "tests/dfx/stability/scripts/test_stability_qwen_image.py", ("diffusion",)), - ("stability_wan22", "tests/dfx/stability/scripts/test_stability_wan22.py", ("diffusion",)), - ("stability_hunyuan_image", "tests/dfx/stability/scripts/test_stability_hunyuan_image.py", ("diffusion",)), + ("stability_qwen3_omni", "tests/dfx/stability/scripts/run_stability_qwen3_omni.py", ("omni",)), + ("stability_qwen3_tts", "tests/dfx/stability/scripts/run_stability_qwen3_tts.py", ("tts",)), + ("stability_minicpmo_4_5", "tests/dfx/stability/scripts/run_stability_minicpmo_4_5.py", ("omni",)), + ("stability_qwen_image", "tests/dfx/stability/scripts/run_stability_qwen_image.py", ("diffusion",)), + ("stability_wan22", "tests/dfx/stability/scripts/run_stability_wan22.py", ("diffusion",)), + ("stability_hunyuan_image", "tests/dfx/stability/scripts/run_stability_hunyuan_image.py", ("diffusion",)), + ("stability_minimax_h3", "tests/dfx/stability/scripts/run_stability_minimax_h3.py", ("diffusion",)), + ("stability_hunyuanvideo15", "tests/dfx/stability/scripts/run_stability_hunyuanvideo15.py", ("diffusion",)), ] @@ -612,6 +618,25 @@ def _stability_model_matches(model_types: list[str], families: tuple[str, ...]) STABILITY_DEFAULT_PYTEST_MARK = "H800" +# Extra shell lines before pytest. MiniMax-H3 FastH3 skipif reads +# VLLM_TEST_MINIMAX_H3_FASTH3_LORA at collection; merge CI downloads the same way. +_FASTH3_LORA_DOWNLOAD = ( + 'VLLM_TEST_MINIMAX_H3_FASTH3_LORA="$(python3 -c ' + "'from huggingface_hub import hf_hub_download; " + "print(hf_hub_download(repo_id=\"FastVideo/FastVideo-FastH3-4-step-Preview-v1-LoRA\", " + "filename=\"dense-datafree/adapter_model.safetensors\"))')\"" +) +STABILITY_JOB_PRELUDE: dict[str, list[str]] = { + "stability_minimax_h3": [ + "if [ -f vllm_omni/diffusion/models/minimax_h3/fasth3.py ]; then", + ' if [ -z "${VLLM_TEST_MINIMAX_H3_FASTH3_LORA:-}" ]; then', + f" {_FASTH3_LORA_DOWNLOAD}", + " fi", + " export VLLM_TEST_MINIMAX_H3_FASTH3_LORA", + "fi", + ], +} + def _shell_quote_pytest_mark(marker_expr: str) -> str: """Shell-safe pytest ``-m`` value for generated bash job scripts.""" @@ -839,6 +864,7 @@ def run_stability_mode(jobs_dir: Path, model_types: list[str]) -> int: + (" (--pytest-mark)" if mark_from_cli else f" (default {STABILITY_DEFAULT_PYTEST_MARK})"), "set -euo pipefail", f'cd "{REPO_ROOT}"', + *STABILITY_JOB_PRELUDE.get(key, []), pytest_line, ] _write_job_script(key, script_lines, jobs_dir) From e5c0afc99432dba767dcb66f28b0fb9aa43ab1af Mon Sep 17 00:00:00 2001 From: tsinghua-code Date: Mon, 21 Sep 2026 20:21:46 +0800 Subject: [PATCH 3/5] [Model] Enable online FP8 linears for LingBot World (#7549) Signed-off-by: tsinghua-code --- recipes/Robbyant/LingBot-World-2.0.md | 43 +++++++++++++++- .../test_lingbot_world_attention.py | 4 ++ .../test_lingbot_world_transformer.py | 51 +++++++++++++++++-- .../test_pipeline_lingbot_world.py | 9 ++-- tests/engine/test_arg_utils.py | 12 ++--- .../models/lingbot_world/pipeline.py | 2 - .../models/lingbot_world/transformer.py | 32 +++++++++--- 7 files changed, 129 insertions(+), 24 deletions(-) diff --git a/recipes/Robbyant/LingBot-World-2.0.md b/recipes/Robbyant/LingBot-World-2.0.md index 7add2f7d8..b20796726 100644 --- a/recipes/Robbyant/LingBot-World-2.0.md +++ b/recipes/Robbyant/LingBot-World-2.0.md @@ -213,6 +213,47 @@ accepts: `s`, `d`, `i`, `j`, `k`, and `l`; - `lingbot.camera_trajectory.v1` for explicit pose/intrinsics trajectories. +## Online FP8 linear layers (experimental) + +Pass `quantization_config={"method": "fp8"}` to the existing `AsyncOmni` +construction to use vLLM's online FP8 linear implementation. On the tested +Hopper-class GPU this dispatches to CUTLASS FP8 GEMM with online per-tensor +activation scaling; no LingBot-specific quantization kernel is introduced. + +Only the transformer's existing vLLM parallel linear layers are eligible: +self-attention, cross-attention, FFN, camera injectors, and C2WS projections. +The ordinary PyTorch linear layers (including the output head and +time/text embeddings), normalization, convolutions, VAE, text encoder, and +AR KV cache are not converted to FP8 by this option. + +Quality-sensitive projections can remain BF16 through fully qualified +`ignored_layers` names. For example, the following configuration tests +retaining the camera/C2WS path in BF16: + +```python +quantization_config = { + "method": "fp8", + "ignored_layers": [ + f"transformer.blocks.{i}.cam_injector_layer{j}" + for i in range(40) + for j in (1, 2) + ] + + [ + "transformer.c2ws_hidden_states_layer1", + "transformer.c2ws_hidden_states_layer2", + ], +} +``` + +Use `transformer.blocks..self_attn.qkv` to exclude the fused Q/K/V +projection as a unit. The example exclusion list is an ablation, not a +universal quality guarantee or a default. Validate the intended scene, seed, +camera trajectory, and session length against BF16 before selecting a policy. +An FP8 kernel speedup alone is not an end-to-end performance claim. +The long-session E8 quality gate in +[#7074](https://github.com/vllm-project/vllm-omni/issues/7074) remains required +for Tier-2 acceptance. + ## Validation Real-checkpoint validation uses 480x832 output, four DMD steps, and seed 42. @@ -246,7 +287,7 @@ tested commit. AR blocks in one request. `max_num_seqs` must be one in both cases. - Stateful streaming VAE decode is not implemented; the realtime example emits latent chunks. -- SP/USP, pipeline/CFG parallelism, HSDP, VAE parallelism, quantization, +- SP/USP, pipeline/CFG parallelism, HSDP, VAE parallelism, quantization methods other than online FP8, Cache-DiT, TeaCache, causal-pretrain, and the 1.3B checkpoint are not claimed. - No AMD GPU, Ascend NPU, or Intel GPU support is claimed. diff --git a/tests/diffusion/models/lingbot_world/test_lingbot_world_attention.py b/tests/diffusion/models/lingbot_world/test_lingbot_world_attention.py index 7e12b3db3..d2d11538c 100644 --- a/tests/diffusion/models/lingbot_world/test_lingbot_world_attention.py +++ b/tests/diffusion/models/lingbot_world/test_lingbot_world_attention.py @@ -134,6 +134,8 @@ def _install_vllm_stubs() -> None: **kwargs, ) -> None: super().__init__() + self.quant_config = kwargs.pop("quant_config", None) + self.prefix = kwargs.pop("prefix", "") del kwargs self.return_bias = return_bias self.weight = nn.Parameter(torch.empty(output_size, input_size)) @@ -164,6 +166,8 @@ def _install_vllm_stubs() -> None: **kwargs, ) -> None: super().__init__() + self.quant_config = kwargs.pop("quant_config", None) + self.prefix = kwargs.pop("prefix", "") del kwargs self.num_heads = total_num_heads self.num_kv_heads = total_num_kv_heads or total_num_heads diff --git a/tests/diffusion/models/lingbot_world/test_lingbot_world_transformer.py b/tests/diffusion/models/lingbot_world/test_lingbot_world_transformer.py index e11c34e35..7087e6171 100644 --- a/tests/diffusion/models/lingbot_world/test_lingbot_world_transformer.py +++ b/tests/diffusion/models/lingbot_world/test_lingbot_world_transformer.py @@ -35,6 +35,8 @@ def _tiny_model( num_layers: int = 2, num_frames_per_block: int = 1, sliding_window_num_frames: int = 3, + quant_config=None, + prefix: str = "", attention_head_dim: int = 2, rope_max_seq_len: int = 16, ): @@ -55,6 +57,8 @@ def _tiny_model( num_frames_per_block=num_frames_per_block, sliding_window_num_frames=sliding_window_num_frames, local_attn_size=-1, + quant_config=quant_config, + prefix=prefix, ) @@ -485,11 +489,52 @@ def test_constructor_rejects_non_null_image_embedding_fields(field: str) -> None @pytest.mark.cpu -def test_constructor_rejects_unsupported_quantization_with_runtime_error() -> None: +@pytest.mark.parametrize("prefix", ["", "transformer"]) +@pytest.mark.parametrize("quantized", [False, True]) +def test_constructor_propagates_quantization_to_parallel_linears(prefix: str, quantized: bool) -> None: module = attention_tests._load_module() + quant_config = object() if quantized else None + model = _tiny_model(module, num_layers=1, quant_config=quant_config, prefix=prefix) + block = model.blocks[0] + quantized_linears = [ + model.c2ws_hidden_states_layer1, + model.c2ws_hidden_states_layer2, + block.self_attn.qkv, + block.self_attn.o, + block.cross_attn.q, + block.cross_attn.k, + block.cross_attn.v, + block.cross_attn.o, + block.ffn[0], + block.ffn[2], + block.cam_injector_layer1, + block.cam_injector_layer2, + ] - with pytest.raises(RuntimeError, match="quant_config.*not supported"): - module.CausalLingBotWorldTransformer3DModel(quant_config=object()) + assert len(quantized_linears) == 12 + assert all(layer.quant_config is quant_config for layer in quantized_linears) + expected_names = { + "c2ws_hidden_states_layer1", + "c2ws_hidden_states_layer2", + *( + f"blocks.0.{name}" + for name in ( + "self_attn.qkv", + "self_attn.o", + "cross_attn.q", + "cross_attn.k", + "cross_attn.v", + "cross_attn.o", + "ffn.0", + "ffn.2", + "cam_injector_layer1", + "cam_injector_layer2", + ) + ), + } + assert {layer.prefix for layer in quantized_linears} == { + f"{prefix}.{name}" if prefix else name for name in expected_names + } @pytest.mark.cpu diff --git a/tests/diffusion/models/lingbot_world/test_pipeline_lingbot_world.py b/tests/diffusion/models/lingbot_world/test_pipeline_lingbot_world.py index b5a57ce75..8c24b8562 100644 --- a/tests/diffusion/models/lingbot_world/test_pipeline_lingbot_world.py +++ b/tests/diffusion/models/lingbot_world/test_pipeline_lingbot_world.py @@ -889,13 +889,14 @@ def test_unsupported_sp_config_fails_before_component_loading(overrides): assert module._loader_state.prefetch_calls == [] -def test_unsupported_quantization_fails_before_component_loading() -> None: +def test_quantization_reaches_transformer_factory() -> None: module = _load_pipeline_module() + quant_config = object() - with pytest.raises(NotImplementedError, match="quantization"): - module.LingBotWorldCausalDMDPipeline(od_config=_od_config(quantization_config=object())) + pipeline = module.LingBotWorldCausalDMDPipeline(od_config=_od_config(quantization_config=quant_config)) - assert module._loader_state.prefetch_calls == [] + assert pipeline.transformer is not None + assert _FakeTransformerFactory.last_call[1:] == (quant_config, "transformer") def test_official_scheduler_config_matches_fixed_dmd_contract() -> None: diff --git a/tests/engine/test_arg_utils.py b/tests/engine/test_arg_utils.py index d40a9c226..3b0192a79 100644 --- a/tests/engine/test_arg_utils.py +++ b/tests/engine/test_arg_utils.py @@ -563,19 +563,19 @@ def test_from_cli_args_preserves_text_encoder_tp_size(): def test_text_encoder_tp_size_reaches_default_diffusion_parallel_config(): """The preserved CLI value must land in DiffusionParallelConfig. - Mirrors the library flow: from_cli_args -> kwargs -> the generic - diffusion fallback, which resolves the field through - ``DiffusionParallelConfig.from_stage_overrides``. + Forward the preserved explicit override to the generic diffusion + fallback, which resolves it through + ``DiffusionParallelConfig.from_stage_overrides``. Serializing all engine + defaults would also forward unrelated LLM-only fields to strict diffusion + ingress, unlike the explicit-kwargs library entrypoint. """ - from dataclasses import asdict - from vllm_omni.config.config_factory import StageConfigFactory engine_args = OmniEngineArgs.from_cli_args( SimpleNamespace(text_encoder_tp_size=2), ) stage_cfg = StageConfigFactory.create_default_diffusion( - {k: v for k, v in asdict(engine_args).items() if v is not None}, + {"text_encoder_tp_size": engine_args.text_encoder_tp_size}, )[0] parallel_config = stage_cfg["engine_args"]["parallel_config"] diff --git a/vllm_omni/diffusion/models/lingbot_world/pipeline.py b/vllm_omni/diffusion/models/lingbot_world/pipeline.py index 8bb83ef80..fec175e69 100644 --- a/vllm_omni/diffusion/models/lingbot_world/pipeline.py +++ b/vllm_omni/diffusion/models/lingbot_world/pipeline.py @@ -241,8 +241,6 @@ def _validate_scheduler_config(config: dict[str, Any]) -> None: def _validate_parallel_config(od_config: OmniDiffusionConfig) -> None: - if getattr(od_config, "quantization_config", None) is not None: - raise NotImplementedError("LingBot World v1 does not support quantization.") parallel_config = getattr(od_config, "parallel_config", None) if parallel_config is None: return diff --git a/vllm_omni/diffusion/models/lingbot_world/transformer.py b/vllm_omni/diffusion/models/lingbot_world/transformer.py index 9a270e611..5b36299d8 100644 --- a/vllm_omni/diffusion/models/lingbot_world/transformer.py +++ b/vllm_omni/diffusion/models/lingbot_world/transformer.py @@ -8,7 +8,7 @@ import math from collections.abc import Iterable from dataclasses import dataclass from types import SimpleNamespace -from typing import Any, Self, cast +from typing import TYPE_CHECKING, Any, Self, cast import torch import torch.nn as nn @@ -36,6 +36,9 @@ from vllm_omni.experimental.ar_diffusion.kv_cache.paged_attention import ( paged_write_attn, ) +if TYPE_CHECKING: + from vllm.model_executor.layers.quantization.base_config import QuantizationConfig + @dataclass class LingBotAttentionCache: @@ -170,6 +173,7 @@ class LingBotSelfAttention(nn.Module): num_heads: int, *, eps: float = 1e-6, + quant_config: QuantizationConfig | None = None, prefix: str = "", ) -> None: super().__init__() @@ -188,6 +192,7 @@ class LingBotSelfAttention(nn.Module): head_size=self.head_dim, total_num_heads=num_heads, bias=True, + quant_config=quant_config, prefix=_projection_prefix(prefix, "qkv"), ) self.num_local_heads = self.qkv.num_heads @@ -205,6 +210,7 @@ class LingBotSelfAttention(nn.Module): bias=True, input_is_parallel=True, return_bias=False, + quant_config=quant_config, prefix=_projection_prefix(prefix, "o"), ) self.norm_q = _LingBotRMSNorm(self.tp_inner_dim, eps) @@ -416,6 +422,7 @@ class LingBotCrossAttention(nn.Module): num_heads: int, *, eps: float = 1e-6, + quant_config: QuantizationConfig | None = None, prefix: str = "", ) -> None: super().__init__() @@ -443,6 +450,7 @@ class LingBotCrossAttention(nn.Module): bias=True, gather_output=False, return_bias=False, + quant_config=quant_config, prefix=_projection_prefix(prefix, "q"), ) self.k = ColumnParallelLinear( @@ -451,6 +459,7 @@ class LingBotCrossAttention(nn.Module): bias=True, gather_output=False, return_bias=False, + quant_config=quant_config, prefix=_projection_prefix(prefix, "k"), ) self.v = ColumnParallelLinear( @@ -459,6 +468,7 @@ class LingBotCrossAttention(nn.Module): bias=True, gather_output=False, return_bias=False, + quant_config=quant_config, prefix=_projection_prefix(prefix, "v"), ) self.o = RowParallelLinear( @@ -467,6 +477,7 @@ class LingBotCrossAttention(nn.Module): bias=True, input_is_parallel=True, return_bias=False, + quant_config=quant_config, prefix=_projection_prefix(prefix, "o"), ) self.norm_q = _LingBotRMSNorm(self.tp_inner_dim, eps) @@ -528,6 +539,7 @@ class LingBotAttentionBlock(nn.Module): ffn_dim: int | None = None, cross_attn_norm: bool = True, eps: float = 1e-6, + quant_config: QuantizationConfig | None = None, prefix: str = "", ) -> None: super().__init__() @@ -538,12 +550,14 @@ class LingBotAttentionBlock(nn.Module): dim, num_heads, eps=eps, + quant_config=quant_config, prefix=_projection_prefix(prefix, "self_attn"), ) self.cross_attn = LingBotCrossAttention( dim, num_heads, eps=eps, + quant_config=quant_config, prefix=_projection_prefix(prefix, "cross_attn"), ) self.norm2 = LayerNorm(dim, eps=eps, elementwise_affine=False) @@ -555,6 +569,7 @@ class LingBotAttentionBlock(nn.Module): bias=True, gather_output=False, return_bias=False, + quant_config=quant_config, prefix=_projection_prefix(prefix, "ffn.0"), ), nn.GELU(approximate="tanh"), @@ -564,6 +579,7 @@ class LingBotAttentionBlock(nn.Module): bias=True, input_is_parallel=True, return_bias=False, + quant_config=quant_config, prefix=_projection_prefix(prefix, "ffn.2"), ), ) @@ -574,6 +590,7 @@ class LingBotAttentionBlock(nn.Module): bias=True, gather_output=False, return_bias=False, + quant_config=quant_config, prefix=_projection_prefix(prefix, "cam_injector_layer1"), ) self.cam_injector_layer2 = RowParallelLinear( @@ -582,6 +599,7 @@ class LingBotAttentionBlock(nn.Module): bias=True, input_is_parallel=True, return_bias=False, + quant_config=quant_config, prefix=_projection_prefix(prefix, "cam_injector_layer2"), ) self.cam_scale_layer = nn.Linear(dim, dim) @@ -823,7 +841,7 @@ class CausalLingBotWorldTransformer3DModel(nn.Module): num_frames_per_block: int = 3, sliding_window_num_frames: int = 18, local_attn_size: int = -1, - quant_config: object | None = None, + quant_config: QuantizationConfig | None = None, prefix: str = "", ) -> None: super().__init__() @@ -853,11 +871,6 @@ class CausalLingBotWorldTransformer3DModel(nn.Module): ): if field_value is not None: raise ValueError(f"{field_name} must be None because LingBot World v2 has no image embedding path.") - if quant_config is not None: - raise RuntimeError( - "quant_config is not supported by the LingBot World transformer; construct the unquantized model." - ) - dim = num_attention_heads * attention_head_dim self.dim = dim self.config = SimpleNamespace( @@ -898,6 +911,7 @@ class CausalLingBotWorldTransformer3DModel(nn.Module): bias=True, gather_output=False, return_bias=False, + quant_config=quant_config, prefix=_projection_prefix(prefix, "c2ws_hidden_states_layer1"), ) self.c2ws_hidden_states_layer2 = RowParallelLinear( @@ -906,6 +920,7 @@ class CausalLingBotWorldTransformer3DModel(nn.Module): bias=True, input_is_parallel=True, return_bias=False, + quant_config=quant_config, prefix=_projection_prefix(prefix, "c2ws_hidden_states_layer2"), ) self.text_embedding = nn.Sequential( @@ -930,6 +945,7 @@ class CausalLingBotWorldTransformer3DModel(nn.Module): ffn_dim=ffn_dim, cross_attn_norm=cross_attn_norm, eps=eps, + quant_config=quant_config, prefix=_projection_prefix(prefix, f"blocks.{index}"), ) for index in range(num_layers) @@ -976,7 +992,7 @@ class CausalLingBotWorldTransformer3DModel(nn.Module): cls, config: dict[str, Any], *, - quant_config: Any | None = None, + quant_config: QuantizationConfig | None = None, prefix: str = "", ) -> Self: checkpoint_contract = { From fa1d03ab8d5111b81968e2173bf61981c43aa0c7 Mon Sep 17 00:00:00 2001 From: Weiming Liao Date: Mon, 21 Sep 2026 21:07:21 +0800 Subject: [PATCH 4/5] [Frontend] Add MiniMax-H3 latent editing workflows (WF-05) (#7898) Signed-off-by: chen hongwei <1792043268@qq.com> Signed-off-by: Weiming Liao Co-authored-by: chen hongwei <1792043268@qq.com> Co-authored-by: chen hongwei <54873389+avicii-forever@users.noreply.github.com> --- apps/ComfyUI-vLLM-Omni/README.md | 15 +- apps/ComfyUI-vLLM-Omni/__init__.py | 6 + .../comfyui_vllm_omni/nodes.py | 92 + .../comfyui_vllm_omni/utils/api_client.py | 49 + .../comfyui_vllm_omni/utils/latent_mask.py | 51 + .../comfyui_vllm_omni/utils/types.py | 4 + .../docs/wf05-h3-latent-editing.md | 17 + ...Omni Latent Mask Editing - Image Mask.json | 499 ++ ...i Latent Mask Editing - Temporal Mask.json | 560 ++ .../vLLM-Omni Latent Mask Editing.json | 511 ++ ...M-Omni MiniMax-H3 Latent Mask Editing.json | 5241 +++++++++++++++++ apps/ComfyUI-vLLM-Omni/web/main.js | 1 + .../features/comfyui/mock_videos_server.py | 115 + .../e2e/features/comfyui/test_latent_mask.py | 80 + .../comfyui/test_latent_mask_editing.py | 99 + 15 files changed, 7339 insertions(+), 1 deletion(-) create mode 100644 apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/latent_mask.py create mode 100644 apps/ComfyUI-vLLM-Omni/docs/wf05-h3-latent-editing.md create mode 100644 apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Image Mask.json create mode 100644 apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Temporal Mask.json create mode 100644 apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing.json create mode 100644 apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni MiniMax-H3 Latent Mask Editing.json create mode 100644 tests/e2e/features/comfyui/mock_videos_server.py create mode 100644 tests/e2e/features/comfyui/test_latent_mask.py create mode 100644 tests/e2e/features/comfyui/test_latent_mask_editing.py diff --git a/apps/ComfyUI-vLLM-Omni/README.md b/apps/ComfyUI-vLLM-Omni/README.md index f33e26b40..c26b2a3de 100644 --- a/apps/ComfyUI-vLLM-Omni/README.md +++ b/apps/ComfyUI-vLLM-Omni/README.md @@ -49,6 +49,7 @@ This extension offers the following nodes based on the output modalities (at **C - **Generate Image** for text-to-image and image-to-image tasks - **Generate Video** for text-to-video, first-frame/image-to-video, and reference-conditioned video +- **Latent Mask Editing** for MiniMax-H3 latent-mask editing (source media plus video/audio noise masks) - **FastH3 Deployment** for routing text-to-video requests to a MiniMax-H3 server with FastH3 fused at startup - **Multimodality Understanding** for multimodality-to-text and multimodality-to-audio tasks - **TTS** and **TTS Voice Clone** for TTS tasks @@ -67,7 +68,7 @@ Every node carries the vLLM-Omni mark in its title bar and is tinted by what it | Amber | AR / Diffusion / Multi-Stage Sampling Params | Sampling parameters that apply to any model | | Purple | Qwen TTS Params, Wan Video Params, MiniMax-H3 Video Params | Parameters that only one model family accepts | | Red | LoRA, FastH3 Deployment | Which weights the server is expected to have loaded | -| Teal | Video References | Reference media | +| Teal | Video References, Latent Mask Editing | Reference media | Recolouring a node by hand (right click -> Colors) overrides its tint, and the choice is kept. @@ -263,6 +264,18 @@ Open the **vLLM-Omni FastH3 Text to Video** template, then: The node records which server the workflow targets; it does not start one, nor switch adapters or attention backends on a running server. +#### Latent-mask editing (MiniMax-H3) + +The [WF-05 template](example_workflows/vLLM-Omni%20MiniMax-H3%20Latent%20Mask%20Editing.json) contains inpainting, object removal, continuation, and extension examples in one graph. See the [workflow guide](docs/wf05-h3-latent-editing.md) for inputs, mask settings, dependencies, and preview limitations. + +Connect a **Latent Mask Editing** node to **Generate Video → latent_edit** to edit a source clip instead of generating from scratch. It uploads the source media and serializes the video/audio noise masks the MiniMax H3 API accepts: + +- `source_video` / `source_audio` — the media to edit. +- `video_mask` — a ComfyUI mask image; `0` preserves a region, `1` regenerates it, fractional values blend. A 2D mask `[H, W]` is applied to every frame; a 3D mask `[T, H, W]` is treated as a temporal mask (one slice per frame) for continuation or extension. It is resized to the video latent grid. +- `audio_mask` — a scalar in `[0, 1]`; `0` keeps the source audio, `1` regenerates it, fractional values blend. + +A non-trivial mask requires its matching source, and a source without a mask is rejected. This node only forwards inputs to the server; the served model must declare latent-mask editing support (MiniMax-H3). + ### TTS (e.g., Qwen TTS series) (Also available at **ComfyUI sidebar->Template->vLLM-Omni->vLLM-Omni TTS**) diff --git a/apps/ComfyUI-vLLM-Omni/__init__.py b/apps/ComfyUI-vLLM-Omni/__init__.py index 8c1b03d87..999f85ab2 100644 --- a/apps/ComfyUI-vLLM-Omni/__init__.py +++ b/apps/ComfyUI-vLLM-Omni/__init__.py @@ -21,7 +21,9 @@ from .comfyui_vllm_omni.nodes import ( VLLMOmniGenerateImage, VLLMOmniGenerateMusic, VLLMOmniGenerateVideo, + VLLMOmniLatentMaskEditing, VLLMOmniMiniMaxH3Params, + VLLMOmniMiniMaxH3TemporalMask, VLLMOmniQwenTTSParams, VLLMOmniRemoteLoRA, VLLMOmniSamplingParamsList, @@ -34,6 +36,7 @@ from .comfyui_vllm_omni.nodes import ( # A dictionary that contains all nodes you want to export with their names NODE_CLASS_MAPPINGS = { + "VLLMOmniMiniMaxH3TemporalMask": VLLMOmniMiniMaxH3TemporalMask, # === Generation === "VLLMOmniGenerateImage": VLLMOmniGenerateImage, "VLLMOmniGenerateVideo": VLLMOmniGenerateVideo, @@ -42,6 +45,7 @@ NODE_CLASS_MAPPINGS = { "VLLMOmniGenerateMusic": VLLMOmniGenerateMusic, "VLLMOmniVoiceClone": VLLMOmniVoiceClone, "VLLMOmniVideoReferences": VLLMOmniVideoReferences, + "VLLMOmniLatentMaskEditing": VLLMOmniLatentMaskEditing, # === Params === "VLLMOmniARSampling": VLLMOmniARSampling, "VLLMOmniDiffusionSampling": VLLMOmniDiffusionSampling, @@ -55,6 +59,7 @@ NODE_CLASS_MAPPINGS = { # A dictionary that contains the friendly/humanly readable titles for the nodes NODE_DISPLAY_NAME_MAPPINGS = { + "VLLMOmniMiniMaxH3TemporalMask": "MiniMax-H3 Temporal Mask", # === Generation === "VLLMOmniGenerateImage": "Generate Image", "VLLMOmniGenerateVideo": "Generate Video", @@ -63,6 +68,7 @@ NODE_DISPLAY_NAME_MAPPINGS = { "VLLMOmniGenerateMusic": "Generate Music", "VLLMOmniVoiceClone": "TTS Voice Cloning", "VLLMOmniVideoReferences": "Video References", + "VLLMOmniLatentMaskEditing": "Latent Mask Editing", # === Params === "VLLMOmniARSampling": "AR Sampling Params", "VLLMOmniDiffusionSampling": "Diffusion Sampling Params", diff --git a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py index 6d5006a11..5bddb9345 100644 --- a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py +++ b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py @@ -1,12 +1,14 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +import math from typing import Literal import torch from comfy_api.input import AudioInput, VideoInput from .utils.api_client import VLLMOmniClient +from .utils.latent_mask import _align_frame_count, _video_latent_t from .utils.logger import get_logger from .utils.models import lookup_model_spec from .utils.types import ( @@ -17,6 +19,7 @@ from .utils.types import ( AutoregressionSamplingParams, DiffusionSamplingParams, FastH3Deployment, + LatentMaskEditing, MiniMaxH3ModelSpecificParams, QwenTTSModelSpecificParams, VideoReferences, @@ -210,6 +213,7 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase): "lora": ("REMOTE_LORA",), "model_params": ("VIDEO_PARAMS",), "fast_h3": ("FASTH3_DEPLOYMENT",), + "latent_edit": ("LATENT_MASK_EDITING",), }, } @@ -262,6 +266,7 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase): model_params: dict | None = None, lora: dict | None = None, fast_h3: dict | None = None, + latent_edit: dict | None = None, **kwargs, ): if kwargs: @@ -350,6 +355,7 @@ class VLLMOmniGenerateVideo(_VLLMOmniGenerateBase): sampling_params=sampling_params, lora=lora, model_params=model_params, + latent_edit=latent_edit, ) return (output,) @@ -1051,3 +1057,89 @@ class VLLMOmniVideoReferences: if value is not None: refs[f"{kind}_{index}"] = value return (refs,) + + +class VLLMOmniLatentMaskEditing: + @classmethod + def INPUT_TYPES(cls): + return { + "required": {}, + "optional": { + "source_video": ("VIDEO",), + "source_audio": ("AUDIO",), + "video_mask": ("MASK",), + "audio_mask": ("FLOAT", {"default": -1.0, "min": -1.0, "max": 1.0, "step": 0.01}), + }, + } + + RETURN_TYPES = ("LATENT_MASK_EDITING",) + RETURN_NAMES = ("latent_edit",) + FUNCTION = "get_latent_edit" + CATEGORY = "vLLM-Omni" + + def get_latent_edit( + self, + source_video: VideoInput | None = None, + source_audio: AudioInput | None = None, + video_mask: torch.Tensor | None = None, + audio_mask: float = -1.0, + **kwargs, + ): + if kwargs: + logger.info("Uncaught kwargs: %s", kwargs) + edit = LatentMaskEditing() + if source_video is not None: + edit["source_video"] = source_video + if source_audio is not None: + edit["source_audio"] = source_audio + if video_mask is not None: + edit["video_mask"] = video_mask + if audio_mask >= 0.0: + edit["audio_mask"] = audio_mask + return (edit,) + + +class VLLMOmniMiniMaxH3TemporalMask: + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "images": ("IMAGE",), + "source_fps": ("FLOAT", {"default": 24.0, "min": 0.01}), + "duration": ("FLOAT", {"default": 5.0, "min": 0.01, "max": 15.0}), + "mode": (["continuation", "extension"],), + "preserve_fraction": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0}), + } + } + + RETURN_TYPES = ("MASK", "FLOAT", "IMAGE", "MASK") + RETURN_NAMES = ("mask", "preview_fps", "preview_images", "preview_mask") + FUNCTION = "build" + CATEGORY = "vLLM-Omni" + + def build(self, images, source_fps, duration, mode, preserve_fraction=0.5): + if images.shape[0] <= 0 or not math.isfinite(source_fps) or source_fps <= 0: + raise ValueError("Source must contain frames and have a positive finite FPS.") + if not math.isfinite(duration) or duration <= 0 or not 0 <= preserve_fraction <= 1: + raise ValueError("Invalid duration or preserve_fraction.") + frames = _align_frame_count(max(1, round(duration * 24))) + source_seconds = images.shape[0] / source_fps + if mode == "extension": + if frames / 24 <= source_seconds: + raise ValueError("Extension output must be longer than the source; increase duration.") + boundary = source_seconds + elif mode == "continuation": + boundary = min(source_seconds, frames / 24) * preserve_fraction + else: + raise ValueError(f"Unknown temporal mask mode: {mode}") + available = min(frames, math.floor(boundary * 24 + 1e-8)) + prefix = 0 if available < 5 else 5 + 17 * ((available - 5) // 17) + preserved = _video_latent_t(prefix) if prefix else 0 + total = _video_latent_t(frames) + mask = torch.ones(total, 1, 1) + mask[:preserved] = 0 + indices = torch.arange(frames, device=images.device) + source_indices = (indices * (source_fps / 24)).floor().long().clamp(max=images.shape[0] - 1) + preview_images = images.index_select(0, source_indices) + preview_mask = mask.index_select(0, torch.arange(frames) * total // frames) + return mask, 24.0, preview_images, preview_mask diff --git a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py index 21935414b..69e1af899 100644 --- a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py +++ b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py @@ -30,6 +30,7 @@ from .format import ( video_to_base64, video_to_bytes, ) +from .latent_mask import scalar_mask_to_json, video_mask_to_grid_json from .logger import get_logger, pretty_printer from .models import lookup_model_spec from .types import ( @@ -283,6 +284,7 @@ class VLLMOmniClient: sampling_params: dict | None = None, model_params: dict | None = None, lora: dict | None = None, + latent_edit: dict | None = None, spec_model: str | None = None, **extra_params, ) -> VideoInput: @@ -386,6 +388,53 @@ class VLLMOmniClient: content_type="image/png", ) + # === latent-mask editing (MiniMax H3) === + if latent_edit is not None: + source_video = latent_edit.get("source_video") + source_audio = latent_edit.get("source_audio") + video_mask = latent_edit.get("video_mask") + audio_mask = latent_edit.get("audio_mask") + + if video_mask is None and audio_mask is None: + raise ValueError("Latent-mask editing requires at least one mask.") + + video_mask_trivial = video_mask is None or bool((video_mask == 1.0).all().item()) + audio_mask_trivial = audio_mask is None or audio_mask == 1.0 + if not video_mask_trivial and source_video is None: + raise ValueError("A non-trivial video mask requires a source video.") + if not audio_mask_trivial and source_audio is None and source_video is None: + raise ValueError("A non-trivial audio mask requires a source audio or a source video with audio.") + + if source_video is not None: + form.add_field( + "source_video", + video_to_bytes(source_video, "source.mp4"), + filename="source.mp4", + content_type="video/mp4", + ) + if source_audio is not None: + form.add_field( + "source_audio", + audio_to_bytes(source_audio, "source_audio.mp3"), + filename="source_audio.mp3", + content_type="audio/mpeg", + ) + if video_mask is not None: + mask_json = video_mask_to_grid_json(video_mask, width=width, height=height, num_frames=num_frames) + form.add_field( + "video_noise_mask", + mask_json.encode("utf-8"), + filename="video-mask.json", + content_type="application/json", + ) + if audio_mask is not None: + form.add_field( + "audio_noise_mask", + scalar_mask_to_json(audio_mask).encode("utf-8"), + filename="audio-mask.json", + content_type="application/json", + ) + if len(keyframe_images) == 2: for image_filename, image in keyframe_images: form.add_field( diff --git a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/latent_mask.py b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/latent_mask.py new file mode 100644 index 000000000..532cf8184 --- /dev/null +++ b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/latent_mask.py @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +import json + +import torch +import torch.nn.functional as F + + +def scalar_mask_to_json(value: float) -> str: + if not 0.0 <= value <= 1.0: + raise ValueError(f"mask value must be in [0, 1], got {value}") + return str(value) + + +def _align_frame_count(frame_count: int) -> int: + if frame_count <= 0: + return 1 + current = int(frame_count) + while current % 17 != 5: + current += 1 + return current + + +def _video_latent_t(frame_count: int) -> int: + if frame_count <= 5: + return 2 + return ((int(frame_count) - 5) // 17) * 5 + 2 + + +def video_mask_to_grid(mask: torch.Tensor, *, width: int, height: int, num_frames: int) -> torch.Tensor: + tv = _video_latent_t(_align_frame_count(num_frames)) + height = int(height) // 32 * 32 + width = int(width) // 32 * 32 + gh, gw = height // 16, width // 16 + + if mask.ndim == 2: + mask = mask.unsqueeze(0) + elif mask.ndim != 3: + raise ValueError(f"expected a 2D or 3D mask tensor, got {mask.ndim}D") + + grid = F.interpolate(mask.unsqueeze(1).float(), size=(gh, gw), mode="area").squeeze(1) + if grid.shape[0] == 1: + grid = grid.expand(tv, gh, gw) + elif grid.shape[0] != tv: + grid = F.interpolate(grid.unsqueeze(0).unsqueeze(0), size=(tv, gh, gw), mode="nearest").squeeze(0).squeeze(0) + return grid + + +def video_mask_to_grid_json(mask: torch.Tensor, *, width: int, height: int, num_frames: int) -> str: + return json.dumps(video_mask_to_grid(mask, width=width, height=height, num_frames=num_frames).tolist()) diff --git a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/types.py b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/types.py index 6d12a98c2..6194613de 100644 --- a/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/types.py +++ b/apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/types.py @@ -40,6 +40,10 @@ class VideoReferences(dict): pass +class LatentMaskEditing(dict): + pass + + class FastH3Deployment(dict): """Descriptor for a server that fused FastH3 at startup.""" diff --git a/apps/ComfyUI-vLLM-Omni/docs/wf05-h3-latent-editing.md b/apps/ComfyUI-vLLM-Omni/docs/wf05-h3-latent-editing.md new file mode 100644 index 000000000..9ed10273f --- /dev/null +++ b/apps/ComfyUI-vLLM-Omni/docs/wf05-h3-latent-editing.md @@ -0,0 +1,17 @@ +# WF-05: MiniMax-H3 latent-mask editing + +Edit a source video with MiniMax-H3 using spatial or temporal masks. The workflow includes object removal, inpainting, continuation, and extension, with a mask preview for each case. + +## Requirements + +- ComfyUI with ComfyUI-vLLM-Omni installed. +- An H3 service using the FL2VA checkpoint partition with latent-mask editing support. Follow the [H3 recipe](../../../recipes/MiniMaxAI/MiniMax-H3.md) for server setup. + +## Use + +Open **vLLM-Omni MiniMax-H3 Latent Mask Editing.json** from the example workflows and set the Generate Video URL/model for the case you want to run. + +1. Upload your source video in Load Video. +2. Adjust the case's mask. For removal and inpainting, set Mask Canvas to the video dimensions, Mask Rectangle to the region size, and Combine Masks x/y to its position. For continuation and extension, set MiniMax-H3 Temporal Mask's mode and duration; continuation also uses `preserve_fraction`. Match the duration in Generate Video, and use a duration longer than the source for extension. Mask values are `0` to preserve and `1` to regenerate. +3. Adjust the prompt to describe the desired content. Set the output dimensions and duration in Generate Video. If editing audio, update the sound description and audio mask (`0` preserves, `1` regenerates the whole clip). +4. Select the case's Mask Preview Save Video node and choose **Execute to selected output nodes** to preview the mask as a red overlay without inference, or select its Result Save Video node to generate the edited video. Running the entire workflow generates all four cases. diff --git a/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Image Mask.json b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Image Mask.json new file mode 100644 index 000000000..17be731bf --- /dev/null +++ b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Image Mask.json @@ -0,0 +1,499 @@ +{ + "id": "latent-mask-editing-7380", + "revision": 0, + "last_node_id": 7, + "last_link_id": 5, + "nodes": [ + { + "id": 1, + "type": "MarkdownNote", + "pos": [ + 40, + 20 + ], + "size": [ + 420, + 210 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Latent-mask editing server prerequisite", + "properties": {}, + "widgets_values": [ + "Requires a vLLM-Omni server that declares latent-mask editing support (MiniMax-H3, EDIT-01). The source clip and video/audio noise masks are uploaded to /v1/videos; 0 preserves, 1 regenerates, fractional blends." + ], + "color": "#432", + "bgcolor": "#000" + }, + { + "id": 2, + "type": "LoadVideo", + "pos": [ + 40, + 280 + ], + "size": [ + 420, + 120 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "localized_name": "file", + "name": "file", + "type": "COMBO", + "widget": { + "name": "file" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [ + 1 + ] + } + ], + "properties": { + "Node name for S&R": "LoadVideo" + }, + "widgets_values": [ + "" + ] + }, + { + "id": 3, + "type": "LoadImageMask", + "pos": [ + 40, + 460 + ], + "size": [ + 420, + 160 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "STRING", + "widget": { + "name": "image" + }, + "link": null + }, + { + "name": "channel", + "type": "STRING", + "widget": { + "name": "channel" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "mask", + "name": "mask", + "type": "MASK", + "links": [ + 2 + ] + } + ], + "properties": { + "Node name for S&R": "LoadImageMask" + }, + "widgets_values": [ + "", + "red" + ] + }, + { + "id": 4, + "type": "VLLMOmniLatentMaskEditing", + "pos": [ + 560, + 260 + ], + "size": [ + 420, + 300 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "localized_name": "source_video", + "name": "source_video", + "type": "VIDEO", + "link": 1, + "shape": 7 + }, + { + "localized_name": "source_audio", + "name": "source_audio", + "type": "AUDIO", + "link": null + }, + { + "localized_name": "video_mask", + "name": "video_mask", + "type": "MASK", + "link": 2, + "shape": 7 + }, + { + "localized_name": "audio_mask", + "name": "audio_mask", + "type": "FLOAT", + "widget": { + "name": "audio_mask" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "latent_edit", + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "links": [ + 3 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniLatentMaskEditing" + }, + "widgets_values": [ + 0.5 + ] + }, + { + "id": 5, + "type": "VLLMOmniMiniMaxH3Params", + "pos": [ + 40, + 680 + ], + "size": [ + 420, + 140 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "localized_name": "audio_flow_shift", + "name": "audio_flow_shift", + "type": "FLOAT", + "widget": { + "name": "audio_flow_shift" + }, + "link": null + }, + { + "localized_name": "flow_shift", + "name": "flow_shift", + "type": "FLOAT", + "widget": { + "name": "flow_shift" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video params", + "name": "video params", + "type": "VIDEO_PARAMS", + "links": [ + 4 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniMiniMaxH3Params" + }, + "widgets_values": [ + 3.0, + 12.0 + ] + }, + { + "id": 6, + "type": "VLLMOmniGenerateVideo", + "pos": [ + 1080, + 200 + ], + "size": [ + 540, + 680 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "localized_name": "frame", + "name": "frame", + "type": "IMAGE", + "link": null, + "shape": 7 + }, + { + "localized_name": "references", + "name": "references", + "type": "VIDEO_REFERENCES", + "link": null, + "shape": 7 + }, + { + "localized_name": "sampling_params", + "name": "sampling_params", + "type": "SAMPLING_PARAMS", + "link": null + }, + { + "localized_name": "lora", + "name": "lora", + "type": "REMOTE_LORA", + "link": null + }, + { + "localized_name": "model_params", + "name": "model_params", + "type": "VIDEO_PARAMS", + "link": 4, + "shape": 7 + }, + { + "localized_name": "fast_h3", + "name": "fast_h3", + "type": "FASTH3_DEPLOYMENT", + "link": null + }, + { + "localized_name": "latent_edit", + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "link": 3, + "shape": 7 + }, + { + "localized_name": "url", + "name": "url", + "type": "STRING", + "widget": { + "name": "url" + }, + "link": null + }, + { + "localized_name": "model", + "name": "model", + "type": "STRING", + "widget": { + "name": "model" + }, + "link": null + }, + { + "localized_name": "prompt", + "name": "prompt", + "type": "STRING", + "widget": { + "name": "prompt" + }, + "link": null + }, + { + "localized_name": "negative_prompt", + "name": "negative_prompt", + "type": "STRING", + "widget": { + "name": "negative_prompt" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "INT", + "widget": { + "name": "fps" + }, + "link": null + }, + { + "localized_name": "duration", + "name": "duration", + "type": "FLOAT", + "widget": { + "name": "duration" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [ + 5 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniGenerateVideo" + }, + "widgets_values": [ + "http://127.0.0.1:8000/v1", + "MiniMaxAI/MiniMax-H3", + "Partially restyle the clip and soundtrack", + "", + 160, + 120, + 24, + 4.458 + ] + }, + { + "id": 7, + "type": "SaveVideo", + "pos": [ + 1700, + 300 + ], + "size": [ + 420, + 160 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 5, + "shape": 7 + }, + { + "localized_name": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "localized_name": "format", + "name": "format", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "properties": { + "Node name for S&R": "SaveVideo" + }, + "widgets_values": [ + "vllm-omni/latent-edit", + "mp4" + ] + } + ], + "links": [ + [ + 1, + 2, + 0, + 4, + 0, + "VIDEO" + ], + [ + 2, + 3, + 0, + 4, + 2, + "MASK" + ], + [ + 3, + 4, + 0, + 6, + 6, + "LATENT_MASK_EDITING" + ], + [ + 4, + 5, + 0, + 6, + 4, + "VIDEO_PARAMS" + ], + [ + 5, + 6, + 0, + 7, + 0, + "VIDEO" + ] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4 +} diff --git a/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Temporal Mask.json b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Temporal Mask.json new file mode 100644 index 000000000..b2b3a6980 --- /dev/null +++ b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing - Temporal Mask.json @@ -0,0 +1,560 @@ +{ + "last_node_id": 9, + "last_link_id": 7, + "nodes": [ + { + "id": 2, + "type": "LoadVideo", + "pos": [ + 40, + 100 + ], + "size": [ + 420, + 120 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "name": "file", + "type": "STRING", + "widget": { + "name": "file" + }, + "link": null + } + ], + "outputs": [ + { + "name": "video", + "type": "VIDEO", + "links": [ + 1 + ] + } + ], + "properties": { + "Node name for S&R": "LoadVideo" + }, + "widgets_values": [ + "" + ] + }, + { + "id": 3, + "type": "SolidMask", + "pos": [ + 40, + 300 + ], + "size": [ + 420, + 160 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + } + ], + "outputs": [ + { + "name": "mask", + "type": "MASK", + "links": [ + 2 + ] + } + ], + "properties": { + "Node name for S&R": "SolidMask" + }, + "widgets_values": [ + 0.0, + 160, + 120 + ] + }, + { + "id": 4, + "type": "SolidMask", + "pos": [ + 40, + 500 + ], + "size": [ + 420, + 160 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + } + ], + "outputs": [ + { + "name": "mask", + "type": "MASK", + "links": [ + 3 + ] + } + ], + "properties": { + "Node name for S&R": "SolidMask" + }, + "widgets_values": [ + 1.0, + 160, + 120 + ] + }, + { + "id": 5, + "type": "BatchMasksNode", + "pos": [ + 250, + 380 + ], + "size": [ + 200, + 100 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "masks.mask0", + "type": "MASK", + "link": 2, + "slot_index": 0 + }, + { + "name": "masks.mask1", + "type": "MASK", + "link": 3, + "slot_index": 1 + } + ], + "outputs": [ + { + "name": "MASK", + "type": "MASK", + "links": [ + 4 + ] + } + ], + "properties": { + "Node name for S&R": "BatchMasksNode" + }, + "widgets_values": [] + }, + { + "id": 6, + "type": "VLLMOmniLatentMaskEditing", + "pos": [ + 450, + 180 + ], + "size": [ + 400, + 120 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "source_video", + "type": "VIDEO", + "link": 1 + }, + { + "name": "source_audio", + "type": "AUDIO", + "link": null + }, + { + "name": "video_mask", + "type": "MASK", + "link": 4 + }, + { + "name": "audio_mask", + "type": "FLOAT", + "widget": { + "name": "audio_mask" + }, + "link": null + } + ], + "outputs": [ + { + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "links": [ + 5 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniLatentMaskEditing" + }, + "widgets_values": [ + -1.0 + ] + }, + { + "id": 7, + "type": "VLLMOmniMiniMaxH3Params", + "pos": [ + 40, + 700 + ], + "size": [ + 420, + 160 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "audio_flow_shift", + "type": "FLOAT", + "widget": { + "name": "audio_flow_shift" + }, + "link": null + }, + { + "name": "flow_shift", + "type": "FLOAT", + "widget": { + "name": "flow_shift" + }, + "link": null + } + ], + "outputs": [ + { + "name": "video params", + "type": "VIDEO_PARAMS", + "links": [ + 6 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniMiniMaxH3Params" + }, + "widgets_values": [ + 3.0, + 12.0 + ] + }, + { + "id": 8, + "type": "VLLMOmniGenerateVideo", + "pos": [ + 700, + 180 + ], + "size": [ + 450, + 380 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "frame", + "type": "IMAGE", + "link": null + }, + { + "name": "references", + "type": "VIDEO_REFERENCES", + "link": null + }, + { + "name": "sampling_params", + "type": "SAMPLING_PARAMS", + "link": null + }, + { + "name": "lora", + "type": "LORA", + "link": null + }, + { + "name": "model_params", + "type": "VIDEO_PARAMS", + "link": 6 + }, + { + "name": "fast_h3", + "type": "FAST_H3", + "link": null + }, + { + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "link": 5 + }, + { + "name": "url", + "type": "STRING", + "widget": { + "name": "url" + }, + "link": null + }, + { + "name": "model", + "type": "STRING", + "widget": { + "name": "model" + }, + "link": null + }, + { + "name": "prompt", + "type": "STRING", + "widget": { + "name": "prompt" + }, + "link": null + }, + { + "name": "negative_prompt", + "type": "STRING", + "widget": { + "name": "negative_prompt" + }, + "link": null + }, + { + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "name": "fps", + "type": "INT", + "widget": { + "name": "fps" + }, + "link": null + }, + { + "name": "duration", + "type": "FLOAT", + "widget": { + "name": "duration" + }, + "link": null + } + ], + "outputs": [ + { + "name": "video", + "type": "VIDEO", + "links": [ + 7 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniGenerateVideo" + }, + "widgets_values": [ + "http://127.0.0.1:8000/v1", + "MiniMaxAI/MiniMax-H3", + "temporal mask: regenerate the ending", + "", + 160, + 120, + 24, + 4.458 + ] + }, + { + "id": 9, + "type": "SaveVideo", + "pos": [ + 900, + 180 + ], + "size": [ + 420, + 200 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "video", + "type": "VIDEO", + "link": 7 + }, + { + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "name": "format", + "type": "STRING", + "widget": { + "name": "format" + }, + "link": null + } + ], + "outputs": [ + { + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "properties": { + "Node name for S&R": "SaveVideo" + }, + "widgets_values": [ + "temporal_mask", + "mp4" + ] + } + ], + "links": [ + [ + 1, + 2, + 0, + 6, + 0, + "VIDEO" + ], + [ + 2, + 3, + 0, + 5, + 0, + "MASK" + ], + [ + 3, + 4, + 0, + 5, + 1, + "MASK" + ], + [ + 4, + 5, + 0, + 6, + 2, + "MASK" + ], + [ + 5, + 6, + 0, + 8, + 6, + "LATENT_MASK_EDITING" + ], + [ + 6, + 7, + 0, + 8, + 4, + "VIDEO_PARAMS" + ], + [ + 7, + 8, + 0, + 9, + 0, + "VIDEO" + ] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4 +} diff --git a/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing.json b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing.json new file mode 100644 index 000000000..4192ef4c5 --- /dev/null +++ b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Latent Mask Editing.json @@ -0,0 +1,511 @@ +{ + "id": "latent-mask-editing-7380", + "revision": 0, + "last_node_id": 7, + "last_link_id": 5, + "nodes": [ + { + "id": 1, + "type": "MarkdownNote", + "pos": [ + 40, + 20 + ], + "size": [ + 420, + 210 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Latent-mask editing server prerequisite", + "properties": {}, + "widgets_values": [ + "Requires a vLLM-Omni server that declares latent-mask editing support (MiniMax-H3, EDIT-01). The source clip and video/audio noise masks are uploaded to /v1/videos; 0 preserves, 1 regenerates, fractional blends." + ], + "color": "#432", + "bgcolor": "#000" + }, + { + "id": 2, + "type": "LoadVideo", + "pos": [ + 40, + 280 + ], + "size": [ + 420, + 120 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "localized_name": "file", + "name": "file", + "type": "COMBO", + "widget": { + "name": "file" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [ + 1 + ] + } + ], + "properties": { + "Node name for S&R": "LoadVideo" + }, + "widgets_values": [ + "" + ] + }, + { + "id": 3, + "type": "SolidMask", + "pos": [ + 40, + 460 + ], + "size": [ + 420, + 160 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "mask", + "name": "mask", + "type": "MASK", + "links": [ + 2 + ] + } + ], + "properties": { + "Node name for S&R": "SolidMask" + }, + "widgets_values": [ + 1.0, + 160, + 120 + ] + }, + { + "id": 4, + "type": "VLLMOmniLatentMaskEditing", + "pos": [ + 560, + 260 + ], + "size": [ + 420, + 300 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "localized_name": "source_video", + "name": "source_video", + "type": "VIDEO", + "link": 1, + "shape": 7 + }, + { + "localized_name": "source_audio", + "name": "source_audio", + "type": "AUDIO", + "link": null + }, + { + "localized_name": "video_mask", + "name": "video_mask", + "type": "MASK", + "link": 2, + "shape": 7 + }, + { + "localized_name": "audio_mask", + "name": "audio_mask", + "type": "FLOAT", + "widget": { + "name": "audio_mask" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "latent_edit", + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "links": [ + 3 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniLatentMaskEditing" + }, + "widgets_values": [ + 0.5 + ] + }, + { + "id": 5, + "type": "VLLMOmniMiniMaxH3Params", + "pos": [ + 40, + 680 + ], + "size": [ + 420, + 140 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "localized_name": "audio_flow_shift", + "name": "audio_flow_shift", + "type": "FLOAT", + "widget": { + "name": "audio_flow_shift" + }, + "link": null + }, + { + "localized_name": "flow_shift", + "name": "flow_shift", + "type": "FLOAT", + "widget": { + "name": "flow_shift" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video params", + "name": "video params", + "type": "VIDEO_PARAMS", + "links": [ + 4 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniMiniMaxH3Params" + }, + "widgets_values": [ + 3.0, + 12.0 + ] + }, + { + "id": 6, + "type": "VLLMOmniGenerateVideo", + "pos": [ + 1080, + 200 + ], + "size": [ + 540, + 680 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "localized_name": "frame", + "name": "frame", + "type": "IMAGE", + "link": null, + "shape": 7 + }, + { + "localized_name": "references", + "name": "references", + "type": "VIDEO_REFERENCES", + "link": null, + "shape": 7 + }, + { + "localized_name": "sampling_params", + "name": "sampling_params", + "type": "SAMPLING_PARAMS", + "link": null + }, + { + "localized_name": "lora", + "name": "lora", + "type": "REMOTE_LORA", + "link": null + }, + { + "localized_name": "model_params", + "name": "model_params", + "type": "VIDEO_PARAMS", + "link": 4, + "shape": 7 + }, + { + "localized_name": "fast_h3", + "name": "fast_h3", + "type": "FASTH3_DEPLOYMENT", + "link": null + }, + { + "localized_name": "latent_edit", + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "link": 3, + "shape": 7 + }, + { + "localized_name": "url", + "name": "url", + "type": "STRING", + "widget": { + "name": "url" + }, + "link": null + }, + { + "localized_name": "model", + "name": "model", + "type": "STRING", + "widget": { + "name": "model" + }, + "link": null + }, + { + "localized_name": "prompt", + "name": "prompt", + "type": "STRING", + "widget": { + "name": "prompt" + }, + "link": null + }, + { + "localized_name": "negative_prompt", + "name": "negative_prompt", + "type": "STRING", + "widget": { + "name": "negative_prompt" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "INT", + "widget": { + "name": "fps" + }, + "link": null + }, + { + "localized_name": "duration", + "name": "duration", + "type": "FLOAT", + "widget": { + "name": "duration" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [ + 5 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniGenerateVideo" + }, + "widgets_values": [ + "http://127.0.0.1:8000/v1", + "MiniMaxAI/MiniMax-H3", + "Partially restyle the clip and soundtrack", + "", + 160, + 120, + 24, + 4.458 + ] + }, + { + "id": 7, + "type": "SaveVideo", + "pos": [ + 1700, + 300 + ], + "size": [ + 420, + 160 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 5, + "shape": 7 + }, + { + "localized_name": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "localized_name": "format", + "name": "format", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "properties": { + "Node name for S&R": "SaveVideo" + }, + "widgets_values": [ + "vllm-omni/latent-edit", + "mp4" + ] + } + ], + "links": [ + [ + 1, + 2, + 0, + 4, + 0, + "VIDEO" + ], + [ + 2, + 3, + 0, + 4, + 2, + "MASK" + ], + [ + 3, + 4, + 0, + 6, + 6, + "LATENT_MASK_EDITING" + ], + [ + 4, + 5, + 0, + 6, + 4, + "VIDEO_PARAMS" + ], + [ + 5, + 6, + 0, + 7, + 0, + "VIDEO" + ] + ], + "groups": [], + "config": {}, + "extra": {}, + "version": 0.4 +} diff --git a/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni MiniMax-H3 Latent Mask Editing.json b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni MiniMax-H3 Latent Mask Editing.json new file mode 100644 index 000000000..fc4341730 --- /dev/null +++ b/apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni MiniMax-H3 Latent Mask Editing.json @@ -0,0 +1,5241 @@ +{ + "id": "eebf673c-c66a-4f20-9e87-b728a82f6e5f", + "revision": 0, + "last_node_id": 99, + "last_link_id": 162, + "nodes": [ + { + "id": 5, + "type": "VLLMOmniMiniMaxH3Params", + "pos": [ + 1510, + 2160 + ], + "size": [ + 300, + 130 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [ + { + "localized_name": "audio_flow_shift", + "name": "audio_flow_shift", + "type": "FLOAT", + "widget": { + "name": "audio_flow_shift" + }, + "link": null + }, + { + "localized_name": "flow_shift", + "name": "flow_shift", + "type": "FLOAT", + "widget": { + "name": "flow_shift" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MiniMax-H3 video params", + "name": "MiniMax-H3 video params", + "type": "VIDEO_PARAMS", + "links": [ + 4 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniMiniMaxH3Params" + }, + "widgets_values": [ + 3, + 12 + ], + "widgets_values_named": { + "audio_flow_shift": 3, + "flow_shift": 12 + } + }, + { + "id": 11, + "type": "VLLMOmniDiffusionSampling", + "pos": [ + 580, + 1870 + ], + "size": [ + 410.68458556883115, + 226 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "localized_name": "n", + "name": "n", + "type": "INT", + "widget": { + "name": "n" + }, + "link": null + }, + { + "localized_name": "num_inference_steps", + "name": "num_inference_steps", + "type": "INT", + "widget": { + "name": "num_inference_steps" + }, + "link": null + }, + { + "localized_name": "guidance_scale", + "name": "guidance_scale", + "type": "FLOAT", + "widget": { + "name": "guidance_scale" + }, + "link": null + }, + { + "localized_name": "true_cfg_scale", + "name": "true_cfg_scale", + "type": "FLOAT", + "widget": { + "name": "true_cfg_scale" + }, + "link": null + }, + { + "localized_name": "vae_use_slicing", + "name": "vae_use_slicing", + "type": "BOOLEAN", + "widget": { + "name": "vae_use_slicing" + }, + "link": null + }, + { + "localized_name": "vae_use_tiling", + "name": "vae_use_tiling", + "type": "BOOLEAN", + "widget": { + "name": "vae_use_tiling" + }, + "link": null + }, + { + "localized_name": "seed", + "name": "seed", + "type": "INT", + "widget": { + "name": "seed" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "diffusion sampling params", + "name": "diffusion sampling params", + "type": "SAMPLING_PARAMS", + "links": [ + 10 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniDiffusionSampling" + }, + "widgets_values": [ + 1, + 50, + 1, + 1, + false, + false, + 0, + "fixed" + ], + "widgets_values_named": { + "n": 1, + "num_inference_steps": 50, + "guidance_scale": 1, + "true_cfg_scale": 1, + "vae_use_slicing": false, + "vae_use_tiling": false, + "seed": 0, + "control_after_generate": "fixed" + } + }, + { + "id": 25, + "type": "VLLMOmniMiniMaxH3Params", + "pos": [ + 1510, + 3760 + ], + "size": [ + 300, + 130 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "localized_name": "audio_flow_shift", + "name": "audio_flow_shift", + "type": "FLOAT", + "widget": { + "name": "audio_flow_shift" + }, + "link": null + }, + { + "localized_name": "flow_shift", + "name": "flow_shift", + "type": "FLOAT", + "widget": { + "name": "flow_shift" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MiniMax-H3 video params", + "name": "MiniMax-H3 video params", + "type": "VIDEO_PARAMS", + "links": [ + 30 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniMiniMaxH3Params" + }, + "widgets_values": [ + 3, + 12 + ], + "widgets_values_named": { + "audio_flow_shift": 3, + "flow_shift": 12 + } + }, + { + "id": 29, + "type": "MarkdownNote", + "pos": [ + 1050, + 3470 + ], + "size": [ + 360, + 330 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Inpainting guide", + "properties": { + "Node name for S&R": "MarkdownNote" + }, + "widgets_values": [ + "Replace the masked region with a curled orange cat gently swaying its tail.\n\nThis case has its own rectangular mask. Leave room for the entire cat and tail motion; check the red overlay before generating.\n\nAudio mask 0 preserves source audio latents. To request new sounds, change the audio prompt and mask together. Fractional audio masks do not selectively retain the school chime while adding meows." + ], + "widgets_values_named": { + "text": "Replace the masked region with a curled orange cat gently swaying its tail.\n\nThis case has its own rectangular mask. Leave room for the entire cat and tail motion; check the red overlay before generating.\n\nAudio mask 0 preserves source audio latents. To request new sounds, change the audio prompt and mask together. Fractional audio masks do not selectively retain the school chime while adding meows." + }, + "color": "#333", + "bgcolor": "#222" + }, + { + "id": 30, + "type": "VLLMOmniDiffusionSampling", + "pos": [ + 580, + 3470 + ], + "size": [ + 410.68458556883115, + 226 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "localized_name": "n", + "name": "n", + "type": "INT", + "widget": { + "name": "n" + }, + "link": null + }, + { + "localized_name": "num_inference_steps", + "name": "num_inference_steps", + "type": "INT", + "widget": { + "name": "num_inference_steps" + }, + "link": null + }, + { + "localized_name": "guidance_scale", + "name": "guidance_scale", + "type": "FLOAT", + "widget": { + "name": "guidance_scale" + }, + "link": null + }, + { + "localized_name": "true_cfg_scale", + "name": "true_cfg_scale", + "type": "FLOAT", + "widget": { + "name": "true_cfg_scale" + }, + "link": null + }, + { + "localized_name": "vae_use_slicing", + "name": "vae_use_slicing", + "type": "BOOLEAN", + "widget": { + "name": "vae_use_slicing" + }, + "link": null + }, + { + "localized_name": "vae_use_tiling", + "name": "vae_use_tiling", + "type": "BOOLEAN", + "widget": { + "name": "vae_use_tiling" + }, + "link": null + }, + { + "localized_name": "seed", + "name": "seed", + "type": "INT", + "widget": { + "name": "seed" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "diffusion sampling params", + "name": "diffusion sampling params", + "type": "SAMPLING_PARAMS", + "links": [ + 33 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniDiffusionSampling" + }, + "widgets_values": [ + 1, + 50, + 1, + 1, + false, + false, + 0, + "fixed" + ], + "widgets_values_named": { + "n": 1, + "num_inference_steps": 50, + "guidance_scale": 1, + "true_cfg_scale": 1, + "vae_use_slicing": false, + "vae_use_tiling": false, + "seed": 0, + "control_after_generate": "fixed" + } + }, + { + "id": 34, + "type": "VLLMOmniMiniMaxH3Params", + "pos": [ + 1510, + 5360 + ], + "size": [ + 300, + 130 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "localized_name": "audio_flow_shift", + "name": "audio_flow_shift", + "type": "FLOAT", + "widget": { + "name": "audio_flow_shift" + }, + "link": null + }, + { + "localized_name": "flow_shift", + "name": "flow_shift", + "type": "FLOAT", + "widget": { + "name": "flow_shift" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MiniMax-H3 video params", + "name": "MiniMax-H3 video params", + "type": "VIDEO_PARAMS", + "links": [ + 38 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniMiniMaxH3Params" + }, + "widgets_values": [ + 3, + 12 + ], + "widgets_values_named": { + "audio_flow_shift": 3, + "flow_shift": 12 + } + }, + { + "id": 45, + "type": "MarkdownNote", + "pos": [ + 1050, + 5070 + ], + "size": [ + 360, + 330 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Continuation guide", + "properties": { + "Node name for S&R": "MarkdownNote" + }, + "widgets_values": [ + "Regenerate the ending with an overcast sky, heavy rain, and distant lightning.\n\nPreserve fraction controls how much of the available source prefix is retained. Match Temporal Mask duration to Generate Video duration; these values are not linked.\n\nThe boundary rounds down to complete H3 chunks. White allows regeneration, but does not force existing subjects to disappear.\n\nThe red preview is schematic. Audio mask affects the entire clip, including the preserved video prefix." + ], + "widgets_values_named": { + "text": "Regenerate the ending with an overcast sky, heavy rain, and distant lightning.\n\nPreserve fraction controls how much of the available source prefix is retained. Match Temporal Mask duration to Generate Video duration; these values are not linked.\n\nThe boundary rounds down to complete H3 chunks. White allows regeneration, but does not force existing subjects to disappear.\n\nThe red preview is schematic. Audio mask affects the entire clip, including the preserved video prefix." + }, + "color": "#333", + "bgcolor": "#222" + }, + { + "id": 46, + "type": "VLLMOmniDiffusionSampling", + "pos": [ + 580, + 5070 + ], + "size": [ + 410.68458556883115, + 226 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "localized_name": "n", + "name": "n", + "type": "INT", + "widget": { + "name": "n" + }, + "link": null + }, + { + "localized_name": "num_inference_steps", + "name": "num_inference_steps", + "type": "INT", + "widget": { + "name": "num_inference_steps" + }, + "link": null + }, + { + "localized_name": "guidance_scale", + "name": "guidance_scale", + "type": "FLOAT", + "widget": { + "name": "guidance_scale" + }, + "link": null + }, + { + "localized_name": "true_cfg_scale", + "name": "true_cfg_scale", + "type": "FLOAT", + "widget": { + "name": "true_cfg_scale" + }, + "link": null + }, + { + "localized_name": "vae_use_slicing", + "name": "vae_use_slicing", + "type": "BOOLEAN", + "widget": { + "name": "vae_use_slicing" + }, + "link": null + }, + { + "localized_name": "vae_use_tiling", + "name": "vae_use_tiling", + "type": "BOOLEAN", + "widget": { + "name": "vae_use_tiling" + }, + "link": null + }, + { + "localized_name": "seed", + "name": "seed", + "type": "INT", + "widget": { + "name": "seed" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "diffusion sampling params", + "name": "diffusion sampling params", + "type": "SAMPLING_PARAMS", + "links": [ + 48 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniDiffusionSampling" + }, + "widgets_values": [ + 1, + 50, + 1, + 1, + false, + false, + 0, + "fixed" + ], + "widgets_values_named": { + "n": 1, + "num_inference_steps": 50, + "guidance_scale": 1, + "true_cfg_scale": 1, + "vae_use_slicing": false, + "vae_use_tiling": false, + "seed": 0, + "control_after_generate": "fixed" + } + }, + { + "id": 49, + "type": "VLLMOmniMiniMaxH3Params", + "pos": [ + 1510, + 6960 + ], + "size": [ + 300, + 130 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "localized_name": "audio_flow_shift", + "name": "audio_flow_shift", + "type": "FLOAT", + "widget": { + "name": "audio_flow_shift" + }, + "link": null + }, + { + "localized_name": "flow_shift", + "name": "flow_shift", + "type": "FLOAT", + "widget": { + "name": "flow_shift" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MiniMax-H3 video params", + "name": "MiniMax-H3 video params", + "type": "VIDEO_PARAMS", + "links": [ + 51 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniMiniMaxH3Params" + }, + "widgets_values": [ + 3, + 12 + ], + "widgets_values_named": { + "audio_flow_shift": 3, + "flow_shift": 12 + } + }, + { + "id": 59, + "type": "VLLMOmniMiniMaxH3TemporalMask", + "pos": [ + 580, + 6040 + ], + "size": [ + 420, + 240 + ], + "flags": {}, + "order": 26, + "mode": 0, + "inputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": 123 + }, + { + "localized_name": "source_fps", + "name": "source_fps", + "type": "FLOAT", + "widget": { + "name": "source_fps" + }, + "link": 124 + }, + { + "localized_name": "duration", + "name": "duration", + "type": "FLOAT", + "widget": { + "name": "duration" + }, + "link": null + }, + { + "localized_name": "mode", + "name": "mode", + "type": "COMBO", + "widget": { + "name": "mode" + }, + "link": null + }, + { + "localized_name": "preserve_fraction", + "name": "preserve_fraction", + "type": "FLOAT", + "widget": { + "name": "preserve_fraction" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "mask", + "name": "mask", + "type": "MASK", + "links": [ + 60 + ] + }, + { + "localized_name": "preview_fps", + "name": "preview_fps", + "type": "FLOAT", + "links": [ + 146 + ] + }, + { + "localized_name": "preview_images", + "name": "preview_images", + "type": "IMAGE", + "links": [ + 153, + 156 + ] + }, + { + "localized_name": "preview_mask", + "name": "preview_mask", + "type": "MASK", + "links": [ + 155 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniMiniMaxH3TemporalMask" + }, + "widgets_values": [ + 24, + 10, + "extension", + 0.5 + ], + "widgets_values_named": { + "source_fps": 24, + "duration": 10, + "mode": "extension", + "preserve_fraction": 0.5 + } + }, + { + "id": 60, + "type": "MarkdownNote", + "pos": [ + 1050, + 6670 + ], + "size": [ + 360, + 330 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Extension guide", + "properties": { + "Node name for S&R": "MarkdownNote" + }, + "widgets_values": [ + "Extend into a summer starry night. Follow a firework rising behind the school with a slow upward camera tilt, ending at its burst.\n\nSet the same duration in Temporal Mask and Generate Video, longer than the source. These values are not linked. The original source length determines the preserved prefix.\n\nThe boundary rounds down to H3 chunks. Preview holds the final source frame under the red overlay; it is not the generated ending.\n\nAudio mask affects the whole clip, not only the extension." + ], + "widgets_values_named": { + "text": "Extend into a summer starry night. Follow a firework rising behind the school with a slow upward camera tilt, ending at its burst.\n\nSet the same duration in Temporal Mask and Generate Video, longer than the source. These values are not linked. The original source length determines the preserved prefix.\n\nThe boundary rounds down to H3 chunks. Preview holds the final source frame under the red overlay; it is not the generated ending.\n\nAudio mask affects the whole clip, not only the extension." + }, + "color": "#333", + "bgcolor": "#222" + }, + { + "id": 2, + "type": "LoadVideo", + "pos": [ + -268.4516045720269, + 1190 + ], + "size": [ + 420, + 314 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "localized_name": "file", + "name": "file", + "type": "COMBO", + "widget": { + "name": "file" + }, + "link": null + }, + { + "localized_name": "choose file to upload", + "name": "upload", + "type": "IMAGEUPLOAD", + "widget": { + "name": "upload" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "VIDEO", + "name": "VIDEO", + "type": "VIDEO", + "links": [ + 1, + 17, + 28, + 36, + 49 + ] + } + ], + "title": "Shared Source Video | All Cases", + "properties": { + "Node name for S&R": "LoadVideo" + }, + "widgets_values": [ + "", + "image" + ], + "widgets_values_named": { + "file": "", + "upload": "image" + } + }, + { + "id": 62, + "type": "MarkdownNote", + "pos": [ + 1050, + 1870 + ], + "size": [ + 360, + 330 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Object Removal guide", + "properties": { + "Node name for S&R": "MarkdownNote" + }, + "widgets_values": [ + "Reconstruct the surrounding wall inside the mask.\n\nSet this case's Mask Canvas, Mask Rectangle, and Rectangle Position to cover the object, handle, and shadow with some background margin. Preview coverage before generating.\n\nSymmetric expansion by d pixels: x/y minus d; width/height plus 2d. A 32-pixel margin improved this school example.\n\nAudio mask 0 preserves source audio latents; reconstruction is not lossless." + ], + "widgets_values_named": { + "text": "Reconstruct the surrounding wall inside the mask.\n\nSet this case's Mask Canvas, Mask Rectangle, and Rectangle Position to cover the object, handle, and shadow with some background margin. Preview coverage before generating.\n\nSymmetric expansion by d pixels: x/y minus d; width/height plus 2d. A 32-pixel margin improved this school example.\n\nAudio mask 0 preserves source audio latents; reconstruction is not lossless." + }, + "color": "#333", + "bgcolor": "#222" + }, + { + "id": 61, + "type": "VLLMOmniDiffusionSampling", + "pos": [ + 580, + 6670 + ], + "size": [ + 410.68458556883115, + 226 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "localized_name": "n", + "name": "n", + "type": "INT", + "widget": { + "name": "n" + }, + "link": null + }, + { + "localized_name": "num_inference_steps", + "name": "num_inference_steps", + "type": "INT", + "widget": { + "name": "num_inference_steps" + }, + "link": null + }, + { + "localized_name": "guidance_scale", + "name": "guidance_scale", + "type": "FLOAT", + "widget": { + "name": "guidance_scale" + }, + "link": null + }, + { + "localized_name": "true_cfg_scale", + "name": "true_cfg_scale", + "type": "FLOAT", + "widget": { + "name": "true_cfg_scale" + }, + "link": null + }, + { + "localized_name": "vae_use_slicing", + "name": "vae_use_slicing", + "type": "BOOLEAN", + "widget": { + "name": "vae_use_slicing" + }, + "link": null + }, + { + "localized_name": "vae_use_tiling", + "name": "vae_use_tiling", + "type": "BOOLEAN", + "widget": { + "name": "vae_use_tiling" + }, + "link": null + }, + { + "localized_name": "seed", + "name": "seed", + "type": "INT", + "widget": { + "name": "seed" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "diffusion sampling params", + "name": "diffusion sampling params", + "type": "SAMPLING_PARAMS", + "links": [ + 61 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniDiffusionSampling" + }, + "widgets_values": [ + 1, + 50, + 1, + 1, + false, + false, + 0, + "fixed" + ], + "widgets_values_named": { + "n": 1, + "num_inference_steps": 50, + "guidance_scale": 1, + "true_cfg_scale": 1, + "vae_use_slicing": false, + "vae_use_tiling": false, + "seed": 0, + "control_after_generate": "fixed" + } + }, + { + "id": 65, + "type": "CreateVideo", + "pos": [ + 1990, + 1190 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 44, + "mode": 0, + "inputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": 130 + }, + { + "localized_name": "audio", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "FLOAT", + "widget": { + "name": "fps" + }, + "link": 131 + }, + { + "localized_name": "bit_depth", + "name": "bit_depth", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "bit_depth" + }, + "link": null + }, + { + "localized_name": "color_space", + "name": "color_space", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "color_space" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "VIDEO", + "name": "VIDEO", + "type": "VIDEO", + "links": [ + 77 + ] + } + ], + "title": "Object Removal | Preview Video", + "properties": { + "Node name for S&R": "CreateVideo" + }, + "widgets_values": [ + 30, + "auto", + "sRGB", + "none" + ], + "widgets_values_named": { + "fps": 30, + "bit_depth": "auto", + "color_space": "sRGB", + "codec": "none" + } + }, + { + "id": 69, + "type": "CreateVideo", + "pos": [ + 1990, + 2790 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 46, + "mode": 0, + "inputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": 137 + }, + { + "localized_name": "audio", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "FLOAT", + "widget": { + "name": "fps" + }, + "link": 138 + }, + { + "localized_name": "bit_depth", + "name": "bit_depth", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "bit_depth" + }, + "link": null + }, + { + "localized_name": "color_space", + "name": "color_space", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "color_space" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "VIDEO", + "name": "VIDEO", + "type": "VIDEO", + "links": [ + 91 + ] + } + ], + "title": "Inpainting | Preview Video", + "properties": { + "Node name for S&R": "CreateVideo" + }, + "widgets_values": [ + 30, + "auto", + "sRGB", + "none" + ], + "widgets_values_named": { + "fps": 30, + "bit_depth": "auto", + "color_space": "sRGB", + "codec": "none" + } + }, + { + "id": 73, + "type": "CreateVideo", + "pos": [ + 1990, + 4390 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 48, + "mode": 0, + "inputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": 152 + }, + { + "localized_name": "audio", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "FLOAT", + "widget": { + "name": "fps" + }, + "link": 142 + }, + { + "localized_name": "bit_depth", + "name": "bit_depth", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "bit_depth" + }, + "link": null + }, + { + "localized_name": "color_space", + "name": "color_space", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "color_space" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "VIDEO", + "name": "VIDEO", + "type": "VIDEO", + "links": [ + 105 + ] + } + ], + "title": "Continuation | Preview Video", + "properties": { + "Node name for S&R": "CreateVideo" + }, + "widgets_values": [ + 30, + "auto", + "sRGB", + "none" + ], + "widgets_values_named": { + "fps": 30, + "bit_depth": "auto", + "color_space": "sRGB", + "codec": "none" + } + }, + { + "id": 77, + "type": "CreateVideo", + "pos": [ + 1990, + 5990 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 50, + "mode": 0, + "inputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": 158 + }, + { + "localized_name": "audio", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "FLOAT", + "widget": { + "name": "fps" + }, + "link": 146 + }, + { + "localized_name": "bit_depth", + "name": "bit_depth", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "bit_depth" + }, + "link": null + }, + { + "localized_name": "color_space", + "name": "color_space", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "color_space" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMBO", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "VIDEO", + "name": "VIDEO", + "type": "VIDEO", + "links": [ + 119 + ] + } + ], + "title": "Extension | Preview Video", + "properties": { + "Node name for S&R": "CreateVideo" + }, + "widgets_values": [ + 30, + "auto", + "sRGB", + "none" + ], + "widgets_values_named": { + "fps": 30, + "bit_depth": "auto", + "color_space": "sRGB", + "codec": "none" + } + }, + { + "id": 79, + "type": "EmptyImage", + "pos": [ + 1090, + 1190 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "localized_name": "batch_size", + "name": "batch_size", + "type": "INT", + "widget": { + "name": "batch_size" + }, + "link": null + }, + { + "localized_name": "color", + "name": "color", + "type": "INT", + "widget": { + "name": "color" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 126 + ] + } + ], + "properties": { + "Node name for S&R": "EmptyImage" + }, + "widgets_values": [ + 32, + 32, + 1, + 16711680 + ], + "widgets_values_named": { + "width": 32, + "height": 32, + "batch_size": 1, + "color": 16711680 + } + }, + { + "id": 80, + "type": "ImageCompositeMasked", + "pos": [ + 1390, + 1190 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 28, + "mode": 0, + "inputs": [ + { + "localized_name": "destination", + "name": "destination", + "type": "IMAGE", + "link": 125 + }, + { + "localized_name": "source", + "name": "source", + "type": "IMAGE", + "link": 126 + }, + { + "localized_name": "mask", + "name": "mask", + "shape": 7, + "type": "MASK", + "link": 127 + }, + { + "localized_name": "x", + "name": "x", + "type": "INT", + "widget": { + "name": "x" + }, + "link": null + }, + { + "localized_name": "y", + "name": "y", + "type": "INT", + "widget": { + "name": "y" + }, + "link": null + }, + { + "localized_name": "resize_source", + "name": "resize_source", + "type": "BOOLEAN", + "widget": { + "name": "resize_source" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 129 + ] + } + ], + "properties": { + "Node name for S&R": "ImageCompositeMasked" + }, + "widgets_values": [ + 0, + 0, + true + ], + "widgets_values_named": { + "x": 0, + "y": 0, + "resize_source": true + } + }, + { + "id": 81, + "type": "ImageBlend", + "pos": [ + 1690, + 1190 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 36, + "mode": 0, + "inputs": [ + { + "localized_name": "image1", + "name": "image1", + "type": "IMAGE", + "link": 128 + }, + { + "localized_name": "image2", + "name": "image2", + "type": "IMAGE", + "link": 129 + }, + { + "localized_name": "blend_factor", + "name": "blend_factor", + "type": "FLOAT", + "widget": { + "name": "blend_factor" + }, + "link": null + }, + { + "localized_name": "blend_mode", + "name": "blend_mode", + "type": "COMBO", + "widget": { + "name": "blend_mode" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 130 + ] + } + ], + "properties": { + "Node name for S&R": "ImageBlend" + }, + "widgets_values": [ + 0.48, + "normal" + ], + "widgets_values_named": { + "blend_factor": 0.48, + "blend_mode": "normal" + } + }, + { + "id": 82, + "type": "EmptyImage", + "pos": [ + 1090, + 2790 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "localized_name": "batch_size", + "name": "batch_size", + "type": "INT", + "widget": { + "name": "batch_size" + }, + "link": null + }, + { + "localized_name": "color", + "name": "color", + "type": "INT", + "widget": { + "name": "color" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 133 + ] + } + ], + "properties": { + "Node name for S&R": "EmptyImage" + }, + "widgets_values": [ + 32, + 32, + 1, + 16711680 + ], + "widgets_values_named": { + "width": 32, + "height": 32, + "batch_size": 1, + "color": 16711680 + } + }, + { + "id": 83, + "type": "ImageCompositeMasked", + "pos": [ + 1390, + 2790 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 30, + "mode": 0, + "inputs": [ + { + "localized_name": "destination", + "name": "destination", + "type": "IMAGE", + "link": 132 + }, + { + "localized_name": "source", + "name": "source", + "type": "IMAGE", + "link": 133 + }, + { + "localized_name": "mask", + "name": "mask", + "shape": 7, + "type": "MASK", + "link": 134 + }, + { + "localized_name": "x", + "name": "x", + "type": "INT", + "widget": { + "name": "x" + }, + "link": null + }, + { + "localized_name": "y", + "name": "y", + "type": "INT", + "widget": { + "name": "y" + }, + "link": null + }, + { + "localized_name": "resize_source", + "name": "resize_source", + "type": "BOOLEAN", + "widget": { + "name": "resize_source" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 136 + ] + } + ], + "properties": { + "Node name for S&R": "ImageCompositeMasked" + }, + "widgets_values": [ + 0, + 0, + true + ], + "widgets_values_named": { + "x": 0, + "y": 0, + "resize_source": true + } + }, + { + "id": 84, + "type": "ImageBlend", + "pos": [ + 1690, + 2790 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 38, + "mode": 0, + "inputs": [ + { + "localized_name": "image1", + "name": "image1", + "type": "IMAGE", + "link": 135 + }, + { + "localized_name": "image2", + "name": "image2", + "type": "IMAGE", + "link": 136 + }, + { + "localized_name": "blend_factor", + "name": "blend_factor", + "type": "FLOAT", + "widget": { + "name": "blend_factor" + }, + "link": null + }, + { + "localized_name": "blend_mode", + "name": "blend_mode", + "type": "COMBO", + "widget": { + "name": "blend_mode" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 137 + ] + } + ], + "properties": { + "Node name for S&R": "ImageBlend" + }, + "widgets_values": [ + 0.48, + "normal" + ], + "widgets_values_named": { + "blend_factor": 0.48, + "blend_mode": "normal" + } + }, + { + "id": 89, + "type": "EmptyImage", + "pos": [ + 1090, + 4390 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "localized_name": "batch_size", + "name": "batch_size", + "type": "INT", + "widget": { + "name": "batch_size" + }, + "link": null + }, + { + "localized_name": "color", + "name": "color", + "type": "INT", + "widget": { + "name": "color" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 148 + ] + } + ], + "properties": { + "Node name for S&R": "EmptyImage" + }, + "widgets_values": [ + 32, + 32, + 1, + 16711680 + ], + "widgets_values_named": { + "width": 32, + "height": 32, + "batch_size": 1, + "color": 16711680 + } + }, + { + "id": 90, + "type": "ImageCompositeMasked", + "pos": [ + 1390, + 4390 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 32, + "mode": 0, + "inputs": [ + { + "localized_name": "destination", + "name": "destination", + "type": "IMAGE", + "link": 147 + }, + { + "localized_name": "source", + "name": "source", + "type": "IMAGE", + "link": 148 + }, + { + "localized_name": "mask", + "name": "mask", + "shape": 7, + "type": "MASK", + "link": 149 + }, + { + "localized_name": "x", + "name": "x", + "type": "INT", + "widget": { + "name": "x" + }, + "link": null + }, + { + "localized_name": "y", + "name": "y", + "type": "INT", + "widget": { + "name": "y" + }, + "link": null + }, + { + "localized_name": "resize_source", + "name": "resize_source", + "type": "BOOLEAN", + "widget": { + "name": "resize_source" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 151 + ] + } + ], + "properties": { + "Node name for S&R": "ImageCompositeMasked" + }, + "widgets_values": [ + 0, + 0, + true + ], + "widgets_values_named": { + "x": 0, + "y": 0, + "resize_source": true + } + }, + { + "id": 91, + "type": "ImageBlend", + "pos": [ + 1690, + 4390 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 40, + "mode": 0, + "inputs": [ + { + "localized_name": "image1", + "name": "image1", + "type": "IMAGE", + "link": 150 + }, + { + "localized_name": "image2", + "name": "image2", + "type": "IMAGE", + "link": 151 + }, + { + "localized_name": "blend_factor", + "name": "blend_factor", + "type": "FLOAT", + "widget": { + "name": "blend_factor" + }, + "link": null + }, + { + "localized_name": "blend_mode", + "name": "blend_mode", + "type": "COMBO", + "widget": { + "name": "blend_mode" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 152 + ] + } + ], + "properties": { + "Node name for S&R": "ImageBlend" + }, + "widgets_values": [ + 0.48, + "normal" + ], + "widgets_values_named": { + "blend_factor": 0.48, + "blend_mode": "normal" + } + }, + { + "id": 92, + "type": "EmptyImage", + "pos": [ + 1090, + 5990 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "localized_name": "batch_size", + "name": "batch_size", + "type": "INT", + "widget": { + "name": "batch_size" + }, + "link": null + }, + { + "localized_name": "color", + "name": "color", + "type": "INT", + "widget": { + "name": "color" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 154 + ] + } + ], + "properties": { + "Node name for S&R": "EmptyImage" + }, + "widgets_values": [ + 32, + 32, + 1, + 16711680 + ], + "widgets_values_named": { + "width": 32, + "height": 32, + "batch_size": 1, + "color": 16711680 + } + }, + { + "id": 93, + "type": "ImageCompositeMasked", + "pos": [ + 1390, + 5990 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 34, + "mode": 0, + "inputs": [ + { + "localized_name": "destination", + "name": "destination", + "type": "IMAGE", + "link": 153 + }, + { + "localized_name": "source", + "name": "source", + "type": "IMAGE", + "link": 154 + }, + { + "localized_name": "mask", + "name": "mask", + "shape": 7, + "type": "MASK", + "link": 155 + }, + { + "localized_name": "x", + "name": "x", + "type": "INT", + "widget": { + "name": "x" + }, + "link": null + }, + { + "localized_name": "y", + "name": "y", + "type": "INT", + "widget": { + "name": "y" + }, + "link": null + }, + { + "localized_name": "resize_source", + "name": "resize_source", + "type": "BOOLEAN", + "widget": { + "name": "resize_source" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 157 + ] + } + ], + "properties": { + "Node name for S&R": "ImageCompositeMasked" + }, + "widgets_values": [ + 0, + 0, + true + ], + "widgets_values_named": { + "x": 0, + "y": 0, + "resize_source": true + } + }, + { + "id": 94, + "type": "ImageBlend", + "pos": [ + 1690, + 5990 + ], + "size": [ + 260, + 150 + ], + "flags": {}, + "order": 42, + "mode": 0, + "inputs": [ + { + "localized_name": "image1", + "name": "image1", + "type": "IMAGE", + "link": 156 + }, + { + "localized_name": "image2", + "name": "image2", + "type": "IMAGE", + "link": 157 + }, + { + "localized_name": "blend_factor", + "name": "blend_factor", + "type": "FLOAT", + "widget": { + "name": "blend_factor" + }, + "link": null + }, + { + "localized_name": "blend_mode", + "name": "blend_mode", + "type": "COMBO", + "widget": { + "name": "blend_mode" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "IMAGE", + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 158 + ] + } + ], + "properties": { + "Node name for S&R": "ImageBlend" + }, + "widgets_values": [ + 0.48, + "normal" + ], + "widgets_values_named": { + "blend_factor": 0.48, + "blend_mode": "normal" + } + }, + { + "id": 1, + "type": "MarkdownNote", + "pos": [ + -268.4516045720269, + 2138.3724768826028 + ], + "size": [ + 420, + 210 + ], + "flags": {}, + "order": 17, + "mode": 0, + "inputs": [], + "outputs": [], + "title": "Usage Guide", + "properties": {}, + "widgets_values": [ + "Use a compatible MiniMax-H3 latent-edit server. Select a source video and set the server URL/model in each Generate Video node.\n\nSelect one Save Preview or Save Video output, then use Execute to selected output nodes. Running the whole workflow submits all four cases.\n\nMask: 0 preserves source latents; 1 regenerates. Audio mask applies to the whole clip, not individual sounds. Preview overlays are silent and do not run inference." + ], + "widgets_values_named": { + "text": "Use a compatible MiniMax-H3 latent-edit server. Select a source video and set the server URL/model in each Generate Video node.\n\nSelect one Save Preview or Save Video output, then use Execute to selected output nodes. Running the whole workflow submits all four cases.\n\nMask: 0 preserves source latents; 1 regenerates. Audio mask applies to the whole clip, not individual sounds. Preview overlays are silent and do not run inference." + }, + "color": "#432", + "bgcolor": "#000" + }, + { + "id": 15, + "type": "GetVideoComponents", + "pos": [ + -268.4516045720269, + 2438.3724768826028 + ], + "size": [ + 193.530859375, + 106 + ], + "flags": {}, + "order": 22, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 17 + } + ], + "outputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "links": [ + 120, + 123, + 125, + 128, + 132, + 135 + ] + }, + { + "localized_name": "audio", + "name": "audio", + "type": "AUDIO", + "links": [] + }, + { + "localized_name": "fps", + "name": "fps", + "type": "FLOAT", + "links": [ + 121, + 124, + 131, + 138 + ] + }, + { + "localized_name": "bit_depth", + "name": "bit_depth", + "type": "COMBO", + "links": [] + }, + { + "localized_name": "color_space", + "name": "color_space", + "type": "COMBO", + "links": [] + } + ], + "properties": { + "Node name for S&R": "GetVideoComponents" + } + }, + { + "id": 95, + "type": "SolidMask", + "pos": [ + 580, + 1240 + ], + "size": [ + 300, + 150 + ], + "flags": {}, + "order": 18, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MASK", + "name": "MASK", + "type": "MASK", + "links": [ + 159 + ] + } + ], + "title": "Mask Canvas", + "properties": { + "Node name for S&R": "SolidMask" + }, + "widgets_values": [ + 0, + 1344, + 768 + ], + "widgets_values_named": { + "value": 0, + "width": 1344, + "height": 768 + } + }, + { + "id": 97, + "type": "SolidMask", + "pos": [ + 580, + 2840 + ], + "size": [ + 300, + 150 + ], + "flags": {}, + "order": 19, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MASK", + "name": "MASK", + "type": "MASK", + "links": [ + 161 + ] + } + ], + "title": "Mask Canvas", + "properties": { + "Node name for S&R": "SolidMask" + }, + "widgets_values": [ + 0, + 1344, + 768 + ], + "widgets_values_named": { + "value": 0, + "width": 1344, + "height": 768 + } + }, + { + "id": 66, + "type": "SaveVideo", + "pos": [ + 2530, + 1240 + ], + "size": [ + 420, + 106 + ], + "flags": {}, + "order": 51, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 77 + }, + { + "localized_name": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "localized_name": "format", + "name": "format", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "format.codec", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format.codec" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "title": "Mask Preview (Object Removal)", + "properties": {}, + "widgets_values": [ + "vllm-omni/WF05-object-removal-mask-preview", + "auto", + "auto", + "auto" + ], + "widgets_values_named": { + "filename_prefix": "vllm-omni/WF05-object-removal-mask-preview", + "format": "auto", + "format.codec": "auto", + "codec": "auto" + } + }, + { + "id": 78, + "type": "SaveVideo", + "pos": [ + 2530, + 6040 + ], + "size": [ + 420, + 106 + ], + "flags": {}, + "order": 54, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 119 + }, + { + "localized_name": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "localized_name": "format", + "name": "format", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "format.codec", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format.codec" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "title": "Mask Preview (Extension)", + "properties": {}, + "widgets_values": [ + "vllm-omni/WF05-extension-mask-preview", + "auto", + "auto", + "auto" + ], + "widgets_values_named": { + "filename_prefix": "vllm-omni/WF05-extension-mask-preview", + "format": "auto", + "format.codec": "auto", + "codec": "auto" + } + }, + { + "id": 6, + "type": "VLLMOmniGenerateVideo", + "pos": [ + 1940, + 1870 + ], + "size": [ + 440, + 680 + ], + "flags": {}, + "order": 35, + "mode": 0, + "inputs": [ + { + "localized_name": "frame", + "name": "frame", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "localized_name": "references", + "name": "references", + "shape": 7, + "type": "VIDEO_REFERENCES", + "link": null + }, + { + "localized_name": "sampling_params", + "name": "sampling_params", + "shape": 7, + "type": "SAMPLING_PARAMS", + "link": 10 + }, + { + "localized_name": "lora", + "name": "lora", + "shape": 7, + "type": "REMOTE_LORA", + "link": null + }, + { + "localized_name": "model_params", + "name": "model_params", + "shape": 7, + "type": "VIDEO_PARAMS", + "link": 4 + }, + { + "localized_name": "fast_h3", + "name": "fast_h3", + "shape": 7, + "type": "FASTH3_DEPLOYMENT", + "link": null + }, + { + "localized_name": "latent_edit", + "name": "latent_edit", + "shape": 7, + "type": "LATENT_MASK_EDITING", + "link": 3 + }, + { + "localized_name": "url", + "name": "url", + "type": "STRING", + "widget": { + "name": "url" + }, + "link": null + }, + { + "localized_name": "model", + "name": "model", + "type": "STRING", + "widget": { + "name": "model" + }, + "link": null + }, + { + "localized_name": "prompt", + "name": "prompt", + "type": "STRING", + "widget": { + "name": "prompt" + }, + "link": null + }, + { + "localized_name": "negative_prompt", + "name": "negative_prompt", + "type": "STRING", + "widget": { + "name": "negative_prompt" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "INT", + "widget": { + "name": "fps" + }, + "link": null + }, + { + "localized_name": "duration", + "name": "duration", + "type": "FLOAT", + "widget": { + "name": "duration" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [ + 5 + ] + } + ], + "title": "Generate Video(Object Removal)", + "properties": { + "Node name for S&R": "VLLMOmniGenerateVideo" + }, + "widgets_values": [ + "http://127.0.0.1:8001/v1", + "MiniMaxAI/MiniMax-H3", + "integrated_multimodal_description: The masked area is a flat, seamless continuation of the surrounding wall, matching its color, texture, and lighting. The wall surface remains consistent throughout the video.\n\noverall_soundscape: \n\nnon_diegetic_music: ", + "", + 1344, + 768, + 24, + 5 + ], + "widgets_values_named": { + "url": "http://127.0.0.1:8001/v1", + "model": "MiniMaxAI/MiniMax-H3", + "prompt": "integrated_multimodal_description: The masked area is a flat, seamless continuation of the surrounding wall, matching its color, texture, and lighting. The wall surface remains consistent throughout the video.\n\noverall_soundscape: \n\nnon_diegetic_music: ", + "negative_prompt": "", + "width": 1344, + "height": 768, + "fps": 24, + "duration": 5 + } + }, + { + "id": 96, + "type": "SolidMask", + "pos": [ + 580, + 1360 + ], + "size": [ + 300, + 150 + ], + "flags": {}, + "order": 20, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MASK", + "name": "MASK", + "type": "MASK", + "links": [ + 160 + ] + } + ], + "title": "Mask Rectangle", + "properties": { + "Node name for S&R": "SolidMask" + }, + "widgets_values": [ + 1, + 180, + 168 + ], + "widgets_values_named": { + "value": 1, + "width": 180, + "height": 168 + } + }, + { + "id": 9, + "type": "MaskComposite", + "pos": [ + 580, + 1540 + ], + "size": [ + 300, + 150 + ], + "flags": {}, + "order": 23, + "mode": 0, + "inputs": [ + { + "localized_name": "destination", + "name": "destination", + "type": "MASK", + "link": 159 + }, + { + "localized_name": "source", + "name": "source", + "type": "MASK", + "link": 160 + }, + { + "localized_name": "x", + "name": "x", + "type": "INT", + "widget": { + "name": "x" + }, + "link": null + }, + { + "localized_name": "y", + "name": "y", + "type": "INT", + "widget": { + "name": "y" + }, + "link": null + }, + { + "localized_name": "operation", + "name": "operation", + "type": "COMBO", + "widget": { + "name": "operation" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MASK", + "name": "MASK", + "type": "MASK", + "links": [ + 63, + 127 + ] + } + ], + "title": "Spatial Mask | Rectangle Position", + "properties": { + "Node name for S&R": "MaskComposite" + }, + "widgets_values": [ + 1074, + 493, + "add" + ], + "widgets_values_named": { + "x": 1074, + "y": 493, + "operation": "add" + } + }, + { + "id": 98, + "type": "SolidMask", + "pos": [ + 580, + 2960 + ], + "size": [ + 300, + 150 + ], + "flags": {}, + "order": 21, + "mode": 0, + "inputs": [ + { + "localized_name": "value", + "name": "value", + "type": "FLOAT", + "widget": { + "name": "value" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MASK", + "name": "MASK", + "type": "MASK", + "links": [ + 162 + ] + } + ], + "title": "Mask Rectangle", + "properties": { + "Node name for S&R": "SolidMask" + }, + "widgets_values": [ + 1, + 180, + 168 + ], + "widgets_values_named": { + "value": 1, + "width": 180, + "height": 168 + } + }, + { + "id": 99, + "type": "MaskComposite", + "pos": [ + 580, + 3140 + ], + "size": [ + 300, + 150 + ], + "flags": {}, + "order": 24, + "mode": 0, + "inputs": [ + { + "localized_name": "destination", + "name": "destination", + "type": "MASK", + "link": 161 + }, + { + "localized_name": "source", + "name": "source", + "type": "MASK", + "link": 162 + }, + { + "localized_name": "x", + "name": "x", + "type": "INT", + "widget": { + "name": "x" + }, + "link": null + }, + { + "localized_name": "y", + "name": "y", + "type": "INT", + "widget": { + "name": "y" + }, + "link": null + }, + { + "localized_name": "operation", + "name": "operation", + "type": "COMBO", + "widget": { + "name": "operation" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "MASK", + "name": "MASK", + "type": "MASK", + "links": [ + 35, + 134 + ] + } + ], + "title": "Spatial Mask | Rectangle Position", + "properties": { + "Node name for S&R": "MaskComposite" + }, + "widgets_values": [ + 1074, + 493, + "add" + ], + "widgets_values_named": { + "x": 1074, + "y": 493, + "operation": "add" + } + }, + { + "id": 70, + "type": "SaveVideo", + "pos": [ + 2530, + 2840 + ], + "size": [ + 420, + 106 + ], + "flags": {}, + "order": 52, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 91 + }, + { + "localized_name": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "localized_name": "format", + "name": "format", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "format.codec", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format.codec" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "title": "Mask Preview (Inpainting)", + "properties": {}, + "widgets_values": [ + "vllm-omni/WF05-inpainting-mask-preview", + "auto", + "auto", + "auto" + ], + "widgets_values_named": { + "filename_prefix": "vllm-omni/WF05-inpainting-mask-preview", + "format": "auto", + "format.codec": "auto", + "codec": "auto" + } + }, + { + "id": 4, + "type": "VLLMOmniLatentMaskEditing", + "pos": [ + 1510, + 1870 + ], + "size": [ + 300, + 200 + ], + "flags": {}, + "order": 27, + "mode": 0, + "inputs": [ + { + "localized_name": "source_video", + "name": "source_video", + "shape": 7, + "type": "VIDEO", + "link": 1 + }, + { + "localized_name": "source_audio", + "name": "source_audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "localized_name": "video_mask", + "name": "video_mask", + "shape": 7, + "type": "MASK", + "link": 63 + }, + { + "localized_name": "audio_mask", + "name": "audio_mask", + "shape": 7, + "type": "FLOAT", + "widget": { + "name": "audio_mask" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "latent_edit", + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "links": [ + 3 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniLatentMaskEditing" + }, + "widgets_values": [ + 0 + ], + "widgets_values_named": { + "audio_mask": 0 + } + }, + { + "id": 7, + "type": "SaveVideo", + "pos": [ + 2530, + 1870 + ], + "size": [ + 420, + 106 + ], + "flags": {}, + "order": 43, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 5 + }, + { + "localized_name": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "localized_name": "format", + "name": "format", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "format.codec", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format.codec" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "title": "Save Video (Object Removal)", + "properties": { + "Node name for S&R": "SaveVideo" + }, + "widgets_values": [ + "vllm-omni/WF05-object-removal", + "mp4", + "auto", + "auto" + ], + "widgets_values_named": { + "filename_prefix": "vllm-omni/WF05-object-removal", + "format": "mp4", + "format.codec": "auto", + "codec": "auto" + } + }, + { + "id": 27, + "type": "SaveVideo", + "pos": [ + 2530, + 3470 + ], + "size": [ + 420, + 106 + ], + "flags": {}, + "order": 45, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 31 + }, + { + "localized_name": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "localized_name": "format", + "name": "format", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "format.codec", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format.codec" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "title": "Save Video (Inpainting)", + "properties": { + "Node name for S&R": "SaveVideo" + }, + "widgets_values": [ + "vllm-omni/WF05-inpainting", + "mp4", + "auto", + "auto" + ], + "widgets_values_named": { + "filename_prefix": "vllm-omni/WF05-inpainting", + "format": "mp4", + "format.codec": "auto", + "codec": "auto" + } + }, + { + "id": 24, + "type": "VLLMOmniLatentMaskEditing", + "pos": [ + 1510, + 3470 + ], + "size": [ + 300, + 200 + ], + "flags": {}, + "order": 29, + "mode": 0, + "inputs": [ + { + "localized_name": "source_video", + "name": "source_video", + "shape": 7, + "type": "VIDEO", + "link": 28 + }, + { + "localized_name": "source_audio", + "name": "source_audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "localized_name": "video_mask", + "name": "video_mask", + "shape": 7, + "type": "MASK", + "link": 35 + }, + { + "localized_name": "audio_mask", + "name": "audio_mask", + "shape": 7, + "type": "FLOAT", + "widget": { + "name": "audio_mask" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "latent_edit", + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "links": [ + 29 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniLatentMaskEditing" + }, + "widgets_values": [ + 0 + ], + "widgets_values_named": { + "audio_mask": 0 + } + }, + { + "id": 26, + "type": "VLLMOmniGenerateVideo", + "pos": [ + 1940, + 3470 + ], + "size": [ + 440, + 680 + ], + "flags": {}, + "order": 37, + "mode": 0, + "inputs": [ + { + "localized_name": "frame", + "name": "frame", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "localized_name": "references", + "name": "references", + "shape": 7, + "type": "VIDEO_REFERENCES", + "link": null + }, + { + "localized_name": "sampling_params", + "name": "sampling_params", + "shape": 7, + "type": "SAMPLING_PARAMS", + "link": 33 + }, + { + "localized_name": "lora", + "name": "lora", + "shape": 7, + "type": "REMOTE_LORA", + "link": null + }, + { + "localized_name": "model_params", + "name": "model_params", + "shape": 7, + "type": "VIDEO_PARAMS", + "link": 30 + }, + { + "localized_name": "fast_h3", + "name": "fast_h3", + "shape": 7, + "type": "FASTH3_DEPLOYMENT", + "link": null + }, + { + "localized_name": "latent_edit", + "name": "latent_edit", + "shape": 7, + "type": "LATENT_MASK_EDITING", + "link": 29 + }, + { + "localized_name": "url", + "name": "url", + "type": "STRING", + "widget": { + "name": "url" + }, + "link": null + }, + { + "localized_name": "model", + "name": "model", + "type": "STRING", + "widget": { + "name": "model" + }, + "link": null + }, + { + "localized_name": "prompt", + "name": "prompt", + "type": "STRING", + "widget": { + "name": "prompt" + }, + "link": null + }, + { + "localized_name": "negative_prompt", + "name": "negative_prompt", + "type": "STRING", + "widget": { + "name": "negative_prompt" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "INT", + "widget": { + "name": "fps" + }, + "link": null + }, + { + "localized_name": "duration", + "name": "duration", + "type": "FLOAT", + "widget": { + "name": "duration" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [ + 31 + ] + } + ], + "title": "Generate Video(Inpainting)", + "properties": { + "Node name for S&R": "VLLMOmniGenerateVideo" + }, + "widgets_values": [ + "http://127.0.0.1:8001/v1", + "MiniMaxAI/MiniMax-H3", + "integrated_multimodal_description: A small orange cat rests curled up in the masked area, gently swaying its tail from side to side. Its appearance matches the surrounding anime style, lighting, and shadows.\n\noverall_soundscape:\n\nnon_diegetic_music: ", + "", + 1344, + 768, + 24, + 5 + ], + "widgets_values_named": { + "url": "http://127.0.0.1:8001/v1", + "model": "MiniMaxAI/MiniMax-H3", + "prompt": "integrated_multimodal_description: A small orange cat rests curled up in the masked area, gently swaying its tail from side to side. Its appearance matches the surrounding anime style, lighting, and shadows.\n\noverall_soundscape:\n\nnon_diegetic_music: ", + "negative_prompt": "", + "width": 1344, + "height": 768, + "fps": 24, + "duration": 5 + } + }, + { + "id": 33, + "type": "VLLMOmniLatentMaskEditing", + "pos": [ + 1510, + 5070 + ], + "size": [ + 300, + 200 + ], + "flags": {}, + "order": 31, + "mode": 0, + "inputs": [ + { + "localized_name": "source_video", + "name": "source_video", + "shape": 7, + "type": "VIDEO", + "link": 36 + }, + { + "localized_name": "source_audio", + "name": "source_audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "localized_name": "video_mask", + "name": "video_mask", + "shape": 7, + "type": "MASK", + "link": 47 + }, + { + "localized_name": "audio_mask", + "name": "audio_mask", + "shape": 7, + "type": "FLOAT", + "widget": { + "name": "audio_mask" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "latent_edit", + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "links": [ + 37 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniLatentMaskEditing" + }, + "widgets_values": [ + 0.8 + ], + "widgets_values_named": { + "audio_mask": 0.8 + } + }, + { + "id": 35, + "type": "VLLMOmniGenerateVideo", + "pos": [ + 1940, + 5070 + ], + "size": [ + 440, + 680 + ], + "flags": {}, + "order": 39, + "mode": 0, + "inputs": [ + { + "localized_name": "frame", + "name": "frame", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "localized_name": "references", + "name": "references", + "shape": 7, + "type": "VIDEO_REFERENCES", + "link": null + }, + { + "localized_name": "sampling_params", + "name": "sampling_params", + "shape": 7, + "type": "SAMPLING_PARAMS", + "link": 48 + }, + { + "localized_name": "lora", + "name": "lora", + "shape": 7, + "type": "REMOTE_LORA", + "link": null + }, + { + "localized_name": "model_params", + "name": "model_params", + "shape": 7, + "type": "VIDEO_PARAMS", + "link": 38 + }, + { + "localized_name": "fast_h3", + "name": "fast_h3", + "shape": 7, + "type": "FASTH3_DEPLOYMENT", + "link": null + }, + { + "localized_name": "latent_edit", + "name": "latent_edit", + "shape": 7, + "type": "LATENT_MASK_EDITING", + "link": 37 + }, + { + "localized_name": "url", + "name": "url", + "type": "STRING", + "widget": { + "name": "url" + }, + "link": null + }, + { + "localized_name": "model", + "name": "model", + "type": "STRING", + "widget": { + "name": "model" + }, + "link": null + }, + { + "localized_name": "prompt", + "name": "prompt", + "type": "STRING", + "widget": { + "name": "prompt" + }, + "link": null + }, + { + "localized_name": "negative_prompt", + "name": "negative_prompt", + "type": "STRING", + "widget": { + "name": "negative_prompt" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "INT", + "widget": { + "name": "fps" + }, + "link": null + }, + { + "localized_name": "duration", + "name": "duration", + "type": "FLOAT", + "widget": { + "name": "duration" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [ + 39 + ] + } + ], + "title": "Generate Video(Continuation)", + "properties": { + "Node name for S&R": "VLLMOmniGenerateVideo" + }, + "widgets_values": [ + "http://127.0.0.1:8001/v1", + "MiniMaxAI/MiniMax-H3", + "integrated_multimodal_description: The sky turns completely overcast, with the sun hidden and no direct sunlight, as torrential rain pours over the school entrance in dense visible streaks and splashes on the wet pavement. Distant lightning flashes in the sky behind the school buildings.\n\noverall_soundscape: Loud, steady rainfall, splashing water, and gusts of wind.\n\nnon_diegetic_music: None.", + "", + 1344, + 768, + 24, + 5 + ], + "widgets_values_named": { + "url": "http://127.0.0.1:8001/v1", + "model": "MiniMaxAI/MiniMax-H3", + "prompt": "integrated_multimodal_description: The sky turns completely overcast, with the sun hidden and no direct sunlight, as torrential rain pours over the school entrance in dense visible streaks and splashes on the wet pavement. Distant lightning flashes in the sky behind the school buildings.\n\noverall_soundscape: Loud, steady rainfall, splashing water, and gusts of wind.\n\nnon_diegetic_music: None.", + "negative_prompt": "", + "width": 1344, + "height": 768, + "fps": 24, + "duration": 5 + } + }, + { + "id": 44, + "type": "VLLMOmniMiniMaxH3TemporalMask", + "pos": [ + 580, + 4440 + ], + "size": [ + 420, + 240 + ], + "flags": {}, + "order": 25, + "mode": 0, + "inputs": [ + { + "localized_name": "images", + "name": "images", + "type": "IMAGE", + "link": 120 + }, + { + "localized_name": "source_fps", + "name": "source_fps", + "type": "FLOAT", + "widget": { + "name": "source_fps" + }, + "link": 121 + }, + { + "localized_name": "duration", + "name": "duration", + "type": "FLOAT", + "widget": { + "name": "duration" + }, + "link": null + }, + { + "localized_name": "mode", + "name": "mode", + "type": "COMBO", + "widget": { + "name": "mode" + }, + "link": null + }, + { + "localized_name": "preserve_fraction", + "name": "preserve_fraction", + "type": "FLOAT", + "widget": { + "name": "preserve_fraction" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "mask", + "name": "mask", + "type": "MASK", + "links": [ + 47 + ] + }, + { + "localized_name": "preview_fps", + "name": "preview_fps", + "type": "FLOAT", + "links": [ + 142 + ] + }, + { + "localized_name": "preview_images", + "name": "preview_images", + "type": "IMAGE", + "links": [ + 147, + 150 + ] + }, + { + "localized_name": "preview_mask", + "name": "preview_mask", + "type": "MASK", + "links": [ + 149 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniMiniMaxH3TemporalMask" + }, + "widgets_values": [ + 24, + 5, + "continuation", + 0.2 + ], + "widgets_values_named": { + "source_fps": 24, + "duration": 5, + "mode": "continuation", + "preserve_fraction": 0.2 + } + }, + { + "id": 36, + "type": "SaveVideo", + "pos": [ + 2530, + 5070 + ], + "size": [ + 420, + 106 + ], + "flags": {}, + "order": 47, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 39 + }, + { + "localized_name": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "localized_name": "format", + "name": "format", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "format.codec", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format.codec" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "title": "Save Video (Continuation)", + "properties": { + "Node name for S&R": "SaveVideo" + }, + "widgets_values": [ + "vllm-omni/WF05-continuation", + "mp4", + "auto", + "auto" + ], + "widgets_values_named": { + "filename_prefix": "vllm-omni/WF05-continuation", + "format": "mp4", + "format.codec": "auto", + "codec": "auto" + } + }, + { + "id": 74, + "type": "SaveVideo", + "pos": [ + 2530, + 4440 + ], + "size": [ + 420, + 106 + ], + "flags": {}, + "order": 53, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 105 + }, + { + "localized_name": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "localized_name": "format", + "name": "format", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "format.codec", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format.codec" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "title": "Mask Preview (Continuation)", + "properties": {}, + "widgets_values": [ + "vllm-omni/WF05-continuation-mask-preview", + "auto", + "auto", + "auto" + ], + "widgets_values_named": { + "filename_prefix": "vllm-omni/WF05-continuation-mask-preview", + "format": "auto", + "format.codec": "auto", + "codec": "auto" + } + }, + { + "id": 50, + "type": "VLLMOmniGenerateVideo", + "pos": [ + 1940, + 6670 + ], + "size": [ + 440, + 680 + ], + "flags": {}, + "order": 41, + "mode": 0, + "inputs": [ + { + "localized_name": "frame", + "name": "frame", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "localized_name": "references", + "name": "references", + "shape": 7, + "type": "VIDEO_REFERENCES", + "link": null + }, + { + "localized_name": "sampling_params", + "name": "sampling_params", + "shape": 7, + "type": "SAMPLING_PARAMS", + "link": 61 + }, + { + "localized_name": "lora", + "name": "lora", + "shape": 7, + "type": "REMOTE_LORA", + "link": null + }, + { + "localized_name": "model_params", + "name": "model_params", + "shape": 7, + "type": "VIDEO_PARAMS", + "link": 51 + }, + { + "localized_name": "fast_h3", + "name": "fast_h3", + "shape": 7, + "type": "FASTH3_DEPLOYMENT", + "link": null + }, + { + "localized_name": "latent_edit", + "name": "latent_edit", + "shape": 7, + "type": "LATENT_MASK_EDITING", + "link": 50 + }, + { + "localized_name": "url", + "name": "url", + "type": "STRING", + "widget": { + "name": "url" + }, + "link": null + }, + { + "localized_name": "model", + "name": "model", + "type": "STRING", + "widget": { + "name": "model" + }, + "link": null + }, + { + "localized_name": "prompt", + "name": "prompt", + "type": "STRING", + "widget": { + "name": "prompt" + }, + "link": null + }, + { + "localized_name": "negative_prompt", + "name": "negative_prompt", + "type": "STRING", + "widget": { + "name": "negative_prompt" + }, + "link": null + }, + { + "localized_name": "width", + "name": "width", + "type": "INT", + "widget": { + "name": "width" + }, + "link": null + }, + { + "localized_name": "height", + "name": "height", + "type": "INT", + "widget": { + "name": "height" + }, + "link": null + }, + { + "localized_name": "fps", + "name": "fps", + "type": "INT", + "widget": { + "name": "fps" + }, + "link": null + }, + { + "localized_name": "duration", + "name": "duration", + "type": "FLOAT", + "widget": { + "name": "duration" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [ + 52 + ] + } + ], + "title": "Generate Video(Extension)", + "properties": { + "Node name for S&R": "VLLMOmniGenerateVideo" + }, + "widgets_values": [ + "http://127.0.0.1:8001/v1", + "MiniMaxAI/MiniMax-H3", + "integrated_multimodal_description: Cut to the same school on a summer night beneath a deep blue sky dotted with stars. A firework rises from behind the school buildings, leaving a glowing trail; the camera slowly tilts upward to follow it into the sky, ending as it bursts into a large bloom of colorful sparks.\n\noverall_soundscape: Soft summer insect sounds, followed by the firework rising with a whistle and bursting with a distant boom and crackling sparks.\n\nnon_diegetic_music: N/A", + "", + 1344, + 768, + 24, + 10 + ], + "widgets_values_named": { + "url": "http://127.0.0.1:8001/v1", + "model": "MiniMaxAI/MiniMax-H3", + "prompt": "integrated_multimodal_description: Cut to the same school on a summer night beneath a deep blue sky dotted with stars. A firework rises from behind the school buildings, leaving a glowing trail; the camera slowly tilts upward to follow it into the sky, ending as it bursts into a large bloom of colorful sparks.\n\noverall_soundscape: Soft summer insect sounds, followed by the firework rising with a whistle and bursting with a distant boom and crackling sparks.\n\nnon_diegetic_music: N/A", + "negative_prompt": "", + "width": 1344, + "height": 768, + "fps": 24, + "duration": 10 + } + }, + { + "id": 48, + "type": "VLLMOmniLatentMaskEditing", + "pos": [ + 1510, + 6670 + ], + "size": [ + 300, + 200 + ], + "flags": {}, + "order": 33, + "mode": 0, + "inputs": [ + { + "localized_name": "source_video", + "name": "source_video", + "shape": 7, + "type": "VIDEO", + "link": 49 + }, + { + "localized_name": "source_audio", + "name": "source_audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "localized_name": "video_mask", + "name": "video_mask", + "shape": 7, + "type": "MASK", + "link": 60 + }, + { + "localized_name": "audio_mask", + "name": "audio_mask", + "shape": 7, + "type": "FLOAT", + "widget": { + "name": "audio_mask" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "latent_edit", + "name": "latent_edit", + "type": "LATENT_MASK_EDITING", + "links": [ + 50 + ] + } + ], + "properties": { + "Node name for S&R": "VLLMOmniLatentMaskEditing" + }, + "widgets_values": [ + 0.9 + ], + "widgets_values_named": { + "audio_mask": 0.9 + } + }, + { + "id": 51, + "type": "SaveVideo", + "pos": [ + 2530, + 6670 + ], + "size": [ + 420, + 338 + ], + "flags": {}, + "order": 49, + "mode": 0, + "inputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "link": 52 + }, + { + "localized_name": "filename_prefix", + "name": "filename_prefix", + "type": "STRING", + "widget": { + "name": "filename_prefix" + }, + "link": null + }, + { + "localized_name": "format", + "name": "format", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "format.codec", + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "format.codec" + }, + "link": null + }, + { + "localized_name": "codec", + "name": "codec", + "shape": 7, + "type": "COMFY_DYNAMICCOMBO_V3", + "widget": { + "name": "codec" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "video", + "name": "video", + "type": "VIDEO", + "links": [] + } + ], + "title": "Save Video (Extension)", + "properties": { + "Node name for S&R": "SaveVideo" + }, + "widgets_values": [ + "vllm-omni/WF05-extension", + "mp4", + "auto", + "auto" + ], + "widgets_values_named": { + "filename_prefix": "vllm-omni/WF05-extension", + "format": "mp4", + "format.codec": "auto", + "codec": "auto" + } + } + ], + "links": [ + [ + 1, + 2, + 0, + 4, + 0, + "VIDEO" + ], + [ + 3, + 4, + 0, + 6, + 6, + "LATENT_MASK_EDITING" + ], + [ + 4, + 5, + 0, + 6, + 4, + "VIDEO_PARAMS" + ], + [ + 5, + 6, + 0, + 7, + 0, + "VIDEO" + ], + [ + 10, + 11, + 0, + 6, + 2, + "SAMPLING_PARAMS" + ], + [ + 17, + 2, + 0, + 15, + 0, + "VIDEO" + ], + [ + 28, + 2, + 0, + 24, + 0, + "VIDEO" + ], + [ + 29, + 24, + 0, + 26, + 6, + "LATENT_MASK_EDITING" + ], + [ + 30, + 25, + 0, + 26, + 4, + "VIDEO_PARAMS" + ], + [ + 31, + 26, + 0, + 27, + 0, + "VIDEO" + ], + [ + 33, + 30, + 0, + 26, + 2, + "SAMPLING_PARAMS" + ], + [ + 35, + 99, + 0, + 24, + 2, + "MASK" + ], + [ + 36, + 2, + 0, + 33, + 0, + "VIDEO" + ], + [ + 37, + 33, + 0, + 35, + 6, + "LATENT_MASK_EDITING" + ], + [ + 38, + 34, + 0, + 35, + 4, + "VIDEO_PARAMS" + ], + [ + 39, + 35, + 0, + 36, + 0, + "VIDEO" + ], + [ + 47, + 44, + 0, + 33, + 2, + "MASK" + ], + [ + 48, + 46, + 0, + 35, + 2, + "SAMPLING_PARAMS" + ], + [ + 49, + 2, + 0, + 48, + 0, + "VIDEO" + ], + [ + 50, + 48, + 0, + 50, + 6, + "LATENT_MASK_EDITING" + ], + [ + 51, + 49, + 0, + 50, + 4, + "VIDEO_PARAMS" + ], + [ + 52, + 50, + 0, + 51, + 0, + "VIDEO" + ], + [ + 60, + 59, + 0, + 48, + 2, + "MASK" + ], + [ + 61, + 61, + 0, + 50, + 2, + "SAMPLING_PARAMS" + ], + [ + 63, + 9, + 0, + 4, + 2, + "MASK" + ], + [ + 77, + 65, + 0, + 66, + 0, + "VIDEO" + ], + [ + 91, + 69, + 0, + 70, + 0, + "VIDEO" + ], + [ + 105, + 73, + 0, + 74, + 0, + "VIDEO" + ], + [ + 119, + 77, + 0, + 78, + 0, + "VIDEO" + ], + [ + 120, + 15, + 0, + 44, + 0, + "IMAGE" + ], + [ + 121, + 15, + 2, + 44, + 1, + "FLOAT" + ], + [ + 123, + 15, + 0, + 59, + 0, + "IMAGE" + ], + [ + 124, + 15, + 2, + 59, + 1, + "FLOAT" + ], + [ + 125, + 15, + 0, + 80, + 0, + "IMAGE" + ], + [ + 126, + 79, + 0, + 80, + 1, + "IMAGE" + ], + [ + 127, + 9, + 0, + 80, + 2, + "MASK" + ], + [ + 128, + 15, + 0, + 81, + 0, + "IMAGE" + ], + [ + 129, + 80, + 0, + 81, + 1, + "IMAGE" + ], + [ + 130, + 81, + 0, + 65, + 0, + "IMAGE" + ], + [ + 131, + 15, + 2, + 65, + 2, + "FLOAT" + ], + [ + 132, + 15, + 0, + 83, + 0, + "IMAGE" + ], + [ + 133, + 82, + 0, + 83, + 1, + "IMAGE" + ], + [ + 134, + 99, + 0, + 83, + 2, + "MASK" + ], + [ + 135, + 15, + 0, + 84, + 0, + "IMAGE" + ], + [ + 136, + 83, + 0, + 84, + 1, + "IMAGE" + ], + [ + 137, + 84, + 0, + 69, + 0, + "IMAGE" + ], + [ + 138, + 15, + 2, + 69, + 2, + "FLOAT" + ], + [ + 142, + 44, + 1, + 73, + 2, + "FLOAT" + ], + [ + 146, + 59, + 1, + 77, + 2, + "FLOAT" + ], + [ + 147, + 44, + 2, + 90, + 0, + "IMAGE" + ], + [ + 148, + 89, + 0, + 90, + 1, + "IMAGE" + ], + [ + 149, + 44, + 3, + 90, + 2, + "MASK" + ], + [ + 150, + 44, + 2, + 91, + 0, + "IMAGE" + ], + [ + 151, + 90, + 0, + 91, + 1, + "IMAGE" + ], + [ + 152, + 91, + 0, + 73, + 0, + "IMAGE" + ], + [ + 153, + 59, + 2, + 93, + 0, + "IMAGE" + ], + [ + 154, + 92, + 0, + 93, + 1, + "IMAGE" + ], + [ + 155, + 59, + 3, + 93, + 2, + "MASK" + ], + [ + 156, + 59, + 2, + 94, + 0, + "IMAGE" + ], + [ + 157, + 93, + 0, + 94, + 1, + "IMAGE" + ], + [ + 158, + 94, + 0, + 77, + 0, + "IMAGE" + ], + [ + 159, + 95, + 0, + 9, + 0, + "MASK" + ], + [ + 160, + 96, + 0, + 9, + 1, + "MASK" + ], + [ + 161, + 97, + 0, + 99, + 0, + "MASK" + ], + [ + 162, + 98, + 0, + 99, + 1, + "MASK" + ] + ], + "groups": [ + { + "id": 1, + "title": "Shared Input", + "bounding": [ + -308.451604572027, + 1080, + 473.7665553206702, + 1508.3724768826028 + ], + "color": "#596477", + "flags": {} + }, + { + "id": 2, + "title": "Output Hub", + "bounding": [ + 2470, + 1080, + 540, + 6400 + ], + "color": "#444", + "flags": {} + }, + { + "id": 10, + "title": "Case 01 | Object Removal", + "bounding": [ + 540, + 1080, + 1880, + 1540 + ], + "color": "#77528d", + "flags": {} + }, + { + "id": 20, + "title": "Case 01 | Object Removal Output", + "bounding": [ + 2490, + 1130, + 500, + 1490 + ], + "color": "#77528d", + "flags": {} + }, + { + "id": 11, + "title": "Case 02 | Inpainting", + "bounding": [ + 540, + 2680, + 1880, + 1540 + ], + "color": "#357e80", + "flags": {} + }, + { + "id": 21, + "title": "Case 02 | Inpainting Output", + "bounding": [ + 2490, + 2730, + 500, + 1490 + ], + "color": "#357e80", + "flags": {} + }, + { + "id": 12, + "title": "Case 03 | Continuation", + "bounding": [ + 540, + 4280, + 1880, + 1540 + ], + "color": "#596477", + "flags": {} + }, + { + "id": 22, + "title": "Case 03 | Continuation Output", + "bounding": [ + 2490, + 4330, + 500, + 1490 + ], + "color": "#596477", + "flags": {} + }, + { + "id": 13, + "title": "Case 04 | Extension", + "bounding": [ + 540, + 5880, + 1880, + 1540 + ], + "color": "#b06634", + "flags": {} + }, + { + "id": 23, + "title": "Case 04 | Extension Output", + "bounding": [ + 2490, + 5930, + 500, + 1490 + ], + "color": "#b06634", + "flags": {} + }, + { + "id": 24, + "title": "Object Removal | Mask Generation", + "bounding": [ + 560, + 1130, + 480, + 620 + ], + "color": "#65794e", + "flags": {} + }, + { + "id": 25, + "title": "Inpainting | Mask Generation", + "bounding": [ + 560, + 2730, + 480, + 620 + ], + "color": "#65794e", + "flags": {} + }, + { + "id": 26, + "title": "Continuation | Mask Generation", + "bounding": [ + 560, + 4330, + 480, + 620 + ], + "color": "#65794e", + "flags": {} + }, + { + "id": 27, + "title": "Extension | Mask Generation", + "bounding": [ + 560, + 5930, + 480, + 620 + ], + "color": "#65794e", + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.9837545257996714, + "offset": [ + -1807.8247536090078, + -6349.252392678693 + ] + } + }, + "version": 0.4 +} diff --git a/apps/ComfyUI-vLLM-Omni/web/main.js b/apps/ComfyUI-vLLM-Omni/web/main.js index 9aba6260e..f8c633e11 100644 --- a/apps/ComfyUI-vLLM-Omni/web/main.js +++ b/apps/ComfyUI-vLLM-Omni/web/main.js @@ -52,6 +52,7 @@ const FAMILY_BY_OUTPUT = { REMOTE_LORA: "deployment", FASTH3_DEPLOYMENT: "deployment", VIDEO_REFERENCES: "media", + LATENT_MASK_EDITING: "media", }; function familyOf(nodeData) { diff --git a/tests/e2e/features/comfyui/mock_videos_server.py b/tests/e2e/features/comfyui/mock_videos_server.py new file mode 100644 index 000000000..c2ddfbb5e --- /dev/null +++ b/tests/e2e/features/comfyui/mock_videos_server.py @@ -0,0 +1,115 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +"""Minimal mock of vLLM-Omni's ``/v1/videos`` API for e2e tests. + +Records every received multipart field to ``MOCK_STATE_FILE`` (JSON) so a test +can assert exactly what the client serialized, and returns a synthetic MP4. +""" + +import asyncio +import json +import os +import subprocess +import uuid + +from fastapi import FastAPI, Request +from fastapi.responses import Response +from starlette.datastructures import UploadFile + +app = FastAPI() +_JOBS: dict[str, dict] = {} +_STATE_FILE = os.environ.get("MOCK_STATE_FILE") + + +def _record(fields: dict) -> None: + if _STATE_FILE: + with open(_STATE_FILE, "w") as f: + json.dump(fields, f) + + +def _synthetic_mp4() -> bytes: + return subprocess.run( + [ + "ffmpeg", + "-hide_banner", + "-loglevel", + "error", + "-y", + "-f", + "lavfi", + "-i", + "testsrc2=size=320x240:rate=24:duration=1", + "-f", + "lavfi", + "-i", + "sine=frequency=440:sample_rate=32000:duration=1", + "-map", + "0:v", + "-map", + "1:a", + "-shortest", + "-c:v", + "libx264", + "-pix_fmt", + "yuv420p", + "-c:a", + "aac", + "-ar", + "32000", + "-ac", + "2", + "-movflags", + "frag_keyframe+empty_moov", + "-f", + "mp4", + "pipe:1", + ], + check=True, + capture_output=True, + ).stdout + + +@app.post("/v1/videos") +async def create_video(request: Request): + form = await request.form() + fields = {} + for key, value in form.items(): + if isinstance(value, UploadFile): + data = await value.read() + fields[key] = { + "file": True, + "size": len(data), + "filename": value.filename, + "content_type": value.content_type, + } + if value.content_type == "application/json": + fields[key]["json"] = json.loads(data) + else: + fields[key] = str(value) + _record(fields) + + job_id = str(uuid.uuid4()) + _JOBS[job_id] = {"status": "queued"} + return {"id": job_id, "status": "queued"} + + +@app.get("/v1/videos/{job_id}") +async def get_video(job_id: str): + if job_id not in _JOBS: + return {"status": "failed", "error": "unknown job"} + return {"id": job_id, "status": "completed"} + + +@app.get("/v1/videos/{job_id}/content") +async def get_video_content(job_id: str): + if job_id not in _JOBS: + return Response(status_code=404) + mp4 = await asyncio.to_thread(_synthetic_mp4) + return Response(content=mp4, media_type="video/mp4") + + +@app.delete("/v1/videos/{job_id}") +async def delete_video(job_id: str): + _JOBS.pop(job_id, None) + return {"deleted": True} diff --git a/tests/e2e/features/comfyui/test_latent_mask.py b/tests/e2e/features/comfyui/test_latent_mask.py new file mode 100644 index 000000000..e208fd5b4 --- /dev/null +++ b/tests/e2e/features/comfyui/test_latent_mask.py @@ -0,0 +1,80 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +"""Unit tests for latent-mask serialization helpers.""" + +import json + +import pytest +import torch +from comfyui_vllm_omni.utils.latent_mask import scalar_mask_to_json, video_mask_to_grid_json + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +def test_scalar_mask_to_json(): + assert scalar_mask_to_json(0.5) == "0.5" + assert scalar_mask_to_json(1.0) == "1.0" + with pytest.raises(ValueError): + scalar_mask_to_json(1.5) + with pytest.raises(ValueError): + scalar_mask_to_json(-0.1) + + +def test_video_mask_grid_shape(): + mask = torch.zeros(256, 448) + grid = json.loads(video_mask_to_grid_json(mask, width=448, height=256, num_frames=107)) + assert len(grid) == 2 + 5 * ((107 - 5) // 17) + assert len(grid[0]) == 256 // 16 + assert len(grid[0][0]) == 448 // 16 + + +def test_video_mask_grid_shape_non_aligned_frames(): + # 96 snaps up to 107 (17n+5), so Tv == 32, matching the server. + grid = json.loads(video_mask_to_grid_json(torch.zeros(256, 448), width=448, height=256, num_frames=96)) + assert len(grid) == 32 + + +def test_video_mask_grid_shape_short_clip(): + # num_frames <= 5 -> Tv == 2. + grid = json.loads(video_mask_to_grid_json(torch.zeros(64, 64), width=64, height=64, num_frames=1)) + assert len(grid) == 2 + + +def test_video_mask_grid_shape_unaligned_width(): + # width=500 floors to 480 (multiple of 32) -> gw == 30, not 31. + grid = json.loads(video_mask_to_grid_json(torch.zeros(64, 64), width=500, height=256, num_frames=22)) + assert len(grid[0][0]) == 30 + + +def test_video_mask_3d_input(): + grid = json.loads(video_mask_to_grid_json(torch.ones(1, 256, 448), width=448, height=256, num_frames=107)) + assert grid[0][0][0] == 1.0 + + +def test_video_mask_temporal(): + # 3D mask: first half 0 (preserve), second half 1 (regenerate). + mask = torch.zeros(7, 64, 64) + mask[4:] = 1.0 + grid = json.loads(video_mask_to_grid_json(mask, width=64, height=64, num_frames=22)) + assert len(grid) == 7 + assert all(v == 0.0 for row in grid[0] for v in row) + assert all(v == 1.0 for row in grid[-1] for v in row) + + +def test_video_mask_temporal_resize(): + # 2 temporal slices resize to tv == 7. + mask = torch.zeros(2, 64, 64) + mask[1] = 1.0 + grid = json.loads(video_mask_to_grid_json(mask, width=64, height=64, num_frames=22)) + assert len(grid) == 7 + + +def test_video_mask_uniform_preserved(): + grid = json.loads(video_mask_to_grid_json(torch.full((64, 64), 0.5), width=64, height=64, num_frames=107)) + assert all(v == 0.5 for row in grid for cell in row for v in cell) + + +def test_video_mask_values_in_range(): + grid = json.loads(video_mask_to_grid_json(torch.rand(128, 128), width=128, height=128, num_frames=22)) + assert all(0.0 <= v <= 1.0 for row in grid for cell in row for v in cell) diff --git a/tests/e2e/features/comfyui/test_latent_mask_editing.py b/tests/e2e/features/comfyui/test_latent_mask_editing.py new file mode 100644 index 000000000..9162974b6 --- /dev/null +++ b/tests/e2e/features/comfyui/test_latent_mask_editing.py @@ -0,0 +1,99 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +"""End-to-end test for latent-mask editing serialization. + +Exercises ``VLLMOmniClient.generate_video`` with a ``latent_edit`` payload +against a mock ``/v1/videos`` server and asserts the multipart fields the +client produced. Runs on CPU: the ComfyUI ``comfy_api`` / ``comfy_extras`` +modules are mocked by this directory's ``conftest.py``. +""" + +import asyncio +import json +import os +import socket +import subprocess +import sys +import time + +import pytest +import torch +from comfy_api.input import VideoInput +from comfyui_vllm_omni.utils.api_client import VLLMOmniClient + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + +_TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) + + +def _free_port() -> int: + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +@pytest.fixture +def mock_server(tmp_path): + port = _free_port() + state_file = tmp_path / "state.json" + env = dict(os.environ, MOCK_STATE_FILE=str(state_file)) + proc = subprocess.Popen( + [sys.executable, "-m", "uvicorn", "mock_videos_server:app", "--host", "127.0.0.1", "--port", str(port)], + cwd=_TESTS_DIR, + env=env, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + try: + deadline = time.time() + 30 + while time.time() < deadline: + try: + with socket.create_connection(("127.0.0.1", port), timeout=1): + break + except OSError: + time.sleep(0.2) + yield f"http://127.0.0.1:{port}/v1", state_file + finally: + proc.terminate() + proc.wait(timeout=10) + + +def test_latent_edit_serialization(mock_server): + base_url, state_file = mock_server + + # A non-trivial video mask requires a source video. The mocked VideoInput + # only needs to provide ``save_to`` for the client's multipart upload. + source_video = VideoInput(b"mock_source_video") + mask = torch.zeros(1, 120, 160) + latent_edit = {"source_video": source_video, "video_mask": mask, "audio_mask": 0.5} + + async def run(): + client = VLLMOmniClient(base_url) + return await client.generate_video( + model="MiniMaxAI/MiniMax-H3", + prompt="restyle the clip", + width=160, + height=120, + num_frames=22, + fps=24, + latent_edit=latent_edit, + ) + + out = asyncio.run(run()) + assert out is not None + + with open(state_file) as f: + fields = json.load(f) + + assert fields["source_video"]["file"] is True + assert fields["source_video"]["content_type"] == "video/mp4" + for name in ("video_noise_mask", "audio_noise_mask"): + assert fields[name]["file"] is True + assert fields[name]["content_type"] == "application/json" + assert fields["audio_noise_mask"]["json"] == 0.5 + + video_mask = fields["video_noise_mask"]["json"] + assert len(video_mask) == 7 # num_frames=22 aligns to 22 (17n+5) -> Tv = 7 + assert len(video_mask[0]) == 6 # height 120 floors to 96 (multiple of 32) -> gh 6 + assert len(video_mask[0][0]) == 10 # width 160 -> gw 10 From 856d19f42cfccb1ae97aea1131b89662368baadb Mon Sep 17 00:00:00 2001 From: Zhou Taichang Date: Mon, 21 Sep 2026 21:12:00 +0800 Subject: [PATCH 5/5] [Benchmark] Add LingBot-World realtime streaming benchmark (#7645) Signed-off-by: tzhouam Co-authored-by: Claude Opus 5 (1M context) --- benchmarks/README.md | 11 + benchmarks/lingbot_world/README.md | 224 +++++ benchmarks/lingbot_world/__init__.py | 2 + .../benchmark_lingbot_world_realtime.py | 642 ++++++++++++++ .../configs/single_gpu_eager.yaml | 45 + .../lingbot_world/configs/usp4_compiled.yaml | 60 ++ benchmarks/lingbot_world/workload.py | 631 ++++++++++++++ .../benchmarks/test_lingbot_world_realtime.py | 798 ++++++++++++++++++ 8 files changed, 2413 insertions(+) create mode 100644 benchmarks/lingbot_world/README.md create mode 100644 benchmarks/lingbot_world/__init__.py create mode 100644 benchmarks/lingbot_world/benchmark_lingbot_world_realtime.py create mode 100644 benchmarks/lingbot_world/configs/single_gpu_eager.yaml create mode 100644 benchmarks/lingbot_world/configs/usp4_compiled.yaml create mode 100644 benchmarks/lingbot_world/workload.py create mode 100644 tests/benchmarks/test_lingbot_world_realtime.py diff --git a/benchmarks/README.md b/benchmarks/README.md index 182bba02d..1a23b1d48 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -41,6 +41,17 @@ LingBot-Video MoE transformer. - **Dense pipeline**: decoded-video MAE, MSE, PSNR, latency, and optional steady-state timings - **MoE transformer**: bitwise router, sparse-block, shared-expert, and full-transformer parity +### [LingBot-World](lingbot_world/README.md) — Realtime Streaming Cadence + +Client-side benchmark for a `WS /v1/realtime/video` world-model session, where one +request is a whole rollout and one AR block streams out as one video chunk. It +measures a cadence rather than a throughput: there is exactly one session, and what +matters is whether the next chunk arrives before the viewer finishes the last one. + +- **Layout**: `lingbot_world/benchmark_lingbot_world_realtime.py` (WS client), `lingbot_world/workload.py` (rollout and metric math), `lingbot_world/configs/` (single-GPU eager and Ulysses-4 compiled deploy configs) +- **Workload**: built-in image-conditioned rollout with a generated camera script, or a JSON spec with per-chunk actions and mid-rollout prompt updates +- **Key metrics**: TTFC (time to first chunk), steady-state chunk inter-arrival percentiles, VIDEO_RTF (wall seconds per video second; lower is better), chunk-deadline attainment, and simulated playback underruns + ### [Distributed](distributed/omni_connectors/README.md) — RDMA Connector Testing RDMA environment setup and transfer tests for `MooncakeTransferEngineConnector`, including pytest-based single-node checks and manual cross-node benchmarks. diff --git a/benchmarks/lingbot_world/README.md b/benchmarks/lingbot_world/README.md new file mode 100644 index 000000000..1dce2d7ce --- /dev/null +++ b/benchmarks/lingbot_world/README.md @@ -0,0 +1,224 @@ +# LingBot-World Benchmarks — Realtime Streaming Cadence + +`benchmark_lingbot_world_realtime.py` measures LingBot-World 2.0 as it is +actually deployed: one `WS /v1/realtime/video` session that streams one video +chunk per AR block, for as long as the world keeps running. + +## Why this is not the diffusion serving benchmark + +[`benchmarks/diffusion`](../diffusion/README.md) measures "one request, one +finished video" and reports request throughput. LingBot-World has neither shape: + +- **One request is the whole rollout.** `prepare_encode` runs once, then every AR + block is four DMD steps plus one clean-KV commit forward, and each block + streams out as a chunk. Throughput in requests per second is meaningless. +- **There is exactly one session.** The stepwise deploy config pins + `max_num_seqs: 1`, and the AR-Diffusion consumer rejects a stage with more than + one replica, because session-affine routing does not exist yet. There is no + concurrency to sweep. +- **What matters is cadence.** A deployment is healthy when chunk *N + 1* arrives + before a viewer finishes watching chunk *N*. That is a latency distribution + against a wall clock, not a throughput number. + +## Quick start + +Serve the model, then point the benchmark at it. The benchmark never starts a +server. + +```bash +# Terminal 1 — one GPU, eager (portable smoke configuration) +vllm serve robbyant/lingbot-world-v2-14b-causal-fast-diffusers \ + --omni \ + --deploy-config benchmarks/lingbot_world/configs/single_gpu_eager.yaml \ + --port 8000 + +# Terminal 2 +pip install websockets +python benchmarks/lingbot_world/benchmark_lingbot_world_realtime.py \ + --host 127.0.0.1 --port 8000 \ + --num-chunks 16 \ + --output-json /tmp/lingbot_world_realtime.json +``` + +With no `--image`, the first frame comes from the shared vLLM image asset, so the +default workload needs no setup and this repository carries no benchmark binary. +Pass `--image /path/to/frame.png` to condition the rollout on your own frame. + +### Realtime topology + +The single-GPU config is a smoke configuration, not a realtime one: one card runs +all five forwards of every block serially and nothing is compiled. Use the +four-GPU config for numbers that mean anything about realtime behavior. + +```bash +vllm serve robbyant/lingbot-world-v2-14b-causal-fast-diffusers \ + --omni \ + --deploy-config benchmarks/lingbot_world/configs/usp4_compiled.yaml \ + --port 8000 + +python benchmarks/lingbot_world/benchmark_lingbot_world_realtime.py \ + --port 8000 --num-chunks 40 --warmup-sessions 1 --sessions 3 \ + --target-fps 12 --output-json /tmp/lingbot_world_repeated.json +``` + +`usp4_compiled.yaml` sets pure Ulysses sequence parallelism over four ranks and +`enforce_eager: false`. The pipeline validates the parallel shape itself: +`sequence_parallel_size` must equal `ulysses_degree`, `ring_degree` and +`allgather_degree` must be 1, `ulysses_mode` must be `strict`, and pipeline, +CFG, and VAE parallel sizes above 1 are rejected. This benchmark config sets +`tensor_parallel_size: 1`; the pipeline validator does not reject larger TP sizes. + +Compiled mode can pay compilation and capture costs on the first rollout. +Use `--warmup-sessions 1` before measured repetitions. Excluding the first six +chunks addresses the attention-window ramp; it does not guarantee that +compilation and capture have finished. + +## Metrics + +| Metric | Meaning | +| --- | --- | +| `ttfc_ms` | **Time to first chunk**: `session.start` sent until the first chunk's bytes arrive. Pays for prompt encode, first-frame VAE encode, and any compile or capture. | +| `interval_all` | Inter-arrival latency across every chunk: mean, median, std, min, p90, p95, p99, max. | +| `interval_steady` | The same summary over steady-state chunks only. | +| `video_rtf` | Wall seconds spent per second of video, matching vLLM-Omni's own RTF (`generation_s / video_duration`). **Lower is better; below 1.0 is real time.** | +| `video_rtfx` | The reciprocal, under the name this repo uses for it: video seconds per wall second, where higher is better. | +| `steady_video_rtf` | `video_rtf` restricted to steady-state chunks. | +| `chunk_deadline_ms` | Wall-clock budget one steady chunk must meet to hold real time: 12 frames ÷ `--target-fps`. Override directly with `--chunk-slo-ms`. | +| `slo_attainment` | Fraction of steady chunks that met the deadline. | +| `playback` | Underrun count, total stall, and stall ratio from a simulated viewer. | + +### Which frame rate counts as real time + +**The checkpoint declares no frame rate.** There is none in `model_index.json`, the +scheduler config, or the transformer config, and the pipeline never reads one. A +chunk is twelve pixel frames; how long those frames *last* is a property of how the +output is played, not of the model. + +That choice moves the verdict without moving anything measured: + +| basis | video per chunk | deadline | RTF at a measured 1357 ms/chunk | +| --- | --- | --- | --- | +| 16 fps | 0.750 s | 750 ms | 1.809 | +| 12 fps | 1.000 s | 1000 ms | 1.357 | + +So `--fps` (sent to the server, which uses it to label the muxed MP4) and +`--target-fps` (the real-time basis for RTF, the deadline, and the playback +simulation) are separate flags. `--target-fps` defaults to `--fps`; set it when the +two differ, and the report prints both. State the basis whenever you quote a number +— an RTF without its fps is not a measurement. (RTF is a cost here, so a bigger number is a +worse result; a steady chunk's RTF is simply its interval divided by its video duration.) +An RTF above 1 means the measured deployment is slower than real time at the +stated target frame rate. The benchmark measures this failure as well as success. + +### Why warmup chunks are not optional + +The DiT attends over a sliding window of `sliding_window_num_frames = 18` latent +frames with `sink_size = 9`, so the key count a block attends over keeps growing +until eighteen latent frames of history exist — six chunks at three frames per +block. **Chunk cost is not stationary before chunk six.** A mean over a short +rollout is therefore lower than anything a running session will sustain, which is +why `--warmup-chunks` defaults to 6 and steady-state metrics are reported +separately. Keep `--num-chunks` comfortably above the warmup, or the benchmark +tells you no chunk reached steady state. The terminal chunk skips next-chunk +preparation, so it is excluded from steady metrics too; it remains in overall +RTF, all-chunk intervals, and playback simulation. + +The default 16-chunk rollout provides only nine steady intervals. Percentiles +are interpolated sample summaries; the report warns when fewer than 100 +intervals contribute, and even 100 samples do not guarantee a stable p99. +Use longer rollouts for tail analysis and at least `--sessions 3` after +`--warmup-sessions 1` for reported comparisons. Session warmup pays compilation +cost; chunk warmup separately excludes the attention-window ramp. The command +above yields 33 steady intervals per session (indices 6 through 38), or 99 across +three sessions. Report each session's mean and the spread across sessions; this +sample still does not establish a reliable p99. + +### Why playback is simulated + +A p99 interval alone cannot say whether a viewer saw a stall: the same p99 is +invisible behind a three-chunk buffer and a visible freeze behind none. The +benchmark replays chunk arrivals against a wall clock — playback starts once +`--playback-buffer-chunks` chunks have landed and then consumes video at `--target-fps` +— and reports every moment the player ran dry. + +### What the client-side number is worth + +On a four-GPU Ulysses run the client's mean inter-arrival over all fifteen +post-first-chunk intervals was **1326.743 ms**, against the server's own +`StageRequestStats.inter_output_latency_ms` of **1327.479 ms** for the same +request — a difference of **0.74 ms**. This cross-check shows agreement in +average output cadence for that run, including warmup and the terminal chunk. +It does not measure encoding or transport latency: a fixed delivery delay can +cancel out of inter-arrival differences. Attributing that latency requires +per-chunk timestamps at both boundaries or server-side profiling. + +Compare matching interval populations when investigating discrepancies; both +measurement bugs and delivery buffering can change the observed cadence. + +### What this benchmark cannot see + +Chunk metadata carries no server-side timestamps, so the benchmark cannot split +DiT time from VAE decode time; use `--enable-diffusion-pipeline-profiler` on the +server for that. It also reports no GPU memory, which is a server-side quantity. + +## Custom rollouts + +`--workload rollout.json` replaces the built-in workload, including mid-rollout +prompt changes, which the CLI flags cannot express: + +```json +{ + "prompt": "The camera moves slowly forward through the scene.", + "image": "/path/to/first_frame.png", + "num_chunks": 24, + "camera_pattern": "orbit", + "fps": 16, + "seed": 42, + "prompt_updates": [ + {"after_chunk": 8, "prompt": "Rain begins to fall", "transition_chunks": 2} + ] +} +``` + +Supply `camera_action_script` directly instead of `num_chunks` to control every +frame's action. It must hold exactly one three-entry action list per chunk; one +AR block is three latent frames, and the server rejects any other shape. +Explicit `--image`, `--prompt`, `--negative-prompt`, `--width`, `--height`, +`--fps`, `--seed`, and `--flow-shift` flags override the matching workload-file +values. `--num-chunks` and `--camera-pattern` apply only to the built-in +workload; when using `--workload`, set those fields in the file instead. +An explicit `camera_action_script` takes precedence over the file's +`num_chunks` and `camera_pattern`. Each prompt update must use a distinct +`after_chunk` boundary so its interaction event ID is unique. + +Camera actions change what the world does, not what it costs, so `--camera-pattern` +(`forward`, `orbit`, `hold`) keeps a rollout representative rather than sweeping a +cost dimension. + +## Options worth knowing + +| Flag | Purpose | +| --- | --- | +| `--target-fps` | Playback rate the real-time verdict is measured against, independent of the mux label. | +| `--sessions N` | N sequential measured rollouts; the report adds spread across them. | +| `--warmup-sessions N` | Unmeasured rollouts first, to pay compile and capture cost. | +| `--print-chunks` | One line per chunk as it arrives. | +| `--save-video PATH` | Remux the last measured session for eyeballing (needs vLLM-Omni). | +| `--ping-interval` | `session.ping` cadence keeping the server's stall clock fresh during a slow first chunk. `0` disables. | +| `--first-chunk-timeout` | Raise it when the server is still compiling or capturing graphs. | + +## Output + +`--output-json` writes the run's configuration, every chunk's arrival time and +size, per-session metrics, and the aggregate, along with the git revision and +hostname so a number can be traced back to what produced it. + +## Tests + +`tests/benchmarks/test_lingbot_world_realtime.py` covers the chunk and frame +arithmetic against the server's own formula, the metric math, and the protocol +handling against a scripted WebSocket server. It needs no GPU: + +```bash +pytest tests/benchmarks/test_lingbot_world_realtime.py +``` diff --git a/benchmarks/lingbot_world/__init__.py b/benchmarks/lingbot_world/__init__.py new file mode 100644 index 000000000..4244ce4b4 --- /dev/null +++ b/benchmarks/lingbot_world/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project diff --git a/benchmarks/lingbot_world/benchmark_lingbot_world_realtime.py b/benchmarks/lingbot_world/benchmark_lingbot_world_realtime.py new file mode 100644 index 000000000..186bc1393 --- /dev/null +++ b/benchmarks/lingbot_world/benchmark_lingbot_world_realtime.py @@ -0,0 +1,642 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""Realtime streaming benchmark for LingBot-World 2.0 over ``WS /v1/realtime/video``. + +Every other diffusion benchmark in this repository measures "one request, one +finished video". LingBot-World is not that: one request is a whole rollout, and +the thing a deployment lives or dies by is whether chunk *N + 1* keeps arriving +before a viewer has finished watching chunk *N*. This benchmark therefore +measures a **cadence**, not a throughput: + +* ``TTFC`` -- session.start to the first chunk's bytes, which pays for prompt + encode, the first-frame VAE encode, and any compile or capture the deploy + config asks for; +* **inter-arrival latency** over all chunks and, separately, over the steady + state after the attention window saturates; +* ``VIDEO_RTF`` -- wall seconds spent per second of video generated, matching how + vLLM-Omni computes RTF everywhere else (``generation_s / video_duration``). + **Lower is better and below 1.0 is real time**; ``VIDEO_RTFX`` is reported + alongside it as the reciprocal, under the repo's own name for that; +* **playback continuity** -- a simulated viewer with a start buffer, because a + p99 that is harmless behind a three-chunk buffer is a visible stall behind + none. + +The server is not started here. Bring one up first (see the README), then point +this script at it. Requires ``websockets``; ``--save-video`` additionally needs +a vLLM-Omni installation for the remux helper. + +Example: + + python benchmarks/lingbot_world/benchmark_lingbot_world_realtime.py \\ + --host 127.0.0.1 --port 8000 \\ + --num-chunks 16 \\ + --output-json /tmp/lingbot_world_realtime.json +""" + +from __future__ import annotations + +import argparse +import asyncio +import base64 +import json +import mimetypes +import os +import platform +import subprocess +import sys +import time +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +REPO_ROOT = Path(__file__).resolve().parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from benchmarks.lingbot_world.workload import ( # noqa: E402 + CAMERA_PATTERNS, + DEFAULT_FLOW_SHIFT, + DEFAULT_FPS, + DEFAULT_HEIGHT, + DEFAULT_MODEL, + DEFAULT_PROMPT, + DEFAULT_SEED, + DEFAULT_WARMUP_CHUNKS, + DEFAULT_WIDTH, + ChunkRecord, + Workload, + aggregate_metrics, + compute_metrics, + default_workload, + load_workload, + steady_chunk_deadline_ms, +) + +# The shared vLLM asset keeps the default workload runnable with no setup and +# adds no binary to this repository. Any real photograph conditions a rollout. +_DEFAULT_IMAGE_ASSET = "2560px-Gfp-wisconsin-madison-the-nature-boardwalk" +_WS_PATH = "/v1/realtime/video" + + +class BenchmarkError(RuntimeError): + """A session failed in a way that invalidates its numbers.""" + + +# --------------------------------------------------------------------------- # +# Input resolution +# --------------------------------------------------------------------------- # + + +def resolve_image_reference(value: str | None) -> str: + """Return an ``http(s)`` or ``data:`` URL for the session's first frame.""" + if value is None: + from vllm.assets.image import ImageAsset + + value = str(ImageAsset(_DEFAULT_IMAGE_ASSET).get_path("jpg")) + if value.startswith(("http://", "https://", "data:")): + return value + path = Path(value).expanduser() + if not path.is_file(): + raise BenchmarkError(f"--image is neither a URL nor a readable file: {value}") + media_type = mimetypes.guess_type(path.name)[0] or "image/png" + return f"data:{media_type};base64,{base64.b64encode(path.read_bytes()).decode()}" + + +def build_workload(args: argparse.Namespace) -> Workload: + if args.workload is not None: + # The file names its own first frame, so resolving happens inside + # load_workload -- never eagerly, which would fetch the default asset a + # --workload run has no use for. + return load_workload( + args.workload, + image_resolver=lambda value: resolve_image_reference(value if args.image is None else args.image), + overrides={ + "width": args.width_override, + "height": args.height_override, + "fps": args.fps_override, + "seed": args.seed_override, + "prompt": args.prompt_override, + "flow_shift": args.flow_shift, + "negative_prompt": args.negative_prompt, + }, + ) + return default_workload( + image_reference=resolve_image_reference(args.image), + num_chunks=args.num_chunks, + camera_pattern=args.camera_pattern, + prompt=args.prompt_override if args.prompt_override is not None else DEFAULT_PROMPT, + width=args.width_override if args.width_override is not None else DEFAULT_WIDTH, + height=args.height_override if args.height_override is not None else DEFAULT_HEIGHT, + fps=args.fps_override if args.fps_override is not None else DEFAULT_FPS, + seed=args.seed_override if args.seed_override is not None else DEFAULT_SEED, + flow_shift=args.flow_shift if args.flow_shift is not None else DEFAULT_FLOW_SHIFT, + negative_prompt=args.negative_prompt, + ) + + +# --------------------------------------------------------------------------- # +# One streaming session +# --------------------------------------------------------------------------- # + + +@dataclass +class SessionResult: + request_id: str | None + records: list[ChunkRecord] + media: bytes + ttfc_s: float | None + video_start_s: float | None + wall_s: float + done_event: dict[str, Any] + + +async def _keepalive(websocket: Any, send_lock: asyncio.Lock, interval: float) -> None: + """Refresh the server's stall clock while a slow first chunk is generating. + + A compiled deploy config records CUDA graphs on the first block, which can + outlast the server's ~60 s stall timeout. ``session.ping`` is the documented + way to say the client is still there. + """ + while True: + await asyncio.sleep(interval) + async with send_lock: + await websocket.send(json.dumps({"type": "session.ping"})) + + +async def run_session( + url: str, + model: str, + workload: Workload, + *, + first_chunk_timeout: float, + chunk_timeout: float, + ping_interval: float, + collect_media: bool, + print_chunks: bool, +) -> SessionResult: + """Drive one rollout and timestamp every chunk as its bytes land.""" + try: + from websockets.asyncio.client import connect + except ImportError: # pragma: no cover - exercised only without websockets + try: + from websockets import connect # type: ignore[attr-defined] + except ImportError as exc: + raise BenchmarkError("This benchmark needs the websockets package: pip install websockets") from exc + + payload = workload.session_start_payload(model) + records: list[ChunkRecord] = [] + media = bytearray() + pending: dict[str, Any] | None = None + request_id: str | None = None + ttfc_s: float | None = None + video_start_s: float | None = None + last_arrival: float | None = None + done_event: dict[str, Any] = {} + send_lock = asyncio.Lock() + keepalive: asyncio.Task[None] | None = None + # Fire remaining prompt updates in chunk order as their boundary passes. + updates = sorted(workload.prompt_updates, key=lambda update: update.after_chunk) + update_index = 0 + + async with connect(url, max_size=None, ping_interval=None) as websocket: + async with send_lock: + await websocket.send(json.dumps(payload, ensure_ascii=False)) + started = time.perf_counter() + if ping_interval > 0: + keepalive = asyncio.create_task(_keepalive(websocket, send_lock, ping_interval)) + + try: + while True: + timeout = first_chunk_timeout if not records else chunk_timeout + try: + deadline = started + (last_arrival if last_arrival is not None else 0.0) + timeout + remaining = deadline - time.perf_counter() + if remaining <= 0: + raise asyncio.TimeoutError + message = await asyncio.wait_for(websocket.recv(), timeout=remaining) + except asyncio.TimeoutError as exc: + raise BenchmarkError( + f"No media chunk for {timeout:g} s after {len(records)} chunk(s); " + "the server may still be loading, compiling, or capturing graphs." + ) from exc + now = time.perf_counter() + + if isinstance(message, (bytes, bytearray)): + if pending is None: + raise BenchmarkError("Received a binary frame with no preceding video.chunk_metadata event.") + if collect_media: + media.extend(message) + if pending.get("kind") == "media": + index = int(pending["generation_chunk_index"]) + arrival = now - started + if ttfc_s is None: + ttfc_s = arrival + records.append( + ChunkRecord( + index=index, + arrival_s=arrival, + inter_arrival_s=arrival - (last_arrival if last_arrival is not None else 0.0), + byte_length=len(message), + num_frames=int(pending["num_frames"]), + ) + ) + last_arrival = arrival + if print_chunks: + print( + f" chunk {index:3d} t={arrival:8.3f}s " + f"interval={records[-1].inter_arrival_s * 1000:8.1f}ms " + f"frames={records[-1].num_frames:3d} bytes={len(message)}", + flush=True, + ) + while update_index < len(updates) and updates[update_index].after_chunk <= index: + async with send_lock: + await websocket.send(json.dumps(updates[update_index].to_payload(), ensure_ascii=False)) + update_index += 1 + pending = None + continue + + event = json.loads(message) + event_type = event.get("type") + if event_type == "video.chunk_metadata": + pending = event + elif event_type == "video.start": + request_id = event.get("request_id") + video_start_s = now - started + elif event_type == "error": + # Say what was asked for alongside what the server refused: + # some builds cap a rollout's num_frames, and the operator + # needs to know which --num-chunks would fit. + raise BenchmarkError( + f"Server rejected the session after {len(records)} chunk(s): " + f"{event.get('message', event)} " + f"(requested {workload.num_chunks} chunks = num_frames {workload.num_frames})" + ) + elif event_type == "session.done": + done_event = event + break + finally: + if keepalive is not None: + keepalive.cancel() + try: + await keepalive + except asyncio.CancelledError: + pass + + wall_s = time.perf_counter() - started + + if not records: + raise BenchmarkError("The session finished without delivering a single media chunk.") + if done_event.get("stopped"): + raise BenchmarkError(f"The session was stopped before it finished: {done_event}") + observed = [record.index for record in records] + if observed != list(range(len(records))): + raise BenchmarkError(f"Chunk indices are not contiguous from zero: {observed}") + if len(records) != workload.num_chunks: + raise BenchmarkError( + f"Expected {workload.num_chunks} chunks for num_frames={workload.num_frames}, received {len(records)}." + ) + + return SessionResult( + request_id=request_id, + records=records, + media=bytes(media), + ttfc_s=ttfc_s, + video_start_s=video_start_s, + wall_s=wall_s, + done_event=done_event, + ) + + +# --------------------------------------------------------------------------- # +# Reporting +# --------------------------------------------------------------------------- # + + +def _git_revision() -> str | None: + try: + return subprocess.check_output( + ["git", "-C", str(REPO_ROOT), "rev-parse", "HEAD"], + text=True, + stderr=subprocess.DEVNULL, + ).strip() + except (OSError, subprocess.CalledProcessError): + return None + + +def _print_row(label: str, value: str) -> None: + print(f"{label:<44}{value:>16}") + + +def print_report( + metrics: dict[str, Any], + *, + workload: Workload, + endpoint: str, + sessions: int, + target_fps: float, +) -> None: + header = " LingBot-World realtime streaming benchmark " + print("\n{s:{c}^{n}}".format(s=header, n=60, c="=")) + _print_row("Endpoint:", endpoint) + _print_row("Geometry:", f"{workload.width}x{workload.height}") + _print_row("Chunks per session:", str(workload.num_chunks)) + _print_row("Sessions:", str(sessions)) + frames = workload.num_frames + _print_row("Real-time basis:", f"{target_fps:g} fps") + _print_row("Video seconds per session:", f"{frames / target_fps:.2f} ({frames} frames)") + if abs(target_fps - workload.fps) > 1e-9: + _print_row("Mux label sent to server:", f"{workload.fps} fps") + + if sessions == 1: + print("{s:{c}^{n}}".format(s=" Cadence ", n=60, c="-")) + if metrics.get("ttfc_ms") is not None: + _print_row("TTFC (session.start -> first chunk):", f"{metrics['ttfc_ms']:.1f} ms") + interval_all = metrics.get("interval_all") or {} + interval_steady = metrics.get("interval_steady") or {} + for title, summary in (("All chunks", interval_all), ("Steady state", interval_steady)): + if not summary: + continue + print(f" {title} (n={int(summary['count'])}):") + _print_row(" mean / median:", f"{summary['mean_ms']:.1f} / {summary['median_ms']:.1f} ms") + _print_row(" std / max:", f"{summary['std_ms']:.1f} / {summary['max_ms']:.1f} ms") + tail = f"{summary['p90_ms']:.0f} / {summary['p95_ms']:.0f} / {summary['p99_ms']:.0f} ms" + _print_row(" p90 / p95 / p99:", tail) + if summary["count"] < 100: + print( + " NOTE: fewer than 100 intervals; tail percentiles are descriptive, not reliable tail estimates." + ) + + print("{s:{c}^{n}}".format(s=" Real time ", n=60, c="-")) + _print_row("VIDEO_RTF (wall s / video s, <1 real time):", f"{metrics['video_rtf']:.3f}") + _print_row("VIDEO_RTFX (video s / wall s):", f"{metrics['video_rtfx']:.3f}") + if metrics.get("steady_video_rtf") is not None: + _print_row("VIDEO_RTF steady state:", f"{metrics['steady_video_rtf']:.3f}") + _print_row("Frames per second:", f"{metrics['frames_per_second']:.2f}") + _print_row("Chunk deadline:", f"{metrics['chunk_deadline_ms']:.1f} ms") + if metrics.get("slo_attainment") is not None: + _print_row( + "Steady chunks within deadline:", + f"{metrics['slo_attainment'] * 100:.1f}% ({metrics['slo_violations']} late)", + ) + if not metrics.get("steady_state_reached", False): + print( + f" NOTE: no chunk reached the steady state; the attention window needs " + f"{metrics['warmup_chunks']} chunks. Raise --num-chunks." + ) + + playback = metrics.get("playback") or {} + print("{s:{c}^{n}}".format(s=" Playback continuity ", n=60, c="-")) + _print_row("Start buffer:", f"{playback.get('buffer_chunks')} chunk(s)") + _print_row("Underruns:", str(playback.get("underrun_count"))) + _print_row("Total stall:", f"{playback.get('total_stall_ms', 0.0):.1f} ms") + _print_row("Stall ratio:", f"{playback.get('stall_ratio', 0.0) * 100:.2f}%") + if playback.get("first_underrun_chunk") is not None: + _print_row("First underrun at chunk:", str(playback["first_underrun_chunk"])) + else: + print("{s:{c}^{n}}".format(s=" Across sessions ", n=60, c="-")) + if metrics.get("ttfc_ms_mean") is not None: + _print_row("TTFC mean / max:", f"{metrics['ttfc_ms_mean']:.1f} / {metrics['ttfc_ms_max']:.1f} ms") + if metrics.get("steady_interval_ms_mean") is not None: + _print_row("Steady interval mean:", f"{metrics['steady_interval_ms_mean']:.1f} ms") + _print_row("Steady interval spread:", f"{metrics['steady_interval_ms_spread']:.1f} ms") + _print_row("VIDEO_RTF mean (lower is better):", f"{metrics['video_rtf_mean']:.3f}") + _print_row( + "VIDEO_RTF best / worst:", + f"{metrics['video_rtf_best']:.3f} / {metrics['video_rtf_worst']:.3f}", + ) + _print_row("Chunk deadline:", f"{metrics['chunk_deadline_ms']:.1f} ms") + _print_row("Underruns (all sessions):", str(metrics["underrun_count"])) + print("=" * 60) + + +def save_video(media: bytes, path: Path, *, fps: int) -> None: + from vllm_omni.diffusion.utils.media_utils import finalize_streaming_video_bytes + + playback_bytes = finalize_streaming_video_bytes(media, input_format="m4s", fps=fps) + path.parent.mkdir(parents=True, exist_ok=True) + tmp_path = path.with_suffix(path.suffix + ".tmp") + tmp_path.write_bytes(playback_bytes) + os.replace(tmp_path, path) + print(f"Saved {len(media)} streamed bytes -> {len(playback_bytes)} playback bytes at {path}") + + +# --------------------------------------------------------------------------- # +# CLI +# --------------------------------------------------------------------------- # + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Benchmark LingBot-World 2.0 realtime streaming over WS /v1/realtime/video.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + endpoint = parser.add_argument_group("endpoint") + endpoint.add_argument("--host", default="127.0.0.1") + endpoint.add_argument("--port", type=int, default=8000) + endpoint.add_argument("--base-url", default=None, help="Full ws:// URL; overrides --host/--port.") + endpoint.add_argument("--model", default=DEFAULT_MODEL) + + workload = parser.add_argument_group("workload") + workload.add_argument("--workload", default=None, help="JSON rollout spec; overrides the built-in workload.") + workload.add_argument("--image", default=None, help="First frame: path or URL. Defaults to the shared vLLM asset.") + workload.add_argument("--prompt", dest="prompt_override", default=None, help=f"Default: {DEFAULT_PROMPT!r}") + workload.add_argument("--negative-prompt", default=None) + workload.add_argument("--num-chunks", type=int, default=16, help="AR blocks per session; ignored with --workload.") + workload.add_argument( + "--camera-pattern", + choices=CAMERA_PATTERNS, + default="forward", + help="Camera pattern for the built-in workload only; ignored with --workload.", + ) + workload.add_argument("--width", dest="width_override", type=int, default=None, help=f"Default: {DEFAULT_WIDTH}") + workload.add_argument("--height", dest="height_override", type=int, default=None, help=f"Default: {DEFAULT_HEIGHT}") + workload.add_argument("--fps", dest="fps_override", type=int, default=None, help=f"Default: {DEFAULT_FPS}") + workload.add_argument("--seed", dest="seed_override", type=int, default=None, help=f"Default: {DEFAULT_SEED}") + workload.add_argument("--flow-shift", type=float, default=None, help=f"Default: {DEFAULT_FLOW_SHIFT}") + + run = parser.add_argument_group("run") + run.add_argument("--sessions", type=int, default=1, help="Sequential measured rollouts.") + run.add_argument("--warmup-sessions", type=int, default=0, help="Unmeasured rollouts first (pays compile cost).") + run.add_argument( + "--warmup-chunks", + type=int, + default=DEFAULT_WARMUP_CHUNKS, + help="Chunks excluded from steady state; the attention window saturates after 6.", + ) + run.add_argument( + "--chunk-slo-ms", + type=float, + default=None, + help="Deadline a steady chunk must meet. Default: one chunk of video at --target-fps.", + ) + run.add_argument( + "--target-fps", + type=float, + default=None, + help=( + "Playback rate used for real-time factor, the chunk deadline, and the playback " + "simulation. The checkpoint declares no frame rate, so this is a consumption " + "choice, not a model property. Default: the --fps sent to the server." + ), + ) + run.add_argument("--playback-buffer-chunks", type=int, default=1, help="Chunks buffered before playback starts.") + run.add_argument("--first-chunk-timeout", type=float, default=900.0) + run.add_argument("--chunk-timeout", type=float, default=300.0) + run.add_argument("--ping-interval", type=float, default=20.0, help="session.ping cadence; 0 disables.") + + output = parser.add_argument_group("output") + output.add_argument("--output-json", default=None) + output.add_argument("--save-video", default=None, help="Write the last measured session's video here.") + output.add_argument("--print-chunks", action="store_true") + + args = parser.parse_args(argv) + if args.sessions < 1: + parser.error("--sessions must be at least 1.") + if args.warmup_sessions < 0: + parser.error("--warmup-sessions must be non-negative.") + if args.warmup_chunks < 0: + parser.error("--warmup-chunks must be non-negative.") + if args.playback_buffer_chunks < 1: + parser.error("--playback-buffer-chunks must be at least 1.") + if args.target_fps is not None and args.target_fps <= 0: + parser.error("--target-fps must be positive.") + if args.workload is None and args.num_chunks < 1: + parser.error("--num-chunks must be at least 1.") + return args + + +def endpoint_url(args: argparse.Namespace) -> str: + if args.base_url is not None: + base = args.base_url.rstrip("/") + for prefix, replacement in (("http://", "ws://"), ("https://", "wss://")): + if base.startswith(prefix): + base = replacement + base[len(prefix) :] + break + return base if base.endswith(_WS_PATH) else base + _WS_PATH + return f"ws://{args.host}:{args.port}{_WS_PATH}" + + +async def run_benchmark(args: argparse.Namespace) -> dict[str, Any]: + workload = build_workload(args) + url = endpoint_url(args) + # The mux label and the real-time basis are different questions; only the + # latter decides whether a run passes. + target_fps = args.target_fps if args.target_fps is not None else float(workload.fps) + if max(1, args.warmup_chunks) >= workload.num_chunks - 1: + print( + f"WARNING: --warmup-chunks={args.warmup_chunks} leaves no nonterminal steady-state chunk in a " + f"{workload.num_chunks}-chunk rollout; steady-state metrics will be empty.", + file=sys.stderr, + ) + + print(f"Connecting to {url}") + print( + f"Rollout: {workload.num_chunks} chunks, num_frames={workload.num_frames}, " + f"{workload.video_seconds:.2f}s of video" + ) + print( + f"Real-time basis: {target_fps:g} fps -> a steady 12-frame chunk is " + f"{steady_chunk_deadline_ms(target_fps):.1f} ms of video" + + ("" if args.target_fps is None else f" (mux label stays {workload.fps} fps)") + ) + + for index in range(args.warmup_sessions): + print(f"Warmup session {index + 1}/{args.warmup_sessions} (not measured)...") + await run_session( + url, + args.model, + workload, + first_chunk_timeout=args.first_chunk_timeout, + chunk_timeout=args.chunk_timeout, + ping_interval=args.ping_interval, + collect_media=False, + print_chunks=False, + ) + + session_metrics: list[dict[str, Any]] = [] + session_payloads: list[dict[str, Any]] = [] + last_result: SessionResult | None = None + for index in range(args.sessions): + if args.sessions > 1: + print(f"Session {index + 1}/{args.sessions}...") + result = await run_session( + url, + args.model, + workload, + first_chunk_timeout=args.first_chunk_timeout, + chunk_timeout=args.chunk_timeout, + ping_interval=args.ping_interval, + collect_media=args.save_video is not None, + print_chunks=args.print_chunks, + ) + last_result = result + metrics = compute_metrics( + result.records, + fps=target_fps, + warmup_chunks=args.warmup_chunks, + slo_ms=args.chunk_slo_ms, + playback_buffer_chunks=args.playback_buffer_chunks, + ttfc_s=result.ttfc_s, + session_wall_s=result.wall_s, + ) + session_metrics.append(metrics) + session_payloads.append( + { + "session_index": index, + "request_id": result.request_id, + "video_start_ms": (result.video_start_s * 1000.0) if result.video_start_s is not None else None, + "chunks": [record.to_dict() for record in result.records], + "metrics": metrics, + } + ) + + aggregate = aggregate_metrics(session_metrics, fps=target_fps) + print_report(aggregate, workload=workload, endpoint=url, sessions=args.sessions, target_fps=target_fps) + + if args.save_video is not None and last_result is not None: + save_video(last_result.media, Path(args.save_video), fps=workload.fps) + + document = { + "benchmark": "lingbot_world_realtime", + "timestamp": datetime.now(timezone.utc).isoformat(), + "endpoint": url, + "model": args.model, + "git_revision": _git_revision(), + "hostname": platform.node(), + "workload": workload.describe(), + "config": { + "sessions": args.sessions, + "warmup_sessions": args.warmup_sessions, + "warmup_chunks": args.warmup_chunks, + "camera_pattern": args.camera_pattern, + "chunk_slo_ms": args.chunk_slo_ms, + "target_fps": target_fps, + "mux_fps": workload.fps, + "playback_buffer_chunks": args.playback_buffer_chunks, + "workload_file": args.workload, + }, + "sessions": session_payloads, + "metrics": aggregate, + } + if args.output_json is not None: + path = Path(args.output_json) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(document, indent=2, sort_keys=True) + "\n") + print(f"Wrote {path}") + return document + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + try: + asyncio.run(run_benchmark(args)) + except BenchmarkError as error: + print(f"ERROR: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/lingbot_world/configs/single_gpu_eager.yaml b/benchmarks/lingbot_world/configs/single_gpu_eager.yaml new file mode 100644 index 000000000..58f36e4f0 --- /dev/null +++ b/benchmarks/lingbot_world/configs/single_gpu_eager.yaml @@ -0,0 +1,45 @@ +# Benchmark deploy config: LingBot-World 2.0 on one GPU, eager. +# +# This is the portable smoke configuration -- it matches the shipped +# vllm_omni/deploy/lingbot_world_v2_stepwise.yaml so a number measured here is +# comparable to the default deployment. It is NOT a realtime configuration: one +# GPU runs all five forwards of every AR block serially and nothing is compiled, +# so expect a chunk interval several times the real-time deadline. +# +# Use configs/usp4_compiled.yaml for the realtime topology. +# +# vllm serve robbyant/lingbot-world-v2-14b-causal-fast-diffusers \ +# --omni --deploy-config benchmarks/lingbot_world/configs/single_gpu_eager.yaml \ +# --port 8000 + +pipeline: lingbot_world +async_chunk: false +distributed_executor_backend: mp +dtype: bfloat16 + +stages: + - stage_id: 0 + devices: "0" + # AR-Diffusion binds one session per runner invocation; the runner rejects + # anything larger. + max_num_seqs: 1 + enforce_eager: true + model_class_name: LingBotWorldCausalDMDPipeline + engine_backend: vllm_omni.experimental.ar_diffusion.engine.ARDiffusionEngine + step_execution: true + streaming_output: true + default_sampling_params: + # The causal-fast checkpoint pins a four-step DMD schedule; this sizes the + # step budget before the pipeline is reached. + num_inference_steps: 4 + output_type: pil + extra_args: + flow_shift: 5.0 + model_config: + # The AR cache geometry is fixed at load time, so a benchmark request must + # ask for exactly this resolution. + ar_diffusion_height: 480 + ar_diffusion_width: 832 + ar_diffusion_kv_config: + gpu_memory_fraction: 0.6 + warmup_cudagraph: true diff --git a/benchmarks/lingbot_world/configs/usp4_compiled.yaml b/benchmarks/lingbot_world/configs/usp4_compiled.yaml new file mode 100644 index 000000000..ed71bc13e --- /dev/null +++ b/benchmarks/lingbot_world/configs/usp4_compiled.yaml @@ -0,0 +1,60 @@ +# Benchmark deploy config: LingBot-World 2.0 on four GPUs, Ulysses SP-4, compiled. +# +# This is the realtime topology. Two things differ from the single-GPU config, +# and both matter to the number the benchmark reports: +# +# parallel_config Pure Ulysses sequence parallelism over four ranks. The +# pipeline validates this: LingBot requires +# sequence_parallel_size == ulysses_degree, ring_degree and +# allgather_degree of 1, ulysses_mode "strict", and +# ulysses_a2a_permute disabled. Pipeline, CFG, and VAE +# parallel sizes above 1 are rejected. This config uses +# TP=1; the validator does not reject larger TP sizes. +# enforce_eager false puts LingBotDMDBlockRunner on the compiled path +# (regional torch.compile with cudagraph step marking). The +# first rollout therefore pays compile and capture cost -- +# use --warmup-sessions 1 before measured repetitions. +# Chunk warmup only excludes the attention-window ramp. +# +# Serve it under a GPU reservation, since it needs four cards: +# +# gpu run --gpus 4 -- vllm serve robbyant/lingbot-world-v2-14b-causal-fast-diffusers \ +# --omni --deploy-config benchmarks/lingbot_world/configs/usp4_compiled.yaml \ +# --port 8000 +# +# `devices` is the rank->card mapping inside the reservation, so it stays +# "0,1,2,3" no matter which physical cards CUDA_VISIBLE_DEVICES selected. + +pipeline: lingbot_world +async_chunk: false +distributed_executor_backend: mp +dtype: bfloat16 + +stages: + - stage_id: 0 + devices: "0,1,2,3" + max_num_seqs: 1 + enforce_eager: false + model_class_name: LingBotWorldCausalDMDPipeline + engine_backend: vllm_omni.experimental.ar_diffusion.engine.ARDiffusionEngine + step_execution: true + streaming_output: true + parallel_config: + tensor_parallel_size: 1 + sequence_parallel_size: 4 + ulysses_degree: 4 + ring_degree: 1 + default_sampling_params: + num_inference_steps: 4 + output_type: pil + extra_args: + flow_shift: 5.0 + model_config: + ar_diffusion_height: 480 + ar_diffusion_width: 832 + ar_diffusion_kv_config: + # Each rank holds its own shard of the paged KV cache, so the per-card + # fraction can stay at the single-GPU value; lower it if the four-rank + # start OOMs on smaller cards. + gpu_memory_fraction: 0.6 + warmup_cudagraph: true diff --git a/benchmarks/lingbot_world/workload.py b/benchmarks/lingbot_world/workload.py new file mode 100644 index 000000000..4b59c8899 --- /dev/null +++ b/benchmarks/lingbot_world/workload.py @@ -0,0 +1,631 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""Workload construction and metric math for the LingBot-World realtime benchmark. + +Nothing here imports torch, vllm, or websockets, so the arithmetic that decides +what a benchmark number *means* is unit-testable without a GPU or a server. + +Two model properties drive every formula in this module, both read from the +checkpoint's ``transformer/config.json`` rather than assumed: + +``num_frames_per_block = 3`` + One AR block commits three latent frames, so one streamed chunk is three + latent frames and a request's ``camera_action_script`` carries exactly one + three-entry action list per chunk. + +``sliding_window_num_frames = 18`` with ``sink_size = 9`` + Self-attention sees nine sink frames plus a nine-frame recent window, so the + key count a block attends over keeps growing until eighteen latent frames of + history exist -- six chunks. Chunk cost is therefore *not* stationary before + chunk six, and a mean taken over a short rollout reports a number no steady + session will ever produce. :data:`DEFAULT_WARMUP_CHUNKS` exists for that + reason and is not a round-number guess. + +The causal VAE expands the first latent frame to one pixel frame and every later +latent frame to four, so chunk zero carries nine pixel frames and every chunk +after it carries twelve. Video-duration math has to respect that asymmetry or +real-time factor comes out wrong on short rollouts. +""" + +from __future__ import annotations + +import json +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +# transformer.config.num_frames_per_block: latent frames committed per AR block. +FRAMES_PER_BLOCK = 3 +# vae_scale_factor_temporal: latent frames -> pixel frames, after the first. +TEMPORAL_COMPRESSION = 4 +# (sink_size 9 + recent window 9) / FRAMES_PER_BLOCK: chunks until the attention +# window saturates and per-chunk cost becomes stationary. +DEFAULT_WARMUP_CHUNKS = 6 + +DEFAULT_MODEL = "robbyant/lingbot-world-v2-14b-causal-fast-diffusers" +DEFAULT_PROMPT = "The camera moves slowly forward through the scene." +# The AR KV cache geometry is fixed at load time, so a request must ask for +# exactly the resolution the deploy config declares. +DEFAULT_WIDTH = 832 +DEFAULT_HEIGHT = 480 +DEFAULT_FPS = 16 +DEFAULT_FLOW_SHIFT = 5.0 +DEFAULT_SEED = 42 + +_HOLD = "hold" +_ACTION_KEYS = ("w", "a", "s", "d") + + +def num_frames_for_chunks(num_chunks: int) -> int: + """Pixel-frame count whose rollout is exactly ``num_chunks`` chunks. + + Inverse of :func:`chunks_for_num_frames`, which is the server's own + ``((num_frames - 1) // 4 + 1) // 3``. + """ + if num_chunks < 1: + raise ValueError("num_chunks must be at least 1.") + return (num_chunks * FRAMES_PER_BLOCK - 1) * TEMPORAL_COMPRESSION + 1 + + +def chunks_for_num_frames(num_frames: int) -> int: + """Chunks the server generates for ``num_frames``, mirroring its own formula.""" + if num_frames < 1: + raise ValueError("num_frames must be at least 1.") + return ((num_frames - 1) // TEMPORAL_COMPRESSION + 1) // FRAMES_PER_BLOCK + + +def pixel_frames_in_chunk(chunk_index: int) -> int: + """Pixel frames carried by one chunk. + + The causal decoder expands the session's opening latent frame once and every + later latent frame four times, so chunk zero is short by three frames. + """ + if chunk_index < 0: + raise ValueError("chunk_index must be non-negative.") + if chunk_index == 0: + return (FRAMES_PER_BLOCK - 1) * TEMPORAL_COMPRESSION + 1 + return FRAMES_PER_BLOCK * TEMPORAL_COMPRESSION + + +def camera_actions_for_chunk(pattern: str, chunk_index: int) -> list[list[str]]: + """One three-entry action list for ``chunk_index``. + + Camera actions select camera embeddings; they change what the world does, + not how much arithmetic it costs. The pattern therefore exists to keep a + benchmark rollout representative rather than to sweep a cost dimension. + """ + if chunk_index < 0: + raise ValueError("chunk_index must be non-negative.") + if pattern == "forward": + return [["w"] for _ in range(FRAMES_PER_BLOCK)] + if pattern == _HOLD: + return [[] for _ in range(FRAMES_PER_BLOCK)] + if pattern == "orbit": + key = _ACTION_KEYS[chunk_index % len(_ACTION_KEYS)] + return [[key] for _ in range(FRAMES_PER_BLOCK)] + raise ValueError(f"Unknown camera pattern {pattern!r}; expected one of {sorted(CAMERA_PATTERNS)}.") + + +CAMERA_PATTERNS = ("forward", "orbit", _HOLD) + + +def build_camera_script(num_chunks: int, pattern: str) -> list[list[list[str]]]: + """Full ``camera_action_script``: one action list per generated chunk.""" + if num_chunks < 1: + raise ValueError("num_chunks must be at least 1.") + return [camera_actions_for_chunk(pattern, index) for index in range(num_chunks)] + + +def _validate_camera_script(script: Sequence[Any]) -> list[list[list[str]]]: + validated: list[list[list[str]]] = [] + for chunk_index, actions in enumerate(script): + if not isinstance(actions, Sequence) or isinstance(actions, (str, bytes)): + raise ValueError(f"camera_action_script[{chunk_index}] must be a list of three action lists.") + if len(actions) != FRAMES_PER_BLOCK: + raise ValueError( + f"camera_action_script[{chunk_index}] must hold exactly {FRAMES_PER_BLOCK} action lists " + f"(got {len(actions)}); one AR block is {FRAMES_PER_BLOCK} latent frames." + ) + frames: list[list[str]] = [] + for frame_index, frame in enumerate(actions): + if not isinstance(frame, Sequence) or isinstance(frame, (str, bytes)): + raise ValueError(f"camera_action_script[{chunk_index}][{frame_index}] must be a list of strings.") + if any(not isinstance(action, str) for action in frame): + raise ValueError(f"camera_action_script[{chunk_index}][{frame_index}] must contain only strings.") + frames.append([str(action) for action in frame]) + validated.append(frames) + if not validated: + raise ValueError("camera_action_script must not be empty.") + return validated + + +@dataclass(frozen=True) +class PromptUpdate: + """A mid-rollout ``session.interaction`` scheduled on a chunk boundary.""" + + after_chunk: int + prompt: str + transition_chunks: int = 3 + + def __post_init__(self) -> None: + if self.after_chunk < 0: + raise ValueError("PromptUpdate.after_chunk must be non-negative.") + if not self.prompt.strip(): + raise ValueError("PromptUpdate.prompt must contain non-whitespace text.") + if self.transition_chunks < 0: + raise ValueError("PromptUpdate.transition_chunks must be non-negative.") + + def to_payload(self) -> dict[str, Any]: + return { + "type": "session.interaction", + "interaction": { + "event_id": f"chunk-{self.after_chunk}", + "event": {"prompt": self.prompt}, + "transition_chunks": self.transition_chunks, + }, + } + + +@dataclass(frozen=True) +class Workload: + """One rollout: what the benchmark asks the server to generate.""" + + prompt: str + image_reference: str + camera_script: list[list[list[str]]] + width: int = DEFAULT_WIDTH + height: int = DEFAULT_HEIGHT + fps: int = DEFAULT_FPS + seed: int = DEFAULT_SEED + flow_shift: float = DEFAULT_FLOW_SHIFT + negative_prompt: str | None = None + prompt_updates: tuple[PromptUpdate, ...] = () + extra_params: Mapping[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + if not self.prompt.strip(): + raise ValueError("Workload.prompt must contain non-whitespace text.") + if not self.image_reference: + raise ValueError("Workload.image_reference is required; LingBot-World is image-conditioned.") + if self.width <= 0 or self.height <= 0 or self.width % 16 or self.height % 16: + raise ValueError("Workload width/height must be positive multiples of 16.") + if self.fps <= 0: + raise ValueError("Workload.fps must be positive.") + object.__setattr__(self, "camera_script", _validate_camera_script(self.camera_script)) + seen_update_chunks: set[int] = set() + for update in self.prompt_updates: + if update.after_chunk in seen_update_chunks: + raise ValueError(f"Duplicate prompt update after chunk {update.after_chunk}; event IDs must be unique.") + seen_update_chunks.add(update.after_chunk) + if update.after_chunk >= self.num_chunks: + raise ValueError( + f"Prompt update after chunk {update.after_chunk} never fires in a {self.num_chunks}-chunk rollout." + ) + + @property + def num_chunks(self) -> int: + return len(self.camera_script) + + @property + def num_frames(self) -> int: + return num_frames_for_chunks(self.num_chunks) + + @property + def video_seconds(self) -> float: + return self.num_frames / self.fps + + def session_start_payload(self, model: str) -> dict[str, Any]: + """The ``session.start`` event for ``WS /v1/realtime/video``.""" + extra_params: dict[str, Any] = { + "flow_shift": self.flow_shift, + "camera_action_script": self.camera_script, + } + extra_params.update(self.extra_params) + payload: dict[str, Any] = { + "type": "session.start", + "model": model, + "prompt": self.prompt, + "image_reference": {"image_url": self.image_reference}, + "width": self.width, + "height": self.height, + "num_frames": self.num_frames, + "fps": self.fps, + "seed": self.seed, + "extra_params": extra_params, + } + if self.negative_prompt is not None: + payload["negative_prompt"] = self.negative_prompt + return payload + + def describe(self) -> dict[str, Any]: + return { + "num_chunks": self.num_chunks, + "num_frames": self.num_frames, + "width": self.width, + "height": self.height, + "fps": self.fps, + "seed": self.seed, + "flow_shift": self.flow_shift, + "video_seconds": self.video_seconds, + "prompt_updates": [ + { + "after_chunk": update.after_chunk, + "transition_chunks": update.transition_chunks, + } + for update in self.prompt_updates + ], + } + + +def default_workload( + *, + image_reference: str, + num_chunks: int, + camera_pattern: str = "forward", + prompt: str = DEFAULT_PROMPT, + width: int = DEFAULT_WIDTH, + height: int = DEFAULT_HEIGHT, + fps: int = DEFAULT_FPS, + seed: int = DEFAULT_SEED, + flow_shift: float = DEFAULT_FLOW_SHIFT, + negative_prompt: str | None = None, +) -> Workload: + """Built-in rollout: a fixed prompt and a generated camera script.""" + return Workload( + prompt=prompt, + image_reference=image_reference, + camera_script=build_camera_script(num_chunks, camera_pattern), + width=width, + height=height, + fps=fps, + seed=seed, + flow_shift=flow_shift, + negative_prompt=negative_prompt, + ) + + +def load_workload( + path: str | Path, + *, + image_resolver: Any = None, + overrides: Mapping[str, Any] | None = None, +) -> Workload: + """Read a rollout spec from JSON. + + ``image_resolver`` turns the spec's ``image`` field into an ``http(s)`` or + ``data:`` URL; the caller owns that because inlining a local file is I/O. + Keys in ``overrides`` that are not ``None`` win over the file, so CLI flags + stay meaningful next to ``--workload``. + """ + document = json.loads(Path(path).read_text()) + if not isinstance(document, dict): + raise ValueError("A workload file must contain a JSON object.") + + merged = dict(document) + for key, value in (overrides or {}).items(): + if value is not None: + merged[key] = value + + camera_script = merged.get("camera_action_script") + if camera_script is None: + num_chunks = merged.get("num_chunks") + if num_chunks is None: + raise ValueError("A workload file needs either camera_action_script or num_chunks.") + camera_script = build_camera_script(int(num_chunks), str(merged.get("camera_pattern", "forward"))) + + image = merged.get("image") or merged.get("image_reference") + if image is None: + raise ValueError("A workload file needs an image or image_reference field.") + image_reference = image_resolver(str(image)) if image_resolver is not None else str(image) + + updates_field = merged.get("prompt_updates") or () + prompt_updates = tuple( + PromptUpdate( + after_chunk=int(entry["after_chunk"]), + prompt=str(entry["prompt"]), + transition_chunks=int(entry.get("transition_chunks", 3)), + ) + for entry in updates_field + ) + + extra_params = merged.get("extra_params") or {} + if not isinstance(extra_params, Mapping): + raise ValueError("Workload extra_params must be a JSON object.") + + return Workload( + prompt=str(merged.get("prompt", DEFAULT_PROMPT)), + image_reference=image_reference, + camera_script=camera_script, + width=int(merged.get("width", DEFAULT_WIDTH)), + height=int(merged.get("height", DEFAULT_HEIGHT)), + fps=int(merged.get("fps", DEFAULT_FPS)), + seed=int(merged.get("seed", DEFAULT_SEED)), + flow_shift=float(merged.get("flow_shift", DEFAULT_FLOW_SHIFT)), + negative_prompt=merged.get("negative_prompt"), + prompt_updates=prompt_updates, + extra_params=dict(extra_params), + ) + + +# --------------------------------------------------------------------------- # +# Metrics +# --------------------------------------------------------------------------- # + + +@dataclass +class ChunkRecord: + """One media chunk as the client observed it.""" + + index: int + arrival_s: float + """Seconds from ``session.start`` being sent to this chunk's bytes arriving.""" + inter_arrival_s: float + """Seconds since the previous chunk; for chunk zero this equals ``arrival_s``.""" + byte_length: int + num_frames: int + + def to_dict(self) -> dict[str, Any]: + return { + "index": self.index, + "arrival_s": self.arrival_s, + "inter_arrival_ms": self.inter_arrival_s * 1000.0, + "byte_length": self.byte_length, + "num_frames": self.num_frames, + } + + +def percentile(values: Sequence[float], fraction: float) -> float: + """Linearly interpolated percentile, so a 10-sample run still reports a p95. + + Implemented here rather than pulled from numpy to keep this module importable + in a bare environment; the benchmark's own numbers must not depend on which + optional package happens to be installed. + """ + if not values: + raise ValueError("percentile() needs at least one value.") + if not 0.0 <= fraction <= 1.0: + raise ValueError("percentile fraction must be within [0, 1].") + ordered = sorted(values) + if len(ordered) == 1: + return ordered[0] + position = fraction * (len(ordered) - 1) + lower = int(position) + upper = min(lower + 1, len(ordered) - 1) + weight = position - lower + return ordered[lower] * (1.0 - weight) + ordered[upper] * weight + + +def _stddev(values: Sequence[float]) -> float: + if len(values) < 2: + return 0.0 + mean = sum(values) / len(values) + return (sum((value - mean) ** 2 for value in values) / (len(values) - 1)) ** 0.5 + + +def _latency_summary(values: Sequence[float]) -> dict[str, float]: + """Millisecond summary of a set of inter-arrival intervals.""" + if not values: + return {} + milliseconds = [value * 1000.0 for value in values] + return { + "count": float(len(milliseconds)), + "mean_ms": sum(milliseconds) / len(milliseconds), + "median_ms": percentile(milliseconds, 0.5), + "std_ms": _stddev(milliseconds), + "min_ms": min(milliseconds), + "p90_ms": percentile(milliseconds, 0.90), + "p95_ms": percentile(milliseconds, 0.95), + "p99_ms": percentile(milliseconds, 0.99), + "max_ms": max(milliseconds), + } + + +@dataclass(frozen=True) +class PlaybackResult: + """What a viewer would have seen, given a start buffer and a playback rate.""" + + underrun_count: int + total_stall_s: float + playback_span_s: float + first_underrun_chunk: int | None + + def to_dict(self) -> dict[str, Any]: + return { + "underrun_count": self.underrun_count, + "total_stall_ms": self.total_stall_s * 1000.0, + "playback_span_s": self.playback_span_s, + "stall_ratio": (self.total_stall_s / self.playback_span_s) if self.playback_span_s > 0 else 0.0, + "first_underrun_chunk": self.first_underrun_chunk, + } + + +def simulate_playback( + records: Sequence[ChunkRecord], + *, + fps: float, + buffer_chunks: int = 1, +) -> PlaybackResult: + """Replay arrivals against a wall clock and count stalls. + + Playback starts when ``buffer_chunks`` chunks have arrived and then consumes + video at ``fps``. A chunk that has not arrived by the moment the player needs + it is an underrun, and the player resumes only when the bytes land -- which + is what a viewer experiences, and what a single p99 number cannot tell you: + the same p99 is harmless behind a three-chunk buffer and fatal behind none. + """ + if fps <= 0: + raise ValueError("fps must be positive.") + if buffer_chunks < 1: + raise ValueError("buffer_chunks must be at least 1.") + if not records: + return PlaybackResult(0, 0.0, 0.0, None) + if buffer_chunks > len(records): + raise ValueError(f"buffer_chunks={buffer_chunks} exceeds the {len(records)} chunks received.") + + ordered = sorted(records, key=lambda record: record.index) + start = ordered[buffer_chunks - 1].arrival_s + cursor = start + underruns = 0 + total_stall = 0.0 + first_underrun: int | None = None + + for record in ordered: + if record.arrival_s > cursor: + stall = record.arrival_s - cursor + underruns += 1 + total_stall += stall + if first_underrun is None: + first_underrun = record.index + cursor = record.arrival_s + cursor += record.num_frames / fps + + return PlaybackResult( + underrun_count=underruns, + total_stall_s=total_stall, + playback_span_s=cursor - start, + first_underrun_chunk=first_underrun, + ) + + +def steady_chunk_deadline_ms(fps: float) -> float: + """Wall-clock budget a steady chunk must meet to hold real time. + + ``fps`` here is the rate the frames are *intended to be played at*, which the + checkpoint does not declare: nothing in ``model_index.json``, the scheduler + config, or the transformer config carries a frame rate, and the pipeline + never reads one. It is a property of how the output is consumed, so both + this deadline and real-time factor scale linearly with the choice and the + caller has to state it rather than inherit it from a muxing default. + """ + if fps <= 0: + raise ValueError("fps must be positive.") + return pixel_frames_in_chunk(1) / fps * 1000.0 + + +def compute_metrics( + records: Sequence[ChunkRecord], + *, + fps: float, + warmup_chunks: int = DEFAULT_WARMUP_CHUNKS, + slo_ms: float | None = None, + playback_buffer_chunks: int = 1, + ttfc_s: float | None = None, + session_wall_s: float | None = None, +) -> dict[str, Any]: + """Summarize one session. + + ``warmup_chunks`` are excluded from the steady-state block because the + attention window has not saturated before then (see the module docstring). + The terminal chunk also stays out of steady metrics because it skips + next-chunk preparation. All chunks stay in the playback simulation, which is + what a session really costs end to end. + """ + if not records: + raise ValueError("compute_metrics() needs at least one chunk record.") + if warmup_chunks < 0: + raise ValueError("warmup_chunks must be non-negative.") + + ordered = sorted(records, key=lambda record: record.index) + deadline_ms = slo_ms if slo_ms is not None else steady_chunk_deadline_ms(fps) + + all_intervals = [record.inter_arrival_s for record in ordered[1:]] + # The terminal chunk skips next-chunk preparation and is not representative + # of sustained generation. Keep it in end-to-end and playback metrics. + steady = [record for record in ordered[:-1] if record.index >= warmup_chunks] + steady_intervals = [record.inter_arrival_s for record in steady if record.index > 0] + + total_frames = sum(record.num_frames for record in ordered) + video_seconds = total_frames / fps + wall_seconds = session_wall_s if session_wall_s is not None else ordered[-1].arrival_s + + metrics: dict[str, Any] = { + "chunks_received": len(ordered), + "warmup_chunks": warmup_chunks, + "steady_chunks": len(steady_intervals), + "steady_state_reached": bool(steady_intervals), + "total_frames": total_frames, + "total_bytes": sum(record.byte_length for record in ordered), + "video_seconds": video_seconds, + "wall_seconds": wall_seconds, + # RTF is processing time over content duration, as vLLM-Omni computes it + # (`patch.py`: generation_s / video_duration; `compute_audio_rtf`: latency + # / audio_duration, "SLO red line < 1"). LOWER IS BETTER and below 1.0 is + # real time. RTFX is the reciprocal the repo reports under its own name, + # kept here because "fraction of real time achieved" reads better on a + # dashboard -- but the two must never be confused for one another. + "video_rtf": (wall_seconds / video_seconds) if video_seconds > 0 else 0.0, + "video_rtfx": (video_seconds / wall_seconds) if wall_seconds > 0 else 0.0, + "frames_per_second": (total_frames / wall_seconds) if wall_seconds > 0 else 0.0, + "chunk_deadline_ms": deadline_ms, + "interval_all": _latency_summary(all_intervals), + "interval_steady": _latency_summary(steady_intervals), + } + if ttfc_s is not None: + metrics["ttfc_ms"] = ttfc_s * 1000.0 + + if steady_intervals: + steady_frames = sum(record.num_frames for record in steady if record.index > 0) + steady_wall = sum(steady_intervals) + steady_video = steady_frames / fps + metrics["steady_video_rtf"] = (steady_wall / steady_video) if steady_video > 0 else 0.0 + metrics["steady_video_rtfx"] = (steady_video / steady_wall) if steady_wall > 0 else 0.0 + within = sum(1 for value in steady_intervals if value * 1000.0 <= deadline_ms) + metrics["slo_attainment"] = within / len(steady_intervals) + metrics["slo_violations"] = len(steady_intervals) - within + + metrics["playback"] = simulate_playback( + ordered, + fps=fps, + buffer_chunks=playback_buffer_chunks, + ).to_dict() + metrics["playback"]["buffer_chunks"] = playback_buffer_chunks + return metrics + + +def aggregate_metrics(sessions: Iterable[Mapping[str, Any]], *, fps: float) -> dict[str, Any]: + """Combine per-session metrics across sequential repeats. + + Steady intervals are pooled rather than averaging per-session means, so a + session that happened to produce fewer steady chunks does not carry the same + weight as a full one. + """ + entries = list(sessions) + if not entries: + raise ValueError("aggregate_metrics() needs at least one session.") + if len(entries) == 1: + single = dict(entries[0]) + single["sessions"] = 1 + return single + + deadline_ms = entries[0]["chunk_deadline_ms"] + if any(entry["chunk_deadline_ms"] != deadline_ms for entry in entries): + raise ValueError("Cannot aggregate sessions with different chunk deadlines.") + ttfcs = [float(entry["ttfc_ms"]) for entry in entries if entry.get("ttfc_ms") is not None] + # Lower is better, so "max" is the worst session, not the best. + rtfs = [float(entry["video_rtf"]) for entry in entries] + steady_means = [ + float(entry["interval_steady"]["mean_ms"]) for entry in entries if entry.get("interval_steady", {}).get("count") + ] + steady_count = sum(entry.get("interval_steady", {}).get("count", 0) for entry in entries) + steady_total_ms = sum( + entry["interval_steady"]["mean_ms"] * entry["interval_steady"]["count"] + for entry in entries + if entry.get("interval_steady", {}).get("count") + ) + return { + "sessions": len(entries), + "chunks_received": sum(int(entry["chunks_received"]) for entry in entries), + "total_frames": sum(int(entry["total_frames"]) for entry in entries), + "video_seconds": sum(float(entry["video_seconds"]) for entry in entries), + "wall_seconds": sum(float(entry["wall_seconds"]) for entry in entries), + "video_rtf_mean": sum(rtfs) / len(rtfs), + "video_rtf_best": min(rtfs), + "video_rtf_worst": max(rtfs), + "ttfc_ms_mean": (sum(ttfcs) / len(ttfcs)) if ttfcs else None, + "ttfc_ms_max": max(ttfcs) if ttfcs else None, + "steady_interval_ms_mean": (steady_total_ms / steady_count) if steady_count else None, + "steady_interval_ms_spread": (max(steady_means) - min(steady_means)) if steady_means else None, + "underrun_count": sum(int(entry["playback"]["underrun_count"]) for entry in entries), + "chunk_deadline_ms": deadline_ms, + } diff --git a/tests/benchmarks/test_lingbot_world_realtime.py b/tests/benchmarks/test_lingbot_world_realtime.py new file mode 100644 index 000000000..9926513b4 --- /dev/null +++ b/tests/benchmarks/test_lingbot_world_realtime.py @@ -0,0 +1,798 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project +"""Unit tests for the LingBot-World realtime streaming benchmark. + +The arithmetic is tested against the server's own formulas rather than against +hand-copied expectations, and the protocol handling runs against a real +WebSocket server that replays ``/v1/realtime/video`` with scripted delays -- so +a benchmark number can be wrong only if the server itself is, not because the +client parsed incorrectly an event or timed the wrong edge. +""" + +from __future__ import annotations + +# ruff: noqa: E402, I001 +import asyncio +import json +import sys +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu] + +REPO_ROOT = Path(__file__).resolve().parents[2] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from benchmarks.lingbot_world.benchmark_lingbot_world_realtime import ( + BenchmarkError, + build_workload, + print_report, + endpoint_url, + parse_args, + resolve_image_reference, + run_session, +) +from benchmarks.lingbot_world.workload import ( + DEFAULT_WARMUP_CHUNKS, + FRAMES_PER_BLOCK, + ChunkRecord, + PromptUpdate, + Workload, + aggregate_metrics, + build_camera_script, + chunks_for_num_frames, + compute_metrics, + default_workload, + load_workload, + num_frames_for_chunks, + percentile, + pixel_frames_in_chunk, + simulate_playback, + steady_chunk_deadline_ms, +) + +_IMAGE = "data:image/png;base64,Zm9v" + + +def _workload(num_chunks: int = 8, fps: int = 16) -> Workload: + return default_workload(image_reference=_IMAGE, num_chunks=num_chunks, fps=fps) + + +# --------------------------------------------------------------------------- # +# Chunk and frame arithmetic +# --------------------------------------------------------------------------- # + + +@pytest.mark.parametrize("num_chunks", list(range(1, 33))) +def test_frame_count_round_trips_through_the_server_formula(num_chunks: int) -> None: + """``num_frames`` must land on a rollout of exactly the requested chunks. + + The server derives chunk count as ``((num_frames - 1) // 4 + 1) // 3``; if + the benchmark's inverse is off by one frame the run silently generates a + different number of blocks than it reports. + """ + assert chunks_for_num_frames(num_frames_for_chunks(num_chunks)) == num_chunks + + +def test_recipe_frame_counts_match_the_documented_chunk_counts() -> None: + # The recipe states three chunks for num_frames 33 and seven for 81. + assert chunks_for_num_frames(33) == 3 + assert chunks_for_num_frames(81) == 7 + assert num_frames_for_chunks(3) == 33 + assert num_frames_for_chunks(7) == 81 + + +def test_first_chunk_is_short_because_the_decoder_expands_it_once() -> None: + assert pixel_frames_in_chunk(0) == 9 + assert pixel_frames_in_chunk(1) == 12 + assert pixel_frames_in_chunk(7) == 12 + # A whole rollout's frames are the sum of its chunks' frames. + assert sum(pixel_frames_in_chunk(index) for index in range(5)) == num_frames_for_chunks(5) + + +def test_camera_script_holds_one_three_entry_action_list_per_chunk() -> None: + script = build_camera_script(4, "orbit") + assert len(script) == 4 + assert all(len(actions) == FRAMES_PER_BLOCK for actions in script) + # orbit advances through the action keys so successive chunks differ. + assert script[0] != script[1] + + +def test_unknown_camera_pattern_is_rejected() -> None: + with pytest.raises(ValueError, match="Unknown camera pattern"): + build_camera_script(2, "sideways") + + +# --------------------------------------------------------------------------- # +# Workload +# --------------------------------------------------------------------------- # + + +def test_session_start_payload_matches_the_documented_protocol() -> None: + workload = _workload(num_chunks=3) + payload = workload.session_start_payload("robbyant/lingbot-world-v2-14b-causal-fast-diffusers") + assert payload["type"] == "session.start" + assert payload["num_frames"] == 33 + assert payload["image_reference"] == {"image_url": _IMAGE} + assert payload["extra_params"]["flow_shift"] == 5.0 + assert len(payload["extra_params"]["camera_action_script"]) == 3 + + +def test_workload_rejects_a_camera_script_with_the_wrong_block_width() -> None: + with pytest.raises(ValueError, match="exactly 3 action lists"): + Workload(prompt="p", image_reference=_IMAGE, camera_script=[[["w"], ["w"]]]) + + +def test_workload_rejects_geometry_the_cache_cannot_serve() -> None: + with pytest.raises(ValueError, match="multiples of 16"): + Workload(prompt="p", image_reference=_IMAGE, camera_script=build_camera_script(1, "forward"), width=831) + + +def test_prompt_update_beyond_the_rollout_is_rejected() -> None: + with pytest.raises(ValueError, match="never fires"): + Workload( + prompt="p", + image_reference=_IMAGE, + camera_script=build_camera_script(2, "forward"), + prompt_updates=(PromptUpdate(after_chunk=5, prompt="later"),), + ) + + +@pytest.mark.parametrize("from_file", [False, True]) +def test_duplicate_prompt_update_boundaries_are_rejected(tmp_path: Path, from_file: bool) -> None: + updates = [ + {"after_chunk": 1, "prompt": "Rain begins"}, + {"after_chunk": 0, "prompt": "Clouds gather"}, + {"after_chunk": 1, "prompt": "Rain stops"}, + ] + with pytest.raises(ValueError, match="Duplicate prompt update after chunk 1"): + if from_file: + path = tmp_path / "rollout.json" + path.write_text(json.dumps({"image": _IMAGE, "num_chunks": 3, "prompt_updates": updates})) + build_workload(parse_args(["--workload", str(path)])) + else: + Workload( + prompt="p", + image_reference=_IMAGE, + camera_script=build_camera_script(3, "forward"), + prompt_updates=( + PromptUpdate(after_chunk=1, prompt="Rain begins"), + PromptUpdate(after_chunk=0, prompt="Clouds gather"), + PromptUpdate(after_chunk=1, prompt="Rain stops"), + ), + ) + + +def test_workload_file_is_loaded_and_cli_overrides_win(tmp_path: Path) -> None: + path = tmp_path / "rollout.json" + path.write_text( + json.dumps( + { + "prompt": "A quiet street", + "image": "https://example.invalid/frame.png", + "num_chunks": 4, + "camera_pattern": "hold", + "fps": 12, + "seed": 7, + "prompt_updates": [{"after_chunk": 1, "prompt": "Rain begins", "transition_chunks": 2}], + } + ) + ) + workload = load_workload(path, overrides={"fps": 24, "seed": None}) + assert workload.prompt == "A quiet street" + assert workload.num_chunks == 4 + assert workload.fps == 24, "an explicit CLI value must beat the file" + assert workload.seed == 7, "a None override must leave the file value alone" + assert workload.prompt_updates[0].transition_chunks == 2 + + +def test_workload_file_without_a_rollout_length_is_rejected(tmp_path: Path) -> None: + path = tmp_path / "rollout.json" + path.write_text(json.dumps({"image": "https://example.invalid/frame.png"})) + with pytest.raises(ValueError, match="camera_action_script or num_chunks"): + load_workload(path) + + +def test_local_image_is_inlined_as_a_data_url(tmp_path: Path) -> None: + image = tmp_path / "frame.png" + image.write_bytes(b"\x89PNG\r\n\x1a\n") + reference = resolve_image_reference(str(image)) + assert reference.startswith("data:image/png;base64,") + assert resolve_image_reference("https://example.invalid/a.png") == "https://example.invalid/a.png" + with pytest.raises(BenchmarkError, match="neither a URL nor a readable file"): + resolve_image_reference(str(tmp_path / "missing.png")) + + +# --------------------------------------------------------------------------- # +# Metric math +# --------------------------------------------------------------------------- # + + +def test_rtf_is_a_cost_not_a_speed() -> None: + """RTF must match vLLM-Omni's own formula, where lower is better. + + ``vllm_omni/metrics/definitions.py::compute_audio_rtf`` is + ``latency / duration`` with the docstring "SLO red line < 1", and + ``benchmarks/patch/patch.py`` sets ``video_rtf = generation_s / + video_duration``. Emitting the reciprocal under the same field name would + silently invert every dashboard that aggregates it, so the direction is + pinned here rather than left to a comment. + """ + # Two chunks, 9 + 12 = 21 frames at 12 fps = 1.75 s of video, generated in 3.5 s. + slow = compute_metrics(_records([1.0, 2.5]), fps=12, warmup_chunks=0, session_wall_s=3.5) + assert slow["video_seconds"] == pytest.approx(1.75) + assert slow["video_rtf"] == pytest.approx(2.0), "twice as slow as real time" + assert slow["video_rtfx"] == pytest.approx(0.5) + + fast = compute_metrics(_records([0.4, 0.475]), fps=12, warmup_chunks=0, session_wall_s=0.875) + assert fast["video_rtf"] == pytest.approx(0.5), "twice as fast as real time" + assert fast["video_rtf"] < 1.0 < slow["video_rtf"] + assert fast["video_rtf"] * fast["video_rtfx"] == pytest.approx(1.0) + + +def test_steady_rtf_is_the_chunk_interval_over_its_deadline() -> None: + """A steady chunk's RTF is just its interval divided by its video duration.""" + metrics = compute_metrics(_records([1.0] + [1.357] * 15), fps=12, warmup_chunks=6) + # 12 frames at 12 fps = 1000 ms of video per steady chunk. + assert metrics["chunk_deadline_ms"] == pytest.approx(1000.0) + assert metrics["steady_video_rtf"] == pytest.approx(1.357, abs=1e-3) + + +def test_percentile_interpolates_so_short_runs_still_report_tails() -> None: + values = [10.0, 20.0, 30.0, 40.0] + assert percentile(values, 0.0) == 10.0 + assert percentile(values, 1.0) == 40.0 + assert percentile(values, 0.5) == 25.0 + assert percentile([5.0], 0.99) == 5.0 + + +def _records(intervals: list[float]) -> list[ChunkRecord]: + """One record per entry; ``intervals[0]`` is the TTFC, the rest are gaps.""" + records: list[ChunkRecord] = [] + arrival = 0.0 + for index, interval in enumerate(intervals): + arrival += interval + records.append( + ChunkRecord( + index=index, + arrival_s=arrival, + inter_arrival_s=interval, + byte_length=1000, + num_frames=pixel_frames_in_chunk(index), + ) + ) + return records + + +def test_steady_state_excludes_the_unsaturated_attention_window() -> None: + """Warmup chunks must not drag the steady-state mean down. + + The first six chunks attend over a growing window, so they are genuinely + cheaper. A benchmark that averages them in reports a cadence the session + will never sustain. + """ + intervals = [1.0] + [0.2] * (DEFAULT_WARMUP_CHUNKS - 1) + [0.8] * 4 + metrics = compute_metrics(_records(intervals), fps=16, warmup_chunks=DEFAULT_WARMUP_CHUNKS) + assert metrics["interval_steady"]["count"] == 3 + assert metrics["interval_steady"]["mean_ms"] == pytest.approx(800.0) + assert metrics["interval_all"]["mean_ms"] < metrics["interval_steady"]["mean_ms"] + + +def test_short_rollouts_report_that_steady_state_was_never_reached() -> None: + metrics = compute_metrics(_records([1.0, 0.5, 0.5]), fps=16, warmup_chunks=DEFAULT_WARMUP_CHUNKS) + assert metrics["steady_state_reached"] is False + assert metrics["interval_steady"] == {} + assert "slo_attainment" not in metrics + + +def test_real_time_factor_and_deadline_use_the_true_frame_counts() -> None: + # Eight chunks: 9 + 7*12 = 93 frames at 16 fps = 5.8125 s of video. + metrics = compute_metrics(_records([1.0] + [0.5] * 7), fps=16, warmup_chunks=0, session_wall_s=5.8125) + assert metrics["total_frames"] == 93 + assert metrics["video_seconds"] == pytest.approx(5.8125) + assert metrics["video_rtf"] == pytest.approx(1.0) + assert metrics["video_rtfx"] == pytest.approx(1.0) + # A steady chunk carries 12 frames, so real time allows 750 ms at 16 fps. + assert metrics["chunk_deadline_ms"] == pytest.approx(750.0) + assert steady_chunk_deadline_ms(12) == pytest.approx(1000.0) + + +def test_slo_attainment_counts_only_steady_chunks() -> None: + intervals = [1.0] + [0.1] * 5 + [0.7, 0.9, 0.7, 0.9] + metrics = compute_metrics(_records(intervals), fps=16, warmup_chunks=6, slo_ms=750.0) + assert metrics["slo_attainment"] == pytest.approx(2 / 3) + assert metrics["slo_violations"] == 1 + + +def test_playback_simulation_finds_stalls_a_percentile_would_hide() -> None: + """One late chunk behind a one-chunk buffer is a visible stall.""" + # 12 frames at 16 fps = 750 ms of playback per chunk; chunk 3 takes 2 s. + intervals = [1.0, 0.5, 0.5, 2.0, 0.5] + result = simulate_playback(_records(intervals), fps=16, buffer_chunks=1) + assert result.underrun_count >= 1 + assert result.first_underrun_chunk == 3 + assert result.total_stall_s > 0.0 + + +def test_a_deeper_buffer_absorbs_the_same_jitter() -> None: + intervals = [1.0, 0.5, 0.5, 1.4, 0.2, 0.2] + fast = simulate_playback(_records(intervals), fps=16, buffer_chunks=1) + buffered = simulate_playback(_records(intervals), fps=16, buffer_chunks=3) + assert buffered.underrun_count < fast.underrun_count + + +def test_playback_rejects_a_buffer_deeper_than_the_rollout() -> None: + with pytest.raises(ValueError, match="exceeds"): + simulate_playback(_records([1.0, 0.5]), fps=16, buffer_chunks=9) + + +def test_aggregate_pools_sessions_and_reports_spread() -> None: + fast = compute_metrics(_records([1.0] + [0.1] * 5 + [0.6] * 4), fps=16, warmup_chunks=6) + slow = compute_metrics(_records([1.0] + [0.1] * 5 + [0.9] * 4), fps=16, warmup_chunks=6) + aggregate = aggregate_metrics([fast, slow], fps=16) + assert aggregate["sessions"] == 2 + assert aggregate["steady_interval_ms_mean"] == pytest.approx(750.0) + assert aggregate["steady_interval_ms_spread"] == pytest.approx(300.0) + assert aggregate["video_rtf_best"] <= aggregate["video_rtf_mean"] <= aggregate["video_rtf_worst"] + + +def test_single_session_aggregate_is_the_session_itself() -> None: + metrics = compute_metrics(_records([1.0] + [0.5] * 3), fps=16, warmup_chunks=0) + aggregate = aggregate_metrics([metrics], fps=16) + assert aggregate["sessions"] == 1 + assert aggregate["video_rtf"] == metrics["video_rtf"] + + +# --------------------------------------------------------------------------- # +# CLI +# --------------------------------------------------------------------------- # + + +def test_endpoint_url_normalizes_http_and_appends_the_ws_path() -> None: + assert endpoint_url(parse_args([])) == "ws://127.0.0.1:8000/v1/realtime/video" + assert endpoint_url(parse_args(["--host", "h", "--port", "9"])) == "ws://h:9/v1/realtime/video" + args = parse_args(["--base-url", "http://example.invalid:8099"]) + assert endpoint_url(args) == "ws://example.invalid:8099/v1/realtime/video" + args = parse_args(["--base-url", "ws://example.invalid:8099/v1/realtime/video"]) + assert endpoint_url(args) == "ws://example.invalid:8099/v1/realtime/video" + + +def test_cli_rejects_a_zero_chunk_buffer() -> None: + with pytest.raises(SystemExit): + parse_args(["--playback-buffer-chunks", "0"]) + with pytest.raises(SystemExit): + parse_args(["--target-fps", "0"]) + + +def test_the_real_time_basis_is_independent_of_the_mux_label() -> None: + """The checkpoint declares no frame rate, so the basis must be statable. + + The same measured cadence has RTF 1.809 at 16 fps and 1.357 at 12 fps; + tying that verdict to the fps sent to the muxer would let a playback label + silently decide whether a run passes. + """ + records = _records([1.0] + [1.357] * 15) + at_16 = compute_metrics(records, fps=16, warmup_chunks=6) + at_12 = compute_metrics(records, fps=12, warmup_chunks=6) + assert at_16["chunk_deadline_ms"] == pytest.approx(750.0) + assert at_12["chunk_deadline_ms"] == pytest.approx(1000.0) + # RTF is cost, so the harsher 16 fps basis scores WORSE (higher). + assert at_16["video_rtf"] > at_12["video_rtf"] + # Same wall clock, same bytes: only the basis moved. + assert at_16["video_rtf"] / at_12["video_rtf"] == pytest.approx(16 / 12) + assert parse_args([]).target_fps is None, "default defers to the workload fps" + + +# --------------------------------------------------------------------------- # +# Protocol handling against a scripted server +# --------------------------------------------------------------------------- # + + +def _serve(script): + """Return an async context manager yielding a ws:// URL for ``script``.""" + from websockets.asyncio.server import Server, serve + + class _Server: + def __init__(self) -> None: + self.server: Server | None = None + self.start_payloads: list[dict] = [] + + async def __aenter__(self) -> str: + async def handler(websocket) -> None: + self.start_payloads.append(json.loads(await websocket.recv())) + await script(websocket, self.start_payloads[-1]) + + self.server = await serve(handler, "127.0.0.1", 0) + port = next(iter(self.server.sockets)).getsockname()[1] + return f"ws://127.0.0.1:{port}/v1/realtime/video" + + async def __aexit__(self, *exc) -> None: + assert self.server is not None + self.server.close() + await self.server.wait_closed() + + return _Server() + + +async def _emit_rollout(websocket, payload, *, intervals=None, chunks=None, trailer=True) -> None: + """Replay the documented event order: start, metadata+binary per chunk, done.""" + num_chunks = chunks if chunks is not None else len(payload["extra_params"]["camera_action_script"]) + await websocket.send(json.dumps({"type": "video.start", "request_id": "req-1", "format": "m4s"})) + for index in range(num_chunks): + if intervals is not None: + await asyncio.sleep(intervals[index]) + frames = pixel_frames_in_chunk(index) + body = bytes([index % 256]) * (100 + index) + await websocket.send( + json.dumps( + { + "type": "video.chunk_metadata", + "request_id": "req-1", + "kind": "media", + "num_frames": frames, + "byte_length": len(body), + "generation_chunk_index": index, + } + ) + ) + await websocket.send(body) + if trailer: + tail = b"\x00" * 8 + await websocket.send( + json.dumps( + { + "type": "video.chunk_metadata", + "request_id": "req-1", + "kind": "trailer", + "num_frames": 0, + "byte_length": len(tail), + "generation_chunk_index": None, + } + ) + ) + await websocket.send(tail) + await websocket.send(json.dumps({"type": "session.done", "stopped": False, "chunks": num_chunks})) + + +def _run(coro): + return asyncio.run(asyncio.wait_for(coro, timeout=30)) + + +def test_session_records_one_chunk_per_block_and_ignores_the_trailer() -> None: + workload = _workload(num_chunks=4) + + async def scenario() -> None: + server = _serve(_emit_rollout) + url = await server.__aenter__() + try: + result = await run_session( + url, + "model", + workload, + first_chunk_timeout=10, + chunk_timeout=10, + ping_interval=0, + collect_media=True, + print_chunks=False, + ) + finally: + await server.__aexit__(None, None, None) + + assert [record.index for record in result.records] == [0, 1, 2, 3] + assert [record.num_frames for record in result.records] == [9, 12, 12, 12] + assert result.request_id == "req-1" + assert result.ttfc_s is not None and result.ttfc_s > 0 + # The trailer's bytes are kept for remuxing but carry no chunk record. + assert len(result.media) == sum(record.byte_length for record in result.records) + 8 + assert server.start_payloads[0]["num_frames"] == workload.num_frames + + _run(scenario()) + + +def test_measured_intervals_track_the_server_cadence() -> None: + workload = _workload(num_chunks=3) + + async def script(websocket, payload) -> None: + await _emit_rollout(websocket, payload, intervals=[0.05, 0.30, 0.05]) + + async def scenario() -> None: + server = _serve(script) + url = await server.__aenter__() + try: + result = await run_session( + url, + "model", + workload, + first_chunk_timeout=10, + chunk_timeout=10, + ping_interval=0, + collect_media=False, + print_chunks=False, + ) + finally: + await server.__aexit__(None, None, None) + + intervals = [record.inter_arrival_s for record in result.records] + assert intervals[1] == pytest.approx(0.30, abs=0.15) + assert intervals[2] < intervals[1], "a fast chunk must not inherit the slow one's interval" + assert result.records[0].arrival_s == pytest.approx(result.ttfc_s) + + _run(scenario()) + + +def test_a_short_rollout_fails_loudly_instead_of_reporting_a_partial_cadence() -> None: + workload = _workload(num_chunks=4) + + async def script(websocket, payload) -> None: + await _emit_rollout(websocket, payload, chunks=2) + + async def scenario() -> None: + server = _serve(script) + url = await server.__aenter__() + try: + with pytest.raises(BenchmarkError, match="Expected 4 chunks"): + await run_session( + url, + "model", + workload, + first_chunk_timeout=10, + chunk_timeout=10, + ping_interval=0, + collect_media=False, + print_chunks=False, + ) + finally: + await server.__aexit__(None, None, None) + + _run(scenario()) + + +def test_a_server_error_event_aborts_the_session() -> None: + workload = _workload(num_chunks=2) + + async def script(websocket, payload) -> None: + await websocket.send(json.dumps({"type": "error", "message": "resolution mismatch"})) + + async def scenario() -> None: + server = _serve(script) + url = await server.__aenter__() + try: + with pytest.raises(BenchmarkError, match="resolution mismatch.*requested 2 chunks"): + await run_session( + url, + "model", + workload, + first_chunk_timeout=10, + chunk_timeout=10, + ping_interval=0, + collect_media=False, + print_chunks=False, + ) + finally: + await server.__aexit__(None, None, None) + + _run(scenario()) + + +def test_a_stalled_server_times_out_rather_than_hanging() -> None: + workload = _workload(num_chunks=2) + + async def script(websocket, payload) -> None: + await websocket.send(json.dumps({"type": "video.start", "request_id": "req-1", "format": "m4s"})) + await asyncio.sleep(5) + + async def scenario() -> None: + server = _serve(script) + url = await server.__aenter__() + try: + with pytest.raises(BenchmarkError, match="No media chunk for"): + await run_session( + url, + "model", + workload, + first_chunk_timeout=0.4, + chunk_timeout=0.4, + ping_interval=0, + collect_media=False, + print_chunks=False, + ) + finally: + await server.__aexit__(None, None, None) + + _run(scenario()) + + +def test_keepalive_pings_reach_the_server_during_a_slow_first_chunk() -> None: + """A compiled first block can outlast the server's stall timeout.""" + workload = _workload(num_chunks=1) + seen: list[str] = [] + + async def script(websocket, payload) -> None: + await websocket.send(json.dumps({"type": "video.start", "request_id": "req-1", "format": "m4s"})) + + async def drain() -> None: + async for message in websocket: + event = json.loads(message) + seen.append(event["type"]) + if event["type"] == "session.ping": + await websocket.send(json.dumps({"type": "session.pong"})) + + drainer = asyncio.create_task(drain()) + await asyncio.sleep(0.35) + await _emit_rollout(websocket, payload, trailer=False) + drainer.cancel() + + async def scenario() -> None: + server = _serve(script) + url = await server.__aenter__() + try: + result = await run_session( + url, + "model", + workload, + first_chunk_timeout=10, + chunk_timeout=10, + ping_interval=0.1, + collect_media=False, + print_chunks=False, + ) + finally: + await server.__aexit__(None, None, None) + assert len(result.records) == 1 + assert "session.ping" in seen + + _run(scenario()) + + +def test_prompt_updates_are_sent_on_their_chunk_boundary() -> None: + workload = Workload( + prompt="start", + image_reference=_IMAGE, + camera_script=build_camera_script(3, "forward"), + prompt_updates=(PromptUpdate(after_chunk=0, prompt="Rain begins", transition_chunks=1),), + ) + received: list[dict] = [] + + async def script(websocket, payload) -> None: + async def drain() -> None: + async for message in websocket: + received.append(json.loads(message)) + + drainer = asyncio.create_task(drain()) + await _emit_rollout(websocket, payload, intervals=[0.02, 0.15, 0.02], trailer=False) + await asyncio.sleep(0.05) + drainer.cancel() + + async def scenario() -> None: + server = _serve(script) + url = await server.__aenter__() + try: + await run_session( + url, + "model", + workload, + first_chunk_timeout=10, + chunk_timeout=10, + ping_interval=0, + collect_media=False, + print_chunks=False, + ) + finally: + await server.__aexit__(None, None, None) + interactions = [event for event in received if event.get("type") == "session.interaction"] + assert len(interactions) == 1 + assert interactions[0]["interaction"]["event"]["prompt"] == "Rain begins" + assert interactions[0]["interaction"]["transition_chunks"] == 1 + + _run(scenario()) + + +@pytest.mark.parametrize("delivered", [0, 1]) +def test_pongs_do_not_extend_media_deadlines(delivered) -> None: + seen = [] + + async def script(websocket, payload) -> None: + if delivered: + await websocket.send( + json.dumps( + { + "type": "video.chunk_metadata", + "kind": "media", + "generation_chunk_index": 0, + "num_frames": 9, + } + ) + ) + await websocket.send(b"media") + async for message in websocket: + seen.append(json.loads(message)["type"]) + await websocket.send(json.dumps({"type": "session.pong"})) + + async def scenario() -> None: + async with _serve(script) as url: + with pytest.raises(BenchmarkError, match=f"after {delivered} chunk"): + await asyncio.wait_for( + run_session( + url, + "model", + _workload(num_chunks=2), + first_chunk_timeout=0.25, + chunk_timeout=0.25, + ping_interval=0.03, + collect_media=False, + print_chunks=False, + ), + timeout=2, + ) + assert "session.ping" in seen + + _run(scenario()) + + +def test_terminal_chunk_is_only_excluded_from_steady_metrics() -> None: + records = _records([1.0] + [0.1] * 5 + [1.2, 0.8, 0.1]) + metrics = compute_metrics(records, fps=12, warmup_chunks=6) + assert metrics["steady_chunks"] == 2 + assert metrics["interval_steady"]["mean_ms"] == pytest.approx(1000) + assert metrics["steady_video_rtf"] == pytest.approx(1) + assert metrics["slo_attainment"] == pytest.approx(0.5) + assert metrics["interval_all"]["count"] == 8 + assert metrics["total_frames"] == 105 + assert metrics["wall_seconds"] == pytest.approx(records[-1].arrival_s) + assert metrics["playback"]["total_stall_ms"] == pytest.approx( + simulate_playback(records, fps=12).total_stall_s * 1000 + ) + + +def test_terminal_chunk_alone_does_not_establish_steady_state() -> None: + metrics = compute_metrics(_records([1.0] * 7), fps=12, warmup_chunks=6) + assert not metrics["steady_state_reached"] + assert "steady_video_rtf" not in metrics + + +def test_aggregate_preserves_custom_deadline_and_weights_intervals() -> None: + fast = compute_metrics(_records([1.0, 0.5, 0.1]), fps=12, warmup_chunks=0, slo_ms=700) + slow = compute_metrics(_records([1.0, 0.9, 0.9, 0.9, 0.1]), fps=12, warmup_chunks=0, slo_ms=700) + aggregate = aggregate_metrics([fast, slow], fps=12) + assert aggregate["chunk_deadline_ms"] == 700 + assert aggregate["steady_interval_ms_mean"] == pytest.approx(800) + slow["chunk_deadline_ms"] = 800 + with pytest.raises(ValueError, match="different chunk deadlines"): + aggregate_metrics([fast, slow], fps=12) + + +@pytest.mark.parametrize( + "flags, expected_shift, expected_negative", + [ + ([], 7.0, "file negative"), + (["--flow-shift", "5", "--negative-prompt", "CLI negative"], 5.0, "CLI negative"), + (["--negative-prompt", ""], 7.0, ""), + ], +) +def test_cli_workload_generation_overrides(tmp_path, flags, expected_shift, expected_negative) -> None: + path = tmp_path / "workload.json" + path.write_text( + json.dumps( + { + "image": _IMAGE, + "num_chunks": 3, + "flow_shift": 7.0, + "negative_prompt": "file negative", + } + ) + ) + workload = build_workload(parse_args(["--workload", str(path), *flags])) + assert workload.flow_shift == expected_shift + assert workload.negative_prompt == expected_negative + + +def test_builtin_workload_keeps_default_flow_shift() -> None: + assert build_workload(parse_args(["--image", _IMAGE])).flow_shift == 5.0 + + +def test_short_sample_report_qualifies_tail_percentiles(capsys) -> None: + metrics = compute_metrics(_records([1.0] * 16), fps=12) + print_report(metrics, workload=_workload(num_chunks=16), endpoint="ws://test", sessions=1, target_fps=12) + assert "not reliable tail estimates" in capsys.readouterr().out