[Frontend] Add MiniMax-H3 latent editing workflows (WF-05) (#7898)

Signed-off-by: chen hongwei <1792043268@qq.com> Signed-off-by: Weiming Liao <liaowm5@gmail.com> Co-authored-by: chen hongwei <1792043268@qq.com> Co-authored-by: chen hongwei <54873389+avicii-forever@users.noreply.github.com>
2026-09-21 13:07:21 +00:00
1 parent e5c0afc994
commit fa1d03ab8d
@@ -0,0 +1,115 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""Minimal mock of vLLM-Omni's ``/v1/videos`` API for e2e tests.
+
+Records every received multipart field to ``MOCK_STATE_FILE`` (JSON) so a test
+can assert exactly what the client serialized, and returns a synthetic MP4.
+"""
+
+import asyncio
+import json
+import os
+import subprocess
+import uuid
+
+from fastapi import FastAPI, Request
+from fastapi.responses import Response
+from starlette.datastructures import UploadFile
+
+app = FastAPI()
+_JOBS: dict[str, dict] = {}
+_STATE_FILE = os.environ.get("MOCK_STATE_FILE")
+
+
+def _record(fields: dict) -> None:
+    if _STATE_FILE:
+        with open(_STATE_FILE, "w") as f:
+            json.dump(fields, f)
+
+
+def _synthetic_mp4() -> bytes:
+    return subprocess.run(
+        [
+            "ffmpeg",
+            "-hide_banner",
+            "-loglevel",
+            "error",
+            "-y",
+            "-f",
+            "lavfi",
+            "-i",
+            "testsrc2=size=320x240:rate=24:duration=1",
+            "-f",
+            "lavfi",
+            "-i",
+            "sine=frequency=440:sample_rate=32000:duration=1",
+            "-map",
+            "0:v",
+            "-map",
+            "1:a",
+            "-shortest",
+            "-c:v",
+            "libx264",
+            "-pix_fmt",
+            "yuv420p",
+            "-c:a",
+            "aac",
+            "-ar",
+            "32000",
+            "-ac",
+            "2",
+            "-movflags",
+            "frag_keyframe+empty_moov",
+            "-f",
+            "mp4",
+            "pipe:1",
+        ],
+        check=True,
+        capture_output=True,
+    ).stdout
+
+
+@app.post("/v1/videos")
+async def create_video(request: Request):
+    form = await request.form()
+    fields = {}
+    for key, value in form.items():
+        if isinstance(value, UploadFile):
+            data = await value.read()
+            fields[key] = {
+                "file": True,
+                "size": len(data),
+                "filename": value.filename,
+                "content_type": value.content_type,
+            }
+            if value.content_type == "application/json":
+                fields[key]["json"] = json.loads(data)
+        else:
+            fields[key] = str(value)
+    _record(fields)
+
+    job_id = str(uuid.uuid4())
+    _JOBS[job_id] = {"status": "queued"}
+    return {"id": job_id, "status": "queued"}
+
+
+@app.get("/v1/videos/{job_id}")
+async def get_video(job_id: str):
+    if job_id not in _JOBS:
+        return {"status": "failed", "error": "unknown job"}
+    return {"id": job_id, "status": "completed"}
+
+
+@app.get("/v1/videos/{job_id}/content")
+async def get_video_content(job_id: str):
+    if job_id not in _JOBS:
+        return Response(status_code=404)
+    mp4 = await asyncio.to_thread(_synthetic_mp4)
+    return Response(content=mp4, media_type="video/mp4")
+
+
+@app.delete("/v1/videos/{job_id}")
+async def delete_video(job_id: str):
+    _JOBS.pop(job_id, None)
+    return {"deleted": True}
@@ -0,0 +1,80 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""Unit tests for latent-mask serialization helpers."""
+
+import json
+
+import pytest
+import torch
+from comfyui_vllm_omni.utils.latent_mask import scalar_mask_to_json, video_mask_to_grid_json
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
+
+def test_scalar_mask_to_json():
+    assert scalar_mask_to_json(0.5) == "0.5"
+    assert scalar_mask_to_json(1.0) == "1.0"
+    with pytest.raises(ValueError):
+        scalar_mask_to_json(1.5)
+    with pytest.raises(ValueError):
+        scalar_mask_to_json(-0.1)
+
+
+def test_video_mask_grid_shape():
+    mask = torch.zeros(256, 448)
+    grid = json.loads(video_mask_to_grid_json(mask, width=448, height=256, num_frames=107))
+    assert len(grid) == 2 + 5 * ((107 - 5) // 17)
+    assert len(grid[0]) == 256 // 16
+    assert len(grid[0][0]) == 448 // 16
+
+
+def test_video_mask_grid_shape_non_aligned_frames():
+    # 96 snaps up to 107 (17n+5), so Tv == 32, matching the server.
+    grid = json.loads(video_mask_to_grid_json(torch.zeros(256, 448), width=448, height=256, num_frames=96))
+    assert len(grid) == 32
+
+
+def test_video_mask_grid_shape_short_clip():
+    # num_frames <= 5 -> Tv == 2.
+    grid = json.loads(video_mask_to_grid_json(torch.zeros(64, 64), width=64, height=64, num_frames=1))
+    assert len(grid) == 2
+
+
+def test_video_mask_grid_shape_unaligned_width():
+    # width=500 floors to 480 (multiple of 32) -> gw == 30, not 31.
+    grid = json.loads(video_mask_to_grid_json(torch.zeros(64, 64), width=500, height=256, num_frames=22))
+    assert len(grid[0][0]) == 30
+
+
+def test_video_mask_3d_input():
+    grid = json.loads(video_mask_to_grid_json(torch.ones(1, 256, 448), width=448, height=256, num_frames=107))
+    assert grid[0][0][0] == 1.0
+
+
+def test_video_mask_temporal():
+    # 3D mask: first half 0 (preserve), second half 1 (regenerate).
+    mask = torch.zeros(7, 64, 64)
+    mask[4:] = 1.0
+    grid = json.loads(video_mask_to_grid_json(mask, width=64, height=64, num_frames=22))
+    assert len(grid) == 7
+    assert all(v == 0.0 for row in grid[0] for v in row)
+    assert all(v == 1.0 for row in grid[-1] for v in row)
+
+
+def test_video_mask_temporal_resize():
+    # 2 temporal slices resize to tv == 7.
+    mask = torch.zeros(2, 64, 64)
+    mask[1] = 1.0
+    grid = json.loads(video_mask_to_grid_json(mask, width=64, height=64, num_frames=22))
+    assert len(grid) == 7
+
+
+def test_video_mask_uniform_preserved():
+    grid = json.loads(video_mask_to_grid_json(torch.full((64, 64), 0.5), width=64, height=64, num_frames=107))
+    assert all(v == 0.5 for row in grid for cell in row for v in cell)
+
+
+def test_video_mask_values_in_range():
+    grid = json.loads(video_mask_to_grid_json(torch.rand(128, 128), width=128, height=128, num_frames=22))
+    assert all(0.0 <= v <= 1.0 for row in grid for cell in row for v in cell)
@@ -0,0 +1,99 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""End-to-end test for latent-mask editing serialization.
+
+Exercises ``VLLMOmniClient.generate_video`` with a ``latent_edit`` payload
+against a mock ``/v1/videos`` server and asserts the multipart fields the
+client produced. Runs on CPU: the ComfyUI ``comfy_api`` / ``comfy_extras``
+modules are mocked by this directory's ``conftest.py``.
+"""
+
+import asyncio
+import json
+import os
+import socket
+import subprocess
+import sys
+import time
+
+import pytest
+import torch
+from comfy_api.input import VideoInput
+from comfyui_vllm_omni.utils.api_client import VLLMOmniClient
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
+_TESTS_DIR = os.path.dirname(os.path.abspath(__file__))
+
+
+def _free_port() -> int:
+    with socket.socket() as s:
+        s.bind(("127.0.0.1", 0))
+        return s.getsockname()[1]
+
+
+@pytest.fixture
+def mock_server(tmp_path):
+    port = _free_port()
+    state_file = tmp_path / "state.json"
+    env = dict(os.environ, MOCK_STATE_FILE=str(state_file))
+    proc = subprocess.Popen(
+        [sys.executable, "-m", "uvicorn", "mock_videos_server:app", "--host", "127.0.0.1", "--port", str(port)],
+        cwd=_TESTS_DIR,
+        env=env,
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+    )
+    try:
+        deadline = time.time() + 30
+        while time.time() < deadline:
+            try:
+                with socket.create_connection(("127.0.0.1", port), timeout=1):
+                    break
+            except OSError:
+                time.sleep(0.2)
+        yield f"http://127.0.0.1:{port}/v1", state_file
+    finally:
+        proc.terminate()
+        proc.wait(timeout=10)
+
+
+def test_latent_edit_serialization(mock_server):
+    base_url, state_file = mock_server
+
+    # A non-trivial video mask requires a source video. The mocked VideoInput
+    # only needs to provide ``save_to`` for the client's multipart upload.
+    source_video = VideoInput(b"mock_source_video")
+    mask = torch.zeros(1, 120, 160)
+    latent_edit = {"source_video": source_video, "video_mask": mask, "audio_mask": 0.5}
+
+    async def run():
+        client = VLLMOmniClient(base_url)
+        return await client.generate_video(
+            model="MiniMaxAI/MiniMax-H3",
+            prompt="restyle the clip",
+            width=160,
+            height=120,
+            num_frames=22,
+            fps=24,
+            latent_edit=latent_edit,
+        )
+
+    out = asyncio.run(run())
+    assert out is not None
+
+    with open(state_file) as f:
+        fields = json.load(f)
+
+    assert fields["source_video"]["file"] is True
+    assert fields["source_video"]["content_type"] == "video/mp4"
+    for name in ("video_noise_mask", "audio_noise_mask"):
+        assert fields[name]["file"] is True
+        assert fields[name]["content_type"] == "application/json"
+    assert fields["audio_noise_mask"]["json"] == 0.5
+
+    video_mask = fields["video_noise_mask"]["json"]
+    assert len(video_mask) == 7  # num_frames=22 aligns to 22 (17n+5) -> Tv = 7
+    assert len(video_mask[0]) == 6  # height 120 floors to 96 (multiple of 32) -> gh 6
+    assert len(video_mask[0][0]) == 10  # width 160 -> gw 10