[Feature][LingBot World]stepwise execution (#6844)
Signed-off-by: BruceLoveDecimal <liuqihao970610@gmail.com> Signed-off-by: liuqihao <liuqihao970610@gmail.com>
This commit is contained in:
1 parent
7f32c326c6
commit
5cd925631c
24 files changed
+2087
-217
No files matched your search
@@ -30,7 +30,6 @@ from typing import Any
|
||||
import numpy as np
|
||||
|
||||
_MODEL = "robbyant/lingbot-world-v2-14b-causal-fast-diffusers"
|
||||
_NUM_INFERENCE_STEPS = 4
|
||||
_MAX_SEQUENCE_LENGTH = 512
|
||||
_MAX_PIXEL_AREA = 480 * 832
|
||||
_MAX_RAW_FRAMES = 117
|
||||
@@ -209,7 +208,6 @@ def build_request(
|
||||
"height": args.height,
|
||||
"width": args.width,
|
||||
"num_frames": args.num_frames,
|
||||
"num_inference_steps": _NUM_INFERENCE_STEPS,
|
||||
"max_sequence_length": _MAX_SEQUENCE_LENGTH,
|
||||
"seed": args.seed,
|
||||
"fps": args.fps,
|
||||
|
||||
@@ -155,7 +155,6 @@ async def run(argv: Sequence[str] | None = None) -> Path:
|
||||
height=args.height,
|
||||
width=args.width,
|
||||
num_frames=9,
|
||||
num_inference_steps=4,
|
||||
max_sequence_length=512,
|
||||
seed=args.seed,
|
||||
output_type="latent",
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# Streaming Video Generation
|
||||
|
||||
This example uses the custom WebSocket endpoint `WS /v1/realtime/video` to receive a video byte stream as chunks are produced.
|
||||
It covers text-only video generation. Image/reference input is intentionally not included for now.
|
||||
It covers text-to-video, and image-conditioned models through `--image-reference`.
|
||||
|
||||
## Start The Server
|
||||
|
||||
@@ -20,7 +20,7 @@ The `--diffusion-streaming-output` CLI flag is forwarded as `streaming_output=Tr
|
||||
|
||||
| Direction | Message | Format | Description |
|
||||
| --- | --- | --- | --- |
|
||||
| Client to server | `session.start` | JSON text: `{"type":"session.start","model":"...","prompt":"...","format":"m4s"}` | Starts generation. `format` is optional and accepts `m4s` (default). Sampling fields such as `width`, `height`, `fps`, `num_frames`, and `extra_params` may be included. |
|
||||
| Client to server | `session.start` | JSON text: `{"type":"session.start","model":"...","prompt":"...","format":"m4s"}` | Starts generation. `format` is optional and accepts `m4s` (default). Sampling fields such as `width`, `height`, `fps`, `num_frames`, and `extra_params` may be included. Image-conditioned models take a first frame as `image_reference`: `{"image_url":"https://..."}` or a `data:` URL. |
|
||||
| Client to server | `session.interaction` | JSON text: `{"type":"session.interaction","interaction":{"event_id":"xxx","event":{"prompt":"..."},"transition_chunks":3}}` | Updates the active prompt midway through generation. `event_id` is optional and `transition_chunks` defaults to the model setting. |
|
||||
| Server to client | `video.start` | JSON text: `{"type":"video.start","request_id":"...","format":"m4s","config":{...}}` | Confirms the session and mirrors the accepted `format`. |
|
||||
| Server to client | `video.chunk_metadata` | JSON text: `{"type":"video.chunk_metadata","request_id":"...","kind":"media","transport_chunk_index":0,"generation_chunk_index":0,"num_frames":9,"byte_length":1234,"started_event_ids":[],"active_event_ids":[],"completed_event_ids":[]}` | Precedes each binary frame and describes the immediately following payload. |
|
||||
@@ -62,7 +62,7 @@ python streaming_video_client.py \
|
||||
The client sends one `session.start` message, prints each received binary video chunk with its byte size and elapsed time, and saves the received bytes to `--output` after `session.done`.
|
||||
The client remuxes the gathered stream to a regular progressive MP4 file so that local playback knows the video duration.
|
||||
|
||||
Schedule midway prompt updates with `--prompt-updates`. Each entry uses `"at"` as seconds on the client clock after the server sends `video.start`:
|
||||
Schedule midway prompt updates with `--prompt-updates`. None are sent unless you ask for them, and only pipelines that implement midway prompt updates accept them (LingBot-World, for one, does not, and rejects the whole session). Each entry uses `"at"` as seconds on the client clock after the server sends `video.start`:
|
||||
|
||||
```bash
|
||||
python streaming_video_client.py \
|
||||
@@ -73,6 +73,19 @@ python streaming_video_client.py \
|
||||
]'
|
||||
```
|
||||
|
||||
The Helios demo continues the default prompt into an underwater storm:
|
||||
|
||||
```bash
|
||||
python streaming_video_client.py \
|
||||
--model BestWishYsh/Helios-Distilled \
|
||||
--prompt-updates '[
|
||||
{"at": 4,
|
||||
"prompt": "An underwater tornado appears and affects the ocean floor in a dramatic and chaotic scene. The water is murky, swirling violently, carrying debris and marine life into the vortex. The tropical fish on the scene all swim in panic, trying to avoid the powerful currents. The camera remains stationary, capturing the intensity of the underwater tornado as it disrupts the serene ocean floor. Close-up shot emphasizing the turbulent motion and destruction."},
|
||||
{"at": 11,
|
||||
"prompt": "The swirling underwater vortex now seizes a heavy, encrusted treasure chest, its lid flapping open as it is smashed onto the ocean floor. Gold coins and silver trinkets spill out, glittering briefly in the murky water before being swept instantly into the violent funnel. The heavy wooden box tumbles end over end, colliding with floating rocks and adding to the debris field. Swirling sediment and bubbles surround the spilling fortune, highlighting the chaotic power of the storm as it ravages the seabed. Close-up shot emphasizing the turbulent motion and destruction."}
|
||||
]'
|
||||
```
|
||||
|
||||
## Run The Gradio Demo
|
||||
|
||||
```bash
|
||||
@@ -106,3 +119,31 @@ Disable it in the CLI example with `--no-helios-distilled-preset`, or override/e
|
||||
python streaming_video_client.py \
|
||||
--extra-params '{"pyramid_num_inference_steps_list":[2, 2, 2]}'
|
||||
```
|
||||
|
||||
### LingBot-World 2.0
|
||||
|
||||
`robbyant/lingbot-world-v2-14b-causal-fast-diffusers` is image-conditioned and runs on the
|
||||
AR-Diffusion engine, so it needs the stepwise deploy config and a first frame. See
|
||||
[the recipe](../../../recipes/Robbyant/LingBot-World-2.0.md) for the constraints on
|
||||
`width`/`height` and `num_frames`.
|
||||
|
||||
```bash
|
||||
vllm serve robbyant/lingbot-world-v2-14b-causal-fast-diffusers \
|
||||
--omni \
|
||||
--deploy-config vllm_omni/deploy/lingbot_world_v2_stepwise.yaml \
|
||||
--port 8000
|
||||
```
|
||||
|
||||
```bash
|
||||
python streaming_video_client.py \
|
||||
--model robbyant/lingbot-world-v2-14b-causal-fast-diffusers \
|
||||
--prompt "The camera moves slowly forward through the scene." \
|
||||
--image-reference /path/to/first_frame.png \
|
||||
--width 832 --height 480 --num-frames 33 --fps 16 --seed 42 \
|
||||
--extra-params '{"camera_action_script":[[["w"],["w"],["w"]],[["a"],[],[]],[[],[],[]]]}' \
|
||||
--output lingbot_world_v2_stream.mp4
|
||||
```
|
||||
|
||||
`camera_action_script` carries one three-latent-frame WASD action list per generated
|
||||
chunk. Mid-session `session.interaction` only updates the prompt, so a LingBot rollout
|
||||
follows the camera script it started with.
|
||||
@@ -1,11 +1,12 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Text-only WebSocket client for `/v1/realtime/video`.
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""WebSocket client for `/v1/realtime/video`.
|
||||
|
||||
The client prints a line for every binary video chunk it receives and writes
|
||||
the received bytes to disk when the session finishes or is interrupted.
|
||||
Image/reference input is intentionally not supported in this example yet.
|
||||
``--image-reference`` sends a first frame for image-conditioned models such as
|
||||
LingBot-World 2.0; text-only models need no image.
|
||||
|
||||
Requirements:
|
||||
pip install av websockets
|
||||
@@ -13,9 +14,11 @@ Requirements:
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import base64
|
||||
import contextlib
|
||||
import io
|
||||
import json
|
||||
import mimetypes
|
||||
import os
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
@@ -53,6 +56,18 @@ class ScheduledPromptUpdate:
|
||||
transition_chunks: int
|
||||
|
||||
|
||||
def _image_reference(value: str) -> dict[str, Any]:
|
||||
"""Accept an http(s) URL as-is, or inline a local image as a data URL."""
|
||||
if value.startswith(("http://", "https://", "data:")):
|
||||
return {"image_url": value}
|
||||
path = Path(value)
|
||||
if not path.is_file():
|
||||
raise argparse.ArgumentTypeError(f"--image-reference is neither a URL nor a readable file: {value}")
|
||||
media_type = mimetypes.guess_type(path.name)[0] or "image/png"
|
||||
encoded = base64.b64encode(path.read_bytes()).decode()
|
||||
return {"image_url": f"data:{media_type};base64,{encoded}"}
|
||||
|
||||
|
||||
def _parse_extra_params(value: str) -> dict[str, Any]:
|
||||
try:
|
||||
parsed = json.loads(value)
|
||||
@@ -116,6 +131,7 @@ def build_session_start(args: argparse.Namespace) -> dict[str, Any]:
|
||||
"prompt": args.prompt,
|
||||
}
|
||||
|
||||
_maybe_set(payload, "image_reference", args.image_reference)
|
||||
_maybe_set(payload, "negative_prompt", args.negative_prompt)
|
||||
_maybe_set(payload, "width", args.width)
|
||||
_maybe_set(payload, "height", args.height)
|
||||
@@ -362,6 +378,12 @@ def parse_args() -> argparse.Namespace:
|
||||
parser.add_argument("--fps", type=int, default=16, help="Frames per second")
|
||||
parser.add_argument("--num-frames", type=int, default=429, help="Number of generated frames")
|
||||
|
||||
parser.add_argument(
|
||||
"--image-reference",
|
||||
type=_image_reference,
|
||||
default=None,
|
||||
help="First frame for image-conditioned models: a local image path or an http(s) URL.",
|
||||
)
|
||||
parser.add_argument("--negative-prompt", default=None, help="Negative prompt")
|
||||
parser.add_argument("--num-inference-steps", type=int, default=None, help="Number of diffusion steps")
|
||||
parser.add_argument("--guidance-scale", type=float, default=1.0, help="Classifier-free guidance scale")
|
||||
@@ -383,36 +405,16 @@ def parse_args() -> argparse.Namespace:
|
||||
default=None,
|
||||
help="JSON object merged into extra_params; overrides preset keys on conflict.",
|
||||
)
|
||||
# Midway prompt updates are pipeline-specific: a model that does not implement
|
||||
# them (LingBot-World, for one) rejects the whole session on the first one, so
|
||||
# nothing is scheduled unless the caller asks for it.
|
||||
parser.add_argument(
|
||||
"--prompt-updates",
|
||||
type=_parse_prompt_updates,
|
||||
default=json.dumps(
|
||||
[
|
||||
{
|
||||
"at": 4,
|
||||
"prompt": (
|
||||
"An underwater tornado appears and affects the ocean floor in a dramatic and chaotic scene. "
|
||||
"The water is murky, swirling violently, carrying debris and marine life into the vortex. "
|
||||
"The tropical fish on the scene all swim in panic, trying to avoid the powerful currents. "
|
||||
"The camera remains stationary, capturing the intensity of the underwater tornado as it disrupts the serene ocean floor. "
|
||||
"Close-up shot emphasizing the turbulent motion and destruction."
|
||||
),
|
||||
},
|
||||
{
|
||||
"at": 11,
|
||||
"prompt": (
|
||||
"The swirling underwater vortex now seizes a heavy, encrusted treasure chest, its lid flapping open as it is smashed onto the ocean floor. "
|
||||
"Gold coins and silver trinkets spill out, glittering briefly in the murky water before being swept instantly into the violent funnel. "
|
||||
"The heavy wooden box tumbles end over end, colliding with floating rocks and adding to the debris field. "
|
||||
"Swirling sediment and bubbles surround the spilling fortune, highlighting the chaotic power of the storm as it ravages the seabed. "
|
||||
"Close-up shot emphasizing the turbulent motion and destruction."
|
||||
),
|
||||
# "transition_chunks": 3,
|
||||
},
|
||||
]
|
||||
),
|
||||
default=[],
|
||||
help=(
|
||||
"JSON array of scheduled prompt updates. Each object requires "
|
||||
"JSON array of scheduled prompt updates; none are sent by default. "
|
||||
"Each object requires "
|
||||
"'at' (seconds after video.start) and 'prompt'. Optional "
|
||||
f"'transition_chunks' defaults to {DEFAULT_TRANSITION_CHUNKS}."
|
||||
),
|
||||
|
||||
Reference in new issue
Block a user