lishunyang
69bcc6416d
Adds Lance (bytedance-research/Lance), a 3B unified AR+diffusion multimodal model on a Qwen2.5-VL backbone. Lance is BAGEL-lineage: it reuses vLLM-Omni's BAGEL Mixture-of-Transformers core verbatim and specializes only the ViT (Qwen2.5-VL vision), the VAE (Wan2.2) and the checkpoint layout. All six single-stage Lance modalities verified end-to-end on 1x B300: t2i, t2v, image_edit, video_edit, x2t_image (image understanding) and x2t_video (video understanding). 3-D mRoPE is wired through BagelRotaryEmbedding for the x2t / video / edit paths. Includes the upstream gap-analysis quality fixes (task-specific system prompts, full Qwen chat template, upstream inference defaults and aspect-ratio buckets). Ships an offline example (examples/offline_inference/lance/), an OpenAI-compatible online-serving example (examples/online_serving/lance/) and an e2e test (tests/e2e/online_serving/test_lance.py). See LANCE_PR.md for the architecture rationale, per-modality status and the two remaining follow-ups (two-stage AR+DiT, video_edit polish). Closes #3697. Signed-off-by: lishunyang <lishunyang03@gmail.com>
107 lines
4.1 KiB
Python
107 lines
4.1 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""Lance chat / system prompts.
|
|
|
|
Matches the upstream Lance training distribution (see
|
|
``bytedance/Lance/data/system_prompt_render.py`` and
|
|
``data/common.py::generate_system_prompt``):
|
|
|
|
* each task has a specific system prompt describing what to attend to;
|
|
* the prompt is wrapped in the Qwen chat template
|
|
``<|im_start|>system\\n…<|im_end|>\\n<|im_start|>user\\n…<|im_end|>\\n``
|
|
``<|im_start|>assistant\\n``;
|
|
* vision tokens are framed by ``<|vision_start|><|video_pad|><|vision_end|>``
|
|
(upstream uses ``<|video_pad|>`` even for image type by default).
|
|
|
|
These are pure string formatting helpers — the runtime pipeline still does
|
|
the actual VAE / ViT prefill separately.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
# Task-specific system prompts (verbatim from upstream's
|
|
# ``generate_system_prompt`` — first variant of each pool).
|
|
SYSTEM_PROMPTS: dict[tuple[str, str], str] = {
|
|
# (task, vision_type) -> system prompt
|
|
("t2i", "image"): (
|
|
"Describe the image by detailing the color, quantity, text, shape, "
|
|
"size, texture, spatial relationships of the objects and background:"
|
|
),
|
|
("t2v", "video"): (
|
|
"Describe the video by detailing the color, quantity, visible text, "
|
|
"shape, size, texture, spatial relationships and motion/camera "
|
|
"movements of the objects and background:"
|
|
),
|
|
("image_edit", "image"): (
|
|
"Describe the key features of the input image (color, shape, size, "
|
|
"texture, objects, background), then explain how the user's text "
|
|
"instruction should alter or modify the image. Generate a new image "
|
|
"that meets the user's requirements while maintaining consistency "
|
|
"with the original input where appropriate."
|
|
),
|
|
("video_edit", "video"): (
|
|
"Describe the key features of the input video (color, shape, size, "
|
|
"texture, objects, background), then explain how the user's text "
|
|
"instruction should alter or modify the video. Generate a new video "
|
|
"that meets the user's requirements while maintaining consistency "
|
|
"with the original input where appropriate."
|
|
),
|
|
("x2t_image", "image"): (
|
|
"Generate a detailed and accurate description of the image, including all the key moments and visual details."
|
|
),
|
|
("x2t_video", "video"): (
|
|
"Generate a detailed and accurate description of the video, including all the key moments and visual details."
|
|
),
|
|
}
|
|
|
|
# Vision-content placeholders (Qwen2.5-VL convention). Upstream uses
|
|
# ``<|video_pad|>`` for image inputs too unless ``force_video_pad=False``
|
|
# (counter-intuitively named; see ``render_qwenvl_prompt`` upstream).
|
|
VISION_START = "<|vision_start|>"
|
|
VISION_END = "<|vision_end|>"
|
|
VIDEO_PAD = "<|video_pad|>"
|
|
IMAGE_PAD = "<|image_pad|>"
|
|
|
|
|
|
def render_lance_prompt(
|
|
task: str,
|
|
user_text: str,
|
|
*,
|
|
vision_token: str | None = None,
|
|
) -> str:
|
|
"""Render a Lance-compatible user-side prompt.
|
|
|
|
``task`` is one of ``t2i``, ``t2v``, ``image_edit``, ``video_edit``,
|
|
``x2t_image``, ``x2t_video``. ``vision_token`` should be the visual
|
|
placeholder to embed inside the user message (e.g. ``VIDEO_PAD`` or a
|
|
``VISION_START + VIDEO_PAD + VISION_END`` block); pass ``None`` for
|
|
text-only inputs.
|
|
|
|
Returns a single string ready to be tokenized; no further wrapping is
|
|
needed by the caller.
|
|
"""
|
|
# ``t2v`` / ``x2t_video`` / ``video_edit`` use video system prompts;
|
|
# the rest are image.
|
|
vision_type = "video" if task in {"t2v", "x2t_video", "video_edit"} else "image"
|
|
sys_prompt = SYSTEM_PROMPTS.get((task, vision_type), "You are a helpful assistant.")
|
|
|
|
if vision_token is None:
|
|
user_msg = user_text
|
|
else:
|
|
# Upstream concatenates the vision block immediately before the text.
|
|
user_msg = f"{vision_token}{user_text}"
|
|
|
|
return (
|
|
f"<|im_start|>system\n{sys_prompt}<|im_end|>\n<|im_start|>user\n{user_msg}<|im_end|>\n<|im_start|>assistant\n"
|
|
)
|
|
|
|
|
|
__all__ = [
|
|
"IMAGE_PAD",
|
|
"SYSTEM_PROMPTS",
|
|
"VIDEO_PAD",
|
|
"VISION_END",
|
|
"VISION_START",
|
|
"render_lance_prompt",
|
|
]
|