[Config Refactor]: Remove bagel yaml (#2936)
Signed-off-by: princepride <wangzhipeng628@gmail.com> Signed-off-by: 汪志鹏 <wangzhipeng628@gmail.com> Co-authored-by: SYLAR <125541396+lishunyang12@users.noreply.github.com>
This commit is contained in:
1 parent
22ffb85063
commit
37e5954acf
31 files changed
+1067
-1420
No files matched your search
@@ -588,6 +588,9 @@ class OmniBagelForConditionalGeneration(BagelForConditionalGeneration):
|
||||
existing = self._ropes_metadata.get(rid)
|
||||
if existing and "image_shape" in existing and "image_shape" not in meta:
|
||||
continue
|
||||
ropes = meta.get("ropes")
|
||||
if ropes:
|
||||
meta["ropes"] = [int(r.item()) if isinstance(r, torch.Tensor) else r for r in ropes]
|
||||
self._ropes_metadata[rid] = meta
|
||||
|
||||
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
|
||||
@@ -731,7 +734,7 @@ class OmniBagelForConditionalGeneration(BagelForConditionalGeneration):
|
||||
positions = self._adjust_positions_for_img2img(positions, input_ids)
|
||||
use_mot = True
|
||||
else:
|
||||
rope = int(positions[seq_len - 1].item()) + 1
|
||||
rope = positions[seq_len - 1] + 1
|
||||
self._ropes_pending.append({"ropes": [rope]})
|
||||
|
||||
if use_mot:
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""BAGEL-7B-MoT pipeline topologies (frozen).
|
||||
|
||||
Two-stage (default):
|
||||
Stage 0: Thinker — multimodal understanding + text generation (AR)
|
||||
Stage 1: DiT — diffusion image generation
|
||||
|
||||
Two-stage think:
|
||||
Same as two-stage but the Thinker decodes <think>...</think> tokens before
|
||||
KV transfer. Uses expand_cfg_prompts_think (companion max_tokens=1) and
|
||||
omits kv_transfer_criteria so transfer happens after EOS, not after prefill.
|
||||
|
||||
Single-stage:
|
||||
Stage 0: DiT — self-contained diffusion stage that handles all modalities
|
||||
(text2img, img2img, img2text, text2text, think) internally via its
|
||||
own LLM, ViT, VAE, and tokenizer.
|
||||
"""
|
||||
|
||||
from vllm_omni.config.stage_config import (
|
||||
PipelineConfig,
|
||||
StageExecutionType,
|
||||
StagePipelineConfig,
|
||||
)
|
||||
|
||||
_PROC = "vllm_omni.model_executor.stage_input_processors.bagel"
|
||||
|
||||
BAGEL_PIPELINE = PipelineConfig(
|
||||
model_type="bagel",
|
||||
model_arch="OmniBagelForConditionalGeneration",
|
||||
hf_architectures=("BagelForConditionalGeneration",),
|
||||
stages=(
|
||||
StagePipelineConfig(
|
||||
stage_id=0,
|
||||
model_stage="thinker",
|
||||
execution_type=StageExecutionType.LLM_AR,
|
||||
input_sources=(),
|
||||
final_output=True,
|
||||
final_output_type="text",
|
||||
owns_tokenizer=True,
|
||||
requires_multimodal_data=True,
|
||||
model_arch="OmniBagelForConditionalGeneration",
|
||||
engine_output_type="text",
|
||||
prompt_expand_func=f"{_PROC}.expand_cfg_prompts",
|
||||
omni_kv_config={
|
||||
"need_send_cache": True,
|
||||
"kv_transfer_criteria": {"type": "prefill_finished"},
|
||||
},
|
||||
sampling_constraints={"detokenize": True},
|
||||
),
|
||||
StagePipelineConfig(
|
||||
stage_id=1,
|
||||
model_stage="dit",
|
||||
execution_type=StageExecutionType.DIFFUSION,
|
||||
input_sources=(0,),
|
||||
final_output=True,
|
||||
final_output_type="image",
|
||||
cfg_kv_collect_func=f"{_PROC}.collect_cfg_kv_caches",
|
||||
omni_kv_config={"need_recv_cache": True},
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
BAGEL_THINK_PIPELINE = PipelineConfig(
|
||||
model_type="bagel_think",
|
||||
model_arch="OmniBagelForConditionalGeneration",
|
||||
hf_architectures=(),
|
||||
stages=(
|
||||
StagePipelineConfig(
|
||||
stage_id=0,
|
||||
model_stage="thinker",
|
||||
execution_type=StageExecutionType.LLM_AR,
|
||||
input_sources=(),
|
||||
final_output=True,
|
||||
final_output_type="text",
|
||||
owns_tokenizer=True,
|
||||
requires_multimodal_data=True,
|
||||
model_arch="OmniBagelForConditionalGeneration",
|
||||
engine_output_type="text",
|
||||
prompt_expand_func=f"{_PROC}.expand_cfg_prompts_think",
|
||||
omni_kv_config={"need_send_cache": True},
|
||||
sampling_constraints={"detokenize": True},
|
||||
),
|
||||
StagePipelineConfig(
|
||||
stage_id=1,
|
||||
model_stage="dit",
|
||||
execution_type=StageExecutionType.DIFFUSION,
|
||||
input_sources=(0,),
|
||||
final_output=True,
|
||||
final_output_type="image",
|
||||
cfg_kv_collect_func=f"{_PROC}.collect_cfg_kv_caches",
|
||||
omni_kv_config={"need_recv_cache": True},
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
BAGEL_SINGLE_STAGE_PIPELINE = PipelineConfig(
|
||||
model_type="bagel_single_stage",
|
||||
model_arch="BagelForConditionalGeneration",
|
||||
hf_architectures=(),
|
||||
stages=(
|
||||
StagePipelineConfig(
|
||||
stage_id=0,
|
||||
model_stage="dit",
|
||||
execution_type=StageExecutionType.DIFFUSION,
|
||||
input_sources=(),
|
||||
final_output=True,
|
||||
final_output_type="image",
|
||||
),
|
||||
),
|
||||
)
|
||||
Reference in new issue
Block a user