[Feature]: Support cfg kv-cache transfer in multi-stage (#1422)

Signed-off-by: princepride <wangzhipeng628@gmail.com>
Signed-off-by: 汪志鹏 <wangzhipeng628@gmail.com>
Co-authored-by: Cursor <cursoragent@cursor.com>

docs/configuration/stage_configs.md

@@ -135,6 +135,14 @@ Each stage in the `stage_args` list contains the following configuration options
 
 A unique identifier for each stage in the multi-stage pipeline. Stages are numbered sequentially starting from 0, and this ID is used to reference stages in inter-stage dependencies (e.g., `engine_input_source`).
 
+### `prompt_expand_func` (Optional)
+
+A custom Python function hook for the LLM stage (Stage 0) that expands a single incoming prompt object into multiple prompts. This is primarily used for multi-modal Classifier-Free Guidance (CFG), where it generates the necessary companion requests (like a negative text prompt) and tags them with internal roles (e.g., `cfg_text`). This ensures the upstream LLM generates the needed contextual hidden states for both the conditional and unconditional generations simultaneously.
+
+### `cfg_kv_collect_func` (Optional)
+
+A custom Python function hook for downstream diffusion stages (Stage 1+) to collect, map, and process the KV caches transferred from the companion requests fired by `prompt_expand_func`. It aggregates the hidden condition states cleanly (e.g., binding them as `cfg_text_past_key_values` and `cfg_text_kv_metadata`), allowing the diffusion runtime to perform CFG smoothly without redundantly evaluating text paths on the DiT workers.
+
 ### `runtime`
 
 Configuration for disaggregated execution of the stage, controlling how the stage is deployed and executed.

docs/design/architecture_overview.md

@@ -92,6 +92,13 @@ The framework achieves high performance through several optimization techniques:
     * **Quantization:** Supports various quantization implementations including FP8 and AWQ.
     * **FusedOps:** Allows for custom and third-party integration.
 
+### Classifier-Free Guidance (CFG) Companion Flow
+
+vLLM-Omni natively models Classifier-Free Guidance (CFG) across disaggregated multi-stage setups via a "companion request" paradigm, eliminating redundant textual/multimodal context computation boundaries:
+1. **Prompt Expansion:** In the initial autoregressive (AR) stage, a customized `prompt_expand_func` hook intercepts incoming generation prompts and pairs them directly with negative companion prompts (e.g., a default negative prompt) on the fly, tagging the secondary prompt with a specific internal role (`cfg_text`).
+2. **Synchronized KV Cache Transfer:** The AR stage evaluates both the primary and companion sequence batches concurrently. The `OmniConnector` captures these specific structural dependencies and reliably passes the positive and negative outcome KV caches seamlessly across stage boundaries via shared memory or network protocols.
+3. **KV Cache Collection & Injection:** Upon reaching the downstream Diffusion (DiT) Engine, an assigned `cfg_kv_collect_func` automatically intercepts the mapped companion caches (`cfg_text_past_key_values`). These auxiliary dependencies are natively gathered and seamlessly bound to the primary generation sequence variables, enabling the DiT Engine to cleanly implement cross-attention CFG guidance over accurate conditioning and unconditioning structures in parallel.
+
 ### Flexibility and Usability
 
 vLLM-Omni is designed to be flexible and straightforward for users:

examples/offline_inference/bagel/end2end.py

@@ -49,7 +49,7 @@ def parse_args():
     parser.add_argument("--cfg-text-scale", type=float, default=4.0, help="Text CFG scale (default: 4.0)")
     parser.add_argument("--cfg-img-scale", type=float, default=1.5, help="Image CFG scale (default: 1.5)")
     parser.add_argument(
-        "--negative-prompt", type=str, default=None, help="Negative prompt (not yet supported, reserved for future)"
+        "--negative-prompt", type=str, default=None, help="Negative prompt for CFG (default: empty prompt)"
     )
 
     args = parser.parse_args()
@@ -162,6 +162,8 @@ def main():
                 # text2img
                 final_prompt_text = f"<|im_start|>{p}<|im_end|>"
                 prompt_dict = {"prompt": final_prompt_text, "modalities": ["image"]}
+                if args.negative_prompt is not None:
+                    prompt_dict["negative_prompt"] = args.negative_prompt
                 formatted_prompts.append(prompt_dict)
 
         params_list = omni.default_sampling_params_list
@@ -170,10 +172,13 @@ def main():
             if len(params_list) > 1:
                 diffusion_params = params_list[1]
                 diffusion_params.num_inference_steps = args.steps  # type: ignore
-                diffusion_params.extra_args = {  # type: ignore
+                extra = {
                     "cfg_text_scale": args.cfg_text_scale,
                     "cfg_img_scale": args.cfg_img_scale,
                 }
+                if args.negative_prompt is not None:
+                    extra["negative_prompt"] = args.negative_prompt
+                diffusion_params.extra_args = extra  # type: ignore
 
         omni_outputs = list(omni.generate(prompts=formatted_prompts, sampling_params_list=params_list))
 

tests/diffusion/test_diffusion_model_runner.py

@@ -56,7 +56,10 @@ def _make_runner(cache_backend, cache_backend_name: str, enable_cache_dit_summar
         enable_cache_dit_summary=enable_cache_dit_summary,
         parallel_config=SimpleNamespace(use_hsdp=False),
     )
-    runner.kv_transfer_manager = SimpleNamespace(receive_kv_cache=lambda req, target_device: None)
+    runner.kv_transfer_manager = SimpleNamespace(
+        receive_kv_cache=lambda req, target_device=None: None,
+        receive_multi_kv_cache=lambda req, cfg_kv_collect_func=None, target_device=None: None,
+    )
     return runner
 
 

tests/e2e/offline_inference/stage_configs/bagel_mooncake_ci.yaml

@@ -4,6 +4,7 @@
 stage_args:
   - stage_id: 0
     stage_type: llm
+    prompt_expand_func: vllm_omni.model_executor.stage_input_processors.bagel.expand_cfg_prompts
     runtime:
       devices: "0"
       max_batch_size: 1
@@ -39,6 +40,7 @@ stage_args:
       to_stage_1: mooncake_connector
   - stage_id: 1
     stage_type: diffusion
+    cfg_kv_collect_func: vllm_omni.model_executor.stage_input_processors.bagel.collect_cfg_kv_caches
     runtime:
       devices: "0"
       max_batch_size: 1

tests/e2e/offline_inference/stage_configs/bagel_sharedmemory_ci.yaml

@@ -4,6 +4,7 @@
 stage_args:
   - stage_id: 0
     stage_type: llm
+    prompt_expand_func: vllm_omni.model_executor.stage_input_processors.bagel.expand_cfg_prompts
     runtime:
       devices: "0"
       max_batch_size: 1
@@ -38,6 +39,7 @@ stage_args:
 
   - stage_id: 1
     stage_type: diffusion
+    cfg_kv_collect_func: vllm_omni.model_executor.stage_input_processors.bagel.collect_cfg_kv_caches
     runtime:
       devices: "0"
       max_batch_size: 1

tests/e2e/offline_inference/test_bagel_text2img.py

@@ -37,16 +37,16 @@ from vllm_omni.entrypoints.omni import Omni
 # "Generated with seed=52, num_inference_steps=15,
 # prompt='A futuristic city skyline at twilight, cyberpunk style'"
 REFERENCE_PIXELS = [
-    {"position": (100, 100), "rgb": (68, 107, 134)},
-    {"position": (400, 50), "rgb": (95, 139, 166)},
-    {"position": (700, 100), "rgb": (99, 122, 151)},
-    {"position": (150, 400), "rgb": (111, 125, 153)},
-    {"position": (512, 512), "rgb": (97, 107, 131)},
-    {"position": (700, 400), "rgb": (48, 64, 98)},
-    {"position": (100, 700), "rgb": (79, 63, 84)},
-    {"position": (400, 700), "rgb": (40, 58, 79)},
-    {"position": (700, 700), "rgb": (60, 75, 103)},
-    {"position": (256, 256), "rgb": (97, 128, 156)},
+    {"position": (100, 100), "rgb": (49, 96, 134)},
+    {"position": (400, 50), "rgb": (63, 127, 167)},
+    {"position": (700, 100), "rgb": (70, 101, 141)},
+    {"position": (150, 400), "rgb": (115, 90, 150)},
+    {"position": (512, 512), "rgb": (98, 86, 119)},
+    {"position": (700, 400), "rgb": (29, 42, 91)},
+    {"position": (100, 700), "rgb": (47, 50, 88)},
+    {"position": (400, 700), "rgb": (36, 52, 91)},
+    {"position": (700, 700), "rgb": (45, 58, 99)},
+    {"position": (256, 256), "rgb": (62, 94, 135)},
 ]
 
 # Maximum allowed difference per color channel
@@ -80,6 +80,10 @@ def _configure_sampling_params(omni: Omni, max_tokens: int = 1, num_inference_st
     params_list[0].max_tokens = max_tokens  # type: ignore
     if len(params_list) > 1:
         params_list[1].num_inference_steps = num_inference_steps  # type: ignore
+        params_list[1].extra_args = {  # type: ignore
+            "cfg_text_scale": 4.0,
+            "cfg_img_scale": 1.5,
+        }
     return params_list
 
 

tests/entrypoints/test_cfg_companion_tracker.py

@@ -0,0 +1,114 @@
+import time
+from types import SimpleNamespace
+
+import pytest
+
+from vllm_omni.entrypoints.cfg_companion_tracker import CfgCompanionTracker
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
+
+def dummy_expand_func(prompt, sp0):
+    if prompt == "expand_me":
+        return [SimpleNamespace(prompt={"prompt": "neg"}, role="cfg_text", request_id_suffix="__cfg_text")]
+    return []
+
+
+@pytest.fixture
+def tracker():
+    sp0 = SimpleNamespace()
+    return CfgCompanionTracker(prompt_expand_func=dummy_expand_func, stage0_sampling_params=sp0, timeout_s=0.1)
+
+
+def test_companion_tracker_initialization(tracker):
+    assert not tracker.is_active
+    assert tracker.num_companions == 0
+
+
+def test_expand_prompts_registers_companions(tracker):
+    request_id_to_prompt = {"req1": "expand_me", "req2": "do_not_expand"}
+
+    pairs = tracker.expand_prompts(request_id_to_prompt)
+
+    assert len(pairs) == 1
+    companion_id, prompt = pairs[0]
+    assert companion_id == "req1__cfg_text"
+    assert prompt == {"prompt": "neg"}
+
+    assert tracker.is_active
+    assert tracker.num_companions == 1
+    assert tracker.is_companion("req1__cfg_text")
+    assert not tracker.is_companion("req2__cfg_text")
+    assert tracker.has_companions("req1")
+    assert not tracker.has_companions("req2")
+
+    comp_map = tracker.get_companion_request_ids("req1")
+    assert comp_map == {"cfg_text": "req1__cfg_text"}
+
+
+def test_companion_lifecycle_success(tracker):
+    request_id_to_prompt = {"req1": "expand_me"}
+    tracker.expand_prompts(request_id_to_prompt)
+
+    # Defer parent
+    engine_outputs = {"out": 123}
+    tracker.defer_parent("req1", engine_outputs, stage_id=0)
+
+    # Initially not done
+    assert not tracker.all_companions_done("req1")
+
+    # Companion completes
+    parent_id = tracker.on_companion_completed("req1__cfg_text")
+
+    # Parent should be returned since all companions are done and it is pending
+    assert parent_id == "req1"
+    assert tracker.all_companions_done("req1")
+
+    # Pop pending parent
+    popped = tracker.pop_pending_parent("req1")
+    assert popped is not None
+    assert popped["engine_outputs"] == engine_outputs
+    assert popped["stage_id"] == 0
+
+
+def test_companion_lifecycle_failure(tracker):
+    request_id_to_prompt = {"req1": "expand_me"}
+    tracker.expand_prompts(request_id_to_prompt)
+
+    tracker.defer_parent("req1", {"out": 123}, stage_id=0)
+
+    # Companion fails
+    parent_id, aborted = tracker.on_companion_error("req1__cfg_text")
+
+    assert parent_id == "req1"
+    assert aborted is True
+    assert tracker.is_parent_failed("req1")
+
+    # Parent should be removed from pending list
+    assert tracker.pop_pending_parent("req1") is None
+
+    # Consume failure
+    tracker.consume_parent_failure("req1")
+    assert not tracker.is_parent_failed("req1")
+
+
+def test_companion_lifecycle_timeout(tracker):
+    request_id_to_prompt = {"req1": "expand_me"}
+    tracker.expand_prompts(request_id_to_prompt)
+
+    tracker.defer_parent("req1", {"out": 123}, stage_id=0)
+
+    # Initially no timeouts
+    timeouts = tracker.check_timeouts()
+    assert len(timeouts) == 0
+
+    # Wait for timeout
+    time.sleep(0.15)
+
+    # Check timeouts again
+    timeouts = tracker.check_timeouts()
+    assert len(timeouts) == 1
+    assert timeouts[0] == "req1"
+
+    # Should be removed from pending
+    assert tracker.pop_pending_parent("req1") is None

vllm_omni/diffusion/data.py

@@ -442,6 +442,9 @@ class OmniDiffusionConfig:
     # Omni configuration (injected from stage config)
     omni_kv_config: dict[str, Any] = field(default_factory=dict)
 
+    # Model-specific function for collecting CFG KV caches (set at runtime)
+    cfg_kv_collect_func: Any | None = None
+
     # Quantization settings
     # Supported methods: "fp8" (FP8 W8A8 on Ada/Hopper, weight-only on older GPUs)
     quantization: str | None = None

vllm_omni/diffusion/models/bagel/pipeline_bagel.py

@@ -327,16 +327,40 @@ class BagelPipeline(nn.Module):
             gen_context["past_key_values"] = injected_kv
             seq_len = injected_kv.key_cache[0].shape[0]
             gen_context["kv_lens"] = [seq_len]
-            gen_context["ropes"] = [seq_len]
+            if req.sampling_params.kv_metadata and "ropes" in req.sampling_params.kv_metadata:
+                gen_context["ropes"] = req.sampling_params.kv_metadata["ropes"]
+            else:
+                gen_context["ropes"] = [seq_len]
 
-            # Disable CFG: single KV cache cannot support 3-branch CFG
-            logger.warning("CFG is disabled when using injected KV Cache")
-            gen_params = BagelGenParams(
-                num_timesteps=gen_params.num_timesteps,
-                timestep_shift=gen_params.timestep_shift,
-                cfg_text_scale=1.0,
-                cfg_img_scale=1.0,
-            )
+            cfg_text_kv = getattr(req.sampling_params, "cfg_text_past_key_values", None)
+            if cfg_text_kv is not None:
+                logger.info("CFG enabled with multi-KV: using injected cfg_text KV Cache")
+                cfg_text_seq_len = cfg_text_kv.key_cache[0].shape[0]
+                cfg_text_context["past_key_values"] = cfg_text_kv
+                cfg_text_context["kv_lens"] = [cfg_text_seq_len]
+                cfg_text_metadata = getattr(req.sampling_params, "cfg_text_kv_metadata", None)
+                if cfg_text_metadata and "ropes" in cfg_text_metadata:
+                    cfg_text_context["ropes"] = cfg_text_metadata["ropes"]
+                else:
+                    cfg_text_context["ropes"] = [cfg_text_seq_len]
+
+                cfg_img_kv = getattr(req.sampling_params, "cfg_img_past_key_values", None) or injected_kv
+                cfg_img_seq_len = cfg_img_kv.key_cache[0].shape[0]
+                cfg_img_context["past_key_values"] = cfg_img_kv
+                cfg_img_context["kv_lens"] = [cfg_img_seq_len]
+                cfg_img_metadata = getattr(req.sampling_params, "cfg_img_kv_metadata", None)
+                if cfg_img_metadata and "ropes" in cfg_img_metadata:
+                    cfg_img_context["ropes"] = cfg_img_metadata["ropes"]
+                else:
+                    cfg_img_context["ropes"] = [cfg_img_seq_len]
+            else:
+                logger.warning("CFG is disabled: only single KV cache available")
+                gen_params = BagelGenParams(
+                    num_timesteps=gen_params.num_timesteps,
+                    timestep_shift=gen_params.timestep_shift,
+                    cfg_text_scale=1.0,
+                    cfg_img_scale=1.0,
+                )
 
         else:
             image_input = (

vllm_omni/diffusion/worker/diffusion_model_runner.py

@@ -191,7 +191,11 @@ class DiffusionModelRunner:
         grad_context = torch.no_grad() if use_hsdp else torch.inference_mode()
         with grad_context:
             # The manager handles the check for need_recv_cache internally
-            self.kv_transfer_manager.receive_kv_cache(req, target_device=getattr(self.pipeline, "device", None))
+            self.kv_transfer_manager.receive_multi_kv_cache(
+                req,
+                cfg_kv_collect_func=getattr(self.od_config, "cfg_kv_collect_func", None),
+                target_device=getattr(self.pipeline, "device", None),
+            )
 
             if req.sampling_params.generator is None and req.sampling_params.seed is not None:
                 if req.sampling_params.generator_device is not None:

vllm_omni/distributed/omni_connectors/kv_transfer_manager.py

@@ -522,3 +522,50 @@ class OmniKVTransferManager:
             self.apply_kv_cache_to_request(req, data)
             return True
         return False
+
+    def receive_multi_kv_cache(
+        self,
+        req: Any,
+        cfg_kv_collect_func: Callable | None = None,
+        target_device: torch.device | None = None,
+    ) -> bool:
+        """Receive primary KV cache and optional CFG companion KV caches.
+
+        First receives the primary KV cache (existing logic). Then, if the
+        request carries cfg_kv_request_ids and a model-specific
+        cfg_kv_collect_func is provided, calls it to fetch and attach the
+        companion KV caches to sampling_params.
+
+        Args:
+            req: Request object with request_id and sampling_params.
+            cfg_kv_collect_func: Model-specific function for collecting
+                CFG KV caches. Signature:
+                (request_id, cfg_request_ids, kv_transfer_manager, target_device)
+                -> dict[str, Any]
+            target_device: Device to move tensors to.
+
+        Returns:
+            True if primary KV cache was received successfully.
+        """
+        primary_ok = self.receive_kv_cache(req, target_device)
+
+        cfg_ids = getattr(getattr(req, "sampling_params", None), "cfg_kv_request_ids", None)
+        if cfg_ids and cfg_kv_collect_func:
+            request_id = getattr(req, "request_id", None) or (
+                req.request_ids[0] if hasattr(req, "request_ids") and req.request_ids else None
+            )
+            try:
+                cfg_kvs = cfg_kv_collect_func(
+                    request_id,
+                    cfg_ids,
+                    self,
+                    target_device,
+                )
+                if cfg_kvs and hasattr(req, "sampling_params") and req.sampling_params is not None:
+                    for key, value in cfg_kvs.items():
+                        setattr(req.sampling_params, key, value)
+                    logger.info("Applied CFG KV caches: %s", list(cfg_kvs.keys()))
+            except Exception:
+                logger.exception("Failed to collect CFG KV caches for %s", request_id)
+
+        return primary_ok

vllm_omni/entrypoints/async_omni_diffusion.py

@@ -60,6 +60,7 @@ class AsyncOmniDiffusion:
         # Capture stage info from kwargs before they might be filtered out
         stage_id = kwargs.get("stage_id")
         engine_input_source = kwargs.get("engine_input_source")
+        cfg_kv_collect_func = kwargs.pop("cfg_kv_collect_func", None)
 
         # Build config
         if od_config is None:
@@ -115,6 +116,9 @@ class AsyncOmniDiffusion:
             else:
                 raise
 
+        if cfg_kv_collect_func is not None:
+            od_config.cfg_kv_collect_func = cfg_kv_collect_func
+
         # Initialize engine
         self.engine: DiffusionEngine = DiffusionEngine.make_engine(od_config)
 

vllm_omni/entrypoints/cfg_companion_tracker.py

@@ -0,0 +1,233 @@
+"""CFG companion request tracker for the Omni orchestrator.
+
+Encapsulates all bookkeeping for Classifier-Free Guidance companion
+requests (prompt expansion, parent/companion ID mapping, completion
+tracking, deferred forwarding, failure propagation, and timeouts)
+so that ``Omni._run_generation`` stays clean.
+"""
+
+from __future__ import annotations
+
+import copy
+import logging
+import os
+import time
+from collections.abc import Callable, Sequence
+from typing import Any
+
+from vllm_omni.distributed.omni_connectors.adapter import try_send_via_connector
+from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniSamplingParams
+
+logger = logging.getLogger(__name__)
+
+
+class CfgCompanionTracker:
+    """Manages CFG companion request lifecycle in the orchestrator scheduling loop."""
+
+    def __init__(
+        self,
+        prompt_expand_func: Callable[..., Any] | None,
+        stage0_sampling_params: Any,
+        timeout_s: float | None = None,
+    ) -> None:
+        self._expand_func = prompt_expand_func
+        self._sp0 = stage0_sampling_params
+        self._timeout_s = (
+            timeout_s if timeout_s is not None else float(os.environ.get("VLLM_CFG_PENDING_TIMEOUT_S", "120"))
+        )
+
+        self._companion_map: dict[str, dict[str, str]] = {}  # parent -> {role: companion_id}
+        self._companion_ids: set[str] = set()
+        self._companion_to_parent: dict[str, str] = {}  # companion -> parent
+        self._done: dict[str, set[str]] = {}  # parent -> completed companion ids
+        self._pending_parents: dict[str, dict[str, Any]] = {}  # parent -> deferred result
+        self._failed_parents: set[str] = set()
+
+    @property
+    def is_active(self) -> bool:
+        return bool(self._companion_ids)
+
+    @property
+    def num_companions(self) -> int:
+        return len(self._companion_ids)
+
+    @property
+    def stage0_sampling_params(self) -> Any:
+        return self._sp0
+
+    def expand_prompts(
+        self,
+        request_id_to_prompt: dict[str, Any],
+    ) -> list[tuple[str, Any]]:
+        """Expand user prompts into ``(companion_id, prompt)`` pairs via model-specific func."""
+        if not self._expand_func:
+            return []
+
+        pairs: list[tuple[str, Any]] = []
+        for rid, prompt in request_id_to_prompt.items():
+            expanded = self._expand_func(prompt, self._sp0)
+            if not expanded:
+                continue
+            role_map: dict[str, str] = {}
+            for ep in expanded:
+                cid = f"{rid}{ep.request_id_suffix}"
+                role_map[ep.role] = cid
+                self._companion_ids.add(cid)
+                self._companion_to_parent[cid] = rid
+                pairs.append((cid, ep.prompt))
+            self._companion_map[rid] = role_map
+            self._done[rid] = set()
+
+        logger.debug(
+            "CFG expansion: %d parent(s) -> %d companion(s)",
+            len(self._companion_map),
+            len(self._companion_ids),
+        )
+        return pairs
+
+    def is_companion(self, req_id: str) -> bool:
+        return req_id in self._companion_ids
+
+    def has_companions(self, parent_id: str) -> bool:
+        return parent_id in self._companion_map
+
+    def all_companions_done(self, parent_id: str) -> bool:
+        role_map = self._companion_map.get(parent_id, {})
+        done_set = self._done.get(parent_id, set())
+        return all(cid in done_set for cid in role_map.values())
+
+    def get_companion_request_ids(self, parent_id: str) -> dict[str, str]:
+        """Return ``{role: companion_request_id}`` for a parent."""
+        return self._companion_map.get(parent_id, {})
+
+    def is_parent_failed(self, parent_id: str) -> bool:
+        return parent_id in self._failed_parents
+
+    # -- Lifecycle events --
+
+    def on_companion_error(self, companion_id: str) -> tuple[str | None, bool]:
+        """Record failure. Returns ``(parent_id, parent_was_aborted)``."""
+        parent_id = self._companion_to_parent.get(companion_id)
+        if parent_id is None:
+            return None, False
+        self._failed_parents.add(parent_id)
+        logger.error("CFG companion %s failed; marking parent %s as failed", companion_id, parent_id)
+        aborted = parent_id in self._pending_parents
+        if aborted:
+            self._pending_parents.pop(parent_id, None)
+        return parent_id, aborted
+
+    def on_companion_completed(self, companion_id: str) -> str | None:
+        """Mark done. Returns parent_id only if parent is pending and all companions finished."""
+        parent_id = self._companion_to_parent.get(companion_id)
+        if parent_id is None:
+            return None
+        self._done[parent_id].add(companion_id)
+        logger.debug("CFG companion %s completed (parent=%s)", companion_id, parent_id)
+        if parent_id in self._pending_parents and self.all_companions_done(parent_id):
+            return parent_id
+        return None
+
+    def consume_parent_failure(self, parent_id: str) -> None:
+        self._failed_parents.discard(parent_id)
+
+    # -- Deferred parent management --
+
+    def defer_parent(self, parent_id: str, engine_outputs: Any, stage_id: int) -> None:
+        """Hold parent result while waiting for companions to finish."""
+        self._pending_parents[parent_id] = {
+            "engine_outputs": engine_outputs,
+            "stage_id": stage_id,
+            "pending_since": time.time(),
+        }
+        logger.debug("Parent %s deferred, waiting for CFG companions", parent_id)
+
+    def pop_pending_parent(self, parent_id: str) -> dict[str, Any] | None:
+        return self._pending_parents.pop(parent_id, None)
+
+    def check_timeouts(self) -> list[str]:
+        """Return and remove parent IDs that exceeded the pending timeout."""
+        if not self._pending_parents:
+            return []
+        now = time.time()
+        timed_out: list[str] = []
+        for pid in list(self._pending_parents):
+            pending_since = self._pending_parents[pid].get("pending_since", now)
+            if now - pending_since > self._timeout_s:
+                self._pending_parents.pop(pid)
+                self._failed_parents.discard(pid)
+                timed_out.append(pid)
+                logger.error("Parent %s timed out waiting for CFG companions (>%.0fs)", pid, self._timeout_s)
+        return timed_out
+
+    # -- Forward parent with CFG KV --
+
+    def forward_parent_with_cfg(
+        self,
+        req_id: str,
+        parent_result: dict[str, Any],
+        stage_list: Sequence[Any],
+        connectors: dict[tuple[str, str], Any],
+        sampling_params_list: Sequence[OmniSamplingParams],
+        request_id_to_prompt: dict[str, Any],
+        final_stage_id_to_prompt: dict[str, int],
+        metrics: Any,
+        remaining_by_stage: list[int],
+    ) -> bool:
+        """Forward a parent request to the next stage with CFG KV request IDs attached."""
+        stage_id = parent_result["stage_id"]
+        next_stage_id = stage_id + 1
+        if next_stage_id > final_stage_id_to_prompt.get(req_id, 0):
+            return True
+
+        next_stage = stage_list[next_stage_id]
+        try:
+            with metrics.stage_postprocess_timer(stage_id, req_id):
+                next_inputs = next_stage.process_engine_inputs(
+                    stage_list,
+                    [request_id_to_prompt[req_id]],
+                    source_outputs_override=parent_result["engine_outputs"],
+                )
+        except Exception as e:
+            logger.exception(
+                "Process engine inputs error for req %s at stage %d: %s",
+                req_id,
+                next_stage_id,
+                e,
+            )
+            return False
+
+        sp_next = copy.deepcopy(sampling_params_list[next_stage_id])
+        if isinstance(sp_next, OmniDiffusionSamplingParams):
+            sp_next.cfg_kv_request_ids = self.get_companion_request_ids(req_id)
+            logger.info(
+                "Attaching cfg_kv_request_ids=%s to request %s",
+                sp_next.cfg_kv_request_ids,
+                req_id,
+            )
+
+        connector_key = (str(stage_id), str(next_stage_id))
+        connector = connectors.get(connector_key)
+        sent_via_connector = False
+        if connector:
+            sent_via_connector = try_send_via_connector(
+                connector=connector,
+                stage_id=stage_id,
+                next_stage_id=next_stage_id,
+                req_id=req_id,
+                next_inputs=next_inputs,
+                sampling_params=sp_next,
+                original_prompt=request_id_to_prompt[req_id],
+                next_stage_queue_submit_fn=stage_list[next_stage_id].submit,
+                metrics=metrics,
+            )
+
+        if not sent_via_connector:
+            raise RuntimeError(
+                f"Failed to send CFG request {req_id} to stage-{next_stage_id} via connector. "
+                "Configure a connector for this edge or inspect connector logs for details."
+            )
+
+        logger.debug("Forwarded CFG-enabled request %s to stage-%d", req_id, next_stage_id)
+        remaining_by_stage[next_stage_id] += 1
+        return True

vllm_omni/entrypoints/omni.py

@@ -34,6 +34,7 @@ from vllm_omni.distributed.ray_utils.utils import (
     get_ray_queue_class,
     try_close_ray,
 )
+from vllm_omni.entrypoints.cfg_companion_tracker import CfgCompanionTracker
 from vllm_omni.entrypoints.omni_stage import OmniStage
 from vllm_omni.entrypoints.stage_utils import SHUTDOWN_TASK, OmniStageTaskType
 from vllm_omni.entrypoints.stage_utils import maybe_load_from_ipc as _load
@@ -935,6 +936,13 @@ class Omni(OmniBase):
         _req_start_ts: dict[str, float] = {}
         _wall_start_ts: float = time.time()
 
+        # CFG companion tracking (prompt expansion + lifecycle management)
+        cfg = CfgCompanionTracker(
+            prompt_expand_func=getattr(self.stage_list[0], "prompt_expand_func", None),
+            stage0_sampling_params=sampling_params_list[0],
+        )
+        expanded_companions = cfg.expand_prompts(request_id_to_prompt)
+
         # Determine the final stage for E2E stats (highest stage_id with final_output=True; fallback to last stage)
         final_stage_id_to_prompt: dict[str, int] = {}
         for rid, prompt in request_id_to_prompt.items():
@@ -976,6 +984,18 @@ class Omni(OmniBase):
             _req_start_ts[req_id] = time.time()
             logger.debug(f"[{self._name}] Enqueued request {req_id} to stage-0")
 
+        # Submit CFG companion requests to stage-0
+        if cfg.is_active:
+            for companion_id, companion_prompt in expanded_companions:
+                task = {
+                    "request_id": companion_id,
+                    "engine_inputs": companion_prompt,
+                    "sampling_params": cfg.stage0_sampling_params,
+                }
+                self.stage_list[0].submit(task)
+                _req_start_ts[companion_id] = time.time()
+                logger.debug(f"[{self._name}] Enqueued CFG companion {companion_id} to stage-0")
+
         pbar = None
         if use_tqdm:
             tqdm_func = use_tqdm if callable(use_tqdm) else tqdm
@@ -987,7 +1007,7 @@ class Omni(OmniBase):
             )
         # For each stage, forward results to next stage; collect finals at the end
         # We pipeline by continually polling output queues in stage order
-        remaining_by_stage: list[int] = [len(request_prompts)] + [0] * (num_stages - 1)
+        remaining_by_stage: list[int] = [len(request_prompts) + cfg.num_companions] + [0] * (num_stages - 1)
         completed_requests = 0
         total_requests = len(request_prompts)
 
@@ -1007,6 +1027,14 @@ class Omni(OmniBase):
                     logger.error(
                         f"[{self._name}] Stage {stage_id} error on request {req_id}: {result['error']}",
                     )
+                    if cfg.is_companion(req_id) and stage_id == 0:
+                        parent_id, parent_aborted = cfg.on_companion_error(req_id)
+                        if parent_aborted:
+                            completed_requests += 1
+                            logger.error(
+                                f"[{self._name}] Parent {parent_id} aborted due to "
+                                f"companion failure ({completed_requests}/{total_requests})",
+                            )
                     continue
 
                 if result.get("type") == "stage_ready":
@@ -1015,6 +1043,30 @@ class Omni(OmniBase):
                     time.sleep(0.05)
                     continue
 
+                # CFG: companion requests only run through Stage-0
+                if cfg.is_companion(req_id) and stage_id == 0:
+                    ready_parent = cfg.on_companion_completed(req_id)
+                    if ready_parent is not None:
+                        success = cfg.forward_parent_with_cfg(
+                            ready_parent,
+                            cfg.pop_pending_parent(ready_parent),
+                            self.stage_list,
+                            self.connectors,
+                            sampling_params_list,
+                            request_id_to_prompt,
+                            final_stage_id_to_prompt,
+                            metrics,
+                            remaining_by_stage,
+                        )
+                        if not success:
+                            cfg.consume_parent_failure(ready_parent)
+                            completed_requests += 1
+                            logger.error(
+                                f"[{self._name}] Parent {ready_parent} dropped due to CFG forwarding failure "
+                                f"({completed_requests}/{total_requests})",
+                            )
+                    continue
+
                 engine_outputs = _load(result, obj_key="engine_outputs", shm_key="engine_outputs_shm")
                 # Mark last output time for this stage whenever we receive outputs
                 metrics.stage_last_ts[stage_id] = max(metrics.stage_last_ts[stage_id] or 0.0, time.time())
@@ -1104,6 +1156,40 @@ class Omni(OmniBase):
 
                 next_stage_id = stage_id + 1
                 if next_stage_id <= final_stage_id_to_prompt[req_id]:
+                    # CFG: if this parent has companions, defer forwarding
+                    if cfg.has_companions(req_id) and stage_id == 0:
+                        if cfg.is_parent_failed(req_id):
+                            cfg.consume_parent_failure(req_id)
+                            completed_requests += 1
+                            logger.error(
+                                f"[{self._name}] Parent {req_id} skipped CFG forwarding due to "
+                                f"companion failure ({completed_requests}/{total_requests})",
+                            )
+                            continue
+
+                        if cfg.all_companions_done(req_id):
+                            success = cfg.forward_parent_with_cfg(
+                                req_id,
+                                {"engine_outputs": engine_outputs, "stage_id": stage_id},
+                                self.stage_list,
+                                self.connectors,
+                                sampling_params_list,
+                                request_id_to_prompt,
+                                final_stage_id_to_prompt,
+                                metrics,
+                                remaining_by_stage,
+                            )
+                            if not success:
+                                cfg.consume_parent_failure(req_id)
+                                completed_requests += 1
+                                logger.error(
+                                    f"[{self._name}] Parent {req_id} dropped due to CFG forwarding failure "
+                                    f"({completed_requests}/{total_requests})",
+                                )
+                        else:
+                            cfg.defer_parent(req_id, engine_outputs, stage_id)
+                        continue
+
                     next_stage: OmniStage = self.stage_list[next_stage_id]
                     try:
                         # Derive inputs for the next stage, record preprocess time
@@ -1112,9 +1198,10 @@ class Omni(OmniBase):
                                 self.stage_list, [request_id_to_prompt[req_id]]
                             )
                     except Exception as e:
+                        completed_requests += 1
                         logger.exception(
                             f"[{self._name}] Process engine inputs error for req {req_id}"
-                            f" at stage {next_stage_id}: {e}",
+                            f" at stage {next_stage_id}: {e} ({completed_requests}/{total_requests})",
                         )
                         continue
                     sp_next = sampling_params_list[next_stage_id]  # type: ignore[index]
@@ -1141,6 +1228,7 @@ class Omni(OmniBase):
                             f"[{self._name}] Failed to send request {req_id} to stage-{next_stage_id} via connector. "
                             "Configure a connector for this edge or inspect connector logs for details."
                         )
+
                     logger.debug(
                         f"[{self._name}] Forwarded request {req_id} to stage-{next_stage_id}",
                     )
@@ -1154,6 +1242,12 @@ class Omni(OmniBase):
                     logger.debug(
                         f"[{self._name}] Request {req_id} fully completed ({completed_requests}/{total_requests})",
                     )
+            for timed_out_id in cfg.check_timeouts():
+                completed_requests += 1
+                logger.error(
+                    f"[{self._name}] Parent {timed_out_id} timed out; counting as failed "
+                    f"({completed_requests}/{total_requests})",
+                )
 
             if not made_progress:
                 time.sleep(0.005)

vllm_omni/entrypoints/omni_diffusion.py

@@ -28,6 +28,7 @@ class OmniDiffusion:
         # Capture stage info from kwargs before they might be filtered out
         stage_id = kwargs.get("stage_id")
         engine_input_source = kwargs.get("engine_input_source")
+        cfg_kv_collect_func = kwargs.pop("cfg_kv_collect_func", None)
 
         if od_config is None:
             od_config = OmniDiffusionConfig.from_kwargs(**kwargs)
@@ -94,6 +95,9 @@ class OmniDiffusion:
             od_config.tf_model_config = TransformerConfig()
             od_config.update_multimodal_support()
 
+        if cfg_kv_collect_func is not None:
+            od_config.cfg_kv_collect_func = cfg_kv_collect_func
+
         self.engine: DiffusionEngine = DiffusionEngine.make_engine(od_config)
 
     def generate(

vllm_omni/entrypoints/omni_stage.py

@@ -47,6 +47,7 @@ from vllm_omni.entrypoints.stage_utils import (
     _resolve_model_tokenizer_paths,
     _to_dict,
     is_profiler_task,
+    load_func_from_config,
     maybe_dump_to_shm,
     set_stage_devices,
 )
@@ -284,6 +285,7 @@ class OmniStage:
         else:
             self.custom_process_input_func = None
 
+        self.prompt_expand_func = load_func_from_config(getattr(stage_config, "prompt_expand_func", None))
         self.final_output = getattr(stage_config, "final_output", False)
         self.final_output_type = getattr(stage_config, "final_output_type", None)
         self.tts_args = _to_dict(getattr(stage_config, "tts_args", {}))
@@ -464,6 +466,7 @@ class OmniStage:
             "connectors_config": connectors_config or {},
             "stage_type": self.stage_type,
             "engine_input_source": self.engine_input_source,
+            "cfg_kv_collect_func": getattr(self.stage_config, "cfg_kv_collect_func", None),
             "final_output": self.final_output,
             "final_output_type": self.final_output_type,
         }
@@ -616,7 +619,11 @@ class OmniStage:
             return None
 
     def process_engine_inputs(
-        self, stage_list: list[Any], prompt: OmniTokensPrompt | TextPrompt = None
+        self,
+        stage_list: list[Any],
+        prompt: OmniTokensPrompt | TextPrompt = None,
+        *,
+        source_outputs_override: Any = None,
     ) -> list[OmniTokensPrompt | TextPrompt]:
         """Process engine inputs for this stage from upstream stage outputs.
 
@@ -627,6 +634,8 @@ class OmniStage:
         Args:
             stage_list: List of all stages in the pipeline
             prompt: Optional original prompt (for multimodal data preservation)
+            source_outputs_override: Use these outputs instead of reading from
+                the source stage's ``engine_outputs`` (for deferred CFG requests).
 
         Returns:
             List of processed engine inputs ready for this stage
@@ -639,7 +648,11 @@ class OmniStage:
             if len(self.engine_input_source) == 0:
                 raise ValueError("engine_input_source is empty")
             source_stage_id = self.engine_input_source[0]
-            source_outputs = stage_list[source_stage_id].engine_outputs
+            source_outputs = (
+                source_outputs_override
+                if source_outputs_override is not None
+                else stage_list[source_stage_id].engine_outputs
+            )
             if not isinstance(prompt, list):
                 prompt = [prompt]
             multi_modal_data = {
@@ -661,6 +674,18 @@ class OmniStage:
 
         else:
             engine_input_source = self.engine_input_source
+            if source_outputs_override is not None and engine_input_source:
+                # Temporarily swap engine_outputs so custom_process_input_func
+                # (which reads stage_list directly) sees the correct data.
+                _source_id = engine_input_source[0]
+                _orig_outputs = stage_list[_source_id].engine_outputs
+                stage_list[_source_id].engine_outputs = source_outputs_override
+                try:
+                    return self.custom_process_input_func(
+                        stage_list, engine_input_source, prompt, self.requires_multimodal_data
+                    )
+                finally:
+                    stage_list[_source_id].engine_outputs = _orig_outputs
             return self.custom_process_input_func(
                 stage_list, engine_input_source, prompt, self.requires_multimodal_data
             )
@@ -706,6 +731,8 @@ def _stage_worker(
     connectors_config = stage_payload.get("connectors_config", {})
     stage_type: Literal["llm", "diffusion"] = stage_payload.get("stage_type", "llm")
 
+    cfg_kv_collect_func = load_func_from_config(stage_payload.get("cfg_kv_collect_func"))
+
     if stage_type != "diffusion":
         _resolve_worker_cls(engine_args)
 
@@ -765,6 +792,7 @@ def _stage_worker(
                 model=model,
                 stage_id=stage_id,
                 engine_input_source=stage_payload.get("engine_input_source", []),
+                cfg_kv_collect_func=cfg_kv_collect_func,
                 **engine_args,
             )
         else:
@@ -1113,6 +1141,7 @@ async def _stage_worker_async(
     final_output = stage_payload.get("final_output", False)
     final_output_type = stage_payload.get("final_output_type", None)
 
+    cfg_kv_collect_func = load_func_from_config(stage_payload.get("cfg_kv_collect_func"))
     # Handle non-standard model directory structures (e.g., tokenizer in root, model in subdir)
     model = _resolve_model_tokenizer_paths(model, engine_args)
 
@@ -1191,10 +1220,13 @@ async def _stage_worker_async(
             od_config["omni_kv_config"]["engine_input_source"] = stage_payload.get("engine_input_source", [])
 
             logger.debug(f"[Stage-%s] Initializing diffusion engine with config: {od_config}", stage_id)
+            _diffusion_kwargs = {k: v for k, v in engine_args.items() if k not in {"od_config", "model"}}
+            if cfg_kv_collect_func is not None:
+                _diffusion_kwargs["cfg_kv_collect_func"] = cfg_kv_collect_func
             stage_engine = AsyncOmniDiffusion(
                 model=model,
                 od_config=od_config,
-                **{k: v for k, v in engine_args.items() if k not in {"od_config", "model"}},
+                **_diffusion_kwargs,
             )
             vllm_config = None  # Diffusion doesn't use vllm_config
         else:

vllm_omni/entrypoints/stage_utils.py

@@ -1,9 +1,11 @@
 from __future__ import annotations
 
 import enum
+import importlib
 import json
 import logging
 import os
+from collections.abc import Callable
 from multiprocessing import shared_memory as _shm
 from typing import Any
 
@@ -12,6 +14,22 @@ from omegaconf import OmegaConf
 logger = logging.getLogger(__name__)
 
 
+def load_func_from_config(func_path: str | None) -> Callable[..., Any] | None:
+    """Dynamically import a callable from a fully-qualified dotted path.
+
+    Args:
+        func_path: Dotted path such as ``"pkg.module.func_name"``, or *None*.
+
+    Returns:
+        The imported callable, or *None* when *func_path* is falsy.
+    """
+    if not func_path:
+        return None
+    module_path, func_name = func_path.rsplit(".", 1)
+    module = importlib.import_module(module_path)
+    return getattr(module, func_name)
+
+
 class OmniStageTaskType(enum.Enum):
     GENERATE = "generate"
     ABORT = "abort"

vllm_omni/inputs/data.py

@@ -229,6 +229,13 @@ class OmniDiffusionSamplingParams:
     kv_metadata: dict[str, Any] | None = None  # Metadata for KV Cache (e.g., kv_lens, ropes)
     need_kv_receive: bool = True  # Flag to indicate if this request expects KV transfer
 
+    # [Omni] Multi-KV for CFG: populated by model-specific cfg_kv_collect_func
+    cfg_text_past_key_values: Any | None = None
+    cfg_img_past_key_values: Any | None = None
+    cfg_text_kv_metadata: dict[str, Any] | None = None
+    cfg_img_kv_metadata: dict[str, Any] | None = None
+    cfg_kv_request_ids: dict[str, str] | None = None
+
     # Component modules
     modules: dict[str, Any] = field(default_factory=dict)
 

vllm_omni/model_executor/stage_configs/bagel.yaml

@@ -3,9 +3,12 @@
 stage_args:
   - stage_id: 0
     stage_type: llm
+    prompt_expand_func: vllm_omni.model_executor.stage_input_processors.bagel.expand_cfg_prompts
     runtime:
       devices: "0"
-      max_batch_size: 1
+      # 2 = 1 user prompt + 1 CFG companion (text-unconditional).
+      # For multi-prompt batches this should scale as batch_size × 2.
+      max_batch_size: 2
     engine_args:
       model_stage: thinker
       model_arch: BagelForConditionalGeneration
@@ -37,6 +40,7 @@ stage_args:
 
   - stage_id: 1
     stage_type: diffusion
+    cfg_kv_collect_func: vllm_omni.model_executor.stage_input_processors.bagel.collect_cfg_kv_caches
     runtime:
       devices: "0"
       max_batch_size: 1

vllm_omni/model_executor/stage_input_processors/bagel.py

@@ -0,0 +1,156 @@
+"""Stage input processor for Bagel: CFG prompt expansion and KV cache collection.
+
+Bagel's 3-branch CFG requires multiple prompts through the AR stage:
+  - gen (conditional): user prompt
+  - cfg_text (text unconditional): negative/empty prompt
+  - cfg_img (image unconditional): user prompt without image (same as gen for text2img)
+
+This module provides model-specific functions referenced by bagel.yaml:
+  - prompt_expand_func  -> expand_cfg_prompts
+  - cfg_kv_collect_func -> collect_cfg_kv_caches
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from types import SimpleNamespace
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+CFG_TEXT_SUFFIX = "__cfg_text"
+
+
+@dataclass
+class ExpandedPrompt:
+    """A single expanded prompt produced by the prompt expansion function."""
+
+    prompt: dict[str, Any] | str
+    role: str
+    request_id_suffix: str
+
+
+def expand_cfg_prompts(
+    prompt: dict[str, Any] | str,
+    sampling_params: Any,
+) -> list[ExpandedPrompt]:
+    """Expand a user prompt into additional prompts needed for Bagel CFG.
+
+    For text2img (modalities contains "image", no multi_modal_data with image):
+      - One extra prompt for cfg_text (negative/empty prompt)
+      - cfg_img reuses gen KV for text2img, so no extra prompt needed
+
+    For text2text or img2text: returns empty list (no expansion needed).
+
+    Args:
+        prompt: The original user prompt (dict or string).
+        sampling_params: The Stage-0 sampling params (may carry extra_args
+            from the diffusion sampling params for negative_prompt).
+
+    Returns:
+        List of ExpandedPrompt. Empty if no expansion needed.
+    """
+    if not isinstance(prompt, dict):
+        return []
+
+    modalities = prompt.get("modalities", [])
+    if "image" not in modalities and "img2img" not in modalities:
+        return []
+
+    neg_prompt = _get_negative_prompt(prompt, sampling_params)
+
+    if "image" in modalities:
+        neg_prompt_dict = {
+            "prompt": neg_prompt,
+            "modalities": prompt.get("modalities", []),
+        }
+        return [
+            ExpandedPrompt(
+                prompt=neg_prompt_dict,
+                role="cfg_text",
+                request_id_suffix=CFG_TEXT_SUFFIX,
+            ),
+        ]
+
+    # img2img: more complex (3 distinct caches). Reserve for future.
+    return []
+
+
+def collect_cfg_kv_caches(
+    request_id: str,
+    cfg_request_ids: dict[str, str],
+    kv_transfer_manager: Any,
+    target_device: Any | None = None,
+) -> dict[str, Any]:
+    """Collect KV caches for all CFG roles from the KV transfer manager.
+
+    Called by the diffusion model runner after receiving the primary KV cache.
+    Uses the kv_transfer_manager to fetch companion KV caches by their
+    request IDs.
+
+    Args:
+        request_id: The original (parent) request ID.
+        cfg_request_ids: Mapping of role -> companion request ID,
+            e.g. {"cfg_text": "req_0__cfg_text"}.
+        kv_transfer_manager: The OmniKVTransferManager instance.
+        target_device: Device to move tensors to.
+
+    Returns:
+        Dict with keys like "cfg_text_past_key_values",
+        "cfg_text_kv_metadata", etc.
+    """
+    result: dict[str, Any] = {}
+
+    for role, companion_rid in cfg_request_ids.items():
+        try:
+            data, size = kv_transfer_manager.receive_kv_cache_for_request(companion_rid, target_device)
+            if data and "layer_blocks" in data:
+                layer_blocks = data["layer_blocks"]
+                kv_obj = SimpleNamespace(**layer_blocks)
+                result[f"{role}_past_key_values"] = kv_obj
+                if "metadata" in data:
+                    result[f"{role}_kv_metadata"] = data["metadata"]
+                logger.info(
+                    "Collected CFG KV cache for role=%s, rid=%s, size=%d bytes",
+                    role,
+                    companion_rid,
+                    size,
+                )
+            else:
+                logger.warning(
+                    "Failed to collect CFG KV cache for role=%s, rid=%s",
+                    role,
+                    companion_rid,
+                )
+        except Exception as e:
+            logger.exception(
+                "Error collecting CFG KV cache for role=%s, rid=%s: %s",
+                role,
+                companion_rid,
+                e,
+            )
+
+    return result
+
+
+def _get_negative_prompt(
+    prompt: dict[str, Any],
+    sampling_params: Any,
+) -> str:
+    """Resolve the negative prompt for CFG from prompt or sampling params.
+
+    An empty string is treated the same as absent (falls through to
+    the Bagel default token pair), because an empty negative prompt is
+    not meaningful for CFG guidance.
+    """
+    neg = prompt.get("negative_prompt")
+    if neg:
+        return neg
+
+    if hasattr(sampling_params, "extra_args") and sampling_params.extra_args:
+        neg = sampling_params.extra_args.get("negative_prompt")
+        if neg:
+            return neg
+
+    return "<|im_start|><|im_end|>"