[Performance][MiniCPM-o] Whole-Euler CUDA graphs for Code2Wav with a shared attention arena (#8007)

docs/serving/realtime_duplex_api.md

@@ -54,7 +54,7 @@ vllm-omni serve openbmb/MiniCPM-o-4_5 \
 ```
 
 `vllm_omni/deploy/minicpmo_4_5.yaml` declares `session_mode: duplex` and
-`duplex_session.max_sessions: 4`. Because the MiniCPM-o 4.5 pipeline declares a
+`duplex_session.max_sessions: 16`. Because the MiniCPM-o 4.5 pipeline declares a
 `duplex_plugin`, `vllm-omni serve` runs it through `DuplexOmni`: the server
 mounts `ws://<host>:8099/v1/realtime?duplex=1` (this page; `ws://<host>:8099/v1/duplex`
 is an alias of the same route), `POST /v1/chat/completions`, `/v1/models` and

tests/config/test_config_factory.py

@@ -1408,12 +1408,12 @@ class TestDeployConfigLoading:
     @pytest.mark.parametrize(
         ("filename", "max_sessions"),
         [
-            ("minicpmo_4_5.yaml", 4),
+            ("minicpmo_4_5.yaml", 16),
             ("minicpmo_4_5_2gpu.yaml", 4),
             ("minicpmo_4_5_3gpu.yaml", 4),
             ("minicpmo_4_5_8x4090.yaml", 1),
-            ("minicpmo_4_5_3gpu_stage1_replicas.yaml", 4),
+            ("minicpmo_4_5_3gpu_stage1_replicas.yaml", 16),
-            ("minicpmo_4_5_4gpu_stage1_replicas.yaml", 4),
+            ("minicpmo_4_5_4gpu_stage1_replicas.yaml", 16),
             ("minicpmo_4_5_8x4090_stage1_replicas.yaml", 1),
         ],
     )
@@ -2886,7 +2886,7 @@ class TestPlatformOverrides:
 
         cuda = _apply_platform_overrides(load_deploy_config(deploy_path), platform="cuda")
         cuda_stages = merge_pipeline_deploy(pipeline, cuda)
-        assert cuda_stages[1].yaml_engine_args["kv_cache_memory_bytes"] == 2 * 1024**3
+        assert cuda_stages[1].yaml_engine_args["kv_cache_memory_bytes"] == 4 * 1024**3
 
         # The CUDA memory budget must not leak into the existing NPU profile.
         npu = _apply_platform_overrides(load_deploy_config(deploy_path), platform="npu")
@@ -2903,7 +2903,7 @@ class TestPlatformOverrides:
                 load_deploy_config(Path(get_deploy_config_path(filename))), platform="cuda"
             )
             replica_stages = merge_pipeline_deploy(pipeline, replica)
-            # Explicit null clears the inherited single-GPU 2 GiB CUDA cap.
+            # Explicit null clears the inherited single-GPU 4 GiB CUDA cap.
             assert replica_stages[1].yaml_engine_args.get("kv_cache_memory_bytes") is None
 
     def test_fish_speech_npu_uses_ascend_kv_block_size(self):

tests/engine/duplex/test_duplex_deploy_config.py

@@ -96,9 +96,9 @@ def test_duplex_session_runtime_rejects_non_positive_values(tmp_path, name: str,
 @pytest.mark.parametrize(
     ("deploy_yaml", "expected"),
     [
-        ("minicpmo_4_5.yaml", 4),
+        ("minicpmo_4_5.yaml", 16),
         ("minicpmo_4_5_8x4090.yaml", 1),
-        ("minicpmo_4_5_3gpu_stage1_replicas.yaml", 4),
+        ("minicpmo_4_5_3gpu_stage1_replicas.yaml", 16),
     ],
 )
 def test_deploy_duplex_max_sessions_tracks_the_deploy_config(deploy_yaml: str, expected: int) -> None:

tests/model_executor/models/minicpmo_4_5/test_cfm_graph_bucketing.py

@@ -16,9 +16,15 @@ import torch
 
 from vllm_omni.model_executor.models.minicpmo_4_5.batched_token2wav import (
     _cfm_pad_frames,
-    _zero_padded_cnn_cache,
     _zero_padded_frames,
 )
+from vllm_omni.model_executor.models.minicpmo_4_5.cuda_graph_wrapper import (
+    _att_keep_ranges,
+    _build_capture_mask,
+    _capture_query_width,
+    _whole_euler_att_segments,
+    _zero_padded_cnn_cache,
+)
 
 pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
 
@@ -139,6 +145,75 @@ def test_varied_chunk_lengths_collapse_onto_few_widths():
     assert len(padded) < len(varied)
 
 
+def test_capture_query_width_collapses_decode_grid_onto_one_bucket():
+    """Whole-Euler capture bucket is the #7416 pad helper with a 64-wide decode grid."""
+    assert _capture_query_width(16, 0) == 16
+    assert _capture_query_width(16, 1) == 16
+    assert {_capture_query_width(w, 64) for w in (16, 32, 48, 50, 64)} == {64}
+    assert _capture_query_width(304, 64) == 320
+    assert _capture_query_width(320, 64) == 320
+
+
+def test_att_keep_ranges_match_the_streaming_trim():
+    """Past ``prompt_len + 100`` frames the cache keeps its first ``prompt_len`` and last 100."""
+    assert _att_keep_ranges(368, (300, 100)) == [(0, 368)]
+    assert _att_keep_ranges(400, (300, 100)) == [(0, 400)]
+    assert _att_keep_ranges(464, (300, 100)) == [(0, 300), (364, 100)]
+    assert _att_keep_ranges(464, None) == [(0, 464)]
+
+
+def test_whole_euler_att_segments_skip_capture_padding():
+    # Steady chunk: capture width == chunk width, so only the trim applies.
+    assert _whole_euler_att_segments(mel_width=64, query_cap=64, offset=400, keep=(300, 100)) == [
+        (0, 300),
+        (364, 100),
+    ]
+    # A 32-wide chunk on the 64 capture: the graph wrote [current 64 | cache 400];
+    # frames 32:64 are capture padding and the cache starts at 64.
+    assert _whole_euler_att_segments(mel_width=32, query_cap=64, offset=400, keep=None) == [(0, 32), (64, 400)]
+    assert _whole_euler_att_segments(mel_width=32, query_cap=64, offset=400, keep=(300, 100)) == [
+        (0, 32),
+        (64, 268),
+        (364, 100),
+    ]
+    # First chunk: no cache, only the tail of the capture width is dropped.
+    assert _whole_euler_att_segments(mel_width=304, query_cap=320, offset=0, keep=None) == [(0, 304)]
+
+
+def test_capture_mask_moves_the_cache_block_past_capture_padding():
+    width, cap, offset, valid = 32, 64, 8, 30
+    caller = torch.ones(2, width, width + offset, dtype=torch.bool)
+    caller[:, :, valid:width] = False
+    mask = _build_capture_mask(
+        attn_mask=caller,
+        batch_size=1,
+        query_cap=cap,
+        offset=offset,
+        mel_width=width,
+        mel_frames=valid,
+        device=torch.device("cpu"),
+    )
+    assert mask.shape == (2, cap, cap + offset)
+    assert mask[:, :, :valid].all()
+    assert not mask[:, :, valid:cap].any()
+    assert mask[:, :, cap:].all()
+    # Capture-padding query rows are never empty, so SDPA cannot produce NaN there.
+    assert mask.any(dim=-1).all()
+
+    implicit = _build_capture_mask(
+        attn_mask=None,
+        batch_size=1,
+        query_cap=cap,
+        offset=offset,
+        mel_width=width,
+        mel_frames=width,
+        device=torch.device("cpu"),
+    )
+    assert implicit[:, :, :width].all()
+    assert not implicit[:, :, width:cap].any()
+    assert implicit[:, :, cap:].all()
+
+
 def test_zero_padded_frames_survives_an_integration_step():
     """The padded region must stay zero after ``x = x + dt * velocity``.
 

tests/model_executor/models/minicpmo_4_5/test_code2wav_batching.py

@@ -172,6 +172,19 @@ def _config(minimum: int = 1, initial: int = 0, *, runtime_prompt_cache_size: in
     )
 
 
+@pytest.mark.parametrize(
+    ("extra", "max_num_seqs", "micro"), [({}, 6, 6), ({}, 64, 16), ({"micro_batch_size": 2}, 6, 2)]
+)
+def test_whole_euler_micro_batch_defaults_to_stage_concurrency(extra, max_num_seqs, micro):
+    config = _config()
+    config.model_config.stage_connector_config["extra"].update(extra)
+    config.scheduler_config = SimpleNamespace(max_num_seqs=max_num_seqs)
+
+    model = MiniCPMO45Code2Wav(vllm_config=config)
+
+    assert model._cfm_graph_config["micro_batch_size"] == micro
+
+
 def _model(initial: int = 0, minimum: int = 1, *, runtime_prompt_cache_size: int = 4, setup_cache_size: int = 1):
     token2wav = _FakeToken2Wav()
     backend = BatchedToken2Wav(token2wav, setup_cache_size=setup_cache_size)
@@ -213,17 +226,43 @@ def test_setup_cache_reuses_read_only_state_for_the_same_exact_batch():
         torch.testing.assert_close(cache[name], expected)
 
 
-def test_setup_cache_keys_batch_size_and_evicts_least_recent_entry():
+def test_setup_solves_the_prompt_once_for_every_batch_size():
     token2wav = _FakeToken2Wav()
     adapter = BatchedToken2Wav(token2wav, setup_cache_size=1)
     prompt = adapter.prepare_prompt("shared", "/fake/prompt.wav")
 
-    adapter.setup_batch(prompt, 1)
+    single = adapter.setup_batch(prompt, 1)
-    adapter.setup_batch(prompt, 2)
+    pair = adapter.setup_batch(prompt, 2)
-    adapter.setup_batch(prompt, 1)
 
-    assert token2wav.flow.encoder.calls == [1, 2, 1]
+    assert token2wav.flow.encoder.calls == [1]
-    assert list(adapter._setup_cache) == [(prompt.cache_key, 1, 0)]
+    assert pair == [single[0], single[0]]
+    assert list(adapter._setup_cache) == [(prompt.cache_key, 0)]
+
+
+def test_setup_cache_evicts_least_recent_prompt():
+    token2wav = _FakeToken2Wav()
+    adapter = BatchedToken2Wav(token2wav, setup_cache_size=1)
+    first = adapter.prepare_prompt("first", "/fake/first.wav")
+    second = adapter.prepare_prompt("second", "/fake/second.wav")
+
+    adapter.setup_batch(first, 1)
+    adapter.setup_batch(second, 2)
+    adapter.setup_batch(first, 1)
+
+    assert token2wav.flow.encoder.calls == [1, 1, 1]
+    assert list(adapter._setup_cache) == [(first.cache_key, 0)]
+
+
+def test_setup_without_cache_still_solves_one_shared_row():
+    token2wav = _FakeToken2Wav()
+    adapter = BatchedToken2Wav(token2wav, setup_cache_size=0)
+    prompt = adapter.prepare_prompt("shared", "/fake/prompt.wav")
+
+    states = adapter.setup_batch(prompt, 3)
+
+    assert token2wav.flow.encoder.calls == [1]
+    assert states[0] is states[1] is states[2]
+    assert not adapter._setup_cache
 
 
 def test_evict_prompt_clears_features_and_setup_state():
@@ -481,9 +520,12 @@ def test_adapter_runs_true_batch_cfg_and_splits_request_caches():
     )
 
     assert token2wav.prompt_calls == 1
-    assert token2wav.flow.encoder.calls == [2, 2]
+    # Setup solves the shared prompt once; the chunk runs both requests together.
-    assert token2wav.flow.decoder.estimator.cfg_batches == [4, 4, 4, 4]
+    assert token2wav.flow.encoder.calls == [1, 2]
-    assert all(order == [1.0, 1.0, 0.0, 0.0] for order in token2wav.flow.decoder.estimator.speaker_order)
+    assert token2wav.flow.decoder.estimator.cfg_batches == [2, 2, 4, 4]
+    speaker_order = token2wav.flow.decoder.estimator.speaker_order
+    assert all(order == [1.0, 0.0] for order in speaker_order[:2])
+    assert all(order == [1.0, 1.0, 0.0, 0.0] for order in speaker_order[2:])
     assert token2wav.hift.calls == [2]
     assert len(audios) == 2
     cache0 = states[0].flow_cache["estimator_cnn_cache"]
@@ -706,6 +748,40 @@ def test_decode_cfm_enters_platform_sdpa_context(monkeypatch):
     assert entered == ["enter", "exit"]
 
 
+def test_eager_cfm_reads_the_step_sizes_to_the_host_once(monkeypatch):
+    """The eager Euler loop must not sync the stream for ``dt`` on every decode."""
+    adapter = BatchedToken2Wav(_FakeToken2Wav())
+    reads: list[str] = []
+
+    def recording(name):
+        original = getattr(torch.Tensor, name)
+
+        def read(self, *args, **kwargs):
+            # The fake estimator records its inputs with ``tolist``; skip those.
+            if sys._getframe(1).f_code.co_filename != __file__:
+                reads.append(name)
+            return original(self, *args, **kwargs)
+
+        return read
+
+    for name in ("item", "tolist"):
+        monkeypatch.setattr(torch.Tensor, name, recording(name))
+
+    def decode():
+        adapter._decode_cfm(
+            torch.ones((1, 1, 2)),
+            torch.ones((1, 1)),
+            torch.zeros((1, 1, 2)),
+            cnn_cache=None,
+            att_cache=None,
+        )
+
+    decode()
+    assert reads == ["tolist"]
+    decode()
+    assert reads == ["tolist"]
+
+
 def test_ragged_decode_bypasses_exact_shape_accelerators():
     adapter = BatchedToken2Wav(_FakeToken2Wav())
     _enable_fake_ragged_kernel(adapter)
@@ -963,6 +1039,44 @@ def test_estimator_cache_stack_split_round_trip_preserves_cfg_rows():
         )
 
 
+@pytest.mark.parametrize("attention_dtype", [torch.float32, torch.bfloat16])
+def test_split_flow_cache_trims_each_request_like_the_batch_trim(attention_dtype):
+    """Trimming per request while splitting equals trimming the stacked batch first."""
+    adapter = BatchedToken2Wav(_FakeToken2Wav(), bfloat16_attention_cache=attention_dtype == torch.bfloat16)
+    batch_size, prompt_len, frames = 3, 5, 12
+    torch.manual_seed(0)
+    stacked = {
+        "conformer_cnn_cache": torch.randn(batch_size, 2, 3),
+        "conformer_att_cache": torch.randn(2, batch_size, 2, 4, 6),
+        "estimator_cnn_cache": torch.randn(4, 2, 2 * batch_size, 3, 2),
+        "estimator_att_cache": torch.randn(4, 2, 2 * batch_size, 2, frames, 6),
+    }
+
+    restored = adapter._split_flow_cache(stacked, batch_size, estimator_att_keep=(prompt_len, 4))
+
+    att = stacked["estimator_att_cache"]
+    trimmed = torch.cat((att[..., :prompt_len, :], att[..., -4:, :]), dim=4)
+    for row, request in enumerate(restored):
+        expected = torch.cat(
+            (trimmed[:, :, row : row + 1], trimmed[:, :, batch_size + row : batch_size + row + 1]),
+            dim=2,
+        ).to(attention_dtype)
+        assert request["estimator_att_cache"].dtype == attention_dtype
+        torch.testing.assert_close(request["estimator_att_cache"], expected)
+        torch.testing.assert_close(
+            request["estimator_cnn_cache"],
+            torch.cat(
+                (
+                    stacked["estimator_cnn_cache"][:, :, row : row + 1],
+                    stacked["estimator_cnn_cache"][:, :, batch_size + row : batch_size + row + 1],
+                ),
+                dim=2,
+            ),
+        )
+    untrimmed = adapter._split_flow_cache(stacked, batch_size, estimator_att_keep=(frames, 4))
+    assert untrimmed[0]["estimator_att_cache"].shape[4] == frames
+
+
 def test_bfloat16_estimator_attention_cache_materializes_only_current_timestep():
     token2wav = _FakeToken2Wav()
     adapter = BatchedToken2Wav(token2wav, bfloat16_attention_cache=True)
@@ -970,9 +1084,8 @@ def test_bfloat16_estimator_attention_cache_materializes_only_current_timestep()
     states = adapter.setup_batch(prompt, 2)
 
     assert all(state.flow_cache["estimator_att_cache"].dtype == torch.bfloat16 for state in states)
-    assert (
+    # Both requests start from the one shared prompt state.
-        states[0].flow_cache["estimator_att_cache"].data_ptr() != states[1].flow_cache["estimator_att_cache"].data_ptr()
+    assert states[0] is states[1]
-    )
 
     stacked = adapter._stack_flow_cache(states)
     assert stacked["estimator_att_cache"].dtype == torch.bfloat16
@@ -991,6 +1104,10 @@ def test_bfloat16_estimator_attention_cache_materializes_only_current_timestep()
         last_chunk=False,
     )
     assert all(state.flow_cache["estimator_att_cache"].dtype == torch.bfloat16 for state in next_states)
+    assert (
+        next_states[0].flow_cache["estimator_att_cache"].data_ptr()
+        != next_states[1].flow_cache["estimator_att_cache"].data_ptr()
+    )
     assert token2wav.flow.decoder.estimator.attention_cache_dtypes[-2:] == [torch.float32, torch.float32]
 
 
@@ -1117,9 +1234,17 @@ def test_initial_empty_segment_marker_initializes_stream_without_audio():
     assert "duplex" in model._states
 
 
-@pytest.mark.parametrize("setup_cache_size,expected_calls", [(0, [2, 2, 1]), (1, [2, 1])])
+@pytest.mark.parametrize("setup_cache_size,expected_calls", [(0, [1, 1, 1]), (1, [1])])
 def test_initial_empty_segment_markers_respect_initial_batch_limit(setup_cache_size, expected_calls):
     model, token2wav = _model(initial=2, setup_cache_size=setup_cache_size)
+    setup_batches: list[int] = []
+    setup_batch = model.backend.setup_batch
+
+    def record_setup(features, batch_size):
+        setup_batches.append(batch_size)
+        return setup_batch(features, batch_size)
+
+    model.backend.setup_batch = record_setup
     names = ["a", "b", "c", "d", "e"]
     boundaries = []
     for name in names:
@@ -1135,6 +1260,8 @@ def test_initial_empty_segment_markers_respect_initial_batch_limit(setup_cache_s
 
     output = _forward(model, boundaries)
 
+    assert setup_batches == [2, 2, 1]
+    # One shared prompt solve per setup, or one in all with the setup cache.
     assert token2wav.flow.encoder.calls == expected_calls
     assert [audio.numel() for audio in output.multimodal_outputs["model_outputs"]] == [0] * len(names)
     assert set(model._states) == set(names)
@@ -1631,8 +1758,8 @@ def test_setup_cache_misses_when_cfm_graph_padding_is_disabled(monkeypatch, disa
     assert unpadded[0] is not padded[0]
     assert reused[0] is unpadded[0]
     assert adapter.flow.encoder.calls == [1, 1]
-    assert padded_keys == [(prompt.cache_key, 1, 16)]
+    assert padded_keys == [(prompt.cache_key, 16)]
-    assert list(adapter._setup_cache) == [(prompt.cache_key, 1, 0)]
+    assert list(adapter._setup_cache) == [(prompt.cache_key, 0)]
 
 
 def test_padded_chunk_keeps_the_cross_chunk_caches_on_the_valid_boundary():

tests/model_executor/models/minicpmo_4_5/test_pipeline.py

@@ -170,7 +170,7 @@ class TestDeployTopology:
         assert stages[1].yaml_engine_args["custom_process_next_stage_input_func"].endswith(expected_processor)
         assert "hf_overrides" not in stages[1].yaml_engine_args
         if filename == "minicpmo_4_5.yaml":
-            assert [stage.yaml_engine_args["max_num_seqs"] for stage in stages] == [4, 4, 4]
+            assert [stage.yaml_engine_args["max_num_seqs"] for stage in stages] == [16, 16, 16]
             memory_utilizations = [stage.yaml_engine_args["gpu_memory_utilization"] for stage in stages]
             assert memory_utilizations == [
                 0.55,

vllm_omni/deploy/minicpmo_4_5.yaml

@@ -7,7 +7,7 @@ session_mode: duplex
 # A smaller window serializes first-packet generation for concurrent chat turns.
 duplex_session:
   # Matches Stage 0/1 max_num_seqs below.
-  max_sessions: 4
+  max_sessions: 16
 
 trust_remote_code: true
 enable_prefix_caching: false
@@ -27,12 +27,17 @@ connectors:
       # HiFTGraphWrapper.derive_capture_bucket_size explicitly includes it.
       enable_hift_graph: true
       enable_cfm_graph: true
+      # Whole-Euler CUDA graphs exist for batch 1, 2, 4, ... up to
+      # micro_batch_size, which defaults to Stage 2 max_num_seqs: all graphs
+      # share one attention-cache arena of that many requests (~0.6 GiB each
+      # in fp32, 10 GiB at 16). Larger batches replay micro-batches;
+      # B > max_graph_batch runs eager.
+      max_graph_batch: 16
       cfm_max_graphs: 32
-      cfm_graph_bucket_frames: 16
 
 stages:
   - stage_id: 0
-    max_num_seqs: 4
+    max_num_seqs: 16
     # Leave headroom on the shared GPU for Talker KV + concurrent Code2Wav.
     gpu_memory_utilization: 0.55
     enforce_eager: false
@@ -62,7 +67,7 @@ stages:
       repetition_penalty: 1.0
 
   - stage_id: 1
-    max_num_seqs: 4
+    max_num_seqs: 16
     # Preserve the checkpoint's full-attention Talker prompt while it fits;
     # native duplex rolls over to a one-chunk window at context capacity.
     # Leave device-level headroom for the colocated HiFi-GAN cuDNN workspace.
@@ -92,7 +97,7 @@ stages:
       detokenize: false
 
   - stage_id: 2
-    max_num_seqs: 4
+    max_num_seqs: 16
     gpu_memory_utilization: 0.18
     enforce_eager: true
     enable_chunked_prefill: false
@@ -113,10 +118,11 @@ platforms:
     stages:
       - stage_id: 1
         # Full-attention occupancy is bounded by max_num_seqs * max_model_len
-        # (4 * 4096 = 16384 tokens). Keep the reported CUDA KV token capacity
+        # (16 * 4096 = 65536 tokens). Keep the reported CUDA KV token capacity
-        # above that bound because physical-preemption replay is unsupported.
+        # above that bound because physical-preemption replay is unsupported
-        # The explicit cap leaves device headroom for concurrent Stage 2.
+        # (2 GiB holds ~35k tokens). The explicit cap leaves device headroom
-        kv_cache_memory_bytes: 2147483648  # 2 GiB
+        # for concurrent Stage 2.
+        kv_cache_memory_bytes: 4294967296  # 4 GiB
 
   npu:
     stages:

vllm_omni/deploy/minicpmo_4_5_2gpu.yaml

@@ -32,7 +32,6 @@ connectors:
       enable_hift_graph: true
       enable_cfm_graph: true
       cfm_max_graphs: 32
-      cfm_graph_bucket_frames: 16
 
 stages:
   - stage_id: 0

vllm_omni/deploy/minicpmo_4_5_3gpu.yaml

@@ -32,7 +32,6 @@ connectors:
       enable_hift_graph: true
       enable_cfm_graph: true
       cfm_max_graphs: 32
-      cfm_graph_bucket_frames: 16
 
 stages:
   - stage_id: 0

vllm_omni/deploy/minicpmo_4_5_8x4090.yaml

@@ -39,7 +39,6 @@ connectors:
       enable_hift_graph: true
       enable_cfm_graph: true
       cfm_max_graphs: 32
-      cfm_graph_bucket_frames: 16
 
 stages:
   - stage_id: 0

vllm_omni/model_executor/models/minicpmo_4_5/batched_token2wav.py

@@ -15,7 +15,16 @@ import torch.nn as nn
 import torch.nn.functional as F
 from vllm.logger import init_logger
 
-from .cuda_graph_wrapper import CFMGraphWrapper, HiFTGraphWrapper
+from .cuda_graph_wrapper import (
+    CFMGraphWrapper,
+    HiFTGraphWrapper,
+    WholeEulerCFMGraphWrapper,
+    _att_keep_ranges,
+    _copy_frame_segments,
+    _euler_step,
+    _euler_timeline,
+    _zero_padded_cnn_cache,
+)
 
 logger = init_logger(__name__)
 
@@ -25,6 +34,17 @@ _SILENCE_TOKEN = 4218
 # than this many codec tokens overflows the PE slice (matrix_ac vs matrix_bd).
 _DEFAULT_RELPOS_MAX_POS = 5000
 _MAX_ENCODE_TOKEN_CAP = 1024
+# Streaming cache trim (stepaudio2 ``Token2wav.stream``): past ``prompt_len +
+# 100`` frames, keep the first ``prompt_len`` frames and the last 100.
+_CACHE_TRIM_SUFFIX = 100
+
+
+def _trim_streaming_cache(cache: torch.Tensor, prompt_len: int) -> torch.Tensor:
+    """The streaming trim of a cache whose frame axis is ``-2``."""
+    ranges = _att_keep_ranges(int(cache.shape[-2]), (prompt_len, _CACHE_TRIM_SUFFIX))
+    if len(ranges) == 1:
+        return cache
+    return torch.cat([cache[..., start : start + length, :] for start, length in ranges], dim=-2)
 
 
 def relpos_encode_token_budget(
@@ -47,6 +67,26 @@ def relpos_encode_token_budget(
     return max(lookahead + 1, min(int(cap), room))
 
 
+def _supports_ragged_kernel(estimator: nn.Module) -> bool:
+    """Whether ``_blocks_forward_chunk_ragged`` can run this estimator's DiT blocks."""
+    blocks = getattr(estimator, "blocks", None)
+    if not blocks or not hasattr(estimator, "in_proj") or not hasattr(estimator, "final_layer"):
+        return False
+    block = blocks[0]
+    attn = getattr(block, "attn", None)
+    conv_blocks = getattr(getattr(block, "conv", None), "block", None)
+    return (
+        hasattr(block, "adaLN_modulation")
+        and all(
+            hasattr(attn, name)
+            for name in ("forward_chunk", "to_q", "to_k", "to_v", "to_heads", "q_norm", "k_norm", "proj")
+        )
+        and conv_blocks is not None
+        and len(conv_blocks) >= 8
+        and all(isinstance(conv_blocks[index], nn.Conv1d) for index in (1, 6))
+    )
+
+
 def _cfm_pad_frames(
     *,
     mel_frames: int,
@@ -98,38 +138,6 @@ def _zero_padded_frames(tensor: torch.Tensor, valid_frames: int | None) -> None:
     tensor[..., valid_frames:] = 0.0
 
 
-def _zero_padded_cnn_cache(
-    cnn_cache: torch.Tensor,
-    estimator: nn.Module,
-    pad_frames: int,
-) -> None:
-    """Clear the cache positions that come from padded frames, in place.
-
-    Each block's CNN cache holds the tail of its convolution output
-    (``new_cnn_cache = x[..., -causal_padding[0]:]``, inside ``stepaudio2``),
-    so when the padding sits at the chunk tail those positions are
-    padding-derived and would otherwise become the next chunk's left context.
-    Padding is at most ``pad_frames`` wide, so only the trailing positions that
-    can come from it are cleared; the valid part of the window is kept.
-    """
-    blocks = estimator.blocks
-    if pad_frames > 0 and blocks and len(blocks) == cnn_cache.shape[0]:
-        width = int(cnn_cache.shape[-1])
-        if width > 0 and all(int(block.conv.block[1].causal_padding[0]) == width for block in blocks):
-            # _estimator_buffers packs equal-width blocks into one tensor.
-            # Clear their shared tail with one write instead of one per block.
-            cnn_cache[..., max(0, width - pad_frames) :] = 0.0
-            return
-
-    for index, block in enumerate(estimator.blocks):
-        width = int(block.conv.block[1].causal_padding[0])
-        if width <= 0:
-            continue
-        zero_from = max(0, width - pad_frames)
-        if zero_from < width:
-            cnn_cache[index][..., zero_from:] = 0.0
-
-
 def plan_token2wav_encode_slices(
     num_frames: int,
     *,
@@ -297,6 +305,7 @@ class BatchedToken2Wav(nn.Module):
             token2wav.speech_window.detach().clone(),
             persistent=False,
         )
+        self._timeline_cache: dict[tuple[torch.device, torch.dtype], tuple[torch.Tensor, list[float]]] = {}
         self.hift_graph_wrapper: HiFTGraphWrapper | None = None
         graph_config = dict(hift_graph_config or {})
         if bool(graph_config.get("enabled", False)):
@@ -312,26 +321,79 @@ class BatchedToken2Wav(nn.Module):
                         "MiniCPM-o HiFT CUDA Graph requires source_cache_len to be divisible by mel_cache_len"
                     )
                 capture_batch_sizes = graph_config.get("capture_batch_sizes", [1])
-                logger.info("Enabling HiFT CUDA Graph with batch sizes %s", capture_batch_sizes)
+                max_serial_batch = int(graph_config.get("max_serial_batch", 4))
+                logger.info(
+                    "Enabling HiFT CUDA Graph with batch sizes %s (max_serial_batch=%d)",
+                    capture_batch_sizes,
+                    max_serial_batch,
+                )
                 self.hift_graph_wrapper = HiFTGraphWrapper(
                     token2wav=token2wav,
                     connector_config=dict(connector_config),
                     capture_batch_sizes=capture_batch_sizes,
+                    max_serial_batch=max_serial_batch,
                 )
                 with torch.inference_mode(), _autocast_disabled(hift_parameter.device):
                     self.hift_graph_wrapper.capture()
                 logger.info("HiFT CUDA Graph captured successfully")
         self._cfm_graph_wrapper: CFMGraphWrapper | None = None
+        self._whole_euler_graph_wrapper: WholeEulerCFMGraphWrapper | None = None
         cfm_graph_cfg = dict(cfm_graph_config or {})
         if bool(cfm_graph_cfg.get("enabled", False)):
             flow_parameter = next(self.flow.parameters(), None)
             if flow_parameter is not None and flow_parameter.device.type == "cuda":
                 estimator = self.flow.decoder.estimator
+                max_graphs = int(cfm_graph_cfg.get("max_graphs", 32))
+                max_serial_batch = int(cfm_graph_cfg.get("max_serial_batch", 4))
+                max_graph_batch_cfg = cfm_graph_cfg.get("max_graph_batch")
+                max_graph_batch = int(max_graph_batch_cfg) if max_graph_batch_cfg is not None else None
+                micro_batch_size_cfg = cfm_graph_cfg.get("micro_batch_size")
+                micro_batch_size = int(micro_batch_size_cfg) if micro_batch_size_cfg is not None else 4
+                if bool(cfm_graph_cfg.get("enable_whole_euler", True)) and self._trt_stepper is None:
+                    capture_frames = cfm_graph_cfg.get("capture_frames")
+                    if capture_frames is None and connector_config is not None:
+                        # The steady chunk's mel width, derived like HiFTGraphWrapper's
+                        # capture bucket: steady chunks run unpadded and shorter
+                        # chunks share their graph under a mask.
+                        capture_frames = int(connector_config["codec_chunk_frames"]) * int(
+                            getattr(self.flow, "token_mel_ratio", 2)
+                        )
+                    query_bucket_frames = int(capture_frames if capture_frames is not None else 64)
+                    self._whole_euler_graph_wrapper = WholeEulerCFMGraphWrapper(
+                        estimator=estimator,
+                        n_timesteps=self.n_timesteps,
+                        inference_cfg_rate=getattr(self.flow.decoder, "inference_cfg_rate", 0.7),
+                        att_cache_dtype=self._estimator_att_cache_dtype,
+                        max_graphs=max_graphs,
+                        max_serial_batch=max_serial_batch,
+                        max_graph_batch=max_graph_batch,
+                        micro_batch_size=micro_batch_size,
+                        query_bucket_frames=query_bucket_frames,
+                        pad_max_rows=cfm_graph_cfg.get("pad_max_rows"),
+                        # Looked up at capture time, so a replaced ragged kernel is the one captured.
+                        ragged_body=(
+                            (lambda *args: self._blocks_forward_chunk_ragged(*args))
+                            if _supports_ragged_kernel(estimator)
+                            else None
+                        ),
+                    )
+                    logger.info(
+                        "Whole-Euler CFM CUDA Graph enabled "
+                        "(max_graphs=%d, max_serial_batch=%d, max_graph_batch=%s, "
+                        "micro_batch_size=%d, query_bucket_frames=%d)",
+                        max_graphs,
+                        max_serial_batch,
+                        str(max_graph_batch),
+                        micro_batch_size,
+                        query_bucket_frames,
+                    )
+                elif self._trt_stepper is not None and bool(cfm_graph_cfg.get("enable_whole_euler", True)):
+                    logger.info("Whole-Euler CFM CUDA Graph disabled because TensorRT stepper is configured")
                 self._cfm_graph_wrapper = CFMGraphWrapper(
                     graph_fn=estimator.blocks_forward_chunk,
-                    max_graphs=int(cfm_graph_cfg.get("max_graphs", 32)),
+                    max_graphs=max_graphs,
                 )
-                logger.info("CFM CUDA Graph enabled (max_graphs=%d)", int(cfm_graph_cfg.get("max_graphs", 32)))
+                logger.info("CFM CUDA Graph enabled (max_graphs=%d)", max_graphs)
             else:
                 logger.info(
                     "CFM CUDA Graph is disabled on device type %s",
@@ -341,7 +403,9 @@ class BatchedToken2Wav(nn.Module):
         # chunk up to a multiple of this many frames so the graph cache key
         # space stays small (0 disables bucketing, e.g. when graphs are off).
         self._cfm_graph_bucket_frames = (
-            int(cfm_graph_cfg.get("bucket_frames", 0)) if self._cfm_graph_wrapper is not None else 0
+            int(cfm_graph_cfg.get("bucket_frames", 0))
+            if (self._cfm_graph_wrapper is not None or self._whole_euler_graph_wrapper is not None)
+            else 0
         )
         if self._cfm_graph_bucket_frames > 1:
             logger.info(
@@ -350,9 +414,7 @@ class BatchedToken2Wav(nn.Module):
             )
         self._prompt_features: dict[tuple[str, str], PromptFeatures] = {}
         self._setup_cache_size = setup_cache_size
-        self._setup_cache: OrderedDict[tuple[tuple[str, str], int, int], tuple[BatchedToken2WavState, ...]] = (
+        self._setup_cache: OrderedDict[tuple[tuple[str, str], int], BatchedToken2WavState] = OrderedDict()
-            OrderedDict()
-        )
 
     def _hift_inference(
         self,
@@ -507,6 +569,7 @@ class BatchedToken2Wav(nn.Module):
         attn_mask: torch.Tensor | None = None,
         valid_lengths: list[int] | None = None,
         valid_frames: int | None = None,
+        time_embedding: torch.Tensor | None = None,
     ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
         if self._trt_stepper is not None and valid_lengths is None:
             out, new_cnn, new_att = self._trt_stepper.step(
@@ -519,7 +582,8 @@ class BatchedToken2Wav(nn.Module):
                 att_cache=att_cache,
             )
             return out.to(mu.dtype), new_cnn, new_att
-        time_embedding = estimator.t_embedder(time).unsqueeze(1)
+        if time_embedding is None:
+            time_embedding = estimator.t_embedder(time).unsqueeze(1)
         width = int(x.shape[-1])
         speaker_features = speakers.unsqueeze(-1).expand(-1, -1, width)
         if valid_frames is not None and valid_frames < width:
@@ -583,14 +647,95 @@ class BatchedToken2Wav(nn.Module):
         valid_lengths: torch.Tensor,
         width: int,
     ) -> torch.Tensor:
+        """The ``width`` frames of ``history`` (b, width + t, c) ending each row's valid frames, as (b, c, width)."""
         indices = valid_lengths[:, None] + torch.arange(width, device=history.device)[None, :]
-        return history.gather(
+        return history.gather(1, indices[:, :, None].expand(-1, -1, int(history.shape[2]))).transpose(1, 2)
-            2,
-            indices[:, None, :].expand(-1, int(history.shape[1]), -1),
-        )
 
+    @staticmethod
+    def _causal_conv_frames_major(conv: nn.Conv1d, history: torch.Tensor) -> torch.Tensor:
+        """``conv`` over ``history`` (b, width + t, c_in) as (b, t, c_out), without a layout change.
+
+        The DiT keeps activations as (b, t, c), which already is NHWC for a
+        height-1 image. Upstream transposes them to (b, c, t) for ``Conv1d``,
+        and cuDNN converts that to NHWC and back around every convolution. As
+        a channels-last conv2d it reads the activations where they are. The
+        channels-last weight is converted once and reused. Other devices run
+        upstream's ``Conv1d`` layout (Ascend has no channels-last format).
+        """
+        if not history.is_cuda:
+            output = F.conv1d(
+                history.transpose(1, 2),
+                conv.weight,
+                conv.bias,
+                conv.stride,
+                conv.padding,
+                conv.dilation,
+                conv.groups,
+            )
+            return output.transpose(1, 2)
+        weight = conv.weight
+        version = (weight.data_ptr(), weight._version, weight.dtype)
+        cached = conv.__dict__.get("_channels_last_weight")
+        if cached is None or cached[0] != version:
+            cached = (version, weight.detach().unsqueeze(2).contiguous(memory_format=torch.channels_last))
+            conv.__dict__["_channels_last_weight"] = cached
+        output = F.conv2d(
+            history.transpose(1, 2).unsqueeze(2),
+            cached[1],
+            conv.bias,
+            (1, conv.stride[0]),
+            (0, conv.padding[0]),
+            (1, conv.dilation[0]),
+            conv.groups,
+        )
+        return output.squeeze(2).transpose(1, 2)
+
+    @staticmethod
+    def _attend_in_place(
+        attn: nn.Module,
+        x: torch.Tensor,
+        att_cache: torch.Tensor | None,
+        attn_mask: torch.Tensor | None,
+        kv: torch.Tensor,
+    ) -> torch.Tensor:
+        """``attn.forward_chunk`` that keeps the new keys and values in ``kv``.
+
+        ``kv`` (b, heads, >= t + cache frames, 2 * head_dim) is the block's new
+        cache, ``[current | cache]`` like upstream's. Upstream concatenates the
+        whole cache behind the current keys and values, then both into the new
+        cache, which the caller copied into ``kv``: four passes over Stage 2's
+        largest tensor. Here only the current chunk is written and attention reads ``kv`` in
+        place. A cache that already sits behind the current chunk in ``kv``
+        (Whole-Euler) is not moved; any other is copied there once. Other
+        devices keep upstream's attention and copy its cache into ``kv``.
+        """
+        if kv.dtype != x.dtype or not x.is_cuda:
+            # A reduced-precision cache attends in the activation dtype.
+            x_att, new_att = attn.forward_chunk(x, att_cache, attn_mask)
+            kv[:, :, : int(new_att.shape[2])].copy_(new_att)
+            return x_att
+        batch, frames, _ = x.shape
+        q = attn.q_norm(attn.to_heads(attn.to_q(x)))
+        k = attn.k_norm(attn.to_heads(attn.to_k(x)))
+        v = attn.to_heads(attn.to_v(x))
+        cached = 0 if att_cache is None else int(att_cache.shape[2])
+        kv = kv[:, :, : frames + cached]
+        behind = kv[:, :, frames:]
+        if cached and (att_cache.data_ptr() != behind.data_ptr() or att_cache.stride() != behind.stride()):
+            behind.copy_(att_cache)
+        head_dim = int(k.shape[3])
+        kv[..., :frames, :head_dim].copy_(k)
+        kv[..., :frames, head_dim:].copy_(v)
+        out = F.scaled_dot_product_attention(
+            q,
+            kv[..., :head_dim],
+            kv[..., head_dim:],
+            attn_mask=None if attn_mask is None else attn_mask.unsqueeze(1),
+        )
+        return attn.proj(out.transpose(1, 2).reshape(batch, frames, -1))
+
+    @staticmethod
     def _blocks_forward_chunk_ragged(
-        self,
         estimator: nn.Module,
         estimator_input: torch.Tensor,
         time_embedding: torch.Tensor,
@@ -599,14 +744,24 @@ class BatchedToken2Wav(nn.Module):
         att_cache: Any,
         cnn_cache_buffer: torch.Tensor,
         att_cache_buffer: torch.Tensor,
-        valid_lengths: list[int],
+        valid_lengths: list[int] | torch.Tensor,
     ) -> torch.Tensor:
-        """Run one padded DiT batch while capturing exact per-row CNN state."""
+        """Run one padded DiT batch while capturing exact per-row CNN state.
-        lengths = torch.tensor(
+
-            (*valid_lengths, *valid_lengths),
+        ``valid_lengths`` is one length per request, or the ``(2B,)`` CFG
-            device=estimator_input.device,
+        lengths tensor a CUDA graph captures as a static input. Upstream's
-            dtype=torch.long,
+        ``DiTBlock.forward_chunk``, with attention over the output cache in
-        )
+        place (``_attend_in_place``) and the causal convolutions on the
+        frames-major activations (``_causal_conv_frames_major``).
+        """
+        if isinstance(valid_lengths, torch.Tensor):
+            lengths = valid_lengths
+        else:
+            lengths = torch.tensor(
+                (*valid_lengths, *valid_lengths),
+                device=estimator_input.device,
+                dtype=torch.long,
+            )
         x = estimator.in_proj(estimator_input.transpose(1, 2))
         for block_index, block in enumerate(estimator.blocks):
             (
@@ -622,49 +777,55 @@ class BatchedToken2Wav(nn.Module):
             ) = block.adaLN_modulation(time_embedding).chunk(9, dim=-1)
 
             normalized = block.norm1(x) * (1 + scale_msa) + shift_msa
-            x_att, new_att = block.attn.forward_chunk(
+            x_att = BatchedToken2Wav._attend_in_place(
+                block.attn,
                 normalized,
                 att_cache[block_index],
                 attn_mask,
+                att_cache_buffer[block_index],
             )
             x = x + gate_msa * x_att
 
+            # CausalConvBlock: [T, conv1, T, LayerNorm, Mish, T, conv2, T]
+            # with the transposes (T) dropped.
+            conv = block.conv
             conv_input = block.norm3(x) * (1 + scale_conv) + shift_conv
             old_cnn = cnn_cache[block_index]
             if old_cnn is None:
-                width = int(block.conv.block[1].causal_padding[0])
                 old_cnn = conv_input.new_zeros(
                     (
                         int(conv_input.shape[0]),
-                        int(block.conv.in_channels + block.conv.out_channels),
+                        int(conv.in_channels + conv.out_channels),
-                        width,
+                        int(conv.block[1].causal_padding[0]),
                     )
                 )
-            old_cnn1, old_cnn2 = old_cnn.split((block.conv.in_channels, block.conv.out_channels), dim=1)
+            width = int(old_cnn.shape[2])
-
+            history = torch.cat((old_cnn[:, : conv.in_channels].transpose(1, 2), conv_input), dim=1)
-            conv1_input = block.conv.block[0](conv_input)
+            new_cnn1 = BatchedToken2Wav._gather_causal_cache(history, lengths, width)
-            conv1_output, _ = block.conv.block[1].forward_chunk(conv1_input, old_cnn1)
+            hidden = conv.block[4](conv.block[3](BatchedToken2Wav._causal_conv_frames_major(conv.block[1], history)))
-            new_cnn1 = self._gather_causal_cache(
+            history = torch.cat((old_cnn[:, conv.in_channels :].transpose(1, 2), hidden), dim=1)
-                torch.cat((old_cnn1, conv1_input), dim=2),
+            new_cnn2 = BatchedToken2Wav._gather_causal_cache(history, lengths, width)
-                lengths,
+            x = x + gate_conv * BatchedToken2Wav._causal_conv_frames_major(conv.block[6], history)
-                int(old_cnn1.shape[2]),
-            )
-
-            conv2_input = block.conv.block[2:6](conv1_output)
-            conv2_output, _ = block.conv.block[6].forward_chunk(conv2_input, old_cnn2)
-            new_cnn2 = self._gather_causal_cache(
-                torch.cat((old_cnn2, conv2_input), dim=2),
-                lengths,
-                int(old_cnn2.shape[2]),
-            )
-            x = x + gate_conv * block.conv.block[7](conv2_output)
             x = x + gate_mlp * block.mlp(block.norm2(x) * (1 + scale_mlp) + shift_mlp)
 
-            cnn_cache_buffer[block_index].copy_(torch.cat((new_cnn1, new_cnn2), dim=1))
+            cnn_cache_buffer[block_index][:, : conv.in_channels].copy_(new_cnn1)
-            att_cache_buffer[block_index][:, :, : int(new_att.shape[2]), :].copy_(new_att)
+            cnn_cache_buffer[block_index][:, conv.in_channels :].copy_(new_cnn2)
 
         return estimator.final_layer(x, time_embedding).transpose(1, 2)
 
+    def _get_timeline(self, device: torch.device, dtype: torch.dtype) -> tuple[torch.Tensor, list[float]]:
+        """The Euler timeline and its step sizes, read to the host once.
+
+        ``_euler_step`` takes ``dt`` as a Python float, so reading it
+        from the device timeline on every decode would sync once per step.
+        """
+        key = (device, dtype)
+        cached = self._timeline_cache.get(key)
+        if cached is None:
+            cached = _euler_timeline(self.n_timesteps, device, dtype)
+            self._timeline_cache[key] = cached
+        return cached
+
     def _decode_cfm(
         self,
         mu: torch.Tensor,
@@ -672,30 +833,44 @@ class BatchedToken2Wav(nn.Module):
         cond: torch.Tensor,
         *,
         cnn_cache: torch.Tensor | None,
-        att_cache: torch.Tensor | None,
+        att_cache: torch.Tensor | list[torch.Tensor] | None,
         valid_lengths: list[int] | None = None,
+        att_keep: tuple[int, int] | None = None,
     ) -> tuple[
         torch.Tensor,
         torch.Tensor,
         torch.Tensor | list[torch.Tensor],
     ]:
+        """Run the CFM Euler solve for one chunk.
+
+        ``att_cache`` may be the stacked estimator cache or one tensor per
+        request (the Whole-Euler path). A Whole-Euler solve returns the new
+        cache in the same form, already trimmed by ``att_keep``; the eager
+        path returns it stacked and untrimmed (ragged: per request, untrimmed).
+        """
         decoder = self.flow.decoder
         estimator = decoder.estimator
         batch_size = int(mu.shape[0])
-        offset = int(att_cache.shape[4]) if att_cache is not None else 0
+        att_rows = att_cache if isinstance(att_cache, list) else None
+        if att_rows is not None:
+            offset = int(att_rows[0].shape[4])
+        else:
+            offset = int(att_cache.shape[4]) if att_cache is not None else 0
         mel_frames = int(mu.shape[2])
+        graphs_active = valid_lengths is None and (
+            self._whole_euler_active()
+            or (
+                self._cfm_graph_wrapper is not None
+                and getattr(self._cfm_graph_wrapper, "enabled", True)
+                and self._trt_stepper is None
+            )
+        )
         pad_frames = _cfm_pad_frames(
             mel_frames=mel_frames,
             offset=offset,
             noise_capacity=int(decoder.rand_noise.shape[2]),
             bucket_frames=self._cfm_graph_bucket_frames,
-            disabled=(
+            disabled=(valid_lengths is not None or not graphs_active),
-                valid_lengths is not None
-                or self._cfm_graph_wrapper is None
-                # `_disable` keeps the wrapper object alive, so check the flag
-                # too: padding under a disabled wrapper is pure overhead.
-                or not self._cfm_graph_wrapper.enabled
-            ),
         )
         if pad_frames:
             # Replicate rather than zero: a repeated last frame is a closer
@@ -718,15 +893,7 @@ class BatchedToken2Wav(nn.Module):
             # is what removes them; zeroing alone would not, since a zero row
             # still occupies part of the softmax denominator.
             x[:, :, mel_frames:] = 0.0
-        timeline = torch.linspace(
+        timeline, dt_steps = self._get_timeline(mu.device, mu.dtype)
-            0,
-            1,
-            self.n_timesteps + 1,
-            device=mu.device,
-            dtype=mu.dtype,
-        )
-        timeline = 1 - torch.cos(timeline * 0.5 * torch.pi)
-        time = timeline[0].expand(batch_size)
         mu_cfg = torch.cat((mu, torch.zeros_like(mu)), dim=0)
         speakers_cfg = torch.cat((speakers, torch.zeros_like(speakers)), dim=0)
         cond_cfg = torch.cat((cond, torch.zeros_like(cond)), dim=0)
@@ -761,19 +928,42 @@ class BatchedToken2Wav(nn.Module):
                 device=mu.device,
             )
             attn_mask[:, :, mel_frames : mel_frames + pad_frames] = False
+
+        if self._whole_euler_active() and (valid_lengths is None or self._whole_euler_ragged_active()):
+            whole_euler_result = self._whole_euler_graph_wrapper.replay(
+                x=x,
+                mu_cfg=mu_cfg,
+                speakers_cfg=speakers_cfg,
+                cond_cfg=cond_cfg,
+                cnn_cache=cnn_cache,
+                att_cache=att_cache,
+                attn_mask=attn_mask,
+                mel_frames=mel_frames,
+                pad_frames=pad_frames,
+                valid_lengths=valid_lengths,
+                att_keep=att_keep,
+            )
+            if whole_euler_result is not None:
+                return whole_euler_result
+        if att_rows is not None:
+            att_cache = self._stack_estimator_att(att_rows)
+
         next_cnn: list[torch.Tensor] = []
         next_att_cache: torch.Tensor | None = None
         ragged_att_cache: list[torch.Tensor] | None = None
-        dt = timeline[1] - timeline[0]
+        time_embeddings = [
+            estimator.t_embedder(timeline[s].expand(2 * batch_size)).unsqueeze(1) for s in range(self.n_timesteps)
+        ]
         with _token2wav_sdpa_context(mu.device):
             for step in range(self.n_timesteps):
                 old_cnn = cnn_cache[step] if cnn_cache is not None else None
                 old_att = att_cache[step] if att_cache is not None else None
+                step_time = timeline[step].expand(2 * batch_size)
                 estimate, step_cnn, step_att = self._estimator_step(
                     estimator,
                     x=torch.cat((x, x), dim=0),
                     mu=mu_cfg,
-                    time=torch.cat((time, time), dim=0),
+                    time=step_time,
                     speakers=speakers_cfg,
                     cond=cond_cfg,
                     cnn_cache=old_cnn,
@@ -781,16 +971,13 @@ class BatchedToken2Wav(nn.Module):
                     attn_mask=attn_mask,
                     valid_lengths=valid_lengths,
                     valid_frames=mel_frames if pad_frames else None,
+                    time_embedding=time_embeddings[step],
                 )
                 if pad_frames:
                     _zero_padded_cnn_cache(step_cnn, estimator, pad_frames)
-                conditional, unconditional = estimate.split(batch_size, dim=0)
+                x = _euler_step(x, estimate, dt_steps[step], decoder.inference_cfg_rate, batch_size)
-                velocity = (1.0 + decoder.inference_cfg_rate) * conditional - decoder.inference_cfg_rate * unconditional
+                if pad_frames:
-                x = x + dt * velocity
+                    _zero_padded_frames(x, mel_frames)
-                _zero_padded_frames(x, mel_frames if pad_frames else None)
-                time = time + dt
-                if step + 1 < self.n_timesteps:
-                    dt = timeline[step + 2] - time[0]
                 next_cnn.append(step_cnn)
                 if valid_lengths is not None:
                     if ragged_att_cache is None:
@@ -842,28 +1029,40 @@ class BatchedToken2Wav(nn.Module):
         assert next_att_cache is not None
         return x, torch.stack(next_cnn), next_att_cache
 
-    def _split_flow_cache(self, cache: dict[str, torch.Tensor], batch_size: int) -> list[dict[str, torch.Tensor]]:
+    def _split_flow_cache(
+        self,
+        cache: dict[str, Any],
+        batch_size: int,
+        *,
+        estimator_att_keep: tuple[int, int] | None = None,
+    ) -> list[dict[str, torch.Tensor]]:
+        """Per-request flow caches from a batch's stacked caches.
+
+        ``estimator_att_cache`` is stacked ``(n_t, depth, 2B, heads, L, d)``
+        with CFG rows ``[cond x B | uncond x B]``, or already one tensor per
+        request (Whole-Euler). The streaming trim ``estimator_att_keep`` is
+        applied while each request is copied out, so no trimmed copy of the
+        whole batch is built first: at 16 requests that copy alone was ~7 GiB.
+        """
+        estimator_att = cache["estimator_att_cache"]
+        stacked_att = isinstance(estimator_att, torch.Tensor)
+        if stacked_att:
+            keep_ranges = _att_keep_ranges(int(estimator_att.shape[4]), estimator_att_keep)
+            request_shape = list(estimator_att.shape)
+            request_shape[2] = 2
+            request_shape[4] = sum(length for _, length in keep_ranges)
+            att_rows = estimator_att.unflatten(2, (2, batch_size))
         result: list[dict[str, torch.Tensor]] = []
         for row in range(batch_size):
-            estimator_att = cache["estimator_att_cache"]
+            if stacked_att:
-            if estimator_att.dtype == self._estimator_att_cache_dtype:
-                request_att = torch.cat(
-                    (
-                        estimator_att[:, :, row : row + 1],
-                        estimator_att[:, :, batch_size + row : batch_size + row + 1],
-                    ),
-                    dim=2,
-                ).detach()
-            else:
-                request_shape = list(estimator_att.shape)
-                request_shape[2] = 2
                 request_att = torch.empty(
                     request_shape,
                     device=estimator_att.device,
                     dtype=self._estimator_att_cache_dtype,
                 )
-                request_att[:, :, 0:1].copy_(estimator_att[:, :, row : row + 1])
+                _copy_frame_segments(request_att, att_rows[:, :, :, row], keep_ranges)
-                request_att[:, :, 1:2].copy_(estimator_att[:, :, batch_size + row : batch_size + row + 1])
+            else:
+                request_att = estimator_att[row]
             result.append(
                 {
                     "conformer_cnn_cache": cache["conformer_cnn_cache"][row : row + 1].detach().clone(),
@@ -880,20 +1079,45 @@ class BatchedToken2Wav(nn.Module):
             )
         return result
 
-    def _stack_flow_cache(self, states: list[BatchedToken2WavState]) -> dict[str, torch.Tensor]:
+    @staticmethod
+    def _stack_estimator_att(rows: list[torch.Tensor]) -> torch.Tensor:
+        """Stack per-request ``(n_t, depth, 2, ...)`` caches into CFG order ``[cond x B | uncond x B]``."""
+        return torch.cat((*[row[:, :, 0:1] for row in rows], *[row[:, :, 1:2] for row in rows]), dim=2)
+
+    def _stack_flow_cache(
+        self,
+        states: list[BatchedToken2WavState],
+        *,
+        include_estimator_att: bool = True,
+    ) -> dict[str, torch.Tensor | None]:
+        """Stack per-request flow caches for one batched call.
+
+        The estimator attention cache is by far the largest (~0.5 GiB per
+        request in fp32). The Whole-Euler path reads it per request, so it
+        skips stacking it (``include_estimator_att=False``).
+        """
         flows = [state.flow_cache for state in states]
         conditional_cnn = [flow["estimator_cnn_cache"][:, :, 0:1] for flow in flows]
         unconditional_cnn = [flow["estimator_cnn_cache"][:, :, 1:2] for flow in flows]
-        conditional_att = [flow["estimator_att_cache"][:, :, 0:1] for flow in flows]
-        unconditional_att = [flow["estimator_att_cache"][:, :, 1:2] for flow in flows]
-        estimator_att = torch.cat((*conditional_att, *unconditional_att), dim=2)
         return {
             "conformer_cnn_cache": torch.cat([flow["conformer_cnn_cache"] for flow in flows], dim=0),
             "conformer_att_cache": torch.cat([flow["conformer_att_cache"] for flow in flows], dim=1),
             "estimator_cnn_cache": torch.cat((*conditional_cnn, *unconditional_cnn), dim=2),
-            "estimator_att_cache": estimator_att,
+            "estimator_att_cache": (
+                self._stack_estimator_att([flow["estimator_att_cache"] for flow in flows])
+                if include_estimator_att
+                else None
+            ),
         }
 
+    def _whole_euler_active(self) -> bool:
+        wrapper = self._whole_euler_graph_wrapper
+        return wrapper is not None and wrapper.enabled and self._trt_stepper is None
+
+    def _whole_euler_ragged_active(self) -> bool:
+        """Whether Whole-Euler graphs also solve ragged (per-row length) batches."""
+        return self._whole_euler_active() and self._whole_euler_graph_wrapper.ragged_body is not None
+
     def _create_initial_states(
         self,
         features: PromptFeatures,
@@ -913,12 +1137,15 @@ class BatchedToken2Wav(nn.Module):
                 att_cache=None,
             )
             projected_speakers = self.flow.spk_embed_affine_layer(F.normalize(speakers, dim=1))
+            # The prompt cache never reaches the trim length; ``att_keep`` only
+            # tells the Whole-Euler arena the steady cache length to reserve.
             _, estimator_cnn, estimator_att = self._decode_cfm(
                 hidden.transpose(1, 2).contiguous(),
                 projected_speakers,
                 prompt_mels.transpose(1, 2).contiguous(),
                 cnn_cache=None,
                 att_cache=None,
+                att_keep=(int(prompt_mels.shape[1]), _CACHE_TRIM_SUFFIX),
             )
         flow_cache = {
             "conformer_cnn_cache": conformer_cnn,
@@ -941,7 +1168,17 @@ class BatchedToken2Wav(nn.Module):
         ]
 
     def setup_batch(self, features: PromptFeatures, batch_size: int) -> list[BatchedToken2WavState]:
-        """Reuse read-only prompt conditioning for the same shape and padding."""
+        """Initial states for ``batch_size`` requests sharing one prompt.
+
+        Every request starts from the same prompt tokens, speaker, mels and
+        noise, so the prompt is solved once and all requests share that one
+        read-only state. Decoding writes a state only where Whole-Euler updates
+        an estimator cache in place, which needs a cache held by one request
+        and allocated with room for its new length; the prompt cache is shared
+        and allocated exactly, and the first chunk always grows it. Solving it
+        per batch size recomputed identical rows and cached one ~0.3 GiB
+        estimator cache copy per request.
+        """
         bucket_frames = (
             self._cfm_graph_bucket_frames
             if self._cfm_graph_wrapper is not None and self._cfm_graph_wrapper.enabled
@@ -949,17 +1186,17 @@ class BatchedToken2Wav(nn.Module):
         )
         # Capture the policy before setup: graph capture may disable the wrapper
         # after padding has already been chosen for these initial states.
-        cache_key = (features.cache_key, batch_size, bucket_frames)
+        cache_key = (features.cache_key, bucket_frames)
-        cached = self._setup_cache.get(cache_key)
+        state = self._setup_cache.get(cache_key)
-        if cached is not None:
+        if state is not None:
             self._setup_cache.move_to_end(cache_key)
-            return list(cached)
+        else:
-        states = self._create_initial_states(features, batch_size)
+            (state,) = self._create_initial_states(features, 1)
-        if self._setup_cache_size > 0:
+            if self._setup_cache_size > 0:
-            self._setup_cache[cache_key] = tuple(states)
+                self._setup_cache[cache_key] = state
-            while len(self._setup_cache) > self._setup_cache_size:
+                while len(self._setup_cache) > self._setup_cache_size:
-                self._setup_cache.popitem(last=False)
+                    self._setup_cache.popitem(last=False)
-        return states
+        return [state] * batch_size
 
     @staticmethod
     def _fade_in_out(
@@ -1058,7 +1295,12 @@ class BatchedToken2Wav(nn.Module):
         flush_encoder: bool = False,
     ) -> tuple[list[torch.Tensor], list[BatchedToken2WavState]]:
         batch_size = int(tokens.shape[0])
-        flow_cache = self._stack_flow_cache(states)
+        # Whole-Euler reads and writes each request's estimator cache directly,
+        # so the batch never holds a stacked copy of it.
+        per_request_att = self._whole_euler_active()
+        flow_cache = self._stack_flow_cache(states, include_estimator_att=not per_request_att)
+        prompt_len = int(features.mels.shape[1])
+        att_keep = (prompt_len, _CACHE_TRIM_SUFFIX)
         speakers = features.speaker_embedding.expand(batch_size, -1)
         with self._autocast(tokens.device):
             hidden, conformer_cnn, conformer_att = self._encode_chunk(
@@ -1074,20 +1316,15 @@ class BatchedToken2Wav(nn.Module):
                 projected_speakers,
                 cond,
                 cnn_cache=flow_cache["estimator_cnn_cache"],
-                att_cache=flow_cache["estimator_att_cache"],
+                att_cache=(
+                    [state.flow_cache["estimator_att_cache"] for state in states]
+                    if per_request_att
+                    else flow_cache["estimator_att_cache"]
+                ),
+                att_keep=att_keep,
             )
 
-        prompt_len = int(features.mels.shape[1])
+        conformer_att = _trim_streaming_cache(conformer_att, prompt_len)
-        if estimator_att.shape[4] > prompt_len + 100:
-            estimator_att = torch.cat(
-                (estimator_att[..., :prompt_len, :], estimator_att[..., -100:, :]),
-                dim=4,
-            )
-        if conformer_att.shape[3] > prompt_len + 100:
-            conformer_att = torch.cat(
-                (conformer_att[..., :prompt_len, :], conformer_att[..., -100:, :]),
-                dim=3,
-            )
         new_flow = self._split_flow_cache(
             {
                 "conformer_cnn_cache": conformer_cnn,
@@ -1096,6 +1333,7 @@ class BatchedToken2Wav(nn.Module):
                 "estimator_att_cache": estimator_att,
             },
             batch_size,
+            estimator_att_keep=att_keep,
         )
         old_mel = torch.cat([state.hift_cache["mel"] for state in states], dim=0)
         old_source = torch.cat([state.hift_cache["source"] for state in states], dim=0)
@@ -1164,10 +1402,17 @@ class BatchedToken2Wav(nn.Module):
                 )
 
         max_frames = self._max_encode_token_frames(states)
-        if any(int(row_tokens.numel()) > max_frames for row_tokens in tokens):
+        per_request_att = self._whole_euler_ragged_active()
+        if (self._whole_euler_active() and not per_request_att) or any(
+            int(row_tokens.numel()) > max_frames for row_tokens in tokens
+        ):
             # decode_batch owns the RelPos-safe encoder slicing used by
-            # non-async Talker dumps. Preserve that path for overlong rows;
+            # non-async Talker dumps. Preserve that path for overlong rows.
-            # ordinary short duplex chunks continue through one padded DiT.
+            # Whole-Euler graphs that cannot run the ragged kernel also take
+            # this path: under concurrency most batches hold some request's
+            # short final chunk, and one padded eager DiT would run the whole
+            # batch ungraphed on a stacked copy of every cache. Otherwise short
+            # duplex chunks share one padded DiT solve, graphed when possible.
             exact_groups: dict[tuple[int, bool], list[int]] = {}
             for row, (row_tokens, last_chunk) in enumerate(zip(tokens, last_chunks, strict=True)):
                 exact_groups.setdefault((int(row_tokens.numel()), last_chunk), []).append(row)
@@ -1193,7 +1438,8 @@ class BatchedToken2Wav(nn.Module):
         conformer_att_rows: list[torch.Tensor | None] = [None] * batch_size
         for (_, last_chunk), rows in encoder_groups.items():
             group_states = [states[row] for row in rows]
-            group_cache = self._stack_flow_cache(group_states)
+            # The encoder reads only the conformer caches.
+            group_cache = self._stack_flow_cache(group_states, include_estimator_att=False)
             group_tokens = torch.stack([tokens[row] for row in rows], dim=0)
             with self._autocast(group_tokens.device):
                 hidden, conformer_cnn, conformer_att = self._encode_chunk(
@@ -1219,7 +1465,10 @@ class BatchedToken2Wav(nn.Module):
         for row, hidden in enumerate(resolved_hidden):
             padded_hidden[row, : int(hidden.shape[1])].copy_(hidden[0])
 
-        flow_cache = self._stack_flow_cache(states)
+        # A graphed ragged solve reads and writes each request's estimator
+        # cache directly (trimmed on the way out), like _decode_batch_once.
+        flow_cache = self._stack_flow_cache(states, include_estimator_att=not per_request_att)
+        prompt_len = int(features.mels.shape[1])
         speakers = features.speaker_embedding.expand(batch_size, -1)
         with self._autocast(padded_hidden.device):
             projected_speakers = self.flow.spk_embed_affine_layer(F.normalize(speakers, dim=1))
@@ -1229,33 +1478,24 @@ class BatchedToken2Wav(nn.Module):
                 projected_speakers,
                 cond,
                 cnn_cache=flow_cache["estimator_cnn_cache"],
-                att_cache=flow_cache["estimator_att_cache"],
+                att_cache=(
+                    [state.flow_cache["estimator_att_cache"] for state in states]
+                    if per_request_att
+                    else flow_cache["estimator_att_cache"]
+                ),
                 valid_lengths=hidden_lengths,
+                att_keep=(prompt_len, _CACHE_TRIM_SUFFIX),
             )
 
-        prompt_len = int(features.mels.shape[1])
         assert isinstance(estimator_att, list)
         new_flow: list[dict[str, torch.Tensor]] = []
         for row, row_estimator_att in enumerate(estimator_att):
             conformer_cnn = conformer_cnn_rows[row]
             conformer_att = conformer_att_rows[row]
             assert conformer_cnn is not None and conformer_att is not None
-            if row_estimator_att.shape[4] > prompt_len + 100:
+            # A graphed solve already trimmed the estimator cache; trimming is idempotent.
-                row_estimator_att = torch.cat(
+            row_estimator_att = _trim_streaming_cache(row_estimator_att, prompt_len)
-                    (
+            conformer_att = _trim_streaming_cache(conformer_att, prompt_len)
-                        row_estimator_att[..., :prompt_len, :],
-                        row_estimator_att[..., -100:, :],
-                    ),
-                    dim=4,
-                )
-            if conformer_att.shape[3] > prompt_len + 100:
-                conformer_att = torch.cat(
-                    (
-                        conformer_att[..., :prompt_len, :],
-                        conformer_att[..., -100:, :],
-                    ),
-                    dim=3,
-                )
             new_flow.append(
                 {
                     "conformer_cnn_cache": conformer_cnn.detach().clone(),

vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py

@@ -268,14 +268,34 @@ class MiniCPMO45Code2Wav(nn.Module):
             raise ValueError(f"Invalid MiniCPM-o connector chunk configuration: {self._connector_config}")
         raw_capture_batch_sizes = extra.get("hift_graph_capture_batch_sizes")
         capture_batch_sizes = [1] if raw_capture_batch_sizes is None else raw_capture_batch_sizes
+        max_serial_batch = extra.get("max_serial_batch")
+        max_serial_batch = 4 if max_serial_batch is None else int(max_serial_batch)
         self._hift_graph_config = {
             "enabled": bool(extra.get("enable_hift_graph", False)),
             "capture_batch_sizes": capture_batch_sizes,
+            "max_serial_batch": max_serial_batch,
         }
+        enable_whole_euler = extra.get("enable_whole_euler")
+        max_graph_batch_raw = extra.get("max_graph_batch")
+        max_graph_batch = int(max_graph_batch_raw) if max_graph_batch_raw is not None else None
+        micro_batch_size_raw = extra.get("micro_batch_size")
+        if micro_batch_size_raw is not None:
+            micro_batch_size = int(micro_batch_size_raw)
+        else:
+            # The Whole-Euler arena reserves one attention cache per micro-batch
+            # row, so size it for the most requests this stage ever batches.
+            max_num_seqs = getattr(getattr(vllm_config, "scheduler_config", None), "max_num_seqs", None)
+            micro_batch_size = min(int(max_num_seqs), max_graph_batch or 16) if max_num_seqs else None
         self._cfm_graph_config = {
             "enabled": bool(extra.get("enable_cfm_graph", False)),
             "max_graphs": int(extra.get("cfm_max_graphs", 32)),
             "bucket_frames": int(extra.get("cfm_graph_bucket_frames", 0)),
+            "capture_frames": extra.get("cfm_graph_capture_frames"),
+            "enable_whole_euler": enable_whole_euler is None or bool(enable_whole_euler),
+            "max_serial_batch": max_serial_batch,
+            "max_graph_batch": max_graph_batch,
+            "micro_batch_size": micro_batch_size,
+            "pad_max_rows": extra.get("whole_euler_pad_max_rows"),
         }
         self._ref_max_seconds = float(extra.get("ref_audio_max_seconds", _REF_MAX_SECONDS))
         if self._ref_max_seconds <= 0:

vllm_omni/platforms/npu/models/minicpmo_4_5_code2wav.py

@@ -114,6 +114,7 @@ def _patched_estimator_step(
     attn_mask=None,
     valid_lengths=None,
     valid_frames=None,
+    time_embedding=None,
 ):
     assert _original_estimator_step is not None
     graph_runner = _backend_graph_runners.get(self)
@@ -137,6 +138,7 @@ def _patched_estimator_step(
             attn_mask=attn_mask,
             valid_lengths=valid_lengths,
             valid_frames=valid_frames,
+            time_embedding=time_embedding,
         )
     if (cnn_cache is None) != (att_cache is None):
         raise ValueError("estimator CNN and attention caches must both be present or absent")
@@ -148,7 +150,8 @@ def _patched_estimator_step(
 
     # The upstream embedder creates a frequency tensor on the host. Keep it
     # outside capture while retaining the tensor-only estimator body in graph.
-    time_embedding = estimator.t_embedder(time).unsqueeze(1)
+    if time_embedding is None:
+        time_embedding = estimator.t_embedder(time).unsqueeze(1)
     if cnn_cache is None:
         return graph_runner.run(
             "cfm_estimator",