[Perf] Optimize Higgs Audio v3 serving (#4204)

Signed-off-by: Sy03 <1370724210@qq.com>
Signed-off-by: Yueqian Lin <linyueqian@outlook.com>
Co-authored-by: Yueqian Lin <linyueqian@outlook.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>

.buildkite/test-nightly.yml

@@ -568,6 +568,55 @@ steps:
                       path: /mnt/hf-cache
                       type: DirectoryOrCreate
 
+  - label: ":full_moon: TTS · Higgs-Audio-V3 · Perf Test"
+    key: nightly-tts-performance-higgs-audio-v3
+    depends_on: upload-nightly-pipeline
+    if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "tts-test"
+    timeout_in_minutes: 180
+    commands:
+      - export BENCHMARK_DIR=tests/dfx/perf/results
+      - export VLLM_USE_DEEP_GEMM="0"
+      - export VLLM_MOE_USE_DEEP_GEMM="0"
+      - |
+        set +e
+        pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_higgs_audio_v3.json
+        EXIT=$$?
+        buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
+        exit $$EXIT
+    agents:
+      queue: "mithril-h100-pool"
+    plugins:
+      - kubernetes:
+          podSpec:
+            containers:
+              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                resources:
+                  limits:
+                    nvidia.com/gpu: 1
+                volumeMounts:
+                  - name: devshm
+                    mountPath: /dev/shm
+                  - name: hf-cache
+                    mountPath: /root/.cache/huggingface
+                env:
+                  - name: HF_HOME
+                    value: /root/.cache/huggingface
+                  - name: HF_TOKEN
+                    valueFrom:
+                      secretKeyRef:
+                        name: hf-token-secret
+                        key: token
+            nodeSelector:
+              node.kubernetes.io/instance-type: gpu-h100-sxm
+            volumes:
+              - name: devshm
+                emptyDir:
+                  medium: Memory
+              - name: hf-cache
+                hostPath:
+                  path: /mnt/hf-cache
+                  type: DirectoryOrCreate
+
   # Diffusion X2I suite: x2i / x2a / x2t and related non-video paths; x2v is only in "Diffusion X2V Model Test" below.
   - group: ":card_index_dividers: Diffusion X2I(&A&T) Model Test"
     key: nightly-diffusion-x2iat-group
@@ -1463,6 +1512,7 @@ steps:
       - nightly-omni-performance-vllm-text
       - nightly-omni-performance-multi-replicas
       - nightly-tts-performance
+      - nightly-tts-performance-higgs-audio-v3
       - nightly-diffusion-x2iat-performance-qwen-image
       - nightly-diffusion-x2iat-performance-qwen-image-edit
       - nightly-diffusion-x2iat-performance-qwen-image-edit-2511
@@ -1476,6 +1526,7 @@ steps:
       - export DEFAULT_INPUT_DIR=tests/dfx/perf/results
       - export DEFAULT_OUTPUT_DIR=tests/dfx/perf/results
       - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance
+      - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance-higgs-audio-v3
       - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-no-async-chunk
       - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-async-chunk
       - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-vllm-text

benchmarks/tts/model_configs.yaml

@@ -33,6 +33,15 @@ models:
       voice_clone:
         task_type: Base
 
+  bosonai/higgs-audio-v3-tts-4b:
+    supported_tasks: [default_voice, voice_clone]
+    backend: openai-audio-speech
+    endpoint: /v1/audio/speech
+    trust_remote_code: true
+    task_extra_body:
+      default_voice: {}
+      voice_clone: {}
+
   openbmb/VoxCPM2:
     supported_tasks: [voice_clone, default_voice]
     trust_remote_code: true

recipes/BosonAI/Higgs-Audio-V3-TTS.md

@@ -52,6 +52,7 @@ vllm-omni serve bosonai/higgs-audio-v3-tts-4b \
 
 The default deploy config `vllm_omni/deploy/higgs_multimodal_qwen3.yaml` is
 loaded automatically by model registry (HF `model_type=higgs_multimodal_qwen3`).
+It matches the high-throughput profile.
 
 **Offline batch inference:**
 
@@ -103,8 +104,28 @@ python examples/online_serving/text_to_speech/higgs_audio_v3/batch_speech_client
 - Voice cloning: `ref_audio` accepts WAV/FLAC/MP3; `ref_text` is optional but improves fidelity.
 - Deploy config: `vllm_omni/deploy/higgs_multimodal_qwen3.yaml` (auto-discovered from `model_type`).
   - `max_num_seqs=16` for both stages.
-  - `enforce_eager=true` for both stages (CUDA graph available for stage 0 but no throughput gain at batch>1).
+  - Stage 0 and Stage 1 default to the same device (`0`) for single-GPU serving.
+  - Stage 0 intentionally keeps `enforce_eager=true`. This preserves the Higgs-specific local MLP CUDA graph path, which is the current high-throughput default.
+  - Stage 1 remains `enforce_eager=true` for the codec decoder.
+- Deploy profiles:
+  - High throughput: `vllm_omni/deploy/higgs_multimodal_qwen3_high_throughput.yaml`.
+    Use this for medium/high concurrency. The auto-discovered
+    `higgs_multimodal_qwen3.yaml` is kept as a compatibility/default alias for
+    this profile.
+  - Low latency: `vllm_omni/deploy/higgs_multimodal_qwen3_low_latency.yaml`.
+    Use this for low-concurrency serving (for example c1-c4). It sets Stage 0
+    `enforce_eager=false` and enables vLLM `FULL_DECODE_ONLY` CUDA graph through
+    YAML `compilation_config`; no environment variable is required.
+  - Profile details: `vllm_omni/deploy/README_higgs_audio_v3.md`.
+- Performance note:
+  - Do not switch the auto-discovered default Stage 0 profile to vLLM
+    `FULL_DECODE_ONLY` CUDA graph without an end-to-end throughput and
+    audio-quality revalidation. On the H20 SeedTTS c16/full-dataset benchmark,
+    the eager Stage 0 path with Higgs local MLP CUDA graph reproduced
+    ~35 audio_s/s (`1088/1088` OK per run, three runs around 134-136s). A
+    separate H20 low-concurrency smoke after the FULL_DECODE audio-feedback
+    capture fix showed lower c1/c4 latency, so the FULL_DECODE profile is kept
+    as an explicit low-latency option rather than the throughput default.
 - Known limitations:
-  - CUDA graph for stage 0 helps batch=1 (-13% RTF) but not batch>1 due to model-owned sampler running outside graph.
   - Stage 1 (code2wav) must use `enforce_eager=true` (`@torch.inference_mode` incompatible with graph capture).
-  - Async streaming (chunk-based) not yet implemented; sync-only pipeline.
+  - Stage 0 full-decode CUDA graph is experimental; sampler, delay-state updates, staging, and request postprocess remain outside the graph.

tests/dfx/perf/tests/test_higgs_audio_v3.json

@@ -0,0 +1,74 @@
+[
+  {
+    "test_name": "test_higgs_audio_v3",
+    "server_params": {
+      "model": "bosonai/higgs-audio-v3-tts-4b",
+      "stage_overrides": {
+        "0": {
+          "gpu_memory_utilization": 0.3
+        },
+        "1": {
+          "gpu_memory_utilization": 0.2
+        }
+      },
+      "extra_cli_args": [
+        "--deploy-config",
+        "vllm_omni/deploy/higgs_multimodal_qwen3.yaml",
+        "--trust-remote-code"
+      ]
+    },
+    "benchmark_params": [
+      {
+        "task": "default_voice",
+        "eval_phase": "latency",
+        "dataset_name": "seed-tts-text",
+        "backend": "openai-audio-speech",
+        "endpoint": "/v1/audio/speech",
+        "dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
+        "num_prompts": [
+          20
+        ],
+        "max_concurrency": [
+          1
+        ],
+        "seed_tts_locale": "en",
+        "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
+        "baseline": {
+          "median_audio_ttfp_ms": [
+            3000
+          ],
+          "median_audio_rtf": [
+            0.5
+          ]
+        }
+      },
+      {
+        "task": "default_voice",
+        "eval_phase": "throughput",
+        "dataset_name": "seed-tts-text",
+        "backend": "openai-audio-speech",
+        "endpoint": "/v1/audio/speech",
+        "dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
+        "num_prompts": [
+          500
+        ],
+        "max_concurrency": [
+          16
+        ],
+        "seed_tts_locale": "en",
+        "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
+        "baseline": {
+          "median_audio_ttfp_ms": [
+            5000
+          ],
+          "median_audio_rtf": [
+            1.2
+          ],
+          "audio_throughput": [
+            30
+          ]
+        }
+      }
+    ]
+  }
+]

tests/distributed/omni_connectors/test_adapter_and_flow.py

@@ -145,7 +145,7 @@ def test_shm_connector_flow(mocker: MockerFixture):
     Using real SharedMemoryConnector (inline mode for simplicity).
     """
     # 1. Setup Connector
-    config = {"shm_threshold_bytes": 1024}  # Large threshold to use inline
+    config = {"shm_threshold_bytes": 1024, "inline_small_payloads": True}
     connector = SharedMemoryConnector(config)
     connectors_map = {("0", "1"): connector}
 

tests/distributed/omni_connectors/test_basic_connectors.py

@@ -75,18 +75,21 @@ def test_create_unknown_connector():
 
 @pytest.fixture
 def shm_connector():
-    config = {"shm_threshold_bytes": 100}  # Small threshold for testing
+    config = {"shm_threshold_bytes": 100, "inline_small_payloads": True}
     return SharedMemoryConnector(config)
 
 
 def test_put_get_inline(shm_connector):
-    """Test transfer for small data (currently always uses SHM)."""
+    """Test inline transfer for small data."""
     data = {"small": "data"}
 
     success, size, metadata = shm_connector.put("stage_0", "stage_1", "req_1", data)
     assert success is True
-    assert "shm" in metadata
+    assert "inline_bytes" in metadata
+    assert "shm" not in metadata
     assert "size" in metadata
+    assert shm_connector._metrics["inline_writes"] == 1
+    assert shm_connector._metrics["shm_writes"] == 0
 
     # Retrieve
     retrieved_data, ret_size = shm_connector.get("stage_0", "stage_1", "req_1", metadata)

tests/distributed/omni_connectors/test_shm_connector.py

@@ -13,7 +13,7 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
 
 @pytest.fixture()
 def connector():
-    c = SharedMemoryConnector({"shm_threshold_bytes": 64})
+    c = SharedMemoryConnector({"shm_threshold_bytes": 0})
     yield c
     c.close()
 

tests/unit/higgs_audio_v3/test_higgs_audio_v3.py

@@ -6,6 +6,10 @@ AC-5 (delay pattern), AC-7 (stage processor), and AC-10 (registry)
 without requiring the actual checkpoint or GPU.
 """
 
+import asyncio
+import time
+from collections import OrderedDict
+
 import pytest
 import torch
 
@@ -232,7 +236,7 @@ class TestDelayPatternBehavior:
 
 
 class TestSamplerMethods:
-    """Test the actual _sample_audio_codes and _apply_delay_pattern_masking methods."""
+    """Test the actual sampler and batched delay masking methods."""
 
     def _make_minimal_talker(self):
         """Create a minimal talker-like object with sampler/masking methods."""
@@ -241,18 +245,65 @@ class TestSamplerMethods:
         class FakeTalker:
             num_codebooks = 8
             codebook_size = 1026
-            _audio_state = {}
 
         t = FakeTalker()
         t._sample_audio_codes = mod.HiggsAudioV3TalkerForConditionalGeneration._sample_audio_codes.__get__(t)
-        t._apply_delay_pattern_masking = (
-            mod.HiggsAudioV3TalkerForConditionalGeneration._apply_delay_pattern_masking.__get__(t)
+        return t
+
+    def _make_batched_sampler_talker(self, num_rows=4):
+        """Create a fake talker with GPU-resident sampler state helpers."""
+        from vllm_omni.model_executor.models.higgs_audio_v3 import higgs_audio_v3_talker as mod
+
+        class FakeTalker:
+            num_codebooks = 8
+            codebook_size = 1026
+
+        t = FakeTalker()
+        t._decode_last_codes = torch.arange(num_rows * t.num_codebooks, dtype=torch.long).view(
+            num_rows, t.num_codebooks
         )
+        t._decode_has_codes = torch.tensor([True, False, True, True])[:num_rows].clone()
+        t._decode_delay_count = torch.tensor([0, 2, 7, 4], dtype=torch.int32)[:num_rows].clone()
+        t._decode_eoc_countdown = torch.tensor([-1, -1, 3, 1], dtype=torch.int32)[:num_rows].clone()
+        t._decode_generation_done = torch.tensor([False, False, False, True])[:num_rows].clone()
+        t._decode_active_audio_count = 0
+        t._codebook_index_cache = {}
+        t._row_index_cache = {}
+        t._boc_frame_cache = {}
+        t._last_audio_codes_buffer = None
+        t._last_audio_host_staging = None
+        t._last_audio_gpu_staging = None
+        t._last_audio_staging_event = None
+        t._audio_staging_events = []
+        t._audio_staging_event_cursor = 0
+        t._last_audio_codes = None
+        t._last_audio_code_valid = []
+        t._postprocess_cursor = 0
+        t._postprocess_audio_rows = 0
+        t._postprocess_audio_active_rows = 0
+
+        cls = mod.HiggsAudioV3TalkerForConditionalGeneration
+        for name in (
+            "_sample_audio_codes",
+            "_ensure_decode_state_capacity",
+            "_get_row_indices",
+            "_get_codebook_indices",
+            "_get_boc_frames",
+            "_get_audio_codes_buffer",
+            "_get_audio_gpu_staging_buffer",
+            "_get_audio_host_staging_buffer",
+            "_apply_delay_pattern_masking_batched",
+            "_update_delay_state_batched",
+            "_prefill_row_mask",
+            "_audio_seed_mask_from_step_input",
+        ):
+            setattr(t, name, getattr(cls, name).__get__(t))
+        t._device_cache_key = cls._device_cache_key
         return t
 
     def test_sample_respects_mask(self):
         """Tokens masked to -inf must never be sampled."""
-        t = self._make_minimal_talker()
+        t = self._make_batched_sampler_talker(num_rows=1)
         logits = torch.full((10, 1026), float("-inf"))
         # Only token 500 is allowed for each row
         logits[:, 500] = 0.0
@@ -262,7 +313,7 @@ class TestSamplerMethods:
 
     def test_sample_all_masked_falls_back_to_argmax(self):
         """All-masked row should fall back to argmax (least-negative logit)."""
-        t = self._make_minimal_talker()
+        t = self._make_batched_sampler_talker(num_rows=1)
         logits = torch.full((2, 1026), float("-inf"))
         # Row 0: all masked
         # Row 1: only token 42 allowed
@@ -275,11 +326,13 @@ class TestSamplerMethods:
         """During delay phase, codebooks beyond delay_count must have only BOC allowed."""
         from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import BOC_ID
 
-        t = self._make_minimal_talker()
+        t = self._make_batched_sampler_talker(num_rows=1)
         # Simulate delay_count=2 for batch row 0
-        t._audio_state[0] = {"num_delay": 2, "num_remaining_delays": None}
+        t._ensure_decode_state_capacity(1, torch.device("cpu"))
+        t._decode_delay_count[0] = 2
+        t._decode_eoc_countdown[0] = -1
         cb_logits = torch.zeros(1, 8, 1026)  # [1 audio row, 8 codebooks, 1026 vocab]
-        t._apply_delay_pattern_masking(cb_logits, [0])
+        t._apply_delay_pattern_masking_batched(cb_logits, torch.tensor([0], dtype=torch.long))
         # CBs 3-7 should have only BOC allowed (everything else -inf)
         for q in range(3, 8):
             row = cb_logits[0, q]
@@ -292,10 +345,12 @@ class TestSamplerMethods:
         """Active codebooks during delay should have BOC disallowed."""
         from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import BOC_ID
 
-        t = self._make_minimal_talker()
-        t._audio_state[0] = {"num_delay": 3, "num_remaining_delays": None}
+        t = self._make_batched_sampler_talker(num_rows=1)
+        t._ensure_decode_state_capacity(1, torch.device("cpu"))
+        t._decode_delay_count[0] = 3
+        t._decode_eoc_countdown[0] = -1
         cb_logits = torch.zeros(1, 8, 1026)
-        t._apply_delay_pattern_masking(cb_logits, [0])
+        t._apply_delay_pattern_masking_batched(cb_logits, torch.tensor([0], dtype=torch.long))
         # CBs 0-3 are active: BOC should be -inf
         for q in range(4):
             assert cb_logits[0, q, BOC_ID].item() == float("-inf")
@@ -304,10 +359,12 @@ class TestSamplerMethods:
         """Only codebook 0 should allow EOC during normal generation."""
         from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import EOC_ID
 
-        t = self._make_minimal_talker()
-        t._audio_state[0] = {"num_delay": 8, "num_remaining_delays": None}
+        t = self._make_batched_sampler_talker(num_rows=1)
+        t._ensure_decode_state_capacity(1, torch.device("cpu"))
+        t._decode_delay_count[0] = 8
+        t._decode_eoc_countdown[0] = -1
         cb_logits = torch.zeros(1, 8, 1026)
-        t._apply_delay_pattern_masking(cb_logits, [0])
+        t._apply_delay_pattern_masking_batched(cb_logits, torch.tensor([0], dtype=torch.long))
         # CB0 should keep EOC
         assert cb_logits[0, 0, EOC_ID].item() != float("-inf")
         # CB1-7 should have EOC masked
@@ -321,11 +378,13 @@ class TestSamplerMethods:
             EOC_ID,
         )
 
-        t = self._make_minimal_talker()
+        t = self._make_batched_sampler_talker(num_rows=1)
         # Ramp-down with 4 remaining delays: lock CBs 0-3 to EOC
-        t._audio_state[0] = {"num_delay": 8, "num_remaining_delays": 4}
+        t._ensure_decode_state_capacity(1, torch.device("cpu"))
+        t._decode_delay_count[0] = 8
+        t._decode_eoc_countdown[0] = 4
         cb_logits = torch.zeros(1, 8, 1026)
-        t._apply_delay_pattern_masking(cb_logits, [0])
+        t._apply_delay_pattern_masking_batched(cb_logits, torch.tensor([0], dtype=torch.long))
         # CBs 0-3 locked: only EOC allowed
         for q in range(4):
             row = cb_logits[0, q]
@@ -337,6 +396,195 @@ class TestSamplerMethods:
             assert cb_logits[0, q, BOC_ID].item() == float("-inf")
             assert cb_logits[0, q, EOC_ID].item() == float("-inf")
 
+    def test_batched_delay_masking_all_rows_matches_sparse_path(self):
+        """The all-row fast path must preserve sparse path masking semantics."""
+        torch.manual_seed(0)
+        sparse = self._make_batched_sampler_talker()
+        all_rows = self._make_batched_sampler_talker()
+        rows = torch.arange(4, dtype=torch.long)
+        logits_sparse = torch.randn(4, 8, 1026)
+        logits_all_rows = logits_sparse.clone()
+
+        sparse._apply_delay_pattern_masking_batched(logits_sparse, rows, all_rows=False)
+        all_rows._apply_delay_pattern_masking_batched(logits_all_rows, rows, all_rows=True)
+
+        assert torch.equal(logits_all_rows, logits_sparse)
+
+    def test_delay_state_update_all_rows_matches_sparse_path(self):
+        """The all-row fast path must update state and staging like index_copy_."""
+        from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
+            EOC_ID,
+        )
+
+        sparse = self._make_batched_sampler_talker()
+        all_rows = self._make_batched_sampler_talker()
+        rows = torch.arange(4, dtype=torch.long)
+        codes = torch.tensor(
+            [
+                [10, 11, 12, 13, 14, 15, 16, 17],
+                [20, 21, 22, 23, 24, 25, 26, 27],
+                [30, 31, EOC_ID, 33, 34, 35, 36, 37],
+                [40, 41, 42, 43, 44, 45, 46, 47],
+            ],
+            dtype=torch.long,
+        )
+        code_row_mask = torch.tensor([True, False, True, True])
+
+        sparse._update_delay_state_batched(
+            codes.clone(), rows, 4, torch.device("cpu"), code_row_mask=code_row_mask, all_rows=False
+        )
+        all_rows._update_delay_state_batched(
+            codes.clone(), rows, 4, torch.device("cpu"), code_row_mask=code_row_mask, all_rows=True
+        )
+
+        assert torch.equal(all_rows._decode_delay_count, sparse._decode_delay_count)
+        assert torch.equal(all_rows._decode_eoc_countdown, sparse._decode_eoc_countdown)
+        assert torch.equal(all_rows._decode_generation_done, sparse._decode_generation_done)
+        assert torch.equal(all_rows._decode_last_codes, sparse._decode_last_codes)
+        assert torch.equal(all_rows._decode_has_codes, sparse._decode_has_codes)
+        assert all_rows._last_audio_codes is None
+        assert sparse._last_audio_codes is None
+        assert torch.equal(all_rows._last_audio_host_staging, sparse._last_audio_host_staging)
+
+    def test_direct_audio_sampler_does_not_override_done_tokens(self):
+        """Done rows are represented directly in GPU sampled tokens."""
+        from vllm_omni.model_executor.models.higgs_audio_v3 import higgs_audio_v3_talker as mod
+
+        t = self._make_batched_sampler_talker(num_rows=2)
+        t._resolve_token_ids = lambda: None
+        t._audio_continuation_id = 99999
+        t._eos_token_id = 151671
+        t._last_logits_hidden = torch.zeros(2, 16)
+        t._last_step_input_ids = torch.tensor([99999, 99999])
+        t._last_step_query_start_loc = None
+        t._decode_has_codes = torch.tensor([True, True])
+        t._decode_generation_done = torch.tensor([False, True])
+        t._decode_delay_count = torch.zeros(2, dtype=torch.long)
+        t._decode_eoc_countdown = torch.full((2,), -1, dtype=torch.long)
+        t._fast_audio_direct_rows = 2
+        t._last_audio_done_flags = None
+        t._fast_audio_sampler_gpu_fallback_reason = lambda **kwargs: None
+        t._audio_codebook_logits_from_rows = lambda hidden, rows, all_rows=False: torch.zeros(2, 8, 1026)
+        t._apply_delay_pattern_masking_batched = lambda cb_logits, audio_rows, all_rows=False: None
+        t._sample_audio_codes = lambda logits_2d: torch.zeros(int(logits_2d.shape[0]), dtype=torch.long)
+        t._update_delay_state_batched = lambda *args, **kwargs: None
+        t.sample = mod.HiggsAudioV3TalkerForConditionalGeneration.sample.__get__(t)
+
+        sampler_output = t.sample(torch.zeros(2, 200000), sampling_metadata=object())
+
+        assert not getattr(
+            mod.HiggsAudioV3TalkerForConditionalGeneration, "supports_sampled_token_ids_cpu_override", False
+        )
+        assert sampler_output.sampled_token_ids.tolist() == [[99999], [151671]]
+
+    def test_direct_audio_sampler_uses_gpu_tokens_for_active_rows(self):
+        """Direct audio sampling leaves scheduler-visible tokens to the GPU tensor."""
+        from vllm_omni.model_executor.models.higgs_audio_v3 import higgs_audio_v3_talker as mod
+
+        t = self._make_batched_sampler_talker(num_rows=2)
+        t._resolve_token_ids = lambda: None
+        t._audio_continuation_id = 99999
+        t._eos_token_id = 151671
+        t._last_logits_hidden = torch.zeros(2, 16)
+        t._last_step_input_ids = torch.tensor([99999, 99999])
+        t._last_step_query_start_loc = None
+        t._decode_has_codes = torch.tensor([True, True])
+        t._decode_generation_done = torch.tensor([False, False])
+        t._decode_delay_count = torch.zeros(2, dtype=torch.long)
+        t._decode_eoc_countdown = torch.full((2,), -1, dtype=torch.long)
+        t._fast_audio_direct_rows = 2
+        t._last_audio_done_flags = [0, 0]
+        t._fast_audio_sampler_gpu_fallback_reason = lambda **kwargs: None
+        t._audio_codebook_logits_from_rows = lambda hidden, rows, all_rows=False: torch.zeros(2, 8, 1026)
+        t._apply_delay_pattern_masking_batched = lambda cb_logits, audio_rows, all_rows=False: None
+        t._sample_audio_codes = lambda logits_2d: torch.zeros(int(logits_2d.shape[0]), dtype=torch.long)
+        t._update_delay_state_batched = lambda *args, **kwargs: None
+        t.sample = mod.HiggsAudioV3TalkerForConditionalGeneration.sample.__get__(t)
+
+        sampler_output = t.sample(torch.zeros(2, 200000), sampling_metadata=object())
+
+        assert sampler_output.sampled_token_ids.tolist() == [[99999], [99999]]
+
+    def test_terminal_rampdown_frame_is_emitted_before_done(self):
+        """The final ramp-down frame must be emitted, not replaced by a done marker."""
+        from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import EOC_ID
+
+        t = self._make_batched_sampler_talker(num_rows=1)
+        t._decode_delay_count[0] = 8
+        t._decode_eoc_countdown[0] = 1
+        t._decode_generation_done[0] = False
+        t._decode_has_codes[0] = True
+        codes = torch.tensor([[10, 11, 12, 13, 14, 15, 16, 17]], dtype=torch.long)
+
+        t._update_delay_state_batched(
+            codes,
+            torch.tensor([0], dtype=torch.long),
+            1,
+            torch.device("cpu"),
+            code_row_mask=torch.tensor([True]),
+            all_rows=True,
+        )
+
+        staged = t._last_audio_host_staging[0]
+        assert staged[t.num_codebooks].item() == 1  # valid
+        assert staged[t.num_codebooks + 1].item() == 1  # done
+        assert not (staged[: t.num_codebooks] == -1).any()
+        assert staged[: t.num_codebooks - 1].eq(EOC_ID).all()
+        assert t._decode_generation_done[0].item() is True
+        assert t._decode_has_codes[0].item() is False
+
+    def test_decode_state_growth_preserves_inflight_rows(self):
+        """Growing decode buffers must copy old rows and initialize new rows inactive."""
+        t = self._make_batched_sampler_talker(num_rows=2)
+        old_codes = t._decode_last_codes.clone()
+        old_has = t._decode_has_codes.clone()
+        old_delay = t._decode_delay_count.clone()
+        old_rem = t._decode_eoc_countdown.clone()
+        old_done = t._decode_generation_done.clone()
+
+        t._ensure_decode_state_capacity(5, torch.device("cpu"))
+
+        assert t._decode_last_codes.shape[0] >= 5
+        assert torch.equal(t._decode_last_codes[:2], old_codes)
+        assert torch.equal(t._decode_has_codes[:2], old_has)
+        assert torch.equal(t._decode_delay_count[:2], old_delay)
+        assert torch.equal(t._decode_eoc_countdown[:2], old_rem)
+        assert torch.equal(t._decode_generation_done[:2], old_done)
+        assert not t._decode_has_codes[2:5].any()
+        assert not t._decode_generation_done[2:5].any()
+        assert t._decode_eoc_countdown[2:5].eq(-1).all()
+
+    def test_mixed_batch_prefill_mask_targets_request_rows(self):
+        """Mixed prefill/decode must reset only prefill request rows."""
+        t = self._make_batched_sampler_talker(num_rows=3)
+        t._last_step_query_start_loc = torch.tensor([0, 1, 4, 5])
+
+        mask = t._prefill_row_mask(3, torch.device("cpu"))
+
+        assert torch.equal(mask, torch.tensor([False, True, False]))
+
+    def test_mixed_batch_seed_mask_maps_token_tails_to_request_rows(self):
+        """Seed detection should return one flag per request, not per token."""
+        t = self._make_batched_sampler_talker(num_rows=3)
+        t._audio_continuation_id = 99999
+        t._last_step_query_start_loc = torch.tensor([0, 1, 4, 5])
+        t._last_step_input_ids = torch.tensor([99999, 11, 12, 13, 99999])
+
+        mask = t._audio_seed_mask_from_step_input(3, torch.device("cpu"))
+
+        assert torch.equal(mask, torch.tensor([True, False, True]))
+
+    def test_mixed_batch_seed_mask_allows_prefill_tail_audio_token(self):
+        """A prefill span ending in <|audio|> starts audio-code generation."""
+        t = self._make_batched_sampler_talker(num_rows=3)
+        t._audio_continuation_id = 99999
+        t._last_step_query_start_loc = torch.tensor([0, 1, 4, 5])
+        t._last_step_input_ids = torch.tensor([99999, 11, 12, 99999, 99999])
+
+        mask = t._audio_seed_mask_from_step_input(3, torch.device("cpu"))
+
+        assert torch.equal(mask, torch.tensor([True, True, True]))
+
 
 # ---- AC-6: Feedback Method Tests ----
 
@@ -348,10 +596,14 @@ class TestFeedbackMethods:
 
         class FakeTalker:
             _last_audio_codes = torch.tensor([[100, 200, 300, 400, 500, 600, 700, 800]])
+            _last_audio_code_valid = [True]
+            _last_audio_host_staging = None
+            _last_audio_staging_event = None
             _postprocess_cursor = 0
 
         t = FakeTalker()
         t.postprocess = mod.HiggsAudioV3TalkerForConditionalGeneration.postprocess.__get__(t)
+        t._postprocess_impl = mod.HiggsAudioV3TalkerForConditionalGeneration._postprocess_impl.__get__(t)
         result = t.postprocess(torch.zeros(1, 64))
         assert "codes" in result
         assert "audio" in result["codes"]
@@ -363,10 +615,14 @@ class TestFeedbackMethods:
 
         class FakeTalker:
             _last_audio_codes = torch.tensor([[-1, -1, -1, -1, -1, -1, -1, -1]])
+            _last_audio_code_valid = [False]
+            _last_audio_host_staging = None
+            _last_audio_staging_event = None
             _postprocess_cursor = 0
 
         t = FakeTalker()
         t.postprocess = mod.HiggsAudioV3TalkerForConditionalGeneration.postprocess.__get__(t)
+        t._postprocess_impl = mod.HiggsAudioV3TalkerForConditionalGeneration._postprocess_impl.__get__(t)
         result = t.postprocess(torch.zeros(1, 64))
         assert result == {}
 
@@ -378,10 +634,14 @@ class TestFeedbackMethods:
             _last_audio_codes = torch.tensor(
                 [[100, 200, 300, 400, 500, 600, 700, 800], [-1, -1, -1, -1, -1, -1, -1, -1]]
             )
+            _last_audio_code_valid = [True, False]
+            _last_audio_host_staging = None
+            _last_audio_staging_event = None
             _postprocess_cursor = 0
 
         t = FakeTalker()
         t.postprocess = mod.HiggsAudioV3TalkerForConditionalGeneration.postprocess.__get__(t)
+        t._postprocess_impl = mod.HiggsAudioV3TalkerForConditionalGeneration._postprocess_impl.__get__(t)
         r1 = t.postprocess(torch.zeros(1, 64))
         assert "codes" in r1
         r2 = t.postprocess(torch.zeros(1, 64))
@@ -405,14 +665,23 @@ class TestAudioFeedback:
             codebook_size = 1026
             _audio_continuation_id = 99999  # fake audio token
             _last_step_query_start_loc = None
-            _audio_state = {}
+            _decode_active_audio_count = 0
+            _decode_last_codes = torch.zeros(4, 8, dtype=torch.long)
+            _decode_has_codes = torch.zeros(4, dtype=torch.bool)
+            _use_external_decode_cudagraph = False
             multimodal_embedding = embed
             model = type("M", (), {"embed_tokens": lambda self, ids: torch.zeros(ids.shape[0], 16)})()
 
         t = FakeTalker()
-        t._decode_last_codes = torch.zeros(256, 8, dtype=torch.long)
-        t._decode_has_codes = torch.zeros(256, dtype=torch.bool)
         t._apply_audio_feedback = mod.HiggsAudioV3TalkerForConditionalGeneration._apply_audio_feedback.__get__(t)
+        t._is_single_token_decode_step = (
+            mod.HiggsAudioV3TalkerForConditionalGeneration._is_single_token_decode_step.__get__(t)
+        )
+        t._step_request_count = mod.HiggsAudioV3TalkerForConditionalGeneration._step_request_count.__get__(t)
+        t._decode_request_token_positions = (
+            mod.HiggsAudioV3TalkerForConditionalGeneration._decode_request_token_positions.__get__(t)
+        )
+        t._ensure_decode_state_capacity = lambda min_bs, device: None
         return t
 
     def test_text_positions_unchanged(self):
@@ -428,9 +697,11 @@ class TestAudioFeedback:
         """Audio position with state should have its embedding replaced."""
         t = self._make_feedback_talker()
         audio_id = t._audio_continuation_id
-        t._audio_state[1] = {"last_codes": torch.zeros(8, dtype=torch.long)}
         t._decode_last_codes[1] = torch.zeros(8, dtype=torch.long)
         t._decode_has_codes[1] = True
+        t._decode_active_audio_count = 1
+        t._ensure_decode_state_capacity = lambda min_bs, device: None
+        t._last_step_query_start_loc = torch.tensor([0, 1, 2, 3])
         input_ids = torch.tensor([1, audio_id, 3])
         hidden = torch.zeros(3, 16)
         result = t._apply_audio_feedback(hidden, input_ids)
@@ -444,12 +715,80 @@ class TestAudioFeedback:
         """Audio position without state should not be replaced."""
         t = self._make_feedback_talker()
         audio_id = t._audio_continuation_id
-        # No state for row 0 — _decode_has_codes[0] defaults to False
+        # No state for row 0
         input_ids = torch.tensor([audio_id])
         hidden = torch.zeros(1, 16)
         result = t._apply_audio_feedback(hidden, input_ids)
         assert torch.equal(result, hidden)
 
+    def test_prefill_span_does_not_receive_audio_feedback(self):
+        """A prefill span can have numel == hidden rows but must not be treated as decode."""
+        t = self._make_feedback_talker()
+        t._last_step_input_ids = torch.tensor([1, 2, 3, 4])
+        t._last_step_query_start_loc = torch.tensor([0, 4])
+        t._decode_last_codes[0] = torch.zeros(8, dtype=torch.long)
+        t._decode_has_codes[0] = True
+        t._decode_active_audio_count = 1
+        hidden = torch.zeros(4, 16)
+
+        result = t._apply_audio_feedback(hidden, t._last_step_input_ids)
+
+        assert torch.equal(result, hidden)
+
+    def test_multi_request_decode_receives_audio_feedback(self):
+        """Concurrent decode has numel > 1 but each request span is exactly one token."""
+        t = self._make_feedback_talker()
+        t._last_step_input_ids = torch.tensor([10, 11, 12, 13])
+        t._last_step_query_start_loc = torch.tensor([0, 1, 2, 3, 4])
+        t._decode_last_codes[2] = torch.zeros(8, dtype=torch.long)
+        t._decode_has_codes[2] = True
+        t._decode_active_audio_count = 1
+        hidden = torch.zeros(4, 16)
+
+        result = t._apply_audio_feedback(hidden, t._last_step_input_ids)
+
+        assert result[2].abs().sum() > 0
+        assert result[0].abs().sum() == 0
+        assert result[1].abs().sum() == 0
+        assert result[3].abs().sum() == 0
+
+    def test_external_decode_graph_capture_keeps_feedback_ops_without_query_start_loc(self):
+        """FULL_DECODE capture has no request qsl but still needs dense decode feedback ops."""
+        t = self._make_feedback_talker()
+        t._use_external_decode_cudagraph = True
+        t._last_step_input_ids = torch.tensor([10, 11, 12, 13])
+        t._last_step_query_start_loc = None
+        t._decode_last_codes[2] = torch.zeros(8, dtype=torch.long)
+        t._decode_has_codes[2] = True
+        hidden = torch.zeros(4, 16)
+
+        result = t._apply_audio_feedback(hidden, t._last_step_input_ids)
+
+        assert result[2].abs().sum() > 0
+        assert result[0].abs().sum() == 0
+        assert result[1].abs().sum() == 0
+        assert result[3].abs().sum() == 0
+
+    def test_mixed_prefill_decode_receives_feedback_only_on_decode_rows(self):
+        """Mixed batches should skip prefill spans but keep audio feedback for decode spans."""
+        t = self._make_feedback_talker()
+        t._last_step_input_ids = torch.tensor([99999, 21, 22, 23, 99999])
+        t._last_step_query_start_loc = torch.tensor([0, 1, 4, 5])
+        t._decode_last_codes[0] = torch.zeros(8, dtype=torch.long)
+        t._decode_last_codes[2] = torch.zeros(8, dtype=torch.long)
+        t._decode_has_codes[0] = True
+        t._decode_has_codes[2] = True
+        t._decode_active_audio_count = 2
+        hidden = torch.zeros(5, 16)
+
+        result = t._apply_audio_feedback(hidden, t._last_step_input_ids)
+
+        assert result[0].abs().sum() > 0
+        assert result[4].abs().sum() > 0
+        assert result[1].abs().sum() == 0
+        assert result[2].abs().sum() == 0
+        assert result[3].abs().sum() == 0
+
 
 # ---- AC-8: Codec Strictness ----
 
@@ -775,6 +1114,23 @@ class TestStageInputProcessor:
         with pytest.raises(ValueError, match="Not enough frames"):
             _revert_delay_pattern(codes)
 
+    def test_talker2code2wav_skips_too_few_frames_without_crashing(self):
+        from vllm_omni.model_executor.stage_input_processors.higgs_audio_v3 import (
+            talker2code2wav,
+        )
+
+        class Output:
+            multimodal_output = {"codes": {"audio": torch.zeros(5, 8, dtype=torch.long)}}
+
+        class TalkerOutput:
+            finished = True
+            outputs = [Output()]
+
+        result = talker2code2wav([TalkerOutput()])
+
+        assert len(result) == 1
+        assert result[0]["prompt_token_ids"] == []
+
     def test_filter_real_code_frames(self):
         from vllm_omni.model_executor.stage_input_processors.higgs_audio_v3 import (
             _filter_real_code_frames,
@@ -826,7 +1182,13 @@ class TestRegistry:
 
         # __file__ = tests/unit/higgs_audio_v3/test_*.py → 4 dirnames to repo root
         repo_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
-        yaml_path = os.path.join(repo_root, "vllm_omni", "deploy", "higgs_multimodal_qwen3.yaml")
+        deploy_dir = os.path.join(repo_root, "vllm_omni", "deploy")
+        for name in (
+            "higgs_multimodal_qwen3.yaml",
+            "higgs_multimodal_qwen3_high_throughput.yaml",
+            "higgs_multimodal_qwen3_low_latency.yaml",
+        ):
+            yaml_path = os.path.join(deploy_dir, name)
             assert os.path.isfile(yaml_path), f"Deploy YAML not found at {yaml_path}"
 
 
@@ -914,298 +1276,89 @@ class TestPromptBuilder:
         assert 151704 not in ids  # <|ref_text|>
 
 
-# ---- Regression: ramp-down done-before-emit off-by-one ----
+class TestVoiceCloneReferenceCache:
+    def test_audio_tokenizer_dir_env_accepts_parent_or_subdir(self, tmp_path, monkeypatch):
+        from vllm_omni.model_executor.models.higgs_audio_v2 import higgs_audio_v2_tokenizer as tok
 
+        parent = tmp_path / "OmniVoice"
+        subdir = parent / "audio_tokenizer"
+        subdir.mkdir(parents=True)
+        (subdir / "config.json").write_text("{}", encoding="utf-8")
 
-def _simulate_delay_state_machine(
-    sampled_codes_per_step: list[torch.Tensor],
-) -> list[torch.Tensor]:
-    """Reproduce the delay-pattern state machine from sample().
+        monkeypatch.setenv("HIGGS_AUDIO_TOKENIZER_PATH", str(parent))
+        assert tok._resolve_audio_tokenizer_dir() == str(subdir)
 
-    Feeds ``sampled_codes_per_step`` (each [Q]) through the exact same
-    logic as HiggsAudioV3TalkerForConditionalGeneration.sample() and
-    returns the list of emitted frames (non -1 codes).
-    """
-    from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import (
-        BOC_ID,
-        EOC_ID,
+        monkeypatch.setenv("HIGGS_AUDIO_TOKENIZER_PATH", str(subdir))
+        assert tok._resolve_audio_tokenizer_dir() == str(subdir)
+
+    def test_higgs_v3_ref_code_cache_returns_clone(self):
+        from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech
+
+        serving = object.__new__(OmniOpenAIServingSpeech)
+        serving._higgs_audio_v3_ref_code_cache = OrderedDict()
+        serving._higgs_audio_v3_ref_code_cache_bytes = 0
+        serving._higgs_audio_v3_ref_code_inflight = {}
+
+        codes = torch.arange(16, dtype=torch.long).reshape(2, 8)
+        serving._put_higgs_audio_v3_ref_codes("ref-a", codes)
+
+        cached = serving._get_higgs_audio_v3_ref_codes("ref-a")
+        assert cached is not None
+        cached.fill_(0)
+
+        cached_again = serving._get_higgs_audio_v3_ref_codes("ref-a")
+        assert cached_again is not None
+        assert torch.equal(cached_again, codes)
+
+    def test_higgs_v3_ref_code_inflight_deduplicates_concurrent_encode(self):
+        from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech
+
+        serving = object.__new__(OmniOpenAIServingSpeech)
+        serving._higgs_audio_v3_ref_code_cache = OrderedDict()
+        serving._higgs_audio_v3_ref_code_cache_bytes = 0
+        serving._higgs_audio_v3_ref_code_inflight = {}
+        calls = 0
+
+        def encode_reference_audio(_wav, _sr):
+            nonlocal calls
+            calls += 1
+            time.sleep(0.05)
+            return torch.arange(16, dtype=torch.long).reshape(2, 8)
+
+        def apply_delay_pattern(codes):
+            return codes + 1
+
+        async def run():
+            return await asyncio.gather(
+                *[
+                    serving._resolve_higgs_audio_v3_ref_codes(
+                        "ref-a",
+                        object(),
+                        24000,
+                        encode_reference_audio,
+                        apply_delay_pattern,
+                    )
+                    for _ in range(3)
+                ]
             )
 
-    num_codebooks = 8
-    bos = BOC_ID
-    eos_stream = EOC_ID
+        results = asyncio.run(run())
+        assert calls == 1
+        assert sum(int(inflight_wait) for _, _, inflight_wait in results) == 2
+        for codes, cache_hit, _ in results:
+            assert cache_hit is False
+            assert torch.equal(codes, torch.arange(16, dtype=torch.long).reshape(2, 8) + 1)
 
-    state: dict = {
-        "num_delay": 0,
-        "num_remaining_delays": None,
-        "audio_out_ids": None,
-        "last_codes": torch.full((num_codebooks,), bos, dtype=torch.long),
-        "should_terminate": False,
-    }
-
-    emitted: list[torch.Tensor] = []
-    for step_codes in sampled_codes_per_step:
-        if state["should_terminate"]:
-            break
-
-        num_delay = state["num_delay"]
-        num_remaining_delays = state["num_remaining_delays"]
-        this_codes = step_codes.clone()
-
-        if num_delay + 1 < num_codebooks:
-            this_codes[num_delay + 1 :] = bos
-            num_delay += 1
-
-        if num_remaining_delays is not None:
-            this_codes[: num_codebooks - num_remaining_delays] = eos_stream
-            num_remaining_delays -= 1
-        else:
-            eos_positions = (this_codes == eos_stream).nonzero(as_tuple=False).reshape(-1)
-            if eos_positions.numel() > 0:
-                last_eos_idx = int(eos_positions[-1].item())
-                this_codes[: last_eos_idx + 1] = eos_stream
-                num_remaining_delays = num_codebooks - last_eos_idx - 1
-
-        if num_remaining_delays is not None and num_remaining_delays <= 0:
-            state["num_delay"] = 0
-            state["num_remaining_delays"] = None
-            state["should_terminate"] = True
-            continue
-
-        state["num_delay"] = num_delay
-        state["num_remaining_delays"] = num_remaining_delays
-        state["last_codes"] = this_codes.clone()
-        emitted.append(this_codes)
-
-    return emitted
-
-
-class TestRampDownOffByOne:
-    """Regression tests for the done-before-emit off-by-one bug.
-
-    When EOC fires early, the ramp-down must emit enough frames so that
-    Stage1's _revert_delay_pattern receives T >= Q = 8.
-    """
-
-    def test_eoc_at_step0_terminates(self):
-        """EOC on cb0 at step 0 terminates; T < Q is handled by Stage1 defensive try/except."""
-        from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import EOC_ID
-
-        Q = 8
-        steps = []
-        for s in range(20):
-            codes = torch.randint(0, 1024, (Q,))
-            if s == 0:
-                codes[0] = EOC_ID
-            steps.append(codes)
-
-        emitted = _simulate_delay_state_machine(steps)
-        assert len(emitted) == Q - 1, f"EOC at step 0: expected {Q - 1} emitted frames, got {len(emitted)}"
-
-    def test_eoc_at_step1_emits_enough_frames(self):
-        """EOC on cb0 at step 1 must produce >= Q emitted frames."""
-        from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import EOC_ID
-
-        Q = 8
-        steps = []
-        for s in range(20):
-            codes = torch.randint(0, 1024, (Q,))
-            if s == 1:
-                codes[0] = EOC_ID
-            steps.append(codes)
-
-        emitted = _simulate_delay_state_machine(steps)
-        assert len(emitted) >= Q, f"EOC at step 1: expected >= {Q} emitted frames, got {len(emitted)}"
-
-    def test_eoc_after_full_rampup_emits_enough(self):
-        """EOC after all codebooks are active produces T >> Q."""
-        from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import EOC_ID
-
-        Q = 8
-        eoc_step = 10
-        steps = []
-        for s in range(30):
-            codes = torch.randint(0, 1024, (Q,))
-            if s == eoc_step:
-                codes[0] = EOC_ID
-            steps.append(codes)
-
-        emitted = _simulate_delay_state_machine(steps)
-        assert len(emitted) >= Q, f"EOC at step {eoc_step}: expected >= {Q} emitted frames, got {len(emitted)}"
-
-    def test_last_rampdown_frame_not_all_negative(self):
-        """The last emitted ramp-down frame must contain real codes, not -1."""
-        from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import EOC_ID
-
-        Q = 8
-        steps = []
-        for s in range(20):
-            codes = torch.randint(0, 1024, (Q,))
-            if s == 1:
-                codes[0] = EOC_ID
-            steps.append(codes)
-
-        emitted = _simulate_delay_state_machine(steps)
-        last = emitted[-1]
-        assert (last >= 0).all(), f"Last ramp-down frame contains negative values: {last.tolist()}"
-
-    def test_emitted_frames_revertible_by_stage1(self):
-        """Emitted frames from EOC-at-step-1 must be revertible by _revert_delay_pattern."""
-        from vllm_omni.model_executor.models.higgs_audio_v3.higgs_audio_v3_talker import EOC_ID
-        from vllm_omni.model_executor.stage_input_processors.higgs_audio_v3 import (
-            _revert_delay_pattern,
+        cached, cache_hit, inflight_wait = asyncio.run(
+            serving._resolve_higgs_audio_v3_ref_codes(
+                "ref-a",
+                object(),
+                24000,
+                encode_reference_audio,
+                apply_delay_pattern,
             )
-
-        Q = 8
-        steps = []
-        for s in range(20):
-            codes = torch.randint(0, 1024, (Q,))
-            if s == 1:
-                codes[0] = EOC_ID
-            steps.append(codes)
-
-        emitted = _simulate_delay_state_machine(steps)
-        # Stack to [T, Q], transpose to [Q, T]
-        codes_qt = torch.stack(emitted, dim=0).t().contiguous()
-        assert codes_qt.shape[0] == Q
-        assert codes_qt.shape[1] >= Q
-
-        # Must not raise
-        reverted = _revert_delay_pattern(codes_qt)
-        assert reverted.shape[0] == Q
-        assert reverted.shape[1] == codes_qt.shape[1] - Q + 1
-
-
-class TestStage1DefensiveHandling:
-    """Stage1 must not crash the server when receiving T < Q frames."""
-
-    def test_revert_delay_pattern_raises_on_insufficient_frames(self):
-        """_revert_delay_pattern correctly raises ValueError for T < Q."""
-        from vllm_omni.model_executor.stage_input_processors.higgs_audio_v3 import (
-            _revert_delay_pattern,
         )
-
-        codes = torch.zeros(8, 7)  # T=7 < Q=8
-        with pytest.raises(ValueError, match="Not enough frames"):
-            _revert_delay_pattern(codes)
-
-    def test_talker2code2wav_survives_insufficient_frames(self):
-        """talker2code2wav must not crash when audio_codes has T < Q frames."""
-        from vllm_omni.model_executor.stage_input_processors.higgs_audio_v3 import (
-            talker2code2wav,
-        )
-
-        # Build a fake talker output with only 7 frames (T=7 < Q=8)
-        class FakeOutput:
-            def __init__(self):
-                self.multimodal_output = {
-                    "codes": {"audio": torch.zeros(7, 8, dtype=torch.long)},
-                }
-
-        class FakeTalkerOutput:
-            finished = True
-            outputs = [FakeOutput()]
-
-        # Must not raise — should return gracefully
-        result = talker2code2wav([FakeTalkerOutput()])
-        assert len(result) == 1
-        # Result is an OmniTokensPrompt with empty prompt_token_ids
-        out = result[0]
-        token_ids = out.prompt_token_ids if hasattr(out, "prompt_token_ids") else out.get("prompt_token_ids", [])
-        assert token_ids == []
-
-
-# ---- Regression: buffer growth + prefill guard ----
-
-
-class TestBufferGrowthPreservesState:
-    """Bug #1: buffer growth in _apply_audio_feedback must not wipe existing state."""
-
-    def _make_feedback_talker_with_buffers(self):
-        from vllm_omni.model_executor.models.higgs_audio_v3 import higgs_audio_v3_talker as mod
-
-        embed = mod.HiggsFusedMultiTextEmbedding(num_codebooks=8, vocab_size=1026, hidden_size=16)
-        torch.nn.init.ones_(embed.weight)
-
-        class FakeTalker:
-            num_codebooks = 8
-            codebook_size = 1026
-            multimodal_embedding = embed
-
-        t = FakeTalker()
-        t._decode_last_codes = torch.zeros(4, 8, dtype=torch.long)
-        t._decode_has_codes = torch.zeros(4, dtype=torch.bool)
-        t._apply_audio_feedback = mod.HiggsAudioV3TalkerForConditionalGeneration._apply_audio_feedback.__get__(t)
-        return t
-
-    def test_buffer_growth_preserves_existing_codes(self):
-        """Growing the buffer must copy old _decode_last_codes data."""
-        t = self._make_feedback_talker_with_buffers()
-        # Set state for slot 2
-        t._decode_last_codes[2] = torch.tensor([100, 200, 300, 400, 500, 600, 700, 800])
-        t._decode_has_codes[2] = True
-
-        # Trigger growth by calling with bs=8 > current capacity 4
-        hidden = torch.randn(8, 16)
-        t._apply_audio_feedback(hidden, torch.zeros(8, dtype=torch.long))
-
-        # After growth, slot 2 data must still be present
-        assert t._decode_has_codes[2].item() is True
-        assert t._decode_last_codes[2, 0].item() == 100
-
-    def test_buffer_growth_new_slots_are_inactive(self):
-        """New slots from buffer growth must have has_codes=False."""
-        t = self._make_feedback_talker_with_buffers()
-        t._decode_has_codes[1] = True
-
-        hidden = torch.randn(8, 16)
-        t._apply_audio_feedback(hidden, torch.zeros(8, dtype=torch.long))
-
-        # Slots 4-7 (new) must be inactive
-        assert not t._decode_has_codes[4].item()
-        assert not t._decode_has_codes[7].item()
-        # Slot 1 (old) must still be active
-        assert t._decode_has_codes[1].item()
-
-
-class TestPrefillGuard:
-    """Bug #2: _apply_audio_feedback must NOT run during prefill.
-
-    The prefill detection uses attn_metadata.max_query_len (== 1 for decode,
-    > 1 for prefill). This correctly handles concurrent decode (N requests,
-    each contributing 1 token → numel=N but max_query_len=1).
-    """
-
-    @staticmethod
-    def _is_prefill_via_max_query_len(max_query_len: int | None, input_ids: torch.Tensor) -> bool:
-        """Mirror the actual detection logic in forward()."""
-        if max_query_len is not None:
-            return int(max_query_len) > 1
-        return input_ids is not None and int(input_ids.numel()) > 1
-
-    def test_prefill_detected_via_max_query_len(self):
-        """Prefill: max_query_len > 1 → is_prefill=True."""
-        input_ids = torch.tensor([1, 2, 3, 4, 5])
-        assert self._is_prefill_via_max_query_len(5, input_ids) is True
-
-    def test_single_decode_detected_via_max_query_len(self):
-        """Single-request decode: max_query_len=1 → is_prefill=False."""
-        input_ids = torch.tensor([42])
-        assert self._is_prefill_via_max_query_len(1, input_ids) is False
-
-    def test_concurrent_decode_not_misclassified(self):
-        """Multi-request decode: numel=4 but max_query_len=1 → is_prefill=False.
-
-        This is the regression caught by reviewer: with c=4 concurrent decode
-        requests, input_ids.numel()==4 but each request contributes exactly 1
-        token, so max_query_len==1 and audio feedback must still apply.
-        """
-        input_ids = torch.tensor([42, 43, 44, 45])  # 4 concurrent decode
-        assert self._is_prefill_via_max_query_len(1, input_ids) is False
-
-    def test_fallback_when_max_query_len_unavailable(self):
-        """When attn_metadata is unavailable, fall back to numel heuristic."""
-        prefill_ids = torch.tensor([1, 2, 3])
-        assert self._is_prefill_via_max_query_len(None, prefill_ids) is True
-        decode_ids = torch.tensor([42])
-        assert self._is_prefill_via_max_query_len(None, decode_ids) is False
+        assert calls == 1
+        assert cache_hit is True
+        assert inflight_wait is False
+        assert torch.equal(cached, torch.arange(16, dtype=torch.long).reshape(2, 8) + 1)

vllm_omni/deploy/README_higgs_audio_v3.md

@@ -0,0 +1,53 @@
+# Higgs-Audio V3 Deploy Profiles
+
+Higgs-Audio V3 has two Stage 0 graph profiles. They are intentionally separate
+because the graph paths are mutually exclusive.
+
+## High Throughput
+
+Use `higgs_multimodal_qwen3_high_throughput.yaml` for medium/high concurrency
+serving. This profile keeps Stage 0 `enforce_eager: true`, which preserves the
+Higgs-specific local MLP CUDA graph path. It is the default production profile
+for throughput-oriented serving.
+
+`higgs_multimodal_qwen3.yaml` is kept as the auto-discovered default deploy
+config for `model_type=higgs_multimodal_qwen3`, and matches the high-throughput
+profile.
+
+```bash
+vllm-omni serve bosonai/higgs-audio-v3-tts-4b \
+    --omni --trust-remote-code \
+    --deploy-config vllm_omni/deploy/higgs_multimodal_qwen3_high_throughput.yaml
+```
+
+## Low Latency
+
+Use `higgs_multimodal_qwen3_low_latency.yaml` for low-concurrency serving
+(for example c1-c4) where Stage 0 decode launch overhead dominates. This profile
+sets Stage 0 `enforce_eager: false` and explicitly enables vLLM
+`FULL_DECODE_ONLY` CUDA graph:
+
+```yaml
+compilation_config:
+  cudagraph_capture_sizes: [1, 2, 4, 8, 16]
+  cudagraph_mode: FULL_DECODE_ONLY
+  cudagraph_num_of_warmups: 1
+```
+
+FULL_DECODE is controlled by deploy configuration, not by an environment
+variable. When this external decode graph is active, the Higgs talker disables
+the local MLP CUDA graph automatically.
+
+```bash
+vllm-omni serve bosonai/higgs-audio-v3-tts-4b \
+    --omni --trust-remote-code \
+    --deploy-config vllm_omni/deploy/higgs_multimodal_qwen3_low_latency.yaml
+```
+
+## Notes
+
+- Stage 1 remains `enforce_eager: true` in both profiles.
+- Keep `VLLM_USE_DEEP_GEMM=0` and `VLLM_MOE_USE_DEEP_GEMM=0` for this model
+  unless DeepGEMM support is revalidated.
+- Revalidate end-to-end throughput and audio quality before changing the default
+  auto-discovered config.

vllm_omni/deploy/higgs_multimodal_qwen3.yaml

@@ -1,5 +1,11 @@
 # Higgs-audio v3 deploy: Stage 0 (talker) -> Stage 1 (code2wav) via shared memory.
 #
+# Auto-discovered default for model_type=higgs_multimodal_qwen3. This file is
+# the high-throughput profile and intentionally keeps Stage 0 enforce_eager=true
+# so the Higgs-specific local MLP CUDA graph path remains active. For low
+# concurrency latency-sensitive serving, use
+# higgs_multimodal_qwen3_low_latency.yaml explicitly.
+#
 # Plain TTS only (no voice cloning). async_chunk streams codec chunks from
 # Stage 0 to Stage 1 as they are produced (TTFA approaches one AR loop's
 # worth of work). Stage 1 honors meta.left_context_size + meta.right_holdback_size
@@ -32,6 +38,7 @@ stages:
     max_num_seqs: 16
     gpu_memory_utilization: 0.6
     enforce_eager: true
+    attention_backend: FLASHINFER
     trust_remote_code: true
     # Stage-0 prefix caching is on by default. The talker's HS opt-out and
     # deferred ``codes.audio`` mm-output write (see higgs_audio_v3_talker.py)

vllm_omni/deploy/higgs_multimodal_qwen3_high_throughput.yaml

@@ -0,0 +1,80 @@
+# Higgs-audio v3 deploy, high-throughput profile.
+#
+# This is the default production profile for medium/high concurrency serving.
+# Stage 0 keeps enforce_eager=true so Higgs' local MLP CUDA graph remains active.
+# Do not enable vLLM FULL_DECODE graph here: it replaces the local MLP graph
+# path instead of stacking on top of it.
+#
+# Plain TTS only (no voice cloning). async_chunk streams codec chunks from
+# Stage 0 to Stage 1 as they are produced (TTFA approaches one AR loop's
+# worth of work). Stage 1 honors meta.left_context_size + meta.right_holdback_size
+# so the per-chunk codec windows stitch into a coherent PCM stream.
+async_chunk: true
+
+connectors:
+  connector_of_shared_memory:
+    name: SharedMemoryConnector
+    extra:
+      shm_threshold_bytes: 65536
+      # Stream PCM bytes back to the client as Stage 1 emits each chunk.
+      codec_streaming: true
+      # Streaming chunk shape for talker2code2wav_async_chunk:
+      #   codec_chunk_frames        steady-state new frames per emit
+      #   codec_left_context_frames sliding past context to avoid leading-edge artifacts
+      #   codec_right_holdback_frames trailing frames held back so the codec
+      #                              has future context for emit-region tail samples
+      #   initial_codec_chunk_frames smaller first chunk to minimize TTFA
+      codec_chunk_frames: 25
+      codec_left_context_frames: 25
+      codec_right_holdback_frames: 4
+      initial_codec_chunk_frames: 1
+      connector_get_sleep_s: 0.01
+      connector_get_max_wait_first_chunk: 5000
+      connector_get_max_wait: 500
+
+stages:
+  - stage_id: 0
+    max_num_seqs: 64
+    gpu_memory_utilization: 0.6
+    enforce_eager: true
+    attention_backend: FLASHINFER
+    trust_remote_code: true
+    # Stage-0 prefix caching is on by default. The talker's HS opt-out and
+    # deferred ``codes.audio`` mm-output write (see higgs_audio_v3_talker.py)
+    # keep per-step bookkeeping negligible; any shared text prefix across
+    # concurrent requests is a net throughput win. Stage 1 (codec decoder)
+    # has no KV cache and stays disabled.
+    enable_prefix_caching: true
+    async_scheduling: false
+    max_num_batched_tokens: 4096
+    max_model_len: 8192
+    devices: "0"
+    output_connectors:
+      to_stage_1: connector_of_shared_memory
+    default_sampling_params:
+      temperature: 1.0
+      top_p: 0.95
+      top_k: 50
+      max_tokens: 2048
+      seed: 42
+      repetition_penalty: 1.0
+
+  - stage_id: 1
+    max_num_seqs: 16
+    gpu_memory_utilization: 0.25
+    enforce_eager: true
+    trust_remote_code: true
+    enable_prefix_caching: false
+    async_scheduling: false
+    max_num_batched_tokens: 65536
+    max_model_len: 65536
+    devices: "0"
+    input_connectors:
+      from_stage_0: connector_of_shared_memory
+    default_sampling_params:
+      temperature: 0.0
+      top_p: 1.0
+      top_k: -1
+      max_tokens: 65536
+      seed: 42
+      repetition_penalty: 1.0

vllm_omni/deploy/higgs_multimodal_qwen3_low_latency.yaml

@@ -0,0 +1,85 @@
+# Higgs-audio v3 deploy, low-latency profile.
+#
+# Use this profile for low-concurrency serving (for example c1-c4) where Stage 0
+# decode launch overhead dominates. Stage 0 enables vLLM FULL_DECODE_ONLY CUDA
+# graph through YAML configuration:
+#
+#   enforce_eager: false
+#   compilation_config.cudagraph_mode: FULL_DECODE_ONLY
+#
+# This is not controlled by an environment variable. The Higgs talker disables
+# its local MLP CUDA graph automatically when the external decode graph is active,
+# because the two graph paths are mutually exclusive.
+#
+# Plain TTS only (no voice cloning). async_chunk streams codec chunks from
+# Stage 0 to Stage 1 as they are produced (TTFA approaches one AR loop's
+# worth of work). Stage 1 honors meta.left_context_size + meta.right_holdback_size
+# so the per-chunk codec windows stitch into a coherent PCM stream.
+async_chunk: true
+
+connectors:
+  connector_of_shared_memory:
+    name: SharedMemoryConnector
+    extra:
+      shm_threshold_bytes: 65536
+      # Stream PCM bytes back to the client as Stage 1 emits each chunk.
+      codec_streaming: true
+      # Streaming chunk shape for talker2code2wav_async_chunk:
+      #   codec_chunk_frames        steady-state new frames per emit
+      #   codec_left_context_frames sliding past context to avoid leading-edge artifacts
+      #   codec_right_holdback_frames trailing frames held back so the codec
+      #                              has future context for emit-region tail samples
+      #   initial_codec_chunk_frames smaller first chunk to minimize TTFA
+      codec_chunk_frames: 25
+      codec_left_context_frames: 25
+      codec_right_holdback_frames: 4
+      initial_codec_chunk_frames: 1
+      connector_get_sleep_s: 0.01
+      connector_get_max_wait_first_chunk: 5000
+      connector_get_max_wait: 500
+
+stages:
+  - stage_id: 0
+    max_num_seqs: 16
+    gpu_memory_utilization: 0.6
+    enforce_eager: false
+    attention_backend: FLASHINFER
+    trust_remote_code: true
+    enable_prefix_caching: true
+    async_scheduling: false
+    compilation_config:
+      cudagraph_capture_sizes: [1, 2, 4, 8, 16]
+      cudagraph_mode: FULL_DECODE_ONLY
+      cudagraph_num_of_warmups: 1
+    max_num_batched_tokens: 4096
+    max_model_len: 8192
+    devices: "0"
+    output_connectors:
+      to_stage_1: connector_of_shared_memory
+    default_sampling_params:
+      temperature: 1.0
+      top_p: 0.95
+      top_k: 50
+      max_tokens: 2048
+      seed: 42
+      repetition_penalty: 1.0
+
+  - stage_id: 1
+    max_num_seqs: 16
+    gpu_memory_utilization: 0.25
+    enforce_eager: true
+    trust_remote_code: true
+    enable_prefix_caching: false
+    async_scheduling: false
+    max_num_batched_tokens: 65536
+    max_model_len: 65536
+    devices: "0"
+    input_connectors:
+      from_stage_0: connector_of_shared_memory
+    default_sampling_params:
+      temperature: 0.0
+      top_p: 1.0
+      top_k: -1
+      max_tokens: 65536
+      seed: 42
+      repetition_penalty: 1.0

vllm_omni/distributed/omni_connectors/connectors/shm_connector.py

@@ -29,6 +29,7 @@ class SharedMemoryConnector(OmniConnectorBase):
         self.stage_id = config.get("stage_id", -1)
         self.device = config.get("device", "cuda:0")
         self.threshold = int(config.get("shm_threshold_bytes", 65536))
+        self.inline_small_payloads = bool(config.get("inline_small_payloads", False))
         self._pending_keys: set[str] = set()
         self._metrics = {
             "puts": 0,
@@ -53,9 +54,11 @@ class SharedMemoryConnector(OmniConnectorBase):
             payload = self.serialize_obj(data)
             size = len(payload)
 
-            # Currently, we always use SHM.
-            if True:
-                # Use Shared Memory
+            # The legacy async-chunk adapter transfers only the connector key
+            # across stages; it cannot deliver inline metadata to the receiver.
+            # Keep key-addressed SHM as the default and only inline payloads
+            # when the caller explicitly enables the metadata-aware path.
+            if size >= self.threshold or not self.inline_small_payloads:
                 lock_file = f"/dev/shm/shm_{put_key}_lockfile.lock"
                 with open(lock_file, "wb+") as lockf:
                     fcntl.flock(lockf, fcntl.LOCK_EX)
@@ -67,9 +70,8 @@ class SharedMemoryConnector(OmniConnectorBase):
                 self._pending_keys.add(put_key)
                 self._metrics["shm_writes"] += 1
             else:
-                # Inline - pass bytes directly to avoid double serialization of the object
-                # We already serialized it to check size, so we pass the bytes.
-                # The Queue will pickle these bytes (fast), avoiding re-serializing the complex object.
+                # Inline small payloads to avoid the mmap + file-lock overhead
+                # that dominates codec chunk transfers.
                 metadata = {"inline_bytes": payload, "size": size}
                 self._metrics["inline_writes"] += 1
 

vllm_omni/entrypoints/openai/serving_speech.py

@@ -78,7 +78,7 @@ _MING_TTS_MODEL_STAGES = {"ming_tts"}
 _MOSS_TTS_MODEL_STAGES = {"moss_tts_nano"}
 _MOSS_TTS_FULL_MODEL_STAGES = {"moss_tts", "moss_tts_codec"}
 _HIGGS_AUDIO_V2_TTS_MODEL_STAGES = {"higgs_audio_v2"}
-_HIGGS_AUDIO_V3_TTS_MODEL_STAGES = {"higgs_audio_v3"}
+_HIGGS_V3_TTS_MODEL_STAGES = {"higgs_audio_v3"}
 _GLM_TTS_MODEL_STAGES = {"glm_tts"}
 _TTS_MODEL_STAGES: set[str] = (
     _VOXTRAL_TTS_MODEL_STAGES
@@ -87,7 +87,7 @@ _TTS_MODEL_STAGES: set[str] = (
     | _COSYVOICE3_TTS_MODEL_STAGES
     | _OMNIVOICE_TTS_MODEL_STAGES
     | _HIGGS_AUDIO_V2_TTS_MODEL_STAGES
-    | _HIGGS_AUDIO_V3_TTS_MODEL_STAGES
+    | _HIGGS_V3_TTS_MODEL_STAGES
     | _COVO_AUDIO_MODEL_STAGES
     | _VOXCPM2_TTS_MODEL_STAGES
     | _MING_TTS_MODEL_STAGES
@@ -123,6 +123,8 @@ _REF_AUDIO_MIN_DURATION = 1.0  # seconds
 _REF_AUDIO_MAX_DURATION = 30.0  # seconds
 _REF_AUDIO_RESOLVE_CACHE_MAX_ENTRIES = 256
 _REF_AUDIO_RESOLVE_CACHE_MAX_BYTES = 256 * 1024 * 1024
+_HIGGS_V3_REF_CODE_CACHE_MAX_ENTRIES = 256
+_HIGGS_V3_REF_CODE_CACHE_MAX_BYTES = 64 * 1024 * 1024
 _QWEN3_TTS_REF_AUDIO_CACHE_KEY = "_qwen3_tts_ref_audio_cache_key"
 _TTS_MAX_INSTRUCTIONS_LENGTH = 500
 _TTS_MAX_NEW_TOKENS_MIN = 1
@@ -288,6 +290,9 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
         self._ref_audio_resolve_cache_max_bytes = _REF_AUDIO_RESOLVE_CACHE_MAX_BYTES
         self._ref_audio_model_artifact_ready: set[str] = set()
         self._request_ref_audio_artifact_keys: dict[str, str] = {}
+        self._higgs_audio_v3_ref_code_cache: OrderedDict[str, tuple[torch.Tensor, int]] = OrderedDict()
+        self._higgs_audio_v3_ref_code_cache_bytes = 0
+        self._higgs_audio_v3_ref_code_inflight: dict[str, asyncio.Task[torch.Tensor]] = {}
         self._speaker_cache = get_speaker_cache()
         self._last_upload_ts = 0
         self._upload_lock = asyncio.Lock()
@@ -646,7 +651,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
             return "moss_tts"
         if model_stage in _HIGGS_AUDIO_V2_TTS_MODEL_STAGES:
             return "higgs_audio_v2"
-        if model_stage in _HIGGS_AUDIO_V3_TTS_MODEL_STAGES:
+        if model_stage in _HIGGS_V3_TTS_MODEL_STAGES:
             return "higgs_audio_v3"
         if model_stage in _GLM_TTS_MODEL_STAGES:
             return "glm_tts"
@@ -2018,17 +2023,18 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
             apply_delay_pattern,
             encode_reference_audio,
         )
-        from vllm_omni.model_executor.models.higgs_audio_v3.ref_audio_cache import (
-            cached_encode_reference_audio,
-        )
 
         wav_list, sr = await self._resolve_ref_audio(request.ref_audio)
+        artifact_key = self._get_resolved_ref_audio_artifact_key(request.ref_audio)
         wav = np.asarray(wav_list, dtype=np.float32)
-        # Cache the codec encode by content hash so identical ref_audio
-        # payloads skip the encode and reuse the same [num_frames,
-        # num_codebooks] codes tensor across requests.
-        ref_codes_raw = await asyncio.to_thread(cached_encode_reference_audio, wav, int(sr), encode_reference_audio)
-        ref_codes_delayed = apply_delay_pattern(ref_codes_raw)
+        ref_codes_delayed, cache_hit, inflight_wait = await self._resolve_higgs_audio_v3_ref_codes(
+            artifact_key,
+            wav,
+            int(sr),
+            encode_reference_audio,
+            apply_delay_pattern,
+        )
+        del cache_hit, inflight_wait
 
         prompt_ids = adapter.build_prompt(
             request.input,
@@ -2044,6 +2050,68 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
         }
         return prompt
 
+    async def _resolve_higgs_audio_v3_ref_codes(
+        self,
+        artifact_key: str | None,
+        wav: np.ndarray,
+        sr: int,
+        encode_reference_audio,
+        apply_delay_pattern,
+    ) -> tuple[torch.Tensor, bool, bool]:
+        ref_codes_delayed = self._get_higgs_audio_v3_ref_codes(artifact_key)
+        if ref_codes_delayed is not None:
+            return ref_codes_delayed, True, False
+        if not artifact_key:
+            ref_codes_raw = await asyncio.to_thread(encode_reference_audio, wav, sr)
+            return apply_delay_pattern(ref_codes_raw), False, False
+
+        task = self._higgs_audio_v3_ref_code_inflight.get(artifact_key)
+        if task is not None:
+            return (await task).clone(), False, True
+
+        async def _encode_and_cache() -> torch.Tensor:
+            ref_codes_raw = await asyncio.to_thread(encode_reference_audio, wav, sr)
+            delayed = apply_delay_pattern(ref_codes_raw)
+            self._put_higgs_audio_v3_ref_codes(artifact_key, delayed)
+            cached = self._get_higgs_audio_v3_ref_codes(artifact_key)
+            return cached if cached is not None else delayed.detach().to("cpu", dtype=torch.long).contiguous()
+
+        task = asyncio.create_task(_encode_and_cache())
+        self._higgs_audio_v3_ref_code_inflight[artifact_key] = task
+        try:
+            return (await task).clone(), False, False
+        finally:
+            if self._higgs_audio_v3_ref_code_inflight.get(artifact_key) is task:
+                self._higgs_audio_v3_ref_code_inflight.pop(artifact_key, None)
+
+    def _get_higgs_audio_v3_ref_codes(self, artifact_key: str | None) -> torch.Tensor | None:
+        if not artifact_key:
+            return None
+        cached = self._higgs_audio_v3_ref_code_cache.get(artifact_key)
+        if cached is None:
+            return None
+        self._higgs_audio_v3_ref_code_cache.move_to_end(artifact_key)
+        return cached[0].clone()
+
+    def _put_higgs_audio_v3_ref_codes(self, artifact_key: str, codes: torch.Tensor) -> None:
+        if _HIGGS_V3_REF_CODE_CACHE_MAX_ENTRIES <= 0 or _HIGGS_V3_REF_CODE_CACHE_MAX_BYTES <= 0 or not artifact_key:
+            return
+        cached_codes = codes.detach().to("cpu", dtype=torch.long).contiguous()
+        size = int(cached_codes.numel() * cached_codes.element_size())
+        if size > _HIGGS_V3_REF_CODE_CACHE_MAX_BYTES:
+            return
+        previous = self._higgs_audio_v3_ref_code_cache.pop(artifact_key, None)
+        if previous is not None:
+            self._higgs_audio_v3_ref_code_cache_bytes -= previous[1]
+        self._higgs_audio_v3_ref_code_cache[artifact_key] = (cached_codes, size)
+        self._higgs_audio_v3_ref_code_cache_bytes += size
+        while len(self._higgs_audio_v3_ref_code_cache) > _HIGGS_V3_REF_CODE_CACHE_MAX_ENTRIES:
+            _, (_, old_size) = self._higgs_audio_v3_ref_code_cache.popitem(last=False)
+            self._higgs_audio_v3_ref_code_cache_bytes -= old_size
+        while self._higgs_audio_v3_ref_code_cache_bytes > _HIGGS_V3_REF_CODE_CACHE_MAX_BYTES:
+            _, (_, old_size) = self._higgs_audio_v3_ref_code_cache.popitem(last=False)
+            self._higgs_audio_v3_ref_code_cache_bytes -= old_size
+
     async def _resolve_higgs_audio_v3_adapter(self):
         """Lazy-load the tokenizer adapter for higgs_audio_v3."""
         cached = getattr(self, "_higgs_audio_v3_adapter", None)

vllm_omni/model_executor/models/higgs_audio_v2/higgs_audio_v2_tokenizer.py

@@ -162,6 +162,54 @@ def _build_delay_pattern(codes: torch.Tensor) -> torch.Tensor:
 _ENCODER_CACHE: Any | None = None
 _K2_OMNIVOICE_REPO = "k2-fsa/OmniVoice"
 _K2_OMNIVOICE_SUBDIR = "audio_tokenizer"
+_AUDIO_TOKENIZER_PATH_ENVS = (
+    "HIGGS_AUDIO_TOKENIZER_PATH",
+    "HIGGS_AUDIO_V2_TOKENIZER_PATH",
+)
+
+
+def _normalize_audio_tokenizer_dir(path: str) -> str | None:
+    if not path:
+        return None
+    expanded = os.path.abspath(os.path.expanduser(path))
+    if os.path.isfile(os.path.join(expanded, "config.json")):
+        return expanded
+    nested = os.path.join(expanded, _K2_OMNIVOICE_SUBDIR)
+    if os.path.isfile(os.path.join(nested, "config.json")):
+        return nested
+    return None
+
+
+def _resolve_audio_tokenizer_dir() -> str | None:
+    """Resolve the Higgs codec encoder dir without requiring online HF access.
+
+    Voice cloning runs in the API server process. Production H20 jobs are often
+    offline, so prefer explicit local directories and already-populated HF cache
+    before falling back to ``snapshot_download``.
+    """
+    for env_name in _AUDIO_TOKENIZER_PATH_ENVS:
+        candidate = _normalize_audio_tokenizer_dir(os.getenv(env_name, ""))
+        if candidate is not None:
+            return candidate
+
+    from huggingface_hub import try_to_load_from_cache
+
+    cached_config = try_to_load_from_cache(
+        repo_id=_K2_OMNIVOICE_REPO,
+        filename=f"{_K2_OMNIVOICE_SUBDIR}/config.json",
+    )
+    if isinstance(cached_config, str) and os.path.isfile(cached_config):
+        return os.path.dirname(cached_config)
+
+    from huggingface_hub.constants import HF_HUB_CACHE
+
+    safe = _K2_OMNIVOICE_REPO.replace("/", "--")
+    snapshots_dir = os.path.join(HF_HUB_CACHE, f"models--{safe}", "snapshots")
+    if os.path.isdir(snapshots_dir):
+        for rev in os.listdir(snapshots_dir):
+            candidate = _normalize_audio_tokenizer_dir(os.path.join(snapshots_dir, rev))
+            if candidate is not None:
+                return candidate
 
 
 def _load_audio_tokenizer():
@@ -180,9 +228,12 @@ def _load_audio_tokenizer():
 
     Only the ``audio_tokenizer/`` subdirectory (~806 MB) is downloaded.
     """
-    from huggingface_hub import snapshot_download
     from transformers import HiggsAudioV2TokenizerModel
 
+    audio_tokenizer_dir = _resolve_audio_tokenizer_dir()
+    if audio_tokenizer_dir is None:
+        from huggingface_hub import snapshot_download
+
         repo_path = snapshot_download(
             _K2_OMNIVOICE_REPO,
             allow_patterns=[f"{_K2_OMNIVOICE_SUBDIR}/*"],

vllm_omni/model_executor/models/higgs_audio_v3/configuration_higgs_audio_v3.py

@@ -93,8 +93,13 @@ class HiggsAudioV3Config(PretrainedConfig):
         tts_token_id: int | None = None,
         text_token_id: int | None = None,
         audio_continuation_id: int | None = None,
+        enable_flashinfer_api_unwrap: bool = True,
+        enable_mlp_cudagraph: bool = True,
         **kwargs: Any,
     ) -> None:
+        # Legacy perf knob removed: Higgs v3 scheduler tokens now come from
+        # SamplerOutput.sampled_token_ids only.
+        kwargs.pop("enable_cpu_token_override", None)
         self.audio_token_id = audio_token_id
         self.mel_per_sample = mel_per_sample
 
@@ -124,6 +129,8 @@ class HiggsAudioV3Config(PretrainedConfig):
         self.tts_token_id = tts_token_id
         self.text_token_id = text_token_id
         self.audio_continuation_id = audio_continuation_id
+        self.enable_flashinfer_api_unwrap = bool(enable_flashinfer_api_unwrap)
+        self.enable_mlp_cudagraph = bool(enable_mlp_cudagraph)
 
         super().__init__(**kwargs)
 

vllm_omni/model_executor/models/higgs_audio_v3/higgs_audio_v3_code2wav.py

@@ -35,6 +35,7 @@ __all__ = [
 
 logger = init_logger(__name__)
 
+
 # Prefix under which codec weights are stored in the v3 checkpoint.
 _CODEC_PREFIX = "tied.embedding.modality_embeddings.0.model."
 

vllm_omni/model_executor/stage_input_processors/higgs_audio_v3.py

@@ -32,6 +32,7 @@ from vllm_omni.data_entry_keys import (
     OmniPayload,
     OmniPayloadStruct,
 )
+from vllm_omni.inputs.data import OmniTokensPrompt
 
 __all__ = ["talker2code2wav", "talker2code2wav_async_chunk"]
 
@@ -48,6 +49,15 @@ _DEFAULT_CODEC_LEFT_CONTEXT_FRAMES = 25
 _DEFAULT_CODEC_RIGHT_HOLDBACK_FRAMES = 4
 
 
+def _empty_code2wav_prompt() -> Any:
+    return OmniTokensPrompt(
+        prompt_token_ids=[],
+        multi_modal_data=None,
+        mm_processor_kwargs=None,
+        additional_information=None,
+    )
+
+
 def _revert_delay_pattern(audio_codes_qt: torch.Tensor) -> torch.Tensor:
     """Reverse the MusicGen-style delay pattern.
 
@@ -91,9 +101,8 @@ def talker2code2wav(
     _requires_multimodal_data: bool = False,
 ) -> list[Any]:
     """Sync: collect all talker codes, revert delay pattern, filter, pass to code2wav."""
-    from vllm_omni.inputs.data import OmniTokensPrompt
 
-    code2wav_inputs: list[OmniTokensPrompt] = []
+    code2wav_inputs: list[Any] = []
     for talker_output in source_outputs:
         if not talker_output.finished:
             continue
@@ -103,14 +112,7 @@ def talker2code2wav(
 
         audio_codes = mm_codes.get("audio")
         if audio_codes is None or not isinstance(audio_codes, torch.Tensor) or audio_codes.numel() == 0:
-            code2wav_inputs.append(
-                OmniTokensPrompt(
-                    prompt_token_ids=[],
-                    multi_modal_data=None,
-                    mm_processor_kwargs=None,
-                    additional_information=None,
-                )
-            )
+            code2wav_inputs.append(_empty_code2wav_prompt())
             continue
 
         audio_codes = audio_codes.to(torch.long)
@@ -137,22 +139,9 @@ def talker2code2wav(
         # Step 1: Revert delay pattern
         try:
             codes_qt = _revert_delay_pattern(codes_qt)
-        except ValueError:
-            logger.warning(
-                "Skipping request: insufficient frames for delay pattern "
-                "reversal (T=%d < Q=%d). This is a request-level error, "
-                "not a server-fatal condition.",
-                codes_qt.shape[1],
-                codes_qt.shape[0],
-            )
-            code2wav_inputs.append(
-                OmniTokensPrompt(
-                    prompt_token_ids=[],
-                    multi_modal_data=None,
-                    mm_processor_kwargs=None,
-                    additional_information=None,
-                )
-            )
+        except ValueError as exc:
+            logger.warning("Skipping invalid Higgs Audio v3 code sequence for Stage 1: %s", exc)
+            code2wav_inputs.append(_empty_code2wav_prompt())
             continue
 
         # Step 2: Replace out-of-range codes (BOC=1024, EOC=1025, -1) with 0.
@@ -172,14 +161,7 @@ def talker2code2wav(
             codes_qt = codes_qt[:, :-1]
 
         if codes_qt.numel() == 0:
-            code2wav_inputs.append(
-                OmniTokensPrompt(
-                    prompt_token_ids=[],
-                    multi_modal_data=None,
-                    mm_processor_kwargs=None,
-                    additional_information=None,
-                )
-            )
+            code2wav_inputs.append(_empty_code2wav_prompt())
             continue
 
         # Code2Wav expects codebook-major flat: [Q * num_frames]

vllm_omni/worker/gpu_ar_model_runner.py

@@ -163,6 +163,8 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin):
         return output_token_ids
 
     def _sampling_metadata_for_model_sampler(self, sampling_metadata):
+        if getattr(self.model, "skips_model_sampler_output_token_history", False):
+            return sampling_metadata
         output_token_ids = self._build_model_sampler_output_token_ids()
         if output_token_ids == sampling_metadata.output_token_ids:
             return sampling_metadata
@@ -568,7 +570,6 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin):
                 use_cascade_attn=cascade_attn_prefix_lens is not None,
                 num_encoder_reqs=len(scheduler_output.scheduled_encoder_inputs),
             )
-
             num_tokens_padded = batch_desc.num_tokens
             num_reqs_padded = batch_desc.num_reqs if batch_desc.num_reqs is not None else num_reqs
             ubatch_slices, ubatch_slices_padded = maybe_create_ubatch_slices(

vllm_omni/worker/gpu_model_runner.py

@@ -57,6 +57,16 @@ class OmniGPUModelRunner(GPUModelRunner):
         # The Omni tensor prefix cache will be allocated
         # when we initialize the metadata builders if enabled
         self.omni_prefix_cache = None
+        self._sampled_token_ids_cpu_override = None
+        self._omni_query_start_loc_model_kwarg = False
+
+    def _to_list(self, sampled_token_ids: torch.Tensor) -> list[list[int]]:
+        override_fn = self._sampled_token_ids_cpu_override
+        if callable(override_fn):
+            sampled = override_fn(sampled_token_ids)
+            if sampled is not None:
+                return sampled
+        return super()._to_list(sampled_token_ids)
 
     def _omni_routed_experts_d2h(self, scheduler_output) -> None:
         """Issue routed-experts D2H copy matching upstream GPUModelRunner pattern.
@@ -105,14 +115,13 @@ class OmniGPUModelRunner(GPUModelRunner):
         )
 
     def initialize_metadata_builders(self, kv_cache_config, kernel_block_sizes):
-        """Override to fix scheduler_metadata buffer size for FA3 + CUDA graph.
+        """Initialize metadata builders and keep FA3 graph metadata buffers sized.
 
-        The upstream FlashAttentionMetadataBuilder pre-allocates
-        scheduler_metadata with (max_num_seqs + 1) entries, but FA3's
-        get_scheduler_metadata() can return up to
-        (max_num_seqs * max_num_splits + 1) entries, causing a RuntimeError
-        during CUDA graph capture.  After calling the parent implementation
-        we resize any too-small buffers.
+        FlashAttentionMetadataBuilder can pre-allocate scheduler_metadata for
+        only max_num_seqs + 1 entries while FA3 with split scheduling may need
+        max_num_seqs * max_num_splits + 1 entries during CUDA graph capture.
+        This runner is shared across Omni models, so preserve the existing
+        workaround for non-Higgs models that still use FA3.
         """
         super().initialize_metadata_builders(kv_cache_config, kernel_block_sizes)
 
@@ -143,6 +152,14 @@ class OmniGPUModelRunner(GPUModelRunner):
     @instrument(span_name="Loading (GPU)")
     def load_model(self, *args, **kwargs) -> None:
         super().load_model(*args, **kwargs)
+        model = getattr(self, "model", None)
+        override_fn = None
+        if bool(getattr(model, "supports_sampled_token_ids_cpu_override", False)):
+            candidate = getattr(model, "consume_sampled_token_ids_cpu_override", None)
+            if callable(candidate):
+                override_fn = candidate
+        self._sampled_token_ids_cpu_override = override_fn
+        self._omni_query_start_loc_model_kwarg = bool(getattr(model, "supports_omni_query_start_loc", False))
         self._maybe_enable_output_token_ids_for_model_sampler()
         self._init_talker_mtp()
         self._prewarm_attention_capture_workspaces()
@@ -1295,6 +1312,13 @@ class OmniGPUModelRunner(GPUModelRunner):
 
             traceback.print_exc()
 
+        if self._omni_query_start_loc_model_kwarg:
+            try:
+                num_reqs = len(self.input_batch.req_ids)
+                model_kwargs_extra["omni_query_start_loc"] = self.query_start_loc.gpu[: num_reqs + 1]
+            except Exception as e:
+                logger.debug("[OMNI] Failed to attach query_start_loc: %s", e)
+
         if getattr(self.model_config, "has_sampling_extra_args", False):
             extra_args_list: list[dict] = []
             for req_id in self.input_batch.req_ids:
@@ -1320,10 +1344,14 @@ class OmniGPUModelRunner(GPUModelRunner):
             # execute the custom postprocess function
             # TODO(Peiqi): do we have a more elegant way to do this?
             if hasattr(self.model, "has_postprocess") and self.model.has_postprocess:
+                postprocess_uses_hidden_states = getattr(self.model, "postprocess_uses_hidden_states", True)
+                postprocess_uses_multimodal_outputs = getattr(self.model, "postprocess_uses_multimodal_outputs", True)
+                postprocess_uses_req_infos = getattr(self.model, "postprocess_uses_req_infos", True)
                 for req_index, req_id in enumerate(self.input_batch.req_ids):
                     if req_ids_filter is not None and req_id not in req_ids_filter:
                         continue
-                    req_infos = self.model_intermediate_buffer.get(req_id, {})
+                    req_infos = self.model_intermediate_buffer.get(req_id, {}) if postprocess_uses_req_infos else {}
+                    if postprocess_uses_hidden_states:
                         if combined_hidden_states:
                             # Combined hidden states contains all hidden states for every request
                             hidden_states_slice = combined_hidden_states[req_id]
@@ -1333,8 +1361,12 @@ class OmniGPUModelRunner(GPUModelRunner):
                             s, e = start_offset, start_offset + sched_tokens
                             # only consider to store data into update dict.
                             hidden_states_slice = hidden_states[s:e]
+                    else:
+                        hidden_states_slice = hidden_states
 
-                    if combined_multimodal_outputs:
+                    if not postprocess_uses_multimodal_outputs:
+                        mm_out = None
+                    elif combined_multimodal_outputs:
                         # NOTE this is a bit ugly, but the mm data is structured as a list of
                         # keys mapping to request IDs, and if enabled, we will always have all
                         # request IDs in every subdict, including for cache misses.
@@ -1807,6 +1839,14 @@ class OmniGPUModelRunner(GPUModelRunner):
     ):
         """Inject omni-specific kwargs into forward and cache model output"""
         model_kwargs_extra = self._build_model_kwargs_extra()
+        update_decode_metadata = getattr(self.model, "update_decode_step_metadata", None)
+        if getattr(self.model, "supports_omni_decode_step_metadata", False) and callable(update_decode_metadata):
+            update_decode_metadata(
+                input_ids=input_ids,
+                positions=positions,
+                inputs_embeds=inputs_embeds,
+                omni_query_start_loc=model_kwargs_extra.get("omni_query_start_loc"),
+            )
 
         model_output = super()._model_forward(
             input_ids=input_ids,