From b039b6a5bd0040ab74d5db1c75ed0ce54871acf4 Mon Sep 17 00:00:00 2001 From: tzhouam Date: Wed, 23 Sep 2026 10:01:21 +0800 Subject: [PATCH] Enable scoped Qwen3.8 Omni inference and CPU TTS compatibility --- benchmarks/edge_harness/probe_qwen35_image.py | 71 +++++ .../results/e2e_recovery_20260923/README.md | 16 + .../evidence/qwen_image.json | 22 ++ .../evidence/qwen_text.json | 22 ++ .../evidence/tts_stream_report.json | 289 ++++++++++++++++++ .../evidence/tts_stream_requests.jsonl | 4 + .../evidence/tts_two_frame.json | 10 + docs/design/omni_edge_e2e_roadmap.md | 5 + tests/config/test_omni_config.py | 17 ++ tests/config/test_vllm_native_pipelines.py | 11 +- tests/core/test_prefix_cache.py | 36 ++- vllm_omni/config/omni_config.py | 5 + vllm_omni/config/pipeline_registry.py | 3 +- vllm_omni/config/vllm_native_pipelines.py | 17 +- vllm_omni/core/prefix_cache.py | 26 +- vllm_omni/worker/sampling_utils.py | 19 +- 16 files changed, 555 insertions(+), 18 deletions(-) create mode 100644 benchmarks/edge_harness/probe_qwen35_image.py create mode 100644 benchmarks/edge_harness/results/e2e_recovery_20260923/README.md create mode 100644 benchmarks/edge_harness/results/e2e_recovery_20260923/evidence/qwen_image.json create mode 100644 benchmarks/edge_harness/results/e2e_recovery_20260923/evidence/qwen_text.json create mode 100644 benchmarks/edge_harness/results/e2e_recovery_20260923/evidence/tts_stream_report.json create mode 100644 benchmarks/edge_harness/results/e2e_recovery_20260923/evidence/tts_stream_requests.jsonl create mode 100644 benchmarks/edge_harness/results/e2e_recovery_20260923/evidence/tts_two_frame.json diff --git a/benchmarks/edge_harness/probe_qwen35_image.py b/benchmarks/edge_harness/probe_qwen35_image.py new file mode 100644 index 000000000..9a210fab3 --- /dev/null +++ b/benchmarks/edge_harness/probe_qwen35_image.py @@ -0,0 +1,71 @@ +"""One real-checkpoint Omni image-to-text probe with a synthetic red image.""" + +import argparse +import json +import os +import time +import traceback +from pathlib import Path + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--model", required=True) + parser.add_argument("--out", type=Path, required=True) + args = parser.parse_args() + os.environ.setdefault("VLLM_USE_FLASHINFER_SAMPLER", "0") + os.environ.setdefault("VLLM_WSL2_ENABLE_PIN_MEMORY", "1") + record = {"model": args.model, "scope": "one synthetic-image Omni image-to-text request", "start": time.time()} + engine = None + try: + from PIL import Image + from transformers import AutoProcessor + from vllm import SamplingParams + + from vllm_omni.entrypoints.omni import Omni + from vllm_omni.windows.aio import install_selector_policy + + install_selector_policy() + processor = AutoProcessor.from_pretrained(args.model, local_files_only=True) + image = Image.new("RGB", (96, 96), (255, 0, 0)) + messages = [{"role": "user", "content": [ + {"type": "image", "image": image}, + {"type": "text", "text": "What is the dominant color in the image? Answer in one word."}, + ]}] + prompt = processor.apply_chat_template( + messages, tokenize=False, add_generation_prompt=True, enable_thinking=False + ) + config = { + "model": args.model, + "language_model_only": False, + "max_model_len": 512, + "max_num_seqs": 1, + "max_num_batched_tokens": 512, + "enforce_eager": True, + "gpu_memory_utilization": 0.85, + "cpu_offload_gb": 2, + "dtype": "bfloat16", + "stage_init_timeout": 180, + } + record["config"] = config + engine = Omni(**config) + outputs = engine.generate( + {"prompt": prompt, "multi_modal_data": {"image": image}}, + SamplingParams(temperature=0, max_tokens=16), + ) + record["outputs"] = [out.outputs[0].text for out in outputs if out.outputs] + if not record["outputs"]: + raise RuntimeError("No image-conditioned output") + record["status"] = "completed" + except Exception as exc: + record.update(status="failed", error=repr(exc), traceback=traceback.format_exc()) + traceback.print_exc() + finally: + if engine is not None: + engine.close() + record["end"] = time.time() + args.out.write_text(json.dumps(record, indent=2) + "\n", encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/benchmarks/edge_harness/results/e2e_recovery_20260923/README.md b/benchmarks/edge_harness/results/e2e_recovery_20260923/README.md new file mode 100644 index 000000000..6ef886176 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_recovery_20260923/README.md @@ -0,0 +1,16 @@ +# 2026-09-23 local E2E recovery + +This is a follow-up to the [60-cell audit](../e2e_profiling_20260922/README.md). It records newly runnable **scoped** paths and checkpoint acquisition; it does not re-label untested device/model combinations as qualified. Checkpoints are stored outside Git under `/home/zhout/project/edge_infer/models` or the local Hugging Face cache. + +| Device / model / artifact | Verified result | Remaining gate | +|---|---|---| +| WSL CPU / Qwen3-TTS 0.6B CustomVoice, revision `85e237c12c027371202489a0ec509ded67b5e4b5` | Two-frame real-weight PCM probe passed (1,920 nonzero finite samples). Full short-utterance streaming completed four requests, including a slow consumer and a request after cancellation. | Measured short request: 3.04 s audio, 17.02 s wall time, RTF 5.60 and four simulated playback underruns. No ASR, speaker, listening or sustained thermal qualification. | +| WSL RTX 5090 Laptop / Qwen3.8-27B NVFP4, local revision `dbb8f445b3145f8a4c18ddc769f032d57d32867c` | Public Omni text request completed 16 tokens. Public Omni image-to-text request with a synthetic red 96×96 image answered `Red`. | One text and one image case only; video, long context, quality suites, power and sustained workload unqualified. | + +Evidence: [text request](evidence/qwen_text.json), [image request](evidence/qwen_image.json), [two-frame audio](evidence/tts_two_frame.json), [stream report](evidence/tts_stream_report.json), [individual stream requests](evidence/tts_stream_requests.jsonl). The image input and exact prompt are reproducible with [the image probe](../../probe_qwen35_image.py). The streaming run used [profile_local_tts.py](../../profile_local_tts.py) with `--length-band short --concurrency 1 --repeats 1 --sustained-seconds 0`; its fixed slow-consumer and cancellation probes also ran. The CPU vLLM wheel was 0.28.0, Omni source reported 0.29.0rc2, and the installed PyTorch CPU build was 2.13.0. Qwen used the installed vLLM 0.29.0 CUDA wheel, WSL kernel `6.18.33.2-microsoft-standard-WSL2`, RTX 5090 Laptop (24,463 MiB), driver 610.71, BF16 compute, eager mode, 2 GiB explicit host offload, one sequence and 512-token context. All values are single-run observations; they are not medians. + +The code changes make vLLM-native `qwen3_5` a one-stage Omni pipeline, project explicit `language_model_only` and `cpu_offload_gb` to the owning stage, skip vLLM's `-1` unmapped-slot sentinel in both prefix-cache write paths, and accept both vLLM 0.28 three-field and 0.29 four-field min-token state. The image probe first refused a 1,024-token context because the available 0.14 GiB KV cache was below the 0.38 GiB requirement. The successful run explicitly used a 512-token context and 0.85 GPU memory utilization. No context or precision was changed silently. + +The missing primary MiniCPM-o 4.5 and InternVLA-A1 checkpoints are being acquired at pinned revisions; no E2E result is claimed until their transfers finish and real checkpoint runs pass. The required InternVLA Cosmos encoder/decoder and Qwen3-VL processor files have been downloaded separately. Existing Spark-X2.5 4B and 1.7B INT8, Qwen3-TTS, Qwen3.8 FP8 and Qwen3.8 NVFP4 artifacts already covered those model families. Additional official Spark 1.7B BF16 and Qwen3.8-27B BF16 transfers were paused when the Hugging Face data path slowed; they are distinct artifact variants, not replacements for the present weights. InternVLA source checkpoints carry a CC BY-NC-SA 4.0 license; the current download is for local research validation. + +The other rows still need actual local devices, a matching complete backend/artifact, memory admission, and model-specific quality and profiling. In particular, no ADB device is attached, native Windows CPU attention operations are missing from the installed extension, AMD iGPU/NPU routes lack a verified stateful whole-model deployment, and there is no qualified joint CPU+iGPU+NPU execution plan. AI Hub component results remain component evidence. diff --git a/benchmarks/edge_harness/results/e2e_recovery_20260923/evidence/qwen_image.json b/benchmarks/edge_harness/results/e2e_recovery_20260923/evidence/qwen_image.json new file mode 100644 index 000000000..605d8875b --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_recovery_20260923/evidence/qwen_image.json @@ -0,0 +1,22 @@ +{ + "model": "/home/zhout/project/edge_infer/models/Qwen3.8-27B-NVFP4", + "scope": "one synthetic-image Omni image-to-text request", + "start": 1790128489.9985843, + "config": { + "model": "/home/zhout/project/edge_infer/models/Qwen3.8-27B-NVFP4", + "language_model_only": false, + "max_model_len": 512, + "max_num_seqs": 1, + "max_num_batched_tokens": 512, + "enforce_eager": true, + "gpu_memory_utilization": 0.85, + "cpu_offload_gb": 2, + "dtype": "bfloat16", + "stage_init_timeout": 180 + }, + "outputs": [ + " Red" + ], + "status": "completed", + "end": 1790128582.8916564 +} diff --git a/benchmarks/edge_harness/results/e2e_recovery_20260923/evidence/qwen_text.json b/benchmarks/edge_harness/results/e2e_recovery_20260923/evidence/qwen_text.json new file mode 100644 index 000000000..1824e2ee6 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_recovery_20260923/evidence/qwen_text.json @@ -0,0 +1,22 @@ +{ + "platform": "Linux-6.18.33.2-microsoft-standard-WSL2-x86_64-with-glibc2.43", + "config": { + "model": "/home/zhout/project/edge_infer/models/Qwen3.8-27B-NVFP4", + "language_model_only": true, + "max_model_len": 512, + "max_num_seqs": 1, + "max_num_batched_tokens": 128, + "enforce_eager": true, + "gpu_memory_utilization": 0.8, + "cpu_offload_gb": 2, + "dtype": "bfloat16", + "stage_init_timeout": 120 + }, + "status": "completed", + "scope": "public Omni, short text only; explicit 2 GiB native offload", + "start": 1790127902.7650325, + "outputs": [ + "OmniRequestOutput(request_id='0_bbb9dc3f-5f14-4eb2-bdde-63131edd4deb', finished=True, stage_id=0, final_output_type='text', prompt_token_ids=[814, 20139, 2136, 42903, 303, 799, 11316, 13], outputs=[CompletionOutput(index=0, text='\\n\\nLocal inference is the process of drawing conclusions or making predictions based solely on information', token_ids=[271, 7090, 42903, 369, 279, 1817, 314, 12945, 29256, 466, 3154, 19292, 3018, 20417, 383, 1928], routed_experts=None, sampling_mask=None, cumulative_logprob=None, logprobs=None, finish_reason=length, stop_reason=None)], images=[], prompt=None, latents=None, metrics={'stage_metrics': {'0': {'stage_id': 0, 'final_output_type': 'text', 'finish_reason': 'length', 'num_tokens_in': 8, 'num_tokens_out': 16, 'stage_gen_time_ms': 4732.486009597778, 'postprocess_time_ms': 0.0, 'audio_frames': 0, 'audio_sample_rate': 0, 'audio_duration_s': 0.0, 'image_pixels': 0, 'denoise_step_latency_ms': 0.0, 'output_unit_type': 'text', 'output_unit_count': 16, 'serving_time_to_first_output_ms': 4751.192092895508, 'image_time_to_first_output_ms': 0.0, 'time_per_output_unit_ms': 0.002590815226236979, 'inter_output_latency_ms': 0.0, 'inter_output_latencies_ms': [], 'vllm_ttft_ms': 0.0, 'vllm_tpot_ms': 0.0, 'vllm_itl_ms': 0.0, 'vllm_itls_ms': [], 'stage_name': 'qwen3_5'}}, 'stage_id': 0, 'final_output_type': 'text', 'num_tokens_in': 8, 'num_tokens_out': 16}, multimodal_output={}, custom_output={}, stage_durations={'queue_wait_ms': 1.3761789887212217, 'preprocess_ms': 17.312199983280152, 'stage_0_gen_ms': 4732.486009597778}, peak_memory_mb=0.0)" + ], + "end": 1790127965.7770884 +} diff --git a/benchmarks/edge_harness/results/e2e_recovery_20260923/evidence/tts_stream_report.json b/benchmarks/edge_harness/results/e2e_recovery_20260923/evidence/tts_stream_report.json new file mode 100644 index 000000000..692ae397d --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_recovery_20260923/evidence/tts_stream_report.json @@ -0,0 +1,289 @@ +{ + "status": "completed", + "argv": [ + "benchmarks/edge_harness/profile_local_tts.py", + "--model", + "/home/zhout/.cache/huggingface/hub/models--Qwen--Qwen3-TTS-12Hz-0.6B-CustomVoice/snapshots/85e237c12c027371202489a0ec509ded67b5e4b5", + "--out", + "/home/zhout/project/edge_infer/analysis/experiments/e2e_recovery_20260923/tts_cpu_full", + "--length-band", + "short", + "--concurrency", + "1", + "--repeats", + "1", + "--sustained-seconds", + "0" + ], + "python": "/home/zhout/project/edge_infer/.venvs/omni-cpu/bin/python", + "host_environment": {}, + "platform": "Linux-6.18.33.2-microsoft-standard-WSL2-x86_64-with-glibc2.43", + "runtime": { + "python": "3.12.13", + "platform": "Linux-6.18.33.2-microsoft-standard-WSL2-x86_64-with-glibc2.43", + "torch": "2.13.0+cpu", + "vllm": "0.28.0", + "vllm_omni": "0.29.0rc2.dev71+g00510ce38.d20260922", + "vllm_omni_file": "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/__init__.py", + "vllm_file": "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/__init__.py", + "checkouts": { + "/home/zhout/project/edge_infer/vllm-omni-edge": "75b65f859b3c2210c6bebae6e2aa6a796fc97153-dirty", + "/home/zhout/project/edge_infer": "758b3af238b19e2c2abc668f23a767bb835a4a22-dirty" + } + }, + "settings": { + "model": "/home/zhout/.cache/huggingface/hub/models--Qwen--Qwen3-TTS-12Hz-0.6B-CustomVoice/snapshots/85e237c12c027371202489a0ec509ded67b5e4b5", + "out": "/home/zhout/project/edge_infer/analysis/experiments/e2e_recovery_20260923/tts_cpu_full", + "repeats": 1, + "sustained_seconds": 0.0, + "length_band": "short", + "concurrency": 1 + }, + "start_unix": 1790128148.8782187, + "requests": 4, + "cache_condition": "existing disk/JIT caches; not cold disk", + "quality_gate": "finite PCM checks only; ASR/speaker/listening reference gates pending", + "playback_scope": "arrival-based simulated playback; no sound-device measurement", + "terminal_audio": "DELTA output required; terminal audio is included once in complete playback metrics and archived separately", + "prompts": { + "short": { + "prompt_token_ids": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "additional_information": { + "task_type": [ + "CustomVoice" + ], + "text": [ + "Hello, this is a speech test." + ], + "language": [ + "English" + ], + "speaker": [ + "Vivian" + ], + "global_request_id": [ + "after_cancel-short-1-4-8721e445" + ] + } + }, + "medium": { + "prompt_token_ids": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "additional_information": { + "task_type": [ + "CustomVoice" + ], + "text": [ + "The garden has trees, flowers, and a small pond. We walk along the path and listen to the birds." + ], + "language": [ + "English" + ], + "speaker": [ + "Vivian" + ], + "global_request_id": [ + "slow_consumer-medium-1-3-9bfebce4" + ] + } + }, + "long": { + "prompt_token_ids": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "additional_information": { + "task_type": [ + "CustomVoice" + ], + "text": [ + "Today we are testing a local speech system. It should speak clearly and keep the audio flowing smoothly. Today we are testing a local speech system. It should speak clearly and keep the audio flowing smoothly. Today we are testing a local speech system. It should speak clearly and keep the audio flowing smoothly. Today we are testing a local speech system. It should speak clearly and keep the audio flowing smoothly. " + ], + "language": [ + "English" + ], + "speaker": [ + "Vivian" + ], + "global_request_id": [ + "cancel-probe-b740da24" + ] + } + } + }, + "startup_s": 31.13160807598615, + "sampling_params": [ + "SamplingParams(n=1, presence_penalty=0.0, frequency_penalty=0.0, repetition_penalty=1.05, temperature=0.9, top_p=1.0, top_k=50, min_p=0.0, seed=None, stop=[], stop_token_ids=[2150], bad_words=[], thinking_token_budget=None, include_stop_str_in_output=False, ignore_eos=False, max_tokens=4096, min_tokens=2, logprobs=None, prompt_logprobs=None, skip_special_tokens=True, spaces_between_special_tokens=True, structured_outputs=None, extra_args=None)", + "SamplingParams(n=1, presence_penalty=0.0, frequency_penalty=0.0, repetition_penalty=1.0, temperature=0.0, top_p=1.0, top_k=0, min_p=0.0, seed=None, stop=[], stop_token_ids=[], bad_words=[], thinking_token_budget=None, include_stop_str_in_output=False, ignore_eos=False, max_tokens=65536, min_tokens=0, logprobs=None, prompt_logprobs=None, skip_special_tokens=True, spaces_between_special_tokens=True, structured_outputs=None, extra_args=None)" + ], + "memory_after_load": { + "samples": 96, + "interval_s": 0.25, + "note": "peaks are lower bounds: anything shorter than interval_s is missed. nvml_device_used is the whole card including other processes; tree_pss includes a share of file-backed pages. These three counters are different accountings and must not be summed.", + "nvml_device_used_peak": null, + "nvml_device_used_min": null, + "tree_rss_peak": 8744435712, + "tree_pss_peak": 7600484352, + "host_available_min": 22244052992 + }, + "sustained_wall_s": 0.0, + "cancel": { + "abort_ack_s": 0.005321010015904903, + "scope": "abort acknowledgement and subsequent request; late-event gate not verified" + }, + "memory": { + "samples": 362, + "interval_s": 0.25, + "note": "peaks are lower bounds: anything shorter than interval_s is missed. nvml_device_used is the whole card including other processes; tree_pss includes a share of file-backed pages. These three counters are different accountings and must not be summed.", + "nvml_device_used_peak": null, + "nvml_device_used_min": null, + "tree_rss_peak": 8744435712, + "tree_pss_peak": 7600484352, + "host_available_min": 19052163072 + }, + "end_unix": 1790128293.2186244 +} diff --git a/benchmarks/edge_harness/results/e2e_recovery_20260923/evidence/tts_stream_requests.jsonl b/benchmarks/edge_harness/results/e2e_recovery_20260923/evidence/tts_stream_requests.jsonl new file mode 100644 index 000000000..a9e9a7cef --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_recovery_20260923/evidence/tts_stream_requests.jsonl @@ -0,0 +1,4 @@ +{"request_id": "warmup-short-1-1", "phase": "warmup", "length_band": "short", "submitted_unix": 1790128180.2599764, "finished_unix": 1790128201.87287, "concurrency": 1, "sr": 24000, "chunks_all": [{"idx": 0, "t_ms": 6325.3543300088495, "samples": 3840, "sr": 24000, "request_id": "warmup-short-1-1", "terminal": false, "sha256": "611772f6716d80def42642c696ab48f1ca0d69d5e2006d12b26fc9278bb951e5", "finite": true}, {"idx": 1, "t_ms": 7813.125300046522, "samples": 7680, "sr": 24000, "request_id": "warmup-short-1-1", "terminal": false, "sha256": "ded437f7a341593a7cf5912c5797d254e25981e4876a92715cbab239ca109196", "finite": true}, {"idx": 2, "t_ms": 11184.011097997427, "samples": 15360, "sr": 24000, "request_id": "warmup-short-1-1", "terminal": false, "sha256": "8d4a5a001369bf8f8c5eee9eae416123249a91ab334eaa53b8a90ec426e68aab", "finite": true}, {"idx": 3, "t_ms": 17789.24036800163, "samples": 30720, "sr": 24000, "request_id": "warmup-short-1-1", "terminal": false, "sha256": "a3ac8dfb6771da4bcbbd962084a8f6c990e8eb0b508e59adfac296a713cfd774", "finite": true}, {"idx": 4, "t_ms": 21612.170481006615, "samples": 19200, "sr": 24000, "request_id": "warmup-short-1-1", "terminal": true, "sha256": "e45b6e57e7703615604e5d57249849e3410bd18bd273f35e46b513e0ecaf8cba", "finite": true}], "finished": true, "terminal_audio_s": 0.8, "prefix_metrics": {"ttfa_ms": 6325.3543300088495, "playback_start_ms": 16669.24036800163, "stall_at_ttfa_ms": 10343.88603799278, "n_chunks": 4, "streamed_audio_s": 2.4, "tail_audio_s": 0.0, "total_wall_s": 21.612890518037602, "rtf_streamed": 7.412183486667345, "rtf_total": 9.005371049182335, "gap_ms": [1487.7709700376727, 3370.885797950905, 6605.229270004202]}, "ttfa_ms": 6325.3543300088495, "playback_start_ms": 19212.170481006615, "stall_at_ttfa_ms": 12886.816150997765, "n_chunks": 5, "streamed_audio_s": 3.2, "tail_audio_s": 0.0, "total_wall_s": 21.612890518037602, "rtf_streamed": 6.753803275314567, "rtf_total": 6.754028286886751, "gap_ms": [1487.7709700376727, 3370.885797950905, 6605.229270004202, 3822.930113004986], "underruns": [[1, 1327.7709700376727], [2, 3050.885797950905], [3, 5965.229270004202], [4, 2542.930113004986]]} +{"request_id": "measured-short-1-2", "phase": "measured", "length_band": "short", "submitted_unix": 1790128201.8761928, "finished_unix": 1790128218.8951125, "concurrency": 1, "sr": 24000, "chunks_all": [{"idx": 0, "t_ms": 1120.8805019850843, "samples": 3840, "sr": 24000, "request_id": "measured-short-1-2", "terminal": false, "sha256": "09fba210f2c2fbc70764a4105247657db958ad05b89de0093bb03899ced744fa", "finite": true}, {"idx": 1, "t_ms": 3047.4736919859424, "samples": 7680, "sr": 24000, "request_id": "measured-short-1-2", "terminal": false, "sha256": "ba3275b0b8a8b6c40ee2071823c95e0d8f960db36aa5dce34a274bce2257dcee", "finite": true}, {"idx": 2, "t_ms": 7024.616775976028, "samples": 15360, "sr": 24000, "request_id": "measured-short-1-2", "terminal": false, "sha256": "50c9d2438d3f66ff937f1b87ba2501399361c1dc0f86db597882dd753f305675", "finite": true}, {"idx": 3, "t_ms": 13397.165296948515, "samples": 30720, "sr": 24000, "request_id": "measured-short-1-2", "terminal": false, "sha256": "b506f53077b79bb319eda197e524e2c30b5a64f77f19ce316ef5b3ec9bd734df", "finite": true}, {"idx": 4, "t_ms": 17018.59503495507, "samples": 15360, "sr": 24000, "request_id": "measured-short-1-2", "terminal": true, "sha256": "6a851fec2bebbf53e80f85506649ec7a77cda587e83936b48bd4583cd5058432", "finite": true}], "finished": true, "terminal_audio_s": 0.64, "prefix_metrics": {"ttfa_ms": 1120.8805019850843, "playback_start_ms": 12277.165296948515, "stall_at_ttfa_ms": 11156.28479496343, "n_chunks": 4, "streamed_audio_s": 2.4, "tail_audio_s": 0.0, "total_wall_s": 17.01891711598728, "rtf_streamed": 5.582152207061881, "rtf_total": 7.0912154649947, "gap_ms": [1926.5931900008582, 3977.143083990086, 6372.548520972487]}, "ttfa_ms": 1120.8805019850843, "playback_start_ms": 14618.59503495507, "stall_at_ttfa_ms": 13497.714532969985, "n_chunks": 5, "streamed_audio_s": 3.04, "tail_audio_s": 0.0, "total_wall_s": 17.01891711598728, "rtf_streamed": 5.598222050972062, "rtf_total": 5.598327998680026, "gap_ms": [1926.5931900008582, 3977.143083990086, 6372.548520972487, 3621.4297380065545], "underruns": [[1, 1766.5931900008582], [2, 3657.143083990086], [3, 5732.548520972487], [4, 2341.4297380065545]]} +{"request_id": "slow_consumer-medium-1-3", "phase": "slow_consumer", "length_band": "medium", "submitted_unix": 1790128218.8962007, "finished_unix": 1790128261.7518387, "concurrency": 1, "sr": 24000, "chunks_all": [{"idx": 0, "t_ms": 1179.83761994401, "samples": 3840, "sr": 24000, "request_id": "slow_consumer-medium-1-3", "terminal": false, "sha256": "0e2e70410d7aa4667b5c50a63629497cdfa68a49f099f85f452558cbda14b160", "finite": true}, {"idx": 1, "t_ms": 2969.4679499953054, "samples": 7680, "sr": 24000, "request_id": "slow_consumer-medium-1-3", "terminal": false, "sha256": "808116d564596b39c5f761022a05c1de083f2e9e5f6f8cdbc1e4198c54635f95", "finite": true}, {"idx": 2, "t_ms": 6295.096705958713, "samples": 15360, "sr": 24000, "request_id": "slow_consumer-medium-1-3", "terminal": false, "sha256": "d638597ac9dd75f4533ca8fe2d082096e13c759fdb311d19d699293efa7e19b5", "finite": true}, {"idx": 3, "t_ms": 13201.034414989408, "samples": 30720, "sr": 24000, "request_id": "slow_consumer-medium-1-3", "terminal": false, "sha256": "59d23b52373cb0db104b21662bf31c4d9fda90fe16eadd992e3c9fea0b9b7d30", "finite": true}, {"idx": 4, "t_ms": 25575.97459194949, "samples": 48000, "sr": 24000, "request_id": "slow_consumer-medium-1-3", "terminal": false, "sha256": "431801ed2529e8f0aaef2103c7e16637b0c8a6c1f73501187acf5b2cbb1fe109", "finite": true}, {"idx": 5, "t_ms": 35489.57968194736, "samples": 48000, "sr": 24000, "request_id": "slow_consumer-medium-1-3", "terminal": false, "sha256": "e32e516425eca3e1af02d98a90772dbf86cc583d8da500f4c2c005bbe91be6bd", "finite": true}, {"idx": 6, "t_ms": 42654.40870099701, "samples": 36480, "sr": 24000, "request_id": "slow_consumer-medium-1-3", "terminal": true, "sha256": "458a53beeccf2057f0c75165d44bd71bd4fefcc946b7c4d9b16a2c0b22a8a9d0", "finite": true}], "finished": true, "terminal_audio_s": 1.52, "prefix_metrics": {"ttfa_ms": 1179.83761994401, "playback_start_ms": 31089.57968194736, "stall_at_ttfa_ms": 29909.74206200335, "n_chunks": 6, "streamed_audio_s": 6.4, "tail_audio_s": 0.0, "total_wall_s": 42.85563437797828, "rtf_streamed": 5.545246825304275, "rtf_total": 6.696192871559106, "gap_ms": [1789.6303300512955, 3325.6287559634075, 6905.937709030695, 12374.94017696008, 9913.605089997873]}, "ttfa_ms": 1179.83761994401, "playback_start_ms": 36254.40870099701, "stall_at_ttfa_ms": 35074.571081053, "n_chunks": 7, "streamed_audio_s": 7.92, "tail_audio_s": 0.0, "total_wall_s": 42.85563437797828, "rtf_streamed": 5.385657664267299, "rtf_total": 5.411064946714429, "gap_ms": [1789.6303300512955, 3325.6287559634075, 6905.937709030695, 12374.94017696008, 9913.605089997873, 7164.829019049648], "underruns": [[1, 1629.6303300512955], [2, 3005.6287559634075], [3, 6265.937709030695], [4, 11094.94017696008], [5, 7913.605089997873], [6, 5164.829019049648]]} +{"request_id": "after_cancel-short-1-4", "phase": "after_cancel", "length_band": "short", "submitted_unix": 1790128262.9201438, "finished_unix": 1790128283.1748633, "concurrency": 1, "sr": 24000, "chunks_all": [{"idx": 0, "t_ms": 1340.9332330338657, "samples": 3840, "sr": 24000, "request_id": "after_cancel-short-1-4", "terminal": false, "sha256": "160202697eaff37db831d7c52316001783920f20c26874517cacd3e196fca957", "finite": true}, {"idx": 1, "t_ms": 3139.497352007311, "samples": 7680, "sr": 24000, "request_id": "after_cancel-short-1-4", "terminal": false, "sha256": "fca24302b351b4acd2a7068e3f46192b04e1c2ff9433f67e5b66687432387be2", "finite": true}, {"idx": 2, "t_ms": 6879.7519620275125, "samples": 15360, "sr": 24000, "request_id": "after_cancel-short-1-4", "terminal": false, "sha256": "9640fe140ee76d90ee40384c2db910a32202cd032ccae9826f22af798282af8c", "finite": true}, {"idx": 3, "t_ms": 14240.559764031786, "samples": 30720, "sr": 24000, "request_id": "after_cancel-short-1-4", "terminal": false, "sha256": "6c7d2a8f0e985dd2852eb871708a63cca07b98de6948f0cc1338a3ea5415f862", "finite": true}, {"idx": 4, "t_ms": 20254.40322502982, "samples": 23040, "sr": 24000, "request_id": "after_cancel-short-1-4", "terminal": true, "sha256": "33c0f46875be1f8c3aaeeb3ab6e7b85b5653b3124142f93f835ff59e1dff427f", "finite": true}], "finished": true, "terminal_audio_s": 0.96, "prefix_metrics": {"ttfa_ms": 1340.9332330338657, "playback_start_ms": 13120.559764031786, "stall_at_ttfa_ms": 11779.62653099792, "n_chunks": 4, "streamed_audio_s": 2.4, "tail_audio_s": 0.0, "total_wall_s": 20.25471683003707, "rtf_streamed": 5.933566568346578, "rtf_total": 8.43946534584878, "gap_ms": [1798.5641189734451, 3740.2546100202017, 7360.807802004274]}, "ttfa_ms": 1340.9332330338657, "playback_start_ms": 17854.40322502982, "stall_at_ttfa_ms": 16513.469991995953, "n_chunks": 5, "streamed_audio_s": 3.36, "tail_audio_s": 0.0, "total_wall_s": 20.25471683003707, "rtf_streamed": 6.028096197925541, "rtf_total": 6.028189532749128, "gap_ms": [1798.5641189734451, 3740.2546100202017, 7360.807802004274, 6013.843460998032], "underruns": [[1, 1638.5641189734451], [2, 3420.2546100202017], [3, 6720.807802004274], [4, 4733.843460998032]]} diff --git a/benchmarks/edge_harness/results/e2e_recovery_20260923/evidence/tts_two_frame.json b/benchmarks/edge_harness/results/e2e_recovery_20260923/evidence/tts_two_frame.json new file mode 100644 index 000000000..8fbd53e86 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_recovery_20260923/evidence/tts_two_frame.json @@ -0,0 +1,10 @@ +{ + "model": "/home/zhout/.cache/huggingface/hub/models--Qwen--Qwen3-TTS-12Hz-0.6B-CustomVoice/snapshots/85e237c12c027371202489a0ec509ded67b5e4b5", + "scope": "two codec frames only; no speech-quality claim", + "start": 1790127913.8721886, + "status": "completed", + "samples": 1920, + "finite": true, + "nonzero": true, + "end": 1790128048.1452236 +} diff --git a/docs/design/omni_edge_e2e_roadmap.md b/docs/design/omni_edge_e2e_roadmap.md index 8951b2306..73bee53c5 100644 --- a/docs/design/omni_edge_e2e_roadmap.md +++ b/docs/design/omni_edge_e2e_roadmap.md @@ -6,6 +6,11 @@ feasibility/implementation blocker. It does not promise that every model fits every device. The [current profiling review](../../benchmarks/edge_harness/results/e2e_profiling_20260922/README.md) records measured execution separately from release qualification. +The [2026-09-23 recovery run](../../benchmarks/edge_harness/results/e2e_recovery_20260923/README.md) +adds a functional CPU TTS stream with playback underruns and a scoped RTX +Qwen3.8 text/image pass. Its missing-checkpoint transfers and remaining device +gates are tracked there; the 2026-09-22 audit remains the historical baseline. + ## Architecture and prerequisites Keep Omni's PipelineConfig, StageRuntime, StageClient, orchestration and admission diff --git a/tests/config/test_omni_config.py b/tests/config/test_omni_config.py index 40229f521..4d980027a 100644 --- a/tests/config/test_omni_config.py +++ b/tests/config/test_omni_config.py @@ -632,6 +632,8 @@ def test_vllm_omni_stage_config_public_fields_use_typed_stage_realizations(): def test_runtime_config_fields_match_structured_runtime_scope(): assert {f.name for f in fields(OmniStageRuntimeConfig)} == { "additional_config", + "backend", + "resource_budget", "distributed_executor_backend", "worker_cls", "devices", @@ -711,6 +713,8 @@ def test_sub_config_fields_match_structured_scopes(): "task_type", "codec_frame_rate_hz", "enforce_eager", + "language_model_only", + "cpu_offload_gb", "max_cudagraph_capture_size", "enable_flashinfer_autotune", "enable_multithread_weight_load", @@ -1273,6 +1277,19 @@ def test_from_pipeline_config_derives_has_sampling_extra_args_from_stage_default assert stage.model_config.has_sampling_extra_args is True +def test_qwen3_5_large_model_execution_overrides_have_a_stage_owner(): + from vllm_omni.engine.stage_init_utils import _project_omni_stage_engine_args + + stage = _from_pipeline_key( + "qwen3_5", + cli_overrides={"language_model_only": True, "cpu_offload_gb": 2.0}, + ).stage_by_id(0) + + args = _project_omni_stage_engine_args(stage) + assert args["language_model_only"] is True + assert args["cpu_offload_gb"] == 2.0 + + def test_diffusion_config_preserves_existing_coercion_hooks(): import torch diff --git a/tests/config/test_vllm_native_pipelines.py b/tests/config/test_vllm_native_pipelines.py index 9ef13a66e..093fdc26f 100644 --- a/tests/config/test_vllm_native_pipelines.py +++ b/tests/config/test_vllm_native_pipelines.py @@ -10,11 +10,9 @@ opaquely. Spark-X2.5 hit exactly that; these tests pin the general case. from __future__ import annotations -import pytest - from vllm_omni.config.pipeline_registry import OMNI_PIPELINES from vllm_omni.config.stage_config import StageExecutionType -from vllm_omni.config.vllm_native_pipelines import LLAMA_PIPELINE +from vllm_omni.config.vllm_native_pipelines import LLAMA_PIPELINE, QWEN3_5_PIPELINE def test_llama_resolves_from_the_registry(): @@ -48,3 +46,10 @@ def test_registering_llama_is_keyed_on_architecture_not_checkpoint(): particular weights.""" assert LLAMA_PIPELINE.model_type == "llama" assert LLAMA_PIPELINE.hf_architectures == ("LlamaForCausalLM",) + + +def test_qwen3_5_conditional_generation_resolves(): + assert OMNI_PIPELINES["qwen3_5"] is QWEN3_5_PIPELINE + assert QWEN3_5_PIPELINE.get_validation_errors() == [] + assert QWEN3_5_PIPELINE.hf_architectures == ("Qwen3_5ForConditionalGeneration",) + assert QWEN3_5_PIPELINE.stages[0].execution_type is StageExecutionType.LLM_AR diff --git a/tests/core/test_prefix_cache.py b/tests/core/test_prefix_cache.py index 4f91a95b7..79924f44a 100644 --- a/tests/core/test_prefix_cache.py +++ b/tests/core/test_prefix_cache.py @@ -4,7 +4,7 @@ from typing import NamedTuple import pytest import torch -from vllm_omni.core.prefix_cache import OmniTensorPrefixCache +from vllm_omni.core.prefix_cache import OmniTensorPrefixCache, _PendingAsyncWrite pytestmark = [pytest.mark.core_model, pytest.mark.cpu] @@ -163,6 +163,40 @@ def test_update_no_multimodal(): assert torch.all(slot_states == new_states) +def test_pending_write_skips_vllm_unmapped_slot_sentinel(): + class CompletedEvent: + def synchronize(self): + pass + + cache = get_omni_pcache_with_mm_tensors({"vision": HIDDEN_SIZE}, seq_len=3) + cache._pending_write = _PendingAsyncWrite( + event=CompletedEvent(), + num_tokens=3, + slots_cpu=torch.tensor([8, -1, 9], dtype=torch.int32), + hidden_cpu=torch.tensor([[1.0, 1.0], [99.0, 99.0], [2.0, 2.0]]), + mm_cpu={"vision": torch.tensor([[3.0, 3.0], [99.0, 99.0], [4.0, 4.0]])}, + ) + + cache._consume_pending_write() + + assert cache._pending_write is None + assert cache.hidden_states_cache.view(-1, HIDDEN_SIZE)[8:10].tolist() == [[1.0, 1.0], [2.0, 2.0]] + assert cache.mm_outputs_cache["vision"].view(-1, HIDDEN_SIZE)[8:10].tolist() == [[3.0, 3.0], [4.0, 4.0]] + + +def test_synchronous_write_skips_vllm_unmapped_slot_sentinel(): + cache = get_omni_pcache_with_mm_tensors({"vision": HIDDEN_SIZE}, seq_len=3) + cache.update_omni_tensor_prefix_cache( + hidden_states=torch.tensor([[1.0, 1.0], [99.0, 99.0], [2.0, 2.0]]), + multimodal_outputs={"vision": torch.tensor([[3.0, 3.0], [99.0, 99.0], [4.0, 4.0]])}, + num_tokens_unpadded=3, + slot_mapping=torch.tensor([8, -1, 9], dtype=torch.int32), + ) + + assert cache.hidden_states_cache.view(-1, HIDDEN_SIZE)[8:10].tolist() == [[1.0, 1.0], [2.0, 2.0]] + assert cache.mm_outputs_cache["vision"].view(-1, HIDDEN_SIZE)[8:10].tolist() == [[3.0, 3.0], [4.0, 4.0]] + + def test_update_uses_precomputed_hidden_states_cpu(): """Precomputed CPU staging should be the only hidden-state cache source.""" cache = get_omni_pcache() diff --git a/vllm_omni/config/omni_config.py b/vllm_omni/config/omni_config.py index 36bd6880e..166309caf 100644 --- a/vllm_omni/config/omni_config.py +++ b/vllm_omni/config/omni_config.py @@ -168,6 +168,9 @@ class _ModelEngineOverrides(TypedDict, total=False): enable_multithread_weight_load: bool num_weight_load_threads: int disable_autocast: bool + # vLLM model execution switches needed by large conditional generators. + language_model_only: bool + cpu_offload_gb: float class _LoadEngineOverrides(TypedDict, total=False): @@ -440,6 +443,8 @@ class OmniStageModelConfig(_TrackExplicitConfigFields): enable_multithread_weight_load: bool = True num_weight_load_threads: int = Field(default=4, ge=1) disable_autocast: bool = False + language_model_only: bool | None = None + cpu_offload_gb: float | None = Field(default=None, ge=0.0) # Per-stage checkpoint/tokenizer subdirectories under the model root # (e.g. Audex stage 0 → checkpoint_folder_audiogen). Mirrors # StagePipelineConfig.model_subdir/tokenizer_subdir on the legacy path. diff --git a/vllm_omni/config/pipeline_registry.py b/vllm_omni/config/pipeline_registry.py index eeddb09e1..d6b08a0b8 100644 --- a/vllm_omni/config/pipeline_registry.py +++ b/vllm_omni/config/pipeline_registry.py @@ -32,6 +32,7 @@ from vllm.logger import init_logger from vllm_omni.config.stage_config import ( PipelineConfig, ) +from vllm_omni.config.vllm_native_pipelines import LLAMA_PIPELINE, QWEN3_5_PIPELINE from vllm_omni.diffusion.models.pi0_pipeline_config import PI0_PIPELINE from vllm_omni.model_executor.models.audex.pipeline import ( AUDEX_S2S_PIPELINE, @@ -111,7 +112,6 @@ from vllm_omni.model_executor.models.qwen3_omni.pipeline import ( resolve_qwen3_omni_pipeline, ) from vllm_omni.model_executor.models.qwen3_tts.pipeline import QWEN3_TTS_PIPELINE, QWEN3_TTS_TALKER_ONLY_PIPELINE -from vllm_omni.config.vllm_native_pipelines import LLAMA_PIPELINE from vllm_omni.model_executor.models.spark2_5.pipeline import SPARK2_5_PIPELINE from vllm_omni.model_executor.models.step_audio2.pipeline import ( STEP_AUDIO2_ASR_PIPELINE, @@ -145,6 +145,7 @@ OMNI_PIPELINES: dict[str, PipelineConfig | PipelineResolverFunc] = { # the same registry as the multi-stage pipelines so that one code path # resolves every model (see the M0 local text mode). "llama": LLAMA_PIPELINE, + "qwen3_5": QWEN3_5_PIPELINE, "spark2_5": SPARK2_5_PIPELINE, "step_audio_2": STEP_AUDIO2_PIPELINE, "step_audio_2_asr": STEP_AUDIO2_ASR_PIPELINE, diff --git a/vllm_omni/config/vllm_native_pipelines.py b/vllm_omni/config/vllm_native_pipelines.py index 201585e78..d5a81bfd2 100644 --- a/vllm_omni/config/vllm_native_pipelines.py +++ b/vllm_omni/config/vllm_native_pipelines.py @@ -15,8 +15,8 @@ the orchestrator resolves a pipeline by ``model_type`` first, and a hit exactly this in M0 -- its model definition had been registered for a while and ``Omni(model=)`` still could not resolve it. -**One stage, and the model_type is the key.** These are autoregressive causal -language models with a single stage that owns the tokenizer and emits text. +**One stage, and the model_type is the key.** These are autoregressive vLLM +models with a single stage that owns the tokenizer and emits text. Because the registry is keyed on ``model_type`` rather than on a checkpoint, registering ``llama`` covers every checkpoint whose config says ``llama`` -- not only the one this was added for. That is the correct granularity (the pipeline @@ -71,3 +71,16 @@ and ``architectures: ["LlamaForCausalLM"]`` -- it is a Llama-shaped 2B model runner no longer forcing omni-only kwargs on a forward that cannot take them (see ``OmniGPUModelRunner._accepted_forward_kwargs``). """ + + +QWEN3_5_PIPELINE = _single_stage_text( + model_type="qwen3_5", + model_arch="Qwen3_5ForConditionalGeneration", + hf_architectures=("Qwen3_5ForConditionalGeneration",), +) +"""Single-stage Qwen3.5 generation using vLLM's model executor. + +Qwen3.8-27B declares ``model_type: qwen3_5`` and this architecture. Pipeline +registration makes the checkpoint reachable through Omni; it does not imply +that its image path or any particular checkpoint has passed an E2E test. +""" diff --git a/vllm_omni/core/prefix_cache.py b/vllm_omni/core/prefix_cache.py index 17a981140..25a8ebd3f 100644 --- a/vllm_omni/core/prefix_cache.py +++ b/vllm_omni/core/prefix_cache.py @@ -279,6 +279,18 @@ class OmniTensorPrefixCache: slots = entry.slots_cpu if slots.dtype != torch.int64: slots = slots.to(torch.int64) + # vLLM uses -1 for tokens without a KV slot (for example, a + # non-attention position in a hybrid model). Such tokens have no + # prefix-cache address either. Preserve row alignment while skipping + # them; index_copy_ rejects the sentinel. + if torch.any(slots < -1): + raise ValueError("prefix-cache slot mapping contains an invalid negative index") + valid = slots >= 0 + has_unmapped_slots = not bool(torch.all(valid)) + if has_unmapped_slots: + slots = slots[valid] + if slots.numel() == 0: + return # NOTE: ``index_copy_(0, idx, src)`` is the row-scatter primitive in # PyTorch. It's semantically identical to ``flat[idx] = src`` but @@ -288,13 +300,13 @@ class OmniTensorPrefixCache: # ``aten::index_put_`` path used by advanced-indexing assignment. if entry.hidden_cpu is not None: flat = self.hidden_states_cache.view(-1, self.hidden_states_cache.shape[-1]) - flat.index_copy_(0, slots, entry.hidden_cpu) + flat.index_copy_(0, slots, entry.hidden_cpu[valid] if has_unmapped_slots else entry.hidden_cpu) for k, src_cpu in entry.mm_cpu.items(): mm_cache = self.mm_outputs_cache.get(k) if mm_cache is None: continue flat = mm_cache.view(-1, mm_cache.shape[-1]) - flat.index_copy_(0, slots, src_cpu) + flat.index_copy_(0, slots, src_cpu[valid] if has_unmapped_slots else src_cpu) def add_prefix_cached_new_req_id(self, req_id: str): """Adds a new request ID to the set of prefix cache hits on the batch.""" @@ -366,6 +378,12 @@ class OmniTensorPrefixCache: feature shape of hidden_states, and cover num_tokens_unpadded. """ unpadded_slot_mapping = slot_mapping[:num_tokens_unpadded] + if torch.any(unpadded_slot_mapping < -1): + raise ValueError("prefix-cache slot mapping contains an invalid negative index") + valid_slots = unpadded_slot_mapping >= 0 + has_unmapped_slots = not bool(torch.all(valid_slots)) + if has_unmapped_slots: + unpadded_slot_mapping = unpadded_slot_mapping[valid_slots] if num_tokens_padded is None: num_tokens_padded = num_tokens_unpadded skip_mm_cache_keys = skip_mm_cache_keys or set() @@ -387,7 +405,7 @@ class OmniTensorPrefixCache: slot_idx = unpadded_slot_mapping if slot_idx.dtype != torch.int64: slot_idx = slot_idx.to(torch.int64) - flat_cache.index_copy_(0, slot_idx, hidden_states) + flat_cache.index_copy_(0, slot_idx, hidden_states[valid_slots] if has_unmapped_slots else hidden_states) logger.debug("Writing to hidden states for %s tokens", num_tokens_unpadded) # Do the same for the stage's cached multimodal outputs @@ -410,7 +428,7 @@ class OmniTensorPrefixCache: slot_idx = unpadded_slot_mapping if slot_idx.dtype != torch.int64: slot_idx = slot_idx.to(torch.int64) - flat_cache.index_copy_(0, slot_idx, mm_state) + flat_cache.index_copy_(0, slot_idx, mm_state[valid_slots] if has_unmapped_slots else mm_state) logger.debug("Writing to mm output cache for %s tokens", num_tokens_unpadded) def stage_deferred_mm_outputs( diff --git a/vllm_omni/worker/sampling_utils.py b/vllm_omni/worker/sampling_utils.py index 39ecae469..4ac20d1cb 100644 --- a/vllm_omni/worker/sampling_utils.py +++ b/vllm_omni/worker/sampling_utils.py @@ -48,7 +48,10 @@ def sanitize_min_tokens_stop_ids(logitsprocs: LogitsProcessors, logits_vocab: in if not min_toks: continue needs_rebuild = False - for _, _, stop_tok_ids, _ in min_toks.values(): + # vLLM 0.28 stores three fields; 0.29 adds the structured-output + # flag. The stop-id set is the third field in both versions. + for state in min_toks.values(): + stop_tok_ids = state[2] oob = [tok for tok in stop_tok_ids if tok >= logits_vocab] if not oob: continue @@ -65,17 +68,19 @@ def sanitize_min_tokens_stop_ids(logitsprocs: LogitsProcessors, logits_vocab: in tok_ids: list[int] = [] restore_reqs: list[int] = [] restore_tok_ids: list[int] = [] - for index, (_, _, stop_tok_ids, uses_structured_output) in min_toks.items(): + for index, state in min_toks.items(): + stop_tok_ids = state[2] reqs.extend([index] * len(stop_tok_ids)) tok_ids.extend(stop_tok_ids) - if uses_structured_output: + if len(state) == 4 and state[3]: restore_reqs.extend([index] * len(stop_tok_ids)) restore_tok_ids.extend(stop_tok_ids) proc.logits_slice = ( proc._device_tensor(reqs, torch.int32), proc._device_tensor(tok_ids, torch.int32), ) - proc.restore_logits_slice = ( - proc._device_tensor(restore_reqs, torch.int32), - proc._device_tensor(restore_tok_ids, torch.int32), - ) + if hasattr(proc, "restore_logits_slice"): + proc.restore_logits_slice = ( + proc._device_tensor(restore_reqs, torch.int32), + proc._device_tensor(restore_tok_ids, torch.int32), + )