anodyine/vllm-omni
1
0 Fork 0
Code Issues 4 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

Enable scoped Qwen3.8 Omni inference and CPU TTS compatibility

Browse Source
This commit is contained in:
tzhouam committed 2026-09-23 10:01:21 +08:00
1 parent 75b65f859b
commit b039b6a5bd
16 files changed
+555 -18
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

benchmarks/edge_harness/probe_qwen35_image.py
+71
View File
@@ -0,0 +1,71 @@
"""One real-checkpoint Omni image-to-text probe with a synthetic red image."""
import argparse
import json
import os
import time
import traceback
from pathlib import Path
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--model", required=True)
parser.add_argument("--out", type=Path, required=True)
args = parser.parse_args()
os.environ.setdefault("VLLM_USE_FLASHINFER_SAMPLER", "0")
os.environ.setdefault("VLLM_WSL2_ENABLE_PIN_MEMORY", "1")
record = {"model": args.model, "scope": "one synthetic-image Omni image-to-text request", "start": time.time()}
engine = None
try:
from PIL import Image
from transformers import AutoProcessor
from vllm import SamplingParams
from vllm_omni.entrypoints.omni import Omni
from vllm_omni.windows.aio import install_selector_policy
install_selector_policy()
processor = AutoProcessor.from_pretrained(args.model, local_files_only=True)
image = Image.new("RGB", (96, 96), (255, 0, 0))
messages = [{"role": "user", "content": [
{"type": "image", "image": image},
{"type": "text", "text": "What is the dominant color in the image? Answer in one word."},
]}]
prompt = processor.apply_chat_template(
messages, tokenize=False, add_generation_prompt=True, enable_thinking=False
)
config = {
"model": args.model,
"language_model_only": False,
"max_model_len": 512,
"max_num_seqs": 1,
"max_num_batched_tokens": 512,
"enforce_eager": True,
"gpu_memory_utilization": 0.85,
"cpu_offload_gb": 2,
"dtype": "bfloat16",
"stage_init_timeout": 180,
}
record["config"] = config
engine = Omni(**config)
outputs = engine.generate(
{"prompt": prompt, "multi_modal_data": {"image": image}},
SamplingParams(temperature=0, max_tokens=16),
)
record["outputs"] = [out.outputs[0].text for out in outputs if out.outputs]
if not record["outputs"]:
raise RuntimeError("No image-conditioned output")
record["status"] = "completed"
except Exception as exc:
record.update(status="failed", error=repr(exc), traceback=traceback.format_exc())
traceback.print_exc()
finally:
if engine is not None:
engine.close()
record["end"] = time.time()
args.out.write_text(json.dumps(record, indent=2) + "\n", encoding="utf-8")
if __name__ == "__main__":
main()
benchmarks/edge_harness/results/e2e_recovery_20260923/README.md
+16
View File
@@ -0,0 +1,16 @@
# 2026-09-23 local E2E recovery
This is a follow-up to the [60-cell audit](../e2e_profiling_20260922/README.md). It records newly runnable **scoped** paths and checkpoint acquisition; it does not re-label untested device/model combinations as qualified. Checkpoints are stored outside Git under `/home/zhout/project/edge_infer/models` or the local Hugging Face cache.
| Device / model / artifact | Verified result | Remaining gate |
|---|---|---|
| WSL CPU / Qwen3-TTS 0.6B CustomVoice, revision `85e237c12c027371202489a0ec509ded67b5e4b5` | Two-frame real-weight PCM probe passed (1,920 nonzero finite samples). Full short-utterance streaming completed four requests, including a slow consumer and a request after cancellation. | Measured short request: 3.04 s audio, 17.02 s wall time, RTF 5.60 and four simulated playback underruns. No ASR, speaker, listening or sustained thermal qualification. |
| WSL RTX 5090 Laptop / Qwen3.8-27B NVFP4, local revision `dbb8f445b3145f8a4c18ddc769f032d57d32867c` | Public Omni text request completed 16 tokens. Public Omni image-to-text request with a synthetic red 96×96 image answered `Red`. | One text and one image case only; video, long context, quality suites, power and sustained workload unqualified. |
Evidence: [text request](evidence/qwen_text.json), [image request](evidence/qwen_image.json), [two-frame audio](evidence/tts_two_frame.json), [stream report](evidence/tts_stream_report.json), [individual stream requests](evidence/tts_stream_requests.jsonl). The image input and exact prompt are reproducible with [the image probe](../../probe_qwen35_image.py). The streaming run used [profile_local_tts.py](../../profile_local_tts.py) with `--length-band short --concurrency 1 --repeats 1 --sustained-seconds 0`; its fixed slow-consumer and cancellation probes also ran. The CPU vLLM wheel was 0.28.0, Omni source reported 0.29.0rc2, and the installed PyTorch CPU build was 2.13.0. Qwen used the installed vLLM 0.29.0 CUDA wheel, WSL kernel `6.18.33.2-microsoft-standard-WSL2`, RTX 5090 Laptop (24,463 MiB), driver 610.71, BF16 compute, eager mode, 2 GiB explicit host offload, one sequence and 512-token context. All values are single-run observations; they are not medians.
The code changes make vLLM-native `qwen3_5` a one-stage Omni pipeline, project explicit `language_model_only` and `cpu_offload_gb` to the owning stage, skip vLLM's `-1` unmapped-slot sentinel in both prefix-cache write paths, and accept both vLLM 0.28 three-field and 0.29 four-field min-token state. The image probe first refused a 1,024-token context because the available 0.14 GiB KV cache was below the 0.38 GiB requirement. The successful run explicitly used a 512-token context and 0.85 GPU memory utilization. No context or precision was changed silently.
The missing primary MiniCPM-o 4.5 and InternVLA-A1 checkpoints are being acquired at pinned revisions; no E2E result is claimed until their transfers finish and real checkpoint runs pass. The required InternVLA Cosmos encoder/decoder and Qwen3-VL processor files have been downloaded separately. Existing Spark-X2.5 4B and 1.7B INT8, Qwen3-TTS, Qwen3.8 FP8 and Qwen3.8 NVFP4 artifacts already covered those model families. Additional official Spark 1.7B BF16 and Qwen3.8-27B BF16 transfers were paused when the Hugging Face data path slowed; they are distinct artifact variants, not replacements for the present weights. InternVLA source checkpoints carry a CC BY-NC-SA 4.0 license; the current download is for local research validation.
The other rows still need actual local devices, a matching complete backend/artifact, memory admission, and model-specific quality and profiling. In particular, no ADB device is attached, native Windows CPU attention operations are missing from the installed extension, AMD iGPU/NPU routes lack a verified stateful whole-model deployment, and there is no qualified joint CPU+iGPU+NPU execution plan. AI Hub component results remain component evidence.
benchmarks/edge_harness/results/e2e_recovery_20260923/evidence/qwen_image.json
+22
View File
@@ -0,0 +1,22 @@
{
"model": "/home/zhout/project/edge_infer/models/Qwen3.8-27B-NVFP4",
"scope": "one synthetic-image Omni image-to-text request",
"start": 1790128489.9985843,
"config": {
"model": "/home/zhout/project/edge_infer/models/Qwen3.8-27B-NVFP4",
"language_model_only": false,
"max_model_len": 512,
"max_num_seqs": 1,
"max_num_batched_tokens": 512,
"enforce_eager": true,
"gpu_memory_utilization": 0.85,
"cpu_offload_gb": 2,
"dtype": "bfloat16",
"stage_init_timeout": 180
},
"outputs": [
" Red"
],
"status": "completed",
"end": 1790128582.8916564
}
benchmarks/edge_harness/results/e2e_recovery_20260923/evidence/qwen_text.json
+22
View File
@@ -0,0 +1,22 @@
{
"platform": "Linux-6.18.33.2-microsoft-standard-WSL2-x86_64-with-glibc2.43",
"config": {
"model": "/home/zhout/project/edge_infer/models/Qwen3.8-27B-NVFP4",
"language_model_only": true,
"max_model_len": 512,
"max_num_seqs": 1,
"max_num_batched_tokens": 128,
"enforce_eager": true,
"gpu_memory_utilization": 0.8,
"cpu_offload_gb": 2,
"dtype": "bfloat16",
"stage_init_timeout": 120
},
"status": "completed",
"scope": "public Omni, short text only; explicit 2 GiB native offload",
"start": 1790127902.7650325,
"outputs": [
"OmniRequestOutput(request_id='0_bbb9dc3f-5f14-4eb2-bdde-63131edd4deb', finished=True, stage_id=0, final_output_type='text', prompt_token_ids=[814, 20139, 2136, 42903, 303, 799, 11316, 13], outputs=[CompletionOutput(index=0, text='\\n\\nLocal inference is the process of drawing conclusions or making predictions based solely on information', token_ids=[271, 7090, 42903, 369, 279, 1817, 314, 12945, 29256, 466, 3154, 19292, 3018, 20417, 383, 1928], routed_experts=None, sampling_mask=None, cumulative_logprob=None, logprobs=None, finish_reason=length, stop_reason=None)], images=[], prompt=None, latents=None, metrics={'stage_metrics': {'0': {'stage_id': 0, 'final_output_type': 'text', 'finish_reason': 'length', 'num_tokens_in': 8, 'num_tokens_out': 16, 'stage_gen_time_ms': 4732.486009597778, 'postprocess_time_ms': 0.0, 'audio_frames': 0, 'audio_sample_rate': 0, 'audio_duration_s': 0.0, 'image_pixels': 0, 'denoise_step_latency_ms': 0.0, 'output_unit_type': 'text', 'output_unit_count': 16, 'serving_time_to_first_output_ms': 4751.192092895508, 'image_time_to_first_output_ms': 0.0, 'time_per_output_unit_ms': 0.002590815226236979, 'inter_output_latency_ms': 0.0, 'inter_output_latencies_ms': [], 'vllm_ttft_ms': 0.0, 'vllm_tpot_ms': 0.0, 'vllm_itl_ms': 0.0, 'vllm_itls_ms': [], 'stage_name': 'qwen3_5'}}, 'stage_id': 0, 'final_output_type': 'text', 'num_tokens_in': 8, 'num_tokens_out': 16}, multimodal_output={}, custom_output={}, stage_durations={'queue_wait_ms': 1.3761789887212217, 'preprocess_ms': 17.312199983280152, 'stage_0_gen_ms': 4732.486009597778}, peak_memory_mb=0.0)"
],
"end": 1790127965.7770884
}
benchmarks/edge_harness/results/e2e_recovery_20260923/evidence/tts_stream_report.json
+289
View File
@@ -0,0 +1,289 @@
{
"status": "completed",
"argv": [
"benchmarks/edge_harness/profile_local_tts.py",
"--model",
"/home/zhout/.cache/huggingface/hub/models--Qwen--Qwen3-TTS-12Hz-0.6B-CustomVoice/snapshots/85e237c12c027371202489a0ec509ded67b5e4b5",
"--out",
"/home/zhout/project/edge_infer/analysis/experiments/e2e_recovery_20260923/tts_cpu_full",
"--length-band",
"short",
"--concurrency",
"1",
"--repeats",
"1",
"--sustained-seconds",
"0"
],
"python": "/home/zhout/project/edge_infer/.venvs/omni-cpu/bin/python",
"host_environment": {},
"platform": "Linux-6.18.33.2-microsoft-standard-WSL2-x86_64-with-glibc2.43",
"runtime": {
"python": "3.12.13",
"platform": "Linux-6.18.33.2-microsoft-standard-WSL2-x86_64-with-glibc2.43",
"torch": "2.13.0+cpu",
"vllm": "0.28.0",
"vllm_omni": "0.29.0rc2.dev71+g00510ce38.d20260922",
"vllm_omni_file": "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/__init__.py",
"vllm_file": "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/__init__.py",
"checkouts": {
"/home/zhout/project/edge_infer/vllm-omni-edge": "75b65f859b3c2210c6bebae6e2aa6a796fc97153-dirty",
"/home/zhout/project/edge_infer": "758b3af238b19e2c2abc668f23a767bb835a4a22-dirty"
}
},
"settings": {
"model": "/home/zhout/.cache/huggingface/hub/models--Qwen--Qwen3-TTS-12Hz-0.6B-CustomVoice/snapshots/85e237c12c027371202489a0ec509ded67b5e4b5",
"out": "/home/zhout/project/edge_infer/analysis/experiments/e2e_recovery_20260923/tts_cpu_full",
"repeats": 1,
"sustained_seconds": 0.0,
"length_band": "short",
"concurrency": 1
},
"start_unix": 1790128148.8782187,
"requests": 4,
"cache_condition": "existing disk/JIT caches; not cold disk",
"quality_gate": "finite PCM checks only; ASR/speaker/listening reference gates pending",
"playback_scope": "arrival-based simulated playback; no sound-device measurement",
"terminal_audio": "DELTA output required; terminal audio is included once in complete playback metrics and archived separately",
"prompts": {
"short": {
"prompt_token_ids": [
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
"additional_information": {
"task_type": [
"CustomVoice"
],
"text": [
"Hello, this is a speech test."
],
"language": [
"English"
],
"speaker": [
"Vivian"
],
"global_request_id": [
"after_cancel-short-1-4-8721e445"
]
}
},
"medium": {
"prompt_token_ids": [
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
"additional_information": {
"task_type": [
"CustomVoice"
],
"text": [
"The garden has trees, flowers, and a small pond. We walk along the path and listen to the birds."
],
"language": [
"English"
],
"speaker": [
"Vivian"
],
"global_request_id": [
"slow_consumer-medium-1-3-9bfebce4"
]
}
},
"long": {
"prompt_token_ids": [
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0,
0
],
"additional_information": {
"task_type": [
"CustomVoice"
],
"text": [
"Today we are testing a local speech system. It should speak clearly and keep the audio flowing smoothly. Today we are testing a local speech system. It should speak clearly and keep the audio flowing smoothly. Today we are testing a local speech system. It should speak clearly and keep the audio flowing smoothly. Today we are testing a local speech system. It should speak clearly and keep the audio flowing smoothly. "
],
"language": [
"English"
],
"speaker": [
"Vivian"
],
"global_request_id": [
"cancel-probe-b740da24"
]
}
}
},
"startup_s": 31.13160807598615,
"sampling_params": [
"SamplingParams(n=1, presence_penalty=0.0, frequency_penalty=0.0, repetition_penalty=1.05, temperature=0.9, top_p=1.0, top_k=50, min_p=0.0, seed=None, stop=[], stop_token_ids=[2150], bad_words=[], thinking_token_budget=None, include_stop_str_in_output=False, ignore_eos=False, max_tokens=4096, min_tokens=2, logprobs=None, prompt_logprobs=None, skip_special_tokens=True, spaces_between_special_tokens=True, structured_outputs=None, extra_args=None)",
"SamplingParams(n=1, presence_penalty=0.0, frequency_penalty=0.0, repetition_penalty=1.0, temperature=0.0, top_p=1.0, top_k=0, min_p=0.0, seed=None, stop=[], stop_token_ids=[], bad_words=[], thinking_token_budget=None, include_stop_str_in_output=False, ignore_eos=False, max_tokens=65536, min_tokens=0, logprobs=None, prompt_logprobs=None, skip_special_tokens=True, spaces_between_special_tokens=True, structured_outputs=None, extra_args=None)"
],
"memory_after_load": {
"samples": 96,
"interval_s": 0.25,
"note": "peaks are lower bounds: anything shorter than interval_s is missed. nvml_device_used is the whole card including other processes; tree_pss includes a share of file-backed pages. These three counters are different accountings and must not be summed.",
"nvml_device_used_peak": null,
"nvml_device_used_min": null,
"tree_rss_peak": 8744435712,
"tree_pss_peak": 7600484352,
"host_available_min": 22244052992
},
"sustained_wall_s": 0.0,
"cancel": {
"abort_ack_s": 0.005321010015904903,
"scope": "abort acknowledgement and subsequent request; late-event gate not verified"
},
"memory": {
"samples": 362,
"interval_s": 0.25,
"note": "peaks are lower bounds: anything shorter than interval_s is missed. nvml_device_used is the whole card including other processes; tree_pss includes a share of file-backed pages. These three counters are different accountings and must not be summed.",
"nvml_device_used_peak": null,
"nvml_device_used_min": null,
"tree_rss_peak": 8744435712,
"tree_pss_peak": 7600484352,
"host_available_min": 19052163072
},
"end_unix": 1790128293.2186244
}
benchmarks/edge_harness/results/e2e_recovery_20260923/evidence/tts_stream_requests.jsonl
+4
View File
@@ -0,0 +1,4 @@
{"request_id": "warmup-short-1-1", "phase": "warmup", "length_band": "short", "submitted_unix": 1790128180.2599764, "finished_unix": 1790128201.87287, "concurrency": 1, "sr": 24000, "chunks_all": [{"idx": 0, "t_ms": 6325.3543300088495, "samples": 3840, "sr": 24000, "request_id": "warmup-short-1-1", "terminal": false, "sha256": "611772f6716d80def42642c696ab48f1ca0d69d5e2006d12b26fc9278bb951e5", "finite": true}, {"idx": 1, "t_ms": 7813.125300046522, "samples": 7680, "sr": 24000, "request_id": "warmup-short-1-1", "terminal": false, "sha256": "ded437f7a341593a7cf5912c5797d254e25981e4876a92715cbab239ca109196", "finite": true}, {"idx": 2, "t_ms": 11184.011097997427, "samples": 15360, "sr": 24000, "request_id": "warmup-short-1-1", "terminal": false, "sha256": "8d4a5a001369bf8f8c5eee9eae416123249a91ab334eaa53b8a90ec426e68aab", "finite": true}, {"idx": 3, "t_ms": 17789.24036800163, "samples": 30720, "sr": 24000, "request_id": "warmup-short-1-1", "terminal": false, "sha256": "a3ac8dfb6771da4bcbbd962084a8f6c990e8eb0b508e59adfac296a713cfd774", "finite": true}, {"idx": 4, "t_ms": 21612.170481006615, "samples": 19200, "sr": 24000, "request_id": "warmup-short-1-1", "terminal": true, "sha256": "e45b6e57e7703615604e5d57249849e3410bd18bd273f35e46b513e0ecaf8cba", "finite": true}], "finished": true, "terminal_audio_s": 0.8, "prefix_metrics": {"ttfa_ms": 6325.3543300088495, "playback_start_ms": 16669.24036800163, "stall_at_ttfa_ms": 10343.88603799278, "n_chunks": 4, "streamed_audio_s": 2.4, "tail_audio_s": 0.0, "total_wall_s": 21.612890518037602, "rtf_streamed": 7.412183486667345, "rtf_total": 9.005371049182335, "gap_ms": [1487.7709700376727, 3370.885797950905, 6605.229270004202]}, "ttfa_ms": 6325.3543300088495, "playback_start_ms": 19212.170481006615, "stall_at_ttfa_ms": 12886.816150997765, "n_chunks": 5, "streamed_audio_s": 3.2, "tail_audio_s": 0.0, "total_wall_s": 21.612890518037602, "rtf_streamed": 6.753803275314567, "rtf_total": 6.754028286886751, "gap_ms": [1487.7709700376727, 3370.885797950905, 6605.229270004202, 3822.930113004986], "underruns": [[1, 1327.7709700376727], [2, 3050.885797950905], [3, 5965.229270004202], [4, 2542.930113004986]]}
{"request_id": "measured-short-1-2", "phase": "measured", "length_band": "short", "submitted_unix": 1790128201.8761928, "finished_unix": 1790128218.8951125, "concurrency": 1, "sr": 24000, "chunks_all": [{"idx": 0, "t_ms": 1120.8805019850843, "samples": 3840, "sr": 24000, "request_id": "measured-short-1-2", "terminal": false, "sha256": "09fba210f2c2fbc70764a4105247657db958ad05b89de0093bb03899ced744fa", "finite": true}, {"idx": 1, "t_ms": 3047.4736919859424, "samples": 7680, "sr": 24000, "request_id": "measured-short-1-2", "terminal": false, "sha256": "ba3275b0b8a8b6c40ee2071823c95e0d8f960db36aa5dce34a274bce2257dcee", "finite": true}, {"idx": 2, "t_ms": 7024.616775976028, "samples": 15360, "sr": 24000, "request_id": "measured-short-1-2", "terminal": false, "sha256": "50c9d2438d3f66ff937f1b87ba2501399361c1dc0f86db597882dd753f305675", "finite": true}, {"idx": 3, "t_ms": 13397.165296948515, "samples": 30720, "sr": 24000, "request_id": "measured-short-1-2", "terminal": false, "sha256": "b506f53077b79bb319eda197e524e2c30b5a64f77f19ce316ef5b3ec9bd734df", "finite": true}, {"idx": 4, "t_ms": 17018.59503495507, "samples": 15360, "sr": 24000, "request_id": "measured-short-1-2", "terminal": true, "sha256": "6a851fec2bebbf53e80f85506649ec7a77cda587e83936b48bd4583cd5058432", "finite": true}], "finished": true, "terminal_audio_s": 0.64, "prefix_metrics": {"ttfa_ms": 1120.8805019850843, "playback_start_ms": 12277.165296948515, "stall_at_ttfa_ms": 11156.28479496343, "n_chunks": 4, "streamed_audio_s": 2.4, "tail_audio_s": 0.0, "total_wall_s": 17.01891711598728, "rtf_streamed": 5.582152207061881, "rtf_total": 7.0912154649947, "gap_ms": [1926.5931900008582, 3977.143083990086, 6372.548520972487]}, "ttfa_ms": 1120.8805019850843, "playback_start_ms": 14618.59503495507, "stall_at_ttfa_ms": 13497.714532969985, "n_chunks": 5, "streamed_audio_s": 3.04, "tail_audio_s": 0.0, "total_wall_s": 17.01891711598728, "rtf_streamed": 5.598222050972062, "rtf_total": 5.598327998680026, "gap_ms": [1926.5931900008582, 3977.143083990086, 6372.548520972487, 3621.4297380065545], "underruns": [[1, 1766.5931900008582], [2, 3657.143083990086], [3, 5732.548520972487], [4, 2341.4297380065545]]}
{"request_id": "slow_consumer-medium-1-3", "phase": "slow_consumer", "length_band": "medium", "submitted_unix": 1790128218.8962007, "finished_unix": 1790128261.7518387, "concurrency": 1, "sr": 24000, "chunks_all": [{"idx": 0, "t_ms": 1179.83761994401, "samples": 3840, "sr": 24000, "request_id": "slow_consumer-medium-1-3", "terminal": false, "sha256": "0e2e70410d7aa4667b5c50a63629497cdfa68a49f099f85f452558cbda14b160", "finite": true}, {"idx": 1, "t_ms": 2969.4679499953054, "samples": 7680, "sr": 24000, "request_id": "slow_consumer-medium-1-3", "terminal": false, "sha256": "808116d564596b39c5f761022a05c1de083f2e9e5f6f8cdbc1e4198c54635f95", "finite": true}, {"idx": 2, "t_ms": 6295.096705958713, "samples": 15360, "sr": 24000, "request_id": "slow_consumer-medium-1-3", "terminal": false, "sha256": "d638597ac9dd75f4533ca8fe2d082096e13c759fdb311d19d699293efa7e19b5", "finite": true}, {"idx": 3, "t_ms": 13201.034414989408, "samples": 30720, "sr": 24000, "request_id": "slow_consumer-medium-1-3", "terminal": false, "sha256": "59d23b52373cb0db104b21662bf31c4d9fda90fe16eadd992e3c9fea0b9b7d30", "finite": true}, {"idx": 4, "t_ms": 25575.97459194949, "samples": 48000, "sr": 24000, "request_id": "slow_consumer-medium-1-3", "terminal": false, "sha256": "431801ed2529e8f0aaef2103c7e16637b0c8a6c1f73501187acf5b2cbb1fe109", "finite": true}, {"idx": 5, "t_ms": 35489.57968194736, "samples": 48000, "sr": 24000, "request_id": "slow_consumer-medium-1-3", "terminal": false, "sha256": "e32e516425eca3e1af02d98a90772dbf86cc583d8da500f4c2c005bbe91be6bd", "finite": true}, {"idx": 6, "t_ms": 42654.40870099701, "samples": 36480, "sr": 24000, "request_id": "slow_consumer-medium-1-3", "terminal": true, "sha256": "458a53beeccf2057f0c75165d44bd71bd4fefcc946b7c4d9b16a2c0b22a8a9d0", "finite": true}], "finished": true, "terminal_audio_s": 1.52, "prefix_metrics": {"ttfa_ms": 1179.83761994401, "playback_start_ms": 31089.57968194736, "stall_at_ttfa_ms": 29909.74206200335, "n_chunks": 6, "streamed_audio_s": 6.4, "tail_audio_s": 0.0, "total_wall_s": 42.85563437797828, "rtf_streamed": 5.545246825304275, "rtf_total": 6.696192871559106, "gap_ms": [1789.6303300512955, 3325.6287559634075, 6905.937709030695, 12374.94017696008, 9913.605089997873]}, "ttfa_ms": 1179.83761994401, "playback_start_ms": 36254.40870099701, "stall_at_ttfa_ms": 35074.571081053, "n_chunks": 7, "streamed_audio_s": 7.92, "tail_audio_s": 0.0, "total_wall_s": 42.85563437797828, "rtf_streamed": 5.385657664267299, "rtf_total": 5.411064946714429, "gap_ms": [1789.6303300512955, 3325.6287559634075, 6905.937709030695, 12374.94017696008, 9913.605089997873, 7164.829019049648], "underruns": [[1, 1629.6303300512955], [2, 3005.6287559634075], [3, 6265.937709030695], [4, 11094.94017696008], [5, 7913.605089997873], [6, 5164.829019049648]]}
{"request_id": "after_cancel-short-1-4", "phase": "after_cancel", "length_band": "short", "submitted_unix": 1790128262.9201438, "finished_unix": 1790128283.1748633, "concurrency": 1, "sr": 24000, "chunks_all": [{"idx": 0, "t_ms": 1340.9332330338657, "samples": 3840, "sr": 24000, "request_id": "after_cancel-short-1-4", "terminal": false, "sha256": "160202697eaff37db831d7c52316001783920f20c26874517cacd3e196fca957", "finite": true}, {"idx": 1, "t_ms": 3139.497352007311, "samples": 7680, "sr": 24000, "request_id": "after_cancel-short-1-4", "terminal": false, "sha256": "fca24302b351b4acd2a7068e3f46192b04e1c2ff9433f67e5b66687432387be2", "finite": true}, {"idx": 2, "t_ms": 6879.7519620275125, "samples": 15360, "sr": 24000, "request_id": "after_cancel-short-1-4", "terminal": false, "sha256": "9640fe140ee76d90ee40384c2db910a32202cd032ccae9826f22af798282af8c", "finite": true}, {"idx": 3, "t_ms": 14240.559764031786, "samples": 30720, "sr": 24000, "request_id": "after_cancel-short-1-4", "terminal": false, "sha256": "6c7d2a8f0e985dd2852eb871708a63cca07b98de6948f0cc1338a3ea5415f862", "finite": true}, {"idx": 4, "t_ms": 20254.40322502982, "samples": 23040, "sr": 24000, "request_id": "after_cancel-short-1-4", "terminal": true, "sha256": "33c0f46875be1f8c3aaeeb3ab6e7b85b5653b3124142f93f835ff59e1dff427f", "finite": true}], "finished": true, "terminal_audio_s": 0.96, "prefix_metrics": {"ttfa_ms": 1340.9332330338657, "playback_start_ms": 13120.559764031786, "stall_at_ttfa_ms": 11779.62653099792, "n_chunks": 4, "streamed_audio_s": 2.4, "tail_audio_s": 0.0, "total_wall_s": 20.25471683003707, "rtf_streamed": 5.933566568346578, "rtf_total": 8.43946534584878, "gap_ms": [1798.5641189734451, 3740.2546100202017, 7360.807802004274]}, "ttfa_ms": 1340.9332330338657, "playback_start_ms": 17854.40322502982, "stall_at_ttfa_ms": 16513.469991995953, "n_chunks": 5, "streamed_audio_s": 3.36, "tail_audio_s": 0.0, "total_wall_s": 20.25471683003707, "rtf_streamed": 6.028096197925541, "rtf_total": 6.028189532749128, "gap_ms": [1798.5641189734451, 3740.2546100202017, 7360.807802004274, 6013.843460998032], "underruns": [[1, 1638.5641189734451], [2, 3420.2546100202017], [3, 6720.807802004274], [4, 4733.843460998032]]}
benchmarks/edge_harness/results/e2e_recovery_20260923/evidence/tts_two_frame.json
+10
View File
@@ -0,0 +1,10 @@
{
"model": "/home/zhout/.cache/huggingface/hub/models--Qwen--Qwen3-TTS-12Hz-0.6B-CustomVoice/snapshots/85e237c12c027371202489a0ec509ded67b5e4b5",
"scope": "two codec frames only; no speech-quality claim",
"start": 1790127913.8721886,
"status": "completed",
"samples": 1920,
"finite": true,
"nonzero": true,
"end": 1790128048.1452236
}
docs/design/omni_edge_e2e_roadmap.md
+5
View File
@@ -6,6 +6,11 @@ feasibility/implementation blocker. It does not promise that every model fits
every device. The [current profiling review](../../benchmarks/edge_harness/results/e2e_profiling_20260922/README.md)
records measured execution separately from release qualification.
The [2026-09-23 recovery run](../../benchmarks/edge_harness/results/e2e_recovery_20260923/README.md)
adds a functional CPU TTS stream with playback underruns and a scoped RTX
Qwen3.8 text/image pass. Its missing-checkpoint transfers and remaining device
gates are tracked there; the 2026-09-22 audit remains the historical baseline.
## Architecture and prerequisites
Keep Omni's PipelineConfig, StageRuntime, StageClient, orchestration and admission
tests/config/test_omni_config.py
+17
View File
@@ -632,6 +632,8 @@ def test_vllm_omni_stage_config_public_fields_use_typed_stage_realizations():
def test_runtime_config_fields_match_structured_runtime_scope():
assert {f.name for f in fields(OmniStageRuntimeConfig)} == {
"additional_config",
"backend",
"resource_budget",
"distributed_executor_backend",
"worker_cls",
"devices",
@@ -711,6 +713,8 @@ def test_sub_config_fields_match_structured_scopes():
"task_type",
"codec_frame_rate_hz",
"enforce_eager",
"language_model_only",
"cpu_offload_gb",
"max_cudagraph_capture_size",
"enable_flashinfer_autotune",
"enable_multithread_weight_load",
@@ -1273,6 +1277,19 @@ def test_from_pipeline_config_derives_has_sampling_extra_args_from_stage_default
assert stage.model_config.has_sampling_extra_args is True
def test_qwen3_5_large_model_execution_overrides_have_a_stage_owner():
from vllm_omni.engine.stage_init_utils import _project_omni_stage_engine_args
stage = _from_pipeline_key(
"qwen3_5",
cli_overrides={"language_model_only": True, "cpu_offload_gb": 2.0},
).stage_by_id(0)
args = _project_omni_stage_engine_args(stage)
assert args["language_model_only"] is True
assert args["cpu_offload_gb"] == 2.0
def test_diffusion_config_preserves_existing_coercion_hooks():
import torch
tests/config/test_vllm_native_pipelines.py
+8 -3
View File
@@ -10,11 +10,9 @@ opaquely. Spark-X2.5 hit exactly that; these tests pin the general case.
from __future__ import annotations
import pytest
from vllm_omni.config.pipeline_registry import OMNI_PIPELINES
from vllm_omni.config.stage_config import StageExecutionType
from vllm_omni.config.vllm_native_pipelines import LLAMA_PIPELINE
from vllm_omni.config.vllm_native_pipelines import LLAMA_PIPELINE, QWEN3_5_PIPELINE
def test_llama_resolves_from_the_registry():
@@ -48,3 +46,10 @@ def test_registering_llama_is_keyed_on_architecture_not_checkpoint():
particular weights."""
assert LLAMA_PIPELINE.model_type == "llama"
assert LLAMA_PIPELINE.hf_architectures == ("LlamaForCausalLM",)
def test_qwen3_5_conditional_generation_resolves():
assert OMNI_PIPELINES["qwen3_5"] is QWEN3_5_PIPELINE
assert QWEN3_5_PIPELINE.get_validation_errors() == []
assert QWEN3_5_PIPELINE.hf_architectures == ("Qwen3_5ForConditionalGeneration",)
assert QWEN3_5_PIPELINE.stages[0].execution_type is StageExecutionType.LLM_AR
tests/core/test_prefix_cache.py
+35 -1
View File
@@ -4,7 +4,7 @@ from typing import NamedTuple
import pytest
import torch
from vllm_omni.core.prefix_cache import OmniTensorPrefixCache
from vllm_omni.core.prefix_cache import OmniTensorPrefixCache, _PendingAsyncWrite
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@@ -163,6 +163,40 @@ def test_update_no_multimodal():
assert torch.all(slot_states == new_states)
def test_pending_write_skips_vllm_unmapped_slot_sentinel():
class CompletedEvent:
def synchronize(self):
pass
cache = get_omni_pcache_with_mm_tensors({"vision": HIDDEN_SIZE}, seq_len=3)
cache._pending_write = _PendingAsyncWrite(
event=CompletedEvent(),
num_tokens=3,
slots_cpu=torch.tensor([8, -1, 9], dtype=torch.int32),
hidden_cpu=torch.tensor([[1.0, 1.0], [99.0, 99.0], [2.0, 2.0]]),
mm_cpu={"vision": torch.tensor([[3.0, 3.0], [99.0, 99.0], [4.0, 4.0]])},
)
cache._consume_pending_write()
assert cache._pending_write is None
assert cache.hidden_states_cache.view(-1, HIDDEN_SIZE)[8:10].tolist() == [[1.0, 1.0], [2.0, 2.0]]
assert cache.mm_outputs_cache["vision"].view(-1, HIDDEN_SIZE)[8:10].tolist() == [[3.0, 3.0], [4.0, 4.0]]
def test_synchronous_write_skips_vllm_unmapped_slot_sentinel():
cache = get_omni_pcache_with_mm_tensors({"vision": HIDDEN_SIZE}, seq_len=3)
cache.update_omni_tensor_prefix_cache(
hidden_states=torch.tensor([[1.0, 1.0], [99.0, 99.0], [2.0, 2.0]]),
multimodal_outputs={"vision": torch.tensor([[3.0, 3.0], [99.0, 99.0], [4.0, 4.0]])},
num_tokens_unpadded=3,
slot_mapping=torch.tensor([8, -1, 9], dtype=torch.int32),
)
assert cache.hidden_states_cache.view(-1, HIDDEN_SIZE)[8:10].tolist() == [[1.0, 1.0], [2.0, 2.0]]
assert cache.mm_outputs_cache["vision"].view(-1, HIDDEN_SIZE)[8:10].tolist() == [[3.0, 3.0], [4.0, 4.0]]
def test_update_uses_precomputed_hidden_states_cpu():
"""Precomputed CPU staging should be the only hidden-state cache source."""
cache = get_omni_pcache()
vllm_omni/config/omni_config.py
+5
View File
@@ -168,6 +168,9 @@ class _ModelEngineOverrides(TypedDict, total=False):
enable_multithread_weight_load: bool
num_weight_load_threads: int
disable_autocast: bool
# vLLM model execution switches needed by large conditional generators.
language_model_only: bool
cpu_offload_gb: float
class _LoadEngineOverrides(TypedDict, total=False):
@@ -440,6 +443,8 @@ class OmniStageModelConfig(_TrackExplicitConfigFields):
enable_multithread_weight_load: bool = True
num_weight_load_threads: int = Field(default=4, ge=1)
disable_autocast: bool = False
language_model_only: bool | None = None
cpu_offload_gb: float | None = Field(default=None, ge=0.0)
# Per-stage checkpoint/tokenizer subdirectories under the model root
# (e.g. Audex stage 0 → checkpoint_folder_audiogen). Mirrors
# StagePipelineConfig.model_subdir/tokenizer_subdir on the legacy path.
vllm_omni/config/pipeline_registry.py
+2 -1
View File
@@ -32,6 +32,7 @@ from vllm.logger import init_logger
from vllm_omni.config.stage_config import (
PipelineConfig,
)
from vllm_omni.config.vllm_native_pipelines import LLAMA_PIPELINE, QWEN3_5_PIPELINE
from vllm_omni.diffusion.models.pi0_pipeline_config import PI0_PIPELINE
from vllm_omni.model_executor.models.audex.pipeline import (
AUDEX_S2S_PIPELINE,
@@ -111,7 +112,6 @@ from vllm_omni.model_executor.models.qwen3_omni.pipeline import (
resolve_qwen3_omni_pipeline,
)
from vllm_omni.model_executor.models.qwen3_tts.pipeline import QWEN3_TTS_PIPELINE, QWEN3_TTS_TALKER_ONLY_PIPELINE
from vllm_omni.config.vllm_native_pipelines import LLAMA_PIPELINE
from vllm_omni.model_executor.models.spark2_5.pipeline import SPARK2_5_PIPELINE
from vllm_omni.model_executor.models.step_audio2.pipeline import (
STEP_AUDIO2_ASR_PIPELINE,
@@ -145,6 +145,7 @@ OMNI_PIPELINES: dict[str, PipelineConfig | PipelineResolverFunc] = {
# the same registry as the multi-stage pipelines so that one code path
# resolves every model (see the M0 local text mode).
"llama": LLAMA_PIPELINE,
"qwen3_5": QWEN3_5_PIPELINE,
"spark2_5": SPARK2_5_PIPELINE,
"step_audio_2": STEP_AUDIO2_PIPELINE,
"step_audio_2_asr": STEP_AUDIO2_ASR_PIPELINE,
vllm_omni/config/vllm_native_pipelines.py
+15 -2
View File
@@ -15,8 +15,8 @@ the orchestrator resolves a pipeline by ``model_type`` first, and a
hit exactly this in M0 -- its model definition had been registered for a while
and ``Omni(model=<checkpoint>)`` still could not resolve it.
**One stage, and the model_type is the key.** These are autoregressive causal
language models with a single stage that owns the tokenizer and emits text.
**One stage, and the model_type is the key.** These are autoregressive vLLM
models with a single stage that owns the tokenizer and emits text.
Because the registry is keyed on ``model_type`` rather than on a checkpoint,
registering ``llama`` covers every checkpoint whose config says ``llama`` -- not
only the one this was added for. That is the correct granularity (the pipeline
@@ -71,3 +71,16 @@ and ``architectures: ["LlamaForCausalLM"]`` -- it is a Llama-shaped 2B model
runner no longer forcing omni-only kwargs on a forward that cannot take them
(see ``OmniGPUModelRunner._accepted_forward_kwargs``).
"""
QWEN3_5_PIPELINE = _single_stage_text(
model_type="qwen3_5",
model_arch="Qwen3_5ForConditionalGeneration",
hf_architectures=("Qwen3_5ForConditionalGeneration",),
)
"""Single-stage Qwen3.5 generation using vLLM's model executor.
Qwen3.8-27B declares ``model_type: qwen3_5`` and this architecture. Pipeline
registration makes the checkpoint reachable through Omni; it does not imply
that its image path or any particular checkpoint has passed an E2E test.
"""
vllm_omni/core/prefix_cache.py
+22 -4
View File
@@ -279,6 +279,18 @@ class OmniTensorPrefixCache:
slots = entry.slots_cpu
if slots.dtype != torch.int64:
slots = slots.to(torch.int64)
# vLLM uses -1 for tokens without a KV slot (for example, a
# non-attention position in a hybrid model). Such tokens have no
# prefix-cache address either. Preserve row alignment while skipping
# them; index_copy_ rejects the sentinel.
if torch.any(slots < -1):
raise ValueError("prefix-cache slot mapping contains an invalid negative index")
valid = slots >= 0
has_unmapped_slots = not bool(torch.all(valid))
if has_unmapped_slots:
slots = slots[valid]
if slots.numel() == 0:
return
# NOTE: ``index_copy_(0, idx, src)`` is the row-scatter primitive in
# PyTorch. It's semantically identical to ``flat[idx] = src`` but
@@ -288,13 +300,13 @@ class OmniTensorPrefixCache:
# ``aten::index_put_`` path used by advanced-indexing assignment.
if entry.hidden_cpu is not None:
flat = self.hidden_states_cache.view(-1, self.hidden_states_cache.shape[-1])
flat.index_copy_(0, slots, entry.hidden_cpu)
flat.index_copy_(0, slots, entry.hidden_cpu[valid] if has_unmapped_slots else entry.hidden_cpu)
for k, src_cpu in entry.mm_cpu.items():
mm_cache = self.mm_outputs_cache.get(k)
if mm_cache is None:
continue
flat = mm_cache.view(-1, mm_cache.shape[-1])
flat.index_copy_(0, slots, src_cpu)
flat.index_copy_(0, slots, src_cpu[valid] if has_unmapped_slots else src_cpu)
def add_prefix_cached_new_req_id(self, req_id: str):
"""Adds a new request ID to the set of prefix cache hits on the batch."""
@@ -366,6 +378,12 @@ class OmniTensorPrefixCache:
feature shape of hidden_states, and cover num_tokens_unpadded.
"""
unpadded_slot_mapping = slot_mapping[:num_tokens_unpadded]
if torch.any(unpadded_slot_mapping < -1):
raise ValueError("prefix-cache slot mapping contains an invalid negative index")
valid_slots = unpadded_slot_mapping >= 0
has_unmapped_slots = not bool(torch.all(valid_slots))
if has_unmapped_slots:
unpadded_slot_mapping = unpadded_slot_mapping[valid_slots]
if num_tokens_padded is None:
num_tokens_padded = num_tokens_unpadded
skip_mm_cache_keys = skip_mm_cache_keys or set()
@@ -387,7 +405,7 @@ class OmniTensorPrefixCache:
slot_idx = unpadded_slot_mapping
if slot_idx.dtype != torch.int64:
slot_idx = slot_idx.to(torch.int64)
flat_cache.index_copy_(0, slot_idx, hidden_states)
flat_cache.index_copy_(0, slot_idx, hidden_states[valid_slots] if has_unmapped_slots else hidden_states)
logger.debug("Writing to hidden states for %s tokens", num_tokens_unpadded)
# Do the same for the stage's cached multimodal outputs
@@ -410,7 +428,7 @@ class OmniTensorPrefixCache:
slot_idx = unpadded_slot_mapping
if slot_idx.dtype != torch.int64:
slot_idx = slot_idx.to(torch.int64)
flat_cache.index_copy_(0, slot_idx, mm_state)
flat_cache.index_copy_(0, slot_idx, mm_state[valid_slots] if has_unmapped_slots else mm_state)
logger.debug("Writing to mm output cache for %s tokens", num_tokens_unpadded)
def stage_deferred_mm_outputs(
vllm_omni/worker/sampling_utils.py
+12 -7
View File
@@ -48,7 +48,10 @@ def sanitize_min_tokens_stop_ids(logitsprocs: LogitsProcessors, logits_vocab: in
if not min_toks:
continue
needs_rebuild = False
for _, _, stop_tok_ids, _ in min_toks.values():
# vLLM 0.28 stores three fields; 0.29 adds the structured-output
# flag. The stop-id set is the third field in both versions.
for state in min_toks.values():
stop_tok_ids = state[2]
oob = [tok for tok in stop_tok_ids if tok >= logits_vocab]
if not oob:
continue
@@ -65,17 +68,19 @@ def sanitize_min_tokens_stop_ids(logitsprocs: LogitsProcessors, logits_vocab: in
tok_ids: list[int] = []
restore_reqs: list[int] = []
restore_tok_ids: list[int] = []
for index, (_, _, stop_tok_ids, uses_structured_output) in min_toks.items():
for index, state in min_toks.items():
stop_tok_ids = state[2]
reqs.extend([index] * len(stop_tok_ids))
tok_ids.extend(stop_tok_ids)
if uses_structured_output:
if len(state) == 4 and state[3]:
restore_reqs.extend([index] * len(stop_tok_ids))
restore_tok_ids.extend(stop_tok_ids)
proc.logits_slice = (
proc._device_tensor(reqs, torch.int32),
proc._device_tensor(tok_ids, torch.int32),
)
proc.restore_logits_slice = (
proc._device_tensor(restore_reqs, torch.int32),
proc._device_tensor(restore_tok_ids, torch.int32),
)
if hasattr(proc, "restore_logits_slice"):
proc.restore_logits_slice = (
proc._device_tensor(restore_reqs, torch.int32),
proc._device_tensor(restore_tok_ids, torch.int32),
)