diff --git a/benchmarks/edge_harness/probe_internvla_actions.py b/benchmarks/edge_harness/probe_internvla_actions.py index db8b69863..c381cb6f0 100644 --- a/benchmarks/edge_harness/probe_internvla_actions.py +++ b/benchmarks/edge_harness/probe_internvla_actions.py @@ -21,6 +21,7 @@ def main() -> None: parser.add_argument("--cosmos-dir", type=Path, required=True) parser.add_argument("--processor-dir", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--actions-output", type=Path, help="Optional float32 .npy actions for reference comparison") parser.add_argument("--device", default="cuda") args = parser.parse_args() @@ -58,6 +59,10 @@ def main() -> None: pipeline = initialize_model(config) if pipeline.runtime_mode() != "real_checkpoint_loaded": raise RuntimeError(f"Unexpected InternVLA runtime mode: {pipeline.runtime_mode()}") + policy_devices = sorted({parameter.device.type for parameter in pipeline.policy.parameters()}) + requested_device_type = torch.device(args.device).type + if policy_devices != [requested_device_type]: + raise RuntimeError(f"InternVLA policy ran on {policy_devices}, not {requested_device_type}") startup_s = time.perf_counter() - started batch_inputs = pipeline._build_fake_batch_inputs() @@ -98,13 +103,23 @@ def main() -> None: raise RuntimeError(f"Unexpected action output: {type(actions).__name__}") if not torch.isfinite(actions).all(): raise RuntimeError("InternVLA returned non-finite actions") + if actions.device.type != requested_device_type: + raise RuntimeError(f"InternVLA actions were on {actions.device}, not {requested_device_type}") - action_bytes = actions.detach().float().cpu().contiguous().numpy().tobytes() + action_array = actions.detach().float().cpu().contiguous().numpy() + action_bytes = action_array.tobytes() + if args.actions_output is not None: + import numpy as np + + args.actions_output.parent.mkdir(parents=True, exist_ok=True) + np.save(args.actions_output, action_array) report = { "scope": "real checkpoint, synthetic zero observations and noise; no task-quality claim", "model_dir": str(model_dir), "runtime_mode": pipeline.runtime_mode(), "device": args.device, + "policy_devices": policy_devices, + "actions_device": str(actions.device), "dtype": str(actions.dtype), "action_shape": list(actions.shape), "finite": True, diff --git a/benchmarks/edge_harness/profile_minicpmo_text_speech.py b/benchmarks/edge_harness/profile_minicpmo_text_speech.py new file mode 100644 index 000000000..3a2a54e35 --- /dev/null +++ b/benchmarks/edge_harness/profile_minicpmo_text_speech.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +"""Measure serial real-weight MiniCPM-o text-to-speech requests on one device.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +import platform +import threading +import time +from pathlib import Path + +import numpy as np +import psutil +import torch +import vllm + +from examples.offline_inference.minicpmo.end2end import get_text_query +from vllm_omni.entrypoints.omni import Omni + + +class MemorySampler: + def __init__(self, interval_s: float = 0.5): + self.interval_s = interval_s + self.stop = threading.Event() + self.samples: list[dict[str, int | float]] = [] + self.thread = threading.Thread(target=self._run, name="memory-sampler", daemon=True) + + def _run(self) -> None: + parent = psutil.Process() + while not self.stop.is_set(): + memory = psutil.virtual_memory() + swap = psutil.swap_memory() + rss_sum = 0 + for proc in [parent, *parent.children(recursive=True)]: + try: + rss_sum += proc.memory_info().rss + except (psutil.NoSuchProcess, psutil.AccessDenied): + pass + self.samples.append( + { + "monotonic_s": time.perf_counter(), + "wsl_used_bytes": memory.used, + "wsl_available_bytes": memory.available, + "swap_used_bytes": swap.used, + "process_tree_rss_sum_bytes": rss_sum, + } + ) + self.stop.wait(self.interval_s) + + def __enter__(self): + self.thread.start() + return self + + def __exit__(self, *_exc): + self.stop.set() + self.thread.join() + + +def _nearest_rank(values: list[float], percentile: float) -> float: + ordered = sorted(values) + return ordered[max(0, math.ceil(percentile * len(ordered)) - 1)] + + +def _run_request(omni: Omni, prompt: dict, memory_sampler: MemorySampler) -> dict: + started = time.perf_counter() + first_text_s = None + first_audio_s = None + text = None + token_ids = None + audio_samples = None + audio_rms = None + request_ids = set() + + for output in omni.generate([prompt], None): + request_ids.add(output.request_id) + if output.final_output_type == "text": + first_text_s = time.perf_counter() - started + text = output.outputs[0].text.strip() + token_ids = list(output.outputs[0].token_ids) + elif output.final_output_type == "audio": + first_audio_s = time.perf_counter() - started + audio = output.outputs[0].multimodal_output["audio"] + if isinstance(audio, list): + audio = torch.cat( + [ + part.flatten() if isinstance(part, torch.Tensor) else torch.as_tensor(part).flatten() + for part in audio + ] + ) + wave = audio.detach().float().cpu().numpy().reshape(-1) + if not np.isfinite(wave).all(): + raise RuntimeError("MiniCPM-o generated non-finite audio") + audio_samples = int(wave.size) + audio_rms = float(np.sqrt(np.mean(wave.astype(np.float64) ** 2))) + + completed = time.perf_counter() + if not text or not token_ids or not audio_samples or audio_rms == 0: + raise RuntimeError("MiniCPM-o did not produce both nonempty text and nonzero audio") + samples = [sample for sample in memory_sampler.samples if started <= sample["monotonic_s"] <= completed] + return { + "request_ids": sorted(request_ids), + "wall_s": completed - started, + "first_text_s": first_text_s, + "first_audio_s": first_audio_s, + "text": text, + "token_ids": token_ids, + "audio_samples": audio_samples, + "audio_duration_s_at_24khz": audio_samples / 24000, + "audio_rms_float": audio_rms, + "sampled_memory": { + "n": len(samples), + "max_wsl_used_bytes": max((s["wsl_used_bytes"] for s in samples), default=None), + "min_wsl_available_bytes": min((s["wsl_available_bytes"] for s in samples), default=None), + "max_swap_used_bytes": max((s["swap_used_bytes"] for s in samples), default=None), + # Sum of RSS can double-count shared pages; it is not unique RAM. + "max_process_tree_rss_sum_bytes": max((s["process_tree_rss_sum_bytes"] for s in samples), default=None), + }, + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", type=Path, required=True) + parser.add_argument("--deploy-config", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--warmups", type=int, default=1) + parser.add_argument("--repeats", type=int, default=20) + args = parser.parse_args() + if args.warmups < 0 or args.repeats < 1: + parser.error("warmups must be nonnegative and repeats must be positive") + + args.output.parent.mkdir(parents=True, exist_ok=True) + prompt = {**get_text_query(use_tts=True).inputs, "modalities": ["text", "audio"]} + with MemorySampler() as sampler: + started = time.perf_counter() + omni = Omni( + model=str(args.model.resolve()), + deploy_config=str(args.deploy_config.resolve()), + trust_remote_code=True, + ) + startup_s = time.perf_counter() - started + try: + rows = [] + with args.output.with_suffix(".jsonl").open("w", encoding="utf-8") as raw: + for index in range(args.warmups + args.repeats): + row = _run_request(omni, prompt, sampler) + row["index"] = index + row["warmup"] = index < args.warmups + rows.append(row) + raw.write(json.dumps(row) + "\n") + raw.flush() + print(f"request={index} warmup={row['warmup']} wall_s={row['wall_s']:.3f}", flush=True) + finally: + omni.close() + + measured = [row for row in rows if not row["warmup"]] + times = [row["wall_s"] for row in measured] + output = { + "scope": "serial one-request-at-a-time MiniCPM-o text-to-speech; no speech-quality claim", + "model": str(args.model.resolve()), + "deploy_config": str(args.deploy_config.resolve()), + "deploy_config_sha256": hashlib.sha256(args.deploy_config.read_bytes()).hexdigest(), + "platform": platform.platform(), + "torch": torch.__version__, + "vllm": vllm.__version__, + "torch_cuda_available": torch.cuda.is_available(), + "environment": { + name: os.environ.get(name) + for name in ("VLLM_TARGET_DEVICE", "VLLM_CPU_KVCACHE_SPACE", "OMP_NUM_THREADS", "CUDA_VISIBLE_DEVICES") + }, + "startup_s": startup_s, + "warmups": args.warmups, + "repeats": args.repeats, + "wall_s_p50_nearest_rank": _nearest_rank(times, 0.5), + "wall_s_p95_nearest_rank": _nearest_rank(times, 0.95), + "requests": rows, + } + args.output.write_text(json.dumps(output, indent=2) + "\n", encoding="utf-8") + print(f"Wrote {args.output}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/README.md b/benchmarks/edge_harness/results/e2e_expansion_20260923/README.md new file mode 100644 index 000000000..bb56c8594 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/README.md @@ -0,0 +1,44 @@ +# 2026-09-23 WSL CPU and native Windows expansion + +This extends the [60-cell device × model matrix](../e2e_profiling_20260922/evidence/summary/README.md) with previously unrun WSL CPU and native Windows paths. The September 22 timing tables remain historical. MiniCPM-o now has a separate 20-request serial text-to-speech timing run; each InternVLA timing remains one observation. None is a sustained or full-modality profile. The [checkpoint manifests](../e2e_recovery_20260923/README.md) pin the real weights. No model, precision or context was silently substituted. + +| Device / model | Verified depth | Observed output and time | Remaining gate | +|---|---|---|---| +| WSL CPU / MiniCPM-o 4.5, revision `503e754207c94da6bb26850b4469f367c9ea3582` | Scoped complete three-stage text-to-text+speech path (P for this input only) | Coherent text and a 5.2 s mono 24 kHz WAV. First request 27.09 s after 73.65 s startup; separate 1-warmup/20-measured serial run: startup 73.89 s, request wall p50 21.86 s/p95 22.97 s. | Speech intelligibility/alignment, image/audio/video inputs, concurrency, interruption, power and thermal behavior. | +| WSL CPU / InternVLA-A1-3B Place_Markpen, revision `0a557c6a503e6545b369bff2f34c76a410bbd190` | Strict real-weight load and full policy forward on **synthetic zero** observations/noise | Finite float32 actions `[1, 50, 32]`; CPU-only PyTorch startup 31.78 s and synchronized forward 3.25 s. Compared with a same-checkpoint CUDA run, relative L2 difference 1.3025% and cosine similarity 0.999917. | Real Place_Markpen observations and reference actions, task tolerance, units/timestamps, repeated latency, memory and deadlines. | +| Native Windows CPU / InternVLA-A1-3B Place_Markpen, same revision | Strict real-weight load and full policy forward on **synthetic zero** observations/noise; policy and actions checked on CPU | Finite float32 actions `[1, 50, 32]`; placement-checked startup 61.66 s and forward 3.36 s. Relative L2 difference from WSL CPU 1.0824%; cosine 0.999942. | Real observations/reference actions and quality tolerance; repeated latency, memory, units/timestamps and deadlines. This is the direct policy CPU path, not native vLLM CPU attention. | +| Native Windows RTX 5090 Laptop / InternVLA-A1-3B Place_Markpen, same revision | Strict real-weight load and full policy forward on **synthetic zero** observations/noise; policy and actions checked on CUDA | Finite float32 actions `[1, 50, 32]`; placement-checked startup 75.54 s and synchronized forward 1.12 s. Relative L2 difference from WSL CUDA 0.8315%; cosine 0.999966. | Real observations/reference actions and quality tolerance; repeated latency, memory, units/timestamps and deadlines. | + +Both runs used Ryzen AI 9 HX 370 on Ubuntu 26.04/WSL2 kernel `6.18.33.2-microsoft-standard-WSL2`, with 30.91 GiB WSL RAM and 8 GiB swap. The CPU environment has Python 3.12.13, PyTorch 2.13.0+cpu (`torch.cuda.is_available() == False`), vLLM 0.28.0 and Transformers 5.14.1; Omni source is 0.29.0rc2. MiniCPM-o used BF16, eager execution, serial requests, `OMP_NUM_THREADS=8`, 1 GiB CPU KV cache **per stage**, 2,048-token thinker / 512-token talker limits, text input only, and no CPU weight offload. Its full three-stage run used swap, so the timing applies only to this constrained machine and workload. The output [text](evidence/minicpmo/0_28854dbf-b6b7-4f1d-a35a-52f254ab9d9e.txt), [WAV](evidence/minicpmo/output_0_28854dbf-b6b7-4f1d-a35a-52f254ab9d9e.wav), [raw log](evidence/minicpmo/minicpmo_cpu_full_compat4.log), [deployment config](evidence/minicpmo/minicpmo_cpu_offline.yaml) and [artifact check](evidence/minicpmo/minicpmo_cpu_artifact_report.json) are retained. The WAV has 124,800 PCM16 frames, 124,147 nonzero samples, peak absolute amplitude 21,850 and RMS 3,319.41; no listening or ASR check was performed. An earlier [one-stage CPU thinker run](evidence/minicpmo/minicpmo_cpu_text.log) also produced coherent text in 8.67 s after 36.02 s startup, but that diagnostic topology is not the speech pipeline. + +The separate [20-request profile](evidence/minicpmo/minicpmo_cpu_profile_20.json) records every measured output and sampled memory; the [JSONL](evidence/minicpmo/minicpmo_cpu_profile_20.jsonl) survives interruption, and the [run log](evidence/minicpmo/minicpmo_cpu_profile_20.log) retains worker details. One warmup was excluded. All 20 measured requests returned the same thinker token sequence, nonzero 124,800-sample audio, and request wall times 20.95–23.33 s; nearest-rank p50/p95 were 21.86/22.97 s (real-time factor 4.20/4.42 for 5.2 s of audio). Sampled WSL available RAM reached a minimum of 3.80 GB and swap use a maximum of 5.75 GB; summed process-tree RSS reached 29.04 GB but double-counts shared pages. These are 0.5 s samples during requests, not load-peak or unique-resident-memory measurements. Audio and text were returned together at request completion, so the profile cannot establish playable-stream latency, and no concurrency or 30-minute thermal run was performed. + +The CPU-only InternVLA run used the same BF16 policy checkpoint, synthetic six-camera history/state input and zero diffusion noise as the CUDA reference. The raw [CPU JSON](evidence/internvla/internvla_pure_cpu_synthetic.json), [CPU log](evidence/internvla/internvla_pure_cpu_synthetic.log), [CPU actions](evidence/internvla/internvla_pure_cpu_actions.npy), [CUDA JSON](evidence/internvla/internvla_cuda_synthetic_with_actions.json), [CUDA actions](evidence/internvla/internvla_cuda_actions.npy) and [numerical comparison](evidence/internvla/internvla_pure_cpu_cuda_parity.json) are retained. The CUDA reference used the WSL RTX 5090 Laptop and PyTorch 2.13.0+cu130; its one forward took 1.33 s. The action difference has maximum absolute error 0.00993 and mean absolute error 0.00117. No reference-task tolerance is available, so this comparison is evidence of numerical proximity, not an accuracy pass. + +The native Windows runs used Windows 11 build 26200, NVIDIA driver 610.71 for the GPU path, Python 3.12.10, PyTorch 2.13.0+cu130, vLLM 0.29.0, Transformers 5.14.1, and the current Omni source. The CPU run explicitly selected the CPU policy despite the CUDA-capable wheel. The placement-checked reruns assert that all policy parameters and the output action tensor remain on the requested device. Both loaded the same checkpoint from the local WSL filesystem through a UNC path; Windows CPU and GPU runs used the same synthetic input and zero diffusion noise as the WSL reference. The native [CPU JSON](evidence/internvla/internvla_windows_cpu_synthetic.json), [GPU JSON](evidence/internvla/internvla_windows_gpu_synthetic.json), [CPU log](evidence/internvla/internvla_windows_cpu_synthetic.log), [GPU log](evidence/internvla/internvla_windows_gpu_synthetic.log), [CPU actions](evidence/internvla/internvla_windows_cpu_actions.npy), [GPU actions](evidence/internvla/internvla_windows_gpu_actions.npy), [CPU/WSL comparison](evidence/internvla/internvla_windows_cpu_wsl_cpu_parity.json), and [GPU/WSL comparison](evidence/internvla/internvla_windows_gpu_wsl_cuda_parity.json) are retained. Native vLLM import initially failed on a Linux-specific CuMem allocator patch; the added Windows guard resolved the import. Native default GBK source decoding also required `PYTHONUTF8=1`. Both platform differences are setup requirements, not model-quality evidence. Power mode and thermal state were not captured. + +The MiniCPM-o CPU environment initially failed in the talker because vLLM 0.28 calls the multimodal processor with a prompt and three-value return contract, while 0.29 expects a `BatchFeature`. The 0.28 MiniCPMV processor also lacks `_call_hf_processor_on_prompts`. The local processor now handles both APIs. The next run failed before vocoder inference because `stepaudio2-minicpmo` was absent from the isolated CPU venv; installing version 0.1.1 with its dependencies enabled the final run. [All failed logs](evidence/minicpmo/) remain available; no failure was relabeled as a pass. + +Reproduce from the repository root with the verified local checkpoint paths: + +```bash +export PYTHONPATH="$PWD" OMP_NUM_THREADS=8 MKL_NUM_THREADS=8 TOKENIZERS_PARALLELISM=false +export CUDA_VISIBLE_DEVICES= VLLM_TARGET_DEVICE=cpu VLLM_CPU_KVCACHE_SPACE=1 VLLM_ENABLE_V1_MULTIPROCESSING=0 +/home/zhout/project/edge_infer/.venvs/omni-cpu/bin/python examples/offline_inference/minicpmo/end2end.py \ + --model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5 \ + --deploy-config benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/minicpmo/minicpmo_cpu_offline.yaml \ + --query-type text --modalities text,audio --num-prompts 1 --output-dir /tmp/minicpmo-cpu + +/home/zhout/project/edge_infer/.venvs/omni-cpu/bin/python benchmarks/edge_harness/probe_internvla_actions.py \ + --model-dir /home/zhout/project/edge_infer/models/InternVLA-A1-3B-FT-Place_Markpen \ + --cosmos-dir /home/zhout/project/edge_infer/models/Cosmos-Tokenizer-CI8x8 \ + --processor-dir /home/zhout/project/edge_infer/models/Qwen3-VL-2B-Instruct-processor \ + --device cpu --output /tmp/internvla-cpu.json --actions-output /tmp/internvla-cpu-actions.npy + +/home/zhout/project/edge_infer/.venvs/omni-cpu/bin/python benchmarks/edge_harness/profile_minicpmo_text_speech.py \ + --model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5 \ + --deploy-config benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/minicpmo/minicpmo_cpu_offline.yaml \ + --warmups 1 --repeats 20 --output /tmp/minicpmo-cpu-profile-20.json +``` + +For native Windows InternVLA, set `PYTHONUTF8=1`, put this checkout on `PYTHONPATH`, and run `benchmarks/edge_harness/probe_internvla_actions.py` with the same model/processor paths as above but Windows-accessible UNC paths; use `--device cpu` or `--device cuda` and separate output paths. diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla/internvla_cuda_actions.npy b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla/internvla_cuda_actions.npy new file mode 100644 index 000000000..56ad4a841 Binary files /dev/null and b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla/internvla_cuda_actions.npy differ diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla/internvla_cuda_synthetic_with_actions.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla/internvla_cuda_synthetic_with_actions.json new file mode 100644 index 000000000..20dd245b8 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla/internvla_cuda_synthetic_with_actions.json @@ -0,0 +1,27 @@ +{ + "scope": "real checkpoint, synthetic zero observations and noise; no task-quality claim", + "model_dir": "/home/zhout/project/edge_infer/models/InternVLA-A1-3B-FT-Place_Markpen", + "runtime_mode": "real_checkpoint_loaded", + "device": "cuda", + "dtype": "torch.float32", + "action_shape": [ + 1, + 50, + 32 + ], + "finite": true, + "action_sha256_float32": "71ddd577e166f417639dd09f8cdaeef5a1bc91a6152c53672542504328675dd6", + "first_action_prefix": [ + 0.02230919525027275, + -0.033485047519207, + 0.005473435390740633, + 0.02993217296898365, + -0.00445316918194294, + -0.01604151912033558, + 0.018104737624526024, + -0.09112925827503204 + ], + "startup_s": 33.11364810500527, + "forward_s": 1.3273328139912337, + "cuda_peak_allocated_bytes": 6683068416 +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla/internvla_cuda_synthetic_with_actions.log b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla/internvla_cuda_synthetic_with_actions.log new file mode 100644 index 000000000..37cca108b --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla/internvla_cuda_synthetic_with_actions.log @@ -0,0 +1,31 @@ +INFO 09-23 15:32:12 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-23 15:32:13 [patch.py:510] inductor factorable-divisibility patch: installed. +INFO 09-23 15:32:13 [patch.py:583] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms. +INFO 09-23 15:32:13 [pipeline_internvla_a1.py:164] Loading InternVLA-A1 weights from /home/zhout/project/edge_infer/models/InternVLA-A1-3B-FT-Place_Markpen +{ + "scope": "real checkpoint, synthetic zero observations and noise; no task-quality claim", + "model_dir": "/home/zhout/project/edge_infer/models/InternVLA-A1-3B-FT-Place_Markpen", + "runtime_mode": "real_checkpoint_loaded", + "device": "cuda", + "dtype": "torch.float32", + "action_shape": [ + 1, + 50, + 32 + ], + "finite": true, + "action_sha256_float32": "71ddd577e166f417639dd09f8cdaeef5a1bc91a6152c53672542504328675dd6", + "first_action_prefix": [ + 0.02230919525027275, + -0.033485047519207, + 0.005473435390740633, + 0.02993217296898365, + -0.00445316918194294, + -0.01604151912033558, + 0.018104737624526024, + -0.09112925827503204 + ], + "startup_s": 33.11364810500527, + "forward_s": 1.3273328139912337, + "cuda_peak_allocated_bytes": 6683068416 +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla/internvla_pure_cpu_actions.npy b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla/internvla_pure_cpu_actions.npy new file mode 100644 index 000000000..560583bcd Binary files /dev/null and b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla/internvla_pure_cpu_actions.npy differ diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla/internvla_pure_cpu_cuda_parity.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla/internvla_pure_cpu_cuda_parity.json new file mode 100644 index 000000000..e86e2dc30 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla/internvla_pure_cpu_cuda_parity.json @@ -0,0 +1,14 @@ +{ + "scope": "same Place_Markpen checkpoint, zero synthetic observations/noise, BF16 policy; CPU-only torch versus CUDA torch float32 actions", + "shape": [ + 1, + 50, + 32 + ], + "finite": true, + "max_abs_error": 0.009932532906532288, + "mean_abs_error": 0.0011705027022981085, + "rmse": 0.0018438395882933343, + "relative_l2_error": 0.013024961700113177, + "cosine_similarity": 0.9999165799446749 +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla/internvla_pure_cpu_synthetic.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla/internvla_pure_cpu_synthetic.json new file mode 100644 index 000000000..12581bcac --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla/internvla_pure_cpu_synthetic.json @@ -0,0 +1,27 @@ +{ + "scope": "real checkpoint, synthetic zero observations and noise; no task-quality claim", + "model_dir": "/home/zhout/project/edge_infer/models/InternVLA-A1-3B-FT-Place_Markpen", + "runtime_mode": "real_checkpoint_loaded", + "device": "cpu", + "dtype": "torch.float32", + "action_shape": [ + 1, + 50, + 32 + ], + "finite": true, + "action_sha256_float32": "db60d6590e242694948f7dcdf873b2fff44819d31b0b178fdad5ec9608d46140", + "first_action_prefix": [ + 0.021246271207928658, + -0.03424469009041786, + 0.005475320853292942, + 0.03046894632279873, + -0.004715746268630028, + -0.015595180913805962, + 0.017074640840291977, + -0.0920763686299324 + ], + "startup_s": 31.779892041988205, + "forward_s": 3.247856915986631, + "cuda_peak_allocated_bytes": null +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla/internvla_pure_cpu_synthetic.log b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla/internvla_pure_cpu_synthetic.log new file mode 100644 index 000000000..f3c5b5498 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla/internvla_pure_cpu_synthetic.log @@ -0,0 +1,42 @@ +/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions: + --> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922 + --> vLLM version 0.28.0 +This will likely cause compatibility issues. + warn_if_misaligned_vllm_version() +W0923 15:49:34.288000 4169445 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0923 15:49:34.327000 4169445 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +INFO 09-23 15:49:36 [importing.py:53] Triton is installed but 0 active driver(s) found (expected 1). Disabling Triton to prevent runtime errors. +WARNING 09-23 15:49:36 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton. +INFO 09-23 15:49:36 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available. +INFO 09-23 15:49:37 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-23 15:49:37 [patch.py:510] inductor factorable-divisibility patch: installed. +INFO 09-23 15:49:37 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled. +INFO 09-23 15:49:37 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable). +INFO 09-23 15:49:38 [pipeline_internvla_a1.py:164] Loading InternVLA-A1 weights from /home/zhout/project/edge_infer/models/InternVLA-A1-3B-FT-Place_Markpen +{ + "scope": "real checkpoint, synthetic zero observations and noise; no task-quality claim", + "model_dir": "/home/zhout/project/edge_infer/models/InternVLA-A1-3B-FT-Place_Markpen", + "runtime_mode": "real_checkpoint_loaded", + "device": "cpu", + "dtype": "torch.float32", + "action_shape": [ + 1, + 50, + 32 + ], + "finite": true, + "action_sha256_float32": "db60d6590e242694948f7dcdf873b2fff44819d31b0b178fdad5ec9608d46140", + "first_action_prefix": [ + 0.021246271207928658, + -0.03424469009041786, + 0.005475320853292942, + 0.03046894632279873, + -0.004715746268630028, + -0.015595180913805962, + 0.017074640840291977, + -0.0920763686299324 + ], + "startup_s": 31.779892041988205, + "forward_s": 3.247856915986631, + "cuda_peak_allocated_bytes": null +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla/internvla_windows_cpu_actions.npy b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla/internvla_windows_cpu_actions.npy new file mode 100644 index 000000000..f47381bc2 Binary files /dev/null and b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla/internvla_windows_cpu_actions.npy differ diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla/internvla_windows_cpu_synthetic.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla/internvla_windows_cpu_synthetic.json new file mode 100644 index 000000000..308fd7fc0 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla/internvla_windows_cpu_synthetic.json @@ -0,0 +1,31 @@ +{ + "scope": "real checkpoint, synthetic zero observations and noise; no task-quality claim", + "model_dir": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\InternVLA-A1-3B-FT-Place_Markpen", + "runtime_mode": "real_checkpoint_loaded", + "device": "cpu", + "policy_devices": [ + "cpu" + ], + "actions_device": "cpu", + "dtype": "torch.float32", + "action_shape": [ + 1, + 50, + 32 + ], + "finite": true, + "action_sha256_float32": "51b01235e490aa4cfc7bf5147bb692a3f7fd7feb6875addeb577e04581686599", + "first_action_prefix": [ + 0.021896986290812492, + -0.034019432961940765, + 0.004346882924437523, + 0.031385887414216995, + -0.0046007889322936535, + -0.016123272478580475, + 0.01791001483798027, + -0.09050343930721283 + ], + "startup_s": 61.65885859995615, + "forward_s": 3.363868599990383, + "cuda_peak_allocated_bytes": null +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla/internvla_windows_cpu_synthetic.log b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla/internvla_windows_cpu_synthetic.log new file mode 100644 index 000000000..510db7259 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla/internvla_windows_cpu_synthetic.log @@ -0,0 +1,34 @@ +INFO 09-23 16:17:04 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-23 16:17:04 [patch.py:510] inductor factorable-divisibility patch: installed. +INFO 09-23 16:17:08 [pipeline_internvla_a1.py:164] Loading InternVLA-A1 weights from \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\InternVLA-A1-3B-FT-Place_Markpen +{ + "scope": "real checkpoint, synthetic zero observations and noise; no task-quality claim", + "model_dir": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\InternVLA-A1-3B-FT-Place_Markpen", + "runtime_mode": "real_checkpoint_loaded", + "device": "cpu", + "policy_devices": [ + "cpu" + ], + "actions_device": "cpu", + "dtype": "torch.float32", + "action_shape": [ + 1, + 50, + 32 + ], + "finite": true, + "action_sha256_float32": "51b01235e490aa4cfc7bf5147bb692a3f7fd7feb6875addeb577e04581686599", + "first_action_prefix": [ + 0.021896986290812492, + -0.034019432961940765, + 0.004346882924437523, + 0.031385887414216995, + -0.0046007889322936535, + -0.016123272478580475, + 0.01791001483798027, + -0.09050343930721283 + ], + "startup_s": 61.65885859995615, + "forward_s": 3.363868599990383, + "cuda_peak_allocated_bytes": null +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla/internvla_windows_cpu_wsl_cpu_parity.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla/internvla_windows_cpu_wsl_cpu_parity.json new file mode 100644 index 000000000..3b5d96577 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla/internvla_windows_cpu_wsl_cpu_parity.json @@ -0,0 +1,13 @@ +{ + "candidate": "internvla_windows_cpu_actions.npy", + "reference": "internvla_pure_cpu_actions.npy", + "shape": [ + 1, + 50, + 32 + ], + "max_abs_error": 0.01507657766342163, + "mean_abs_error": 0.0009196984453774348, + "relative_l2_error": 0.010824427638738232, + "cosine_similarity": 0.9999416791155367 +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla/internvla_windows_gpu_actions.npy b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla/internvla_windows_gpu_actions.npy new file mode 100644 index 000000000..666d89e72 Binary files /dev/null and b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla/internvla_windows_gpu_actions.npy differ diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla/internvla_windows_gpu_synthetic.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla/internvla_windows_gpu_synthetic.json new file mode 100644 index 000000000..254b516b4 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla/internvla_windows_gpu_synthetic.json @@ -0,0 +1,31 @@ +{ + "scope": "real checkpoint, synthetic zero observations and noise; no task-quality claim", + "model_dir": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\InternVLA-A1-3B-FT-Place_Markpen", + "runtime_mode": "real_checkpoint_loaded", + "device": "cuda", + "policy_devices": [ + "cuda" + ], + "actions_device": "cuda:0", + "dtype": "torch.float32", + "action_shape": [ + 1, + 50, + 32 + ], + "finite": true, + "action_sha256_float32": "873432b6e4cef2b8822f7f5a720fef069c7ba9c6a3f45a831b2cf04239196337", + "first_action_prefix": [ + 0.021501213312149048, + -0.033786751329898834, + 0.004070071969181299, + 0.03125888481736183, + -0.004352160729467869, + -0.016927657648921013, + 0.017821842804551125, + -0.09145286679267883 + ], + "startup_s": 75.53822049999144, + "forward_s": 1.1152496999129653, + "cuda_peak_allocated_bytes": 6683068416 +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla/internvla_windows_gpu_synthetic.log b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla/internvla_windows_gpu_synthetic.log new file mode 100644 index 000000000..596af495a --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla/internvla_windows_gpu_synthetic.log @@ -0,0 +1,34 @@ +INFO 09-23 16:15:09 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-23 16:15:10 [patch.py:510] inductor factorable-divisibility patch: installed. +INFO 09-23 16:15:14 [pipeline_internvla_a1.py:164] Loading InternVLA-A1 weights from \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\InternVLA-A1-3B-FT-Place_Markpen +{ + "scope": "real checkpoint, synthetic zero observations and noise; no task-quality claim", + "model_dir": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\InternVLA-A1-3B-FT-Place_Markpen", + "runtime_mode": "real_checkpoint_loaded", + "device": "cuda", + "policy_devices": [ + "cuda" + ], + "actions_device": "cuda:0", + "dtype": "torch.float32", + "action_shape": [ + 1, + 50, + 32 + ], + "finite": true, + "action_sha256_float32": "873432b6e4cef2b8822f7f5a720fef069c7ba9c6a3f45a831b2cf04239196337", + "first_action_prefix": [ + 0.021501213312149048, + -0.033786751329898834, + 0.004070071969181299, + 0.03125888481736183, + -0.004352160729467869, + -0.016927657648921013, + 0.017821842804551125, + -0.09145286679267883 + ], + "startup_s": 75.53822049999144, + "forward_s": 1.1152496999129653, + "cuda_peak_allocated_bytes": 6683068416 +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla/internvla_windows_gpu_wsl_cuda_parity.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla/internvla_windows_gpu_wsl_cuda_parity.json new file mode 100644 index 000000000..7a7169c9b --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/internvla/internvla_windows_gpu_wsl_cuda_parity.json @@ -0,0 +1,13 @@ +{ + "candidate": "internvla_windows_gpu_actions.npy", + "reference": "internvla_cuda_actions.npy", + "shape": [ + 1, + 50, + 32 + ], + "max_abs_error": 0.008046535775065422, + "mean_abs_error": 0.0008131423041140806, + "relative_l2_error": 0.008314707744017184, + "cosine_similarity": 0.9999661845811505 +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/minicpmo/0_28854dbf-b6b7-4f1d-a35a-52f254ab9d9e.txt b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/minicpmo/0_28854dbf-b6b7-4f1d-a35a-52f254ab9d9e.txt new file mode 100644 index 000000000..1e65f7dd2 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/minicpmo/0_28854dbf-b6b7-4f1d-a35a-52f254ab9d9e.txt @@ -0,0 +1,15 @@ +Prompt: +<|im_start|>system +You are MiniCPM-o, a helpful multimodal assistant that can understand images, audio and video, and respond in text and speech.<|im_end|> +<|im_start|>user +Say hello, then introduce vLLM-Omni in one sentence.<|im_end|> +<|im_start|>assistant + + + + +<|tts_bos|> +vllm_text_output: +Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video. +vllm_token_ids: +[9707, 11, 358, 1079, 348, 4086, 44, 23620, 76, 7751, 11, 264, 3460, 79049, 57597, 1614, 12875, 315, 8660, 323, 23163, 1467, 11, 5335, 11, 7699, 11, 323, 2766, 13, 151704, 151645] diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/minicpmo/0_9ba10952-b263-4057-a17f-7f09c33fcff8.txt b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/minicpmo/0_9ba10952-b263-4057-a17f-7f09c33fcff8.txt new file mode 100644 index 000000000..b44e4e4a4 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/minicpmo/0_9ba10952-b263-4057-a17f-7f09c33fcff8.txt @@ -0,0 +1,15 @@ +Prompt: +<|im_start|>system +You are MiniCPM-o, a helpful multimodal assistant that can understand images, audio and video, and respond in text and speech.<|im_end|> +<|im_start|>user +Say hello, then introduce vLLM-Omni in one sentence.<|im_end|> +<|im_start|>assistant + + + + + +vllm_text_output: +Hello! vLLM-Omni is a versatile, high-performance large language model designed for efficient and accurate text generation and understanding. +vllm_token_ids: +[9707, 0, 348, 4086, 44, 23620, 76, 7751, 374, 264, 31945, 11, 1550, 57474, 3460, 4128, 1614, 6188, 369, 11050, 323, 13382, 1467, 9471, 323, 8660, 13, 151645] diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/minicpmo/minicpmo_cpu_artifact_report.json b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/minicpmo/minicpmo_cpu_artifact_report.json new file mode 100644 index 000000000..e9a7ada9c --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/minicpmo/minicpmo_cpu_artifact_report.json @@ -0,0 +1,14 @@ +{ + "scope": "one real-weight MiniCPM-o 4.5 three-stage WSL CPU text-to-speech request; speech meaning unverified", + "cpu_text_tokens": 32, + "cpu_cuda_text_token_ids_equal": true, + "wave_frames": 124800, + "wave_sample_rate_hz": 24000, + "wave_channels": 1, + "wave_sample_width_bytes": 2, + "wave_duration_s": 5.2, + "wave_nonzero_samples": 124147, + "wave_peak_abs_pcm16": 21850, + "wave_rms_pcm16": 3319.410484644662, + "wave_sha256": "ded62a7285136fe16f28c7e6eb546048a6deb7e385a7a974031032c55df2de40" +} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/minicpmo/minicpmo_cpu_full.log b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/minicpmo/minicpmo_cpu_full.log new file mode 100644 index 000000000..992ec3606 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/minicpmo/minicpmo_cpu_full.log @@ -0,0 +1,433 @@ +W0923 15:36:58.480000 4167421 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0923 15:36:58.503000 4167421 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +WARNING 09-23 15:36:59 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton. +INFO 09-23 15:36:59 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available. +/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions: + --> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922 + --> vLLM version 0.28.0 +This will likely cause compatibility issues. + warn_if_misaligned_vllm_version() +INFO 09-23 15:37:00 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-23 15:37:00 [patch.py:510] inductor factorable-divisibility patch: installed. +INFO 09-23 15:37:00 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled. +INFO 09-23 15:37:01 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable). +==================== + vllm version: 0.28.0 + ==================== +INFO 09-23 15:37:01 [omni_base.py:216] [Omni] Initializing with model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5 +INFO 09-23 15:37:01 [async_omni_engine.py:172] [AsyncOmniEngine] Initializing with model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5 +INFO 09-23 15:37:01 [async_omni_engine.py:290] [AsyncOmniEngine] Launching Orchestrator thread with 3 stages +INFO 09-23 15:37:01 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn +INFO 09-23 15:37:01 [initialization.py:356] Loaded OmniTransferConfig with 1 connector configurations +WARNING 09-23 15:37:01 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass) +WARNING 09-23 15:37:01 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass) +WARNING 09-23 15:37:01 [arg_utils.py:1719] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM. +INFO 09-23 15:37:01 [model.py:672] Resolved architecture: MiniCPMO45OmniForConditionalGeneration +INFO 09-23 15:37:01 [model.py:1965] Using max model len 2048 +WARNING 09-23 15:37:01 [model.py:963] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False. +INFO 09-23 15:37:02 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode. +INFO 09-23 15:37:02 [model.py:875] Disabled mm_prefix attention mode because multimodal inputs are configuration-disabled. Attention backends without mm_prefix support may now be selected. +INFO 09-23 15:37:02 [scheduler.py:242] Chunked prefill is enabled with max_num_batched_tokens=2048. +WARNING 09-23 15:37:02 [vllm.py:1285] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none +WARNING 09-23 15:37:02 [vllm.py:1354] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored. +INFO 09-23 15:37:02 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +WARNING 09-23 15:37:02 [vllm.py:652] Model Runner V2 requires Triton; using the V1 model runner instead. +INFO 09-23 15:37:02 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant +WARNING 09-23 15:37:02 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass) +WARNING 09-23 15:37:02 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass) +WARNING 09-23 15:37:02 [arg_utils.py:1719] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM. +INFO 09-23 15:37:02 [model.py:672] Resolved architecture: MiniCPMO45OmniForConditionalGeneration +INFO 09-23 15:37:02 [model.py:1965] Using max model len 512 +WARNING 09-23 15:37:02 [model.py:963] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False. +INFO 09-23 15:37:02 [model.py:1965] Using max model len 512 +INFO 09-23 15:37:02 [scheduler.py:242] Chunked prefill is enabled with max_num_batched_tokens=512. +INFO 09-23 15:37:02 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +WARNING 09-23 15:37:02 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass) +WARNING 09-23 15:37:02 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass) +WARNING 09-23 15:37:02 [arg_utils.py:1719] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM. +INFO 09-23 15:37:02 [model.py:672] Resolved architecture: MiniCPMO45Code2Wav +INFO 09-23 15:37:02 [model.py:1965] Using max model len 4096 +WARNING 09-23 15:37:02 [arg_utils.py:2664] This model does not officially support disabling chunked prefill. Disabling this manually may cause the engine to crash or produce incorrect outputs. +INFO 09-23 15:37:02 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +INFO 09-23 15:37:02 [stage_runtime.py:853] [stage_init] Stage-0 set runtime devices: 0 +W0923 15:37:06.286000 4167451 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0923 15:37:06.314000 4167451 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +WARNING 09-23 15:37:07 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton. +INFO 09-23 15:37:07 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available. +/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions: + --> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922 + --> vLLM version 0.28.0 +This will likely cause compatibility issues. + warn_if_misaligned_vllm_version() +INFO 09-23 15:37:08 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-23 15:37:08 [patch.py:510] inductor factorable-divisibility patch: installed. +INFO 09-23 15:37:08 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled. +INFO 09-23 15:37:08 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable). +(StageEngineCoreProc_stage0_replica0 pid=4167451) INFO 09-23 15:37:09 [core.py:122] Initializing a V1 LLM engine (v0.28.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=2048, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=True, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cpu, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': None, 'compile_ranges_endpoints': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': None, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto') +(StageEngineCoreProc_stage0_replica0 pid=4167451) INFO 09-23 15:37:09 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode. +[W923 15:37:09.092897341 utils.cpp:68] Warning: NUMA binding: Using MEMBIND policy for memory allocation on the NUMA nodes (0). Memory allocations will be strictly bound to these NUMA nodes. (function init_cpu_memory_env) +(StageEngineCoreProc_stage0_replica0 pid=4167451) WARNING 09-23 15:37:09 [vllm.py:652] Model Runner V2 requires Triton; using the V1 model runner instead. +(StageEngineCoreProc_stage0_replica0 pid=4167451) WARNING 09-23 15:37:09 [cpu_worker.py:127] libiomp is not found in LD_PRELOAD. For best performance, please follow the section `set LD_PRELOAD` in https://docs.vllm.ai/en/latest/getting_started/installation/cpu/ to setup required pre-loaded libraries. +(StageEngineCoreProc_stage0_replica0 pid=4167451) INFO 09-23 15:37:09 [parallel_state.py:1638] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_a31cf537bd4d45dea02606fe108bbbce backend=gloo +(StageEngineCoreProc_stage0_replica0 pid=4167451) INFO 09-23 15:37:09 [parallel_state.py:1982] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A +(StageEngineCoreProc_stage0_replica0 pid=4167451) INFO 09-23 15:37:09 [cpu_model_runner.py:131] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5... +(StageEngineCoreProc_stage0_replica0 pid=4167451) WARNING 09-23 15:37:09 [vllm.py:1285] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none +(StageEngineCoreProc_stage0_replica0 pid=4167451) WARNING 09-23 15:37:09 [vllm.py:1354] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored. +(StageEngineCoreProc_stage0_replica0 pid=4167451) INFO 09-23 15:37:09 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +(StageEngineCoreProc_stage0_replica0 pid=4167451) INFO 09-23 15:37:09 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant +(StageEngineCoreProc_stage0_replica0 pid=4167451) INFO 09-23 15:37:09 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +(StageEngineCoreProc_stage0_replica0 pid=4167451) INFO 09-23 15:37:09 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +(StageEngineCoreProc_stage0_replica0 pid=4167451) INFO 09-23 15:37:10 [selector.py:217] Using HND KV cache layout for CPU_ATTN backend. +(StageEngineCoreProc_stage0_replica0 pid=4167451) INFO 09-23 15:37:13 [weight_utils.py:858] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 27.02 GiB. +(StageEngineCoreProc_stage0_replica0 pid=4167451) INFO 09-23 15:37:13 [weight_utils.py:881] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch. +(StageEngineCoreProc_stage0_replica0 pid=4167451) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00 18646 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory'] +(StageEngineCoreProc_stage0_replica0 pid=4167451) INFO 09-23 15:37:35 [cpu_worker.py:255] Explicitly set (1.0/30.91) GiB for KV cache on node 0. +(StageEngineCoreProc_stage0_replica0 pid=4167451) INFO 09-23 15:37:35 [kv_cache_utils.py:1869] GPU KV cache size: 7,168 tokens, Maximum concurrency for 2,048 tokens per request: 3.50x +(StageEngineCoreProc_stage0_replica0 pid=4167451) INFO 09-23 15:37:35 [memory.py:91] Released 0 MiB after warm-up (anonymous RSS 19661 -> 19661 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory'] +(StageEngineCoreProc_stage0_replica0 pid=4167451) INFO 09-23 15:37:35 [core.py:354] init engine (profile, create kv cache, warmup model) took 0.44 s +(StageEngineCoreProc_stage0_replica0 pid=4167451) WARNING 09-23 15:37:35 [scheduler.py:182] Using custom scheduler class vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled. +(StageEngineCoreProc_stage0_replica0 pid=4167451) INFO 09-23 15:37:35 [factory.py:46] Created connector: SharedMemoryConnector +(StageEngineCoreProc_stage0_replica0 pid=4167451) INFO 09-23 15:37:46 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +INFO 09-23 15:37:46 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed +INFO 09-23 15:37:46 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-0 [rep-0] initializing EngineCore +INFO 09-23 15:37:46 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-0 [rep-0] EngineCore running +INFO 09-23 15:37:46 [stage_runtime.py:958] [StageRuntime] Stage 0 initialized +INFO 09-23 15:37:46 [stage_runtime.py:853] [stage_init] Stage-1 set runtime devices: 0 +W0923 15:37:53.025000 4167571 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0923 15:37:53.066000 4167571 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +WARNING 09-23 15:37:54 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton. +INFO 09-23 15:37:54 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available. +/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions: + --> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922 + --> vLLM version 0.28.0 +This will likely cause compatibility issues. + warn_if_misaligned_vllm_version() +INFO 09-23 15:37:55 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-23 15:37:56 [patch.py:510] inductor factorable-divisibility patch: installed. +INFO 09-23 15:37:56 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled. +INFO 09-23 15:37:56 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable). +(StageEngineCoreProc_stage1_replica0 pid=4167571) INFO 09-23 15:37:56 [core.py:122] Initializing a V1 LLM engine (v0.28.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=512, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=True, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cpu, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': None, 'compile_ranges_endpoints': [512], 'inductor_compile_config': {'enable_auto_functionalized_v2': False}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': None, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto') +[W923 15:37:57.845305821 utils.cpp:68] Warning: NUMA binding: Using MEMBIND policy for memory allocation on the NUMA nodes (0). Memory allocations will be strictly bound to these NUMA nodes. (function init_cpu_memory_env) +(StageEngineCoreProc_stage1_replica0 pid=4167571) WARNING 09-23 15:37:57 [vllm.py:652] Model Runner V2 requires Triton; using the V1 model runner instead. +(StageEngineCoreProc_stage1_replica0 pid=4167571) WARNING 09-23 15:37:57 [cpu_worker.py:127] libiomp is not found in LD_PRELOAD. For best performance, please follow the section `set LD_PRELOAD` in https://docs.vllm.ai/en/latest/getting_started/installation/cpu/ to setup required pre-loaded libraries. +(StageEngineCoreProc_stage1_replica0 pid=4167571) INFO 09-23 15:37:57 [parallel_state.py:1638] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_ed0f981d16804c67838c632666ad27c7 backend=gloo +(StageEngineCoreProc_stage1_replica0 pid=4167571) INFO 09-23 15:37:57 [parallel_state.py:1982] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A +(StageEngineCoreProc_stage1_replica0 pid=4167571) WARNING 09-23 15:37:57 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it. +(StageEngineCoreProc_stage1_replica0 pid=4167571) [transformers] `MiniCPMOProcessor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +(StageEngineCoreProc_stage1_replica0 pid=4167571) [transformers] Requested torchvision backend is not available. Falling back to pil backend. +(StageEngineCoreProc_stage1_replica0 pid=4167571) [transformers] `MiniCPMOProcessor` defines `feature_extractor_class = 'WhisperFeatureExtractor'`, which is deprecated. Register the correct mapping in `AutoFeatureExtractor` instead. +(StageEngineCoreProc_stage1_replica0 pid=4167571) WARNING 09-23 15:38:00 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it. +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] StageEngineCoreProc failed to start. +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] Traceback (most recent call last): +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_core_proc.py", line 189, in run_stage_core +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] engine_core = StageEngineCoreProc( +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] return func(*args, **kwargs) +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/core.py", line 1073, in __init__ +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] super().__init__( +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/core.py", line 133, in __init__ +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] self.model_executor = executor_class(vllm_config) +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] return func(*args, **kwargs) +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/executor/abstract.py", line 110, in __init__ +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] self._init_executor() +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/executor/uniproc_executor.py", line 69, in _init_executor +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] self.driver_worker.init_device() +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 335, in init_device +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] self.worker.init_device() # type: ignore +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/init_timeline.py", line 343, in wrapper +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] return fn(*args, **kwargs) +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/platforms/cpu/worker/cpu_ar_worker.py", line 30, in init_device +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] super().init_device() # type: ignore[misc] +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/cpu_worker.py", line 190, in init_device +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] self.model_runner = CPUModelRunner(self.vllm_config, torch.device("cpu")) +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/cpu_model_runner.py", line 31, in __init__ +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] super().__init__(vllm_config, device) +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/gpu_model_runner.py", line 915, in __init__ +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] MultiModalBudget(self.vllm_config, self.mm_registry) +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/encoder_budget.py", line 94, in __init__ +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] all_mm_max_toks_per_item = get_mm_max_toks_per_item( +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/encoder_budget.py", line 33, in get_mm_max_toks_per_item +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] mm_inputs = mm_registry.get_dummy_mm_inputs( +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/registry.py", line 263, in get_dummy_mm_inputs +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] mm_inputs = processor.apply( +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py", line 3419, in apply +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] return super().apply(inputs, timing_ctx) +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/processing/processor.py", line 1849, in apply +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] ) = self._cached_apply_hf_processor(inputs, timing_ctx) +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/processing/processor.py", line 1628, in _cached_apply_hf_processor +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] ) = self._apply_hf_processor_main( +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] TypeError: MiniCPMO45OmniLLMMultiModalProcessor._apply_hf_processor_main() got an unexpected keyword argument 'prompt' +(StageEngineCoreProc_stage1_replica0 pid=4167571) Process StageEngineCoreProc_stage1_replica0: +(StageEngineCoreProc_stage1_replica0 pid=4167571) Traceback (most recent call last): +(StageEngineCoreProc_stage1_replica0 pid=4167571) File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/multiprocessing/process.py", line 314, in _bootstrap +(StageEngineCoreProc_stage1_replica0 pid=4167571) self.run() +(StageEngineCoreProc_stage1_replica0 pid=4167571) File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/multiprocessing/process.py", line 108, in run +(StageEngineCoreProc_stage1_replica0 pid=4167571) self._target(*self._args, **self._kwargs) +(StageEngineCoreProc_stage1_replica0 pid=4167571) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_core_proc.py", line 189, in run_stage_core +(StageEngineCoreProc_stage1_replica0 pid=4167571) engine_core = StageEngineCoreProc( +(StageEngineCoreProc_stage1_replica0 pid=4167571) ^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167571) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper +(StageEngineCoreProc_stage1_replica0 pid=4167571) return func(*args, **kwargs) +(StageEngineCoreProc_stage1_replica0 pid=4167571) ^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167571) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/core.py", line 1073, in __init__ +(StageEngineCoreProc_stage1_replica0 pid=4167571) super().__init__( +(StageEngineCoreProc_stage1_replica0 pid=4167571) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/core.py", line 133, in __init__ +(StageEngineCoreProc_stage1_replica0 pid=4167571) self.model_executor = executor_class(vllm_config) +(StageEngineCoreProc_stage1_replica0 pid=4167571) ^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167571) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper +(StageEngineCoreProc_stage1_replica0 pid=4167571) return func(*args, **kwargs) +(StageEngineCoreProc_stage1_replica0 pid=4167571) ^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167571) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/executor/abstract.py", line 110, in __init__ +(StageEngineCoreProc_stage1_replica0 pid=4167571) self._init_executor() +(StageEngineCoreProc_stage1_replica0 pid=4167571) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/executor/uniproc_executor.py", line 69, in _init_executor +(StageEngineCoreProc_stage1_replica0 pid=4167571) self.driver_worker.init_device() +(StageEngineCoreProc_stage1_replica0 pid=4167571) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 335, in init_device +(StageEngineCoreProc_stage1_replica0 pid=4167571) self.worker.init_device() # type: ignore +(StageEngineCoreProc_stage1_replica0 pid=4167571) ^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167571) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/init_timeline.py", line 343, in wrapper +(StageEngineCoreProc_stage1_replica0 pid=4167571) return fn(*args, **kwargs) +(StageEngineCoreProc_stage1_replica0 pid=4167571) ^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167571) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/platforms/cpu/worker/cpu_ar_worker.py", line 30, in init_device +(StageEngineCoreProc_stage1_replica0 pid=4167571) super().init_device() # type: ignore[misc] +(StageEngineCoreProc_stage1_replica0 pid=4167571) ^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167571) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/cpu_worker.py", line 190, in init_device +(StageEngineCoreProc_stage1_replica0 pid=4167571) self.model_runner = CPUModelRunner(self.vllm_config, torch.device("cpu")) +(StageEngineCoreProc_stage1_replica0 pid=4167571) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167571) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/cpu_model_runner.py", line 31, in __init__ +(StageEngineCoreProc_stage1_replica0 pid=4167571) super().__init__(vllm_config, device) +(StageEngineCoreProc_stage1_replica0 pid=4167571) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/gpu_model_runner.py", line 915, in __init__ +(StageEngineCoreProc_stage1_replica0 pid=4167571) MultiModalBudget(self.vllm_config, self.mm_registry) +(StageEngineCoreProc_stage1_replica0 pid=4167571) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/encoder_budget.py", line 94, in __init__ +(StageEngineCoreProc_stage1_replica0 pid=4167571) all_mm_max_toks_per_item = get_mm_max_toks_per_item( +(StageEngineCoreProc_stage1_replica0 pid=4167571) ^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167571) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/encoder_budget.py", line 33, in get_mm_max_toks_per_item +(StageEngineCoreProc_stage1_replica0 pid=4167571) mm_inputs = mm_registry.get_dummy_mm_inputs( +(StageEngineCoreProc_stage1_replica0 pid=4167571) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167571) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/registry.py", line 263, in get_dummy_mm_inputs +(StageEngineCoreProc_stage1_replica0 pid=4167571) mm_inputs = processor.apply( +(StageEngineCoreProc_stage1_replica0 pid=4167571) ^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167571) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py", line 3419, in apply +(StageEngineCoreProc_stage1_replica0 pid=4167571) return super().apply(inputs, timing_ctx) +(StageEngineCoreProc_stage1_replica0 pid=4167571) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167571) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/processing/processor.py", line 1849, in apply +(StageEngineCoreProc_stage1_replica0 pid=4167571) ) = self._cached_apply_hf_processor(inputs, timing_ctx) +(StageEngineCoreProc_stage1_replica0 pid=4167571) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167571) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/processing/processor.py", line 1628, in _cached_apply_hf_processor +(StageEngineCoreProc_stage1_replica0 pid=4167571) ) = self._apply_hf_processor_main( +(StageEngineCoreProc_stage1_replica0 pid=4167571) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167571) TypeError: MiniCPMO45OmniLLMMultiModalProcessor._apply_hf_processor_main() got an unexpected keyword argument 'prompt' +ERROR 09-23 15:38:03 [stage_runtime.py:265] [StageRuntime] Stage initialization failed; shutting down 1 initialized client(s) +ERROR 09-23 15:38:03 [stage_runtime.py:265] Traceback (most recent call last): +ERROR 09-23 15:38:03 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 252, in initialize +ERROR 09-23 15:38:03 [stage_runtime.py:265] initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout) +ERROR 09-23 15:38:03 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +ERROR 09-23 15:38:03 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 589, in _initialize_stage_replicas +ERROR 09-23 15:38:03 [stage_runtime.py:265] raise primary_exc +ERROR 09-23 15:38:03 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 550, in _init_group +ERROR 09-23 15:38:03 [stage_runtime.py:265] client = self._initialize_replica( +ERROR 09-23 15:38:03 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^ +ERROR 09-23 15:38:03 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 828, in _initialize_replica +ERROR 09-23 15:38:03 [stage_runtime.py:265] return self._initialize_local_llm_replica(plan, stage_init_timeout) +ERROR 09-23 15:38:03 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +ERROR 09-23 15:38:03 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 934, in _initialize_local_llm_replica +ERROR 09-23 15:38:03 [stage_runtime.py:265] launch_stack.close() +ERROR 09-23 15:38:03 [stage_runtime.py:265] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 618, in close +ERROR 09-23 15:38:03 [stage_runtime.py:265] self.__exit__(None, None, None) +ERROR 09-23 15:38:03 [stage_runtime.py:265] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 610, in __exit__ +ERROR 09-23 15:38:03 [stage_runtime.py:265] raise exc_details[1] +ERROR 09-23 15:38:03 [stage_runtime.py:265] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 595, in __exit__ +ERROR 09-23 15:38:03 [stage_runtime.py:265] if cb(*exc_details): +ERROR 09-23 15:38:03 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^ +ERROR 09-23 15:38:03 [stage_runtime.py:265] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 144, in __exit__ +ERROR 09-23 15:38:03 [stage_runtime.py:265] next(self.gen) +ERROR 09-23 15:38:03 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_startup.py", line 1211, in launch_stage_replica +ERROR 09-23 15:38:03 [stage_runtime.py:265] wait_for_engine_startup( +ERROR 09-23 15:38:03 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/utils.py", line 1286, in wait_for_engine_startup +ERROR 09-23 15:38:03 [stage_runtime.py:265] raise RuntimeError( +ERROR 09-23 15:38:03 [stage_runtime.py:265] RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {} +INFO 09-23 15:38:03 [core_client.py:689] [shutdown] MPClient: start timeout=default +INFO 09-23 15:38:03 [core_client.py:691] [shutdown] MPClient: stopping engine manager +INFO 09-23 15:38:03 [utils.py:615] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage0_replica0 +WARNING 09-23 15:38:08 [utils.py:635] [shutdown] Process manager: force killing remaining processes count=1 +WARNING 09-23 15:38:08 [utils.py:640] [shutdown] Process manager: force killing remaining process StageEngineCoreProc_stage0_replica0 pid 4167451 +INFO 09-23 15:38:08 [core_client.py:693] [shutdown] MPClient: engine manager stopped +INFO 09-23 15:38:08 [core_client.py:694] [shutdown] MPClient: cleaning up background resources +INFO 09-23 15:38:08 [core_client.py:696] [shutdown] MPClient: complete +ERROR 09-23 15:38:08 [async_omni_engine.py:474] [AsyncOmniEngine] Orchestrator thread crashed +ERROR 09-23 15:38:08 [async_omni_engine.py:474] Traceback (most recent call last): +ERROR 09-23 15:38:08 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 468, in _bootstrap_orchestrator +ERROR 09-23 15:38:08 [async_omni_engine.py:474] loop.run_until_complete(_run_orchestrator()) +ERROR 09-23 15:38:08 [async_omni_engine.py:474] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/base_events.py", line 691, in run_until_complete +ERROR 09-23 15:38:08 [async_omni_engine.py:474] return future.result() +ERROR 09-23 15:38:08 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^ +ERROR 09-23 15:38:08 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 424, in _run_orchestrator +ERROR 09-23 15:38:08 [async_omni_engine.py:474] self._initialize_stages(stage_init_timeout) +ERROR 09-23 15:38:08 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 350, in _initialize_stages +ERROR 09-23 15:38:08 [async_omni_engine.py:474] self._initialize_stages_impl(stage_init_timeout) +ERROR 09-23 15:38:08 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 371, in _initialize_stages_impl +ERROR 09-23 15:38:08 [async_omni_engine.py:474] self._runtime.initialize() +ERROR 09-23 15:38:08 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 278, in initialize +ERROR 09-23 15:38:08 [async_omni_engine.py:474] raise exc +ERROR 09-23 15:38:08 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 252, in initialize +ERROR 09-23 15:38:08 [async_omni_engine.py:474] initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout) +ERROR 09-23 15:38:08 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +ERROR 09-23 15:38:08 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 589, in _initialize_stage_replicas +ERROR 09-23 15:38:08 [async_omni_engine.py:474] raise primary_exc +ERROR 09-23 15:38:08 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 550, in _init_group +ERROR 09-23 15:38:08 [async_omni_engine.py:474] client = self._initialize_replica( +ERROR 09-23 15:38:08 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^^^^^^^^^^ +ERROR 09-23 15:38:08 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 828, in _initialize_replica +ERROR 09-23 15:38:08 [async_omni_engine.py:474] return self._initialize_local_llm_replica(plan, stage_init_timeout) +ERROR 09-23 15:38:08 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +ERROR 09-23 15:38:08 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 934, in _initialize_local_llm_replica +ERROR 09-23 15:38:08 [async_omni_engine.py:474] launch_stack.close() +ERROR 09-23 15:38:08 [async_omni_engine.py:474] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 618, in close +ERROR 09-23 15:38:08 [async_omni_engine.py:474] self.__exit__(None, None, None) +ERROR 09-23 15:38:08 [async_omni_engine.py:474] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 610, in __exit__ +ERROR 09-23 15:38:08 [async_omni_engine.py:474] raise exc_details[1] +ERROR 09-23 15:38:08 [async_omni_engine.py:474] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 595, in __exit__ +ERROR 09-23 15:38:08 [async_omni_engine.py:474] if cb(*exc_details): +ERROR 09-23 15:38:08 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^ +ERROR 09-23 15:38:08 [async_omni_engine.py:474] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 144, in __exit__ +ERROR 09-23 15:38:08 [async_omni_engine.py:474] next(self.gen) +ERROR 09-23 15:38:08 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_startup.py", line 1211, in launch_stage_replica +ERROR 09-23 15:38:08 [async_omni_engine.py:474] wait_for_engine_startup( +ERROR 09-23 15:38:08 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/utils.py", line 1286, in wait_for_engine_startup +ERROR 09-23 15:38:08 [async_omni_engine.py:474] raise RuntimeError( +ERROR 09-23 15:38:08 [async_omni_engine.py:474] RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {} +INFO 09-23 15:38:08 [async_omni_engine.py:1807] [AsyncOmniEngine] Shutting down Orchestrator +Exception in thread orchestrator: +Traceback (most recent call last): + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/threading.py", line 1075, in _bootstrap_inner + self.run() + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/threading.py", line 1012, in run + self._target(*self._args, **self._kwargs) + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 468, in _bootstrap_orchestrator + loop.run_until_complete(_run_orchestrator()) + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/base_events.py", line 691, in run_until_complete + return future.result() + ^^^^^^^^^^^^^^^ + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 424, in _run_orchestrator + self._initialize_stages(stage_init_timeout) + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 350, in _initialize_stages + self._initialize_stages_impl(stage_init_timeout) + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 371, in _initialize_stages_impl + self._runtime.initialize() + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 278, in initialize + raise exc + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 252, in initialize + initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 589, in _initialize_stage_replicas + raise primary_exc + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 550, in _init_group + client = self._initialize_replica( + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 828, in _initialize_replica + return self._initialize_local_llm_replica(plan, stage_init_timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 934, in _initialize_local_llm_replica + launch_stack.close() + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 618, in close + self.__exit__(None, None, None) + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 610, in __exit__ + raise exc_details[1] + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 595, in __exit__ + if cb(*exc_details): + ^^^^^^^^^^^^^^^^ + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 144, in __exit__ + next(self.gen) + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_startup.py", line 1211, in launch_stage_replica + wait_for_engine_startup( + File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/utils.py", line 1286, in wait_for_engine_startup + raise RuntimeError( +RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {} +Traceback (most recent call last): + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/threading.py", line 1075, in _bootstrap_inner + self.run() + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/threading.py", line 1012, in run + self._target(*self._args, **self._kwargs) + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 468, in _bootstrap_orchestrator + loop.run_until_complete(_run_orchestrator()) + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/base_events.py", line 691, in run_until_complete + return future.result() + ^^^^^^^^^^^^^^^ + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 424, in _run_orchestrator + self._initialize_stages(stage_init_timeout) + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 350, in _initialize_stages + self._initialize_stages_impl(stage_init_timeout) + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 371, in _initialize_stages_impl + self._runtime.initialize() + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 278, in initialize + raise exc + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 252, in initialize + initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 589, in _initialize_stage_replicas + raise primary_exc + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 550, in _init_group + client = self._initialize_replica( + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 828, in _initialize_replica + return self._initialize_local_llm_replica(plan, stage_init_timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 934, in _initialize_local_llm_replica + launch_stack.close() + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 618, in close + self.__exit__(None, None, None) + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 610, in __exit__ + raise exc_details[1] + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 595, in __exit__ + if cb(*exc_details): + ^^^^^^^^^^^^^^^^ + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 144, in __exit__ + next(self.gen) + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_startup.py", line 1211, in launch_stage_replica + wait_for_engine_startup( + File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/utils.py", line 1286, in wait_for_engine_startup + raise RuntimeError( +RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {} + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/home/zhout/project/edge_infer/vllm-omni-edge/examples/offline_inference/minicpmo/end2end.py", line 519, in + main(args) + File "/home/zhout/project/edge_infer/vllm-omni-edge/examples/offline_inference/minicpmo/end2end.py", line 283, in main + omni = Omni(**omni_kwargs) + ^^^^^^^^^^^^^^^^^^^ + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/entrypoints/omni_base.py", line 223, in __init__ + self.engine = AsyncOmniEngine( + ^^^^^^^^^^^^^^^^ + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 305, in __init__ + self._wait_for_orchestrator_init(startup_future, startup_timeout) + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 539, in _wait_for_orchestrator_init + startup_future.result( + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/concurrent/futures/_base.py", line 456, in result + return self.__get_result() + ^^^^^^^^^^^^^^^^^^^ + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/concurrent/futures/_base.py", line 401, in __get_result + raise self._exception +RuntimeError: Orchestrator initialization failed: Engine core initialization failed. See root cause above. Failed core proc(s): {} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/minicpmo/minicpmo_cpu_full_compat.log b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/minicpmo/minicpmo_cpu_full_compat.log new file mode 100644 index 000000000..0492db191 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/minicpmo/minicpmo_cpu_full_compat.log @@ -0,0 +1,456 @@ +W0923 15:39:14.030000 4167733 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0923 15:39:14.055000 4167733 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +WARNING 09-23 15:39:14 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton. +INFO 09-23 15:39:14 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available. +/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions: + --> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922 + --> vLLM version 0.28.0 +This will likely cause compatibility issues. + warn_if_misaligned_vllm_version() +INFO 09-23 15:39:15 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-23 15:39:16 [patch.py:510] inductor factorable-divisibility patch: installed. +INFO 09-23 15:39:16 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled. +INFO 09-23 15:39:16 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable). +==================== + vllm version: 0.28.0 + ==================== +INFO 09-23 15:39:16 [omni_base.py:216] [Omni] Initializing with model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5 +INFO 09-23 15:39:16 [async_omni_engine.py:172] [AsyncOmniEngine] Initializing with model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5 +INFO 09-23 15:39:16 [async_omni_engine.py:290] [AsyncOmniEngine] Launching Orchestrator thread with 3 stages +INFO 09-23 15:39:16 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn +INFO 09-23 15:39:16 [initialization.py:356] Loaded OmniTransferConfig with 1 connector configurations +WARNING 09-23 15:39:16 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass) +WARNING 09-23 15:39:16 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass) +WARNING 09-23 15:39:16 [arg_utils.py:1719] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM. +INFO 09-23 15:39:16 [model.py:672] Resolved architecture: MiniCPMO45OmniForConditionalGeneration +INFO 09-23 15:39:16 [model.py:1965] Using max model len 2048 +WARNING 09-23 15:39:16 [model.py:963] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False. +INFO 09-23 15:39:17 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode. +INFO 09-23 15:39:17 [model.py:875] Disabled mm_prefix attention mode because multimodal inputs are configuration-disabled. Attention backends without mm_prefix support may now be selected. +INFO 09-23 15:39:17 [scheduler.py:242] Chunked prefill is enabled with max_num_batched_tokens=2048. +WARNING 09-23 15:39:17 [vllm.py:1285] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none +WARNING 09-23 15:39:17 [vllm.py:1354] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored. +INFO 09-23 15:39:17 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +WARNING 09-23 15:39:17 [vllm.py:652] Model Runner V2 requires Triton; using the V1 model runner instead. +INFO 09-23 15:39:17 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant +WARNING 09-23 15:39:17 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass) +WARNING 09-23 15:39:17 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass) +WARNING 09-23 15:39:17 [arg_utils.py:1719] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM. +INFO 09-23 15:39:17 [model.py:672] Resolved architecture: MiniCPMO45OmniForConditionalGeneration +INFO 09-23 15:39:17 [model.py:1965] Using max model len 512 +WARNING 09-23 15:39:17 [model.py:963] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False. +INFO 09-23 15:39:17 [model.py:1965] Using max model len 512 +INFO 09-23 15:39:17 [scheduler.py:242] Chunked prefill is enabled with max_num_batched_tokens=512. +INFO 09-23 15:39:17 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +WARNING 09-23 15:39:17 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass) +WARNING 09-23 15:39:17 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass) +WARNING 09-23 15:39:17 [arg_utils.py:1719] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM. +INFO 09-23 15:39:17 [model.py:672] Resolved architecture: MiniCPMO45Code2Wav +INFO 09-23 15:39:17 [model.py:1965] Using max model len 4096 +WARNING 09-23 15:39:17 [arg_utils.py:2664] This model does not officially support disabling chunked prefill. Disabling this manually may cause the engine to crash or produce incorrect outputs. +INFO 09-23 15:39:17 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +INFO 09-23 15:39:17 [stage_runtime.py:853] [stage_init] Stage-0 set runtime devices: 0 +W0923 15:39:21.181000 4167762 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0923 15:39:21.207000 4167762 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +WARNING 09-23 15:39:22 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton. +INFO 09-23 15:39:22 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available. +/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions: + --> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922 + --> vLLM version 0.28.0 +This will likely cause compatibility issues. + warn_if_misaligned_vllm_version() +INFO 09-23 15:39:23 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-23 15:39:23 [patch.py:510] inductor factorable-divisibility patch: installed. +INFO 09-23 15:39:23 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled. +INFO 09-23 15:39:23 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable). +(StageEngineCoreProc_stage0_replica0 pid=4167762) INFO 09-23 15:39:23 [core.py:122] Initializing a V1 LLM engine (v0.28.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=2048, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=True, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cpu, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': None, 'compile_ranges_endpoints': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': None, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto') +(StageEngineCoreProc_stage0_replica0 pid=4167762) INFO 09-23 15:39:24 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode. +[W923 15:39:24.921595339 utils.cpp:68] Warning: NUMA binding: Using MEMBIND policy for memory allocation on the NUMA nodes (0). Memory allocations will be strictly bound to these NUMA nodes. (function init_cpu_memory_env) +(StageEngineCoreProc_stage0_replica0 pid=4167762) WARNING 09-23 15:39:24 [vllm.py:652] Model Runner V2 requires Triton; using the V1 model runner instead. +(StageEngineCoreProc_stage0_replica0 pid=4167762) WARNING 09-23 15:39:24 [cpu_worker.py:127] libiomp is not found in LD_PRELOAD. For best performance, please follow the section `set LD_PRELOAD` in https://docs.vllm.ai/en/latest/getting_started/installation/cpu/ to setup required pre-loaded libraries. +(StageEngineCoreProc_stage0_replica0 pid=4167762) INFO 09-23 15:39:24 [parallel_state.py:1638] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_af47bac558064cc0b15208cedc8aabf6 backend=gloo +(StageEngineCoreProc_stage0_replica0 pid=4167762) INFO 09-23 15:39:24 [parallel_state.py:1982] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A +(StageEngineCoreProc_stage0_replica0 pid=4167762) INFO 09-23 15:39:24 [cpu_model_runner.py:131] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5... +(StageEngineCoreProc_stage0_replica0 pid=4167762) WARNING 09-23 15:39:24 [vllm.py:1285] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none +(StageEngineCoreProc_stage0_replica0 pid=4167762) WARNING 09-23 15:39:24 [vllm.py:1354] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored. +(StageEngineCoreProc_stage0_replica0 pid=4167762) INFO 09-23 15:39:24 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +(StageEngineCoreProc_stage0_replica0 pid=4167762) INFO 09-23 15:39:24 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant +(StageEngineCoreProc_stage0_replica0 pid=4167762) INFO 09-23 15:39:24 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +(StageEngineCoreProc_stage0_replica0 pid=4167762) INFO 09-23 15:39:24 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +(StageEngineCoreProc_stage0_replica0 pid=4167762) INFO 09-23 15:39:24 [selector.py:217] Using HND KV cache layout for CPU_ATTN backend. +(StageEngineCoreProc_stage0_replica0 pid=4167762) INFO 09-23 15:39:28 [weight_utils.py:858] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 27.03 GiB. +(StageEngineCoreProc_stage0_replica0 pid=4167762) INFO 09-23 15:39:28 [weight_utils.py:881] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch. +(StageEngineCoreProc_stage0_replica0 pid=4167762) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00 19293 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory'] +(StageEngineCoreProc_stage0_replica0 pid=4167762) INFO 09-23 15:39:43 [cpu_worker.py:255] Explicitly set (1.0/30.91) GiB for KV cache on node 0. +(StageEngineCoreProc_stage0_replica0 pid=4167762) INFO 09-23 15:39:43 [kv_cache_utils.py:1869] GPU KV cache size: 7,168 tokens, Maximum concurrency for 2,048 tokens per request: 3.50x +(StageEngineCoreProc_stage0_replica0 pid=4167762) INFO 09-23 15:39:43 [memory.py:91] Released 0 MiB after warm-up (anonymous RSS 20303 -> 20303 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory'] +(StageEngineCoreProc_stage0_replica0 pid=4167762) INFO 09-23 15:39:43 [core.py:354] init engine (profile, create kv cache, warmup model) took 0.16 s +(StageEngineCoreProc_stage0_replica0 pid=4167762) WARNING 09-23 15:39:43 [scheduler.py:182] Using custom scheduler class vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled. +(StageEngineCoreProc_stage0_replica0 pid=4167762) INFO 09-23 15:39:43 [factory.py:46] Created connector: SharedMemoryConnector +(StageEngineCoreProc_stage0_replica0 pid=4167762) INFO 09-23 15:39:43 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +INFO 09-23 15:39:43 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed +INFO 09-23 15:39:43 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-0 [rep-0] initializing EngineCore +INFO 09-23 15:39:44 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-0 [rep-0] EngineCore running +INFO 09-23 15:39:44 [stage_runtime.py:958] [StageRuntime] Stage 0 initialized +INFO 09-23 15:39:44 [stage_runtime.py:853] [stage_init] Stage-1 set runtime devices: 0 +W0923 15:39:52.020000 4167858 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0923 15:39:52.071000 4167858 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +WARNING 09-23 15:39:53 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton. +INFO 09-23 15:39:53 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available. +/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions: + --> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922 + --> vLLM version 0.28.0 +This will likely cause compatibility issues. + warn_if_misaligned_vllm_version() +INFO 09-23 15:39:55 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-23 15:39:55 [patch.py:510] inductor factorable-divisibility patch: installed. +INFO 09-23 15:39:55 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled. +INFO 09-23 15:39:56 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable). +(StageEngineCoreProc_stage1_replica0 pid=4167858) INFO 09-23 15:39:56 [core.py:122] Initializing a V1 LLM engine (v0.28.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=512, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=True, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cpu, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': None, 'compile_ranges_endpoints': [512], 'inductor_compile_config': {'enable_auto_functionalized_v2': False}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': None, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto') +[W923 15:39:57.775880442 utils.cpp:68] Warning: NUMA binding: Using MEMBIND policy for memory allocation on the NUMA nodes (0). Memory allocations will be strictly bound to these NUMA nodes. (function init_cpu_memory_env) +(StageEngineCoreProc_stage1_replica0 pid=4167858) WARNING 09-23 15:39:57 [vllm.py:652] Model Runner V2 requires Triton; using the V1 model runner instead. +(StageEngineCoreProc_stage1_replica0 pid=4167858) WARNING 09-23 15:39:57 [cpu_worker.py:127] libiomp is not found in LD_PRELOAD. For best performance, please follow the section `set LD_PRELOAD` in https://docs.vllm.ai/en/latest/getting_started/installation/cpu/ to setup required pre-loaded libraries. +(StageEngineCoreProc_stage1_replica0 pid=4167858) INFO 09-23 15:39:57 [parallel_state.py:1638] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_c4c7bdcb69a6490196643cc73405e482 backend=gloo +(StageEngineCoreProc_stage1_replica0 pid=4167858) INFO 09-23 15:39:57 [parallel_state.py:1982] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A +(StageEngineCoreProc_stage1_replica0 pid=4167858) WARNING 09-23 15:39:57 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it. +(StageEngineCoreProc_stage1_replica0 pid=4167858) [transformers] `MiniCPMOProcessor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +(StageEngineCoreProc_stage1_replica0 pid=4167858) [transformers] Requested torchvision backend is not available. Falling back to pil backend. +(StageEngineCoreProc_stage1_replica0 pid=4167858) [transformers] `MiniCPMOProcessor` defines `feature_extractor_class = 'WhisperFeatureExtractor'`, which is deprecated. Register the correct mapping in `AutoFeatureExtractor` instead. +(StageEngineCoreProc_stage1_replica0 pid=4167858) WARNING 09-23 15:40:00 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it. +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] StageEngineCoreProc failed to start. +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] Traceback (most recent call last): +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_core_proc.py", line 189, in run_stage_core +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] engine_core = StageEngineCoreProc( +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] return func(*args, **kwargs) +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/core.py", line 1073, in __init__ +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] super().__init__( +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/core.py", line 133, in __init__ +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] self.model_executor = executor_class(vllm_config) +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] return func(*args, **kwargs) +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/executor/abstract.py", line 110, in __init__ +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] self._init_executor() +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/executor/uniproc_executor.py", line 69, in _init_executor +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] self.driver_worker.init_device() +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 335, in init_device +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] self.worker.init_device() # type: ignore +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/init_timeline.py", line 343, in wrapper +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] return fn(*args, **kwargs) +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/platforms/cpu/worker/cpu_ar_worker.py", line 30, in init_device +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] super().init_device() # type: ignore[misc] +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/cpu_worker.py", line 190, in init_device +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] self.model_runner = CPUModelRunner(self.vllm_config, torch.device("cpu")) +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/cpu_model_runner.py", line 31, in __init__ +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] super().__init__(vllm_config, device) +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/gpu_model_runner.py", line 915, in __init__ +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] MultiModalBudget(self.vllm_config, self.mm_registry) +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/encoder_budget.py", line 94, in __init__ +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] all_mm_max_toks_per_item = get_mm_max_toks_per_item( +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/encoder_budget.py", line 33, in get_mm_max_toks_per_item +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] mm_inputs = mm_registry.get_dummy_mm_inputs( +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/registry.py", line 263, in get_dummy_mm_inputs +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] mm_inputs = processor.apply( +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py", line 3419, in apply +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] return super().apply(inputs, timing_ctx) +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/processing/processor.py", line 1849, in apply +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] ) = self._cached_apply_hf_processor(inputs, timing_ctx) +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/processing/processor.py", line 1628, in _cached_apply_hf_processor +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] ) = self._apply_hf_processor_main( +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py", line 3440, in _apply_hf_processor_main +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] mm_inputs = self.process_mm_inputs(mm_data, hf_processor_mm_kwargs) +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py", line 3595, in process_mm_inputs +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] **self.process_images(mm_data, mm_kwargs), +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py", line 3552, in process_images +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] image_inputs = self._call_hf_processor_on_prompts( +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py", line 3610, in _call_hf_processor_on_prompts +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] return MiniCPMVMultiModalProcessor._call_hf_processor_on_prompts( +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] AttributeError: type object 'MiniCPMVMultiModalProcessor' has no attribute '_call_hf_processor_on_prompts' +(StageEngineCoreProc_stage1_replica0 pid=4167858) Process StageEngineCoreProc_stage1_replica0: +(StageEngineCoreProc_stage1_replica0 pid=4167858) Traceback (most recent call last): +(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/multiprocessing/process.py", line 314, in _bootstrap +(StageEngineCoreProc_stage1_replica0 pid=4167858) self.run() +(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/multiprocessing/process.py", line 108, in run +(StageEngineCoreProc_stage1_replica0 pid=4167858) self._target(*self._args, **self._kwargs) +(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_core_proc.py", line 189, in run_stage_core +(StageEngineCoreProc_stage1_replica0 pid=4167858) engine_core = StageEngineCoreProc( +(StageEngineCoreProc_stage1_replica0 pid=4167858) ^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper +(StageEngineCoreProc_stage1_replica0 pid=4167858) return func(*args, **kwargs) +(StageEngineCoreProc_stage1_replica0 pid=4167858) ^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/core.py", line 1073, in __init__ +(StageEngineCoreProc_stage1_replica0 pid=4167858) super().__init__( +(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/core.py", line 133, in __init__ +(StageEngineCoreProc_stage1_replica0 pid=4167858) self.model_executor = executor_class(vllm_config) +(StageEngineCoreProc_stage1_replica0 pid=4167858) ^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper +(StageEngineCoreProc_stage1_replica0 pid=4167858) return func(*args, **kwargs) +(StageEngineCoreProc_stage1_replica0 pid=4167858) ^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/executor/abstract.py", line 110, in __init__ +(StageEngineCoreProc_stage1_replica0 pid=4167858) self._init_executor() +(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/executor/uniproc_executor.py", line 69, in _init_executor +(StageEngineCoreProc_stage1_replica0 pid=4167858) self.driver_worker.init_device() +(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 335, in init_device +(StageEngineCoreProc_stage1_replica0 pid=4167858) self.worker.init_device() # type: ignore +(StageEngineCoreProc_stage1_replica0 pid=4167858) ^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/init_timeline.py", line 343, in wrapper +(StageEngineCoreProc_stage1_replica0 pid=4167858) return fn(*args, **kwargs) +(StageEngineCoreProc_stage1_replica0 pid=4167858) ^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/platforms/cpu/worker/cpu_ar_worker.py", line 30, in init_device +(StageEngineCoreProc_stage1_replica0 pid=4167858) super().init_device() # type: ignore[misc] +(StageEngineCoreProc_stage1_replica0 pid=4167858) ^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/cpu_worker.py", line 190, in init_device +(StageEngineCoreProc_stage1_replica0 pid=4167858) self.model_runner = CPUModelRunner(self.vllm_config, torch.device("cpu")) +(StageEngineCoreProc_stage1_replica0 pid=4167858) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/cpu_model_runner.py", line 31, in __init__ +(StageEngineCoreProc_stage1_replica0 pid=4167858) super().__init__(vllm_config, device) +(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/gpu_model_runner.py", line 915, in __init__ +(StageEngineCoreProc_stage1_replica0 pid=4167858) MultiModalBudget(self.vllm_config, self.mm_registry) +(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/encoder_budget.py", line 94, in __init__ +(StageEngineCoreProc_stage1_replica0 pid=4167858) all_mm_max_toks_per_item = get_mm_max_toks_per_item( +(StageEngineCoreProc_stage1_replica0 pid=4167858) ^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/encoder_budget.py", line 33, in get_mm_max_toks_per_item +(StageEngineCoreProc_stage1_replica0 pid=4167858) mm_inputs = mm_registry.get_dummy_mm_inputs( +(StageEngineCoreProc_stage1_replica0 pid=4167858) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/registry.py", line 263, in get_dummy_mm_inputs +(StageEngineCoreProc_stage1_replica0 pid=4167858) mm_inputs = processor.apply( +(StageEngineCoreProc_stage1_replica0 pid=4167858) ^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py", line 3419, in apply +(StageEngineCoreProc_stage1_replica0 pid=4167858) return super().apply(inputs, timing_ctx) +(StageEngineCoreProc_stage1_replica0 pid=4167858) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/processing/processor.py", line 1849, in apply +(StageEngineCoreProc_stage1_replica0 pid=4167858) ) = self._cached_apply_hf_processor(inputs, timing_ctx) +(StageEngineCoreProc_stage1_replica0 pid=4167858) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/processing/processor.py", line 1628, in _cached_apply_hf_processor +(StageEngineCoreProc_stage1_replica0 pid=4167858) ) = self._apply_hf_processor_main( +(StageEngineCoreProc_stage1_replica0 pid=4167858) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py", line 3440, in _apply_hf_processor_main +(StageEngineCoreProc_stage1_replica0 pid=4167858) mm_inputs = self.process_mm_inputs(mm_data, hf_processor_mm_kwargs) +(StageEngineCoreProc_stage1_replica0 pid=4167858) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py", line 3595, in process_mm_inputs +(StageEngineCoreProc_stage1_replica0 pid=4167858) **self.process_images(mm_data, mm_kwargs), +(StageEngineCoreProc_stage1_replica0 pid=4167858) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py", line 3552, in process_images +(StageEngineCoreProc_stage1_replica0 pid=4167858) image_inputs = self._call_hf_processor_on_prompts( +(StageEngineCoreProc_stage1_replica0 pid=4167858) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py", line 3610, in _call_hf_processor_on_prompts +(StageEngineCoreProc_stage1_replica0 pid=4167858) return MiniCPMVMultiModalProcessor._call_hf_processor_on_prompts( +(StageEngineCoreProc_stage1_replica0 pid=4167858) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4167858) AttributeError: type object 'MiniCPMVMultiModalProcessor' has no attribute '_call_hf_processor_on_prompts' +ERROR 09-23 15:40:03 [stage_runtime.py:265] [StageRuntime] Stage initialization failed; shutting down 1 initialized client(s) +ERROR 09-23 15:40:03 [stage_runtime.py:265] Traceback (most recent call last): +ERROR 09-23 15:40:03 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 252, in initialize +ERROR 09-23 15:40:03 [stage_runtime.py:265] initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout) +ERROR 09-23 15:40:03 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +ERROR 09-23 15:40:03 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 589, in _initialize_stage_replicas +ERROR 09-23 15:40:03 [stage_runtime.py:265] raise primary_exc +ERROR 09-23 15:40:03 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 550, in _init_group +ERROR 09-23 15:40:03 [stage_runtime.py:265] client = self._initialize_replica( +ERROR 09-23 15:40:03 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^ +ERROR 09-23 15:40:03 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 828, in _initialize_replica +ERROR 09-23 15:40:03 [stage_runtime.py:265] return self._initialize_local_llm_replica(plan, stage_init_timeout) +ERROR 09-23 15:40:03 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +ERROR 09-23 15:40:03 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 934, in _initialize_local_llm_replica +ERROR 09-23 15:40:03 [stage_runtime.py:265] launch_stack.close() +ERROR 09-23 15:40:03 [stage_runtime.py:265] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 618, in close +ERROR 09-23 15:40:03 [stage_runtime.py:265] self.__exit__(None, None, None) +ERROR 09-23 15:40:03 [stage_runtime.py:265] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 610, in __exit__ +ERROR 09-23 15:40:03 [stage_runtime.py:265] raise exc_details[1] +ERROR 09-23 15:40:03 [stage_runtime.py:265] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 595, in __exit__ +ERROR 09-23 15:40:03 [stage_runtime.py:265] if cb(*exc_details): +ERROR 09-23 15:40:03 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^ +ERROR 09-23 15:40:03 [stage_runtime.py:265] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 144, in __exit__ +ERROR 09-23 15:40:03 [stage_runtime.py:265] next(self.gen) +ERROR 09-23 15:40:03 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_startup.py", line 1211, in launch_stage_replica +ERROR 09-23 15:40:03 [stage_runtime.py:265] wait_for_engine_startup( +ERROR 09-23 15:40:03 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/utils.py", line 1286, in wait_for_engine_startup +ERROR 09-23 15:40:03 [stage_runtime.py:265] raise RuntimeError( +ERROR 09-23 15:40:03 [stage_runtime.py:265] RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {} +INFO 09-23 15:40:03 [core_client.py:689] [shutdown] MPClient: start timeout=default +INFO 09-23 15:40:03 [core_client.py:691] [shutdown] MPClient: stopping engine manager +INFO 09-23 15:40:03 [utils.py:615] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage0_replica0 +INFO 09-23 15:40:07 [core_client.py:693] [shutdown] MPClient: engine manager stopped +INFO 09-23 15:40:07 [core_client.py:694] [shutdown] MPClient: cleaning up background resources +INFO 09-23 15:40:07 [core_client.py:696] [shutdown] MPClient: complete +ERROR 09-23 15:40:07 [async_omni_engine.py:474] [AsyncOmniEngine] Orchestrator thread crashed +ERROR 09-23 15:40:07 [async_omni_engine.py:474] Traceback (most recent call last): +ERROR 09-23 15:40:07 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 468, in _bootstrap_orchestrator +ERROR 09-23 15:40:07 [async_omni_engine.py:474] loop.run_until_complete(_run_orchestrator()) +ERROR 09-23 15:40:07 [async_omni_engine.py:474] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/base_events.py", line 691, in run_until_complete +ERROR 09-23 15:40:07 [async_omni_engine.py:474] return future.result() +ERROR 09-23 15:40:07 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^ +ERROR 09-23 15:40:07 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 424, in _run_orchestrator +ERROR 09-23 15:40:07 [async_omni_engine.py:474] self._initialize_stages(stage_init_timeout) +ERROR 09-23 15:40:07 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 350, in _initialize_stages +ERROR 09-23 15:40:07 [async_omni_engine.py:474] self._initialize_stages_impl(stage_init_timeout) +ERROR 09-23 15:40:07 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 371, in _initialize_stages_impl +ERROR 09-23 15:40:07 [async_omni_engine.py:474] self._runtime.initialize() +ERROR 09-23 15:40:07 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 278, in initialize +ERROR 09-23 15:40:07 [async_omni_engine.py:474] raise exc +ERROR 09-23 15:40:07 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 252, in initialize +ERROR 09-23 15:40:07 [async_omni_engine.py:474] initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout) +ERROR 09-23 15:40:07 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +ERROR 09-23 15:40:07 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 589, in _initialize_stage_replicas +ERROR 09-23 15:40:07 [async_omni_engine.py:474] raise primary_exc +ERROR 09-23 15:40:07 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 550, in _init_group +ERROR 09-23 15:40:07 [async_omni_engine.py:474] client = self._initialize_replica( +ERROR 09-23 15:40:07 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^^^^^^^^^^ +ERROR 09-23 15:40:07 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 828, in _initialize_replica +ERROR 09-23 15:40:07 [async_omni_engine.py:474] return self._initialize_local_llm_replica(plan, stage_init_timeout) +ERROR 09-23 15:40:07 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +ERROR 09-23 15:40:07 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 934, in _initialize_local_llm_replica +ERROR 09-23 15:40:07 [async_omni_engine.py:474] launch_stack.close() +ERROR 09-23 15:40:07 [async_omni_engine.py:474] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 618, in close +ERROR 09-23 15:40:07 [async_omni_engine.py:474] self.__exit__(None, None, None) +ERROR 09-23 15:40:07 [async_omni_engine.py:474] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 610, in __exit__ +ERROR 09-23 15:40:07 [async_omni_engine.py:474] raise exc_details[1] +ERROR 09-23 15:40:07 [async_omni_engine.py:474] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 595, in __exit__ +ERROR 09-23 15:40:07 [async_omni_engine.py:474] if cb(*exc_details): +ERROR 09-23 15:40:07 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^ +ERROR 09-23 15:40:07 [async_omni_engine.py:474] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 144, in __exit__ +ERROR 09-23 15:40:07 [async_omni_engine.py:474] next(self.gen) +ERROR 09-23 15:40:07 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_startup.py", line 1211, in launch_stage_replica +ERROR 09-23 15:40:07 [async_omni_engine.py:474] wait_for_engine_startup( +ERROR 09-23 15:40:07 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/utils.py", line 1286, in wait_for_engine_startup +ERROR 09-23 15:40:07 [async_omni_engine.py:474] raise RuntimeError( +ERROR 09-23 15:40:07 [async_omni_engine.py:474] RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {} +INFO 09-23 15:40:07 [async_omni_engine.py:1807] [AsyncOmniEngine] Shutting down Orchestrator +Exception in thread orchestrator: +Traceback (most recent call last): + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/threading.py", line 1075, in _bootstrap_inner + self.run() + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/threading.py", line 1012, in run + self._target(*self._args, **self._kwargs) + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 468, in _bootstrap_orchestrator + loop.run_until_complete(_run_orchestrator()) + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/base_events.py", line 691, in run_until_complete + return future.result() + ^^^^^^^^^^^^^^^ + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 424, in _run_orchestrator + self._initialize_stages(stage_init_timeout) + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 350, in _initialize_stages + self._initialize_stages_impl(stage_init_timeout) + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 371, in _initialize_stages_impl + self._runtime.initialize() + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 278, in initialize + raise exc + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 252, in initialize + initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 589, in _initialize_stage_replicas + raise primary_exc + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 550, in _init_group + client = self._initialize_replica( + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 828, in _initialize_replica + return self._initialize_local_llm_replica(plan, stage_init_timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 934, in _initialize_local_llm_replica + launch_stack.close() + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 618, in close + self.__exit__(None, None, None) + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 610, in __exit__ + raise exc_details[1] + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 595, in __exit__ + if cb(*exc_details): + ^^^^^^^^^^^^^^^^ + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 144, in __exit__ + next(self.gen) + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_startup.py", line 1211, in launch_stage_replica + wait_for_engine_startup( + File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/utils.py", line 1286, in wait_for_engine_startup + raise RuntimeError( +RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {} +Traceback (most recent call last): + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/threading.py", line 1075, in _bootstrap_inner + self.run() + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/threading.py", line 1012, in run + self._target(*self._args, **self._kwargs) + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 468, in _bootstrap_orchestrator + loop.run_until_complete(_run_orchestrator()) + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/base_events.py", line 691, in run_until_complete + return future.result() + ^^^^^^^^^^^^^^^ + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 424, in _run_orchestrator + self._initialize_stages(stage_init_timeout) + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 350, in _initialize_stages + self._initialize_stages_impl(stage_init_timeout) + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 371, in _initialize_stages_impl + self._runtime.initialize() + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 278, in initialize + raise exc + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 252, in initialize + initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 589, in _initialize_stage_replicas + raise primary_exc + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 550, in _init_group + client = self._initialize_replica( + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 828, in _initialize_replica + return self._initialize_local_llm_replica(plan, stage_init_timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 934, in _initialize_local_llm_replica + launch_stack.close() + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 618, in close + self.__exit__(None, None, None) + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 610, in __exit__ + raise exc_details[1] + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 595, in __exit__ + if cb(*exc_details): + ^^^^^^^^^^^^^^^^ + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 144, in __exit__ + next(self.gen) + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_startup.py", line 1211, in launch_stage_replica + wait_for_engine_startup( + File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/utils.py", line 1286, in wait_for_engine_startup + raise RuntimeError( +RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {} + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/home/zhout/project/edge_infer/vllm-omni-edge/examples/offline_inference/minicpmo/end2end.py", line 519, in + main(args) + File "/home/zhout/project/edge_infer/vllm-omni-edge/examples/offline_inference/minicpmo/end2end.py", line 283, in main + omni = Omni(**omni_kwargs) + ^^^^^^^^^^^^^^^^^^^ + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/entrypoints/omni_base.py", line 223, in __init__ + self.engine = AsyncOmniEngine( + ^^^^^^^^^^^^^^^^ + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 305, in __init__ + self._wait_for_orchestrator_init(startup_future, startup_timeout) + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 539, in _wait_for_orchestrator_init + startup_future.result( + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/concurrent/futures/_base.py", line 456, in result + return self.__get_result() + ^^^^^^^^^^^^^^^^^^^ + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/concurrent/futures/_base.py", line 401, in __get_result + raise self._exception +RuntimeError: Orchestrator initialization failed: Engine core initialization failed. See root cause above. Failed core proc(s): {} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/minicpmo/minicpmo_cpu_full_compat2.log b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/minicpmo/minicpmo_cpu_full_compat2.log new file mode 100644 index 000000000..1061767d8 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/minicpmo/minicpmo_cpu_full_compat2.log @@ -0,0 +1,490 @@ +W0923 15:41:18.737000 4168074 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0923 15:41:18.764000 4168074 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +WARNING 09-23 15:41:19 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton. +INFO 09-23 15:41:19 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available. +/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions: + --> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922 + --> vLLM version 0.28.0 +This will likely cause compatibility issues. + warn_if_misaligned_vllm_version() +INFO 09-23 15:41:20 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-23 15:41:21 [patch.py:510] inductor factorable-divisibility patch: installed. +INFO 09-23 15:41:21 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled. +INFO 09-23 15:41:21 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable). +==================== + vllm version: 0.28.0 + ==================== +INFO 09-23 15:41:21 [omni_base.py:216] [Omni] Initializing with model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5 +INFO 09-23 15:41:21 [async_omni_engine.py:172] [AsyncOmniEngine] Initializing with model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5 +INFO 09-23 15:41:21 [async_omni_engine.py:290] [AsyncOmniEngine] Launching Orchestrator thread with 3 stages +INFO 09-23 15:41:21 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn +INFO 09-23 15:41:21 [initialization.py:356] Loaded OmniTransferConfig with 1 connector configurations +WARNING 09-23 15:41:21 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass) +WARNING 09-23 15:41:21 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass) +WARNING 09-23 15:41:21 [arg_utils.py:1719] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM. +INFO 09-23 15:41:21 [model.py:672] Resolved architecture: MiniCPMO45OmniForConditionalGeneration +INFO 09-23 15:41:21 [model.py:1965] Using max model len 2048 +WARNING 09-23 15:41:21 [model.py:963] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False. +INFO 09-23 15:41:22 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode. +INFO 09-23 15:41:22 [model.py:875] Disabled mm_prefix attention mode because multimodal inputs are configuration-disabled. Attention backends without mm_prefix support may now be selected. +INFO 09-23 15:41:22 [scheduler.py:242] Chunked prefill is enabled with max_num_batched_tokens=2048. +WARNING 09-23 15:41:22 [vllm.py:1285] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none +WARNING 09-23 15:41:22 [vllm.py:1354] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored. +INFO 09-23 15:41:22 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +WARNING 09-23 15:41:22 [vllm.py:652] Model Runner V2 requires Triton; using the V1 model runner instead. +INFO 09-23 15:41:22 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant +WARNING 09-23 15:41:22 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass) +WARNING 09-23 15:41:22 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass) +WARNING 09-23 15:41:22 [arg_utils.py:1719] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM. +INFO 09-23 15:41:22 [model.py:672] Resolved architecture: MiniCPMO45OmniForConditionalGeneration +INFO 09-23 15:41:22 [model.py:1965] Using max model len 512 +WARNING 09-23 15:41:22 [model.py:963] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False. +INFO 09-23 15:41:22 [model.py:1965] Using max model len 512 +INFO 09-23 15:41:22 [scheduler.py:242] Chunked prefill is enabled with max_num_batched_tokens=512. +INFO 09-23 15:41:22 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +WARNING 09-23 15:41:22 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass) +WARNING 09-23 15:41:22 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass) +WARNING 09-23 15:41:22 [arg_utils.py:1719] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM. +INFO 09-23 15:41:22 [model.py:672] Resolved architecture: MiniCPMO45Code2Wav +INFO 09-23 15:41:22 [model.py:1965] Using max model len 4096 +WARNING 09-23 15:41:22 [arg_utils.py:2664] This model does not officially support disabling chunked prefill. Disabling this manually may cause the engine to crash or produce incorrect outputs. +INFO 09-23 15:41:22 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +INFO 09-23 15:41:22 [stage_runtime.py:853] [stage_init] Stage-0 set runtime devices: 0 +W0923 15:41:26.220000 4168102 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0923 15:41:26.247000 4168102 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +WARNING 09-23 15:41:27 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton. +INFO 09-23 15:41:27 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available. +/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions: + --> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922 + --> vLLM version 0.28.0 +This will likely cause compatibility issues. + warn_if_misaligned_vllm_version() +INFO 09-23 15:41:28 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-23 15:41:28 [patch.py:510] inductor factorable-divisibility patch: installed. +INFO 09-23 15:41:28 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled. +INFO 09-23 15:41:28 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable). +(StageEngineCoreProc_stage0_replica0 pid=4168102) INFO 09-23 15:41:28 [core.py:122] Initializing a V1 LLM engine (v0.28.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=2048, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=True, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cpu, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': None, 'compile_ranges_endpoints': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': None, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto') +(StageEngineCoreProc_stage0_replica0 pid=4168102) INFO 09-23 15:41:29 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode. +[W923 15:41:29.029555315 utils.cpp:68] Warning: NUMA binding: Using MEMBIND policy for memory allocation on the NUMA nodes (0). Memory allocations will be strictly bound to these NUMA nodes. (function init_cpu_memory_env) +(StageEngineCoreProc_stage0_replica0 pid=4168102) WARNING 09-23 15:41:29 [vllm.py:652] Model Runner V2 requires Triton; using the V1 model runner instead. +(StageEngineCoreProc_stage0_replica0 pid=4168102) WARNING 09-23 15:41:29 [cpu_worker.py:127] libiomp is not found in LD_PRELOAD. For best performance, please follow the section `set LD_PRELOAD` in https://docs.vllm.ai/en/latest/getting_started/installation/cpu/ to setup required pre-loaded libraries. +(StageEngineCoreProc_stage0_replica0 pid=4168102) INFO 09-23 15:41:29 [parallel_state.py:1638] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_aece089fe58d4103be76afbba283ffff backend=gloo +(StageEngineCoreProc_stage0_replica0 pid=4168102) INFO 09-23 15:41:29 [parallel_state.py:1982] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A +(StageEngineCoreProc_stage0_replica0 pid=4168102) INFO 09-23 15:41:29 [cpu_model_runner.py:131] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5... +(StageEngineCoreProc_stage0_replica0 pid=4168102) WARNING 09-23 15:41:29 [vllm.py:1285] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none +(StageEngineCoreProc_stage0_replica0 pid=4168102) WARNING 09-23 15:41:29 [vllm.py:1354] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored. +(StageEngineCoreProc_stage0_replica0 pid=4168102) INFO 09-23 15:41:29 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +(StageEngineCoreProc_stage0_replica0 pid=4168102) INFO 09-23 15:41:29 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant +(StageEngineCoreProc_stage0_replica0 pid=4168102) INFO 09-23 15:41:29 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +(StageEngineCoreProc_stage0_replica0 pid=4168102) INFO 09-23 15:41:29 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +(StageEngineCoreProc_stage0_replica0 pid=4168102) INFO 09-23 15:41:29 [selector.py:217] Using HND KV cache layout for CPU_ATTN backend. +(StageEngineCoreProc_stage0_replica0 pid=4168102) INFO 09-23 15:41:33 [weight_utils.py:858] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 27.06 GiB. +(StageEngineCoreProc_stage0_replica0 pid=4168102) INFO 09-23 15:41:33 [weight_utils.py:881] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch. +(StageEngineCoreProc_stage0_replica0 pid=4168102) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00 19293 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory'] +(StageEngineCoreProc_stage0_replica0 pid=4168102) INFO 09-23 15:41:50 [cpu_worker.py:255] Explicitly set (1.0/30.91) GiB for KV cache on node 0. +(StageEngineCoreProc_stage0_replica0 pid=4168102) INFO 09-23 15:41:50 [kv_cache_utils.py:1869] GPU KV cache size: 7,168 tokens, Maximum concurrency for 2,048 tokens per request: 3.50x +(StageEngineCoreProc_stage0_replica0 pid=4168102) INFO 09-23 15:41:50 [memory.py:91] Released 0 MiB after warm-up (anonymous RSS 20303 -> 20303 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory'] +(StageEngineCoreProc_stage0_replica0 pid=4168102) INFO 09-23 15:41:50 [core.py:354] init engine (profile, create kv cache, warmup model) took 0.20 s +(StageEngineCoreProc_stage0_replica0 pid=4168102) WARNING 09-23 15:41:50 [scheduler.py:182] Using custom scheduler class vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled. +(StageEngineCoreProc_stage0_replica0 pid=4168102) INFO 09-23 15:41:50 [factory.py:46] Created connector: SharedMemoryConnector +(StageEngineCoreProc_stage0_replica0 pid=4168102) INFO 09-23 15:41:51 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +INFO 09-23 15:41:51 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed +INFO 09-23 15:41:51 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-0 [rep-0] initializing EngineCore +INFO 09-23 15:41:51 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-0 [rep-0] EngineCore running +INFO 09-23 15:41:51 [stage_runtime.py:958] [StageRuntime] Stage 0 initialized +INFO 09-23 15:41:51 [stage_runtime.py:853] [stage_init] Stage-1 set runtime devices: 0 +W0923 15:41:59.494000 4168210 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0923 15:41:59.538000 4168210 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +WARNING 09-23 15:42:00 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton. +INFO 09-23 15:42:00 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available. +/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions: + --> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922 + --> vLLM version 0.28.0 +This will likely cause compatibility issues. + warn_if_misaligned_vllm_version() +INFO 09-23 15:42:02 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-23 15:42:02 [patch.py:510] inductor factorable-divisibility patch: installed. +INFO 09-23 15:42:02 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled. +INFO 09-23 15:42:02 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable). +(StageEngineCoreProc_stage1_replica0 pid=4168210) INFO 09-23 15:42:03 [core.py:122] Initializing a V1 LLM engine (v0.28.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=512, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=True, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cpu, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': None, 'compile_ranges_endpoints': [512], 'inductor_compile_config': {'enable_auto_functionalized_v2': False}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': None, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto') +[W923 15:42:04.364823574 utils.cpp:68] Warning: NUMA binding: Using MEMBIND policy for memory allocation on the NUMA nodes (0). Memory allocations will be strictly bound to these NUMA nodes. (function init_cpu_memory_env) +(StageEngineCoreProc_stage1_replica0 pid=4168210) WARNING 09-23 15:42:04 [vllm.py:652] Model Runner V2 requires Triton; using the V1 model runner instead. +(StageEngineCoreProc_stage1_replica0 pid=4168210) WARNING 09-23 15:42:04 [cpu_worker.py:127] libiomp is not found in LD_PRELOAD. For best performance, please follow the section `set LD_PRELOAD` in https://docs.vllm.ai/en/latest/getting_started/installation/cpu/ to setup required pre-loaded libraries. +(StageEngineCoreProc_stage1_replica0 pid=4168210) INFO 09-23 15:42:04 [parallel_state.py:1638] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_8835b78aaeee4981b46f13fffdc72e04 backend=gloo +(StageEngineCoreProc_stage1_replica0 pid=4168210) INFO 09-23 15:42:04 [parallel_state.py:1982] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A +(StageEngineCoreProc_stage1_replica0 pid=4168210) WARNING 09-23 15:42:04 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it. +(StageEngineCoreProc_stage1_replica0 pid=4168210) [transformers] `MiniCPMOProcessor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +(StageEngineCoreProc_stage1_replica0 pid=4168210) [transformers] Requested torchvision backend is not available. Falling back to pil backend. +(StageEngineCoreProc_stage1_replica0 pid=4168210) [transformers] `MiniCPMOProcessor` defines `feature_extractor_class = 'WhisperFeatureExtractor'`, which is deprecated. Register the correct mapping in `AutoFeatureExtractor` instead. +(StageEngineCoreProc_stage1_replica0 pid=4168210) [transformers] Image processor MiniCPMVImageProcessor: kwargs ['max_slice_nums'] were applied for backward compatibility. To avoid this warning, add them to valid_kwargs: create a custom TypedDict extending ImagesKwargs with these keys and set it as the `valid_kwargs` class attribute. +(StageEngineCoreProc_stage1_replica0 pid=4168210) WARNING 09-23 15:42:12 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it. +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] StageEngineCoreProc failed to start. +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] Traceback (most recent call last): +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/processing/context.py", line 286, in call_hf_processor +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] output = hf_processor(**data, **allowed_kwargs) +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/transformers_utils/processors/minicpmo.py", line 107, in __call__ +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] model_inputs = self._convert_omni_to_inputs( +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/transformers_utils/processors/minicpmo.py", line 397, in _convert_omni_to_inputs +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] assert len(image_tags) == len(image_sizes[index]) +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] AssertionError +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] The above exception was the direct cause of the following exception: +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] Traceback (most recent call last): +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_core_proc.py", line 189, in run_stage_core +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] engine_core = StageEngineCoreProc( +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] return func(*args, **kwargs) +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/core.py", line 1073, in __init__ +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] super().__init__( +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/core.py", line 133, in __init__ +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] self.model_executor = executor_class(vllm_config) +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] return func(*args, **kwargs) +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/executor/abstract.py", line 110, in __init__ +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] self._init_executor() +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/executor/uniproc_executor.py", line 69, in _init_executor +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] self.driver_worker.init_device() +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 335, in init_device +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] self.worker.init_device() # type: ignore +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/init_timeline.py", line 343, in wrapper +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] return fn(*args, **kwargs) +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/platforms/cpu/worker/cpu_ar_worker.py", line 30, in init_device +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] super().init_device() # type: ignore[misc] +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/cpu_worker.py", line 190, in init_device +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] self.model_runner = CPUModelRunner(self.vllm_config, torch.device("cpu")) +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/cpu_model_runner.py", line 31, in __init__ +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] super().__init__(vllm_config, device) +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/gpu_model_runner.py", line 915, in __init__ +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] MultiModalBudget(self.vllm_config, self.mm_registry) +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/encoder_budget.py", line 94, in __init__ +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] all_mm_max_toks_per_item = get_mm_max_toks_per_item( +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/encoder_budget.py", line 33, in get_mm_max_toks_per_item +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] mm_inputs = mm_registry.get_dummy_mm_inputs( +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/registry.py", line 263, in get_dummy_mm_inputs +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] mm_inputs = processor.apply( +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py", line 3420, in apply +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] return super().apply(inputs, timing_ctx) +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/processing/processor.py", line 1849, in apply +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ) = self._cached_apply_hf_processor(inputs, timing_ctx) +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/processing/processor.py", line 1628, in _cached_apply_hf_processor +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ) = self._apply_hf_processor_main( +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py", line 3444, in _apply_hf_processor_main +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] prompt_ids = self._apply_hf_processor_text_only(prompt, tokenization_kwargs or {}) +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/processing/processor.py", line 1336, in _apply_hf_processor_text_only +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] prompt_ids, _, _ = self._apply_hf_processor_text_mm( +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/processing/processor.py", line 1301, in _apply_hf_processor_text_mm +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] processed_data = self._call_hf_processor( +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/processing/processor.py", line 1258, in _call_hf_processor +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] return self.info.ctx.call_hf_processor( +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/processing/context.py", line 293, in call_hf_processor +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] raise ValueError(msg) from exc +(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ValueError: Failed to apply MiniCPMOProcessor on data={'text': '()(./)()'} with kwargs={'truncation': False, 'return_tensors': 'pt'} +(StageEngineCoreProc_stage1_replica0 pid=4168210) Process StageEngineCoreProc_stage1_replica0: +(StageEngineCoreProc_stage1_replica0 pid=4168210) Traceback (most recent call last): +(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/processing/context.py", line 286, in call_hf_processor +(StageEngineCoreProc_stage1_replica0 pid=4168210) output = hf_processor(**data, **allowed_kwargs) +(StageEngineCoreProc_stage1_replica0 pid=4168210) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/transformers_utils/processors/minicpmo.py", line 107, in __call__ +(StageEngineCoreProc_stage1_replica0 pid=4168210) model_inputs = self._convert_omni_to_inputs( +(StageEngineCoreProc_stage1_replica0 pid=4168210) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/transformers_utils/processors/minicpmo.py", line 397, in _convert_omni_to_inputs +(StageEngineCoreProc_stage1_replica0 pid=4168210) assert len(image_tags) == len(image_sizes[index]) +(StageEngineCoreProc_stage1_replica0 pid=4168210) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4168210) AssertionError +(StageEngineCoreProc_stage1_replica0 pid=4168210) +(StageEngineCoreProc_stage1_replica0 pid=4168210) The above exception was the direct cause of the following exception: +(StageEngineCoreProc_stage1_replica0 pid=4168210) +(StageEngineCoreProc_stage1_replica0 pid=4168210) Traceback (most recent call last): +(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/multiprocessing/process.py", line 314, in _bootstrap +(StageEngineCoreProc_stage1_replica0 pid=4168210) self.run() +(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/multiprocessing/process.py", line 108, in run +(StageEngineCoreProc_stage1_replica0 pid=4168210) self._target(*self._args, **self._kwargs) +(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_core_proc.py", line 189, in run_stage_core +(StageEngineCoreProc_stage1_replica0 pid=4168210) engine_core = StageEngineCoreProc( +(StageEngineCoreProc_stage1_replica0 pid=4168210) ^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper +(StageEngineCoreProc_stage1_replica0 pid=4168210) return func(*args, **kwargs) +(StageEngineCoreProc_stage1_replica0 pid=4168210) ^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/core.py", line 1073, in __init__ +(StageEngineCoreProc_stage1_replica0 pid=4168210) super().__init__( +(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/core.py", line 133, in __init__ +(StageEngineCoreProc_stage1_replica0 pid=4168210) self.model_executor = executor_class(vllm_config) +(StageEngineCoreProc_stage1_replica0 pid=4168210) ^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper +(StageEngineCoreProc_stage1_replica0 pid=4168210) return func(*args, **kwargs) +(StageEngineCoreProc_stage1_replica0 pid=4168210) ^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/executor/abstract.py", line 110, in __init__ +(StageEngineCoreProc_stage1_replica0 pid=4168210) self._init_executor() +(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/executor/uniproc_executor.py", line 69, in _init_executor +(StageEngineCoreProc_stage1_replica0 pid=4168210) self.driver_worker.init_device() +(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 335, in init_device +(StageEngineCoreProc_stage1_replica0 pid=4168210) self.worker.init_device() # type: ignore +(StageEngineCoreProc_stage1_replica0 pid=4168210) ^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/init_timeline.py", line 343, in wrapper +(StageEngineCoreProc_stage1_replica0 pid=4168210) return fn(*args, **kwargs) +(StageEngineCoreProc_stage1_replica0 pid=4168210) ^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/platforms/cpu/worker/cpu_ar_worker.py", line 30, in init_device +(StageEngineCoreProc_stage1_replica0 pid=4168210) super().init_device() # type: ignore[misc] +(StageEngineCoreProc_stage1_replica0 pid=4168210) ^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/cpu_worker.py", line 190, in init_device +(StageEngineCoreProc_stage1_replica0 pid=4168210) self.model_runner = CPUModelRunner(self.vllm_config, torch.device("cpu")) +(StageEngineCoreProc_stage1_replica0 pid=4168210) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/cpu_model_runner.py", line 31, in __init__ +(StageEngineCoreProc_stage1_replica0 pid=4168210) super().__init__(vllm_config, device) +(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/gpu_model_runner.py", line 915, in __init__ +(StageEngineCoreProc_stage1_replica0 pid=4168210) MultiModalBudget(self.vllm_config, self.mm_registry) +(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/encoder_budget.py", line 94, in __init__ +(StageEngineCoreProc_stage1_replica0 pid=4168210) all_mm_max_toks_per_item = get_mm_max_toks_per_item( +(StageEngineCoreProc_stage1_replica0 pid=4168210) ^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/encoder_budget.py", line 33, in get_mm_max_toks_per_item +(StageEngineCoreProc_stage1_replica0 pid=4168210) mm_inputs = mm_registry.get_dummy_mm_inputs( +(StageEngineCoreProc_stage1_replica0 pid=4168210) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/registry.py", line 263, in get_dummy_mm_inputs +(StageEngineCoreProc_stage1_replica0 pid=4168210) mm_inputs = processor.apply( +(StageEngineCoreProc_stage1_replica0 pid=4168210) ^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py", line 3420, in apply +(StageEngineCoreProc_stage1_replica0 pid=4168210) return super().apply(inputs, timing_ctx) +(StageEngineCoreProc_stage1_replica0 pid=4168210) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/processing/processor.py", line 1849, in apply +(StageEngineCoreProc_stage1_replica0 pid=4168210) ) = self._cached_apply_hf_processor(inputs, timing_ctx) +(StageEngineCoreProc_stage1_replica0 pid=4168210) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/processing/processor.py", line 1628, in _cached_apply_hf_processor +(StageEngineCoreProc_stage1_replica0 pid=4168210) ) = self._apply_hf_processor_main( +(StageEngineCoreProc_stage1_replica0 pid=4168210) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py", line 3444, in _apply_hf_processor_main +(StageEngineCoreProc_stage1_replica0 pid=4168210) prompt_ids = self._apply_hf_processor_text_only(prompt, tokenization_kwargs or {}) +(StageEngineCoreProc_stage1_replica0 pid=4168210) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/processing/processor.py", line 1336, in _apply_hf_processor_text_only +(StageEngineCoreProc_stage1_replica0 pid=4168210) prompt_ids, _, _ = self._apply_hf_processor_text_mm( +(StageEngineCoreProc_stage1_replica0 pid=4168210) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/processing/processor.py", line 1301, in _apply_hf_processor_text_mm +(StageEngineCoreProc_stage1_replica0 pid=4168210) processed_data = self._call_hf_processor( +(StageEngineCoreProc_stage1_replica0 pid=4168210) ^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/processing/processor.py", line 1258, in _call_hf_processor +(StageEngineCoreProc_stage1_replica0 pid=4168210) return self.info.ctx.call_hf_processor( +(StageEngineCoreProc_stage1_replica0 pid=4168210) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/processing/context.py", line 293, in call_hf_processor +(StageEngineCoreProc_stage1_replica0 pid=4168210) raise ValueError(msg) from exc +(StageEngineCoreProc_stage1_replica0 pid=4168210) ValueError: Failed to apply MiniCPMOProcessor on data={'text': '()(./)()'} with kwargs={'truncation': False, 'return_tensors': 'pt'} +ERROR 09-23 15:42:14 [stage_runtime.py:265] [StageRuntime] Stage initialization failed; shutting down 1 initialized client(s) +ERROR 09-23 15:42:14 [stage_runtime.py:265] Traceback (most recent call last): +ERROR 09-23 15:42:14 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 252, in initialize +ERROR 09-23 15:42:14 [stage_runtime.py:265] initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout) +ERROR 09-23 15:42:14 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +ERROR 09-23 15:42:14 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 589, in _initialize_stage_replicas +ERROR 09-23 15:42:14 [stage_runtime.py:265] raise primary_exc +ERROR 09-23 15:42:14 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 550, in _init_group +ERROR 09-23 15:42:14 [stage_runtime.py:265] client = self._initialize_replica( +ERROR 09-23 15:42:14 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^ +ERROR 09-23 15:42:14 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 828, in _initialize_replica +ERROR 09-23 15:42:14 [stage_runtime.py:265] return self._initialize_local_llm_replica(plan, stage_init_timeout) +ERROR 09-23 15:42:14 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +ERROR 09-23 15:42:14 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 934, in _initialize_local_llm_replica +ERROR 09-23 15:42:14 [stage_runtime.py:265] launch_stack.close() +ERROR 09-23 15:42:14 [stage_runtime.py:265] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 618, in close +ERROR 09-23 15:42:14 [stage_runtime.py:265] self.__exit__(None, None, None) +ERROR 09-23 15:42:14 [stage_runtime.py:265] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 610, in __exit__ +ERROR 09-23 15:42:14 [stage_runtime.py:265] raise exc_details[1] +ERROR 09-23 15:42:14 [stage_runtime.py:265] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 595, in __exit__ +ERROR 09-23 15:42:14 [stage_runtime.py:265] if cb(*exc_details): +ERROR 09-23 15:42:14 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^ +ERROR 09-23 15:42:14 [stage_runtime.py:265] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 144, in __exit__ +ERROR 09-23 15:42:14 [stage_runtime.py:265] next(self.gen) +ERROR 09-23 15:42:14 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_startup.py", line 1211, in launch_stage_replica +ERROR 09-23 15:42:14 [stage_runtime.py:265] wait_for_engine_startup( +ERROR 09-23 15:42:14 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/utils.py", line 1286, in wait_for_engine_startup +ERROR 09-23 15:42:14 [stage_runtime.py:265] raise RuntimeError( +ERROR 09-23 15:42:14 [stage_runtime.py:265] RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {} +INFO 09-23 15:42:14 [core_client.py:689] [shutdown] MPClient: start timeout=default +INFO 09-23 15:42:14 [core_client.py:691] [shutdown] MPClient: stopping engine manager +INFO 09-23 15:42:14 [utils.py:615] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage0_replica0 +WARNING 09-23 15:42:19 [utils.py:635] [shutdown] Process manager: force killing remaining processes count=1 +WARNING 09-23 15:42:19 [utils.py:640] [shutdown] Process manager: force killing remaining process StageEngineCoreProc_stage0_replica0 pid 4168102 +INFO 09-23 15:42:19 [core_client.py:693] [shutdown] MPClient: engine manager stopped +INFO 09-23 15:42:19 [core_client.py:694] [shutdown] MPClient: cleaning up background resources +INFO 09-23 15:42:19 [core_client.py:696] [shutdown] MPClient: complete +ERROR 09-23 15:42:19 [async_omni_engine.py:474] [AsyncOmniEngine] Orchestrator thread crashed +ERROR 09-23 15:42:19 [async_omni_engine.py:474] Traceback (most recent call last): +ERROR 09-23 15:42:19 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 468, in _bootstrap_orchestrator +ERROR 09-23 15:42:19 [async_omni_engine.py:474] loop.run_until_complete(_run_orchestrator()) +ERROR 09-23 15:42:19 [async_omni_engine.py:474] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/base_events.py", line 691, in run_until_complete +ERROR 09-23 15:42:19 [async_omni_engine.py:474] return future.result() +ERROR 09-23 15:42:19 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^ +ERROR 09-23 15:42:19 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 424, in _run_orchestrator +ERROR 09-23 15:42:19 [async_omni_engine.py:474] self._initialize_stages(stage_init_timeout) +ERROR 09-23 15:42:19 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 350, in _initialize_stages +ERROR 09-23 15:42:19 [async_omni_engine.py:474] self._initialize_stages_impl(stage_init_timeout) +ERROR 09-23 15:42:19 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 371, in _initialize_stages_impl +ERROR 09-23 15:42:19 [async_omni_engine.py:474] self._runtime.initialize() +ERROR 09-23 15:42:19 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 278, in initialize +ERROR 09-23 15:42:19 [async_omni_engine.py:474] raise exc +ERROR 09-23 15:42:19 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 252, in initialize +ERROR 09-23 15:42:19 [async_omni_engine.py:474] initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout) +ERROR 09-23 15:42:19 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +ERROR 09-23 15:42:19 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 589, in _initialize_stage_replicas +ERROR 09-23 15:42:19 [async_omni_engine.py:474] raise primary_exc +ERROR 09-23 15:42:19 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 550, in _init_group +ERROR 09-23 15:42:19 [async_omni_engine.py:474] client = self._initialize_replica( +ERROR 09-23 15:42:19 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^^^^^^^^^^ +ERROR 09-23 15:42:19 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 828, in _initialize_replica +ERROR 09-23 15:42:19 [async_omni_engine.py:474] return self._initialize_local_llm_replica(plan, stage_init_timeout) +ERROR 09-23 15:42:19 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +ERROR 09-23 15:42:19 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 934, in _initialize_local_llm_replica +ERROR 09-23 15:42:19 [async_omni_engine.py:474] launch_stack.close() +ERROR 09-23 15:42:19 [async_omni_engine.py:474] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 618, in close +ERROR 09-23 15:42:19 [async_omni_engine.py:474] self.__exit__(None, None, None) +ERROR 09-23 15:42:19 [async_omni_engine.py:474] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 610, in __exit__ +ERROR 09-23 15:42:19 [async_omni_engine.py:474] raise exc_details[1] +ERROR 09-23 15:42:19 [async_omni_engine.py:474] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 595, in __exit__ +ERROR 09-23 15:42:19 [async_omni_engine.py:474] if cb(*exc_details): +ERROR 09-23 15:42:19 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^ +ERROR 09-23 15:42:19 [async_omni_engine.py:474] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 144, in __exit__ +ERROR 09-23 15:42:19 [async_omni_engine.py:474] next(self.gen) +ERROR 09-23 15:42:19 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_startup.py", line 1211, in launch_stage_replica +ERROR 09-23 15:42:19 [async_omni_engine.py:474] wait_for_engine_startup( +ERROR 09-23 15:42:19 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/utils.py", line 1286, in wait_for_engine_startup +ERROR 09-23 15:42:19 [async_omni_engine.py:474] raise RuntimeError( +ERROR 09-23 15:42:19 [async_omni_engine.py:474] RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {} +INFO 09-23 15:42:19 [async_omni_engine.py:1807] [AsyncOmniEngine] Shutting down Orchestrator +Exception in thread orchestrator: +Traceback (most recent call last): + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/threading.py", line 1075, in _bootstrap_inner + self.run() + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/threading.py", line 1012, in run + self._target(*self._args, **self._kwargs) + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 468, in _bootstrap_orchestrator + loop.run_until_complete(_run_orchestrator()) + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/base_events.py", line 691, in run_until_complete + return future.result() + ^^^^^^^^^^^^^^^ + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 424, in _run_orchestrator + self._initialize_stages(stage_init_timeout) + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 350, in _initialize_stages + self._initialize_stages_impl(stage_init_timeout) + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 371, in _initialize_stages_impl + self._runtime.initialize() + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 278, in initialize + raise exc + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 252, in initialize + initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 589, in _initialize_stage_replicas + raise primary_exc + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 550, in _init_group + client = self._initialize_replica( + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 828, in _initialize_replica + return self._initialize_local_llm_replica(plan, stage_init_timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 934, in _initialize_local_llm_replica + launch_stack.close() + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 618, in close + self.__exit__(None, None, None) + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 610, in __exit__ + raise exc_details[1] + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 595, in __exit__ + if cb(*exc_details): + ^^^^^^^^^^^^^^^^ + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 144, in __exit__ + next(self.gen) + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_startup.py", line 1211, in launch_stage_replica + wait_for_engine_startup( + File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/utils.py", line 1286, in wait_for_engine_startup + raise RuntimeError( +RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {} +Traceback (most recent call last): + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/threading.py", line 1075, in _bootstrap_inner + self.run() + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/threading.py", line 1012, in run + self._target(*self._args, **self._kwargs) + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 468, in _bootstrap_orchestrator + loop.run_until_complete(_run_orchestrator()) + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/base_events.py", line 691, in run_until_complete + return future.result() + ^^^^^^^^^^^^^^^ + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 424, in _run_orchestrator + self._initialize_stages(stage_init_timeout) + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 350, in _initialize_stages + self._initialize_stages_impl(stage_init_timeout) + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 371, in _initialize_stages_impl + self._runtime.initialize() + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 278, in initialize + raise exc + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 252, in initialize + initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 589, in _initialize_stage_replicas + raise primary_exc + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 550, in _init_group + client = self._initialize_replica( + ^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 828, in _initialize_replica + return self._initialize_local_llm_replica(plan, stage_init_timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 934, in _initialize_local_llm_replica + launch_stack.close() + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 618, in close + self.__exit__(None, None, None) + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 610, in __exit__ + raise exc_details[1] + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 595, in __exit__ + if cb(*exc_details): + ^^^^^^^^^^^^^^^^ + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 144, in __exit__ + next(self.gen) + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_startup.py", line 1211, in launch_stage_replica + wait_for_engine_startup( + File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/utils.py", line 1286, in wait_for_engine_startup + raise RuntimeError( +RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {} + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "/home/zhout/project/edge_infer/vllm-omni-edge/examples/offline_inference/minicpmo/end2end.py", line 519, in + main(args) + File "/home/zhout/project/edge_infer/vllm-omni-edge/examples/offline_inference/minicpmo/end2end.py", line 283, in main + omni = Omni(**omni_kwargs) + ^^^^^^^^^^^^^^^^^^^ + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/entrypoints/omni_base.py", line 223, in __init__ + self.engine = AsyncOmniEngine( + ^^^^^^^^^^^^^^^^ + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 305, in __init__ + self._wait_for_orchestrator_init(startup_future, startup_timeout) + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 539, in _wait_for_orchestrator_init + startup_future.result( + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/concurrent/futures/_base.py", line 456, in result + return self.__get_result() + ^^^^^^^^^^^^^^^^^^^ + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/concurrent/futures/_base.py", line 401, in __get_result + raise self._exception +RuntimeError: Orchestrator initialization failed: Engine core initialization failed. See root cause above. Failed core proc(s): {} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/minicpmo/minicpmo_cpu_full_compat3.log b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/minicpmo/minicpmo_cpu_full_compat3.log new file mode 100644 index 000000000..a11c09236 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/minicpmo/minicpmo_cpu_full_compat3.log @@ -0,0 +1,587 @@ +W0923 15:43:20.838000 4168432 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0923 15:43:20.864000 4168432 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +WARNING 09-23 15:43:21 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton. +INFO 09-23 15:43:21 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available. +/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions: + --> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922 + --> vLLM version 0.28.0 +This will likely cause compatibility issues. + warn_if_misaligned_vllm_version() +INFO 09-23 15:43:22 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-23 15:43:22 [patch.py:510] inductor factorable-divisibility patch: installed. +INFO 09-23 15:43:22 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled. +INFO 09-23 15:43:23 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable). +==================== + vllm version: 0.28.0 + ==================== +INFO 09-23 15:43:23 [omni_base.py:216] [Omni] Initializing with model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5 +INFO 09-23 15:43:23 [async_omni_engine.py:172] [AsyncOmniEngine] Initializing with model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5 +INFO 09-23 15:43:23 [async_omni_engine.py:290] [AsyncOmniEngine] Launching Orchestrator thread with 3 stages +INFO 09-23 15:43:23 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn +INFO 09-23 15:43:23 [initialization.py:356] Loaded OmniTransferConfig with 1 connector configurations +WARNING 09-23 15:43:23 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass) +WARNING 09-23 15:43:23 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass) +WARNING 09-23 15:43:23 [arg_utils.py:1719] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM. +INFO 09-23 15:43:23 [model.py:672] Resolved architecture: MiniCPMO45OmniForConditionalGeneration +INFO 09-23 15:43:23 [model.py:1965] Using max model len 2048 +WARNING 09-23 15:43:23 [model.py:963] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False. +INFO 09-23 15:43:24 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode. +INFO 09-23 15:43:24 [model.py:875] Disabled mm_prefix attention mode because multimodal inputs are configuration-disabled. Attention backends without mm_prefix support may now be selected. +INFO 09-23 15:43:24 [scheduler.py:242] Chunked prefill is enabled with max_num_batched_tokens=2048. +WARNING 09-23 15:43:24 [vllm.py:1285] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none +WARNING 09-23 15:43:24 [vllm.py:1354] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored. +INFO 09-23 15:43:24 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +WARNING 09-23 15:43:24 [vllm.py:652] Model Runner V2 requires Triton; using the V1 model runner instead. +INFO 09-23 15:43:24 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant +WARNING 09-23 15:43:24 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass) +WARNING 09-23 15:43:24 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass) +WARNING 09-23 15:43:24 [arg_utils.py:1719] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM. +INFO 09-23 15:43:24 [model.py:672] Resolved architecture: MiniCPMO45OmniForConditionalGeneration +INFO 09-23 15:43:24 [model.py:1965] Using max model len 512 +WARNING 09-23 15:43:24 [model.py:963] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False. +INFO 09-23 15:43:24 [model.py:1965] Using max model len 512 +INFO 09-23 15:43:24 [scheduler.py:242] Chunked prefill is enabled with max_num_batched_tokens=512. +INFO 09-23 15:43:24 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +WARNING 09-23 15:43:24 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass) +WARNING 09-23 15:43:24 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass) +WARNING 09-23 15:43:24 [arg_utils.py:1719] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM. +INFO 09-23 15:43:24 [model.py:672] Resolved architecture: MiniCPMO45Code2Wav +INFO 09-23 15:43:24 [model.py:1965] Using max model len 4096 +WARNING 09-23 15:43:24 [arg_utils.py:2664] This model does not officially support disabling chunked prefill. Disabling this manually may cause the engine to crash or produce incorrect outputs. +INFO 09-23 15:43:24 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +INFO 09-23 15:43:24 [stage_runtime.py:853] [stage_init] Stage-0 set runtime devices: 0 +W0923 15:43:28.412000 4168462 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0923 15:43:28.440000 4168462 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +WARNING 09-23 15:43:29 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton. +INFO 09-23 15:43:29 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available. +/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions: + --> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922 + --> vLLM version 0.28.0 +This will likely cause compatibility issues. + warn_if_misaligned_vllm_version() +INFO 09-23 15:43:30 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-23 15:43:30 [patch.py:510] inductor factorable-divisibility patch: installed. +INFO 09-23 15:43:30 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled. +INFO 09-23 15:43:30 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable). +(StageEngineCoreProc_stage0_replica0 pid=4168462) INFO 09-23 15:43:31 [core.py:122] Initializing a V1 LLM engine (v0.28.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=2048, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=True, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cpu, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': None, 'compile_ranges_endpoints': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': None, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto') +(StageEngineCoreProc_stage0_replica0 pid=4168462) INFO 09-23 15:43:32 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode. +[W923 15:43:32.228349802 utils.cpp:68] Warning: NUMA binding: Using MEMBIND policy for memory allocation on the NUMA nodes (0). Memory allocations will be strictly bound to these NUMA nodes. (function init_cpu_memory_env) +(StageEngineCoreProc_stage0_replica0 pid=4168462) WARNING 09-23 15:43:32 [vllm.py:652] Model Runner V2 requires Triton; using the V1 model runner instead. +(StageEngineCoreProc_stage0_replica0 pid=4168462) WARNING 09-23 15:43:32 [cpu_worker.py:127] libiomp is not found in LD_PRELOAD. For best performance, please follow the section `set LD_PRELOAD` in https://docs.vllm.ai/en/latest/getting_started/installation/cpu/ to setup required pre-loaded libraries. +(StageEngineCoreProc_stage0_replica0 pid=4168462) INFO 09-23 15:43:32 [parallel_state.py:1638] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_f8d63e300ac346b79645aff6c0c6e5a8 backend=gloo +(StageEngineCoreProc_stage0_replica0 pid=4168462) INFO 09-23 15:43:32 [parallel_state.py:1982] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A +(StageEngineCoreProc_stage0_replica0 pid=4168462) INFO 09-23 15:43:32 [cpu_model_runner.py:131] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5... +(StageEngineCoreProc_stage0_replica0 pid=4168462) WARNING 09-23 15:43:32 [vllm.py:1285] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none +(StageEngineCoreProc_stage0_replica0 pid=4168462) WARNING 09-23 15:43:32 [vllm.py:1354] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored. +(StageEngineCoreProc_stage0_replica0 pid=4168462) INFO 09-23 15:43:32 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +(StageEngineCoreProc_stage0_replica0 pid=4168462) INFO 09-23 15:43:32 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant +(StageEngineCoreProc_stage0_replica0 pid=4168462) INFO 09-23 15:43:32 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +(StageEngineCoreProc_stage0_replica0 pid=4168462) INFO 09-23 15:43:32 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +(StageEngineCoreProc_stage0_replica0 pid=4168462) INFO 09-23 15:43:32 [selector.py:217] Using HND KV cache layout for CPU_ATTN backend. +(StageEngineCoreProc_stage0_replica0 pid=4168462) INFO 09-23 15:43:35 [weight_utils.py:858] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 27.11 GiB. +(StageEngineCoreProc_stage0_replica0 pid=4168462) INFO 09-23 15:43:35 [weight_utils.py:881] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch. +(StageEngineCoreProc_stage0_replica0 pid=4168462) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00 18645 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory'] +(StageEngineCoreProc_stage0_replica0 pid=4168462) INFO 09-23 15:43:57 [cpu_worker.py:255] Explicitly set (1.0/30.91) GiB for KV cache on node 0. +(StageEngineCoreProc_stage0_replica0 pid=4168462) INFO 09-23 15:43:57 [kv_cache_utils.py:1869] GPU KV cache size: 7,168 tokens, Maximum concurrency for 2,048 tokens per request: 3.50x +(StageEngineCoreProc_stage0_replica0 pid=4168462) INFO 09-23 15:43:57 [memory.py:91] Released 0 MiB after warm-up (anonymous RSS 19662 -> 19662 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory'] +(StageEngineCoreProc_stage0_replica0 pid=4168462) INFO 09-23 15:43:57 [core.py:354] init engine (profile, create kv cache, warmup model) took 0.47 s +(StageEngineCoreProc_stage0_replica0 pid=4168462) WARNING 09-23 15:43:57 [scheduler.py:182] Using custom scheduler class vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled. +(StageEngineCoreProc_stage0_replica0 pid=4168462) INFO 09-23 15:43:58 [factory.py:46] Created connector: SharedMemoryConnector +(StageEngineCoreProc_stage0_replica0 pid=4168462) INFO 09-23 15:44:08 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +INFO 09-23 15:44:08 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed +INFO 09-23 15:44:08 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-0 [rep-0] initializing EngineCore +INFO 09-23 15:44:08 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-0 [rep-0] EngineCore running +INFO 09-23 15:44:08 [stage_runtime.py:958] [StageRuntime] Stage 0 initialized +INFO 09-23 15:44:08 [stage_runtime.py:853] [stage_init] Stage-1 set runtime devices: 0 +W0923 15:44:14.751000 4168582 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0923 15:44:14.792000 4168582 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +WARNING 09-23 15:44:16 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton. +INFO 09-23 15:44:16 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available. +/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions: + --> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922 + --> vLLM version 0.28.0 +This will likely cause compatibility issues. + warn_if_misaligned_vllm_version() +INFO 09-23 15:44:18 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-23 15:44:18 [patch.py:510] inductor factorable-divisibility patch: installed. +INFO 09-23 15:44:18 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled. +INFO 09-23 15:44:18 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable). +(StageEngineCoreProc_stage1_replica0 pid=4168582) INFO 09-23 15:44:19 [core.py:122] Initializing a V1 LLM engine (v0.28.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=512, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=True, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cpu, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': None, 'compile_ranges_endpoints': [512], 'inductor_compile_config': {'enable_auto_functionalized_v2': False}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': None, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto') +[W923 15:44:20.299950745 utils.cpp:68] Warning: NUMA binding: Using MEMBIND policy for memory allocation on the NUMA nodes (0). Memory allocations will be strictly bound to these NUMA nodes. (function init_cpu_memory_env) +(StageEngineCoreProc_stage1_replica0 pid=4168582) WARNING 09-23 15:44:20 [vllm.py:652] Model Runner V2 requires Triton; using the V1 model runner instead. +(StageEngineCoreProc_stage1_replica0 pid=4168582) WARNING 09-23 15:44:20 [cpu_worker.py:127] libiomp is not found in LD_PRELOAD. For best performance, please follow the section `set LD_PRELOAD` in https://docs.vllm.ai/en/latest/getting_started/installation/cpu/ to setup required pre-loaded libraries. +(StageEngineCoreProc_stage1_replica0 pid=4168582) INFO 09-23 15:44:20 [parallel_state.py:1638] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_c745d19853d047d0b9c64ec8c42c5ccc backend=gloo +(StageEngineCoreProc_stage1_replica0 pid=4168582) INFO 09-23 15:44:20 [parallel_state.py:1982] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A +(StageEngineCoreProc_stage1_replica0 pid=4168582) WARNING 09-23 15:44:20 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it. +(StageEngineCoreProc_stage1_replica0 pid=4168582) [transformers] `MiniCPMOProcessor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +(StageEngineCoreProc_stage1_replica0 pid=4168582) [transformers] Requested torchvision backend is not available. Falling back to pil backend. +(StageEngineCoreProc_stage1_replica0 pid=4168582) [transformers] `MiniCPMOProcessor` defines `feature_extractor_class = 'WhisperFeatureExtractor'`, which is deprecated. Register the correct mapping in `AutoFeatureExtractor` instead. +(StageEngineCoreProc_stage1_replica0 pid=4168582) [transformers] Image processor MiniCPMVImageProcessor: kwargs ['max_slice_nums'] were applied for backward compatibility. To avoid this warning, add them to valid_kwargs: create a custom TypedDict extending ImagesKwargs with these keys and set it as the `valid_kwargs` class attribute. +(StageEngineCoreProc_stage1_replica0 pid=4168582) WARNING 09-23 15:44:28 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it. +(StageEngineCoreProc_stage1_replica0 pid=4168582) WARNING 09-23 15:44:28 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it. +(StageEngineCoreProc_stage1_replica0 pid=4168582) WARNING 09-23 15:44:28 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it. +(StageEngineCoreProc_stage1_replica0 pid=4168582) INFO 09-23 15:44:29 [factory.py:46] Created connector: SharedMemoryConnector +(StageEngineCoreProc_stage1_replica0 pid=4168582) INFO 09-23 15:44:29 [cpu_model_runner.py:131] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5... +(StageEngineCoreProc_stage1_replica0 pid=4168582) WARNING 09-23 15:44:29 [vllm.py:1285] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none +(StageEngineCoreProc_stage1_replica0 pid=4168582) WARNING 09-23 15:44:29 [vllm.py:1354] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored. +(StageEngineCoreProc_stage1_replica0 pid=4168582) INFO 09-23 15:44:29 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +(StageEngineCoreProc_stage1_replica0 pid=4168582) INFO 09-23 15:44:29 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant +(StageEngineCoreProc_stage1_replica0 pid=4168582) INFO 09-23 15:44:29 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +(StageEngineCoreProc_stage1_replica0 pid=4168582) INFO 09-23 15:44:29 [selector.py:217] Using HND KV cache layout for CPU_ATTN backend. +(StageEngineCoreProc_stage1_replica0 pid=4168582) INFO 09-23 15:44:29 [weight_utils.py:858] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 8.79 GiB. +(StageEngineCoreProc_stage1_replica0 pid=4168582) INFO 09-23 15:44:29 [weight_utils.py:888] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre) and the checkpoint size (17.46 GiB) exceeds 90% of available RAM (8.79 GiB). +(StageEngineCoreProc_stage1_replica0 pid=4168582) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00 1447 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory'] +(StageEngineCoreProc_stage1_replica0 pid=4168582) INFO 09-23 15:44:32 [cpu_worker.py:255] Explicitly set (1.0/30.91) GiB for KV cache on node 0. +(StageEngineCoreProc_stage1_replica0 pid=4168582) INFO 09-23 15:44:32 [kv_cache_utils.py:1869] GPU KV cache size: 17,408 tokens, Maximum concurrency for 512 tokens per request: 34.00x +(StageEngineCoreProc_stage1_replica0 pid=4168582) INFO 09-23 15:44:33 [memory.py:91] Released -0 MiB after warm-up (anonymous RSS 2469 -> 2469 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory'] +(StageEngineCoreProc_stage1_replica0 pid=4168582) INFO 09-23 15:44:33 [core.py:354] init engine (profile, create kv cache, warmup model) took 0.31 s +(StageEngineCoreProc_stage1_replica0 pid=4168582) WARNING 09-23 15:44:33 [scheduler.py:182] Using custom scheduler class vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled. +(StageEngineCoreProc_stage1_replica0 pid=4168582) WARNING 09-23 15:44:33 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it. +(StageEngineCoreProc_stage1_replica0 pid=4168582) WARNING 09-23 15:44:33 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it. +(StageEngineCoreProc_stage1_replica0 pid=4168582) INFO 09-23 15:44:33 [factory.py:46] Created connector: SharedMemoryConnector +(StageEngineCoreProc_stage1_replica0 pid=4168582) INFO 09-23 15:44:33 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +INFO 09-23 15:44:33 [stage_runtime.py:943] [StageRuntime] Stage 1 engine startup completed +INFO 09-23 15:44:33 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-1 [rep-0] initializing EngineCore +INFO 09-23 15:44:33 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-1 [rep-0] EngineCore running +INFO 09-23 15:44:33 [stage_runtime.py:958] [StageRuntime] Stage 1 initialized +INFO 09-23 15:44:33 [stage_runtime.py:853] [stage_init] Stage-2 set runtime devices: 0 +W0923 15:44:39.263000 4168702 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0923 15:44:39.290000 4168702 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +WARNING 09-23 15:44:40 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton. +INFO 09-23 15:44:40 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available. +/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions: + --> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922 + --> vLLM version 0.28.0 +This will likely cause compatibility issues. + warn_if_misaligned_vllm_version() +INFO 09-23 15:44:41 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-23 15:44:42 [patch.py:510] inductor factorable-divisibility patch: installed. +INFO 09-23 15:44:42 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled. +INFO 09-23 15:44:42 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable). +(StageEngineCoreProc_stage2_replica0 pid=4168702) INFO 09-23 15:44:42 [core.py:122] Initializing a V1 LLM engine (v0.28.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=4096, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=True, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cpu, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=False, pooler_config=None, compilation_config={'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': None, 'compile_ranges_endpoints': [4096], 'inductor_compile_config': {'enable_auto_functionalized_v2': False}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': None, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto') +[W923 15:44:42.892659826 utils.cpp:68] Warning: NUMA binding: Using MEMBIND policy for memory allocation on the NUMA nodes (0). Memory allocations will be strictly bound to these NUMA nodes. (function init_cpu_memory_env) +(StageEngineCoreProc_stage2_replica0 pid=4168702) WARNING 09-23 15:44:42 [vllm.py:652] Model Runner V2 requires Triton; using the V1 model runner instead. +(StageEngineCoreProc_stage2_replica0 pid=4168702) WARNING 09-23 15:44:42 [cpu_worker.py:127] libiomp is not found in LD_PRELOAD. For best performance, please follow the section `set LD_PRELOAD` in https://docs.vllm.ai/en/latest/getting_started/installation/cpu/ to setup required pre-loaded libraries. +(StageEngineCoreProc_stage2_replica0 pid=4168702) INFO 09-23 15:44:42 [parallel_state.py:1638] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_8bf2c7708ac8464e8a951a827bd46d53 backend=gloo +(StageEngineCoreProc_stage2_replica0 pid=4168702) INFO 09-23 15:44:42 [parallel_state.py:1982] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A +(StageEngineCoreProc_stage2_replica0 pid=4168702) INFO 09-23 15:44:42 [factory.py:46] Created connector: SharedMemoryConnector +(StageEngineCoreProc_stage2_replica0 pid=4168702) INFO 09-23 15:44:42 [cpu_model_runner.py:131] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5... +(StageEngineCoreProc_stage2_replica0 pid=4168702) INFO 09-23 15:44:42 [weight_utils.py:858] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 6.82 GiB. +(StageEngineCoreProc_stage2_replica0 pid=4168702) INFO 09-23 15:44:42 [weight_utils.py:888] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre) and the checkpoint size (17.46 GiB) exceeds 90% of available RAM (6.82 GiB). +(StageEngineCoreProc_stage2_replica0 pid=4168702) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00 + main(args) + File "/home/zhout/project/edge_infer/vllm-omni-edge/examples/offline_inference/minicpmo/end2end.py", line 283, in main + omni = Omni(**omni_kwargs) + ^^^^^^^^^^^^^^^^^^^ + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/entrypoints/omni_base.py", line 223, in __init__ + self.engine = AsyncOmniEngine( + ^^^^^^^^^^^^^^^^ + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 305, in __init__ + self._wait_for_orchestrator_init(startup_future, startup_timeout) + File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 539, in _wait_for_orchestrator_init + startup_future.result( + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/concurrent/futures/_base.py", line 456, in result + return self.__get_result() + ^^^^^^^^^^^^^^^^^^^ + File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/concurrent/futures/_base.py", line 401, in __get_result + raise self._exception +RuntimeError: Orchestrator initialization failed: Engine core initialization failed. See root cause above. Failed core proc(s): {} diff --git a/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/minicpmo/minicpmo_cpu_full_compat4.log b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/minicpmo/minicpmo_cpu_full_compat4.log new file mode 100644 index 000000000..d68e5ee05 --- /dev/null +++ b/benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/minicpmo/minicpmo_cpu_full_compat4.log @@ -0,0 +1,234 @@ +W0923 15:45:57.706000 4168920 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0923 15:45:57.732000 4168920 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +WARNING 09-23 15:45:59 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton. +INFO 09-23 15:45:59 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available. +/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions: + --> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922 + --> vLLM version 0.28.0 +This will likely cause compatibility issues. + warn_if_misaligned_vllm_version() +INFO 09-23 15:46:00 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-23 15:46:00 [patch.py:510] inductor factorable-divisibility patch: installed. +INFO 09-23 15:46:00 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled. +INFO 09-23 15:46:00 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable). +==================== + vllm version: 0.28.0 + ==================== +INFO 09-23 15:46:00 [omni_base.py:216] [Omni] Initializing with model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5 +INFO 09-23 15:46:00 [async_omni_engine.py:172] [AsyncOmniEngine] Initializing with model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5 +INFO 09-23 15:46:01 [async_omni_engine.py:290] [AsyncOmniEngine] Launching Orchestrator thread with 3 stages +INFO 09-23 15:46:01 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn +INFO 09-23 15:46:01 [initialization.py:356] Loaded OmniTransferConfig with 1 connector configurations +WARNING 09-23 15:46:01 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass) +WARNING 09-23 15:46:01 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass) +WARNING 09-23 15:46:01 [arg_utils.py:1719] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM. +INFO 09-23 15:46:01 [model.py:672] Resolved architecture: MiniCPMO45OmniForConditionalGeneration +INFO 09-23 15:46:01 [model.py:1965] Using max model len 2048 +WARNING 09-23 15:46:01 [model.py:963] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False. +INFO 09-23 15:46:01 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode. +INFO 09-23 15:46:01 [model.py:875] Disabled mm_prefix attention mode because multimodal inputs are configuration-disabled. Attention backends without mm_prefix support may now be selected. +INFO 09-23 15:46:01 [scheduler.py:242] Chunked prefill is enabled with max_num_batched_tokens=2048. +WARNING 09-23 15:46:01 [vllm.py:1285] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none +WARNING 09-23 15:46:01 [vllm.py:1354] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored. +INFO 09-23 15:46:01 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +WARNING 09-23 15:46:01 [vllm.py:652] Model Runner V2 requires Triton; using the V1 model runner instead. +INFO 09-23 15:46:01 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant +WARNING 09-23 15:46:01 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass) +WARNING 09-23 15:46:01 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass) +WARNING 09-23 15:46:01 [arg_utils.py:1719] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM. +INFO 09-23 15:46:01 [model.py:672] Resolved architecture: MiniCPMO45OmniForConditionalGeneration +INFO 09-23 15:46:01 [model.py:1965] Using max model len 512 +WARNING 09-23 15:46:01 [model.py:963] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False. +INFO 09-23 15:46:01 [model.py:1965] Using max model len 512 +INFO 09-23 15:46:01 [scheduler.py:242] Chunked prefill is enabled with max_num_batched_tokens=512. +INFO 09-23 15:46:01 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +WARNING 09-23 15:46:01 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass) +WARNING 09-23 15:46:01 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass) +WARNING 09-23 15:46:01 [arg_utils.py:1719] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM. +INFO 09-23 15:46:01 [model.py:672] Resolved architecture: MiniCPMO45Code2Wav +INFO 09-23 15:46:01 [model.py:1965] Using max model len 4096 +WARNING 09-23 15:46:01 [arg_utils.py:2664] This model does not officially support disabling chunked prefill. Disabling this manually may cause the engine to crash or produce incorrect outputs. +INFO 09-23 15:46:01 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +INFO 09-23 15:46:01 [stage_runtime.py:853] [stage_init] Stage-0 set runtime devices: 0 +W0923 15:46:05.282000 4168948 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0923 15:46:05.308000 4168948 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +WARNING 09-23 15:46:06 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton. +INFO 09-23 15:46:06 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available. +/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions: + --> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922 + --> vLLM version 0.28.0 +This will likely cause compatibility issues. + warn_if_misaligned_vllm_version() +INFO 09-23 15:46:07 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-23 15:46:07 [patch.py:510] inductor factorable-divisibility patch: installed. +INFO 09-23 15:46:07 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled. +INFO 09-23 15:46:07 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable). +(StageEngineCoreProc_stage0_replica0 pid=4168948) INFO 09-23 15:46:07 [core.py:122] Initializing a V1 LLM engine (v0.28.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=2048, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=True, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cpu, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': None, 'compile_ranges_endpoints': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': None, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto') +(StageEngineCoreProc_stage0_replica0 pid=4168948) INFO 09-23 15:46:08 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode. +[W923 15:46:08.986759514 utils.cpp:68] Warning: NUMA binding: Using MEMBIND policy for memory allocation on the NUMA nodes (0). Memory allocations will be strictly bound to these NUMA nodes. (function init_cpu_memory_env) +(StageEngineCoreProc_stage0_replica0 pid=4168948) WARNING 09-23 15:46:08 [vllm.py:652] Model Runner V2 requires Triton; using the V1 model runner instead. +(StageEngineCoreProc_stage0_replica0 pid=4168948) WARNING 09-23 15:46:08 [cpu_worker.py:127] libiomp is not found in LD_PRELOAD. For best performance, please follow the section `set LD_PRELOAD` in https://docs.vllm.ai/en/latest/getting_started/installation/cpu/ to setup required pre-loaded libraries. +(StageEngineCoreProc_stage0_replica0 pid=4168948) INFO 09-23 15:46:08 [parallel_state.py:1638] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_a3ba980eb94d4dfcb8fd518afa3242cb backend=gloo +(StageEngineCoreProc_stage0_replica0 pid=4168948) INFO 09-23 15:46:08 [parallel_state.py:1982] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A +(StageEngineCoreProc_stage0_replica0 pid=4168948) INFO 09-23 15:46:08 [cpu_model_runner.py:131] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5... +(StageEngineCoreProc_stage0_replica0 pid=4168948) WARNING 09-23 15:46:08 [vllm.py:1285] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none +(StageEngineCoreProc_stage0_replica0 pid=4168948) WARNING 09-23 15:46:08 [vllm.py:1354] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored. +(StageEngineCoreProc_stage0_replica0 pid=4168948) INFO 09-23 15:46:08 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +(StageEngineCoreProc_stage0_replica0 pid=4168948) INFO 09-23 15:46:08 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant +(StageEngineCoreProc_stage0_replica0 pid=4168948) INFO 09-23 15:46:08 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +(StageEngineCoreProc_stage0_replica0 pid=4168948) INFO 09-23 15:46:08 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +(StageEngineCoreProc_stage0_replica0 pid=4168948) INFO 09-23 15:46:08 [selector.py:217] Using HND KV cache layout for CPU_ATTN backend. +(StageEngineCoreProc_stage0_replica0 pid=4168948) INFO 09-23 15:46:12 [weight_utils.py:858] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 27.06 GiB. +(StageEngineCoreProc_stage0_replica0 pid=4168948) INFO 09-23 15:46:12 [weight_utils.py:881] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch. +(StageEngineCoreProc_stage0_replica0 pid=4168948) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00 19304 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory'] +(StageEngineCoreProc_stage0_replica0 pid=4168948) INFO 09-23 15:46:23 [cpu_worker.py:255] Explicitly set (1.0/30.91) GiB for KV cache on node 0. +(StageEngineCoreProc_stage0_replica0 pid=4168948) INFO 09-23 15:46:23 [kv_cache_utils.py:1869] GPU KV cache size: 7,168 tokens, Maximum concurrency for 2,048 tokens per request: 3.50x +(StageEngineCoreProc_stage0_replica0 pid=4168948) INFO 09-23 15:46:23 [memory.py:91] Released 0 MiB after warm-up (anonymous RSS 20314 -> 20314 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory'] +(StageEngineCoreProc_stage0_replica0 pid=4168948) INFO 09-23 15:46:23 [core.py:354] init engine (profile, create kv cache, warmup model) took 0.52 s +(StageEngineCoreProc_stage0_replica0 pid=4168948) WARNING 09-23 15:46:23 [scheduler.py:182] Using custom scheduler class vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled. +(StageEngineCoreProc_stage0_replica0 pid=4168948) INFO 09-23 15:46:23 [factory.py:46] Created connector: SharedMemoryConnector +(StageEngineCoreProc_stage0_replica0 pid=4168948) INFO 09-23 15:46:24 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +INFO 09-23 15:46:24 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed +INFO 09-23 15:46:24 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-0 [rep-0] initializing EngineCore +INFO 09-23 15:46:24 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-0 [rep-0] EngineCore running +INFO 09-23 15:46:24 [stage_runtime.py:958] [StageRuntime] Stage 0 initialized +INFO 09-23 15:46:24 [stage_runtime.py:853] [stage_init] Stage-1 set runtime devices: 0 +W0923 15:46:31.117000 4169043 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0923 15:46:31.158000 4169043 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +WARNING 09-23 15:46:32 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton. +INFO 09-23 15:46:32 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available. +/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions: + --> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922 + --> vLLM version 0.28.0 +This will likely cause compatibility issues. + warn_if_misaligned_vllm_version() +INFO 09-23 15:46:34 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-23 15:46:34 [patch.py:510] inductor factorable-divisibility patch: installed. +INFO 09-23 15:46:34 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled. +INFO 09-23 15:46:34 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable). +(StageEngineCoreProc_stage1_replica0 pid=4169043) INFO 09-23 15:46:35 [core.py:122] Initializing a V1 LLM engine (v0.28.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=512, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=True, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cpu, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': None, 'compile_ranges_endpoints': [512], 'inductor_compile_config': {'enable_auto_functionalized_v2': False}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': None, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto') +[W923 15:46:36.481340156 utils.cpp:68] Warning: NUMA binding: Using MEMBIND policy for memory allocation on the NUMA nodes (0). Memory allocations will be strictly bound to these NUMA nodes. (function init_cpu_memory_env) +(StageEngineCoreProc_stage1_replica0 pid=4169043) WARNING 09-23 15:46:36 [vllm.py:652] Model Runner V2 requires Triton; using the V1 model runner instead. +(StageEngineCoreProc_stage1_replica0 pid=4169043) WARNING 09-23 15:46:36 [cpu_worker.py:127] libiomp is not found in LD_PRELOAD. For best performance, please follow the section `set LD_PRELOAD` in https://docs.vllm.ai/en/latest/getting_started/installation/cpu/ to setup required pre-loaded libraries. +(StageEngineCoreProc_stage1_replica0 pid=4169043) INFO 09-23 15:46:36 [parallel_state.py:1638] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_1e217dea20904802b2167eaa6f6f2a31 backend=gloo +(StageEngineCoreProc_stage1_replica0 pid=4169043) INFO 09-23 15:46:36 [parallel_state.py:1982] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A +(StageEngineCoreProc_stage1_replica0 pid=4169043) WARNING 09-23 15:46:36 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it. +(StageEngineCoreProc_stage1_replica0 pid=4169043) [transformers] `MiniCPMOProcessor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +(StageEngineCoreProc_stage1_replica0 pid=4169043) [transformers] Requested torchvision backend is not available. Falling back to pil backend. +(StageEngineCoreProc_stage1_replica0 pid=4169043) [transformers] `MiniCPMOProcessor` defines `feature_extractor_class = 'WhisperFeatureExtractor'`, which is deprecated. Register the correct mapping in `AutoFeatureExtractor` instead. +(StageEngineCoreProc_stage1_replica0 pid=4169043) [transformers] Image processor MiniCPMVImageProcessor: kwargs ['max_slice_nums'] were applied for backward compatibility. To avoid this warning, add them to valid_kwargs: create a custom TypedDict extending ImagesKwargs with these keys and set it as the `valid_kwargs` class attribute. +(StageEngineCoreProc_stage1_replica0 pid=4169043) WARNING 09-23 15:46:45 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it. +(StageEngineCoreProc_stage1_replica0 pid=4169043) WARNING 09-23 15:46:45 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it. +(StageEngineCoreProc_stage1_replica0 pid=4169043) WARNING 09-23 15:46:45 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it. +(StageEngineCoreProc_stage1_replica0 pid=4169043) INFO 09-23 15:46:45 [factory.py:46] Created connector: SharedMemoryConnector +(StageEngineCoreProc_stage1_replica0 pid=4169043) INFO 09-23 15:46:45 [cpu_model_runner.py:131] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5... +(StageEngineCoreProc_stage1_replica0 pid=4169043) WARNING 09-23 15:46:45 [vllm.py:1285] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none +(StageEngineCoreProc_stage1_replica0 pid=4169043) WARNING 09-23 15:46:45 [vllm.py:1354] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored. +(StageEngineCoreProc_stage1_replica0 pid=4169043) INFO 09-23 15:46:45 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +(StageEngineCoreProc_stage1_replica0 pid=4169043) INFO 09-23 15:46:45 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant +(StageEngineCoreProc_stage1_replica0 pid=4169043) INFO 09-23 15:46:45 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +(StageEngineCoreProc_stage1_replica0 pid=4169043) INFO 09-23 15:46:45 [selector.py:217] Using HND KV cache layout for CPU_ATTN backend. +(StageEngineCoreProc_stage1_replica0 pid=4169043) INFO 09-23 15:46:45 [weight_utils.py:858] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 8.10 GiB. +(StageEngineCoreProc_stage1_replica0 pid=4169043) INFO 09-23 15:46:45 [weight_utils.py:888] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre) and the checkpoint size (17.46 GiB) exceeds 90% of available RAM (8.10 GiB). +(StageEngineCoreProc_stage1_replica0 pid=4169043) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00 1462 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory'] +(StageEngineCoreProc_stage1_replica0 pid=4169043) INFO 09-23 15:46:49 [cpu_worker.py:255] Explicitly set (1.0/30.91) GiB for KV cache on node 0. +(StageEngineCoreProc_stage1_replica0 pid=4169043) INFO 09-23 15:46:49 [kv_cache_utils.py:1869] GPU KV cache size: 17,408 tokens, Maximum concurrency for 512 tokens per request: 34.00x +(StageEngineCoreProc_stage1_replica0 pid=4169043) INFO 09-23 15:46:49 [memory.py:91] Released 0 MiB after warm-up (anonymous RSS 2484 -> 2484 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory'] +(StageEngineCoreProc_stage1_replica0 pid=4169043) INFO 09-23 15:46:49 [core.py:354] init engine (profile, create kv cache, warmup model) took 0.16 s +(StageEngineCoreProc_stage1_replica0 pid=4169043) WARNING 09-23 15:46:49 [scheduler.py:182] Using custom scheduler class vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled. +(StageEngineCoreProc_stage1_replica0 pid=4169043) WARNING 09-23 15:46:49 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it. +(StageEngineCoreProc_stage1_replica0 pid=4169043) WARNING 09-23 15:46:50 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it. +(StageEngineCoreProc_stage1_replica0 pid=4169043) INFO 09-23 15:46:50 [factory.py:46] Created connector: SharedMemoryConnector +(StageEngineCoreProc_stage1_replica0 pid=4169043) INFO 09-23 15:46:50 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +INFO 09-23 15:46:50 [stage_runtime.py:943] [StageRuntime] Stage 1 engine startup completed +INFO 09-23 15:46:50 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-1 [rep-0] initializing EngineCore +INFO 09-23 15:46:50 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-1 [rep-0] EngineCore running +INFO 09-23 15:46:50 [stage_runtime.py:958] [StageRuntime] Stage 1 initialized +INFO 09-23 15:46:50 [stage_runtime.py:853] [stage_init] Stage-2 set runtime devices: 0 +W0923 15:46:55.369000 4169162 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0923 15:46:55.399000 4169162 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +WARNING 09-23 15:46:56 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton. +INFO 09-23 15:46:56 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available. +/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions: + --> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922 + --> vLLM version 0.28.0 +This will likely cause compatibility issues. + warn_if_misaligned_vllm_version() +INFO 09-23 15:46:57 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-23 15:46:57 [patch.py:510] inductor factorable-divisibility patch: installed. +INFO 09-23 15:46:57 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled. +INFO 09-23 15:46:57 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable). +(StageEngineCoreProc_stage2_replica0 pid=4169162) INFO 09-23 15:46:58 [core.py:122] Initializing a V1 LLM engine (v0.28.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=4096, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=True, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cpu, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=False, pooler_config=None, compilation_config={'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': None, 'compile_ranges_endpoints': [4096], 'inductor_compile_config': {'enable_auto_functionalized_v2': False}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': None, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto') +[W923 15:46:58.367332736 utils.cpp:68] Warning: NUMA binding: Using MEMBIND policy for memory allocation on the NUMA nodes (0). Memory allocations will be strictly bound to these NUMA nodes. (function init_cpu_memory_env) +(StageEngineCoreProc_stage2_replica0 pid=4169162) WARNING 09-23 15:46:58 [vllm.py:652] Model Runner V2 requires Triton; using the V1 model runner instead. +(StageEngineCoreProc_stage2_replica0 pid=4169162) WARNING 09-23 15:46:58 [cpu_worker.py:127] libiomp is not found in LD_PRELOAD. For best performance, please follow the section `set LD_PRELOAD` in https://docs.vllm.ai/en/latest/getting_started/installation/cpu/ to setup required pre-loaded libraries. +(StageEngineCoreProc_stage2_replica0 pid=4169162) INFO 09-23 15:46:58 [parallel_state.py:1638] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_47ba1048249e4d048745ce4f237328fc backend=gloo +(StageEngineCoreProc_stage2_replica0 pid=4169162) INFO 09-23 15:46:58 [parallel_state.py:1982] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A +(StageEngineCoreProc_stage2_replica0 pid=4169162) INFO 09-23 15:46:58 [factory.py:46] Created connector: SharedMemoryConnector +(StageEngineCoreProc_stage2_replica0 pid=4169162) INFO 09-23 15:46:58 [cpu_model_runner.py:131] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5... +(StageEngineCoreProc_stage2_replica0 pid=4169162) INFO 09-23 15:46:58 [weight_utils.py:858] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 6.17 GiB. +(StageEngineCoreProc_stage2_replica0 pid=4169162) INFO 09-23 15:46:58 [weight_utils.py:888] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre) and the checkpoint size (17.46 GiB) exceeds 90% of available RAM (6.17 GiB). +(StageEngineCoreProc_stage2_replica0 pid=4169162) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00 3969 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory'] +(StageEngineCoreProc_stage2_replica0 pid=4169162) INFO 09-23 15:47:08 [memory.py:91] Released 0 MiB after warm-up (anonymous RSS 3969 -> 3969 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory'] +(StageEngineCoreProc_stage2_replica0 pid=4169162) INFO 09-23 15:47:08 [core.py:354] init engine (profile, create kv cache, warmup model) took 0.00 s +(StageEngineCoreProc_stage2_replica0 pid=4169162) WARNING 09-23 15:47:09 [scheduler.py:182] Using custom scheduler class vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled. +(StageEngineCoreProc_stage2_replica0 pid=4169162) INFO 09-23 15:47:09 [factory.py:46] Created connector: SharedMemoryConnector +(StageEngineCoreProc_stage2_replica0 pid=4169162) WARNING 09-23 15:47:10 [vllm.py:1285] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none +(StageEngineCoreProc_stage2_replica0 pid=4169162) WARNING 09-23 15:47:10 [vllm.py:1354] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored. +INFO 09-23 15:47:10 [stage_runtime.py:943] [StageRuntime] Stage 2 engine startup completed +INFO 09-23 15:47:10 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-2 [rep-0] initializing EngineCore +(StageEngineCoreProc_stage2_replica0 pid=4169162) INFO 09-23 15:47:10 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +(StageEngineCoreProc_stage2_replica0 pid=4169162) INFO 09-23 15:47:10 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant +INFO 09-23 15:47:10 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-2 [rep-0] EngineCore running +INFO 09-23 15:47:10 [stage_runtime.py:958] [StageRuntime] Stage 2 initialized +INFO 09-23 15:47:14 [orchestrator.py:604] [Orchestrator] Starting event loop +INFO 09-23 15:47:14 [async_omni_engine.py:323] [AsyncOmniEngine] Orchestrator ready with 3 stages +INFO 09-23 15:47:14 [omni_base.py:235] [Omni] AsyncOmniEngine initialized in 73.65 seconds +INFO 09-23 15:47:14 [omni_base.py:255] [Omni] Initialized with 3 stages for model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5 +WARNING 09-23 15:47:14 [input_processor.py:296] Passing raw prompts to InputProcessor is deprecated and will be removed in v0.18. You should instead pass the outputs of Renderer.render_cmpl() or Renderer.render_chat(). + Processed prompts: 0%| | 0/1 [00:00 is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0923 15:59:43.041000 4170603 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +WARNING 09-23 15:59:44 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton. +INFO 09-23 15:59:44 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available. +/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions: + --> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922 + --> vLLM version 0.28.0 +This will likely cause compatibility issues. + warn_if_misaligned_vllm_version() +INFO 09-23 15:59:46 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-23 15:59:46 [patch.py:510] inductor factorable-divisibility patch: installed. +INFO 09-23 15:59:46 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled. +INFO 09-23 15:59:46 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable). +INFO 09-23 15:59:47 [omni_base.py:216] [Omni] Initializing with model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5 +INFO 09-23 15:59:47 [async_omni_engine.py:172] [AsyncOmniEngine] Initializing with model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5 +INFO 09-23 15:59:47 [async_omni_engine.py:290] [AsyncOmniEngine] Launching Orchestrator thread with 3 stages +INFO 09-23 15:59:47 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn +INFO 09-23 15:59:47 [initialization.py:356] Loaded OmniTransferConfig with 1 connector configurations +WARNING 09-23 15:59:47 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass) +WARNING 09-23 15:59:47 [arg_utils.py:1719] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM. +INFO 09-23 15:59:47 [model.py:672] Resolved architecture: MiniCPMO45OmniForConditionalGeneration +INFO 09-23 15:59:47 [model.py:1965] Using max model len 2048 +WARNING 09-23 15:59:47 [model.py:963] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False. +INFO 09-23 15:59:48 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode. +INFO 09-23 15:59:48 [model.py:875] Disabled mm_prefix attention mode because multimodal inputs are configuration-disabled. Attention backends without mm_prefix support may now be selected. +INFO 09-23 15:59:48 [scheduler.py:242] Chunked prefill is enabled with max_num_batched_tokens=2048. +WARNING 09-23 15:59:48 [vllm.py:1285] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none +WARNING 09-23 15:59:48 [vllm.py:1354] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored. +INFO 09-23 15:59:48 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +WARNING 09-23 15:59:48 [vllm.py:652] Model Runner V2 requires Triton; using the V1 model runner instead. +INFO 09-23 15:59:48 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant +WARNING 09-23 15:59:48 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass) +WARNING 09-23 15:59:48 [arg_utils.py:1719] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM. +INFO 09-23 15:59:48 [model.py:672] Resolved architecture: MiniCPMO45OmniForConditionalGeneration +INFO 09-23 15:59:48 [model.py:1965] Using max model len 512 +WARNING 09-23 15:59:48 [model.py:963] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False. +INFO 09-23 15:59:48 [model.py:1965] Using max model len 512 +INFO 09-23 15:59:48 [scheduler.py:242] Chunked prefill is enabled with max_num_batched_tokens=512. +INFO 09-23 15:59:48 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +WARNING 09-23 15:59:48 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass) +WARNING 09-23 15:59:48 [arg_utils.py:1719] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM. +INFO 09-23 15:59:48 [model.py:672] Resolved architecture: MiniCPMO45Code2Wav +INFO 09-23 15:59:48 [model.py:1965] Using max model len 4096 +WARNING 09-23 15:59:48 [arg_utils.py:2664] This model does not officially support disabling chunked prefill. Disabling this manually may cause the engine to crash or produce incorrect outputs. +INFO 09-23 15:59:48 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +INFO 09-23 15:59:48 [stage_runtime.py:853] [stage_init] Stage-0 set runtime devices: 0 +W0923 15:59:51.983000 4170635 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0923 15:59:52.010000 4170635 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +WARNING 09-23 15:59:53 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton. +INFO 09-23 15:59:53 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available. +/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions: + --> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922 + --> vLLM version 0.28.0 +This will likely cause compatibility issues. + warn_if_misaligned_vllm_version() +INFO 09-23 15:59:54 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-23 15:59:54 [patch.py:510] inductor factorable-divisibility patch: installed. +INFO 09-23 15:59:54 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled. +INFO 09-23 15:59:54 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable). +(StageEngineCoreProc_stage0_replica0 pid=4170635) INFO 09-23 15:59:54 [core.py:122] Initializing a V1 LLM engine (v0.28.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=2048, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=True, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cpu, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': None, 'compile_ranges_endpoints': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': None, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto') +(StageEngineCoreProc_stage0_replica0 pid=4170635) INFO 09-23 15:59:55 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode. +[W923 15:59:55.874166122 utils.cpp:68] Warning: NUMA binding: Using MEMBIND policy for memory allocation on the NUMA nodes (0). Memory allocations will be strictly bound to these NUMA nodes. (function init_cpu_memory_env) +(StageEngineCoreProc_stage0_replica0 pid=4170635) WARNING 09-23 15:59:55 [vllm.py:652] Model Runner V2 requires Triton; using the V1 model runner instead. +(StageEngineCoreProc_stage0_replica0 pid=4170635) WARNING 09-23 15:59:55 [cpu_worker.py:127] libiomp is not found in LD_PRELOAD. For best performance, please follow the section `set LD_PRELOAD` in https://docs.vllm.ai/en/latest/getting_started/installation/cpu/ to setup required pre-loaded libraries. +(StageEngineCoreProc_stage0_replica0 pid=4170635) INFO 09-23 15:59:55 [parallel_state.py:1638] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_36a1b45f19114691ab18107ff3f31f93 backend=gloo +(StageEngineCoreProc_stage0_replica0 pid=4170635) INFO 09-23 15:59:55 [parallel_state.py:1982] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A +(StageEngineCoreProc_stage0_replica0 pid=4170635) INFO 09-23 15:59:55 [cpu_model_runner.py:131] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5... +(StageEngineCoreProc_stage0_replica0 pid=4170635) WARNING 09-23 15:59:55 [vllm.py:1285] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none +(StageEngineCoreProc_stage0_replica0 pid=4170635) WARNING 09-23 15:59:55 [vllm.py:1354] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored. +(StageEngineCoreProc_stage0_replica0 pid=4170635) INFO 09-23 15:59:55 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +(StageEngineCoreProc_stage0_replica0 pid=4170635) INFO 09-23 15:59:55 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant +(StageEngineCoreProc_stage0_replica0 pid=4170635) INFO 09-23 15:59:55 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +(StageEngineCoreProc_stage0_replica0 pid=4170635) INFO 09-23 15:59:55 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +(StageEngineCoreProc_stage0_replica0 pid=4170635) INFO 09-23 15:59:55 [selector.py:217] Using HND KV cache layout for CPU_ATTN backend. +(StageEngineCoreProc_stage0_replica0 pid=4170635) INFO 09-23 15:59:59 [weight_utils.py:858] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 27.11 GiB. +(StageEngineCoreProc_stage0_replica0 pid=4170635) INFO 09-23 15:59:59 [weight_utils.py:881] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch. +(StageEngineCoreProc_stage0_replica0 pid=4170635) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00 19304 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory'] +(StageEngineCoreProc_stage0_replica0 pid=4170635) INFO 09-23 16:00:14 [cpu_worker.py:255] Explicitly set (1.0/30.91) GiB for KV cache on node 0. +(StageEngineCoreProc_stage0_replica0 pid=4170635) INFO 09-23 16:00:14 [kv_cache_utils.py:1869] GPU KV cache size: 7,168 tokens, Maximum concurrency for 2,048 tokens per request: 3.50x +(StageEngineCoreProc_stage0_replica0 pid=4170635) INFO 09-23 16:00:14 [memory.py:91] Released 0 MiB after warm-up (anonymous RSS 20314 -> 20314 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory'] +(StageEngineCoreProc_stage0_replica0 pid=4170635) INFO 09-23 16:00:14 [core.py:354] init engine (profile, create kv cache, warmup model) took 0.17 s +(StageEngineCoreProc_stage0_replica0 pid=4170635) WARNING 09-23 16:00:14 [scheduler.py:182] Using custom scheduler class vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled. +(StageEngineCoreProc_stage0_replica0 pid=4170635) INFO 09-23 16:00:14 [factory.py:46] Created connector: SharedMemoryConnector +(StageEngineCoreProc_stage0_replica0 pid=4170635) INFO 09-23 16:00:15 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +INFO 09-23 16:00:15 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed +INFO 09-23 16:00:15 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-0 [rep-0] initializing EngineCore +INFO 09-23 16:00:15 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-0 [rep-0] EngineCore running +INFO 09-23 16:00:15 [stage_runtime.py:958] [StageRuntime] Stage 0 initialized +INFO 09-23 16:00:15 [stage_runtime.py:853] [stage_init] Stage-1 set runtime devices: 0 +W0923 16:00:20.857000 4170732 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0923 16:00:20.889000 4170732 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +WARNING 09-23 16:00:22 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton. +INFO 09-23 16:00:22 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available. +/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions: + --> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922 + --> vLLM version 0.28.0 +This will likely cause compatibility issues. + warn_if_misaligned_vllm_version() +INFO 09-23 16:00:23 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-23 16:00:24 [patch.py:510] inductor factorable-divisibility patch: installed. +INFO 09-23 16:00:24 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled. +INFO 09-23 16:00:24 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable). +(StageEngineCoreProc_stage1_replica0 pid=4170732) INFO 09-23 16:00:24 [core.py:122] Initializing a V1 LLM engine (v0.28.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=512, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=True, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cpu, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': None, 'compile_ranges_endpoints': [512], 'inductor_compile_config': {'enable_auto_functionalized_v2': False}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': None, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto') +[W923 16:00:25.714479784 utils.cpp:68] Warning: NUMA binding: Using MEMBIND policy for memory allocation on the NUMA nodes (0). Memory allocations will be strictly bound to these NUMA nodes. (function init_cpu_memory_env) +(StageEngineCoreProc_stage1_replica0 pid=4170732) WARNING 09-23 16:00:25 [vllm.py:652] Model Runner V2 requires Triton; using the V1 model runner instead. +(StageEngineCoreProc_stage1_replica0 pid=4170732) WARNING 09-23 16:00:25 [cpu_worker.py:127] libiomp is not found in LD_PRELOAD. For best performance, please follow the section `set LD_PRELOAD` in https://docs.vllm.ai/en/latest/getting_started/installation/cpu/ to setup required pre-loaded libraries. +(StageEngineCoreProc_stage1_replica0 pid=4170732) INFO 09-23 16:00:25 [parallel_state.py:1638] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_7f40d35b6b1a418eb13eb3038a5e4d8e backend=gloo +(StageEngineCoreProc_stage1_replica0 pid=4170732) INFO 09-23 16:00:25 [parallel_state.py:1982] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A +(StageEngineCoreProc_stage1_replica0 pid=4170732) WARNING 09-23 16:00:25 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it. +(StageEngineCoreProc_stage1_replica0 pid=4170732) [transformers] `MiniCPMOProcessor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +(StageEngineCoreProc_stage1_replica0 pid=4170732) [transformers] Requested torchvision backend is not available. Falling back to pil backend. +(StageEngineCoreProc_stage1_replica0 pid=4170732) [transformers] `MiniCPMOProcessor` defines `feature_extractor_class = 'WhisperFeatureExtractor'`, which is deprecated. Register the correct mapping in `AutoFeatureExtractor` instead. +(StageEngineCoreProc_stage1_replica0 pid=4170732) [transformers] Image processor MiniCPMVImageProcessor: kwargs ['max_slice_nums'] were applied for backward compatibility. To avoid this warning, add them to valid_kwargs: create a custom TypedDict extending ImagesKwargs with these keys and set it as the `valid_kwargs` class attribute. +(StageEngineCoreProc_stage1_replica0 pid=4170732) WARNING 09-23 16:00:34 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it. +(StageEngineCoreProc_stage1_replica0 pid=4170732) WARNING 09-23 16:00:34 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it. +(StageEngineCoreProc_stage1_replica0 pid=4170732) WARNING 09-23 16:00:34 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it. +(StageEngineCoreProc_stage1_replica0 pid=4170732) INFO 09-23 16:00:34 [factory.py:46] Created connector: SharedMemoryConnector +(StageEngineCoreProc_stage1_replica0 pid=4170732) INFO 09-23 16:00:34 [cpu_model_runner.py:131] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5... +(StageEngineCoreProc_stage1_replica0 pid=4170732) WARNING 09-23 16:00:34 [vllm.py:1285] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none +(StageEngineCoreProc_stage1_replica0 pid=4170732) WARNING 09-23 16:00:34 [vllm.py:1354] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored. +(StageEngineCoreProc_stage1_replica0 pid=4170732) INFO 09-23 16:00:34 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +(StageEngineCoreProc_stage1_replica0 pid=4170732) INFO 09-23 16:00:34 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant +(StageEngineCoreProc_stage1_replica0 pid=4170732) INFO 09-23 16:00:34 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +(StageEngineCoreProc_stage1_replica0 pid=4170732) INFO 09-23 16:00:34 [selector.py:217] Using HND KV cache layout for CPU_ATTN backend. +(StageEngineCoreProc_stage1_replica0 pid=4170732) INFO 09-23 16:00:35 [weight_utils.py:858] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 8.14 GiB. +(StageEngineCoreProc_stage1_replica0 pid=4170732) INFO 09-23 16:00:35 [weight_utils.py:888] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre) and the checkpoint size (17.46 GiB) exceeds 90% of available RAM (8.14 GiB). +(StageEngineCoreProc_stage1_replica0 pid=4170732) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00 1461 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory'] +(StageEngineCoreProc_stage1_replica0 pid=4170732) INFO 09-23 16:00:37 [cpu_worker.py:255] Explicitly set (1.0/30.91) GiB for KV cache on node 0. +(StageEngineCoreProc_stage1_replica0 pid=4170732) INFO 09-23 16:00:37 [kv_cache_utils.py:1869] GPU KV cache size: 17,408 tokens, Maximum concurrency for 512 tokens per request: 34.00x +(StageEngineCoreProc_stage1_replica0 pid=4170732) INFO 09-23 16:00:39 [memory.py:91] Released 0 MiB after warm-up (anonymous RSS 2483 -> 2483 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory'] +(StageEngineCoreProc_stage1_replica0 pid=4170732) INFO 09-23 16:00:39 [core.py:354] init engine (profile, create kv cache, warmup model) took 1.50 s +(StageEngineCoreProc_stage1_replica0 pid=4170732) WARNING 09-23 16:00:39 [scheduler.py:182] Using custom scheduler class vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled. +(StageEngineCoreProc_stage1_replica0 pid=4170732) WARNING 09-23 16:00:39 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it. +(StageEngineCoreProc_stage1_replica0 pid=4170732) WARNING 09-23 16:00:39 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it. +(StageEngineCoreProc_stage1_replica0 pid=4170732) INFO 09-23 16:00:39 [factory.py:46] Created connector: SharedMemoryConnector +(StageEngineCoreProc_stage1_replica0 pid=4170732) INFO 09-23 16:00:39 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +INFO 09-23 16:00:39 [stage_runtime.py:943] [StageRuntime] Stage 1 engine startup completed +INFO 09-23 16:00:39 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-1 [rep-0] initializing EngineCore +INFO 09-23 16:00:40 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-1 [rep-0] EngineCore running +INFO 09-23 16:00:40 [stage_runtime.py:958] [StageRuntime] Stage 1 initialized +INFO 09-23 16:00:40 [stage_runtime.py:853] [stage_init] Stage-2 set runtime devices: 0 +W0923 16:00:44.331000 4170848 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0923 16:00:44.358000 4170848 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +WARNING 09-23 16:00:45 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton. +INFO 09-23 16:00:45 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available. +/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions: + --> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922 + --> vLLM version 0.28.0 +This will likely cause compatibility issues. + warn_if_misaligned_vllm_version() +INFO 09-23 16:00:46 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-23 16:00:46 [patch.py:510] inductor factorable-divisibility patch: installed. +INFO 09-23 16:00:46 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled. +INFO 09-23 16:00:46 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable). +(StageEngineCoreProc_stage2_replica0 pid=4170848) INFO 09-23 16:00:47 [core.py:122] Initializing a V1 LLM engine (v0.28.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=4096, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=True, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cpu, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=False, pooler_config=None, compilation_config={'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': None, 'compile_ranges_endpoints': [4096], 'inductor_compile_config': {'enable_auto_functionalized_v2': False}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': None, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto') +[W923 16:00:47.844157663 utils.cpp:68] Warning: NUMA binding: Using MEMBIND policy for memory allocation on the NUMA nodes (0). Memory allocations will be strictly bound to these NUMA nodes. (function init_cpu_memory_env) +(StageEngineCoreProc_stage2_replica0 pid=4170848) WARNING 09-23 16:00:47 [vllm.py:652] Model Runner V2 requires Triton; using the V1 model runner instead. +(StageEngineCoreProc_stage2_replica0 pid=4170848) WARNING 09-23 16:00:47 [cpu_worker.py:127] libiomp is not found in LD_PRELOAD. For best performance, please follow the section `set LD_PRELOAD` in https://docs.vllm.ai/en/latest/getting_started/installation/cpu/ to setup required pre-loaded libraries. +(StageEngineCoreProc_stage2_replica0 pid=4170848) INFO 09-23 16:00:47 [parallel_state.py:1638] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_d02d9dab0eb747c2a451e899387fbf98 backend=gloo +(StageEngineCoreProc_stage2_replica0 pid=4170848) INFO 09-23 16:00:47 [parallel_state.py:1982] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A +(StageEngineCoreProc_stage2_replica0 pid=4170848) INFO 09-23 16:00:47 [factory.py:46] Created connector: SharedMemoryConnector +(StageEngineCoreProc_stage2_replica0 pid=4170848) INFO 09-23 16:00:47 [cpu_model_runner.py:131] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5... +(StageEngineCoreProc_stage2_replica0 pid=4170848) INFO 09-23 16:00:47 [weight_utils.py:858] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 6.67 GiB. +(StageEngineCoreProc_stage2_replica0 pid=4170848) INFO 09-23 16:00:47 [weight_utils.py:888] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre) and the checkpoint size (17.46 GiB) exceeds 90% of available RAM (6.67 GiB). +(StageEngineCoreProc_stage2_replica0 pid=4170848) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00 3964 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory'] +(StageEngineCoreProc_stage2_replica0 pid=4170848) INFO 09-23 16:00:55 [memory.py:91] Released 0 MiB after warm-up (anonymous RSS 3964 -> 3964 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory'] +(StageEngineCoreProc_stage2_replica0 pid=4170848) INFO 09-23 16:00:55 [core.py:354] init engine (profile, create kv cache, warmup model) took 0.00 s +(StageEngineCoreProc_stage2_replica0 pid=4170848) WARNING 09-23 16:00:56 [scheduler.py:182] Using custom scheduler class vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled. +(StageEngineCoreProc_stage2_replica0 pid=4170848) INFO 09-23 16:00:56 [factory.py:46] Created connector: SharedMemoryConnector +(StageEngineCoreProc_stage2_replica0 pid=4170848) WARNING 09-23 16:00:56 [vllm.py:1285] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none +(StageEngineCoreProc_stage2_replica0 pid=4170848) WARNING 09-23 16:00:56 [vllm.py:1354] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored. +(StageEngineCoreProc_stage2_replica0 pid=4170848) INFO 09-23 16:00:56 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +(StageEngineCoreProc_stage2_replica0 pid=4170848) INFO 09-23 16:00:56 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant +INFO 09-23 16:00:56 [stage_runtime.py:943] [StageRuntime] Stage 2 engine startup completed +INFO 09-23 16:00:56 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-2 [rep-0] initializing EngineCore +INFO 09-23 16:00:56 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-2 [rep-0] EngineCore running +INFO 09-23 16:00:56 [stage_runtime.py:958] [StageRuntime] Stage 2 initialized +INFO 09-23 16:01:00 [orchestrator.py:604] [Orchestrator] Starting event loop +INFO 09-23 16:01:00 [async_omni_engine.py:323] [AsyncOmniEngine] Orchestrator ready with 3 stages +INFO 09-23 16:01:00 [omni_base.py:235] [Omni] AsyncOmniEngine initialized in 73.88 seconds +INFO 09-23 16:01:00 [omni_base.py:255] [Omni] Initialized with 3 stages for model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5 +WARNING 09-23 16:01:01 [input_processor.py:296] Passing raw prompts to InputProcessor is deprecated and will be removed in v0.18. You should instead pass the outputs of Renderer.render_cmpl() or Renderer.render_chat(). + Processed prompts: 0%| | 0/1 [00:00 vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922 + --> vLLM version 0.28.0 +This will likely cause compatibility issues. + warn_if_misaligned_vllm_version() +W0923 15:34:52.858000 4167184 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0923 15:34:52.905000 4167184 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +WARNING 09-23 15:34:54 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton. +INFO 09-23 15:34:54 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available. +INFO 09-23 15:34:56 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-23 15:34:56 [patch.py:510] inductor factorable-divisibility patch: installed. +INFO 09-23 15:34:56 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled. +INFO 09-23 15:34:56 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable). +==================== + vllm version: 0.28.0 + ==================== +INFO 09-23 15:34:56 [omni_base.py:216] [Omni] Initializing with model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5 +INFO 09-23 15:34:56 [async_omni_engine.py:172] [AsyncOmniEngine] Initializing with model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5 +INFO 09-23 15:34:57 [async_omni_engine.py:290] [AsyncOmniEngine] Launching Orchestrator thread with 1 stages +INFO 09-23 15:34:57 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn +INFO 09-23 15:34:57 [initialization.py:356] Loaded OmniTransferConfig with 0 connector configurations +WARNING 09-23 15:34:57 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass) +WARNING 09-23 15:34:57 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass) +WARNING 09-23 15:34:57 [arg_utils.py:1719] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM. +INFO 09-23 15:34:57 [model.py:672] Resolved architecture: MiniCPMO45OmniForConditionalGeneration +INFO 09-23 15:34:57 [model.py:1965] Using max model len 2048 +WARNING 09-23 15:34:57 [model.py:963] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False. +INFO 09-23 15:34:58 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode. +INFO 09-23 15:34:58 [model.py:875] Disabled mm_prefix attention mode because multimodal inputs are configuration-disabled. Attention backends without mm_prefix support may now be selected. +INFO 09-23 15:34:58 [scheduler.py:242] Chunked prefill is enabled with max_num_batched_tokens=2048. +WARNING 09-23 15:34:58 [vllm.py:1285] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none +WARNING 09-23 15:34:58 [vllm.py:1354] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored. +INFO 09-23 15:34:58 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +WARNING 09-23 15:34:58 [vllm.py:652] Model Runner V2 requires Triton; using the V1 model runner instead. +INFO 09-23 15:34:58 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant +INFO 09-23 15:34:58 [stage_runtime.py:853] [stage_init] Stage-0 set runtime devices: 0 +/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions: + --> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922 + --> vLLM version 0.28.0 +This will likely cause compatibility issues. + warn_if_misaligned_vllm_version() +W0923 15:35:01.933000 4167214 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +W0923 15:35:01.959000 4167214 torch/utils/_pytree.py:630] is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release. +WARNING 09-23 15:35:02 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton. +INFO 09-23 15:35:02 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available. +INFO 09-23 15:35:04 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed. +INFO 09-23 15:35:04 [patch.py:510] inductor factorable-divisibility patch: installed. +INFO 09-23 15:35:04 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled. +INFO 09-23 15:35:04 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable). +(StageEngineCoreProc_stage0_replica0 pid=4167214) INFO 09-23 15:35:04 [core.py:122] Initializing a V1 LLM engine (v0.28.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=2048, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=True, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cpu, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': , 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': None, 'compile_ranges_endpoints': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False}, 'inductor_passes': {}, 'cudagraph_mode': , 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': None, 'dynamic_shapes_config': {'type': , 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto') +(StageEngineCoreProc_stage0_replica0 pid=4167214) INFO 09-23 15:35:05 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode. +[W923 15:35:05.941299642 utils.cpp:68] Warning: NUMA binding: Using MEMBIND policy for memory allocation on the NUMA nodes (0). Memory allocations will be strictly bound to these NUMA nodes. (function init_cpu_memory_env) +(StageEngineCoreProc_stage0_replica0 pid=4167214) WARNING 09-23 15:35:05 [vllm.py:652] Model Runner V2 requires Triton; using the V1 model runner instead. +(StageEngineCoreProc_stage0_replica0 pid=4167214) WARNING 09-23 15:35:05 [cpu_worker.py:127] libiomp is not found in LD_PRELOAD. For best performance, please follow the section `set LD_PRELOAD` in https://docs.vllm.ai/en/latest/getting_started/installation/cpu/ to setup required pre-loaded libraries. +(StageEngineCoreProc_stage0_replica0 pid=4167214) INFO 09-23 15:35:05 [parallel_state.py:1638] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_5a5a70874ca14e3ab8e90cde5a3e0034 backend=gloo +(StageEngineCoreProc_stage0_replica0 pid=4167214) INFO 09-23 15:35:05 [parallel_state.py:1982] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A +(StageEngineCoreProc_stage0_replica0 pid=4167214) INFO 09-23 15:35:05 [cpu_model_runner.py:131] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5... +(StageEngineCoreProc_stage0_replica0 pid=4167214) WARNING 09-23 15:35:05 [vllm.py:1285] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none +(StageEngineCoreProc_stage0_replica0 pid=4167214) WARNING 09-23 15:35:05 [vllm.py:1354] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored. +(StageEngineCoreProc_stage0_replica0 pid=4167214) INFO 09-23 15:35:05 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +(StageEngineCoreProc_stage0_replica0 pid=4167214) INFO 09-23 15:35:05 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant +(StageEngineCoreProc_stage0_replica0 pid=4167214) INFO 09-23 15:35:05 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +(StageEngineCoreProc_stage0_replica0 pid=4167214) INFO 09-23 15:35:05 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +(StageEngineCoreProc_stage0_replica0 pid=4167214) INFO 09-23 15:35:05 [selector.py:217] Using HND KV cache layout for CPU_ATTN backend. +(StageEngineCoreProc_stage0_replica0 pid=4167214) INFO 09-23 15:35:09 [weight_utils.py:858] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 27.01 GiB. +(StageEngineCoreProc_stage0_replica0 pid=4167214) INFO 09-23 15:35:09 [weight_utils.py:881] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch. +(StageEngineCoreProc_stage0_replica0 pid=4167214) Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00 19278 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory'] +(StageEngineCoreProc_stage0_replica0 pid=4167214) INFO 09-23 15:35:29 [cpu_worker.py:255] Explicitly set (2.0/30.91) GiB for KV cache on node 0. +(StageEngineCoreProc_stage0_replica0 pid=4167214) INFO 09-23 15:35:29 [kv_cache_utils.py:1869] GPU KV cache size: 14,464 tokens, Maximum concurrency for 2,048 tokens per request: 7.06x +(StageEngineCoreProc_stage0_replica0 pid=4167214) INFO 09-23 15:35:29 [memory.py:91] Released 0 MiB after warm-up (anonymous RSS 21316 -> 21316 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory'] +(StageEngineCoreProc_stage0_replica0 pid=4167214) INFO 09-23 15:35:29 [core.py:354] init engine (profile, create kv cache, warmup model) took 0.27 s +(StageEngineCoreProc_stage0_replica0 pid=4167214) WARNING 09-23 15:35:29 [scheduler.py:182] Using custom scheduler class vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled. +(StageEngineCoreProc_stage0_replica0 pid=4167214) INFO 09-23 15:35:29 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']) +INFO 09-23 15:35:29 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed +INFO 09-23 15:35:30 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-0 [rep-0] initializing EngineCore +INFO 09-23 15:35:30 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-0 [rep-0] EngineCore running +INFO 09-23 15:35:30 [stage_runtime.py:958] [StageRuntime] Stage 0 initialized +INFO 09-23 15:35:32 [orchestrator.py:604] [Orchestrator] Starting event loop +INFO 09-23 15:35:32 [async_omni_engine.py:323] [AsyncOmniEngine] Orchestrator ready with 1 stages +INFO 09-23 15:35:32 [omni_base.py:235] [Omni] AsyncOmniEngine initialized in 36.02 seconds +INFO 09-23 15:35:32 [omni_base.py:255] [Omni] Initialized with 1 stages for model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5 +WARNING 09-23 15:35:33 [input_processor.py:296] Passing raw prompts to InputProcessor is deprecated and will be removed in v0.18. You should instead pass the outputs of Renderer.render_cmpl() or Renderer.render_chat(). + Processed prompts: 0%| | 0/1 [00:00./)", + tokenization_kwargs={}, + enable_hf_prompt_update=False, + ) + + prompt_ids, mm_data, updated = result + assert prompt_ids == [len("()")] + assert isinstance(mm_data, BatchFeature) and not mm_data + assert updated is False + + +def test_vllm_029_contract_returns_batch_feature(): + result = MiniCPMO45OmniLLMMultiModalProcessor._apply_hf_processor_main( + _processor(), _EmptyItems(), {} + ) + + assert isinstance(result, BatchFeature) + torch.testing.assert_close(result["input_ids"], torch.tensor([[5]])) diff --git a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py index 315ae159d..f0397618b 100644 --- a/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py +++ b/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py @@ -21,6 +21,7 @@ import inspect import math import os import warnings +from collections import defaultdict from collections.abc import Callable, Iterable, Mapping, Sequence from dataclasses import dataclass, replace from functools import partial @@ -3422,24 +3423,37 @@ class MiniCPMO45OmniLLMMultiModalProcessor(BaseMultiModalProcessor[MiniCPMO45Omn self, mm_items: MultiModalDataItems, hf_processor_mm_kwargs: Mapping[str, object], - ) -> BatchFeature: + prompt: str | list[int] | None = None, + tokenization_kwargs: Mapping[str, object] | None = None, + *, + enable_hf_prompt_update: bool | None = None, + ) -> BatchFeature | tuple[list[int], BatchFeature, bool]: """ Process each modality independently because the MiniCPM processor asserts that image tags and image sizes have matching lengths. + + vLLM 0.28 passes the prompt here and expects a three-tuple; 0.29 + tokenizes the prompt separately and expects only the BatchFeature. """ valid_mm_items = mm_items.select({key for key, count in mm_items.get_all_counts().items() if count > 0}) mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items) + mm_inputs = self.process_mm_inputs(mm_data, hf_processor_mm_kwargs) + if prompt is not None: + if isinstance(prompt, str): + # vLLM 0.28's generic text-only path invokes the HF + # multimodal processor without media, which rejects MiniCPM + # modality placeholders in dummy prompts. + prompt_ids = self.info.get_tokenizer().encode(prompt) + else: + prompt_ids = self._apply_hf_processor_tokens_only(prompt) + processed_data = BatchFeature(dict(mm_inputs)) + processed_data.update(passthrough_data) + return prompt_ids, processed_data, False + tokenizer = self.info.get_tokenizer() prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts()) - input_ids = torch.tensor([tokenizer.encode(prompt_text)]) - mm_inputs = self.process_mm_inputs(mm_data, hf_processor_mm_kwargs) - processed_data = BatchFeature( - { - "input_ids": input_ids, - **mm_inputs, - } - ) + processed_data = BatchFeature({"input_ids": torch.tensor([tokenizer.encode(prompt_text)]), **mm_inputs}) processed_data.update(passthrough_data) return processed_data @@ -3597,13 +3611,32 @@ class MiniCPMO45OmniLLMMultiModalProcessor(BaseMultiModalProcessor[MiniCPMO45Omn ) -> dict[str, NestedTensors]: from vllm.model_executor.models.minicpmv import MiniCPMVMultiModalProcessor - return MiniCPMVMultiModalProcessor._call_hf_processor_on_prompts( - self, - prompts, - mm_data, - {key: value for key, value in mm_kwargs.items() if key != "use_tts"}, - out_keys=out_keys, - ) + mm_kwargs = {key: value for key, value in mm_kwargs.items() if key != "use_tts"} + call_prompts = getattr(MiniCPMVMultiModalProcessor, "_call_hf_processor_on_prompts", None) + if call_prompts is not None: + return call_prompts(self, prompts, mm_data, mm_kwargs, out_keys=out_keys) + + # vLLM 0.28 predates this MiniCPMV helper. Keep the same per-modality + # processing contract while using its available HF processor context. + if self.info.get_model_version() in {(2, 6), (4, 0), (4, 5), (4, 6)}: + inputs = self.info.ctx.call_hf_processor( + self.info.get_hf_processor(**mm_kwargs), + dict(text=prompts, **mm_data), + mm_kwargs, + ) + else: + inputs = defaultdict(list) + for i, prompt in enumerate(prompts): + inputs_one = self.info.ctx.call_hf_processor( + self.info.get_hf_processor(**mm_kwargs), + dict(text=prompt, **{key: value[i] for key, value in mm_data.items()}), + mm_kwargs, + ) + for key, value in inputs_one.items(): + if len(value) != 1: + raise ValueError(f"Expected one {key} item per MiniCPM-o prompt, got {len(value)}") + inputs[key].append(value[0]) + return {key: inputs[key] for key in out_keys} def _get_prompt_updates( self, diff --git a/vllm_omni/patch.py b/vllm_omni/patch.py index 9c8da61f2..0a863c027 100644 --- a/vllm_omni/patch.py +++ b/vllm_omni/patch.py @@ -536,6 +536,12 @@ _patch_inductor_factorable_divisibility() # (not a slot or C extension). If CuMemAllocator is rewritten in C/Cython, # this monkey-patch will silently become a no-op. def _patch_cumem_free_callback_cuda() -> None: + if sys.platform == "win32": + # The native Windows vLLM build may not load libcudart before this + # module is imported. This Linux CUDA atexit workaround is not needed + # to import Omni on that platform. + _PATCH_LOGGER.debug("[cumem-cuda] native Windows; skipping patch") + return try: import torch