Qualify scoped MiniCPM-o CPU speech and Windows InternVLA policy paths
This commit is contained in:
tzhouam committed
1 parent
3f2cd52939
commit
2f5d0edcf7
37 files changed
+4618
-27
No files matched your search
@@ -21,6 +21,7 @@ def main() -> None:
|
||||
parser.add_argument("--cosmos-dir", type=Path, required=True)
|
||||
parser.add_argument("--processor-dir", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--actions-output", type=Path, help="Optional float32 .npy actions for reference comparison")
|
||||
parser.add_argument("--device", default="cuda")
|
||||
args = parser.parse_args()
|
||||
|
||||
@@ -58,6 +59,10 @@ def main() -> None:
|
||||
pipeline = initialize_model(config)
|
||||
if pipeline.runtime_mode() != "real_checkpoint_loaded":
|
||||
raise RuntimeError(f"Unexpected InternVLA runtime mode: {pipeline.runtime_mode()}")
|
||||
policy_devices = sorted({parameter.device.type for parameter in pipeline.policy.parameters()})
|
||||
requested_device_type = torch.device(args.device).type
|
||||
if policy_devices != [requested_device_type]:
|
||||
raise RuntimeError(f"InternVLA policy ran on {policy_devices}, not {requested_device_type}")
|
||||
startup_s = time.perf_counter() - started
|
||||
|
||||
batch_inputs = pipeline._build_fake_batch_inputs()
|
||||
@@ -98,13 +103,23 @@ def main() -> None:
|
||||
raise RuntimeError(f"Unexpected action output: {type(actions).__name__}")
|
||||
if not torch.isfinite(actions).all():
|
||||
raise RuntimeError("InternVLA returned non-finite actions")
|
||||
if actions.device.type != requested_device_type:
|
||||
raise RuntimeError(f"InternVLA actions were on {actions.device}, not {requested_device_type}")
|
||||
|
||||
action_bytes = actions.detach().float().cpu().contiguous().numpy().tobytes()
|
||||
action_array = actions.detach().float().cpu().contiguous().numpy()
|
||||
action_bytes = action_array.tobytes()
|
||||
if args.actions_output is not None:
|
||||
import numpy as np
|
||||
|
||||
args.actions_output.parent.mkdir(parents=True, exist_ok=True)
|
||||
np.save(args.actions_output, action_array)
|
||||
report = {
|
||||
"scope": "real checkpoint, synthetic zero observations and noise; no task-quality claim",
|
||||
"model_dir": str(model_dir),
|
||||
"runtime_mode": pipeline.runtime_mode(),
|
||||
"device": args.device,
|
||||
"policy_devices": policy_devices,
|
||||
"actions_device": str(actions.device),
|
||||
"dtype": str(actions.dtype),
|
||||
"action_shape": list(actions.shape),
|
||||
"finite": True,
|
||||
|
||||
@@ -0,0 +1,187 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Measure serial real-weight MiniCPM-o text-to-speech requests on one device."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import platform
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import psutil
|
||||
import torch
|
||||
import vllm
|
||||
|
||||
from examples.offline_inference.minicpmo.end2end import get_text_query
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
|
||||
|
||||
class MemorySampler:
|
||||
def __init__(self, interval_s: float = 0.5):
|
||||
self.interval_s = interval_s
|
||||
self.stop = threading.Event()
|
||||
self.samples: list[dict[str, int | float]] = []
|
||||
self.thread = threading.Thread(target=self._run, name="memory-sampler", daemon=True)
|
||||
|
||||
def _run(self) -> None:
|
||||
parent = psutil.Process()
|
||||
while not self.stop.is_set():
|
||||
memory = psutil.virtual_memory()
|
||||
swap = psutil.swap_memory()
|
||||
rss_sum = 0
|
||||
for proc in [parent, *parent.children(recursive=True)]:
|
||||
try:
|
||||
rss_sum += proc.memory_info().rss
|
||||
except (psutil.NoSuchProcess, psutil.AccessDenied):
|
||||
pass
|
||||
self.samples.append(
|
||||
{
|
||||
"monotonic_s": time.perf_counter(),
|
||||
"wsl_used_bytes": memory.used,
|
||||
"wsl_available_bytes": memory.available,
|
||||
"swap_used_bytes": swap.used,
|
||||
"process_tree_rss_sum_bytes": rss_sum,
|
||||
}
|
||||
)
|
||||
self.stop.wait(self.interval_s)
|
||||
|
||||
def __enter__(self):
|
||||
self.thread.start()
|
||||
return self
|
||||
|
||||
def __exit__(self, *_exc):
|
||||
self.stop.set()
|
||||
self.thread.join()
|
||||
|
||||
|
||||
def _nearest_rank(values: list[float], percentile: float) -> float:
|
||||
ordered = sorted(values)
|
||||
return ordered[max(0, math.ceil(percentile * len(ordered)) - 1)]
|
||||
|
||||
|
||||
def _run_request(omni: Omni, prompt: dict, memory_sampler: MemorySampler) -> dict:
|
||||
started = time.perf_counter()
|
||||
first_text_s = None
|
||||
first_audio_s = None
|
||||
text = None
|
||||
token_ids = None
|
||||
audio_samples = None
|
||||
audio_rms = None
|
||||
request_ids = set()
|
||||
|
||||
for output in omni.generate([prompt], None):
|
||||
request_ids.add(output.request_id)
|
||||
if output.final_output_type == "text":
|
||||
first_text_s = time.perf_counter() - started
|
||||
text = output.outputs[0].text.strip()
|
||||
token_ids = list(output.outputs[0].token_ids)
|
||||
elif output.final_output_type == "audio":
|
||||
first_audio_s = time.perf_counter() - started
|
||||
audio = output.outputs[0].multimodal_output["audio"]
|
||||
if isinstance(audio, list):
|
||||
audio = torch.cat(
|
||||
[
|
||||
part.flatten() if isinstance(part, torch.Tensor) else torch.as_tensor(part).flatten()
|
||||
for part in audio
|
||||
]
|
||||
)
|
||||
wave = audio.detach().float().cpu().numpy().reshape(-1)
|
||||
if not np.isfinite(wave).all():
|
||||
raise RuntimeError("MiniCPM-o generated non-finite audio")
|
||||
audio_samples = int(wave.size)
|
||||
audio_rms = float(np.sqrt(np.mean(wave.astype(np.float64) ** 2)))
|
||||
|
||||
completed = time.perf_counter()
|
||||
if not text or not token_ids or not audio_samples or audio_rms == 0:
|
||||
raise RuntimeError("MiniCPM-o did not produce both nonempty text and nonzero audio")
|
||||
samples = [sample for sample in memory_sampler.samples if started <= sample["monotonic_s"] <= completed]
|
||||
return {
|
||||
"request_ids": sorted(request_ids),
|
||||
"wall_s": completed - started,
|
||||
"first_text_s": first_text_s,
|
||||
"first_audio_s": first_audio_s,
|
||||
"text": text,
|
||||
"token_ids": token_ids,
|
||||
"audio_samples": audio_samples,
|
||||
"audio_duration_s_at_24khz": audio_samples / 24000,
|
||||
"audio_rms_float": audio_rms,
|
||||
"sampled_memory": {
|
||||
"n": len(samples),
|
||||
"max_wsl_used_bytes": max((s["wsl_used_bytes"] for s in samples), default=None),
|
||||
"min_wsl_available_bytes": min((s["wsl_available_bytes"] for s in samples), default=None),
|
||||
"max_swap_used_bytes": max((s["swap_used_bytes"] for s in samples), default=None),
|
||||
# Sum of RSS can double-count shared pages; it is not unique RAM.
|
||||
"max_process_tree_rss_sum_bytes": max((s["process_tree_rss_sum_bytes"] for s in samples), default=None),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--model", type=Path, required=True)
|
||||
parser.add_argument("--deploy-config", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--warmups", type=int, default=1)
|
||||
parser.add_argument("--repeats", type=int, default=20)
|
||||
args = parser.parse_args()
|
||||
if args.warmups < 0 or args.repeats < 1:
|
||||
parser.error("warmups must be nonnegative and repeats must be positive")
|
||||
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
prompt = {**get_text_query(use_tts=True).inputs, "modalities": ["text", "audio"]}
|
||||
with MemorySampler() as sampler:
|
||||
started = time.perf_counter()
|
||||
omni = Omni(
|
||||
model=str(args.model.resolve()),
|
||||
deploy_config=str(args.deploy_config.resolve()),
|
||||
trust_remote_code=True,
|
||||
)
|
||||
startup_s = time.perf_counter() - started
|
||||
try:
|
||||
rows = []
|
||||
with args.output.with_suffix(".jsonl").open("w", encoding="utf-8") as raw:
|
||||
for index in range(args.warmups + args.repeats):
|
||||
row = _run_request(omni, prompt, sampler)
|
||||
row["index"] = index
|
||||
row["warmup"] = index < args.warmups
|
||||
rows.append(row)
|
||||
raw.write(json.dumps(row) + "\n")
|
||||
raw.flush()
|
||||
print(f"request={index} warmup={row['warmup']} wall_s={row['wall_s']:.3f}", flush=True)
|
||||
finally:
|
||||
omni.close()
|
||||
|
||||
measured = [row for row in rows if not row["warmup"]]
|
||||
times = [row["wall_s"] for row in measured]
|
||||
output = {
|
||||
"scope": "serial one-request-at-a-time MiniCPM-o text-to-speech; no speech-quality claim",
|
||||
"model": str(args.model.resolve()),
|
||||
"deploy_config": str(args.deploy_config.resolve()),
|
||||
"deploy_config_sha256": hashlib.sha256(args.deploy_config.read_bytes()).hexdigest(),
|
||||
"platform": platform.platform(),
|
||||
"torch": torch.__version__,
|
||||
"vllm": vllm.__version__,
|
||||
"torch_cuda_available": torch.cuda.is_available(),
|
||||
"environment": {
|
||||
name: os.environ.get(name)
|
||||
for name in ("VLLM_TARGET_DEVICE", "VLLM_CPU_KVCACHE_SPACE", "OMP_NUM_THREADS", "CUDA_VISIBLE_DEVICES")
|
||||
},
|
||||
"startup_s": startup_s,
|
||||
"warmups": args.warmups,
|
||||
"repeats": args.repeats,
|
||||
"wall_s_p50_nearest_rank": _nearest_rank(times, 0.5),
|
||||
"wall_s_p95_nearest_rank": _nearest_rank(times, 0.95),
|
||||
"requests": rows,
|
||||
}
|
||||
args.output.write_text(json.dumps(output, indent=2) + "\n", encoding="utf-8")
|
||||
print(f"Wrote {args.output}", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,44 @@
|
||||
# 2026-09-23 WSL CPU and native Windows expansion
|
||||
|
||||
This extends the [60-cell device × model matrix](../e2e_profiling_20260922/evidence/summary/README.md) with previously unrun WSL CPU and native Windows paths. The September 22 timing tables remain historical. MiniCPM-o now has a separate 20-request serial text-to-speech timing run; each InternVLA timing remains one observation. None is a sustained or full-modality profile. The [checkpoint manifests](../e2e_recovery_20260923/README.md) pin the real weights. No model, precision or context was silently substituted.
|
||||
|
||||
| Device / model | Verified depth | Observed output and time | Remaining gate |
|
||||
|---|---|---|---|
|
||||
| WSL CPU / MiniCPM-o 4.5, revision `503e754207c94da6bb26850b4469f367c9ea3582` | Scoped complete three-stage text-to-text+speech path (P for this input only) | Coherent text and a 5.2 s mono 24 kHz WAV. First request 27.09 s after 73.65 s startup; separate 1-warmup/20-measured serial run: startup 73.89 s, request wall p50 21.86 s/p95 22.97 s. | Speech intelligibility/alignment, image/audio/video inputs, concurrency, interruption, power and thermal behavior. |
|
||||
| WSL CPU / InternVLA-A1-3B Place_Markpen, revision `0a557c6a503e6545b369bff2f34c76a410bbd190` | Strict real-weight load and full policy forward on **synthetic zero** observations/noise | Finite float32 actions `[1, 50, 32]`; CPU-only PyTorch startup 31.78 s and synchronized forward 3.25 s. Compared with a same-checkpoint CUDA run, relative L2 difference 1.3025% and cosine similarity 0.999917. | Real Place_Markpen observations and reference actions, task tolerance, units/timestamps, repeated latency, memory and deadlines. |
|
||||
| Native Windows CPU / InternVLA-A1-3B Place_Markpen, same revision | Strict real-weight load and full policy forward on **synthetic zero** observations/noise; policy and actions checked on CPU | Finite float32 actions `[1, 50, 32]`; placement-checked startup 61.66 s and forward 3.36 s. Relative L2 difference from WSL CPU 1.0824%; cosine 0.999942. | Real observations/reference actions and quality tolerance; repeated latency, memory, units/timestamps and deadlines. This is the direct policy CPU path, not native vLLM CPU attention. |
|
||||
| Native Windows RTX 5090 Laptop / InternVLA-A1-3B Place_Markpen, same revision | Strict real-weight load and full policy forward on **synthetic zero** observations/noise; policy and actions checked on CUDA | Finite float32 actions `[1, 50, 32]`; placement-checked startup 75.54 s and synchronized forward 1.12 s. Relative L2 difference from WSL CUDA 0.8315%; cosine 0.999966. | Real observations/reference actions and quality tolerance; repeated latency, memory, units/timestamps and deadlines. |
|
||||
|
||||
Both runs used Ryzen AI 9 HX 370 on Ubuntu 26.04/WSL2 kernel `6.18.33.2-microsoft-standard-WSL2`, with 30.91 GiB WSL RAM and 8 GiB swap. The CPU environment has Python 3.12.13, PyTorch 2.13.0+cpu (`torch.cuda.is_available() == False`), vLLM 0.28.0 and Transformers 5.14.1; Omni source is 0.29.0rc2. MiniCPM-o used BF16, eager execution, serial requests, `OMP_NUM_THREADS=8`, 1 GiB CPU KV cache **per stage**, 2,048-token thinker / 512-token talker limits, text input only, and no CPU weight offload. Its full three-stage run used swap, so the timing applies only to this constrained machine and workload. The output [text](evidence/minicpmo/0_28854dbf-b6b7-4f1d-a35a-52f254ab9d9e.txt), [WAV](evidence/minicpmo/output_0_28854dbf-b6b7-4f1d-a35a-52f254ab9d9e.wav), [raw log](evidence/minicpmo/minicpmo_cpu_full_compat4.log), [deployment config](evidence/minicpmo/minicpmo_cpu_offline.yaml) and [artifact check](evidence/minicpmo/minicpmo_cpu_artifact_report.json) are retained. The WAV has 124,800 PCM16 frames, 124,147 nonzero samples, peak absolute amplitude 21,850 and RMS 3,319.41; no listening or ASR check was performed. An earlier [one-stage CPU thinker run](evidence/minicpmo/minicpmo_cpu_text.log) also produced coherent text in 8.67 s after 36.02 s startup, but that diagnostic topology is not the speech pipeline.
|
||||
|
||||
The separate [20-request profile](evidence/minicpmo/minicpmo_cpu_profile_20.json) records every measured output and sampled memory; the [JSONL](evidence/minicpmo/minicpmo_cpu_profile_20.jsonl) survives interruption, and the [run log](evidence/minicpmo/minicpmo_cpu_profile_20.log) retains worker details. One warmup was excluded. All 20 measured requests returned the same thinker token sequence, nonzero 124,800-sample audio, and request wall times 20.95–23.33 s; nearest-rank p50/p95 were 21.86/22.97 s (real-time factor 4.20/4.42 for 5.2 s of audio). Sampled WSL available RAM reached a minimum of 3.80 GB and swap use a maximum of 5.75 GB; summed process-tree RSS reached 29.04 GB but double-counts shared pages. These are 0.5 s samples during requests, not load-peak or unique-resident-memory measurements. Audio and text were returned together at request completion, so the profile cannot establish playable-stream latency, and no concurrency or 30-minute thermal run was performed.
|
||||
|
||||
The CPU-only InternVLA run used the same BF16 policy checkpoint, synthetic six-camera history/state input and zero diffusion noise as the CUDA reference. The raw [CPU JSON](evidence/internvla/internvla_pure_cpu_synthetic.json), [CPU log](evidence/internvla/internvla_pure_cpu_synthetic.log), [CPU actions](evidence/internvla/internvla_pure_cpu_actions.npy), [CUDA JSON](evidence/internvla/internvla_cuda_synthetic_with_actions.json), [CUDA actions](evidence/internvla/internvla_cuda_actions.npy) and [numerical comparison](evidence/internvla/internvla_pure_cpu_cuda_parity.json) are retained. The CUDA reference used the WSL RTX 5090 Laptop and PyTorch 2.13.0+cu130; its one forward took 1.33 s. The action difference has maximum absolute error 0.00993 and mean absolute error 0.00117. No reference-task tolerance is available, so this comparison is evidence of numerical proximity, not an accuracy pass.
|
||||
|
||||
The native Windows runs used Windows 11 build 26200, NVIDIA driver 610.71 for the GPU path, Python 3.12.10, PyTorch 2.13.0+cu130, vLLM 0.29.0, Transformers 5.14.1, and the current Omni source. The CPU run explicitly selected the CPU policy despite the CUDA-capable wheel. The placement-checked reruns assert that all policy parameters and the output action tensor remain on the requested device. Both loaded the same checkpoint from the local WSL filesystem through a UNC path; Windows CPU and GPU runs used the same synthetic input and zero diffusion noise as the WSL reference. The native [CPU JSON](evidence/internvla/internvla_windows_cpu_synthetic.json), [GPU JSON](evidence/internvla/internvla_windows_gpu_synthetic.json), [CPU log](evidence/internvla/internvla_windows_cpu_synthetic.log), [GPU log](evidence/internvla/internvla_windows_gpu_synthetic.log), [CPU actions](evidence/internvla/internvla_windows_cpu_actions.npy), [GPU actions](evidence/internvla/internvla_windows_gpu_actions.npy), [CPU/WSL comparison](evidence/internvla/internvla_windows_cpu_wsl_cpu_parity.json), and [GPU/WSL comparison](evidence/internvla/internvla_windows_gpu_wsl_cuda_parity.json) are retained. Native vLLM import initially failed on a Linux-specific CuMem allocator patch; the added Windows guard resolved the import. Native default GBK source decoding also required `PYTHONUTF8=1`. Both platform differences are setup requirements, not model-quality evidence. Power mode and thermal state were not captured.
|
||||
|
||||
The MiniCPM-o CPU environment initially failed in the talker because vLLM 0.28 calls the multimodal processor with a prompt and three-value return contract, while 0.29 expects a `BatchFeature`. The 0.28 MiniCPMV processor also lacks `_call_hf_processor_on_prompts`. The local processor now handles both APIs. The next run failed before vocoder inference because `stepaudio2-minicpmo` was absent from the isolated CPU venv; installing version 0.1.1 with its dependencies enabled the final run. [All failed logs](evidence/minicpmo/) remain available; no failure was relabeled as a pass.
|
||||
|
||||
Reproduce from the repository root with the verified local checkpoint paths:
|
||||
|
||||
```bash
|
||||
export PYTHONPATH="$PWD" OMP_NUM_THREADS=8 MKL_NUM_THREADS=8 TOKENIZERS_PARALLELISM=false
|
||||
export CUDA_VISIBLE_DEVICES= VLLM_TARGET_DEVICE=cpu VLLM_CPU_KVCACHE_SPACE=1 VLLM_ENABLE_V1_MULTIPROCESSING=0
|
||||
/home/zhout/project/edge_infer/.venvs/omni-cpu/bin/python examples/offline_inference/minicpmo/end2end.py \
|
||||
--model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5 \
|
||||
--deploy-config benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/minicpmo/minicpmo_cpu_offline.yaml \
|
||||
--query-type text --modalities text,audio --num-prompts 1 --output-dir /tmp/minicpmo-cpu
|
||||
|
||||
/home/zhout/project/edge_infer/.venvs/omni-cpu/bin/python benchmarks/edge_harness/probe_internvla_actions.py \
|
||||
--model-dir /home/zhout/project/edge_infer/models/InternVLA-A1-3B-FT-Place_Markpen \
|
||||
--cosmos-dir /home/zhout/project/edge_infer/models/Cosmos-Tokenizer-CI8x8 \
|
||||
--processor-dir /home/zhout/project/edge_infer/models/Qwen3-VL-2B-Instruct-processor \
|
||||
--device cpu --output /tmp/internvla-cpu.json --actions-output /tmp/internvla-cpu-actions.npy
|
||||
|
||||
/home/zhout/project/edge_infer/.venvs/omni-cpu/bin/python benchmarks/edge_harness/profile_minicpmo_text_speech.py \
|
||||
--model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5 \
|
||||
--deploy-config benchmarks/edge_harness/results/e2e_expansion_20260923/evidence/minicpmo/minicpmo_cpu_offline.yaml \
|
||||
--warmups 1 --repeats 20 --output /tmp/minicpmo-cpu-profile-20.json
|
||||
```
|
||||
|
||||
For native Windows InternVLA, set `PYTHONUTF8=1`, put this checkout on `PYTHONPATH`, and run `benchmarks/edge_harness/probe_internvla_actions.py` with the same model/processor paths as above but Windows-accessible UNC paths; use `--device cpu` or `--device cuda` and separate output paths.
|
||||
BIN
Binary file not shown.
+27
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"scope": "real checkpoint, synthetic zero observations and noise; no task-quality claim",
|
||||
"model_dir": "/home/zhout/project/edge_infer/models/InternVLA-A1-3B-FT-Place_Markpen",
|
||||
"runtime_mode": "real_checkpoint_loaded",
|
||||
"device": "cuda",
|
||||
"dtype": "torch.float32",
|
||||
"action_shape": [
|
||||
1,
|
||||
50,
|
||||
32
|
||||
],
|
||||
"finite": true,
|
||||
"action_sha256_float32": "71ddd577e166f417639dd09f8cdaeef5a1bc91a6152c53672542504328675dd6",
|
||||
"first_action_prefix": [
|
||||
0.02230919525027275,
|
||||
-0.033485047519207,
|
||||
0.005473435390740633,
|
||||
0.02993217296898365,
|
||||
-0.00445316918194294,
|
||||
-0.01604151912033558,
|
||||
0.018104737624526024,
|
||||
-0.09112925827503204
|
||||
],
|
||||
"startup_s": 33.11364810500527,
|
||||
"forward_s": 1.3273328139912337,
|
||||
"cuda_peak_allocated_bytes": 6683068416
|
||||
}
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
INFO 09-23 15:32:12 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 15:32:13 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 15:32:13 [patch.py:583] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
|
||||
INFO 09-23 15:32:13 [pipeline_internvla_a1.py:164] Loading InternVLA-A1 weights from /home/zhout/project/edge_infer/models/InternVLA-A1-3B-FT-Place_Markpen
|
||||
{
|
||||
"scope": "real checkpoint, synthetic zero observations and noise; no task-quality claim",
|
||||
"model_dir": "/home/zhout/project/edge_infer/models/InternVLA-A1-3B-FT-Place_Markpen",
|
||||
"runtime_mode": "real_checkpoint_loaded",
|
||||
"device": "cuda",
|
||||
"dtype": "torch.float32",
|
||||
"action_shape": [
|
||||
1,
|
||||
50,
|
||||
32
|
||||
],
|
||||
"finite": true,
|
||||
"action_sha256_float32": "71ddd577e166f417639dd09f8cdaeef5a1bc91a6152c53672542504328675dd6",
|
||||
"first_action_prefix": [
|
||||
0.02230919525027275,
|
||||
-0.033485047519207,
|
||||
0.005473435390740633,
|
||||
0.02993217296898365,
|
||||
-0.00445316918194294,
|
||||
-0.01604151912033558,
|
||||
0.018104737624526024,
|
||||
-0.09112925827503204
|
||||
],
|
||||
"startup_s": 33.11364810500527,
|
||||
"forward_s": 1.3273328139912337,
|
||||
"cuda_peak_allocated_bytes": 6683068416
|
||||
}
|
||||
BIN
Binary file not shown.
+14
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"scope": "same Place_Markpen checkpoint, zero synthetic observations/noise, BF16 policy; CPU-only torch versus CUDA torch float32 actions",
|
||||
"shape": [
|
||||
1,
|
||||
50,
|
||||
32
|
||||
],
|
||||
"finite": true,
|
||||
"max_abs_error": 0.009932532906532288,
|
||||
"mean_abs_error": 0.0011705027022981085,
|
||||
"rmse": 0.0018438395882933343,
|
||||
"relative_l2_error": 0.013024961700113177,
|
||||
"cosine_similarity": 0.9999165799446749
|
||||
}
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"scope": "real checkpoint, synthetic zero observations and noise; no task-quality claim",
|
||||
"model_dir": "/home/zhout/project/edge_infer/models/InternVLA-A1-3B-FT-Place_Markpen",
|
||||
"runtime_mode": "real_checkpoint_loaded",
|
||||
"device": "cpu",
|
||||
"dtype": "torch.float32",
|
||||
"action_shape": [
|
||||
1,
|
||||
50,
|
||||
32
|
||||
],
|
||||
"finite": true,
|
||||
"action_sha256_float32": "db60d6590e242694948f7dcdf873b2fff44819d31b0b178fdad5ec9608d46140",
|
||||
"first_action_prefix": [
|
||||
0.021246271207928658,
|
||||
-0.03424469009041786,
|
||||
0.005475320853292942,
|
||||
0.03046894632279873,
|
||||
-0.004715746268630028,
|
||||
-0.015595180913805962,
|
||||
0.017074640840291977,
|
||||
-0.0920763686299324
|
||||
],
|
||||
"startup_s": 31.779892041988205,
|
||||
"forward_s": 3.247856915986631,
|
||||
"cuda_peak_allocated_bytes": null
|
||||
}
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions:
|
||||
--> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922
|
||||
--> vLLM version 0.28.0
|
||||
This will likely cause compatibility issues.
|
||||
warn_if_misaligned_vllm_version()
|
||||
W0923 15:49:34.288000 4169445 torch/utils/_pytree.py:630] <enum 'KernelPreference'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
W0923 15:49:34.327000 4169445 torch/utils/_pytree.py:630] <enum 'ScaleCalculationMode'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
INFO 09-23 15:49:36 [importing.py:53] Triton is installed but 0 active driver(s) found (expected 1). Disabling Triton to prevent runtime errors.
|
||||
WARNING 09-23 15:49:36 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton.
|
||||
INFO 09-23 15:49:36 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available.
|
||||
INFO 09-23 15:49:37 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 15:49:37 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 15:49:37 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled.
|
||||
INFO 09-23 15:49:37 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable).
|
||||
INFO 09-23 15:49:38 [pipeline_internvla_a1.py:164] Loading InternVLA-A1 weights from /home/zhout/project/edge_infer/models/InternVLA-A1-3B-FT-Place_Markpen
|
||||
{
|
||||
"scope": "real checkpoint, synthetic zero observations and noise; no task-quality claim",
|
||||
"model_dir": "/home/zhout/project/edge_infer/models/InternVLA-A1-3B-FT-Place_Markpen",
|
||||
"runtime_mode": "real_checkpoint_loaded",
|
||||
"device": "cpu",
|
||||
"dtype": "torch.float32",
|
||||
"action_shape": [
|
||||
1,
|
||||
50,
|
||||
32
|
||||
],
|
||||
"finite": true,
|
||||
"action_sha256_float32": "db60d6590e242694948f7dcdf873b2fff44819d31b0b178fdad5ec9608d46140",
|
||||
"first_action_prefix": [
|
||||
0.021246271207928658,
|
||||
-0.03424469009041786,
|
||||
0.005475320853292942,
|
||||
0.03046894632279873,
|
||||
-0.004715746268630028,
|
||||
-0.015595180913805962,
|
||||
0.017074640840291977,
|
||||
-0.0920763686299324
|
||||
],
|
||||
"startup_s": 31.779892041988205,
|
||||
"forward_s": 3.247856915986631,
|
||||
"cuda_peak_allocated_bytes": null
|
||||
}
|
||||
BIN
Binary file not shown.
+31
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"scope": "real checkpoint, synthetic zero observations and noise; no task-quality claim",
|
||||
"model_dir": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\InternVLA-A1-3B-FT-Place_Markpen",
|
||||
"runtime_mode": "real_checkpoint_loaded",
|
||||
"device": "cpu",
|
||||
"policy_devices": [
|
||||
"cpu"
|
||||
],
|
||||
"actions_device": "cpu",
|
||||
"dtype": "torch.float32",
|
||||
"action_shape": [
|
||||
1,
|
||||
50,
|
||||
32
|
||||
],
|
||||
"finite": true,
|
||||
"action_sha256_float32": "51b01235e490aa4cfc7bf5147bb692a3f7fd7feb6875addeb577e04581686599",
|
||||
"first_action_prefix": [
|
||||
0.021896986290812492,
|
||||
-0.034019432961940765,
|
||||
0.004346882924437523,
|
||||
0.031385887414216995,
|
||||
-0.0046007889322936535,
|
||||
-0.016123272478580475,
|
||||
0.01791001483798027,
|
||||
-0.09050343930721283
|
||||
],
|
||||
"startup_s": 61.65885859995615,
|
||||
"forward_s": 3.363868599990383,
|
||||
"cuda_peak_allocated_bytes": null
|
||||
}
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
INFO 09-23 16:17:04 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 16:17:04 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 16:17:08 [pipeline_internvla_a1.py:164] Loading InternVLA-A1 weights from \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\InternVLA-A1-3B-FT-Place_Markpen
|
||||
{
|
||||
"scope": "real checkpoint, synthetic zero observations and noise; no task-quality claim",
|
||||
"model_dir": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\InternVLA-A1-3B-FT-Place_Markpen",
|
||||
"runtime_mode": "real_checkpoint_loaded",
|
||||
"device": "cpu",
|
||||
"policy_devices": [
|
||||
"cpu"
|
||||
],
|
||||
"actions_device": "cpu",
|
||||
"dtype": "torch.float32",
|
||||
"action_shape": [
|
||||
1,
|
||||
50,
|
||||
32
|
||||
],
|
||||
"finite": true,
|
||||
"action_sha256_float32": "51b01235e490aa4cfc7bf5147bb692a3f7fd7feb6875addeb577e04581686599",
|
||||
"first_action_prefix": [
|
||||
0.021896986290812492,
|
||||
-0.034019432961940765,
|
||||
0.004346882924437523,
|
||||
0.031385887414216995,
|
||||
-0.0046007889322936535,
|
||||
-0.016123272478580475,
|
||||
0.01791001483798027,
|
||||
-0.09050343930721283
|
||||
],
|
||||
"startup_s": 61.65885859995615,
|
||||
"forward_s": 3.363868599990383,
|
||||
"cuda_peak_allocated_bytes": null
|
||||
}
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"candidate": "internvla_windows_cpu_actions.npy",
|
||||
"reference": "internvla_pure_cpu_actions.npy",
|
||||
"shape": [
|
||||
1,
|
||||
50,
|
||||
32
|
||||
],
|
||||
"max_abs_error": 0.01507657766342163,
|
||||
"mean_abs_error": 0.0009196984453774348,
|
||||
"relative_l2_error": 0.010824427638738232,
|
||||
"cosine_similarity": 0.9999416791155367
|
||||
}
|
||||
BIN
Binary file not shown.
+31
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"scope": "real checkpoint, synthetic zero observations and noise; no task-quality claim",
|
||||
"model_dir": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\InternVLA-A1-3B-FT-Place_Markpen",
|
||||
"runtime_mode": "real_checkpoint_loaded",
|
||||
"device": "cuda",
|
||||
"policy_devices": [
|
||||
"cuda"
|
||||
],
|
||||
"actions_device": "cuda:0",
|
||||
"dtype": "torch.float32",
|
||||
"action_shape": [
|
||||
1,
|
||||
50,
|
||||
32
|
||||
],
|
||||
"finite": true,
|
||||
"action_sha256_float32": "873432b6e4cef2b8822f7f5a720fef069c7ba9c6a3f45a831b2cf04239196337",
|
||||
"first_action_prefix": [
|
||||
0.021501213312149048,
|
||||
-0.033786751329898834,
|
||||
0.004070071969181299,
|
||||
0.03125888481736183,
|
||||
-0.004352160729467869,
|
||||
-0.016927657648921013,
|
||||
0.017821842804551125,
|
||||
-0.09145286679267883
|
||||
],
|
||||
"startup_s": 75.53822049999144,
|
||||
"forward_s": 1.1152496999129653,
|
||||
"cuda_peak_allocated_bytes": 6683068416
|
||||
}
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
INFO 09-23 16:15:09 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 16:15:10 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 16:15:14 [pipeline_internvla_a1.py:164] Loading InternVLA-A1 weights from \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\InternVLA-A1-3B-FT-Place_Markpen
|
||||
{
|
||||
"scope": "real checkpoint, synthetic zero observations and noise; no task-quality claim",
|
||||
"model_dir": "\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\InternVLA-A1-3B-FT-Place_Markpen",
|
||||
"runtime_mode": "real_checkpoint_loaded",
|
||||
"device": "cuda",
|
||||
"policy_devices": [
|
||||
"cuda"
|
||||
],
|
||||
"actions_device": "cuda:0",
|
||||
"dtype": "torch.float32",
|
||||
"action_shape": [
|
||||
1,
|
||||
50,
|
||||
32
|
||||
],
|
||||
"finite": true,
|
||||
"action_sha256_float32": "873432b6e4cef2b8822f7f5a720fef069c7ba9c6a3f45a831b2cf04239196337",
|
||||
"first_action_prefix": [
|
||||
0.021501213312149048,
|
||||
-0.033786751329898834,
|
||||
0.004070071969181299,
|
||||
0.03125888481736183,
|
||||
-0.004352160729467869,
|
||||
-0.016927657648921013,
|
||||
0.017821842804551125,
|
||||
-0.09145286679267883
|
||||
],
|
||||
"startup_s": 75.53822049999144,
|
||||
"forward_s": 1.1152496999129653,
|
||||
"cuda_peak_allocated_bytes": 6683068416
|
||||
}
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"candidate": "internvla_windows_gpu_actions.npy",
|
||||
"reference": "internvla_cuda_actions.npy",
|
||||
"shape": [
|
||||
1,
|
||||
50,
|
||||
32
|
||||
],
|
||||
"max_abs_error": 0.008046535775065422,
|
||||
"mean_abs_error": 0.0008131423041140806,
|
||||
"relative_l2_error": 0.008314707744017184,
|
||||
"cosine_similarity": 0.9999661845811505
|
||||
}
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
Prompt:
|
||||
<|im_start|>system
|
||||
You are MiniCPM-o, a helpful multimodal assistant that can understand images, audio and video, and respond in text and speech.<|im_end|>
|
||||
<|im_start|>user
|
||||
Say hello, then introduce vLLM-Omni in one sentence.<|im_end|>
|
||||
<|im_start|>assistant
|
||||
<think>
|
||||
|
||||
</think>
|
||||
|
||||
<|tts_bos|>
|
||||
vllm_text_output:
|
||||
Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video.
|
||||
vllm_token_ids:
|
||||
[9707, 11, 358, 1079, 348, 4086, 44, 23620, 76, 7751, 11, 264, 3460, 79049, 57597, 1614, 12875, 315, 8660, 323, 23163, 1467, 11, 5335, 11, 7699, 11, 323, 2766, 13, 151704, 151645]
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
Prompt:
|
||||
<|im_start|>system
|
||||
You are MiniCPM-o, a helpful multimodal assistant that can understand images, audio and video, and respond in text and speech.<|im_end|>
|
||||
<|im_start|>user
|
||||
Say hello, then introduce vLLM-Omni in one sentence.<|im_end|>
|
||||
<|im_start|>assistant
|
||||
<think>
|
||||
|
||||
</think>
|
||||
|
||||
|
||||
vllm_text_output:
|
||||
Hello! vLLM-Omni is a versatile, high-performance large language model designed for efficient and accurate text generation and understanding.
|
||||
vllm_token_ids:
|
||||
[9707, 0, 348, 4086, 44, 23620, 76, 7751, 374, 264, 31945, 11, 1550, 57474, 3460, 4128, 1614, 6188, 369, 11050, 323, 13382, 1467, 9471, 323, 8660, 13, 151645]
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"scope": "one real-weight MiniCPM-o 4.5 three-stage WSL CPU text-to-speech request; speech meaning unverified",
|
||||
"cpu_text_tokens": 32,
|
||||
"cpu_cuda_text_token_ids_equal": true,
|
||||
"wave_frames": 124800,
|
||||
"wave_sample_rate_hz": 24000,
|
||||
"wave_channels": 1,
|
||||
"wave_sample_width_bytes": 2,
|
||||
"wave_duration_s": 5.2,
|
||||
"wave_nonzero_samples": 124147,
|
||||
"wave_peak_abs_pcm16": 21850,
|
||||
"wave_rms_pcm16": 3319.410484644662,
|
||||
"wave_sha256": "ded62a7285136fe16f28c7e6eb546048a6deb7e385a7a974031032c55df2de40"
|
||||
}
|
||||
+433
@@ -0,0 +1,433 @@
|
||||
W0923 15:36:58.480000 4167421 torch/utils/_pytree.py:630] <enum 'KernelPreference'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
W0923 15:36:58.503000 4167421 torch/utils/_pytree.py:630] <enum 'ScaleCalculationMode'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
WARNING 09-23 15:36:59 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton.
|
||||
INFO 09-23 15:36:59 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available.
|
||||
/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions:
|
||||
--> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922
|
||||
--> vLLM version 0.28.0
|
||||
This will likely cause compatibility issues.
|
||||
warn_if_misaligned_vllm_version()
|
||||
INFO 09-23 15:37:00 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 15:37:00 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 15:37:00 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled.
|
||||
INFO 09-23 15:37:01 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable).
|
||||
====================
|
||||
vllm version: 0.28.0
|
||||
====================
|
||||
INFO 09-23 15:37:01 [omni_base.py:216] [Omni] Initializing with model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5
|
||||
INFO 09-23 15:37:01 [async_omni_engine.py:172] [AsyncOmniEngine] Initializing with model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5
|
||||
INFO 09-23 15:37:01 [async_omni_engine.py:290] [AsyncOmniEngine] Launching Orchestrator thread with 3 stages
|
||||
INFO 09-23 15:37:01 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
INFO 09-23 15:37:01 [initialization.py:356] Loaded OmniTransferConfig with 1 connector configurations
|
||||
WARNING 09-23 15:37:01 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 15:37:01 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
WARNING 09-23 15:37:01 [arg_utils.py:1719] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM.
|
||||
INFO 09-23 15:37:01 [model.py:672] Resolved architecture: MiniCPMO45OmniForConditionalGeneration
|
||||
INFO 09-23 15:37:01 [model.py:1965] Using max model len 2048
|
||||
WARNING 09-23 15:37:01 [model.py:963] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False.
|
||||
INFO 09-23 15:37:02 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode.
|
||||
INFO 09-23 15:37:02 [model.py:875] Disabled mm_prefix attention mode because multimodal inputs are configuration-disabled. Attention backends without mm_prefix support may now be selected.
|
||||
INFO 09-23 15:37:02 [scheduler.py:242] Chunked prefill is enabled with max_num_batched_tokens=2048.
|
||||
WARNING 09-23 15:37:02 [vllm.py:1285] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
WARNING 09-23 15:37:02 [vllm.py:1354] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
INFO 09-23 15:37:02 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
WARNING 09-23 15:37:02 [vllm.py:652] Model Runner V2 requires Triton; using the V1 model runner instead.
|
||||
INFO 09-23 15:37:02 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
WARNING 09-23 15:37:02 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 15:37:02 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
WARNING 09-23 15:37:02 [arg_utils.py:1719] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM.
|
||||
INFO 09-23 15:37:02 [model.py:672] Resolved architecture: MiniCPMO45OmniForConditionalGeneration
|
||||
INFO 09-23 15:37:02 [model.py:1965] Using max model len 512
|
||||
WARNING 09-23 15:37:02 [model.py:963] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False.
|
||||
INFO 09-23 15:37:02 [model.py:1965] Using max model len 512
|
||||
INFO 09-23 15:37:02 [scheduler.py:242] Chunked prefill is enabled with max_num_batched_tokens=512.
|
||||
INFO 09-23 15:37:02 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
WARNING 09-23 15:37:02 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 15:37:02 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
WARNING 09-23 15:37:02 [arg_utils.py:1719] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM.
|
||||
INFO 09-23 15:37:02 [model.py:672] Resolved architecture: MiniCPMO45Code2Wav
|
||||
INFO 09-23 15:37:02 [model.py:1965] Using max model len 4096
|
||||
WARNING 09-23 15:37:02 [arg_utils.py:2664] This model does not officially support disabling chunked prefill. Disabling this manually may cause the engine to crash or produce incorrect outputs.
|
||||
INFO 09-23 15:37:02 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
INFO 09-23 15:37:02 [stage_runtime.py:853] [stage_init] Stage-0 set runtime devices: 0
|
||||
W0923 15:37:06.286000 4167451 torch/utils/_pytree.py:630] <enum 'KernelPreference'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
W0923 15:37:06.314000 4167451 torch/utils/_pytree.py:630] <enum 'ScaleCalculationMode'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
WARNING 09-23 15:37:07 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton.
|
||||
INFO 09-23 15:37:07 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available.
|
||||
/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions:
|
||||
--> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922
|
||||
--> vLLM version 0.28.0
|
||||
This will likely cause compatibility issues.
|
||||
warn_if_misaligned_vllm_version()
|
||||
INFO 09-23 15:37:08 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 15:37:08 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 15:37:08 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled.
|
||||
INFO 09-23 15:37:08 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable).
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167451) INFO 09-23 15:37:09 [core.py:122] Initializing a V1 LLM engine (v0.28.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=2048, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=True, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cpu, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': None, 'compile_ranges_endpoints': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': None, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167451) INFO 09-23 15:37:09 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode.
|
||||
[W923 15:37:09.092897341 utils.cpp:68] Warning: NUMA binding: Using MEMBIND policy for memory allocation on the NUMA nodes (0). Memory allocations will be strictly bound to these NUMA nodes. (function init_cpu_memory_env)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167451) WARNING 09-23 15:37:09 [vllm.py:652] Model Runner V2 requires Triton; using the V1 model runner instead.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167451) WARNING 09-23 15:37:09 [cpu_worker.py:127] libiomp is not found in LD_PRELOAD. For best performance, please follow the section `set LD_PRELOAD` in https://docs.vllm.ai/en/latest/getting_started/installation/cpu/ to setup required pre-loaded libraries.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167451) INFO 09-23 15:37:09 [parallel_state.py:1638] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_a31cf537bd4d45dea02606fe108bbbce backend=gloo
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167451) INFO 09-23 15:37:09 [parallel_state.py:1982] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167451) INFO 09-23 15:37:09 [cpu_model_runner.py:131] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5...
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167451) WARNING 09-23 15:37:09 [vllm.py:1285] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167451) WARNING 09-23 15:37:09 [vllm.py:1354] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167451) INFO 09-23 15:37:09 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167451) INFO 09-23 15:37:09 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167451) INFO 09-23 15:37:09 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167451) INFO 09-23 15:37:09 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167451) INFO 09-23 15:37:10 [selector.py:217] Using HND KV cache layout for CPU_ATTN backend.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167451) INFO 09-23 15:37:13 [weight_utils.py:858] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 27.02 GiB.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167451) INFO 09-23 15:37:13 [weight_utils.py:881] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167451) ␍Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167451) ␍Loading safetensors checkpoint shards: 25% Completed | 1/4 [00:00<00:01, 2.14it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167451) ␍Loading safetensors checkpoint shards: 50% Completed | 2/4 [00:01<00:01, 1.43it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167451) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:02<00:00, 1.61it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167451) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:02<00:00, 1.61it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167451)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167451) INFO 09-23 15:37:27 [default_loader.py:430] Loading weights took 13.38 seconds
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167451) INFO 09-23 15:37:34 [memory.py:91] Released 511 MiB after weight load (anonymous RSS 19157 -> 18646 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory']
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167451) INFO 09-23 15:37:35 [cpu_worker.py:255] Explicitly set (1.0/30.91) GiB for KV cache on node 0.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167451) INFO 09-23 15:37:35 [kv_cache_utils.py:1869] GPU KV cache size: 7,168 tokens, Maximum concurrency for 2,048 tokens per request: 3.50x
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167451) INFO 09-23 15:37:35 [memory.py:91] Released 0 MiB after warm-up (anonymous RSS 19661 -> 19661 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory']
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167451) INFO 09-23 15:37:35 [core.py:354] init engine (profile, create kv cache, warmup model) took 0.44 s
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167451) WARNING 09-23 15:37:35 [scheduler.py:182] Using custom scheduler class vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167451) INFO 09-23 15:37:35 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167451) INFO 09-23 15:37:46 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
INFO 09-23 15:37:46 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed
|
||||
INFO 09-23 15:37:46 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-0 [rep-0] initializing EngineCore
|
||||
INFO 09-23 15:37:46 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-0 [rep-0] EngineCore running
|
||||
INFO 09-23 15:37:46 [stage_runtime.py:958] [StageRuntime] Stage 0 initialized
|
||||
INFO 09-23 15:37:46 [stage_runtime.py:853] [stage_init] Stage-1 set runtime devices: 0
|
||||
W0923 15:37:53.025000 4167571 torch/utils/_pytree.py:630] <enum 'KernelPreference'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
W0923 15:37:53.066000 4167571 torch/utils/_pytree.py:630] <enum 'ScaleCalculationMode'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
WARNING 09-23 15:37:54 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton.
|
||||
INFO 09-23 15:37:54 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available.
|
||||
/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions:
|
||||
--> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922
|
||||
--> vLLM version 0.28.0
|
||||
This will likely cause compatibility issues.
|
||||
warn_if_misaligned_vllm_version()
|
||||
INFO 09-23 15:37:55 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 15:37:56 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 15:37:56 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled.
|
||||
INFO 09-23 15:37:56 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable).
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) INFO 09-23 15:37:56 [core.py:122] Initializing a V1 LLM engine (v0.28.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=512, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=True, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cpu, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': None, 'compile_ranges_endpoints': [512], 'inductor_compile_config': {'enable_auto_functionalized_v2': False}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': None, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
[W923 15:37:57.845305821 utils.cpp:68] Warning: NUMA binding: Using MEMBIND policy for memory allocation on the NUMA nodes (0). Memory allocations will be strictly bound to these NUMA nodes. (function init_cpu_memory_env)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) WARNING 09-23 15:37:57 [vllm.py:652] Model Runner V2 requires Triton; using the V1 model runner instead.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) WARNING 09-23 15:37:57 [cpu_worker.py:127] libiomp is not found in LD_PRELOAD. For best performance, please follow the section `set LD_PRELOAD` in https://docs.vllm.ai/en/latest/getting_started/installation/cpu/ to setup required pre-loaded libraries.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) INFO 09-23 15:37:57 [parallel_state.py:1638] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_ed0f981d16804c67838c632666ad27c7 backend=gloo
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) INFO 09-23 15:37:57 [parallel_state.py:1982] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) WARNING 09-23 15:37:57 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) [transformers] `MiniCPMOProcessor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) [transformers] Requested torchvision backend is not available. Falling back to pil backend.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) [transformers] `MiniCPMOProcessor` defines `feature_extractor_class = 'WhisperFeatureExtractor'`, which is deprecated. Register the correct mapping in `AutoFeatureExtractor` instead.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) WARNING 09-23 15:38:00 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] StageEngineCoreProc failed to start.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] Traceback (most recent call last):
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_core_proc.py", line 189, in run_stage_core
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] engine_core = StageEngineCoreProc(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] return func(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/core.py", line 1073, in __init__
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] super().__init__(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/core.py", line 133, in __init__
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] self.model_executor = executor_class(vllm_config)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] return func(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/executor/abstract.py", line 110, in __init__
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] self._init_executor()
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/executor/uniproc_executor.py", line 69, in _init_executor
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] self.driver_worker.init_device()
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 335, in init_device
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] self.worker.init_device() # type: ignore
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/init_timeline.py", line 343, in wrapper
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] return fn(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/platforms/cpu/worker/cpu_ar_worker.py", line 30, in init_device
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] super().init_device() # type: ignore[misc]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/cpu_worker.py", line 190, in init_device
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] self.model_runner = CPUModelRunner(self.vllm_config, torch.device("cpu"))
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/cpu_model_runner.py", line 31, in __init__
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] super().__init__(vllm_config, device)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/gpu_model_runner.py", line 915, in __init__
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] MultiModalBudget(self.vllm_config, self.mm_registry)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/encoder_budget.py", line 94, in __init__
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] all_mm_max_toks_per_item = get_mm_max_toks_per_item(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/encoder_budget.py", line 33, in get_mm_max_toks_per_item
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] mm_inputs = mm_registry.get_dummy_mm_inputs(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/registry.py", line 263, in get_dummy_mm_inputs
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] mm_inputs = processor.apply(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py", line 3419, in apply
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] return super().apply(inputs, timing_ctx)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/processing/processor.py", line 1849, in apply
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] ) = self._cached_apply_hf_processor(inputs, timing_ctx)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/processing/processor.py", line 1628, in _cached_apply_hf_processor
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] ) = self._apply_hf_processor_main(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ERROR 09-23 15:38:00 [stage_engine_core_proc.py:242] TypeError: MiniCPMO45OmniLLMMultiModalProcessor._apply_hf_processor_main() got an unexpected keyword argument 'prompt'
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) Process StageEngineCoreProc_stage1_replica0:
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) Traceback (most recent call last):
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/multiprocessing/process.py", line 314, in _bootstrap
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) self.run()
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/multiprocessing/process.py", line 108, in run
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) self._target(*self._args, **self._kwargs)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_core_proc.py", line 189, in run_stage_core
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) engine_core = StageEngineCoreProc(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) return func(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/core.py", line 1073, in __init__
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) super().__init__(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/core.py", line 133, in __init__
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) self.model_executor = executor_class(vllm_config)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) return func(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/executor/abstract.py", line 110, in __init__
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) self._init_executor()
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/executor/uniproc_executor.py", line 69, in _init_executor
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) self.driver_worker.init_device()
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 335, in init_device
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) self.worker.init_device() # type: ignore
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/init_timeline.py", line 343, in wrapper
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) return fn(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/platforms/cpu/worker/cpu_ar_worker.py", line 30, in init_device
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) super().init_device() # type: ignore[misc]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/cpu_worker.py", line 190, in init_device
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) self.model_runner = CPUModelRunner(self.vllm_config, torch.device("cpu"))
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/cpu_model_runner.py", line 31, in __init__
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) super().__init__(vllm_config, device)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/gpu_model_runner.py", line 915, in __init__
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) MultiModalBudget(self.vllm_config, self.mm_registry)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/encoder_budget.py", line 94, in __init__
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) all_mm_max_toks_per_item = get_mm_max_toks_per_item(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/encoder_budget.py", line 33, in get_mm_max_toks_per_item
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) mm_inputs = mm_registry.get_dummy_mm_inputs(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/registry.py", line 263, in get_dummy_mm_inputs
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) mm_inputs = processor.apply(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py", line 3419, in apply
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) return super().apply(inputs, timing_ctx)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/processing/processor.py", line 1849, in apply
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ) = self._cached_apply_hf_processor(inputs, timing_ctx)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/processing/processor.py", line 1628, in _cached_apply_hf_processor
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ) = self._apply_hf_processor_main(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167571) TypeError: MiniCPMO45OmniLLMMultiModalProcessor._apply_hf_processor_main() got an unexpected keyword argument 'prompt'
|
||||
ERROR 09-23 15:38:03 [stage_runtime.py:265] [StageRuntime] Stage initialization failed; shutting down 1 initialized client(s)
|
||||
ERROR 09-23 15:38:03 [stage_runtime.py:265] Traceback (most recent call last):
|
||||
ERROR 09-23 15:38:03 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 252, in initialize
|
||||
ERROR 09-23 15:38:03 [stage_runtime.py:265] initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout)
|
||||
ERROR 09-23 15:38:03 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 15:38:03 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 589, in _initialize_stage_replicas
|
||||
ERROR 09-23 15:38:03 [stage_runtime.py:265] raise primary_exc
|
||||
ERROR 09-23 15:38:03 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 550, in _init_group
|
||||
ERROR 09-23 15:38:03 [stage_runtime.py:265] client = self._initialize_replica(
|
||||
ERROR 09-23 15:38:03 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 15:38:03 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 828, in _initialize_replica
|
||||
ERROR 09-23 15:38:03 [stage_runtime.py:265] return self._initialize_local_llm_replica(plan, stage_init_timeout)
|
||||
ERROR 09-23 15:38:03 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 15:38:03 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 934, in _initialize_local_llm_replica
|
||||
ERROR 09-23 15:38:03 [stage_runtime.py:265] launch_stack.close()
|
||||
ERROR 09-23 15:38:03 [stage_runtime.py:265] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 618, in close
|
||||
ERROR 09-23 15:38:03 [stage_runtime.py:265] self.__exit__(None, None, None)
|
||||
ERROR 09-23 15:38:03 [stage_runtime.py:265] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 610, in __exit__
|
||||
ERROR 09-23 15:38:03 [stage_runtime.py:265] raise exc_details[1]
|
||||
ERROR 09-23 15:38:03 [stage_runtime.py:265] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 595, in __exit__
|
||||
ERROR 09-23 15:38:03 [stage_runtime.py:265] if cb(*exc_details):
|
||||
ERROR 09-23 15:38:03 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 15:38:03 [stage_runtime.py:265] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 144, in __exit__
|
||||
ERROR 09-23 15:38:03 [stage_runtime.py:265] next(self.gen)
|
||||
ERROR 09-23 15:38:03 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_startup.py", line 1211, in launch_stage_replica
|
||||
ERROR 09-23 15:38:03 [stage_runtime.py:265] wait_for_engine_startup(
|
||||
ERROR 09-23 15:38:03 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/utils.py", line 1286, in wait_for_engine_startup
|
||||
ERROR 09-23 15:38:03 [stage_runtime.py:265] raise RuntimeError(
|
||||
ERROR 09-23 15:38:03 [stage_runtime.py:265] RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {}
|
||||
INFO 09-23 15:38:03 [core_client.py:689] [shutdown] MPClient: start timeout=default
|
||||
INFO 09-23 15:38:03 [core_client.py:691] [shutdown] MPClient: stopping engine manager
|
||||
INFO 09-23 15:38:03 [utils.py:615] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage0_replica0
|
||||
WARNING 09-23 15:38:08 [utils.py:635] [shutdown] Process manager: force killing remaining processes count=1
|
||||
WARNING 09-23 15:38:08 [utils.py:640] [shutdown] Process manager: force killing remaining process StageEngineCoreProc_stage0_replica0 pid 4167451
|
||||
INFO 09-23 15:38:08 [core_client.py:693] [shutdown] MPClient: engine manager stopped
|
||||
INFO 09-23 15:38:08 [core_client.py:694] [shutdown] MPClient: cleaning up background resources
|
||||
INFO 09-23 15:38:08 [core_client.py:696] [shutdown] MPClient: complete
|
||||
ERROR 09-23 15:38:08 [async_omni_engine.py:474] [AsyncOmniEngine] Orchestrator thread crashed
|
||||
ERROR 09-23 15:38:08 [async_omni_engine.py:474] Traceback (most recent call last):
|
||||
ERROR 09-23 15:38:08 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 468, in _bootstrap_orchestrator
|
||||
ERROR 09-23 15:38:08 [async_omni_engine.py:474] loop.run_until_complete(_run_orchestrator())
|
||||
ERROR 09-23 15:38:08 [async_omni_engine.py:474] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/base_events.py", line 691, in run_until_complete
|
||||
ERROR 09-23 15:38:08 [async_omni_engine.py:474] return future.result()
|
||||
ERROR 09-23 15:38:08 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 15:38:08 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 424, in _run_orchestrator
|
||||
ERROR 09-23 15:38:08 [async_omni_engine.py:474] self._initialize_stages(stage_init_timeout)
|
||||
ERROR 09-23 15:38:08 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 350, in _initialize_stages
|
||||
ERROR 09-23 15:38:08 [async_omni_engine.py:474] self._initialize_stages_impl(stage_init_timeout)
|
||||
ERROR 09-23 15:38:08 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 371, in _initialize_stages_impl
|
||||
ERROR 09-23 15:38:08 [async_omni_engine.py:474] self._runtime.initialize()
|
||||
ERROR 09-23 15:38:08 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 278, in initialize
|
||||
ERROR 09-23 15:38:08 [async_omni_engine.py:474] raise exc
|
||||
ERROR 09-23 15:38:08 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 252, in initialize
|
||||
ERROR 09-23 15:38:08 [async_omni_engine.py:474] initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout)
|
||||
ERROR 09-23 15:38:08 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 15:38:08 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 589, in _initialize_stage_replicas
|
||||
ERROR 09-23 15:38:08 [async_omni_engine.py:474] raise primary_exc
|
||||
ERROR 09-23 15:38:08 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 550, in _init_group
|
||||
ERROR 09-23 15:38:08 [async_omni_engine.py:474] client = self._initialize_replica(
|
||||
ERROR 09-23 15:38:08 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 15:38:08 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 828, in _initialize_replica
|
||||
ERROR 09-23 15:38:08 [async_omni_engine.py:474] return self._initialize_local_llm_replica(plan, stage_init_timeout)
|
||||
ERROR 09-23 15:38:08 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 15:38:08 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 934, in _initialize_local_llm_replica
|
||||
ERROR 09-23 15:38:08 [async_omni_engine.py:474] launch_stack.close()
|
||||
ERROR 09-23 15:38:08 [async_omni_engine.py:474] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 618, in close
|
||||
ERROR 09-23 15:38:08 [async_omni_engine.py:474] self.__exit__(None, None, None)
|
||||
ERROR 09-23 15:38:08 [async_omni_engine.py:474] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 610, in __exit__
|
||||
ERROR 09-23 15:38:08 [async_omni_engine.py:474] raise exc_details[1]
|
||||
ERROR 09-23 15:38:08 [async_omni_engine.py:474] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 595, in __exit__
|
||||
ERROR 09-23 15:38:08 [async_omni_engine.py:474] if cb(*exc_details):
|
||||
ERROR 09-23 15:38:08 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 15:38:08 [async_omni_engine.py:474] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 144, in __exit__
|
||||
ERROR 09-23 15:38:08 [async_omni_engine.py:474] next(self.gen)
|
||||
ERROR 09-23 15:38:08 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_startup.py", line 1211, in launch_stage_replica
|
||||
ERROR 09-23 15:38:08 [async_omni_engine.py:474] wait_for_engine_startup(
|
||||
ERROR 09-23 15:38:08 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/utils.py", line 1286, in wait_for_engine_startup
|
||||
ERROR 09-23 15:38:08 [async_omni_engine.py:474] raise RuntimeError(
|
||||
ERROR 09-23 15:38:08 [async_omni_engine.py:474] RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {}
|
||||
INFO 09-23 15:38:08 [async_omni_engine.py:1807] [AsyncOmniEngine] Shutting down Orchestrator
|
||||
Exception in thread orchestrator:
|
||||
Traceback (most recent call last):
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
|
||||
self.run()
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/threading.py", line 1012, in run
|
||||
self._target(*self._args, **self._kwargs)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 468, in _bootstrap_orchestrator
|
||||
loop.run_until_complete(_run_orchestrator())
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/base_events.py", line 691, in run_until_complete
|
||||
return future.result()
|
||||
^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 424, in _run_orchestrator
|
||||
self._initialize_stages(stage_init_timeout)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 350, in _initialize_stages
|
||||
self._initialize_stages_impl(stage_init_timeout)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 371, in _initialize_stages_impl
|
||||
self._runtime.initialize()
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 278, in initialize
|
||||
raise exc
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 252, in initialize
|
||||
initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 589, in _initialize_stage_replicas
|
||||
raise primary_exc
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 550, in _init_group
|
||||
client = self._initialize_replica(
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 828, in _initialize_replica
|
||||
return self._initialize_local_llm_replica(plan, stage_init_timeout)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 934, in _initialize_local_llm_replica
|
||||
launch_stack.close()
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 618, in close
|
||||
self.__exit__(None, None, None)
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 610, in __exit__
|
||||
raise exc_details[1]
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 595, in __exit__
|
||||
if cb(*exc_details):
|
||||
^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 144, in __exit__
|
||||
next(self.gen)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_startup.py", line 1211, in launch_stage_replica
|
||||
wait_for_engine_startup(
|
||||
File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/utils.py", line 1286, in wait_for_engine_startup
|
||||
raise RuntimeError(
|
||||
RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {}
|
||||
Traceback (most recent call last):
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
|
||||
self.run()
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/threading.py", line 1012, in run
|
||||
self._target(*self._args, **self._kwargs)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 468, in _bootstrap_orchestrator
|
||||
loop.run_until_complete(_run_orchestrator())
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/base_events.py", line 691, in run_until_complete
|
||||
return future.result()
|
||||
^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 424, in _run_orchestrator
|
||||
self._initialize_stages(stage_init_timeout)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 350, in _initialize_stages
|
||||
self._initialize_stages_impl(stage_init_timeout)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 371, in _initialize_stages_impl
|
||||
self._runtime.initialize()
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 278, in initialize
|
||||
raise exc
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 252, in initialize
|
||||
initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 589, in _initialize_stage_replicas
|
||||
raise primary_exc
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 550, in _init_group
|
||||
client = self._initialize_replica(
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 828, in _initialize_replica
|
||||
return self._initialize_local_llm_replica(plan, stage_init_timeout)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 934, in _initialize_local_llm_replica
|
||||
launch_stack.close()
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 618, in close
|
||||
self.__exit__(None, None, None)
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 610, in __exit__
|
||||
raise exc_details[1]
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 595, in __exit__
|
||||
if cb(*exc_details):
|
||||
^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 144, in __exit__
|
||||
next(self.gen)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_startup.py", line 1211, in launch_stage_replica
|
||||
wait_for_engine_startup(
|
||||
File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/utils.py", line 1286, in wait_for_engine_startup
|
||||
raise RuntimeError(
|
||||
RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {}
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
|
||||
Traceback (most recent call last):
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/examples/offline_inference/minicpmo/end2end.py", line 519, in <module>
|
||||
main(args)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/examples/offline_inference/minicpmo/end2end.py", line 283, in main
|
||||
omni = Omni(**omni_kwargs)
|
||||
^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/entrypoints/omni_base.py", line 223, in __init__
|
||||
self.engine = AsyncOmniEngine(
|
||||
^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 305, in __init__
|
||||
self._wait_for_orchestrator_init(startup_future, startup_timeout)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 539, in _wait_for_orchestrator_init
|
||||
startup_future.result(
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/concurrent/futures/_base.py", line 456, in result
|
||||
return self.__get_result()
|
||||
^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/concurrent/futures/_base.py", line 401, in __get_result
|
||||
raise self._exception
|
||||
RuntimeError: Orchestrator initialization failed: Engine core initialization failed. See root cause above. Failed core proc(s): {}
|
||||
+456
@@ -0,0 +1,456 @@
|
||||
W0923 15:39:14.030000 4167733 torch/utils/_pytree.py:630] <enum 'KernelPreference'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
W0923 15:39:14.055000 4167733 torch/utils/_pytree.py:630] <enum 'ScaleCalculationMode'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
WARNING 09-23 15:39:14 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton.
|
||||
INFO 09-23 15:39:14 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available.
|
||||
/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions:
|
||||
--> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922
|
||||
--> vLLM version 0.28.0
|
||||
This will likely cause compatibility issues.
|
||||
warn_if_misaligned_vllm_version()
|
||||
INFO 09-23 15:39:15 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 15:39:16 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 15:39:16 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled.
|
||||
INFO 09-23 15:39:16 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable).
|
||||
====================
|
||||
vllm version: 0.28.0
|
||||
====================
|
||||
INFO 09-23 15:39:16 [omni_base.py:216] [Omni] Initializing with model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5
|
||||
INFO 09-23 15:39:16 [async_omni_engine.py:172] [AsyncOmniEngine] Initializing with model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5
|
||||
INFO 09-23 15:39:16 [async_omni_engine.py:290] [AsyncOmniEngine] Launching Orchestrator thread with 3 stages
|
||||
INFO 09-23 15:39:16 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
INFO 09-23 15:39:16 [initialization.py:356] Loaded OmniTransferConfig with 1 connector configurations
|
||||
WARNING 09-23 15:39:16 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 15:39:16 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
WARNING 09-23 15:39:16 [arg_utils.py:1719] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM.
|
||||
INFO 09-23 15:39:16 [model.py:672] Resolved architecture: MiniCPMO45OmniForConditionalGeneration
|
||||
INFO 09-23 15:39:16 [model.py:1965] Using max model len 2048
|
||||
WARNING 09-23 15:39:16 [model.py:963] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False.
|
||||
INFO 09-23 15:39:17 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode.
|
||||
INFO 09-23 15:39:17 [model.py:875] Disabled mm_prefix attention mode because multimodal inputs are configuration-disabled. Attention backends without mm_prefix support may now be selected.
|
||||
INFO 09-23 15:39:17 [scheduler.py:242] Chunked prefill is enabled with max_num_batched_tokens=2048.
|
||||
WARNING 09-23 15:39:17 [vllm.py:1285] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
WARNING 09-23 15:39:17 [vllm.py:1354] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
INFO 09-23 15:39:17 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
WARNING 09-23 15:39:17 [vllm.py:652] Model Runner V2 requires Triton; using the V1 model runner instead.
|
||||
INFO 09-23 15:39:17 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
WARNING 09-23 15:39:17 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 15:39:17 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
WARNING 09-23 15:39:17 [arg_utils.py:1719] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM.
|
||||
INFO 09-23 15:39:17 [model.py:672] Resolved architecture: MiniCPMO45OmniForConditionalGeneration
|
||||
INFO 09-23 15:39:17 [model.py:1965] Using max model len 512
|
||||
WARNING 09-23 15:39:17 [model.py:963] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False.
|
||||
INFO 09-23 15:39:17 [model.py:1965] Using max model len 512
|
||||
INFO 09-23 15:39:17 [scheduler.py:242] Chunked prefill is enabled with max_num_batched_tokens=512.
|
||||
INFO 09-23 15:39:17 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
WARNING 09-23 15:39:17 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 15:39:17 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
WARNING 09-23 15:39:17 [arg_utils.py:1719] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM.
|
||||
INFO 09-23 15:39:17 [model.py:672] Resolved architecture: MiniCPMO45Code2Wav
|
||||
INFO 09-23 15:39:17 [model.py:1965] Using max model len 4096
|
||||
WARNING 09-23 15:39:17 [arg_utils.py:2664] This model does not officially support disabling chunked prefill. Disabling this manually may cause the engine to crash or produce incorrect outputs.
|
||||
INFO 09-23 15:39:17 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
INFO 09-23 15:39:17 [stage_runtime.py:853] [stage_init] Stage-0 set runtime devices: 0
|
||||
W0923 15:39:21.181000 4167762 torch/utils/_pytree.py:630] <enum 'KernelPreference'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
W0923 15:39:21.207000 4167762 torch/utils/_pytree.py:630] <enum 'ScaleCalculationMode'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
WARNING 09-23 15:39:22 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton.
|
||||
INFO 09-23 15:39:22 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available.
|
||||
/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions:
|
||||
--> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922
|
||||
--> vLLM version 0.28.0
|
||||
This will likely cause compatibility issues.
|
||||
warn_if_misaligned_vllm_version()
|
||||
INFO 09-23 15:39:23 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 15:39:23 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 15:39:23 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled.
|
||||
INFO 09-23 15:39:23 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable).
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167762) INFO 09-23 15:39:23 [core.py:122] Initializing a V1 LLM engine (v0.28.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=2048, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=True, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cpu, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': None, 'compile_ranges_endpoints': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': None, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167762) INFO 09-23 15:39:24 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode.
|
||||
[W923 15:39:24.921595339 utils.cpp:68] Warning: NUMA binding: Using MEMBIND policy for memory allocation on the NUMA nodes (0). Memory allocations will be strictly bound to these NUMA nodes. (function init_cpu_memory_env)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167762) WARNING 09-23 15:39:24 [vllm.py:652] Model Runner V2 requires Triton; using the V1 model runner instead.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167762) WARNING 09-23 15:39:24 [cpu_worker.py:127] libiomp is not found in LD_PRELOAD. For best performance, please follow the section `set LD_PRELOAD` in https://docs.vllm.ai/en/latest/getting_started/installation/cpu/ to setup required pre-loaded libraries.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167762) INFO 09-23 15:39:24 [parallel_state.py:1638] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_af47bac558064cc0b15208cedc8aabf6 backend=gloo
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167762) INFO 09-23 15:39:24 [parallel_state.py:1982] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167762) INFO 09-23 15:39:24 [cpu_model_runner.py:131] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5...
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167762) WARNING 09-23 15:39:24 [vllm.py:1285] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167762) WARNING 09-23 15:39:24 [vllm.py:1354] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167762) INFO 09-23 15:39:24 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167762) INFO 09-23 15:39:24 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167762) INFO 09-23 15:39:24 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167762) INFO 09-23 15:39:24 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167762) INFO 09-23 15:39:24 [selector.py:217] Using HND KV cache layout for CPU_ATTN backend.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167762) INFO 09-23 15:39:28 [weight_utils.py:858] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 27.03 GiB.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167762) INFO 09-23 15:39:28 [weight_utils.py:881] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167762) ␍Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167762) ␍Loading safetensors checkpoint shards: 25% Completed | 1/4 [00:01<00:03, 1.04s/it]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167762) ␍Loading safetensors checkpoint shards: 50% Completed | 2/4 [00:02<00:02, 1.10s/it]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167762) ␍Loading safetensors checkpoint shards: 75% Completed | 3/4 [00:03<00:01, 1.14s/it]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167762) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:04<00:00, 1.24s/it]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167762) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:04<00:00, 1.19s/it]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167762)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167762) INFO 09-23 15:39:40 [default_loader.py:430] Loading weights took 11.63 seconds
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167762) INFO 09-23 15:39:43 [memory.py:91] Released 512 MiB after weight load (anonymous RSS 19805 -> 19293 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory']
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167762) INFO 09-23 15:39:43 [cpu_worker.py:255] Explicitly set (1.0/30.91) GiB for KV cache on node 0.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167762) INFO 09-23 15:39:43 [kv_cache_utils.py:1869] GPU KV cache size: 7,168 tokens, Maximum concurrency for 2,048 tokens per request: 3.50x
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167762) INFO 09-23 15:39:43 [memory.py:91] Released 0 MiB after warm-up (anonymous RSS 20303 -> 20303 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory']
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167762) INFO 09-23 15:39:43 [core.py:354] init engine (profile, create kv cache, warmup model) took 0.16 s
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167762) WARNING 09-23 15:39:43 [scheduler.py:182] Using custom scheduler class vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167762) INFO 09-23 15:39:43 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167762) INFO 09-23 15:39:43 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
INFO 09-23 15:39:43 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed
|
||||
INFO 09-23 15:39:43 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-0 [rep-0] initializing EngineCore
|
||||
INFO 09-23 15:39:44 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-0 [rep-0] EngineCore running
|
||||
INFO 09-23 15:39:44 [stage_runtime.py:958] [StageRuntime] Stage 0 initialized
|
||||
INFO 09-23 15:39:44 [stage_runtime.py:853] [stage_init] Stage-1 set runtime devices: 0
|
||||
W0923 15:39:52.020000 4167858 torch/utils/_pytree.py:630] <enum 'KernelPreference'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
W0923 15:39:52.071000 4167858 torch/utils/_pytree.py:630] <enum 'ScaleCalculationMode'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
WARNING 09-23 15:39:53 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton.
|
||||
INFO 09-23 15:39:53 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available.
|
||||
/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions:
|
||||
--> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922
|
||||
--> vLLM version 0.28.0
|
||||
This will likely cause compatibility issues.
|
||||
warn_if_misaligned_vllm_version()
|
||||
INFO 09-23 15:39:55 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 15:39:55 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 15:39:55 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled.
|
||||
INFO 09-23 15:39:56 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable).
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) INFO 09-23 15:39:56 [core.py:122] Initializing a V1 LLM engine (v0.28.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=512, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=True, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cpu, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': None, 'compile_ranges_endpoints': [512], 'inductor_compile_config': {'enable_auto_functionalized_v2': False}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': None, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
[W923 15:39:57.775880442 utils.cpp:68] Warning: NUMA binding: Using MEMBIND policy for memory allocation on the NUMA nodes (0). Memory allocations will be strictly bound to these NUMA nodes. (function init_cpu_memory_env)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) WARNING 09-23 15:39:57 [vllm.py:652] Model Runner V2 requires Triton; using the V1 model runner instead.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) WARNING 09-23 15:39:57 [cpu_worker.py:127] libiomp is not found in LD_PRELOAD. For best performance, please follow the section `set LD_PRELOAD` in https://docs.vllm.ai/en/latest/getting_started/installation/cpu/ to setup required pre-loaded libraries.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) INFO 09-23 15:39:57 [parallel_state.py:1638] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_c4c7bdcb69a6490196643cc73405e482 backend=gloo
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) INFO 09-23 15:39:57 [parallel_state.py:1982] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) WARNING 09-23 15:39:57 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) [transformers] `MiniCPMOProcessor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) [transformers] Requested torchvision backend is not available. Falling back to pil backend.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) [transformers] `MiniCPMOProcessor` defines `feature_extractor_class = 'WhisperFeatureExtractor'`, which is deprecated. Register the correct mapping in `AutoFeatureExtractor` instead.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) WARNING 09-23 15:40:00 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] StageEngineCoreProc failed to start.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] Traceback (most recent call last):
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_core_proc.py", line 189, in run_stage_core
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] engine_core = StageEngineCoreProc(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] return func(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/core.py", line 1073, in __init__
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] super().__init__(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/core.py", line 133, in __init__
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] self.model_executor = executor_class(vllm_config)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] return func(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/executor/abstract.py", line 110, in __init__
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] self._init_executor()
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/executor/uniproc_executor.py", line 69, in _init_executor
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] self.driver_worker.init_device()
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 335, in init_device
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] self.worker.init_device() # type: ignore
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/init_timeline.py", line 343, in wrapper
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] return fn(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/platforms/cpu/worker/cpu_ar_worker.py", line 30, in init_device
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] super().init_device() # type: ignore[misc]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/cpu_worker.py", line 190, in init_device
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] self.model_runner = CPUModelRunner(self.vllm_config, torch.device("cpu"))
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/cpu_model_runner.py", line 31, in __init__
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] super().__init__(vllm_config, device)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/gpu_model_runner.py", line 915, in __init__
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] MultiModalBudget(self.vllm_config, self.mm_registry)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/encoder_budget.py", line 94, in __init__
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] all_mm_max_toks_per_item = get_mm_max_toks_per_item(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/encoder_budget.py", line 33, in get_mm_max_toks_per_item
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] mm_inputs = mm_registry.get_dummy_mm_inputs(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/registry.py", line 263, in get_dummy_mm_inputs
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] mm_inputs = processor.apply(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py", line 3419, in apply
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] return super().apply(inputs, timing_ctx)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/processing/processor.py", line 1849, in apply
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] ) = self._cached_apply_hf_processor(inputs, timing_ctx)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/processing/processor.py", line 1628, in _cached_apply_hf_processor
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] ) = self._apply_hf_processor_main(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py", line 3440, in _apply_hf_processor_main
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] mm_inputs = self.process_mm_inputs(mm_data, hf_processor_mm_kwargs)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py", line 3595, in process_mm_inputs
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] **self.process_images(mm_data, mm_kwargs),
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py", line 3552, in process_images
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] image_inputs = self._call_hf_processor_on_prompts(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py", line 3610, in _call_hf_processor_on_prompts
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] return MiniCPMVMultiModalProcessor._call_hf_processor_on_prompts(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ERROR 09-23 15:40:00 [stage_engine_core_proc.py:242] AttributeError: type object 'MiniCPMVMultiModalProcessor' has no attribute '_call_hf_processor_on_prompts'
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) Process StageEngineCoreProc_stage1_replica0:
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) Traceback (most recent call last):
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/multiprocessing/process.py", line 314, in _bootstrap
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) self.run()
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/multiprocessing/process.py", line 108, in run
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) self._target(*self._args, **self._kwargs)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_core_proc.py", line 189, in run_stage_core
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) engine_core = StageEngineCoreProc(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) return func(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/core.py", line 1073, in __init__
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) super().__init__(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/core.py", line 133, in __init__
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) self.model_executor = executor_class(vllm_config)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) return func(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/executor/abstract.py", line 110, in __init__
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) self._init_executor()
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/executor/uniproc_executor.py", line 69, in _init_executor
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) self.driver_worker.init_device()
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 335, in init_device
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) self.worker.init_device() # type: ignore
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/init_timeline.py", line 343, in wrapper
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) return fn(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/platforms/cpu/worker/cpu_ar_worker.py", line 30, in init_device
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) super().init_device() # type: ignore[misc]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/cpu_worker.py", line 190, in init_device
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) self.model_runner = CPUModelRunner(self.vllm_config, torch.device("cpu"))
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/cpu_model_runner.py", line 31, in __init__
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) super().__init__(vllm_config, device)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/gpu_model_runner.py", line 915, in __init__
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) MultiModalBudget(self.vllm_config, self.mm_registry)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/encoder_budget.py", line 94, in __init__
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) all_mm_max_toks_per_item = get_mm_max_toks_per_item(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/encoder_budget.py", line 33, in get_mm_max_toks_per_item
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) mm_inputs = mm_registry.get_dummy_mm_inputs(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/registry.py", line 263, in get_dummy_mm_inputs
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) mm_inputs = processor.apply(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py", line 3419, in apply
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) return super().apply(inputs, timing_ctx)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/processing/processor.py", line 1849, in apply
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ) = self._cached_apply_hf_processor(inputs, timing_ctx)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/processing/processor.py", line 1628, in _cached_apply_hf_processor
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ) = self._apply_hf_processor_main(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py", line 3440, in _apply_hf_processor_main
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) mm_inputs = self.process_mm_inputs(mm_data, hf_processor_mm_kwargs)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py", line 3595, in process_mm_inputs
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) **self.process_images(mm_data, mm_kwargs),
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py", line 3552, in process_images
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) image_inputs = self._call_hf_processor_on_prompts(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py", line 3610, in _call_hf_processor_on_prompts
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) return MiniCPMVMultiModalProcessor._call_hf_processor_on_prompts(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4167858) AttributeError: type object 'MiniCPMVMultiModalProcessor' has no attribute '_call_hf_processor_on_prompts'
|
||||
ERROR 09-23 15:40:03 [stage_runtime.py:265] [StageRuntime] Stage initialization failed; shutting down 1 initialized client(s)
|
||||
ERROR 09-23 15:40:03 [stage_runtime.py:265] Traceback (most recent call last):
|
||||
ERROR 09-23 15:40:03 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 252, in initialize
|
||||
ERROR 09-23 15:40:03 [stage_runtime.py:265] initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout)
|
||||
ERROR 09-23 15:40:03 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 15:40:03 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 589, in _initialize_stage_replicas
|
||||
ERROR 09-23 15:40:03 [stage_runtime.py:265] raise primary_exc
|
||||
ERROR 09-23 15:40:03 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 550, in _init_group
|
||||
ERROR 09-23 15:40:03 [stage_runtime.py:265] client = self._initialize_replica(
|
||||
ERROR 09-23 15:40:03 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 15:40:03 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 828, in _initialize_replica
|
||||
ERROR 09-23 15:40:03 [stage_runtime.py:265] return self._initialize_local_llm_replica(plan, stage_init_timeout)
|
||||
ERROR 09-23 15:40:03 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 15:40:03 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 934, in _initialize_local_llm_replica
|
||||
ERROR 09-23 15:40:03 [stage_runtime.py:265] launch_stack.close()
|
||||
ERROR 09-23 15:40:03 [stage_runtime.py:265] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 618, in close
|
||||
ERROR 09-23 15:40:03 [stage_runtime.py:265] self.__exit__(None, None, None)
|
||||
ERROR 09-23 15:40:03 [stage_runtime.py:265] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 610, in __exit__
|
||||
ERROR 09-23 15:40:03 [stage_runtime.py:265] raise exc_details[1]
|
||||
ERROR 09-23 15:40:03 [stage_runtime.py:265] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 595, in __exit__
|
||||
ERROR 09-23 15:40:03 [stage_runtime.py:265] if cb(*exc_details):
|
||||
ERROR 09-23 15:40:03 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 15:40:03 [stage_runtime.py:265] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 144, in __exit__
|
||||
ERROR 09-23 15:40:03 [stage_runtime.py:265] next(self.gen)
|
||||
ERROR 09-23 15:40:03 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_startup.py", line 1211, in launch_stage_replica
|
||||
ERROR 09-23 15:40:03 [stage_runtime.py:265] wait_for_engine_startup(
|
||||
ERROR 09-23 15:40:03 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/utils.py", line 1286, in wait_for_engine_startup
|
||||
ERROR 09-23 15:40:03 [stage_runtime.py:265] raise RuntimeError(
|
||||
ERROR 09-23 15:40:03 [stage_runtime.py:265] RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {}
|
||||
INFO 09-23 15:40:03 [core_client.py:689] [shutdown] MPClient: start timeout=default
|
||||
INFO 09-23 15:40:03 [core_client.py:691] [shutdown] MPClient: stopping engine manager
|
||||
INFO 09-23 15:40:03 [utils.py:615] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage0_replica0
|
||||
INFO 09-23 15:40:07 [core_client.py:693] [shutdown] MPClient: engine manager stopped
|
||||
INFO 09-23 15:40:07 [core_client.py:694] [shutdown] MPClient: cleaning up background resources
|
||||
INFO 09-23 15:40:07 [core_client.py:696] [shutdown] MPClient: complete
|
||||
ERROR 09-23 15:40:07 [async_omni_engine.py:474] [AsyncOmniEngine] Orchestrator thread crashed
|
||||
ERROR 09-23 15:40:07 [async_omni_engine.py:474] Traceback (most recent call last):
|
||||
ERROR 09-23 15:40:07 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 468, in _bootstrap_orchestrator
|
||||
ERROR 09-23 15:40:07 [async_omni_engine.py:474] loop.run_until_complete(_run_orchestrator())
|
||||
ERROR 09-23 15:40:07 [async_omni_engine.py:474] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/base_events.py", line 691, in run_until_complete
|
||||
ERROR 09-23 15:40:07 [async_omni_engine.py:474] return future.result()
|
||||
ERROR 09-23 15:40:07 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 15:40:07 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 424, in _run_orchestrator
|
||||
ERROR 09-23 15:40:07 [async_omni_engine.py:474] self._initialize_stages(stage_init_timeout)
|
||||
ERROR 09-23 15:40:07 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 350, in _initialize_stages
|
||||
ERROR 09-23 15:40:07 [async_omni_engine.py:474] self._initialize_stages_impl(stage_init_timeout)
|
||||
ERROR 09-23 15:40:07 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 371, in _initialize_stages_impl
|
||||
ERROR 09-23 15:40:07 [async_omni_engine.py:474] self._runtime.initialize()
|
||||
ERROR 09-23 15:40:07 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 278, in initialize
|
||||
ERROR 09-23 15:40:07 [async_omni_engine.py:474] raise exc
|
||||
ERROR 09-23 15:40:07 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 252, in initialize
|
||||
ERROR 09-23 15:40:07 [async_omni_engine.py:474] initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout)
|
||||
ERROR 09-23 15:40:07 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 15:40:07 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 589, in _initialize_stage_replicas
|
||||
ERROR 09-23 15:40:07 [async_omni_engine.py:474] raise primary_exc
|
||||
ERROR 09-23 15:40:07 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 550, in _init_group
|
||||
ERROR 09-23 15:40:07 [async_omni_engine.py:474] client = self._initialize_replica(
|
||||
ERROR 09-23 15:40:07 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 15:40:07 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 828, in _initialize_replica
|
||||
ERROR 09-23 15:40:07 [async_omni_engine.py:474] return self._initialize_local_llm_replica(plan, stage_init_timeout)
|
||||
ERROR 09-23 15:40:07 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 15:40:07 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 934, in _initialize_local_llm_replica
|
||||
ERROR 09-23 15:40:07 [async_omni_engine.py:474] launch_stack.close()
|
||||
ERROR 09-23 15:40:07 [async_omni_engine.py:474] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 618, in close
|
||||
ERROR 09-23 15:40:07 [async_omni_engine.py:474] self.__exit__(None, None, None)
|
||||
ERROR 09-23 15:40:07 [async_omni_engine.py:474] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 610, in __exit__
|
||||
ERROR 09-23 15:40:07 [async_omni_engine.py:474] raise exc_details[1]
|
||||
ERROR 09-23 15:40:07 [async_omni_engine.py:474] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 595, in __exit__
|
||||
ERROR 09-23 15:40:07 [async_omni_engine.py:474] if cb(*exc_details):
|
||||
ERROR 09-23 15:40:07 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 15:40:07 [async_omni_engine.py:474] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 144, in __exit__
|
||||
ERROR 09-23 15:40:07 [async_omni_engine.py:474] next(self.gen)
|
||||
ERROR 09-23 15:40:07 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_startup.py", line 1211, in launch_stage_replica
|
||||
ERROR 09-23 15:40:07 [async_omni_engine.py:474] wait_for_engine_startup(
|
||||
ERROR 09-23 15:40:07 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/utils.py", line 1286, in wait_for_engine_startup
|
||||
ERROR 09-23 15:40:07 [async_omni_engine.py:474] raise RuntimeError(
|
||||
ERROR 09-23 15:40:07 [async_omni_engine.py:474] RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {}
|
||||
INFO 09-23 15:40:07 [async_omni_engine.py:1807] [AsyncOmniEngine] Shutting down Orchestrator
|
||||
Exception in thread orchestrator:
|
||||
Traceback (most recent call last):
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
|
||||
self.run()
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/threading.py", line 1012, in run
|
||||
self._target(*self._args, **self._kwargs)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 468, in _bootstrap_orchestrator
|
||||
loop.run_until_complete(_run_orchestrator())
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/base_events.py", line 691, in run_until_complete
|
||||
return future.result()
|
||||
^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 424, in _run_orchestrator
|
||||
self._initialize_stages(stage_init_timeout)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 350, in _initialize_stages
|
||||
self._initialize_stages_impl(stage_init_timeout)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 371, in _initialize_stages_impl
|
||||
self._runtime.initialize()
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 278, in initialize
|
||||
raise exc
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 252, in initialize
|
||||
initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 589, in _initialize_stage_replicas
|
||||
raise primary_exc
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 550, in _init_group
|
||||
client = self._initialize_replica(
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 828, in _initialize_replica
|
||||
return self._initialize_local_llm_replica(plan, stage_init_timeout)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 934, in _initialize_local_llm_replica
|
||||
launch_stack.close()
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 618, in close
|
||||
self.__exit__(None, None, None)
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 610, in __exit__
|
||||
raise exc_details[1]
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 595, in __exit__
|
||||
if cb(*exc_details):
|
||||
^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 144, in __exit__
|
||||
next(self.gen)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_startup.py", line 1211, in launch_stage_replica
|
||||
wait_for_engine_startup(
|
||||
File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/utils.py", line 1286, in wait_for_engine_startup
|
||||
raise RuntimeError(
|
||||
RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {}
|
||||
Traceback (most recent call last):
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
|
||||
self.run()
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/threading.py", line 1012, in run
|
||||
self._target(*self._args, **self._kwargs)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 468, in _bootstrap_orchestrator
|
||||
loop.run_until_complete(_run_orchestrator())
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/base_events.py", line 691, in run_until_complete
|
||||
return future.result()
|
||||
^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 424, in _run_orchestrator
|
||||
self._initialize_stages(stage_init_timeout)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 350, in _initialize_stages
|
||||
self._initialize_stages_impl(stage_init_timeout)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 371, in _initialize_stages_impl
|
||||
self._runtime.initialize()
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 278, in initialize
|
||||
raise exc
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 252, in initialize
|
||||
initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 589, in _initialize_stage_replicas
|
||||
raise primary_exc
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 550, in _init_group
|
||||
client = self._initialize_replica(
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 828, in _initialize_replica
|
||||
return self._initialize_local_llm_replica(plan, stage_init_timeout)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 934, in _initialize_local_llm_replica
|
||||
launch_stack.close()
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 618, in close
|
||||
self.__exit__(None, None, None)
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 610, in __exit__
|
||||
raise exc_details[1]
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 595, in __exit__
|
||||
if cb(*exc_details):
|
||||
^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 144, in __exit__
|
||||
next(self.gen)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_startup.py", line 1211, in launch_stage_replica
|
||||
wait_for_engine_startup(
|
||||
File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/utils.py", line 1286, in wait_for_engine_startup
|
||||
raise RuntimeError(
|
||||
RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {}
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
|
||||
Traceback (most recent call last):
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/examples/offline_inference/minicpmo/end2end.py", line 519, in <module>
|
||||
main(args)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/examples/offline_inference/minicpmo/end2end.py", line 283, in main
|
||||
omni = Omni(**omni_kwargs)
|
||||
^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/entrypoints/omni_base.py", line 223, in __init__
|
||||
self.engine = AsyncOmniEngine(
|
||||
^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 305, in __init__
|
||||
self._wait_for_orchestrator_init(startup_future, startup_timeout)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 539, in _wait_for_orchestrator_init
|
||||
startup_future.result(
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/concurrent/futures/_base.py", line 456, in result
|
||||
return self.__get_result()
|
||||
^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/concurrent/futures/_base.py", line 401, in __get_result
|
||||
raise self._exception
|
||||
RuntimeError: Orchestrator initialization failed: Engine core initialization failed. See root cause above. Failed core proc(s): {}
|
||||
+490
@@ -0,0 +1,490 @@
|
||||
W0923 15:41:18.737000 4168074 torch/utils/_pytree.py:630] <enum 'KernelPreference'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
W0923 15:41:18.764000 4168074 torch/utils/_pytree.py:630] <enum 'ScaleCalculationMode'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
WARNING 09-23 15:41:19 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton.
|
||||
INFO 09-23 15:41:19 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available.
|
||||
/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions:
|
||||
--> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922
|
||||
--> vLLM version 0.28.0
|
||||
This will likely cause compatibility issues.
|
||||
warn_if_misaligned_vllm_version()
|
||||
INFO 09-23 15:41:20 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 15:41:21 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 15:41:21 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled.
|
||||
INFO 09-23 15:41:21 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable).
|
||||
====================
|
||||
vllm version: 0.28.0
|
||||
====================
|
||||
INFO 09-23 15:41:21 [omni_base.py:216] [Omni] Initializing with model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5
|
||||
INFO 09-23 15:41:21 [async_omni_engine.py:172] [AsyncOmniEngine] Initializing with model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5
|
||||
INFO 09-23 15:41:21 [async_omni_engine.py:290] [AsyncOmniEngine] Launching Orchestrator thread with 3 stages
|
||||
INFO 09-23 15:41:21 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
INFO 09-23 15:41:21 [initialization.py:356] Loaded OmniTransferConfig with 1 connector configurations
|
||||
WARNING 09-23 15:41:21 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 15:41:21 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
WARNING 09-23 15:41:21 [arg_utils.py:1719] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM.
|
||||
INFO 09-23 15:41:21 [model.py:672] Resolved architecture: MiniCPMO45OmniForConditionalGeneration
|
||||
INFO 09-23 15:41:21 [model.py:1965] Using max model len 2048
|
||||
WARNING 09-23 15:41:21 [model.py:963] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False.
|
||||
INFO 09-23 15:41:22 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode.
|
||||
INFO 09-23 15:41:22 [model.py:875] Disabled mm_prefix attention mode because multimodal inputs are configuration-disabled. Attention backends without mm_prefix support may now be selected.
|
||||
INFO 09-23 15:41:22 [scheduler.py:242] Chunked prefill is enabled with max_num_batched_tokens=2048.
|
||||
WARNING 09-23 15:41:22 [vllm.py:1285] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
WARNING 09-23 15:41:22 [vllm.py:1354] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
INFO 09-23 15:41:22 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
WARNING 09-23 15:41:22 [vllm.py:652] Model Runner V2 requires Triton; using the V1 model runner instead.
|
||||
INFO 09-23 15:41:22 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
WARNING 09-23 15:41:22 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 15:41:22 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
WARNING 09-23 15:41:22 [arg_utils.py:1719] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM.
|
||||
INFO 09-23 15:41:22 [model.py:672] Resolved architecture: MiniCPMO45OmniForConditionalGeneration
|
||||
INFO 09-23 15:41:22 [model.py:1965] Using max model len 512
|
||||
WARNING 09-23 15:41:22 [model.py:963] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False.
|
||||
INFO 09-23 15:41:22 [model.py:1965] Using max model len 512
|
||||
INFO 09-23 15:41:22 [scheduler.py:242] Chunked prefill is enabled with max_num_batched_tokens=512.
|
||||
INFO 09-23 15:41:22 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
WARNING 09-23 15:41:22 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 15:41:22 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
WARNING 09-23 15:41:22 [arg_utils.py:1719] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM.
|
||||
INFO 09-23 15:41:22 [model.py:672] Resolved architecture: MiniCPMO45Code2Wav
|
||||
INFO 09-23 15:41:22 [model.py:1965] Using max model len 4096
|
||||
WARNING 09-23 15:41:22 [arg_utils.py:2664] This model does not officially support disabling chunked prefill. Disabling this manually may cause the engine to crash or produce incorrect outputs.
|
||||
INFO 09-23 15:41:22 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
INFO 09-23 15:41:22 [stage_runtime.py:853] [stage_init] Stage-0 set runtime devices: 0
|
||||
W0923 15:41:26.220000 4168102 torch/utils/_pytree.py:630] <enum 'KernelPreference'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
W0923 15:41:26.247000 4168102 torch/utils/_pytree.py:630] <enum 'ScaleCalculationMode'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
WARNING 09-23 15:41:27 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton.
|
||||
INFO 09-23 15:41:27 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available.
|
||||
/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions:
|
||||
--> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922
|
||||
--> vLLM version 0.28.0
|
||||
This will likely cause compatibility issues.
|
||||
warn_if_misaligned_vllm_version()
|
||||
INFO 09-23 15:41:28 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 15:41:28 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 15:41:28 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled.
|
||||
INFO 09-23 15:41:28 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable).
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168102) INFO 09-23 15:41:28 [core.py:122] Initializing a V1 LLM engine (v0.28.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=2048, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=True, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cpu, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': None, 'compile_ranges_endpoints': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': None, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168102) INFO 09-23 15:41:29 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode.
|
||||
[W923 15:41:29.029555315 utils.cpp:68] Warning: NUMA binding: Using MEMBIND policy for memory allocation on the NUMA nodes (0). Memory allocations will be strictly bound to these NUMA nodes. (function init_cpu_memory_env)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168102) WARNING 09-23 15:41:29 [vllm.py:652] Model Runner V2 requires Triton; using the V1 model runner instead.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168102) WARNING 09-23 15:41:29 [cpu_worker.py:127] libiomp is not found in LD_PRELOAD. For best performance, please follow the section `set LD_PRELOAD` in https://docs.vllm.ai/en/latest/getting_started/installation/cpu/ to setup required pre-loaded libraries.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168102) INFO 09-23 15:41:29 [parallel_state.py:1638] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_aece089fe58d4103be76afbba283ffff backend=gloo
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168102) INFO 09-23 15:41:29 [parallel_state.py:1982] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168102) INFO 09-23 15:41:29 [cpu_model_runner.py:131] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5...
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168102) WARNING 09-23 15:41:29 [vllm.py:1285] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168102) WARNING 09-23 15:41:29 [vllm.py:1354] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168102) INFO 09-23 15:41:29 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168102) INFO 09-23 15:41:29 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168102) INFO 09-23 15:41:29 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168102) INFO 09-23 15:41:29 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168102) INFO 09-23 15:41:29 [selector.py:217] Using HND KV cache layout for CPU_ATTN backend.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168102) INFO 09-23 15:41:33 [weight_utils.py:858] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 27.06 GiB.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168102) INFO 09-23 15:41:33 [weight_utils.py:881] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168102) ␍Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168102) ␍Loading safetensors checkpoint shards: 25% Completed | 1/4 [00:00<00:01, 2.20it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168102) ␍Loading safetensors checkpoint shards: 50% Completed | 2/4 [00:01<00:01, 1.51it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168102) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:02<00:00, 1.67it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168102) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:02<00:00, 1.68it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168102)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168102) INFO 09-23 15:41:47 [default_loader.py:430] Loading weights took 13.79 seconds
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168102) INFO 09-23 15:41:50 [memory.py:91] Released 520 MiB after weight load (anonymous RSS 19813 -> 19293 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory']
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168102) INFO 09-23 15:41:50 [cpu_worker.py:255] Explicitly set (1.0/30.91) GiB for KV cache on node 0.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168102) INFO 09-23 15:41:50 [kv_cache_utils.py:1869] GPU KV cache size: 7,168 tokens, Maximum concurrency for 2,048 tokens per request: 3.50x
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168102) INFO 09-23 15:41:50 [memory.py:91] Released 0 MiB after warm-up (anonymous RSS 20303 -> 20303 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory']
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168102) INFO 09-23 15:41:50 [core.py:354] init engine (profile, create kv cache, warmup model) took 0.20 s
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168102) WARNING 09-23 15:41:50 [scheduler.py:182] Using custom scheduler class vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168102) INFO 09-23 15:41:50 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168102) INFO 09-23 15:41:51 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
INFO 09-23 15:41:51 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed
|
||||
INFO 09-23 15:41:51 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-0 [rep-0] initializing EngineCore
|
||||
INFO 09-23 15:41:51 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-0 [rep-0] EngineCore running
|
||||
INFO 09-23 15:41:51 [stage_runtime.py:958] [StageRuntime] Stage 0 initialized
|
||||
INFO 09-23 15:41:51 [stage_runtime.py:853] [stage_init] Stage-1 set runtime devices: 0
|
||||
W0923 15:41:59.494000 4168210 torch/utils/_pytree.py:630] <enum 'KernelPreference'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
W0923 15:41:59.538000 4168210 torch/utils/_pytree.py:630] <enum 'ScaleCalculationMode'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
WARNING 09-23 15:42:00 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton.
|
||||
INFO 09-23 15:42:00 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available.
|
||||
/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions:
|
||||
--> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922
|
||||
--> vLLM version 0.28.0
|
||||
This will likely cause compatibility issues.
|
||||
warn_if_misaligned_vllm_version()
|
||||
INFO 09-23 15:42:02 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 15:42:02 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 15:42:02 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled.
|
||||
INFO 09-23 15:42:02 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable).
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) INFO 09-23 15:42:03 [core.py:122] Initializing a V1 LLM engine (v0.28.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=512, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=True, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cpu, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': None, 'compile_ranges_endpoints': [512], 'inductor_compile_config': {'enable_auto_functionalized_v2': False}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': None, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
[W923 15:42:04.364823574 utils.cpp:68] Warning: NUMA binding: Using MEMBIND policy for memory allocation on the NUMA nodes (0). Memory allocations will be strictly bound to these NUMA nodes. (function init_cpu_memory_env)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) WARNING 09-23 15:42:04 [vllm.py:652] Model Runner V2 requires Triton; using the V1 model runner instead.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) WARNING 09-23 15:42:04 [cpu_worker.py:127] libiomp is not found in LD_PRELOAD. For best performance, please follow the section `set LD_PRELOAD` in https://docs.vllm.ai/en/latest/getting_started/installation/cpu/ to setup required pre-loaded libraries.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) INFO 09-23 15:42:04 [parallel_state.py:1638] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_8835b78aaeee4981b46f13fffdc72e04 backend=gloo
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) INFO 09-23 15:42:04 [parallel_state.py:1982] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) WARNING 09-23 15:42:04 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) [transformers] `MiniCPMOProcessor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) [transformers] Requested torchvision backend is not available. Falling back to pil backend.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) [transformers] `MiniCPMOProcessor` defines `feature_extractor_class = 'WhisperFeatureExtractor'`, which is deprecated. Register the correct mapping in `AutoFeatureExtractor` instead.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) [transformers] Image processor MiniCPMVImageProcessor: kwargs ['max_slice_nums'] were applied for backward compatibility. To avoid this warning, add them to valid_kwargs: create a custom TypedDict extending ImagesKwargs with these keys and set it as the `valid_kwargs` class attribute.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) WARNING 09-23 15:42:12 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] StageEngineCoreProc failed to start.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] Traceback (most recent call last):
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/processing/context.py", line 286, in call_hf_processor
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] output = hf_processor(**data, **allowed_kwargs)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/transformers_utils/processors/minicpmo.py", line 107, in __call__
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] model_inputs = self._convert_omni_to_inputs(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/transformers_utils/processors/minicpmo.py", line 397, in _convert_omni_to_inputs
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] assert len(image_tags) == len(image_sizes[index])
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] AssertionError
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] The above exception was the direct cause of the following exception:
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] Traceback (most recent call last):
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_core_proc.py", line 189, in run_stage_core
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] engine_core = StageEngineCoreProc(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] return func(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/core.py", line 1073, in __init__
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] super().__init__(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/core.py", line 133, in __init__
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] self.model_executor = executor_class(vllm_config)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] return func(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/executor/abstract.py", line 110, in __init__
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] self._init_executor()
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/executor/uniproc_executor.py", line 69, in _init_executor
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] self.driver_worker.init_device()
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 335, in init_device
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] self.worker.init_device() # type: ignore
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/init_timeline.py", line 343, in wrapper
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] return fn(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/platforms/cpu/worker/cpu_ar_worker.py", line 30, in init_device
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] super().init_device() # type: ignore[misc]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/cpu_worker.py", line 190, in init_device
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] self.model_runner = CPUModelRunner(self.vllm_config, torch.device("cpu"))
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/cpu_model_runner.py", line 31, in __init__
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] super().__init__(vllm_config, device)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/gpu_model_runner.py", line 915, in __init__
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] MultiModalBudget(self.vllm_config, self.mm_registry)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/encoder_budget.py", line 94, in __init__
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] all_mm_max_toks_per_item = get_mm_max_toks_per_item(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/encoder_budget.py", line 33, in get_mm_max_toks_per_item
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] mm_inputs = mm_registry.get_dummy_mm_inputs(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/registry.py", line 263, in get_dummy_mm_inputs
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] mm_inputs = processor.apply(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py", line 3420, in apply
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] return super().apply(inputs, timing_ctx)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/processing/processor.py", line 1849, in apply
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ) = self._cached_apply_hf_processor(inputs, timing_ctx)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/processing/processor.py", line 1628, in _cached_apply_hf_processor
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ) = self._apply_hf_processor_main(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py", line 3444, in _apply_hf_processor_main
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] prompt_ids = self._apply_hf_processor_text_only(prompt, tokenization_kwargs or {})
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/processing/processor.py", line 1336, in _apply_hf_processor_text_only
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] prompt_ids, _, _ = self._apply_hf_processor_text_mm(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/processing/processor.py", line 1301, in _apply_hf_processor_text_mm
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] processed_data = self._call_hf_processor(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/processing/processor.py", line 1258, in _call_hf_processor
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] return self.info.ctx.call_hf_processor(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/processing/context.py", line 293, in call_hf_processor
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] raise ValueError(msg) from exc
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ERROR 09-23 15:42:12 [stage_engine_core_proc.py:242] ValueError: Failed to apply MiniCPMOProcessor on data={'text': '(<audio>./</audio>)(<image>./</image>)(<video>./</video>)'} with kwargs={'truncation': False, 'return_tensors': 'pt'}
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) Process StageEngineCoreProc_stage1_replica0:
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) Traceback (most recent call last):
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/processing/context.py", line 286, in call_hf_processor
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) output = hf_processor(**data, **allowed_kwargs)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/transformers_utils/processors/minicpmo.py", line 107, in __call__
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) model_inputs = self._convert_omni_to_inputs(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/transformers_utils/processors/minicpmo.py", line 397, in _convert_omni_to_inputs
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) assert len(image_tags) == len(image_sizes[index])
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) AssertionError
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) The above exception was the direct cause of the following exception:
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) Traceback (most recent call last):
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/multiprocessing/process.py", line 314, in _bootstrap
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) self.run()
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/multiprocessing/process.py", line 108, in run
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) self._target(*self._args, **self._kwargs)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_core_proc.py", line 189, in run_stage_core
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) engine_core = StageEngineCoreProc(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) return func(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/core.py", line 1073, in __init__
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) super().__init__(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/core.py", line 133, in __init__
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) self.model_executor = executor_class(vllm_config)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) return func(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/executor/abstract.py", line 110, in __init__
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) self._init_executor()
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/executor/uniproc_executor.py", line 69, in _init_executor
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) self.driver_worker.init_device()
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/worker_base.py", line 335, in init_device
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) self.worker.init_device() # type: ignore
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/init_timeline.py", line 343, in wrapper
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) return fn(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/platforms/cpu/worker/cpu_ar_worker.py", line 30, in init_device
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) super().init_device() # type: ignore[misc]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/cpu_worker.py", line 190, in init_device
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) self.model_runner = CPUModelRunner(self.vllm_config, torch.device("cpu"))
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/cpu_model_runner.py", line 31, in __init__
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) super().__init__(vllm_config, device)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/gpu_model_runner.py", line 915, in __init__
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) MultiModalBudget(self.vllm_config, self.mm_registry)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/encoder_budget.py", line 94, in __init__
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) all_mm_max_toks_per_item = get_mm_max_toks_per_item(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/encoder_budget.py", line 33, in get_mm_max_toks_per_item
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) mm_inputs = mm_registry.get_dummy_mm_inputs(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/registry.py", line 263, in get_dummy_mm_inputs
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) mm_inputs = processor.apply(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py", line 3420, in apply
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) return super().apply(inputs, timing_ctx)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/processing/processor.py", line 1849, in apply
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ) = self._cached_apply_hf_processor(inputs, timing_ctx)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/processing/processor.py", line 1628, in _cached_apply_hf_processor
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ) = self._apply_hf_processor_main(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_omni_llm.py", line 3444, in _apply_hf_processor_main
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) prompt_ids = self._apply_hf_processor_text_only(prompt, tokenization_kwargs or {})
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/processing/processor.py", line 1336, in _apply_hf_processor_text_only
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) prompt_ids, _, _ = self._apply_hf_processor_text_mm(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/processing/processor.py", line 1301, in _apply_hf_processor_text_mm
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) processed_data = self._call_hf_processor(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/processing/processor.py", line 1258, in _call_hf_processor
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) return self.info.ctx.call_hf_processor(
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/multimodal/processing/context.py", line 293, in call_hf_processor
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) raise ValueError(msg) from exc
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168210) ValueError: Failed to apply MiniCPMOProcessor on data={'text': '(<audio>./</audio>)(<image>./</image>)(<video>./</video>)'} with kwargs={'truncation': False, 'return_tensors': 'pt'}
|
||||
ERROR 09-23 15:42:14 [stage_runtime.py:265] [StageRuntime] Stage initialization failed; shutting down 1 initialized client(s)
|
||||
ERROR 09-23 15:42:14 [stage_runtime.py:265] Traceback (most recent call last):
|
||||
ERROR 09-23 15:42:14 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 252, in initialize
|
||||
ERROR 09-23 15:42:14 [stage_runtime.py:265] initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout)
|
||||
ERROR 09-23 15:42:14 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 15:42:14 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 589, in _initialize_stage_replicas
|
||||
ERROR 09-23 15:42:14 [stage_runtime.py:265] raise primary_exc
|
||||
ERROR 09-23 15:42:14 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 550, in _init_group
|
||||
ERROR 09-23 15:42:14 [stage_runtime.py:265] client = self._initialize_replica(
|
||||
ERROR 09-23 15:42:14 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 15:42:14 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 828, in _initialize_replica
|
||||
ERROR 09-23 15:42:14 [stage_runtime.py:265] return self._initialize_local_llm_replica(plan, stage_init_timeout)
|
||||
ERROR 09-23 15:42:14 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 15:42:14 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 934, in _initialize_local_llm_replica
|
||||
ERROR 09-23 15:42:14 [stage_runtime.py:265] launch_stack.close()
|
||||
ERROR 09-23 15:42:14 [stage_runtime.py:265] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 618, in close
|
||||
ERROR 09-23 15:42:14 [stage_runtime.py:265] self.__exit__(None, None, None)
|
||||
ERROR 09-23 15:42:14 [stage_runtime.py:265] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 610, in __exit__
|
||||
ERROR 09-23 15:42:14 [stage_runtime.py:265] raise exc_details[1]
|
||||
ERROR 09-23 15:42:14 [stage_runtime.py:265] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 595, in __exit__
|
||||
ERROR 09-23 15:42:14 [stage_runtime.py:265] if cb(*exc_details):
|
||||
ERROR 09-23 15:42:14 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 15:42:14 [stage_runtime.py:265] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 144, in __exit__
|
||||
ERROR 09-23 15:42:14 [stage_runtime.py:265] next(self.gen)
|
||||
ERROR 09-23 15:42:14 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_startup.py", line 1211, in launch_stage_replica
|
||||
ERROR 09-23 15:42:14 [stage_runtime.py:265] wait_for_engine_startup(
|
||||
ERROR 09-23 15:42:14 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/utils.py", line 1286, in wait_for_engine_startup
|
||||
ERROR 09-23 15:42:14 [stage_runtime.py:265] raise RuntimeError(
|
||||
ERROR 09-23 15:42:14 [stage_runtime.py:265] RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {}
|
||||
INFO 09-23 15:42:14 [core_client.py:689] [shutdown] MPClient: start timeout=default
|
||||
INFO 09-23 15:42:14 [core_client.py:691] [shutdown] MPClient: stopping engine manager
|
||||
INFO 09-23 15:42:14 [utils.py:615] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage0_replica0
|
||||
WARNING 09-23 15:42:19 [utils.py:635] [shutdown] Process manager: force killing remaining processes count=1
|
||||
WARNING 09-23 15:42:19 [utils.py:640] [shutdown] Process manager: force killing remaining process StageEngineCoreProc_stage0_replica0 pid 4168102
|
||||
INFO 09-23 15:42:19 [core_client.py:693] [shutdown] MPClient: engine manager stopped
|
||||
INFO 09-23 15:42:19 [core_client.py:694] [shutdown] MPClient: cleaning up background resources
|
||||
INFO 09-23 15:42:19 [core_client.py:696] [shutdown] MPClient: complete
|
||||
ERROR 09-23 15:42:19 [async_omni_engine.py:474] [AsyncOmniEngine] Orchestrator thread crashed
|
||||
ERROR 09-23 15:42:19 [async_omni_engine.py:474] Traceback (most recent call last):
|
||||
ERROR 09-23 15:42:19 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 468, in _bootstrap_orchestrator
|
||||
ERROR 09-23 15:42:19 [async_omni_engine.py:474] loop.run_until_complete(_run_orchestrator())
|
||||
ERROR 09-23 15:42:19 [async_omni_engine.py:474] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/base_events.py", line 691, in run_until_complete
|
||||
ERROR 09-23 15:42:19 [async_omni_engine.py:474] return future.result()
|
||||
ERROR 09-23 15:42:19 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 15:42:19 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 424, in _run_orchestrator
|
||||
ERROR 09-23 15:42:19 [async_omni_engine.py:474] self._initialize_stages(stage_init_timeout)
|
||||
ERROR 09-23 15:42:19 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 350, in _initialize_stages
|
||||
ERROR 09-23 15:42:19 [async_omni_engine.py:474] self._initialize_stages_impl(stage_init_timeout)
|
||||
ERROR 09-23 15:42:19 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 371, in _initialize_stages_impl
|
||||
ERROR 09-23 15:42:19 [async_omni_engine.py:474] self._runtime.initialize()
|
||||
ERROR 09-23 15:42:19 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 278, in initialize
|
||||
ERROR 09-23 15:42:19 [async_omni_engine.py:474] raise exc
|
||||
ERROR 09-23 15:42:19 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 252, in initialize
|
||||
ERROR 09-23 15:42:19 [async_omni_engine.py:474] initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout)
|
||||
ERROR 09-23 15:42:19 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 15:42:19 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 589, in _initialize_stage_replicas
|
||||
ERROR 09-23 15:42:19 [async_omni_engine.py:474] raise primary_exc
|
||||
ERROR 09-23 15:42:19 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 550, in _init_group
|
||||
ERROR 09-23 15:42:19 [async_omni_engine.py:474] client = self._initialize_replica(
|
||||
ERROR 09-23 15:42:19 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 15:42:19 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 828, in _initialize_replica
|
||||
ERROR 09-23 15:42:19 [async_omni_engine.py:474] return self._initialize_local_llm_replica(plan, stage_init_timeout)
|
||||
ERROR 09-23 15:42:19 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 15:42:19 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 934, in _initialize_local_llm_replica
|
||||
ERROR 09-23 15:42:19 [async_omni_engine.py:474] launch_stack.close()
|
||||
ERROR 09-23 15:42:19 [async_omni_engine.py:474] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 618, in close
|
||||
ERROR 09-23 15:42:19 [async_omni_engine.py:474] self.__exit__(None, None, None)
|
||||
ERROR 09-23 15:42:19 [async_omni_engine.py:474] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 610, in __exit__
|
||||
ERROR 09-23 15:42:19 [async_omni_engine.py:474] raise exc_details[1]
|
||||
ERROR 09-23 15:42:19 [async_omni_engine.py:474] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 595, in __exit__
|
||||
ERROR 09-23 15:42:19 [async_omni_engine.py:474] if cb(*exc_details):
|
||||
ERROR 09-23 15:42:19 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 15:42:19 [async_omni_engine.py:474] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 144, in __exit__
|
||||
ERROR 09-23 15:42:19 [async_omni_engine.py:474] next(self.gen)
|
||||
ERROR 09-23 15:42:19 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_startup.py", line 1211, in launch_stage_replica
|
||||
ERROR 09-23 15:42:19 [async_omni_engine.py:474] wait_for_engine_startup(
|
||||
ERROR 09-23 15:42:19 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/utils.py", line 1286, in wait_for_engine_startup
|
||||
ERROR 09-23 15:42:19 [async_omni_engine.py:474] raise RuntimeError(
|
||||
ERROR 09-23 15:42:19 [async_omni_engine.py:474] RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {}
|
||||
INFO 09-23 15:42:19 [async_omni_engine.py:1807] [AsyncOmniEngine] Shutting down Orchestrator
|
||||
Exception in thread orchestrator:
|
||||
Traceback (most recent call last):
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
|
||||
self.run()
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/threading.py", line 1012, in run
|
||||
self._target(*self._args, **self._kwargs)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 468, in _bootstrap_orchestrator
|
||||
loop.run_until_complete(_run_orchestrator())
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/base_events.py", line 691, in run_until_complete
|
||||
return future.result()
|
||||
^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 424, in _run_orchestrator
|
||||
self._initialize_stages(stage_init_timeout)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 350, in _initialize_stages
|
||||
self._initialize_stages_impl(stage_init_timeout)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 371, in _initialize_stages_impl
|
||||
self._runtime.initialize()
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 278, in initialize
|
||||
raise exc
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 252, in initialize
|
||||
initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 589, in _initialize_stage_replicas
|
||||
raise primary_exc
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 550, in _init_group
|
||||
client = self._initialize_replica(
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 828, in _initialize_replica
|
||||
return self._initialize_local_llm_replica(plan, stage_init_timeout)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 934, in _initialize_local_llm_replica
|
||||
launch_stack.close()
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 618, in close
|
||||
self.__exit__(None, None, None)
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 610, in __exit__
|
||||
raise exc_details[1]
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 595, in __exit__
|
||||
if cb(*exc_details):
|
||||
^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 144, in __exit__
|
||||
next(self.gen)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_startup.py", line 1211, in launch_stage_replica
|
||||
wait_for_engine_startup(
|
||||
File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/utils.py", line 1286, in wait_for_engine_startup
|
||||
raise RuntimeError(
|
||||
RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {}
|
||||
Traceback (most recent call last):
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
|
||||
self.run()
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/threading.py", line 1012, in run
|
||||
self._target(*self._args, **self._kwargs)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 468, in _bootstrap_orchestrator
|
||||
loop.run_until_complete(_run_orchestrator())
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/base_events.py", line 691, in run_until_complete
|
||||
return future.result()
|
||||
^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 424, in _run_orchestrator
|
||||
self._initialize_stages(stage_init_timeout)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 350, in _initialize_stages
|
||||
self._initialize_stages_impl(stage_init_timeout)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 371, in _initialize_stages_impl
|
||||
self._runtime.initialize()
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 278, in initialize
|
||||
raise exc
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 252, in initialize
|
||||
initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 589, in _initialize_stage_replicas
|
||||
raise primary_exc
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 550, in _init_group
|
||||
client = self._initialize_replica(
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 828, in _initialize_replica
|
||||
return self._initialize_local_llm_replica(plan, stage_init_timeout)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 934, in _initialize_local_llm_replica
|
||||
launch_stack.close()
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 618, in close
|
||||
self.__exit__(None, None, None)
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 610, in __exit__
|
||||
raise exc_details[1]
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 595, in __exit__
|
||||
if cb(*exc_details):
|
||||
^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 144, in __exit__
|
||||
next(self.gen)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_startup.py", line 1211, in launch_stage_replica
|
||||
wait_for_engine_startup(
|
||||
File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/utils.py", line 1286, in wait_for_engine_startup
|
||||
raise RuntimeError(
|
||||
RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {}
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
|
||||
Traceback (most recent call last):
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/examples/offline_inference/minicpmo/end2end.py", line 519, in <module>
|
||||
main(args)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/examples/offline_inference/minicpmo/end2end.py", line 283, in main
|
||||
omni = Omni(**omni_kwargs)
|
||||
^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/entrypoints/omni_base.py", line 223, in __init__
|
||||
self.engine = AsyncOmniEngine(
|
||||
^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 305, in __init__
|
||||
self._wait_for_orchestrator_init(startup_future, startup_timeout)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 539, in _wait_for_orchestrator_init
|
||||
startup_future.result(
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/concurrent/futures/_base.py", line 456, in result
|
||||
return self.__get_result()
|
||||
^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/concurrent/futures/_base.py", line 401, in __get_result
|
||||
raise self._exception
|
||||
RuntimeError: Orchestrator initialization failed: Engine core initialization failed. See root cause above. Failed core proc(s): {}
|
||||
+587
@@ -0,0 +1,587 @@
|
||||
W0923 15:43:20.838000 4168432 torch/utils/_pytree.py:630] <enum 'KernelPreference'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
W0923 15:43:20.864000 4168432 torch/utils/_pytree.py:630] <enum 'ScaleCalculationMode'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
WARNING 09-23 15:43:21 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton.
|
||||
INFO 09-23 15:43:21 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available.
|
||||
/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions:
|
||||
--> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922
|
||||
--> vLLM version 0.28.0
|
||||
This will likely cause compatibility issues.
|
||||
warn_if_misaligned_vllm_version()
|
||||
INFO 09-23 15:43:22 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 15:43:22 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 15:43:22 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled.
|
||||
INFO 09-23 15:43:23 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable).
|
||||
====================
|
||||
vllm version: 0.28.0
|
||||
====================
|
||||
INFO 09-23 15:43:23 [omni_base.py:216] [Omni] Initializing with model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5
|
||||
INFO 09-23 15:43:23 [async_omni_engine.py:172] [AsyncOmniEngine] Initializing with model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5
|
||||
INFO 09-23 15:43:23 [async_omni_engine.py:290] [AsyncOmniEngine] Launching Orchestrator thread with 3 stages
|
||||
INFO 09-23 15:43:23 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
INFO 09-23 15:43:23 [initialization.py:356] Loaded OmniTransferConfig with 1 connector configurations
|
||||
WARNING 09-23 15:43:23 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 15:43:23 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
WARNING 09-23 15:43:23 [arg_utils.py:1719] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM.
|
||||
INFO 09-23 15:43:23 [model.py:672] Resolved architecture: MiniCPMO45OmniForConditionalGeneration
|
||||
INFO 09-23 15:43:23 [model.py:1965] Using max model len 2048
|
||||
WARNING 09-23 15:43:23 [model.py:963] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False.
|
||||
INFO 09-23 15:43:24 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode.
|
||||
INFO 09-23 15:43:24 [model.py:875] Disabled mm_prefix attention mode because multimodal inputs are configuration-disabled. Attention backends without mm_prefix support may now be selected.
|
||||
INFO 09-23 15:43:24 [scheduler.py:242] Chunked prefill is enabled with max_num_batched_tokens=2048.
|
||||
WARNING 09-23 15:43:24 [vllm.py:1285] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
WARNING 09-23 15:43:24 [vllm.py:1354] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
INFO 09-23 15:43:24 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
WARNING 09-23 15:43:24 [vllm.py:652] Model Runner V2 requires Triton; using the V1 model runner instead.
|
||||
INFO 09-23 15:43:24 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
WARNING 09-23 15:43:24 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 15:43:24 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
WARNING 09-23 15:43:24 [arg_utils.py:1719] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM.
|
||||
INFO 09-23 15:43:24 [model.py:672] Resolved architecture: MiniCPMO45OmniForConditionalGeneration
|
||||
INFO 09-23 15:43:24 [model.py:1965] Using max model len 512
|
||||
WARNING 09-23 15:43:24 [model.py:963] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False.
|
||||
INFO 09-23 15:43:24 [model.py:1965] Using max model len 512
|
||||
INFO 09-23 15:43:24 [scheduler.py:242] Chunked prefill is enabled with max_num_batched_tokens=512.
|
||||
INFO 09-23 15:43:24 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
WARNING 09-23 15:43:24 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 15:43:24 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
WARNING 09-23 15:43:24 [arg_utils.py:1719] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM.
|
||||
INFO 09-23 15:43:24 [model.py:672] Resolved architecture: MiniCPMO45Code2Wav
|
||||
INFO 09-23 15:43:24 [model.py:1965] Using max model len 4096
|
||||
WARNING 09-23 15:43:24 [arg_utils.py:2664] This model does not officially support disabling chunked prefill. Disabling this manually may cause the engine to crash or produce incorrect outputs.
|
||||
INFO 09-23 15:43:24 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
INFO 09-23 15:43:24 [stage_runtime.py:853] [stage_init] Stage-0 set runtime devices: 0
|
||||
W0923 15:43:28.412000 4168462 torch/utils/_pytree.py:630] <enum 'KernelPreference'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
W0923 15:43:28.440000 4168462 torch/utils/_pytree.py:630] <enum 'ScaleCalculationMode'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
WARNING 09-23 15:43:29 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton.
|
||||
INFO 09-23 15:43:29 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available.
|
||||
/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions:
|
||||
--> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922
|
||||
--> vLLM version 0.28.0
|
||||
This will likely cause compatibility issues.
|
||||
warn_if_misaligned_vllm_version()
|
||||
INFO 09-23 15:43:30 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 15:43:30 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 15:43:30 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled.
|
||||
INFO 09-23 15:43:30 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable).
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168462) INFO 09-23 15:43:31 [core.py:122] Initializing a V1 LLM engine (v0.28.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=2048, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=True, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cpu, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': None, 'compile_ranges_endpoints': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': None, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168462) INFO 09-23 15:43:32 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode.
|
||||
[W923 15:43:32.228349802 utils.cpp:68] Warning: NUMA binding: Using MEMBIND policy for memory allocation on the NUMA nodes (0). Memory allocations will be strictly bound to these NUMA nodes. (function init_cpu_memory_env)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168462) WARNING 09-23 15:43:32 [vllm.py:652] Model Runner V2 requires Triton; using the V1 model runner instead.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168462) WARNING 09-23 15:43:32 [cpu_worker.py:127] libiomp is not found in LD_PRELOAD. For best performance, please follow the section `set LD_PRELOAD` in https://docs.vllm.ai/en/latest/getting_started/installation/cpu/ to setup required pre-loaded libraries.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168462) INFO 09-23 15:43:32 [parallel_state.py:1638] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_f8d63e300ac346b79645aff6c0c6e5a8 backend=gloo
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168462) INFO 09-23 15:43:32 [parallel_state.py:1982] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168462) INFO 09-23 15:43:32 [cpu_model_runner.py:131] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5...
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168462) WARNING 09-23 15:43:32 [vllm.py:1285] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168462) WARNING 09-23 15:43:32 [vllm.py:1354] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168462) INFO 09-23 15:43:32 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168462) INFO 09-23 15:43:32 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168462) INFO 09-23 15:43:32 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168462) INFO 09-23 15:43:32 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168462) INFO 09-23 15:43:32 [selector.py:217] Using HND KV cache layout for CPU_ATTN backend.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168462) INFO 09-23 15:43:35 [weight_utils.py:858] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 27.11 GiB.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168462) INFO 09-23 15:43:35 [weight_utils.py:881] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168462) ␍Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168462) ␍Loading safetensors checkpoint shards: 25% Completed | 1/4 [00:00<00:01, 2.15it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168462) ␍Loading safetensors checkpoint shards: 50% Completed | 2/4 [00:01<00:01, 1.32it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168462) ␍Loading safetensors checkpoint shards: 75% Completed | 3/4 [00:02<00:00, 1.11it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168462) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:03<00:00, 1.01it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168462) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:03<00:00, 1.10it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168462)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168462) INFO 09-23 15:43:51 [default_loader.py:430] Loading weights took 15.33 seconds
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168462) INFO 09-23 15:43:57 [memory.py:91] Released 511 MiB after weight load (anonymous RSS 19156 -> 18645 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory']
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168462) INFO 09-23 15:43:57 [cpu_worker.py:255] Explicitly set (1.0/30.91) GiB for KV cache on node 0.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168462) INFO 09-23 15:43:57 [kv_cache_utils.py:1869] GPU KV cache size: 7,168 tokens, Maximum concurrency for 2,048 tokens per request: 3.50x
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168462) INFO 09-23 15:43:57 [memory.py:91] Released 0 MiB after warm-up (anonymous RSS 19662 -> 19662 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory']
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168462) INFO 09-23 15:43:57 [core.py:354] init engine (profile, create kv cache, warmup model) took 0.47 s
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168462) WARNING 09-23 15:43:57 [scheduler.py:182] Using custom scheduler class vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168462) INFO 09-23 15:43:58 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168462) INFO 09-23 15:44:08 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
INFO 09-23 15:44:08 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed
|
||||
INFO 09-23 15:44:08 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-0 [rep-0] initializing EngineCore
|
||||
INFO 09-23 15:44:08 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-0 [rep-0] EngineCore running
|
||||
INFO 09-23 15:44:08 [stage_runtime.py:958] [StageRuntime] Stage 0 initialized
|
||||
INFO 09-23 15:44:08 [stage_runtime.py:853] [stage_init] Stage-1 set runtime devices: 0
|
||||
W0923 15:44:14.751000 4168582 torch/utils/_pytree.py:630] <enum 'KernelPreference'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
W0923 15:44:14.792000 4168582 torch/utils/_pytree.py:630] <enum 'ScaleCalculationMode'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
WARNING 09-23 15:44:16 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton.
|
||||
INFO 09-23 15:44:16 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available.
|
||||
/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions:
|
||||
--> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922
|
||||
--> vLLM version 0.28.0
|
||||
This will likely cause compatibility issues.
|
||||
warn_if_misaligned_vllm_version()
|
||||
INFO 09-23 15:44:18 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 15:44:18 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 15:44:18 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled.
|
||||
INFO 09-23 15:44:18 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable).
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168582) INFO 09-23 15:44:19 [core.py:122] Initializing a V1 LLM engine (v0.28.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=512, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=True, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cpu, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': None, 'compile_ranges_endpoints': [512], 'inductor_compile_config': {'enable_auto_functionalized_v2': False}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': None, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
[W923 15:44:20.299950745 utils.cpp:68] Warning: NUMA binding: Using MEMBIND policy for memory allocation on the NUMA nodes (0). Memory allocations will be strictly bound to these NUMA nodes. (function init_cpu_memory_env)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168582) WARNING 09-23 15:44:20 [vllm.py:652] Model Runner V2 requires Triton; using the V1 model runner instead.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168582) WARNING 09-23 15:44:20 [cpu_worker.py:127] libiomp is not found in LD_PRELOAD. For best performance, please follow the section `set LD_PRELOAD` in https://docs.vllm.ai/en/latest/getting_started/installation/cpu/ to setup required pre-loaded libraries.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168582) INFO 09-23 15:44:20 [parallel_state.py:1638] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_c745d19853d047d0b9c64ec8c42c5ccc backend=gloo
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168582) INFO 09-23 15:44:20 [parallel_state.py:1982] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168582) WARNING 09-23 15:44:20 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168582) [transformers] `MiniCPMOProcessor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168582) [transformers] Requested torchvision backend is not available. Falling back to pil backend.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168582) [transformers] `MiniCPMOProcessor` defines `feature_extractor_class = 'WhisperFeatureExtractor'`, which is deprecated. Register the correct mapping in `AutoFeatureExtractor` instead.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168582) [transformers] Image processor MiniCPMVImageProcessor: kwargs ['max_slice_nums'] were applied for backward compatibility. To avoid this warning, add them to valid_kwargs: create a custom TypedDict extending ImagesKwargs with these keys and set it as the `valid_kwargs` class attribute.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168582) WARNING 09-23 15:44:28 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168582) WARNING 09-23 15:44:28 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168582) WARNING 09-23 15:44:28 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168582) INFO 09-23 15:44:29 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168582) INFO 09-23 15:44:29 [cpu_model_runner.py:131] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5...
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168582) WARNING 09-23 15:44:29 [vllm.py:1285] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168582) WARNING 09-23 15:44:29 [vllm.py:1354] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168582) INFO 09-23 15:44:29 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168582) INFO 09-23 15:44:29 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168582) INFO 09-23 15:44:29 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168582) INFO 09-23 15:44:29 [selector.py:217] Using HND KV cache layout for CPU_ATTN backend.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168582) INFO 09-23 15:44:29 [weight_utils.py:858] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 8.79 GiB.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168582) INFO 09-23 15:44:29 [weight_utils.py:888] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre) and the checkpoint size (17.46 GiB) exceeds 90% of available RAM (8.79 GiB).
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168582) ␍Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168582) ␍Loading safetensors checkpoint shards: 25% Completed | 1/4 [00:01<00:03, 1.03s/it]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168582) ␍Loading safetensors checkpoint shards: 50% Completed | 2/4 [00:01<00:01, 1.25it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168582) ␍Loading safetensors checkpoint shards: 75% Completed | 3/4 [00:02<00:00, 1.51it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168582) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:02<00:00, 1.39it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168582) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:02<00:00, 1.34it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168582)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168582) INFO 09-23 15:44:32 [default_loader.py:430] Loading weights took 3.12 seconds
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168582) INFO 09-23 15:44:32 [memory.py:91] Released 76 MiB after weight load (anonymous RSS 1523 -> 1447 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory']
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168582) INFO 09-23 15:44:32 [cpu_worker.py:255] Explicitly set (1.0/30.91) GiB for KV cache on node 0.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168582) INFO 09-23 15:44:32 [kv_cache_utils.py:1869] GPU KV cache size: 17,408 tokens, Maximum concurrency for 512 tokens per request: 34.00x
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168582) INFO 09-23 15:44:33 [memory.py:91] Released -0 MiB after warm-up (anonymous RSS 2469 -> 2469 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory']
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168582) INFO 09-23 15:44:33 [core.py:354] init engine (profile, create kv cache, warmup model) took 0.31 s
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168582) WARNING 09-23 15:44:33 [scheduler.py:182] Using custom scheduler class vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168582) WARNING 09-23 15:44:33 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168582) WARNING 09-23 15:44:33 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168582) INFO 09-23 15:44:33 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4168582) INFO 09-23 15:44:33 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
INFO 09-23 15:44:33 [stage_runtime.py:943] [StageRuntime] Stage 1 engine startup completed
|
||||
INFO 09-23 15:44:33 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-1 [rep-0] initializing EngineCore
|
||||
INFO 09-23 15:44:33 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-1 [rep-0] EngineCore running
|
||||
INFO 09-23 15:44:33 [stage_runtime.py:958] [StageRuntime] Stage 1 initialized
|
||||
INFO 09-23 15:44:33 [stage_runtime.py:853] [stage_init] Stage-2 set runtime devices: 0
|
||||
W0923 15:44:39.263000 4168702 torch/utils/_pytree.py:630] <enum 'KernelPreference'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
W0923 15:44:39.290000 4168702 torch/utils/_pytree.py:630] <enum 'ScaleCalculationMode'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
WARNING 09-23 15:44:40 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton.
|
||||
INFO 09-23 15:44:40 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available.
|
||||
/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions:
|
||||
--> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922
|
||||
--> vLLM version 0.28.0
|
||||
This will likely cause compatibility issues.
|
||||
warn_if_misaligned_vllm_version()
|
||||
INFO 09-23 15:44:41 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 15:44:42 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 15:44:42 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled.
|
||||
INFO 09-23 15:44:42 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable).
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) INFO 09-23 15:44:42 [core.py:122] Initializing a V1 LLM engine (v0.28.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=4096, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=True, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cpu, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=False, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': None, 'compile_ranges_endpoints': [4096], 'inductor_compile_config': {'enable_auto_functionalized_v2': False}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': None, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
[W923 15:44:42.892659826 utils.cpp:68] Warning: NUMA binding: Using MEMBIND policy for memory allocation on the NUMA nodes (0). Memory allocations will be strictly bound to these NUMA nodes. (function init_cpu_memory_env)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) WARNING 09-23 15:44:42 [vllm.py:652] Model Runner V2 requires Triton; using the V1 model runner instead.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) WARNING 09-23 15:44:42 [cpu_worker.py:127] libiomp is not found in LD_PRELOAD. For best performance, please follow the section `set LD_PRELOAD` in https://docs.vllm.ai/en/latest/getting_started/installation/cpu/ to setup required pre-loaded libraries.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) INFO 09-23 15:44:42 [parallel_state.py:1638] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_8bf2c7708ac8464e8a951a827bd46d53 backend=gloo
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) INFO 09-23 15:44:42 [parallel_state.py:1982] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) INFO 09-23 15:44:42 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) INFO 09-23 15:44:42 [cpu_model_runner.py:131] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5...
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) INFO 09-23 15:44:42 [weight_utils.py:858] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 6.82 GiB.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) INFO 09-23 15:44:42 [weight_utils.py:888] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre) and the checkpoint size (17.46 GiB) exceeds 90% of available RAM (6.82 GiB).
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ␍Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:00<00:00, 218.12it/s]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) INFO 09-23 15:44:42 [step_audio2_token2wav.py:163] Loading Token2Wav models from: /home/zhout/project/edge_infer/models/MiniCPM-o-4_5/assets/token2wav
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] StageEngineCoreProc failed to start.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] Traceback (most recent call last):
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_core_proc.py", line 189, in run_stage_core
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] engine_core = StageEngineCoreProc(
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] return func(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/core.py", line 1073, in __init__
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] super().__init__(
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/core.py", line 133, in __init__
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] self.model_executor = executor_class(vllm_config)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] return func(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/executor/abstract.py", line 110, in __init__
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] self._init_executor()
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/executor/uniproc_executor.py", line 74, in _init_executor
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] self.driver_worker.load_model()
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/patch.py", line 738, in load_model
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] out = base_load(self, *args, **kwargs)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/gpu_worker.py", line 457, in load_model
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] self.model_runner.load_model(load_dummy_weights=load_dummy_weights)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] return func(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/init_timeline.py", line 343, in wrapper
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] return fn(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/worker/gpu_model_runner.py", line 188, in load_model
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] super().load_model(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] return func(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/cpu_model_runner.py", line 132, in load_model
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] self.model = get_model(vllm_config=self.vllm_config)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/model_executor/model_loader/__init__.py", line 137, in get_model
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] return loader.load_model(
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] return func(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/model_executor/model_loader/base_loader.py", line 64, in load_model
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] self.load_weights(model, model_config)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] return func(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/model_executor/model_loader/default_loader.py", line 427, in load_weights
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] loaded_weights = model.load_weights(self.get_all_weights(model_config, model))
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py", line 823, in load_weights
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] self._build_backend()
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py", line 864, in _build_backend
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] token2wav = Token2wav(
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] ^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_token2wav.py", line 79, in __init__
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] self._core._ensure_models_loaded()
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/step_audio2/step_audio2_token2wav.py", line 183, in _ensure_models_loaded
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] configs = load_hyperpyyaml(io.StringIO(flow_yaml))
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/hyperpyyaml/core.py", line 189, in load_hyperpyyaml
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] hparams = yaml.load(yaml_stream, Loader=loader)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/yaml/__init__.py", line 81, in load
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] return loader.get_single_data()
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/ruamel/yaml/constructor.py", line 119, in get_single_data
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] return self.construct_document(node)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/ruamel/yaml/constructor.py", line 123, in construct_document
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] data = self.construct_object(node)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/ruamel/yaml/constructor.py", line 150, in construct_object
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] data = self.construct_non_recursive_object(node)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/ruamel/yaml/constructor.py", line 191, in construct_non_recursive_object
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] for _dummy in generator:
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] ^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/ruamel/yaml/constructor.py", line 629, in construct_yaml_map
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] value = self.construct_mapping(node)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/ruamel/yaml/constructor.py", line 425, in construct_mapping
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] return BaseConstructor.construct_mapping(self, node, deep=deep)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/ruamel/yaml/constructor.py", line 247, in construct_mapping
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] value = self.construct_object(value_node, deep=deep)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/ruamel/yaml/constructor.py", line 150, in construct_object
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] data = self.construct_non_recursive_object(node)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/ruamel/yaml/constructor.py", line 186, in construct_non_recursive_object
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] data = constructor(self, tag_suffix, node)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/hyperpyyaml/core.py", line 473, in _construct_object
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] raise ImportError("There is no such class as %s" % callable_string)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ERROR 09-23 15:44:44 [stage_engine_core_proc.py:242] ImportError: There is no such class as stepaudio2.cosyvoice2.flow.flow.CausalMaskedDiffWithXvec
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) Process StageEngineCoreProc_stage2_replica0:
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) Traceback (most recent call last):
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/multiprocessing/process.py", line 314, in _bootstrap
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) self.run()
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/multiprocessing/process.py", line 108, in run
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) self._target(*self._args, **self._kwargs)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_core_proc.py", line 189, in run_stage_core
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) engine_core = StageEngineCoreProc(
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) return func(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/core.py", line 1073, in __init__
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) super().__init__(
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/core.py", line 133, in __init__
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) self.model_executor = executor_class(vllm_config)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) return func(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/executor/abstract.py", line 110, in __init__
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) self._init_executor()
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/executor/uniproc_executor.py", line 74, in _init_executor
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) self.driver_worker.load_model()
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/patch.py", line 738, in load_model
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) out = base_load(self, *args, **kwargs)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/gpu_worker.py", line 457, in load_model
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) self.model_runner.load_model(load_dummy_weights=load_dummy_weights)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) return func(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/init_timeline.py", line 343, in wrapper
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) return fn(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/worker/gpu_model_runner.py", line 188, in load_model
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) super().load_model(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) return func(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/worker/cpu_model_runner.py", line 132, in load_model
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) self.model = get_model(vllm_config=self.vllm_config)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/model_executor/model_loader/__init__.py", line 137, in get_model
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) return loader.load_model(
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) return func(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/model_executor/model_loader/base_loader.py", line 64, in load_model
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) self.load_weights(model, model_config)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) return func(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/model_executor/model_loader/default_loader.py", line 427, in load_weights
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) loaded_weights = model.load_weights(self.get_all_weights(model_config, model))
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py", line 823, in load_weights
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) self._build_backend()
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py", line 864, in _build_backend
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) token2wav = Token2wav(
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_token2wav.py", line 79, in __init__
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) self._core._ensure_models_loaded()
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/step_audio2/step_audio2_token2wav.py", line 183, in _ensure_models_loaded
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) configs = load_hyperpyyaml(io.StringIO(flow_yaml))
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/hyperpyyaml/core.py", line 189, in load_hyperpyyaml
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) hparams = yaml.load(yaml_stream, Loader=loader)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/yaml/__init__.py", line 81, in load
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) return loader.get_single_data()
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/ruamel/yaml/constructor.py", line 119, in get_single_data
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) return self.construct_document(node)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/ruamel/yaml/constructor.py", line 123, in construct_document
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) data = self.construct_object(node)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/ruamel/yaml/constructor.py", line 150, in construct_object
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) data = self.construct_non_recursive_object(node)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/ruamel/yaml/constructor.py", line 191, in construct_non_recursive_object
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) for _dummy in generator:
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/ruamel/yaml/constructor.py", line 629, in construct_yaml_map
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) value = self.construct_mapping(node)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/ruamel/yaml/constructor.py", line 425, in construct_mapping
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) return BaseConstructor.construct_mapping(self, node, deep=deep)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/ruamel/yaml/constructor.py", line 247, in construct_mapping
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) value = self.construct_object(value_node, deep=deep)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/ruamel/yaml/constructor.py", line 150, in construct_object
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) data = self.construct_non_recursive_object(node)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/ruamel/yaml/constructor.py", line 186, in construct_non_recursive_object
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) data = constructor(self, tag_suffix, node)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/hyperpyyaml/core.py", line 473, in _construct_object
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) raise ImportError("There is no such class as %s" % callable_string)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4168702) ImportError: There is no such class as stepaudio2.cosyvoice2.flow.flow.CausalMaskedDiffWithXvec
|
||||
ERROR 09-23 15:44:46 [stage_runtime.py:265] [StageRuntime] Stage initialization failed; shutting down 2 initialized client(s)
|
||||
ERROR 09-23 15:44:46 [stage_runtime.py:265] Traceback (most recent call last):
|
||||
ERROR 09-23 15:44:46 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 252, in initialize
|
||||
ERROR 09-23 15:44:46 [stage_runtime.py:265] initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout)
|
||||
ERROR 09-23 15:44:46 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 15:44:46 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 589, in _initialize_stage_replicas
|
||||
ERROR 09-23 15:44:46 [stage_runtime.py:265] raise primary_exc
|
||||
ERROR 09-23 15:44:46 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 550, in _init_group
|
||||
ERROR 09-23 15:44:46 [stage_runtime.py:265] client = self._initialize_replica(
|
||||
ERROR 09-23 15:44:46 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 15:44:46 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 828, in _initialize_replica
|
||||
ERROR 09-23 15:44:46 [stage_runtime.py:265] return self._initialize_local_llm_replica(plan, stage_init_timeout)
|
||||
ERROR 09-23 15:44:46 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 15:44:46 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 934, in _initialize_local_llm_replica
|
||||
ERROR 09-23 15:44:46 [stage_runtime.py:265] launch_stack.close()
|
||||
ERROR 09-23 15:44:46 [stage_runtime.py:265] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 618, in close
|
||||
ERROR 09-23 15:44:46 [stage_runtime.py:265] self.__exit__(None, None, None)
|
||||
ERROR 09-23 15:44:46 [stage_runtime.py:265] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 610, in __exit__
|
||||
ERROR 09-23 15:44:46 [stage_runtime.py:265] raise exc_details[1]
|
||||
ERROR 09-23 15:44:46 [stage_runtime.py:265] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 595, in __exit__
|
||||
ERROR 09-23 15:44:46 [stage_runtime.py:265] if cb(*exc_details):
|
||||
ERROR 09-23 15:44:46 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 15:44:46 [stage_runtime.py:265] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 144, in __exit__
|
||||
ERROR 09-23 15:44:46 [stage_runtime.py:265] next(self.gen)
|
||||
ERROR 09-23 15:44:46 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_startup.py", line 1211, in launch_stage_replica
|
||||
ERROR 09-23 15:44:46 [stage_runtime.py:265] wait_for_engine_startup(
|
||||
ERROR 09-23 15:44:46 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/utils.py", line 1286, in wait_for_engine_startup
|
||||
ERROR 09-23 15:44:46 [stage_runtime.py:265] raise RuntimeError(
|
||||
ERROR 09-23 15:44:46 [stage_runtime.py:265] RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {}
|
||||
INFO 09-23 15:44:46 [core_client.py:689] [shutdown] MPClient: start timeout=default
|
||||
INFO 09-23 15:44:46 [core_client.py:691] [shutdown] MPClient: stopping engine manager
|
||||
INFO 09-23 15:44:46 [utils.py:615] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage1_replica0
|
||||
INFO 09-23 15:44:49 [core_client.py:693] [shutdown] MPClient: engine manager stopped
|
||||
INFO 09-23 15:44:49 [core_client.py:694] [shutdown] MPClient: cleaning up background resources
|
||||
INFO 09-23 15:44:49 [core_client.py:696] [shutdown] MPClient: complete
|
||||
INFO 09-23 15:44:49 [core_client.py:689] [shutdown] MPClient: start timeout=default
|
||||
INFO 09-23 15:44:49 [utils.py:615] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage0_replica0
|
||||
WARNING 09-23 15:44:54 [utils.py:635] [shutdown] Process manager: force killing remaining processes count=1
|
||||
WARNING 09-23 15:44:54 [utils.py:640] [shutdown] Process manager: force killing remaining process StageEngineCoreProc_stage0_replica0 pid 4168462
|
||||
ERROR 09-23 15:44:54 [async_omni_engine.py:474] [AsyncOmniEngine] Orchestrator thread crashed
|
||||
ERROR 09-23 15:44:54 [async_omni_engine.py:474] Traceback (most recent call last):
|
||||
ERROR 09-23 15:44:54 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 468, in _bootstrap_orchestrator
|
||||
ERROR 09-23 15:44:54 [async_omni_engine.py:474] loop.run_until_complete(_run_orchestrator())
|
||||
ERROR 09-23 15:44:54 [async_omni_engine.py:474] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/base_events.py", line 691, in run_until_complete
|
||||
ERROR 09-23 15:44:54 [async_omni_engine.py:474] return future.result()
|
||||
ERROR 09-23 15:44:54 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 15:44:54 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 424, in _run_orchestrator
|
||||
ERROR 09-23 15:44:54 [async_omni_engine.py:474] self._initialize_stages(stage_init_timeout)
|
||||
ERROR 09-23 15:44:54 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 350, in _initialize_stages
|
||||
ERROR 09-23 15:44:54 [async_omni_engine.py:474] self._initialize_stages_impl(stage_init_timeout)
|
||||
ERROR 09-23 15:44:54 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 371, in _initialize_stages_impl
|
||||
ERROR 09-23 15:44:54 [async_omni_engine.py:474] self._runtime.initialize()
|
||||
ERROR 09-23 15:44:54 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 278, in initialize
|
||||
ERROR 09-23 15:44:54 [async_omni_engine.py:474] raise exc
|
||||
ERROR 09-23 15:44:54 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 252, in initialize
|
||||
ERROR 09-23 15:44:54 [async_omni_engine.py:474] initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout)
|
||||
ERROR 09-23 15:44:54 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 15:44:54 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 589, in _initialize_stage_replicas
|
||||
ERROR 09-23 15:44:54 [async_omni_engine.py:474] raise primary_exc
|
||||
ERROR 09-23 15:44:54 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 550, in _init_group
|
||||
ERROR 09-23 15:44:54 [async_omni_engine.py:474] client = self._initialize_replica(
|
||||
ERROR 09-23 15:44:54 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 15:44:54 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 828, in _initialize_replica
|
||||
ERROR 09-23 15:44:54 [async_omni_engine.py:474] return self._initialize_local_llm_replica(plan, stage_init_timeout)
|
||||
ERROR 09-23 15:44:54 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 15:44:54 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 934, in _initialize_local_llm_replica
|
||||
ERROR 09-23 15:44:54 [async_omni_engine.py:474] launch_stack.close()
|
||||
ERROR 09-23 15:44:54 [async_omni_engine.py:474] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 618, in close
|
||||
ERROR 09-23 15:44:54 [async_omni_engine.py:474] self.__exit__(None, None, None)
|
||||
ERROR 09-23 15:44:54 [async_omni_engine.py:474] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 610, in __exit__
|
||||
ERROR 09-23 15:44:54 [async_omni_engine.py:474] raise exc_details[1]
|
||||
ERROR 09-23 15:44:54 [async_omni_engine.py:474] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 595, in __exit__
|
||||
ERROR 09-23 15:44:54 [async_omni_engine.py:474] if cb(*exc_details):
|
||||
ERROR 09-23 15:44:54 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 15:44:54 [async_omni_engine.py:474] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 144, in __exit__
|
||||
ERROR 09-23 15:44:54 [async_omni_engine.py:474] next(self.gen)
|
||||
ERROR 09-23 15:44:54 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_startup.py", line 1211, in launch_stage_replica
|
||||
ERROR 09-23 15:44:54 [async_omni_engine.py:474] wait_for_engine_startup(
|
||||
ERROR 09-23 15:44:54 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/utils.py", line 1286, in wait_for_engine_startup
|
||||
ERROR 09-23 15:44:54 [async_omni_engine.py:474] raise RuntimeError(
|
||||
ERROR 09-23 15:44:54 [async_omni_engine.py:474] RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {}
|
||||
INFO 09-23 15:44:54 [async_omni_engine.py:1807] [AsyncOmniEngine] Shutting down Orchestrator
|
||||
Exception in thread orchestrator:
|
||||
Traceback (most recent call last):
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
|
||||
self.run()
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/threading.py", line 1012, in run
|
||||
self._target(*self._args, **self._kwargs)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 468, in _bootstrap_orchestrator
|
||||
loop.run_until_complete(_run_orchestrator())
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/base_events.py", line 691, in run_until_complete
|
||||
return future.result()
|
||||
^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 424, in _run_orchestrator
|
||||
self._initialize_stages(stage_init_timeout)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 350, in _initialize_stages
|
||||
self._initialize_stages_impl(stage_init_timeout)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 371, in _initialize_stages_impl
|
||||
self._runtime.initialize()
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 278, in initialize
|
||||
raise exc
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 252, in initialize
|
||||
initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 589, in _initialize_stage_replicas
|
||||
raise primary_exc
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 550, in _init_group
|
||||
client = self._initialize_replica(
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 828, in _initialize_replica
|
||||
return self._initialize_local_llm_replica(plan, stage_init_timeout)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 934, in _initialize_local_llm_replica
|
||||
launch_stack.close()
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 618, in close
|
||||
self.__exit__(None, None, None)
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 610, in __exit__
|
||||
raise exc_details[1]
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 595, in __exit__
|
||||
if cb(*exc_details):
|
||||
^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 144, in __exit__
|
||||
next(self.gen)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_startup.py", line 1211, in launch_stage_replica
|
||||
wait_for_engine_startup(
|
||||
File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/utils.py", line 1286, in wait_for_engine_startup
|
||||
raise RuntimeError(
|
||||
RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {}
|
||||
Traceback (most recent call last):
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
|
||||
self.run()
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/threading.py", line 1012, in run
|
||||
self._target(*self._args, **self._kwargs)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 468, in _bootstrap_orchestrator
|
||||
loop.run_until_complete(_run_orchestrator())
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/base_events.py", line 691, in run_until_complete
|
||||
return future.result()
|
||||
^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 424, in _run_orchestrator
|
||||
self._initialize_stages(stage_init_timeout)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 350, in _initialize_stages
|
||||
self._initialize_stages_impl(stage_init_timeout)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 371, in _initialize_stages_impl
|
||||
self._runtime.initialize()
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 278, in initialize
|
||||
raise exc
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 252, in initialize
|
||||
initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 589, in _initialize_stage_replicas
|
||||
raise primary_exc
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 550, in _init_group
|
||||
client = self._initialize_replica(
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 828, in _initialize_replica
|
||||
return self._initialize_local_llm_replica(plan, stage_init_timeout)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 934, in _initialize_local_llm_replica
|
||||
launch_stack.close()
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 618, in close
|
||||
self.__exit__(None, None, None)
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 610, in __exit__
|
||||
raise exc_details[1]
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 595, in __exit__
|
||||
if cb(*exc_details):
|
||||
^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 144, in __exit__
|
||||
next(self.gen)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_startup.py", line 1211, in launch_stage_replica
|
||||
wait_for_engine_startup(
|
||||
File "/home/zhout/project/edge_infer/.venvs/omni-cpu/lib/python3.12/site-packages/vllm/v1/engine/utils.py", line 1286, in wait_for_engine_startup
|
||||
raise RuntimeError(
|
||||
RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {}
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
|
||||
Traceback (most recent call last):
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/examples/offline_inference/minicpmo/end2end.py", line 519, in <module>
|
||||
main(args)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/examples/offline_inference/minicpmo/end2end.py", line 283, in main
|
||||
omni = Omni(**omni_kwargs)
|
||||
^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/entrypoints/omni_base.py", line 223, in __init__
|
||||
self.engine = AsyncOmniEngine(
|
||||
^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 305, in __init__
|
||||
self._wait_for_orchestrator_init(startup_future, startup_timeout)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 539, in _wait_for_orchestrator_init
|
||||
startup_future.result(
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/concurrent/futures/_base.py", line 456, in result
|
||||
return self.__get_result()
|
||||
^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/concurrent/futures/_base.py", line 401, in __get_result
|
||||
raise self._exception
|
||||
RuntimeError: Orchestrator initialization failed: Engine core initialization failed. See root cause above. Failed core proc(s): {}
|
||||
+234
@@ -0,0 +1,234 @@
|
||||
W0923 15:45:57.706000 4168920 torch/utils/_pytree.py:630] <enum 'KernelPreference'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
W0923 15:45:57.732000 4168920 torch/utils/_pytree.py:630] <enum 'ScaleCalculationMode'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
WARNING 09-23 15:45:59 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton.
|
||||
INFO 09-23 15:45:59 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available.
|
||||
/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions:
|
||||
--> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922
|
||||
--> vLLM version 0.28.0
|
||||
This will likely cause compatibility issues.
|
||||
warn_if_misaligned_vllm_version()
|
||||
INFO 09-23 15:46:00 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 15:46:00 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 15:46:00 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled.
|
||||
INFO 09-23 15:46:00 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable).
|
||||
====================
|
||||
vllm version: 0.28.0
|
||||
====================
|
||||
INFO 09-23 15:46:00 [omni_base.py:216] [Omni] Initializing with model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5
|
||||
INFO 09-23 15:46:00 [async_omni_engine.py:172] [AsyncOmniEngine] Initializing with model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5
|
||||
INFO 09-23 15:46:01 [async_omni_engine.py:290] [AsyncOmniEngine] Launching Orchestrator thread with 3 stages
|
||||
INFO 09-23 15:46:01 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
INFO 09-23 15:46:01 [initialization.py:356] Loaded OmniTransferConfig with 1 connector configurations
|
||||
WARNING 09-23 15:46:01 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 15:46:01 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
WARNING 09-23 15:46:01 [arg_utils.py:1719] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM.
|
||||
INFO 09-23 15:46:01 [model.py:672] Resolved architecture: MiniCPMO45OmniForConditionalGeneration
|
||||
INFO 09-23 15:46:01 [model.py:1965] Using max model len 2048
|
||||
WARNING 09-23 15:46:01 [model.py:963] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False.
|
||||
INFO 09-23 15:46:01 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode.
|
||||
INFO 09-23 15:46:01 [model.py:875] Disabled mm_prefix attention mode because multimodal inputs are configuration-disabled. Attention backends without mm_prefix support may now be selected.
|
||||
INFO 09-23 15:46:01 [scheduler.py:242] Chunked prefill is enabled with max_num_batched_tokens=2048.
|
||||
WARNING 09-23 15:46:01 [vllm.py:1285] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
WARNING 09-23 15:46:01 [vllm.py:1354] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
INFO 09-23 15:46:01 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
WARNING 09-23 15:46:01 [vllm.py:652] Model Runner V2 requires Triton; using the V1 model runner instead.
|
||||
INFO 09-23 15:46:01 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
WARNING 09-23 15:46:01 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 15:46:01 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
WARNING 09-23 15:46:01 [arg_utils.py:1719] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM.
|
||||
INFO 09-23 15:46:01 [model.py:672] Resolved architecture: MiniCPMO45OmniForConditionalGeneration
|
||||
INFO 09-23 15:46:01 [model.py:1965] Using max model len 512
|
||||
WARNING 09-23 15:46:01 [model.py:963] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False.
|
||||
INFO 09-23 15:46:01 [model.py:1965] Using max model len 512
|
||||
INFO 09-23 15:46:01 [scheduler.py:242] Chunked prefill is enabled with max_num_batched_tokens=512.
|
||||
INFO 09-23 15:46:01 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
WARNING 09-23 15:46:01 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 15:46:01 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
WARNING 09-23 15:46:01 [arg_utils.py:1719] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM.
|
||||
INFO 09-23 15:46:01 [model.py:672] Resolved architecture: MiniCPMO45Code2Wav
|
||||
INFO 09-23 15:46:01 [model.py:1965] Using max model len 4096
|
||||
WARNING 09-23 15:46:01 [arg_utils.py:2664] This model does not officially support disabling chunked prefill. Disabling this manually may cause the engine to crash or produce incorrect outputs.
|
||||
INFO 09-23 15:46:01 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
INFO 09-23 15:46:01 [stage_runtime.py:853] [stage_init] Stage-0 set runtime devices: 0
|
||||
W0923 15:46:05.282000 4168948 torch/utils/_pytree.py:630] <enum 'KernelPreference'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
W0923 15:46:05.308000 4168948 torch/utils/_pytree.py:630] <enum 'ScaleCalculationMode'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
WARNING 09-23 15:46:06 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton.
|
||||
INFO 09-23 15:46:06 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available.
|
||||
/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions:
|
||||
--> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922
|
||||
--> vLLM version 0.28.0
|
||||
This will likely cause compatibility issues.
|
||||
warn_if_misaligned_vllm_version()
|
||||
INFO 09-23 15:46:07 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 15:46:07 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 15:46:07 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled.
|
||||
INFO 09-23 15:46:07 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable).
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168948) INFO 09-23 15:46:07 [core.py:122] Initializing a V1 LLM engine (v0.28.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=2048, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=True, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cpu, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': None, 'compile_ranges_endpoints': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': None, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168948) INFO 09-23 15:46:08 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode.
|
||||
[W923 15:46:08.986759514 utils.cpp:68] Warning: NUMA binding: Using MEMBIND policy for memory allocation on the NUMA nodes (0). Memory allocations will be strictly bound to these NUMA nodes. (function init_cpu_memory_env)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168948) WARNING 09-23 15:46:08 [vllm.py:652] Model Runner V2 requires Triton; using the V1 model runner instead.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168948) WARNING 09-23 15:46:08 [cpu_worker.py:127] libiomp is not found in LD_PRELOAD. For best performance, please follow the section `set LD_PRELOAD` in https://docs.vllm.ai/en/latest/getting_started/installation/cpu/ to setup required pre-loaded libraries.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168948) INFO 09-23 15:46:08 [parallel_state.py:1638] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_a3ba980eb94d4dfcb8fd518afa3242cb backend=gloo
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168948) INFO 09-23 15:46:08 [parallel_state.py:1982] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168948) INFO 09-23 15:46:08 [cpu_model_runner.py:131] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5...
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168948) WARNING 09-23 15:46:08 [vllm.py:1285] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168948) WARNING 09-23 15:46:08 [vllm.py:1354] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168948) INFO 09-23 15:46:08 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168948) INFO 09-23 15:46:08 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168948) INFO 09-23 15:46:08 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168948) INFO 09-23 15:46:08 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168948) INFO 09-23 15:46:08 [selector.py:217] Using HND KV cache layout for CPU_ATTN backend.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168948) INFO 09-23 15:46:12 [weight_utils.py:858] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 27.06 GiB.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168948) INFO 09-23 15:46:12 [weight_utils.py:881] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168948) ␍Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168948) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:00<00:00, 233.50it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168948)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168948) INFO 09-23 15:46:20 [default_loader.py:430] Loading weights took 7.98 seconds
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168948) INFO 09-23 15:46:23 [memory.py:91] Released 518 MiB after weight load (anonymous RSS 19822 -> 19304 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory']
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168948) INFO 09-23 15:46:23 [cpu_worker.py:255] Explicitly set (1.0/30.91) GiB for KV cache on node 0.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168948) INFO 09-23 15:46:23 [kv_cache_utils.py:1869] GPU KV cache size: 7,168 tokens, Maximum concurrency for 2,048 tokens per request: 3.50x
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168948) INFO 09-23 15:46:23 [memory.py:91] Released 0 MiB after warm-up (anonymous RSS 20314 -> 20314 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory']
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168948) INFO 09-23 15:46:23 [core.py:354] init engine (profile, create kv cache, warmup model) took 0.52 s
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168948) WARNING 09-23 15:46:23 [scheduler.py:182] Using custom scheduler class vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168948) INFO 09-23 15:46:23 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4168948) INFO 09-23 15:46:24 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
INFO 09-23 15:46:24 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed
|
||||
INFO 09-23 15:46:24 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-0 [rep-0] initializing EngineCore
|
||||
INFO 09-23 15:46:24 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-0 [rep-0] EngineCore running
|
||||
INFO 09-23 15:46:24 [stage_runtime.py:958] [StageRuntime] Stage 0 initialized
|
||||
INFO 09-23 15:46:24 [stage_runtime.py:853] [stage_init] Stage-1 set runtime devices: 0
|
||||
W0923 15:46:31.117000 4169043 torch/utils/_pytree.py:630] <enum 'KernelPreference'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
W0923 15:46:31.158000 4169043 torch/utils/_pytree.py:630] <enum 'ScaleCalculationMode'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
WARNING 09-23 15:46:32 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton.
|
||||
INFO 09-23 15:46:32 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available.
|
||||
/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions:
|
||||
--> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922
|
||||
--> vLLM version 0.28.0
|
||||
This will likely cause compatibility issues.
|
||||
warn_if_misaligned_vllm_version()
|
||||
INFO 09-23 15:46:34 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 15:46:34 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 15:46:34 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled.
|
||||
INFO 09-23 15:46:34 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable).
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4169043) INFO 09-23 15:46:35 [core.py:122] Initializing a V1 LLM engine (v0.28.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=512, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=True, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cpu, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': None, 'compile_ranges_endpoints': [512], 'inductor_compile_config': {'enable_auto_functionalized_v2': False}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': None, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
[W923 15:46:36.481340156 utils.cpp:68] Warning: NUMA binding: Using MEMBIND policy for memory allocation on the NUMA nodes (0). Memory allocations will be strictly bound to these NUMA nodes. (function init_cpu_memory_env)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4169043) WARNING 09-23 15:46:36 [vllm.py:652] Model Runner V2 requires Triton; using the V1 model runner instead.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4169043) WARNING 09-23 15:46:36 [cpu_worker.py:127] libiomp is not found in LD_PRELOAD. For best performance, please follow the section `set LD_PRELOAD` in https://docs.vllm.ai/en/latest/getting_started/installation/cpu/ to setup required pre-loaded libraries.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4169043) INFO 09-23 15:46:36 [parallel_state.py:1638] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_1e217dea20904802b2167eaa6f6f2a31 backend=gloo
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4169043) INFO 09-23 15:46:36 [parallel_state.py:1982] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4169043) WARNING 09-23 15:46:36 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4169043) [transformers] `MiniCPMOProcessor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4169043) [transformers] Requested torchvision backend is not available. Falling back to pil backend.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4169043) [transformers] `MiniCPMOProcessor` defines `feature_extractor_class = 'WhisperFeatureExtractor'`, which is deprecated. Register the correct mapping in `AutoFeatureExtractor` instead.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4169043) [transformers] Image processor MiniCPMVImageProcessor: kwargs ['max_slice_nums'] were applied for backward compatibility. To avoid this warning, add them to valid_kwargs: create a custom TypedDict extending ImagesKwargs with these keys and set it as the `valid_kwargs` class attribute.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4169043) WARNING 09-23 15:46:45 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4169043) WARNING 09-23 15:46:45 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4169043) WARNING 09-23 15:46:45 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4169043) INFO 09-23 15:46:45 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4169043) INFO 09-23 15:46:45 [cpu_model_runner.py:131] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5...
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4169043) WARNING 09-23 15:46:45 [vllm.py:1285] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4169043) WARNING 09-23 15:46:45 [vllm.py:1354] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4169043) INFO 09-23 15:46:45 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4169043) INFO 09-23 15:46:45 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4169043) INFO 09-23 15:46:45 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4169043) INFO 09-23 15:46:45 [selector.py:217] Using HND KV cache layout for CPU_ATTN backend.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4169043) INFO 09-23 15:46:45 [weight_utils.py:858] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 8.10 GiB.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4169043) INFO 09-23 15:46:45 [weight_utils.py:888] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre) and the checkpoint size (17.46 GiB) exceeds 90% of available RAM (8.10 GiB).
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4169043) ␍Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4169043) ␍Loading safetensors checkpoint shards: 25% Completed | 1/4 [00:00<00:01, 2.28it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4169043) ␍Loading safetensors checkpoint shards: 50% Completed | 2/4 [00:01<00:01, 1.42it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4169043) ␍Loading safetensors checkpoint shards: 75% Completed | 3/4 [00:02<00:00, 1.41it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4169043) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:03<00:00, 1.03s/it]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4169043) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:03<00:00, 1.12it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4169043)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4169043) INFO 09-23 15:46:49 [default_loader.py:430] Loading weights took 3.83 seconds
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4169043) INFO 09-23 15:46:49 [memory.py:91] Released 68 MiB after weight load (anonymous RSS 1530 -> 1462 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory']
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4169043) INFO 09-23 15:46:49 [cpu_worker.py:255] Explicitly set (1.0/30.91) GiB for KV cache on node 0.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4169043) INFO 09-23 15:46:49 [kv_cache_utils.py:1869] GPU KV cache size: 17,408 tokens, Maximum concurrency for 512 tokens per request: 34.00x
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4169043) INFO 09-23 15:46:49 [memory.py:91] Released 0 MiB after warm-up (anonymous RSS 2484 -> 2484 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory']
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4169043) INFO 09-23 15:46:49 [core.py:354] init engine (profile, create kv cache, warmup model) took 0.16 s
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4169043) WARNING 09-23 15:46:49 [scheduler.py:182] Using custom scheduler class vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4169043) WARNING 09-23 15:46:49 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4169043) WARNING 09-23 15:46:50 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4169043) INFO 09-23 15:46:50 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4169043) INFO 09-23 15:46:50 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
INFO 09-23 15:46:50 [stage_runtime.py:943] [StageRuntime] Stage 1 engine startup completed
|
||||
INFO 09-23 15:46:50 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-1 [rep-0] initializing EngineCore
|
||||
INFO 09-23 15:46:50 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-1 [rep-0] EngineCore running
|
||||
INFO 09-23 15:46:50 [stage_runtime.py:958] [StageRuntime] Stage 1 initialized
|
||||
INFO 09-23 15:46:50 [stage_runtime.py:853] [stage_init] Stage-2 set runtime devices: 0
|
||||
W0923 15:46:55.369000 4169162 torch/utils/_pytree.py:630] <enum 'KernelPreference'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
W0923 15:46:55.399000 4169162 torch/utils/_pytree.py:630] <enum 'ScaleCalculationMode'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
WARNING 09-23 15:46:56 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton.
|
||||
INFO 09-23 15:46:56 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available.
|
||||
/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions:
|
||||
--> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922
|
||||
--> vLLM version 0.28.0
|
||||
This will likely cause compatibility issues.
|
||||
warn_if_misaligned_vllm_version()
|
||||
INFO 09-23 15:46:57 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 15:46:57 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 15:46:57 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled.
|
||||
INFO 09-23 15:46:57 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable).
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4169162) INFO 09-23 15:46:58 [core.py:122] Initializing a V1 LLM engine (v0.28.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=4096, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=True, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cpu, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=False, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': None, 'compile_ranges_endpoints': [4096], 'inductor_compile_config': {'enable_auto_functionalized_v2': False}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': None, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
[W923 15:46:58.367332736 utils.cpp:68] Warning: NUMA binding: Using MEMBIND policy for memory allocation on the NUMA nodes (0). Memory allocations will be strictly bound to these NUMA nodes. (function init_cpu_memory_env)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4169162) WARNING 09-23 15:46:58 [vllm.py:652] Model Runner V2 requires Triton; using the V1 model runner instead.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4169162) WARNING 09-23 15:46:58 [cpu_worker.py:127] libiomp is not found in LD_PRELOAD. For best performance, please follow the section `set LD_PRELOAD` in https://docs.vllm.ai/en/latest/getting_started/installation/cpu/ to setup required pre-loaded libraries.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4169162) INFO 09-23 15:46:58 [parallel_state.py:1638] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_47ba1048249e4d048745ce4f237328fc backend=gloo
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4169162) INFO 09-23 15:46:58 [parallel_state.py:1982] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4169162) INFO 09-23 15:46:58 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4169162) INFO 09-23 15:46:58 [cpu_model_runner.py:131] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5...
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4169162) INFO 09-23 15:46:58 [weight_utils.py:858] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 6.17 GiB.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4169162) INFO 09-23 15:46:58 [weight_utils.py:888] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre) and the checkpoint size (17.46 GiB) exceeds 90% of available RAM (6.17 GiB).
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4169162) ␍Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4169162) ␍Loading safetensors checkpoint shards: 25% Completed | 1/4 [00:00<00:01, 2.66it/s]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4169162) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:00<00:00, 9.33it/s]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4169162)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4169162) INFO 09-23 15:46:59 [step_audio2_token2wav.py:163] Loading Token2Wav models from: /home/zhout/project/edge_infer/models/MiniCPM-o-4_5/assets/token2wav
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4169162) INFO 09-23 15:47:08 [step_audio2_token2wav.py:201] Token2Wav models loaded successfully
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4169162) INFO 09-23 15:47:08 [default_loader.py:430] Loading weights took 10.48 seconds
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4169162) INFO 09-23 15:47:08 [memory.py:91] Released 515 MiB after weight load (anonymous RSS 4484 -> 3969 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory']
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4169162) INFO 09-23 15:47:08 [memory.py:91] Released 0 MiB after warm-up (anonymous RSS 3969 -> 3969 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory']
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4169162) INFO 09-23 15:47:08 [core.py:354] init engine (profile, create kv cache, warmup model) took 0.00 s
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4169162) WARNING 09-23 15:47:09 [scheduler.py:182] Using custom scheduler class vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4169162) INFO 09-23 15:47:09 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4169162) WARNING 09-23 15:47:10 [vllm.py:1285] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4169162) WARNING 09-23 15:47:10 [vllm.py:1354] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
INFO 09-23 15:47:10 [stage_runtime.py:943] [StageRuntime] Stage 2 engine startup completed
|
||||
INFO 09-23 15:47:10 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-2 [rep-0] initializing EngineCore
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4169162) INFO 09-23 15:47:10 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4169162) INFO 09-23 15:47:10 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
INFO 09-23 15:47:10 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-2 [rep-0] EngineCore running
|
||||
INFO 09-23 15:47:10 [stage_runtime.py:958] [StageRuntime] Stage 2 initialized
|
||||
INFO 09-23 15:47:14 [orchestrator.py:604] [Orchestrator] Starting event loop
|
||||
INFO 09-23 15:47:14 [async_omni_engine.py:323] [AsyncOmniEngine] Orchestrator ready with 3 stages
|
||||
INFO 09-23 15:47:14 [omni_base.py:235] [Omni] AsyncOmniEngine initialized in 73.65 seconds
|
||||
INFO 09-23 15:47:14 [omni_base.py:255] [Omni] Initialized with 3 stages for model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5
|
||||
WARNING 09-23 15:47:14 [input_processor.py:296] Passing raw prompts to InputProcessor is deprecated and will be removed in v0.18. You should instead pass the outputs of Renderer.render_cmpl() or Renderer.render_chat().
|
||||
␍Processed prompts: 0%| | 0/1 [00:00<?, ?it/s](StageEngineCoreProc_stage0_replica0 pid=4168948) WARNING 09-23 15:47:18 [gpu_ar_model_runner.py:1464] prefer_model_sampler model MiniCPMO45OmniForConditionalGeneration returned None from sample(); falling back to the default sampler.
|
||||
␍Processed prompts: 100%|██████████| 1/1 [00:27<00:00, 27.08s/it]␍Processed prompts: 100%|██████████| 1/1 [00:27<00:00, 27.09s/it]
|
||||
query type: text
|
||||
use_tts (tts_bos): True
|
||||
Request ID: 0_28854dbf-b6b7-4f1d-a35a-52f254ab9d9e, Text saved to /home/zhout/project/edge_infer/analysis/experiments/e2e_expansion_20260923/minicpmo_cpu_full_compat4/0_28854dbf-b6b7-4f1d-a35a-52f254ab9d9e.txt
|
||||
Request ID: 0_28854dbf-b6b7-4f1d-a35a-52f254ab9d9e, Saved audio to /home/zhout/project/edge_infer/analysis/experiments/e2e_expansion_20260923/minicpmo_cpu_full_compat4/output_0_28854dbf-b6b7-4f1d-a35a-52f254ab9d9e.wav
|
||||
INFO 09-23 15:47:41 [omni_base.py:781] [Omni] Shutting down
|
||||
INFO 09-23 15:47:41 [async_omni_engine.py:1807] [AsyncOmniEngine] Shutting down Orchestrator
|
||||
INFO 09-23 15:47:41 [orchestrator.py:705] [Orchestrator] Received shutdown signal
|
||||
INFO 09-23 15:47:41 [orchestrator.py:3092] [Orchestrator] Shutting down all 3 client(s)
|
||||
INFO 09-23 15:47:41 [core_client.py:689] [shutdown] MPClient: start timeout=default
|
||||
INFO 09-23 15:47:41 [core_client.py:691] [shutdown] MPClient: stopping engine manager
|
||||
INFO 09-23 15:47:41 [utils.py:615] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage0_replica0
|
||||
WARNING 09-23 15:47:46 [utils.py:635] [shutdown] Process manager: force killing remaining processes count=1
|
||||
WARNING 09-23 15:47:46 [utils.py:640] [shutdown] Process manager: force killing remaining process StageEngineCoreProc_stage0_replica0 pid 4168948
|
||||
INFO 09-23 15:47:46 [core_client.py:693] [shutdown] MPClient: engine manager stopped
|
||||
INFO 09-23 15:47:46 [core_client.py:694] [shutdown] MPClient: cleaning up background resources
|
||||
INFO 09-23 15:47:46 [core_client.py:696] [shutdown] MPClient: complete
|
||||
INFO 09-23 15:47:46 [stage_pool.py:1354] [StagePool] Stage 0 replica 0 shut down
|
||||
INFO 09-23 15:47:46 [core_client.py:689] [shutdown] MPClient: start timeout=default
|
||||
INFO 09-23 15:47:46 [utils.py:615] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage1_replica0
|
||||
INFO 09-23 15:47:49 [stage_pool.py:1354] [StagePool] Stage 1 replica 0 shut down
|
||||
INFO 09-23 15:47:49 [core_client.py:689] [shutdown] MPClient: start timeout=default
|
||||
INFO 09-23 15:47:49 [utils.py:615] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage2_replica0
|
||||
INFO 09-23 15:47:53 [stage_pool.py:1354] [StagePool] Stage 2 replica 0 shut down
|
||||
/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/multiprocessing/resource_tracker.py:279: UserWarning: resource_tracker: There appear to be 1 leaked shared_memory objects to clean up at shutdown
|
||||
warnings.warn('resource_tracker: There appear to be %d '
|
||||
+127
@@ -0,0 +1,127 @@
|
||||
# MiniCPM-o 4.5 offline text-to-text+speech probe on WSL CPU.
|
||||
# This limits context, concurrency and multimodal inputs. Run with
|
||||
# VLLM_TARGET_DEVICE=cpu and VLLM_CPU_KVCACHE_SPACE=1 (GiB per stage).
|
||||
# The CUDA/NPU overrides below are inherited but inactive on this run.
|
||||
pipeline: minicpmo_4_5
|
||||
async_chunk: true
|
||||
|
||||
trust_remote_code: true
|
||||
enable_prefix_caching: false
|
||||
|
||||
connectors:
|
||||
connector_of_shared_memory:
|
||||
name: SharedMemoryConnector
|
||||
extra:
|
||||
connector_get_sleep_s: 0.01
|
||||
connector_get_max_wait_first_chunk: 3000
|
||||
connector_get_max_wait: 300
|
||||
codec_chunk_frames: 25
|
||||
codec_left_context_frames: 3
|
||||
# HiFT capture buckets are derived from codec_chunk_frames and
|
||||
# codec_left_context_frames. (#5869)
|
||||
# A future initial_codec_chunk_frames shape will use lazy capture unless
|
||||
# HiFTGraphWrapper.derive_capture_bucket_size explicitly includes it.
|
||||
enable_hift_graph: false
|
||||
enable_cfm_graph: false
|
||||
cfm_max_graphs: 32
|
||||
|
||||
stages:
|
||||
- stage_id: 0
|
||||
max_num_seqs: 1
|
||||
# Bound the thinker budget; no weights are offloaded from CPU.
|
||||
gpu_memory_utilization: 0.58
|
||||
cpu_offload_gb: 0
|
||||
enforce_eager: true
|
||||
tensor_parallel_size: 1
|
||||
max_num_batched_tokens: 2048
|
||||
max_model_len: 2048
|
||||
devices: "0"
|
||||
# Present in the parent profile; this probe submits text only.
|
||||
interleave_mm_strings: true
|
||||
media_io_kwargs:
|
||||
video:
|
||||
fps: 1
|
||||
num_frames: 128
|
||||
limit_mm_per_prompt:
|
||||
# No image, audio or video inputs are admitted in this text-to-speech probe.
|
||||
image: 0
|
||||
audio: 0
|
||||
video: 0
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 64
|
||||
seed: 42
|
||||
detokenize: true
|
||||
# 1.0 for MCQ accuracy; 1.1 can bias short A–D answers.
|
||||
repetition_penalty: 1.0
|
||||
|
||||
- stage_id: 1
|
||||
max_num_seqs: 1
|
||||
# One short utterance; this does not qualify the full 4096-token Talker.
|
||||
gpu_memory_utilization: 0.10
|
||||
enforce_eager: true
|
||||
max_num_batched_tokens: 512
|
||||
# The checkpoint supports 4096, but this probe explicitly admits 512.
|
||||
max_model_len: 512
|
||||
devices: "0"
|
||||
output_connectors:
|
||||
to_stage_2: connector_of_shared_memory
|
||||
# Talker is a single-vocab codec LM. These knobs go to vLLM's Sampler
|
||||
# (head_code logits). EOS is tts_config.eos_token_id (num_audio_tokens-1).
|
||||
# Values track upstream utils.TTSSamplingParams (min_p/win_size/tau_r are
|
||||
# declared there but never read, so they stay out). repetition_penalty is
|
||||
# consumed by the Talker, which rescores it over upstream's 16-frame window
|
||||
# instead of the Sampler's whole-stream presence penalty.
|
||||
# min_tokens is upstream's min_new_token.
|
||||
default_sampling_params:
|
||||
temperature: 0.8
|
||||
top_p: 0.85
|
||||
top_k: 25
|
||||
repetition_penalty: 1.05
|
||||
min_tokens: 50
|
||||
max_tokens: 128
|
||||
seed: 42
|
||||
detokenize: false
|
||||
|
||||
- stage_id: 2
|
||||
max_num_seqs: 1
|
||||
gpu_memory_utilization: 0.10
|
||||
enforce_eager: true
|
||||
enable_chunked_prefill: false
|
||||
max_num_batched_tokens: 4096
|
||||
max_model_len: 4096
|
||||
devices: "0"
|
||||
input_connectors:
|
||||
from_stage_1: connector_of_shared_memory
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 4096
|
||||
detokenize: true
|
||||
|
||||
platforms:
|
||||
cuda:
|
||||
stages:
|
||||
- stage_id: 1
|
||||
# The cap must still admit the single 512-token Talker session.
|
||||
kv_cache_memory_bytes: 268435456 # 256 MiB for the 512-token probe
|
||||
|
||||
npu:
|
||||
stages:
|
||||
- stage_id: 0
|
||||
max_num_batched_tokens: 8192
|
||||
compilation_config:
|
||||
cudagraph_mode: PIECEWISE
|
||||
- stage_id: 1
|
||||
max_num_batched_tokens: 8192
|
||||
compilation_config:
|
||||
cudagraph_mode: PIECEWISE
|
||||
- stage_id: 2
|
||||
# Stage 2 stays outer-eager and captures only exact-shape CFM DiT
|
||||
# estimator kernels through the NPU platform patch.
|
||||
additional_config:
|
||||
code2wav_enable_npu_graph: true
|
||||
code2wav_max_npu_graphs: 32
|
||||
+1178
File diff suppressed because it is too large.
Load diff
+21
@@ -0,0 +1,21 @@
|
||||
{"request_ids": ["0_3db5a3d3-747f-4482-a1d8-0845b7bac692"], "wall_s": 28.94222697499208, "first_text_s": 28.905936644005124, "first_audio_s": 28.905941563018132, "text": "Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video.", "token_ids": [9707, 11, 358, 1079, 348, 4086, 44, 23620, 76, 7751, 11, 264, 3460, 79049, 57597, 1614, 12875, 315, 8660, 323, 23163, 1467, 11, 5335, 11, 7699, 11, 323, 2766, 13, 151704, 151645], "audio_samples": 124800, "audio_duration_s_at_24khz": 5.2, "audio_rms_float": 0.10130035807634666, "sampled_memory": {"n": 57, "max_wsl_used_bytes": 29457465344, "min_wsl_available_bytes": 3727511552, "max_swap_used_bytes": 5707304960, "max_process_tree_rss_sum_bytes": 29116563456}, "index": 0, "warmup": true}
|
||||
{"request_ids": ["0_698c61b3-b2bf-4b99-9661-e9d0dbef9941"], "wall_s": 22.969852273003198, "first_text_s": 22.96952056302689, "first_audio_s": 22.9695245110197, "text": "Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video.", "token_ids": [9707, 11, 358, 1079, 348, 4086, 44, 23620, 76, 7751, 11, 264, 3460, 79049, 57597, 1614, 12875, 315, 8660, 323, 23163, 1467, 11, 5335, 11, 7699, 11, 323, 2766, 13, 151704, 151645], "audio_samples": 124800, "audio_duration_s_at_24khz": 5.2, "audio_rms_float": 0.10129267401123347, "sampled_memory": {"n": 45, "max_wsl_used_bytes": 29384687616, "min_wsl_available_bytes": 3800289280, "max_swap_used_bytes": 5705441280, "max_process_tree_rss_sum_bytes": 29043417088}, "index": 1, "warmup": false}
|
||||
{"request_ids": ["0_d84be9b7-44e7-4227-8034-4bc39b1778e3"], "wall_s": 21.431721268047113, "first_text_s": 21.431443120003678, "first_audio_s": 21.43144537403714, "text": "Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video.", "token_ids": [9707, 11, 358, 1079, 348, 4086, 44, 23620, 76, 7751, 11, 264, 3460, 79049, 57597, 1614, 12875, 315, 8660, 323, 23163, 1467, 11, 5335, 11, 7699, 11, 323, 2766, 13, 151704, 151645], "audio_samples": 124800, "audio_duration_s_at_24khz": 5.2, "audio_rms_float": 0.10128706657968871, "sampled_memory": {"n": 43, "max_wsl_used_bytes": 29370408960, "min_wsl_available_bytes": 3814567936, "max_swap_used_bytes": 5704372224, "max_process_tree_rss_sum_bytes": 29041909760}, "index": 2, "warmup": false}
|
||||
{"request_ids": ["0_c61278e1-bb87-42a7-be2a-e1b389758895"], "wall_s": 22.179198982019443, "first_text_s": 22.178632276016288, "first_audio_s": 22.178635031043086, "text": "Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video.", "token_ids": [9707, 11, 358, 1079, 348, 4086, 44, 23620, 76, 7751, 11, 264, 3460, 79049, 57597, 1614, 12875, 315, 8660, 323, 23163, 1467, 11, 5335, 11, 7699, 11, 323, 2766, 13, 151704, 151645], "audio_samples": 124800, "audio_duration_s_at_24khz": 5.2, "audio_rms_float": 0.1013044282129624, "sampled_memory": {"n": 44, "max_wsl_used_bytes": 29376024576, "min_wsl_available_bytes": 3808952320, "max_swap_used_bytes": 5704454144, "max_process_tree_rss_sum_bytes": 29042581504}, "index": 3, "warmup": false}
|
||||
{"request_ids": ["0_7ffbbb83-3d6e-4bf9-8647-985099d7e917"], "wall_s": 20.954937847040128, "first_text_s": 20.954634269990493, "first_audio_s": 20.954636403999757, "text": "Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video.", "token_ids": [9707, 11, 358, 1079, 348, 4086, 44, 23620, 76, 7751, 11, 264, 3460, 79049, 57597, 1614, 12875, 315, 8660, 323, 23163, 1467, 11, 5335, 11, 7699, 11, 323, 2766, 13, 151704, 151645], "audio_samples": 124800, "audio_duration_s_at_24khz": 5.2, "audio_rms_float": 0.10130348975702128, "sampled_memory": {"n": 41, "max_wsl_used_bytes": 29339131904, "min_wsl_available_bytes": 3845844992, "max_swap_used_bytes": 5704404992, "max_process_tree_rss_sum_bytes": 29033603072}, "index": 4, "warmup": false}
|
||||
{"request_ids": ["0_6af6781c-6a98-4de8-aec0-3a87cf0b4ae6"], "wall_s": 21.663487254001666, "first_text_s": 21.662724583991803, "first_audio_s": 21.66272842104081, "text": "Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video.", "token_ids": [9707, 11, 358, 1079, 348, 4086, 44, 23620, 76, 7751, 11, 264, 3460, 79049, 57597, 1614, 12875, 315, 8660, 323, 23163, 1467, 11, 5335, 11, 7699, 11, 323, 2766, 13, 151704, 151645], "audio_samples": 124800, "audio_duration_s_at_24khz": 5.2, "audio_rms_float": 0.101298162520547, "sampled_memory": {"n": 43, "max_wsl_used_bytes": 29336969216, "min_wsl_available_bytes": 3848007680, "max_swap_used_bytes": 5745725440, "max_process_tree_rss_sum_bytes": 29032423424}, "index": 5, "warmup": false}
|
||||
{"request_ids": ["0_0e9649e0-ca16-4674-ad0b-ee0ea5c17989"], "wall_s": 21.83056641201256, "first_text_s": 21.83021873200778, "first_audio_s": 21.83022158703534, "text": "Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video.", "token_ids": [9707, 11, 358, 1079, 348, 4086, 44, 23620, 76, 7751, 11, 264, 3460, 79049, 57597, 1614, 12875, 315, 8660, 323, 23163, 1467, 11, 5335, 11, 7699, 11, 323, 2766, 13, 151704, 151645], "audio_samples": 124800, "audio_duration_s_at_24khz": 5.2, "audio_rms_float": 0.10130055496869259, "sampled_memory": {"n": 43, "max_wsl_used_bytes": 29318987776, "min_wsl_available_bytes": 3865989120, "max_swap_used_bytes": 5745344512, "max_process_tree_rss_sum_bytes": 28968747008}, "index": 6, "warmup": false}
|
||||
{"request_ids": ["0_9ccf6e38-8aff-46f8-8ef9-e45132bb530b"], "wall_s": 21.832306760014035, "first_text_s": 21.83202002500184, "first_audio_s": 21.83202205901034, "text": "Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video.", "token_ids": [9707, 11, 358, 1079, 348, 4086, 44, 23620, 76, 7751, 11, 264, 3460, 79049, 57597, 1614, 12875, 315, 8660, 323, 23163, 1467, 11, 5335, 11, 7699, 11, 323, 2766, 13, 151704, 151645], "audio_samples": 124800, "audio_duration_s_at_24khz": 5.2, "audio_rms_float": 0.10130224602764161, "sampled_memory": {"n": 44, "max_wsl_used_bytes": 29306101760, "min_wsl_available_bytes": 3878875136, "max_swap_used_bytes": 5744496640, "max_process_tree_rss_sum_bytes": 28969312256}, "index": 7, "warmup": false}
|
||||
{"request_ids": ["0_a3e00dd7-047a-417b-bbcf-0eaf08502f72"], "wall_s": 21.37202463694848, "first_text_s": 21.371750255988445, "first_audio_s": 21.37175232998561, "text": "Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video.", "token_ids": [9707, 11, 358, 1079, 348, 4086, 44, 23620, 76, 7751, 11, 264, 3460, 79049, 57597, 1614, 12875, 315, 8660, 323, 23163, 1467, 11, 5335, 11, 7699, 11, 323, 2766, 13, 151704, 151645], "audio_samples": 124800, "audio_duration_s_at_24khz": 5.2, "audio_rms_float": 0.10130658512152362, "sampled_memory": {"n": 42, "max_wsl_used_bytes": 29298372608, "min_wsl_available_bytes": 3886604288, "max_swap_used_bytes": 5742460928, "max_process_tree_rss_sum_bytes": 28969422848}, "index": 8, "warmup": false}
|
||||
{"request_ids": ["0_dda0c7f6-f58e-4a75-964e-f51e43745429"], "wall_s": 21.641810890985653, "first_text_s": 21.641490402980708, "first_audio_s": 21.641492767026648, "text": "Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video.", "token_ids": [9707, 11, 358, 1079, 348, 4086, 44, 23620, 76, 7751, 11, 264, 3460, 79049, 57597, 1614, 12875, 315, 8660, 323, 23163, 1467, 11, 5335, 11, 7699, 11, 323, 2766, 13, 151704, 151645], "audio_samples": 124800, "audio_duration_s_at_24khz": 5.2, "audio_rms_float": 0.10130041554975792, "sampled_memory": {"n": 43, "max_wsl_used_bytes": 29298696192, "min_wsl_available_bytes": 3886280704, "max_swap_used_bytes": 5742014464, "max_process_tree_rss_sum_bytes": 28969246720}, "index": 9, "warmup": false}
|
||||
{"request_ids": ["0_0271d8d2-8b77-4da6-8ada-476002c81f8a"], "wall_s": 22.77722787496168, "first_text_s": 22.77689273795113, "first_audio_s": 22.776894591981545, "text": "Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video.", "token_ids": [9707, 11, 358, 1079, 348, 4086, 44, 23620, 76, 7751, 11, 264, 3460, 79049, 57597, 1614, 12875, 315, 8660, 323, 23163, 1467, 11, 5335, 11, 7699, 11, 323, 2766, 13, 151704, 151645], "audio_samples": 124800, "audio_duration_s_at_24khz": 5.2, "audio_rms_float": 0.10129139283031244, "sampled_memory": {"n": 45, "max_wsl_used_bytes": 29306675200, "min_wsl_available_bytes": 3878301696, "max_swap_used_bytes": 5741182976, "max_process_tree_rss_sum_bytes": 28969017344}, "index": 10, "warmup": false}
|
||||
{"request_ids": ["0_5e0e137a-c535-4eb8-aa17-e092e884e49c"], "wall_s": 22.47308399801841, "first_text_s": 22.472258438006975, "first_audio_s": 22.472261073009577, "text": "Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video.", "token_ids": [9707, 11, 358, 1079, 348, 4086, 44, 23620, 76, 7751, 11, 264, 3460, 79049, 57597, 1614, 12875, 315, 8660, 323, 23163, 1467, 11, 5335, 11, 7699, 11, 323, 2766, 13, 151704, 151645], "audio_samples": 124800, "audio_duration_s_at_24khz": 5.2, "audio_rms_float": 0.10130044210140426, "sampled_memory": {"n": 45, "max_wsl_used_bytes": 29309440000, "min_wsl_available_bytes": 3875536896, "max_swap_used_bytes": 5745393664, "max_process_tree_rss_sum_bytes": 28968665088}, "index": 11, "warmup": false}
|
||||
{"request_ids": ["0_4155ef4f-1021-46bd-97be-624734cef687"], "wall_s": 21.860765409015585, "first_text_s": 21.86014178901678, "first_audio_s": 21.860144073027186, "text": "Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video.", "token_ids": [9707, 11, 358, 1079, 348, 4086, 44, 23620, 76, 7751, 11, 264, 3460, 79049, 57597, 1614, 12875, 315, 8660, 323, 23163, 1467, 11, 5335, 11, 7699, 11, 323, 2766, 13, 151704, 151645], "audio_samples": 124800, "audio_duration_s_at_24khz": 5.2, "audio_rms_float": 0.10129724559693552, "sampled_memory": {"n": 43, "max_wsl_used_bytes": 29308547072, "min_wsl_available_bytes": 3876429824, "max_swap_used_bytes": 5745082368, "max_process_tree_rss_sum_bytes": 28954423296}, "index": 12, "warmup": false}
|
||||
{"request_ids": ["0_f2a11efe-86d2-46b6-940a-6baf54edfa44"], "wall_s": 21.485359954996966, "first_text_s": 21.48501689301338, "first_audio_s": 21.485019097977784, "text": "Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video.", "token_ids": [9707, 11, 358, 1079, 348, 4086, 44, 23620, 76, 7751, 11, 264, 3460, 79049, 57597, 1614, 12875, 315, 8660, 323, 23163, 1467, 11, 5335, 11, 7699, 11, 323, 2766, 13, 151704, 151645], "audio_samples": 124800, "audio_duration_s_at_24khz": 5.2, "audio_rms_float": 0.10129644935542514, "sampled_memory": {"n": 43, "max_wsl_used_bytes": 29296164864, "min_wsl_available_bytes": 3888812032, "max_swap_used_bytes": 5716574208, "max_process_tree_rss_sum_bytes": 28909273088}, "index": 13, "warmup": false}
|
||||
{"request_ids": ["0_9c94433c-760d-42f7-b7ac-dae748f3123b"], "wall_s": 22.231876905017998, "first_text_s": 22.231531900004484, "first_audio_s": 22.231534104037564, "text": "Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video.", "token_ids": [9707, 11, 358, 1079, 348, 4086, 44, 23620, 76, 7751, 11, 264, 3460, 79049, 57597, 1614, 12875, 315, 8660, 323, 23163, 1467, 11, 5335, 11, 7699, 11, 323, 2766, 13, 151704, 151645], "audio_samples": 124800, "audio_duration_s_at_24khz": 5.2, "audio_rms_float": 0.10129699068805799, "sampled_memory": {"n": 44, "max_wsl_used_bytes": 29298913280, "min_wsl_available_bytes": 3886063616, "max_swap_used_bytes": 5716029440, "max_process_tree_rss_sum_bytes": 28909584384}, "index": 14, "warmup": false}
|
||||
{"request_ids": ["0_0bf3748f-76d4-40ea-990c-031a13ac7362"], "wall_s": 22.94262919801986, "first_text_s": 22.942252181994263, "first_audio_s": 22.94225467700744, "text": "Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video.", "token_ids": [9707, 11, 358, 1079, 348, 4086, 44, 23620, 76, 7751, 11, 264, 3460, 79049, 57597, 1614, 12875, 315, 8660, 323, 23163, 1467, 11, 5335, 11, 7699, 11, 323, 2766, 13, 151704, 151645], "audio_samples": 124800, "audio_duration_s_at_24khz": 5.2, "audio_rms_float": 0.10129511725657585, "sampled_memory": {"n": 45, "max_wsl_used_bytes": 29295939584, "min_wsl_available_bytes": 3889037312, "max_swap_used_bytes": 5716090880, "max_process_tree_rss_sum_bytes": 28908597248}, "index": 15, "warmup": false}
|
||||
{"request_ids": ["0_911b8c68-5f63-474f-8122-d94ae21058a0"], "wall_s": 23.334465862018988, "first_text_s": 23.333876583026722, "first_audio_s": 23.33387897798093, "text": "Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video.", "token_ids": [9707, 11, 358, 1079, 348, 4086, 44, 23620, 76, 7751, 11, 264, 3460, 79049, 57597, 1614, 12875, 315, 8660, 323, 23163, 1467, 11, 5335, 11, 7699, 11, 323, 2766, 13, 151704, 151645], "audio_samples": 124800, "audio_duration_s_at_24khz": 5.2, "audio_rms_float": 0.10129396359028461, "sampled_memory": {"n": 46, "max_wsl_used_bytes": 29288583168, "min_wsl_available_bytes": 3896393728, "max_swap_used_bytes": 5715787776, "max_process_tree_rss_sum_bytes": 28908490752}, "index": 16, "warmup": false}
|
||||
{"request_ids": ["0_77344a4b-cafe-4d8b-8f12-22992942db04"], "wall_s": 22.133186558028683, "first_text_s": 22.13281638501212, "first_audio_s": 22.132818069017958, "text": "Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video.", "token_ids": [9707, 11, 358, 1079, 348, 4086, 44, 23620, 76, 7751, 11, 264, 3460, 79049, 57597, 1614, 12875, 315, 8660, 323, 23163, 1467, 11, 5335, 11, 7699, 11, 323, 2766, 13, 151704, 151645], "audio_samples": 124800, "audio_duration_s_at_24khz": 5.2, "audio_rms_float": 0.10129915562878435, "sampled_memory": {"n": 44, "max_wsl_used_bytes": 29293105152, "min_wsl_available_bytes": 3891871744, "max_swap_used_bytes": 5715550208, "max_process_tree_rss_sum_bytes": 28907925504}, "index": 17, "warmup": false}
|
||||
{"request_ids": ["0_1c580670-cd86-4428-bc4c-37abe8543f1f"], "wall_s": 21.847536417015363, "first_text_s": 21.84722417400917, "first_audio_s": 21.847226137993857, "text": "Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video.", "token_ids": [9707, 11, 358, 1079, 348, 4086, 44, 23620, 76, 7751, 11, 264, 3460, 79049, 57597, 1614, 12875, 315, 8660, 323, 23163, 1467, 11, 5335, 11, 7699, 11, 323, 2766, 13, 151704, 151645], "audio_samples": 124800, "audio_duration_s_at_24khz": 5.2, "audio_rms_float": 0.10131189393013892, "sampled_memory": {"n": 43, "max_wsl_used_bytes": 29288325120, "min_wsl_available_bytes": 3896651776, "max_swap_used_bytes": 5715267584, "max_process_tree_rss_sum_bytes": 28908838912}, "index": 18, "warmup": false}
|
||||
{"request_ids": ["0_5b6b4e0b-df82-47bb-8786-6dc98e64acff"], "wall_s": 22.6926162270247, "first_text_s": 22.692229292006232, "first_audio_s": 22.692231115011964, "text": "Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video.", "token_ids": [9707, 11, 358, 1079, 348, 4086, 44, 23620, 76, 7751, 11, 264, 3460, 79049, 57597, 1614, 12875, 315, 8660, 323, 23163, 1467, 11, 5335, 11, 7699, 11, 323, 2766, 13, 151704, 151645], "audio_samples": 124800, "audio_duration_s_at_24khz": 5.2, "audio_rms_float": 0.1013045516284855, "sampled_memory": {"n": 45, "max_wsl_used_bytes": 29296058368, "min_wsl_available_bytes": 3888918528, "max_swap_used_bytes": 5715034112, "max_process_tree_rss_sum_bytes": 28908634112}, "index": 19, "warmup": false}
|
||||
{"request_ids": ["0_8e1057dc-fffc-4366-8a07-ef84e9ef324c"], "wall_s": 22.936057427024934, "first_text_s": 22.935763849003706, "first_audio_s": 22.935765352041926, "text": "Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video.", "token_ids": [9707, 11, 358, 1079, 348, 4086, 44, 23620, 76, 7751, 11, 264, 3460, 79049, 57597, 1614, 12875, 315, 8660, 323, 23163, 1467, 11, 5335, 11, 7699, 11, 323, 2766, 13, 151704, 151645], "audio_samples": 124800, "audio_duration_s_at_24khz": 5.2, "audio_rms_float": 0.10130242674324817, "sampled_memory": {"n": 46, "max_wsl_used_bytes": 29296902144, "min_wsl_available_bytes": 3888074752, "max_swap_used_bytes": 5714731008, "max_process_tree_rss_sum_bytes": 28908433408}, "index": 20, "warmup": false}
|
||||
+271
@@ -0,0 +1,271 @@
|
||||
W0923 15:59:42.999000 4170603 torch/utils/_pytree.py:630] <enum 'KernelPreference'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
W0923 15:59:43.041000 4170603 torch/utils/_pytree.py:630] <enum 'ScaleCalculationMode'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
WARNING 09-23 15:59:44 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton.
|
||||
INFO 09-23 15:59:44 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available.
|
||||
/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions:
|
||||
--> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922
|
||||
--> vLLM version 0.28.0
|
||||
This will likely cause compatibility issues.
|
||||
warn_if_misaligned_vllm_version()
|
||||
INFO 09-23 15:59:46 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 15:59:46 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 15:59:46 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled.
|
||||
INFO 09-23 15:59:46 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable).
|
||||
INFO 09-23 15:59:47 [omni_base.py:216] [Omni] Initializing with model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5
|
||||
INFO 09-23 15:59:47 [async_omni_engine.py:172] [AsyncOmniEngine] Initializing with model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5
|
||||
INFO 09-23 15:59:47 [async_omni_engine.py:290] [AsyncOmniEngine] Launching Orchestrator thread with 3 stages
|
||||
INFO 09-23 15:59:47 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
INFO 09-23 15:59:47 [initialization.py:356] Loaded OmniTransferConfig with 1 connector configurations
|
||||
WARNING 09-23 15:59:47 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 15:59:47 [arg_utils.py:1719] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM.
|
||||
INFO 09-23 15:59:47 [model.py:672] Resolved architecture: MiniCPMO45OmniForConditionalGeneration
|
||||
INFO 09-23 15:59:47 [model.py:1965] Using max model len 2048
|
||||
WARNING 09-23 15:59:47 [model.py:963] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False.
|
||||
INFO 09-23 15:59:48 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode.
|
||||
INFO 09-23 15:59:48 [model.py:875] Disabled mm_prefix attention mode because multimodal inputs are configuration-disabled. Attention backends without mm_prefix support may now be selected.
|
||||
INFO 09-23 15:59:48 [scheduler.py:242] Chunked prefill is enabled with max_num_batched_tokens=2048.
|
||||
WARNING 09-23 15:59:48 [vllm.py:1285] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
WARNING 09-23 15:59:48 [vllm.py:1354] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
INFO 09-23 15:59:48 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
WARNING 09-23 15:59:48 [vllm.py:652] Model Runner V2 requires Triton; using the V1 model runner instead.
|
||||
INFO 09-23 15:59:48 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
WARNING 09-23 15:59:48 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 15:59:48 [arg_utils.py:1719] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM.
|
||||
INFO 09-23 15:59:48 [model.py:672] Resolved architecture: MiniCPMO45OmniForConditionalGeneration
|
||||
INFO 09-23 15:59:48 [model.py:1965] Using max model len 512
|
||||
WARNING 09-23 15:59:48 [model.py:963] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False.
|
||||
INFO 09-23 15:59:48 [model.py:1965] Using max model len 512
|
||||
INFO 09-23 15:59:48 [scheduler.py:242] Chunked prefill is enabled with max_num_batched_tokens=512.
|
||||
INFO 09-23 15:59:48 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
WARNING 09-23 15:59:48 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 15:59:48 [arg_utils.py:1719] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM.
|
||||
INFO 09-23 15:59:48 [model.py:672] Resolved architecture: MiniCPMO45Code2Wav
|
||||
INFO 09-23 15:59:48 [model.py:1965] Using max model len 4096
|
||||
WARNING 09-23 15:59:48 [arg_utils.py:2664] This model does not officially support disabling chunked prefill. Disabling this manually may cause the engine to crash or produce incorrect outputs.
|
||||
INFO 09-23 15:59:48 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
INFO 09-23 15:59:48 [stage_runtime.py:853] [stage_init] Stage-0 set runtime devices: 0
|
||||
W0923 15:59:51.983000 4170635 torch/utils/_pytree.py:630] <enum 'KernelPreference'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
W0923 15:59:52.010000 4170635 torch/utils/_pytree.py:630] <enum 'ScaleCalculationMode'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
WARNING 09-23 15:59:53 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton.
|
||||
INFO 09-23 15:59:53 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available.
|
||||
/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions:
|
||||
--> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922
|
||||
--> vLLM version 0.28.0
|
||||
This will likely cause compatibility issues.
|
||||
warn_if_misaligned_vllm_version()
|
||||
INFO 09-23 15:59:54 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 15:59:54 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 15:59:54 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled.
|
||||
INFO 09-23 15:59:54 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable).
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4170635) INFO 09-23 15:59:54 [core.py:122] Initializing a V1 LLM engine (v0.28.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=2048, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=True, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cpu, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': None, 'compile_ranges_endpoints': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': None, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4170635) INFO 09-23 15:59:55 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode.
|
||||
[W923 15:59:55.874166122 utils.cpp:68] Warning: NUMA binding: Using MEMBIND policy for memory allocation on the NUMA nodes (0). Memory allocations will be strictly bound to these NUMA nodes. (function init_cpu_memory_env)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4170635) WARNING 09-23 15:59:55 [vllm.py:652] Model Runner V2 requires Triton; using the V1 model runner instead.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4170635) WARNING 09-23 15:59:55 [cpu_worker.py:127] libiomp is not found in LD_PRELOAD. For best performance, please follow the section `set LD_PRELOAD` in https://docs.vllm.ai/en/latest/getting_started/installation/cpu/ to setup required pre-loaded libraries.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4170635) INFO 09-23 15:59:55 [parallel_state.py:1638] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_36a1b45f19114691ab18107ff3f31f93 backend=gloo
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4170635) INFO 09-23 15:59:55 [parallel_state.py:1982] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4170635) INFO 09-23 15:59:55 [cpu_model_runner.py:131] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5...
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4170635) WARNING 09-23 15:59:55 [vllm.py:1285] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4170635) WARNING 09-23 15:59:55 [vllm.py:1354] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4170635) INFO 09-23 15:59:55 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4170635) INFO 09-23 15:59:55 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4170635) INFO 09-23 15:59:55 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4170635) INFO 09-23 15:59:55 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4170635) INFO 09-23 15:59:55 [selector.py:217] Using HND KV cache layout for CPU_ATTN backend.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4170635) INFO 09-23 15:59:59 [weight_utils.py:858] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 27.11 GiB.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4170635) INFO 09-23 15:59:59 [weight_utils.py:881] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4170635) ␍Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4170635) ␍Loading safetensors checkpoint shards: 25% Completed | 1/4 [00:01<00:03, 1.01s/it]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4170635) ␍Loading safetensors checkpoint shards: 50% Completed | 2/4 [00:02<00:02, 1.07s/it]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4170635) ␍Loading safetensors checkpoint shards: 75% Completed | 3/4 [00:03<00:01, 1.09s/it]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4170635) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:04<00:00, 1.21s/it]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4170635) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:04<00:00, 1.16s/it]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4170635)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4170635) INFO 09-23 16:00:12 [default_loader.py:430] Loading weights took 12.84 seconds
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4170635) INFO 09-23 16:00:14 [memory.py:91] Released 516 MiB after weight load (anonymous RSS 19821 -> 19304 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory']
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4170635) INFO 09-23 16:00:14 [cpu_worker.py:255] Explicitly set (1.0/30.91) GiB for KV cache on node 0.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4170635) INFO 09-23 16:00:14 [kv_cache_utils.py:1869] GPU KV cache size: 7,168 tokens, Maximum concurrency for 2,048 tokens per request: 3.50x
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4170635) INFO 09-23 16:00:14 [memory.py:91] Released 0 MiB after warm-up (anonymous RSS 20314 -> 20314 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory']
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4170635) INFO 09-23 16:00:14 [core.py:354] init engine (profile, create kv cache, warmup model) took 0.17 s
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4170635) WARNING 09-23 16:00:14 [scheduler.py:182] Using custom scheduler class vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4170635) INFO 09-23 16:00:14 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4170635) INFO 09-23 16:00:15 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
INFO 09-23 16:00:15 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed
|
||||
INFO 09-23 16:00:15 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-0 [rep-0] initializing EngineCore
|
||||
INFO 09-23 16:00:15 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-0 [rep-0] EngineCore running
|
||||
INFO 09-23 16:00:15 [stage_runtime.py:958] [StageRuntime] Stage 0 initialized
|
||||
INFO 09-23 16:00:15 [stage_runtime.py:853] [stage_init] Stage-1 set runtime devices: 0
|
||||
W0923 16:00:20.857000 4170732 torch/utils/_pytree.py:630] <enum 'KernelPreference'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
W0923 16:00:20.889000 4170732 torch/utils/_pytree.py:630] <enum 'ScaleCalculationMode'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
WARNING 09-23 16:00:22 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton.
|
||||
INFO 09-23 16:00:22 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available.
|
||||
/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions:
|
||||
--> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922
|
||||
--> vLLM version 0.28.0
|
||||
This will likely cause compatibility issues.
|
||||
warn_if_misaligned_vllm_version()
|
||||
INFO 09-23 16:00:23 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 16:00:24 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 16:00:24 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled.
|
||||
INFO 09-23 16:00:24 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable).
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4170732) INFO 09-23 16:00:24 [core.py:122] Initializing a V1 LLM engine (v0.28.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=512, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=True, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cpu, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': None, 'compile_ranges_endpoints': [512], 'inductor_compile_config': {'enable_auto_functionalized_v2': False}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': None, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
[W923 16:00:25.714479784 utils.cpp:68] Warning: NUMA binding: Using MEMBIND policy for memory allocation on the NUMA nodes (0). Memory allocations will be strictly bound to these NUMA nodes. (function init_cpu_memory_env)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4170732) WARNING 09-23 16:00:25 [vllm.py:652] Model Runner V2 requires Triton; using the V1 model runner instead.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4170732) WARNING 09-23 16:00:25 [cpu_worker.py:127] libiomp is not found in LD_PRELOAD. For best performance, please follow the section `set LD_PRELOAD` in https://docs.vllm.ai/en/latest/getting_started/installation/cpu/ to setup required pre-loaded libraries.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4170732) INFO 09-23 16:00:25 [parallel_state.py:1638] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_7f40d35b6b1a418eb13eb3038a5e4d8e backend=gloo
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4170732) INFO 09-23 16:00:25 [parallel_state.py:1982] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4170732) WARNING 09-23 16:00:25 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4170732) [transformers] `MiniCPMOProcessor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4170732) [transformers] Requested torchvision backend is not available. Falling back to pil backend.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4170732) [transformers] `MiniCPMOProcessor` defines `feature_extractor_class = 'WhisperFeatureExtractor'`, which is deprecated. Register the correct mapping in `AutoFeatureExtractor` instead.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4170732) [transformers] Image processor MiniCPMVImageProcessor: kwargs ['max_slice_nums'] were applied for backward compatibility. To avoid this warning, add them to valid_kwargs: create a custom TypedDict extending ImagesKwargs with these keys and set it as the `valid_kwargs` class attribute.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4170732) WARNING 09-23 16:00:34 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4170732) WARNING 09-23 16:00:34 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4170732) WARNING 09-23 16:00:34 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4170732) INFO 09-23 16:00:34 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4170732) INFO 09-23 16:00:34 [cpu_model_runner.py:131] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5...
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4170732) WARNING 09-23 16:00:34 [vllm.py:1285] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4170732) WARNING 09-23 16:00:34 [vllm.py:1354] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4170732) INFO 09-23 16:00:34 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4170732) INFO 09-23 16:00:34 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4170732) INFO 09-23 16:00:34 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4170732) INFO 09-23 16:00:34 [selector.py:217] Using HND KV cache layout for CPU_ATTN backend.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4170732) INFO 09-23 16:00:35 [weight_utils.py:858] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 8.14 GiB.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4170732) INFO 09-23 16:00:35 [weight_utils.py:888] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre) and the checkpoint size (17.46 GiB) exceeds 90% of available RAM (8.14 GiB).
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4170732) ␍Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4170732) ␍Loading safetensors checkpoint shards: 25% Completed | 1/4 [00:00<00:01, 2.01it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4170732) ␍Loading safetensors checkpoint shards: 50% Completed | 2/4 [00:01<00:01, 1.28it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4170732) ␍Loading safetensors checkpoint shards: 75% Completed | 3/4 [00:01<00:00, 2.02it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4170732) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:02<00:00, 1.76it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4170732) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:02<00:00, 1.73it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4170732)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4170732) INFO 09-23 16:00:37 [default_loader.py:430] Loading weights took 2.47 seconds
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4170732) INFO 09-23 16:00:37 [memory.py:91] Released 71 MiB after weight load (anonymous RSS 1532 -> 1461 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory']
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4170732) INFO 09-23 16:00:37 [cpu_worker.py:255] Explicitly set (1.0/30.91) GiB for KV cache on node 0.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4170732) INFO 09-23 16:00:37 [kv_cache_utils.py:1869] GPU KV cache size: 17,408 tokens, Maximum concurrency for 512 tokens per request: 34.00x
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4170732) INFO 09-23 16:00:39 [memory.py:91] Released 0 MiB after warm-up (anonymous RSS 2483 -> 2483 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory']
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4170732) INFO 09-23 16:00:39 [core.py:354] init engine (profile, create kv cache, warmup model) took 1.50 s
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4170732) WARNING 09-23 16:00:39 [scheduler.py:182] Using custom scheduler class vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4170732) WARNING 09-23 16:00:39 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4170732) WARNING 09-23 16:00:39 [cpu_worker.py:153] CPU backend doesn't allow to use `torch.set_num_threads` after the thread binding, skip it.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4170732) INFO 09-23 16:00:39 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4170732) INFO 09-23 16:00:39 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
INFO 09-23 16:00:39 [stage_runtime.py:943] [StageRuntime] Stage 1 engine startup completed
|
||||
INFO 09-23 16:00:39 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-1 [rep-0] initializing EngineCore
|
||||
INFO 09-23 16:00:40 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-1 [rep-0] EngineCore running
|
||||
INFO 09-23 16:00:40 [stage_runtime.py:958] [StageRuntime] Stage 1 initialized
|
||||
INFO 09-23 16:00:40 [stage_runtime.py:853] [stage_init] Stage-2 set runtime devices: 0
|
||||
W0923 16:00:44.331000 4170848 torch/utils/_pytree.py:630] <enum 'KernelPreference'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
W0923 16:00:44.358000 4170848 torch/utils/_pytree.py:630] <enum 'ScaleCalculationMode'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
WARNING 09-23 16:00:45 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton.
|
||||
INFO 09-23 16:00:45 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available.
|
||||
/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions:
|
||||
--> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922
|
||||
--> vLLM version 0.28.0
|
||||
This will likely cause compatibility issues.
|
||||
warn_if_misaligned_vllm_version()
|
||||
INFO 09-23 16:00:46 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 16:00:46 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 16:00:46 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled.
|
||||
INFO 09-23 16:00:46 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable).
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4170848) INFO 09-23 16:00:47 [core.py:122] Initializing a V1 LLM engine (v0.28.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=4096, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=True, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cpu, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=False, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': None, 'compile_ranges_endpoints': [4096], 'inductor_compile_config': {'enable_auto_functionalized_v2': False}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': None, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
[W923 16:00:47.844157663 utils.cpp:68] Warning: NUMA binding: Using MEMBIND policy for memory allocation on the NUMA nodes (0). Memory allocations will be strictly bound to these NUMA nodes. (function init_cpu_memory_env)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4170848) WARNING 09-23 16:00:47 [vllm.py:652] Model Runner V2 requires Triton; using the V1 model runner instead.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4170848) WARNING 09-23 16:00:47 [cpu_worker.py:127] libiomp is not found in LD_PRELOAD. For best performance, please follow the section `set LD_PRELOAD` in https://docs.vllm.ai/en/latest/getting_started/installation/cpu/ to setup required pre-loaded libraries.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4170848) INFO 09-23 16:00:47 [parallel_state.py:1638] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_d02d9dab0eb747c2a451e899387fbf98 backend=gloo
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4170848) INFO 09-23 16:00:47 [parallel_state.py:1982] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4170848) INFO 09-23 16:00:47 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4170848) INFO 09-23 16:00:47 [cpu_model_runner.py:131] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5...
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4170848) INFO 09-23 16:00:47 [weight_utils.py:858] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 6.67 GiB.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4170848) INFO 09-23 16:00:47 [weight_utils.py:888] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre) and the checkpoint size (17.46 GiB) exceeds 90% of available RAM (6.67 GiB).
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4170848) ␍Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4170848) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:00<00:00, 222.65it/s]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4170848)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4170848) INFO 09-23 16:00:47 [step_audio2_token2wav.py:163] Loading Token2Wav models from: /home/zhout/project/edge_infer/models/MiniCPM-o-4_5/assets/token2wav
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4170848) INFO 09-23 16:00:55 [step_audio2_token2wav.py:201] Token2Wav models loaded successfully
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4170848) INFO 09-23 16:00:55 [default_loader.py:430] Loading weights took 7.55 seconds
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4170848) INFO 09-23 16:00:55 [memory.py:91] Released 516 MiB after weight load (anonymous RSS 4480 -> 3964 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory']
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4170848) INFO 09-23 16:00:55 [memory.py:91] Released 0 MiB after warm-up (anonymous RSS 3964 -> 3964 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory']
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4170848) INFO 09-23 16:00:55 [core.py:354] init engine (profile, create kv cache, warmup model) took 0.00 s
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4170848) WARNING 09-23 16:00:56 [scheduler.py:182] Using custom scheduler class vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4170848) INFO 09-23 16:00:56 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4170848) WARNING 09-23 16:00:56 [vllm.py:1285] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4170848) WARNING 09-23 16:00:56 [vllm.py:1354] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4170848) INFO 09-23 16:00:56 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4170848) INFO 09-23 16:00:56 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
INFO 09-23 16:00:56 [stage_runtime.py:943] [StageRuntime] Stage 2 engine startup completed
|
||||
INFO 09-23 16:00:56 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-2 [rep-0] initializing EngineCore
|
||||
INFO 09-23 16:00:56 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-2 [rep-0] EngineCore running
|
||||
INFO 09-23 16:00:56 [stage_runtime.py:958] [StageRuntime] Stage 2 initialized
|
||||
INFO 09-23 16:01:00 [orchestrator.py:604] [Orchestrator] Starting event loop
|
||||
INFO 09-23 16:01:00 [async_omni_engine.py:323] [AsyncOmniEngine] Orchestrator ready with 3 stages
|
||||
INFO 09-23 16:01:00 [omni_base.py:235] [Omni] AsyncOmniEngine initialized in 73.88 seconds
|
||||
INFO 09-23 16:01:00 [omni_base.py:255] [Omni] Initialized with 3 stages for model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5
|
||||
WARNING 09-23 16:01:01 [input_processor.py:296] Passing raw prompts to InputProcessor is deprecated and will be removed in v0.18. You should instead pass the outputs of Renderer.render_cmpl() or Renderer.render_chat().
|
||||
␍Processed prompts: 0%| | 0/1 [00:00<?, ?it/s](StageEngineCoreProc_stage0_replica0 pid=4170635) WARNING 09-23 16:01:06 [gpu_ar_model_runner.py:1464] prefer_model_sampler model MiniCPMO45OmniForConditionalGeneration returned None from sample(); falling back to the default sampler.
|
||||
␍Processed prompts: 100%|██████████| 1/1 [00:28<00:00, 28.58s/it]␍Processed prompts: 100%|██████████| 1/1 [00:28<00:00, 28.58s/it]
|
||||
request=0 warmup=True wall_s=28.942
|
||||
␍Processed prompts: 0%| | 0/1 [00:00<?, ?it/s]␍Processed prompts: 100%|██████████| 1/1 [00:22<00:00, 22.96s/it]␍Processed prompts: 100%|██████████| 1/1 [00:22<00:00, 22.96s/it]
|
||||
request=1 warmup=False wall_s=22.970
|
||||
␍Processed prompts: 0%| | 0/1 [00:00<?, ?it/s]␍Processed prompts: 100%|██████████| 1/1 [00:21<00:00, 21.43s/it]␍Processed prompts: 100%|██████████| 1/1 [00:21<00:00, 21.43s/it]
|
||||
request=2 warmup=False wall_s=21.432
|
||||
␍Processed prompts: 0%| | 0/1 [00:00<?, ?it/s]␍Processed prompts: 100%|██████████| 1/1 [00:22<00:00, 22.18s/it]␍Processed prompts: 100%|██████████| 1/1 [00:22<00:00, 22.18s/it]
|
||||
request=3 warmup=False wall_s=22.179
|
||||
␍Processed prompts: 0%| | 0/1 [00:00<?, ?it/s]␍Processed prompts: 100%|██████████| 1/1 [00:20<00:00, 20.94s/it]␍Processed prompts: 100%|██████████| 1/1 [00:20<00:00, 20.94s/it]
|
||||
request=4 warmup=False wall_s=20.955
|
||||
␍Processed prompts: 0%| | 0/1 [00:00<?, ?it/s]␍Processed prompts: 100%|██████████| 1/1 [00:21<00:00, 21.66s/it]␍Processed prompts: 100%|██████████| 1/1 [00:21<00:00, 21.66s/it]
|
||||
request=5 warmup=False wall_s=21.663
|
||||
␍Processed prompts: 0%| | 0/1 [00:00<?, ?it/s]␍Processed prompts: 100%|██████████| 1/1 [00:21<00:00, 21.83s/it]␍Processed prompts: 100%|██████████| 1/1 [00:21<00:00, 21.83s/it]
|
||||
request=6 warmup=False wall_s=21.831
|
||||
␍Processed prompts: 0%| | 0/1 [00:00<?, ?it/s]␍Processed prompts: 100%|██████████| 1/1 [00:21<00:00, 21.83s/it]␍Processed prompts: 100%|██████████| 1/1 [00:21<00:00, 21.83s/it]
|
||||
request=7 warmup=False wall_s=21.832
|
||||
␍Processed prompts: 0%| | 0/1 [00:00<?, ?it/s]␍Processed prompts: 100%|██████████| 1/1 [00:21<00:00, 21.36s/it]␍Processed prompts: 100%|██████████| 1/1 [00:21<00:00, 21.36s/it]
|
||||
request=8 warmup=False wall_s=21.372
|
||||
␍Processed prompts: 0%| | 0/1 [00:00<?, ?it/s]␍Processed prompts: 100%|██████████| 1/1 [00:21<00:00, 21.64s/it]␍Processed prompts: 100%|██████████| 1/1 [00:21<00:00, 21.64s/it]
|
||||
request=9 warmup=False wall_s=21.642
|
||||
␍Processed prompts: 0%| | 0/1 [00:00<?, ?it/s]␍Processed prompts: 100%|██████████| 1/1 [00:22<00:00, 22.77s/it]␍Processed prompts: 100%|██████████| 1/1 [00:22<00:00, 22.78s/it]
|
||||
request=10 warmup=False wall_s=22.777
|
||||
␍Processed prompts: 0%| | 0/1 [00:00<?, ?it/s]␍Processed prompts: 100%|██████████| 1/1 [00:22<00:00, 22.47s/it]␍Processed prompts: 100%|██████████| 1/1 [00:22<00:00, 22.47s/it]
|
||||
request=11 warmup=False wall_s=22.473
|
||||
␍Processed prompts: 0%| | 0/1 [00:00<?, ?it/s]␍Processed prompts: 100%|██████████| 1/1 [00:21<00:00, 21.86s/it]␍Processed prompts: 100%|██████████| 1/1 [00:21<00:00, 21.86s/it]
|
||||
request=12 warmup=False wall_s=21.861
|
||||
␍Processed prompts: 0%| | 0/1 [00:00<?, ?it/s]␍Processed prompts: 100%|██████████| 1/1 [00:21<00:00, 21.48s/it]␍Processed prompts: 100%|██████████| 1/1 [00:21<00:00, 21.48s/it]
|
||||
request=13 warmup=False wall_s=21.485
|
||||
␍Processed prompts: 0%| | 0/1 [00:00<?, ?it/s]␍Processed prompts: 100%|██████████| 1/1 [00:22<00:00, 22.23s/it]␍Processed prompts: 100%|██████████| 1/1 [00:22<00:00, 22.23s/it]
|
||||
request=14 warmup=False wall_s=22.232
|
||||
␍Processed prompts: 0%| | 0/1 [00:00<?, ?it/s]␍Processed prompts: 100%|██████████| 1/1 [00:22<00:00, 22.93s/it]␍Processed prompts: 100%|██████████| 1/1 [00:22<00:00, 22.93s/it]
|
||||
request=15 warmup=False wall_s=22.943
|
||||
␍Processed prompts: 0%| | 0/1 [00:00<?, ?it/s]␍Processed prompts: 100%|██████████| 1/1 [00:23<00:00, 23.32s/it]␍Processed prompts: 100%|██████████| 1/1 [00:23<00:00, 23.32s/it]
|
||||
request=16 warmup=False wall_s=23.334
|
||||
␍Processed prompts: 0%| | 0/1 [00:00<?, ?it/s]␍Processed prompts: 100%|██████████| 1/1 [00:22<00:00, 22.13s/it]␍Processed prompts: 100%|██████████| 1/1 [00:22<00:00, 22.13s/it]
|
||||
request=17 warmup=False wall_s=22.133
|
||||
␍Processed prompts: 0%| | 0/1 [00:00<?, ?it/s]␍Processed prompts: 100%|██████████| 1/1 [00:21<00:00, 21.84s/it]␍Processed prompts: 100%|██████████| 1/1 [00:21<00:00, 21.84s/it]
|
||||
request=18 warmup=False wall_s=21.848
|
||||
␍Processed prompts: 0%| | 0/1 [00:00<?, ?it/s]␍Processed prompts: 100%|██████████| 1/1 [00:22<00:00, 22.69s/it]␍Processed prompts: 100%|██████████| 1/1 [00:22<00:00, 22.69s/it]
|
||||
request=19 warmup=False wall_s=22.693
|
||||
␍Processed prompts: 0%| | 0/1 [00:00<?, ?it/s]␍Processed prompts: 100%|██████████| 1/1 [00:22<00:00, 22.92s/it]␍Processed prompts: 100%|██████████| 1/1 [00:22<00:00, 22.92s/it]
|
||||
request=20 warmup=False wall_s=22.936
|
||||
INFO 09-23 16:08:52 [omni_base.py:781] [Omni] Shutting down
|
||||
INFO 09-23 16:08:52 [async_omni_engine.py:1807] [AsyncOmniEngine] Shutting down Orchestrator
|
||||
INFO 09-23 16:08:52 [orchestrator.py:705] [Orchestrator] Received shutdown signal
|
||||
INFO 09-23 16:08:52 [orchestrator.py:3092] [Orchestrator] Shutting down all 3 client(s)
|
||||
INFO 09-23 16:08:52 [core_client.py:689] [shutdown] MPClient: start timeout=default
|
||||
INFO 09-23 16:08:52 [core_client.py:691] [shutdown] MPClient: stopping engine manager
|
||||
INFO 09-23 16:08:52 [utils.py:615] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage0_replica0
|
||||
WARNING 09-23 16:08:57 [utils.py:635] [shutdown] Process manager: force killing remaining processes count=1
|
||||
WARNING 09-23 16:08:57 [utils.py:640] [shutdown] Process manager: force killing remaining process StageEngineCoreProc_stage0_replica0 pid 4170635
|
||||
INFO 09-23 16:08:57 [core_client.py:693] [shutdown] MPClient: engine manager stopped
|
||||
INFO 09-23 16:08:57 [core_client.py:694] [shutdown] MPClient: cleaning up background resources
|
||||
INFO 09-23 16:08:57 [core_client.py:696] [shutdown] MPClient: complete
|
||||
INFO 09-23 16:08:57 [stage_pool.py:1354] [StagePool] Stage 0 replica 0 shut down
|
||||
INFO 09-23 16:08:57 [core_client.py:689] [shutdown] MPClient: start timeout=default
|
||||
INFO 09-23 16:08:57 [utils.py:615] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage1_replica0
|
||||
INFO 09-23 16:09:00 [stage_pool.py:1354] [StagePool] Stage 1 replica 0 shut down
|
||||
INFO 09-23 16:09:00 [core_client.py:689] [shutdown] MPClient: start timeout=default
|
||||
INFO 09-23 16:09:00 [utils.py:615] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage2_replica0
|
||||
INFO 09-23 16:09:04 [stage_pool.py:1354] [StagePool] Stage 2 replica 0 shut down
|
||||
Wrote /home/zhout/project/edge_infer/analysis/experiments/e2e_expansion_20260923/minicpmo_cpu_profile_20.json
|
||||
/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/multiprocessing/resource_tracker.py:279: UserWarning: resource_tracker: There appear to be 1 leaked semaphore objects to clean up at shutdown
|
||||
warnings.warn('resource_tracker: There appear to be %d '
|
||||
/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/multiprocessing/resource_tracker.py:279: UserWarning: resource_tracker: There appear to be 21 leaked shared_memory objects to clean up at shutdown
|
||||
warnings.warn('resource_tracker: There appear to be %d '
|
||||
+106
@@ -0,0 +1,106 @@
|
||||
/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions:
|
||||
--> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922
|
||||
--> vLLM version 0.28.0
|
||||
This will likely cause compatibility issues.
|
||||
warn_if_misaligned_vllm_version()
|
||||
W0923 15:34:52.858000 4167184 torch/utils/_pytree.py:630] <enum 'KernelPreference'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
W0923 15:34:52.905000 4167184 torch/utils/_pytree.py:630] <enum 'ScaleCalculationMode'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
WARNING 09-23 15:34:54 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton.
|
||||
INFO 09-23 15:34:54 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available.
|
||||
INFO 09-23 15:34:56 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 15:34:56 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 15:34:56 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled.
|
||||
INFO 09-23 15:34:56 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable).
|
||||
====================
|
||||
vllm version: 0.28.0
|
||||
====================
|
||||
INFO 09-23 15:34:56 [omni_base.py:216] [Omni] Initializing with model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5
|
||||
INFO 09-23 15:34:56 [async_omni_engine.py:172] [AsyncOmniEngine] Initializing with model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5
|
||||
INFO 09-23 15:34:57 [async_omni_engine.py:290] [AsyncOmniEngine] Launching Orchestrator thread with 1 stages
|
||||
INFO 09-23 15:34:57 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
INFO 09-23 15:34:57 [initialization.py:356] Loaded OmniTransferConfig with 0 connector configurations
|
||||
WARNING 09-23 15:34:57 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 15:34:57 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
WARNING 09-23 15:34:57 [arg_utils.py:1719] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM.
|
||||
INFO 09-23 15:34:57 [model.py:672] Resolved architecture: MiniCPMO45OmniForConditionalGeneration
|
||||
INFO 09-23 15:34:57 [model.py:1965] Using max model len 2048
|
||||
WARNING 09-23 15:34:57 [model.py:963] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False.
|
||||
INFO 09-23 15:34:58 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode.
|
||||
INFO 09-23 15:34:58 [model.py:875] Disabled mm_prefix attention mode because multimodal inputs are configuration-disabled. Attention backends without mm_prefix support may now be selected.
|
||||
INFO 09-23 15:34:58 [scheduler.py:242] Chunked prefill is enabled with max_num_batched_tokens=2048.
|
||||
WARNING 09-23 15:34:58 [vllm.py:1285] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
WARNING 09-23 15:34:58 [vllm.py:1354] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
INFO 09-23 15:34:58 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
WARNING 09-23 15:34:58 [vllm.py:652] Model Runner V2 requires Triton; using the V1 model runner instead.
|
||||
INFO 09-23 15:34:58 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
INFO 09-23 15:34:58 [stage_runtime.py:853] [stage_init] Stage-0 set runtime devices: 0
|
||||
/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/version.py:55: RuntimeWarning: vLLM and vLLM-Omni appear to have mismatched major/minor versions:
|
||||
--> vLLM-Omni version 0.29.0rc2.dev71+g00510ce38.d20260922
|
||||
--> vLLM version 0.28.0
|
||||
This will likely cause compatibility issues.
|
||||
warn_if_misaligned_vllm_version()
|
||||
W0923 15:35:01.933000 4167214 torch/utils/_pytree.py:630] <enum 'KernelPreference'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
W0923 15:35:01.959000 4167214 torch/utils/_pytree.py:630] <enum 'ScaleCalculationMode'> is an Enum subclass and is now natively supported by torch.compile as an opaque value type. Calling register_constant() on Enum subclasses is deprecated and will be an error in a future release.
|
||||
WARNING 09-23 15:35:02 [importing.py:65] Triton is installed, but doesn't include CPU backend. Disabling Triton.
|
||||
INFO 09-23 15:35:02 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available.
|
||||
INFO 09-23 15:35:04 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 15:35:04 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 15:35:04 [patch.py:640] CPU AOT compile-cache load shim installed: compiled models are reused across engine starts instead of being recompiled.
|
||||
INFO 09-23 15:35:04 [patch.py:750] CPU weight-load memory release installed (set VLLM_OMNI_CPU_RELEASE_MEMORY=0 to disable).
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167214) INFO 09-23 15:35:04 [core.py:122] Initializing a V1 LLM engine (v0.28.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=2048, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=True, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cpu, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': None, 'compile_ranges_endpoints': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': None, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167214) INFO 09-23 15:35:05 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode.
|
||||
[W923 15:35:05.941299642 utils.cpp:68] Warning: NUMA binding: Using MEMBIND policy for memory allocation on the NUMA nodes (0). Memory allocations will be strictly bound to these NUMA nodes. (function init_cpu_memory_env)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167214) WARNING 09-23 15:35:05 [vllm.py:652] Model Runner V2 requires Triton; using the V1 model runner instead.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167214) WARNING 09-23 15:35:05 [cpu_worker.py:127] libiomp is not found in LD_PRELOAD. For best performance, please follow the section `set LD_PRELOAD` in https://docs.vllm.ai/en/latest/getting_started/installation/cpu/ to setup required pre-loaded libraries.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167214) INFO 09-23 15:35:05 [parallel_state.py:1638] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_5a5a70874ca14e3ab8e90cde5a3e0034 backend=gloo
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167214) INFO 09-23 15:35:05 [parallel_state.py:1982] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167214) INFO 09-23 15:35:05 [cpu_model_runner.py:131] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5...
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167214) WARNING 09-23 15:35:05 [vllm.py:1285] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167214) WARNING 09-23 15:35:05 [vllm.py:1354] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167214) INFO 09-23 15:35:05 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167214) INFO 09-23 15:35:05 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167214) INFO 09-23 15:35:05 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167214) INFO 09-23 15:35:05 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167214) INFO 09-23 15:35:05 [selector.py:217] Using HND KV cache layout for CPU_ATTN backend.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167214) INFO 09-23 15:35:09 [weight_utils.py:858] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 27.01 GiB.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167214) INFO 09-23 15:35:09 [weight_utils.py:881] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167214) ␍Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167214) ␍Loading safetensors checkpoint shards: 25% Completed | 1/4 [00:00<00:02, 1.01it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167214) ␍Loading safetensors checkpoint shards: 50% Completed | 2/4 [00:02<00:02, 1.05s/it]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167214) ␍Loading safetensors checkpoint shards: 75% Completed | 3/4 [00:03<00:01, 1.09s/it]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167214) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:04<00:00, 1.20s/it]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167214) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:04<00:00, 1.15s/it]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167214)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167214) INFO 09-23 15:35:23 [default_loader.py:430] Loading weights took 13.77 seconds
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167214) INFO 09-23 15:35:29 [memory.py:91] Released 512 MiB after weight load (anonymous RSS 19790 -> 19278 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory']
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167214) INFO 09-23 15:35:29 [cpu_worker.py:255] Explicitly set (2.0/30.91) GiB for KV cache on node 0.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167214) INFO 09-23 15:35:29 [kv_cache_utils.py:1869] GPU KV cache size: 14,464 tokens, Maximum concurrency for 2,048 tokens per request: 7.06x
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167214) INFO 09-23 15:35:29 [memory.py:91] Released 0 MiB after warm-up (anonymous RSS 21316 -> 21316 MiB) via ['libc.so.6:malloc_trim', 'libtcmalloc_minimal.so.4:MallocExtension_ReleaseFreeMemory']
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167214) INFO 09-23 15:35:29 [core.py:354] init engine (profile, create kv cache, warmup model) took 0.27 s
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167214) WARNING 09-23 15:35:29 [scheduler.py:182] Using custom scheduler class vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4167214) INFO 09-23 15:35:29 [kernel.py:308] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
INFO 09-23 15:35:29 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed
|
||||
INFO 09-23 15:35:30 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-0 [rep-0] initializing EngineCore
|
||||
INFO 09-23 15:35:30 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-0 [rep-0] EngineCore running
|
||||
INFO 09-23 15:35:30 [stage_runtime.py:958] [StageRuntime] Stage 0 initialized
|
||||
INFO 09-23 15:35:32 [orchestrator.py:604] [Orchestrator] Starting event loop
|
||||
INFO 09-23 15:35:32 [async_omni_engine.py:323] [AsyncOmniEngine] Orchestrator ready with 1 stages
|
||||
INFO 09-23 15:35:32 [omni_base.py:235] [Omni] AsyncOmniEngine initialized in 36.02 seconds
|
||||
INFO 09-23 15:35:32 [omni_base.py:255] [Omni] Initialized with 1 stages for model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5
|
||||
WARNING 09-23 15:35:33 [input_processor.py:296] Passing raw prompts to InputProcessor is deprecated and will be removed in v0.18. You should instead pass the outputs of Renderer.render_cmpl() or Renderer.render_chat().
|
||||
␍Processed prompts: 0%| | 0/1 [00:00<?, ?it/s](StageEngineCoreProc_stage0_replica0 pid=4167214) WARNING 09-23 15:35:34 [gpu_ar_model_runner.py:1464] prefer_model_sampler model MiniCPMO45OmniForConditionalGeneration returned None from sample(); falling back to the default sampler.
|
||||
␍Processed prompts: 100%|██████████| 1/1 [00:08<00:00, 8.67s/it]␍Processed prompts: 100%|██████████| 1/1 [00:08<00:00, 8.67s/it]
|
||||
query type: text
|
||||
use_tts (tts_bos): False
|
||||
Request ID: 0_9ba10952-b263-4057-a17f-7f09c33fcff8, Text saved to /home/zhout/project/edge_infer/analysis/experiments/e2e_expansion_20260923/minicpmo_cpu_text/0_9ba10952-b263-4057-a17f-7f09c33fcff8.txt
|
||||
INFO 09-23 15:35:41 [omni_base.py:781] [Omni] Shutting down
|
||||
INFO 09-23 15:35:41 [async_omni_engine.py:1807] [AsyncOmniEngine] Shutting down Orchestrator
|
||||
INFO 09-23 15:35:41 [orchestrator.py:705] [Orchestrator] Received shutdown signal
|
||||
INFO 09-23 15:35:41 [orchestrator.py:3092] [Orchestrator] Shutting down all 1 client(s)
|
||||
INFO 09-23 15:35:41 [core_client.py:689] [shutdown] MPClient: start timeout=default
|
||||
INFO 09-23 15:35:41 [core_client.py:691] [shutdown] MPClient: stopping engine manager
|
||||
INFO 09-23 15:35:41 [utils.py:615] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage0_replica0
|
||||
INFO 09-23 15:35:46 [core_client.py:693] [shutdown] MPClient: engine manager stopped
|
||||
INFO 09-23 15:35:46 [core_client.py:694] [shutdown] MPClient: cleaning up background resources
|
||||
INFO 09-23 15:35:46 [core_client.py:696] [shutdown] MPClient: complete
|
||||
INFO 09-23 15:35:46 [stage_pool.py:1354] [StagePool] Stage 0 replica 0 shut down
|
||||
BIN
Binary file not shown.
@@ -1,6 +1,6 @@
|
||||
# E2E check and profiling
|
||||
|
||||
All 60 pairings have a disposition. The timing tables below remain the 2026-09-22 measurements. The configuration matrix and affected per-cell reasons incorporate the [2026-09-23 scoped recovery runs](../../../e2e_recovery_20260923/README.md); those single-request or short-stream runs are not additional completed profiling protocols. Profiling completion does not imply full release qualification.
|
||||
All 60 pairings have a disposition. The timing tables below remain the 2026-09-22 measurements. The configuration matrix and affected per-cell reasons incorporate the [2026-09-23 scoped recovery runs](../../../e2e_recovery_20260923/README.md) and [CPU expansion](../../../e2e_expansion_20260923/README.md). The CPU expansion adds a separate 20-request serial MiniCPM-o text-to-speech profile; other new paths are single-request or short-stream runs. Profiling completion does not imply full release qualification.
|
||||
|
||||
## Configuration × model
|
||||
|
||||
@@ -8,10 +8,10 @@ NOT E2E means no complete device/model workload was verified. “Scoped E2E” i
|
||||
|
||||
| Device | Spark-X2.5 | Qwen3-TTS 0.6B CustomVoice | Qwen3.8-27B | MiniCPM-o 4.5 | InternVLA-A1 |
|
||||
|---|---|---|---|---|---|
|
||||
| PC HX370: CPU / WSL | profile completed | scoped E2E: short stream | NOT E2E | NOT E2E | NOT E2E |
|
||||
| PC HX370: CPU / native Windows | NOT E2E | NOT E2E | NOT E2E | NOT E2E | NOT E2E |
|
||||
| PC HX370: CPU / WSL | profile completed | scoped E2E: short stream | NOT E2E | scoped E2E: text → speech | synthetic policy pass |
|
||||
| PC HX370: CPU / native Windows | NOT E2E | NOT E2E | NOT E2E | NOT E2E | synthetic policy pass |
|
||||
| PC HX370 + RTX 5090 Laptop / WSL | profile completed | profile completed | scoped E2E: text + image | scoped E2E: text → speech | synthetic policy pass |
|
||||
| PC HX370 + RTX 5090 Laptop / Windows | profile completed | profile completed | NOT E2E | NOT E2E | NOT E2E |
|
||||
| PC HX370 + RTX 5090 Laptop / Windows | profile completed | profile completed | NOT E2E | NOT E2E | synthetic policy pass |
|
||||
| PC HX370 + Radeon 890M / Windows worker | NOT E2E | NOT E2E | NOT E2E | NOT E2E | NOT E2E |
|
||||
| PC HX370 + AMD NPU / Windows worker | NOT E2E | NOT E2E | NOT E2E | NOT E2E | NOT E2E |
|
||||
| PC CPU+iGPU+NPU +/- discrete GPU, joint model execution | NOT E2E | NOT E2E | NOT E2E | NOT E2E | NOT E2E |
|
||||
@@ -28,13 +28,13 @@ NOT E2E means no complete device/model workload was verified. “Scoped E2E” i
|
||||
| PC HX370: CPU / WSL | Spark-X2.5 | PROFILE_COMPLETED | not fully qualified: reference-quality and remaining reliability/performance gates must be reviewed | Qualify a matching vLLM 0.29 CPU build and token parity separately. |
|
||||
| PC HX370: CPU / WSL | Qwen3-TTS 0.6B CustomVoice | SCOPED_E2E | The 2026-09-23 compatibility fix enabled real-weight two-frame PCM and four short-stream requests, including cancellation recovery. One measured request generated 3.04 s of audio in 17.02 s (RTF 5.60) with four simulated playback underruns. The CPU wheel remains vLLM 0.28 versus Omni 0.29. | Qualify a matching CPU runtime, speech quality, uninterrupted playback and sustained load; [recovery evidence](../../../e2e_recovery_20260923/README.md). |
|
||||
| PC HX370: CPU / WSL | Qwen3.8-27B | NOT_E2E_PROFILEABLE | No compatible CPU artifact/deployment for this model was found in the checked model roots. Available Qwen 27B FP8/NVFP4 CUDA artifacts do not establish a CPU route. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
| PC HX370: CPU / WSL | MiniCPM-o 4.5 | NOT_E2E_PROFILEABLE | The exact MiniCPM-o 4.5 checkpoint is now downloaded and SHA-256 verified, but no CPU memory admission or complete CPU pipeline has been run. The RTX result does not establish CPU support. | Validate a CPU execution plan and complete text/speech path under its actual memory budget; [weight evidence](../../../e2e_recovery_20260923/evidence/weights/minicpmo.json). |
|
||||
| PC HX370: CPU / WSL | InternVLA-A1 | NOT_E2E_PROFILEABLE | Base and Place_Markpen checkpoints are now SHA-256 verified, but the policy has only run on CUDA with synthetic inputs. No CPU policy forward or task-quality result exists. | Admit the full CPU model and run the policy on real observations with reference actions; [recovery evidence](../../../e2e_recovery_20260923/README.md). |
|
||||
| PC HX370: CPU / WSL | MiniCPM-o 4.5 | SCOPED_E2E | The verified checkpoint completed real three-stage text-to-text+speech requests on the vLLM 0.28 CPU backend: coherent text and nonzero 5.2 s WAVs. A separate 1-warmup/20-measured serial run had request wall p50 21.86 s/p95 22.97 s, with one consistent thinker token sequence. WSL swap reached 5.75 GB; speech meaning was not checked. | Validate speech intelligibility, other input modalities, concurrency, interruption, admission/loading peak and sustained thermal behavior before broader CPU qualification; [CPU evidence](../../../e2e_expansion_20260923/README.md). |
|
||||
| PC HX370: CPU / WSL | InternVLA-A1 | SYNTHETIC_POLICY_PASS | The Place_Markpen checkpoint strict-loaded in CPU-only PyTorch and produced finite `[1, 50, 32]` actions from zero observations/noise in one 3.25 s forward. Compared with the same CUDA checkpoint/input, relative L2 error was 1.30% and cosine similarity 0.999917; no task-quality tolerance is established. | Validate real observations against reference actions, units/timestamps, repeated latency and memory before robot-task qualification; [CPU evidence](../../../e2e_expansion_20260923/README.md). |
|
||||
| PC HX370: CPU / native Windows | Spark-X2.5 | NOT_E2E_PROFILEABLE | The installed native Windows vLLM extension lacks cpu_attn_reshape_and_cache and cpu_attn_get_scheduler_metadata. The separate ORT CPU graph worker is not a whole-model text/TTS adapter; a qualified CPU build/binding is missing. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
| PC HX370: CPU / native Windows | Qwen3-TTS 0.6B CustomVoice | NOT_E2E_PROFILEABLE | The installed native Windows vLLM extension lacks cpu_attn_reshape_and_cache and cpu_attn_get_scheduler_metadata. The separate ORT CPU graph worker is not a whole-model text/TTS adapter; a qualified CPU build/binding is missing. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
| PC HX370: CPU / native Windows | Qwen3.8-27B | NOT_E2E_PROFILEABLE | The installed native Windows vLLM extension lacks cpu_attn_reshape_and_cache and cpu_attn_get_scheduler_metadata. The separate ORT CPU graph worker is not a whole-model text/TTS adapter; a qualified CPU build/binding is missing. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
| PC HX370: CPU / native Windows | MiniCPM-o 4.5 | NOT_E2E_PROFILEABLE | The exact checkpoint is verified in the local Linux model store, but the installed native Windows vLLM extension lacks required CPU attention operators; no native full-model run or memory admission exists. | Establish a native Windows CPU backend and model access, then run the full requested path; [weight evidence](../../../e2e_recovery_20260923/evidence/weights/minicpmo.json). |
|
||||
| PC HX370: CPU / native Windows | InternVLA-A1 | NOT_E2E_PROFILEABLE | Base and fine-tuned checkpoints are verified in the local Linux model store. No native Windows policy run, model access or qualified CPU backend exists; the installed vLLM CPU attention operators are incomplete. | Validate a native policy backend, real observations and reference actions; [recovery evidence](../../../e2e_recovery_20260923/README.md). |
|
||||
| PC HX370: CPU / native Windows | InternVLA-A1 | SYNTHETIC_POLICY_PASS | The Place_Markpen checkpoint strict-loaded from a UNC path and the direct CPU policy produced finite `[1, 50, 32]` actions on synthetic zero observations/noise. Policy/output CPU placement was checked; one forward took 3.36 s. Relative L2 difference from WSL CPU was 1.08% (no task tolerance). This direct policy route does not require native vLLM CPU attention operators. | Validate real observations/reference actions, units/timestamps, memory and repeated latency before task qualification; [native evidence](../../../e2e_expansion_20260923/README.md). |
|
||||
| PC HX370 + RTX 5090 Laptop / WSL | Spark-X2.5 | PROFILE_COMPLETED | not fully qualified: reference-quality and remaining reliability/performance gates must be reviewed | Broaden only with a named workload/device/precision qualification. |
|
||||
| PC HX370 + RTX 5090 Laptop / WSL | Qwen3-TTS 0.6B CustomVoice | PROFILE_COMPLETED | not fully qualified: reference-quality and remaining reliability/performance gates must be reviewed | Resolve and measure playback stalls; run quality, interruption and thermal tests. |
|
||||
| PC HX370 + RTX 5090 Laptop / WSL | Qwen3.8-27B | SCOPED_E2E | A new one-stage Omni binding ran the real NVFP4 checkpoint: one 16-token text request and one synthetic red-image request (answer `Red`) at an explicit 512-token context with host offload. This is not the full multimodal, long-context or quality workload. | Qualify image/video suites, longer contexts, latency distributions and sustained power; [recovery evidence](../../../e2e_recovery_20260923/README.md). |
|
||||
@@ -44,7 +44,7 @@ NOT E2E means no complete device/model workload was verified. “Scoped E2E” i
|
||||
| PC HX370 + RTX 5090 Laptop / Windows | Qwen3-TTS 0.6B CustomVoice | PROFILE_COMPLETED | not fully qualified: reference-quality and remaining reliability/performance gates must be reviewed | Resolve and measure playback stalls; run quality, interruption and thermal tests. |
|
||||
| PC HX370 + RTX 5090 Laptop / Windows | Qwen3.8-27B | NOT_E2E_PROFILEABLE | The Omni pipeline binding now exists and passed scoped text/image cases in WSL, but no native Windows Omni run has verified the NVFP4 checkpoint, memory admission or modality behavior. | Run the same checkpoint and explicit context on native Windows, then broaden quality and profiling; [WSL-only evidence](../../../e2e_recovery_20260923/README.md). |
|
||||
| PC HX370 + RTX 5090 Laptop / Windows | MiniCPM-o 4.5 | NOT_E2E_PROFILEABLE | The exact checkpoint is verified and one constrained three-stage text/speech request passed in WSL; no native Windows execution, memory admission or speech-quality check exists. | Repeat the real three-stage pipeline on native Windows and verify speech and sustained behavior; [WSL-only evidence](../../../e2e_recovery_20260923/README.md). |
|
||||
| PC HX370 + RTX 5090 Laptop / Windows | InternVLA-A1 | NOT_E2E_PROFILEABLE | Base and fine-tuned checkpoints are verified, but the only policy forward used synthetic observations under WSL/CUDA; no native Windows run or real-task qualification exists. | Run a native Windows policy forward, then compare real observations against reference actions; [WSL-only evidence](../../../e2e_recovery_20260923/README.md). |
|
||||
| PC HX370 + RTX 5090 Laptop / Windows | InternVLA-A1 | SYNTHETIC_POLICY_PASS | The Place_Markpen checkpoint strict-loaded from a UNC path and produced finite `[1, 50, 32]` actions on synthetic zero observations/noise. Policy/output CUDA placement was checked; one native forward took 1.12 s, with 6.68 GB peak allocated. Relative L2 difference from WSL CUDA was 0.83% (no task tolerance). | Validate real observations/reference actions, units/timestamps, memory and repeated latency before task qualification; [native evidence](../../../e2e_expansion_20260923/README.md). |
|
||||
| PC HX370 + Radeon 890M / Windows worker | Spark-X2.5 | NOT_E2E_PROFILEABLE | No placement- and quality-verified whole-model graph bundle/stateful adapter for this model on this AMD route. Small backend graphs do not establish model support. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
| PC HX370 + Radeon 890M / Windows worker | Qwen3-TTS 0.6B CustomVoice | NOT_E2E_PROFILEABLE | No placement- and quality-verified whole-model graph bundle/stateful adapter for this model on this AMD route. Small backend graphs do not establish model support. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
| PC HX370 + Radeon 890M / Windows worker | Qwen3.8-27B | NOT_E2E_PROFILEABLE | FP32 448x448 vision tower: DirectML 85/113 nodes; normalized max error 9.095e-06 versus torch CPU. Historical component path, not full 27B or current graph-stage image-to-text integration. | Recover the hashed tower bundle and qualify its current Omni graph handoff with the language stage. |
|
||||
|
||||
@@ -13,6 +13,15 @@ three-stage text-to-speech functional pass after correcting request embeddings,
|
||||
and an InternVLA real-weight synthetic action forward. Model-quality and device gates
|
||||
remain open; the 2026-09-22 audit remains the historical baseline.
|
||||
|
||||
The [WSL CPU expansion](../../benchmarks/edge_harness/results/e2e_expansion_20260923/README.md)
|
||||
adds constrained three-stage MiniCPM-o text-to-speech requests and a CPU-only
|
||||
InternVLA synthetic policy forward. A subsequent 20-request serial MiniCPM-o
|
||||
profile measured p50 21.86 s/p95 22.97 s request wall time after one warmup, with
|
||||
swap use under the 30.91 GiB WSL limit. It does not qualify speech quality,
|
||||
concurrency, streaming, loading peak or sustained behavior. Native Windows
|
||||
InternVLA direct-policy runs separately passed on CPU and RTX 5090 Laptop with
|
||||
synthetic inputs; AMD and mobile cells do not inherit these passes.
|
||||
|
||||
## Architecture and prerequisites
|
||||
|
||||
Keep Omni's PipelineConfig, StageRuntime, StageClient, orchestration and admission
|
||||
@@ -68,8 +77,8 @@ capability and lifecycle implementation; a state-handle type alone is insufficie
|
||||
| M1 | S25 Spark | Device-local Omni-compatible controller; embedding, prefill, continuous decode, ring/full KV, head and sampler; resident state | Reference checks, at least 128 output tokens, 512-window/1024-bucket transitions, cancellation, resident memory and sustained profile |
|
||||
| M2 | Qwen3-TTS | Resolve desktop playback and shutdown findings and CPU compatibility; then mobile talker/predictor plus GPU vocoder | Complete PCM and tail, history/chunk correctness, interruption/recovery, quality checks, no post-startup underruns and RTF below 1 in the declared workload |
|
||||
| M3 | AMD stages | Connect compatible real encoders or other useful coarse stages through current Omni workers; start from the existing 890M vision evidence | Actual node/device placement, numerical and task quality, complete downstream output, shared-memory and handoff costs; reject unhelpful splits |
|
||||
| M4a | MiniCPM-o 4.5 | Three-stage desktop text+WAV path produced coherent text and nonzero WAV under a constrained 24 GiB profile after fixing uninitialized request embeddings; add encoder and persistent-state reference before mobile artifacts | Speech intelligibility/alignment, separate text, image and audio-understanding suites, then combined streaming and interruption |
|
||||
| M4b | InternVLA-A1 | Real weights, observation preprocessing, shared prefix state, iterative action head and versioned action metadata; one strict-load synthetic action forward now passes on RTX 5090 Laptop | Reference action agreement on real observations, units/order/timestamps, observation age, stale-input handling and application-specific deadline reporting |
|
||||
| M4a | MiniCPM-o 4.5 | Three-stage desktop text+WAV produced coherent text and nonzero WAV on RTX 5090 Laptop and WSL CPU under separate constrained plans. The CPU plan needed vLLM 0.28/0.29 processor compatibility and used swap; 20 serial CPU requests measured p50 21.86 s/p95 22.97 s. Add encoder and persistent-state reference before mobile artifacts | Speech intelligibility/alignment, CPU loading-peak admission, concurrency, separate text, image and audio-understanding suites, then combined streaming and interruption |
|
||||
| M4b | InternVLA-A1 | Real weights, observation preprocessing, shared prefix state, iterative action head and versioned action metadata; strict-load synthetic policy forwards now pass on WSL and native Windows, each on CPU and RTX 5090 Laptop | Reference action agreement on real observations, units/order/timestamps, observation age, stale-input handling and application-specific deadline reporting |
|
||||
| Independent | Qwen3.8-27B | Public Omni pipeline binding; text, then image/video and longer contexts; same-model CPU/iGPU artifact and explicit offload evaluation | Modality-specific quality, actual loading/runtime memory and measured performance; each precision/context receives its own qualification |
|
||||
|
||||
The initial M0 desktop text acceptance remains scoped to its tested checkpoint,
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
"""MiniCPM-o processor contract across vLLM 0.28 CPU and 0.29 CUDA."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
from transformers.feature_extraction_utils import BatchFeature
|
||||
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni_llm import (
|
||||
MiniCPMO45OmniLLMMultiModalProcessor,
|
||||
)
|
||||
|
||||
|
||||
class _EmptyItems:
|
||||
def get_all_counts(self):
|
||||
return {}
|
||||
|
||||
def select(self, _keys):
|
||||
return self
|
||||
|
||||
|
||||
def _processor():
|
||||
tokenizer = SimpleNamespace(encode=lambda prompt: [len(prompt)])
|
||||
return SimpleNamespace(
|
||||
_get_hf_mm_data=lambda _items: ({}, {}),
|
||||
process_mm_inputs=lambda _data, _kwargs: {},
|
||||
info=SimpleNamespace(get_tokenizer=lambda: tokenizer),
|
||||
dummy_inputs=SimpleNamespace(get_dummy_text=lambda _counts: "dummy"),
|
||||
)
|
||||
|
||||
|
||||
def test_vllm_028_contract_tokenizes_prompt_without_hf_multimodal_call():
|
||||
result = MiniCPMO45OmniLLMMultiModalProcessor._apply_hf_processor_main(
|
||||
_processor(),
|
||||
_EmptyItems(),
|
||||
{},
|
||||
prompt="(<audio>./</audio>)",
|
||||
tokenization_kwargs={},
|
||||
enable_hf_prompt_update=False,
|
||||
)
|
||||
|
||||
prompt_ids, mm_data, updated = result
|
||||
assert prompt_ids == [len("(<audio>./</audio>)")]
|
||||
assert isinstance(mm_data, BatchFeature) and not mm_data
|
||||
assert updated is False
|
||||
|
||||
|
||||
def test_vllm_029_contract_returns_batch_feature():
|
||||
result = MiniCPMO45OmniLLMMultiModalProcessor._apply_hf_processor_main(
|
||||
_processor(), _EmptyItems(), {}
|
||||
)
|
||||
|
||||
assert isinstance(result, BatchFeature)
|
||||
torch.testing.assert_close(result["input_ids"], torch.tensor([[5]]))
|
||||
@@ -21,6 +21,7 @@ import inspect
|
||||
import math
|
||||
import os
|
||||
import warnings
|
||||
from collections import defaultdict
|
||||
from collections.abc import Callable, Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass, replace
|
||||
from functools import partial
|
||||
@@ -3422,24 +3423,37 @@ class MiniCPMO45OmniLLMMultiModalProcessor(BaseMultiModalProcessor[MiniCPMO45Omn
|
||||
self,
|
||||
mm_items: MultiModalDataItems,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
prompt: str | list[int] | None = None,
|
||||
tokenization_kwargs: Mapping[str, object] | None = None,
|
||||
*,
|
||||
enable_hf_prompt_update: bool | None = None,
|
||||
) -> BatchFeature | tuple[list[int], BatchFeature, bool]:
|
||||
"""
|
||||
Process each modality independently because the MiniCPM processor
|
||||
asserts that image tags and image sizes have matching lengths.
|
||||
|
||||
vLLM 0.28 passes the prompt here and expects a three-tuple; 0.29
|
||||
tokenizes the prompt separately and expects only the BatchFeature.
|
||||
"""
|
||||
valid_mm_items = mm_items.select({key for key, count in mm_items.get_all_counts().items() if count > 0})
|
||||
mm_data, passthrough_data = self._get_hf_mm_data(valid_mm_items)
|
||||
|
||||
mm_inputs = self.process_mm_inputs(mm_data, hf_processor_mm_kwargs)
|
||||
if prompt is not None:
|
||||
if isinstance(prompt, str):
|
||||
# vLLM 0.28's generic text-only path invokes the HF
|
||||
# multimodal processor without media, which rejects MiniCPM
|
||||
# modality placeholders in dummy prompts.
|
||||
prompt_ids = self.info.get_tokenizer().encode(prompt)
|
||||
else:
|
||||
prompt_ids = self._apply_hf_processor_tokens_only(prompt)
|
||||
processed_data = BatchFeature(dict(mm_inputs))
|
||||
processed_data.update(passthrough_data)
|
||||
return prompt_ids, processed_data, False
|
||||
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
prompt_text = self.dummy_inputs.get_dummy_text(mm_items.get_all_counts())
|
||||
input_ids = torch.tensor([tokenizer.encode(prompt_text)])
|
||||
mm_inputs = self.process_mm_inputs(mm_data, hf_processor_mm_kwargs)
|
||||
processed_data = BatchFeature(
|
||||
{
|
||||
"input_ids": input_ids,
|
||||
**mm_inputs,
|
||||
}
|
||||
)
|
||||
processed_data = BatchFeature({"input_ids": torch.tensor([tokenizer.encode(prompt_text)]), **mm_inputs})
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
|
||||
@@ -3597,13 +3611,32 @@ class MiniCPMO45OmniLLMMultiModalProcessor(BaseMultiModalProcessor[MiniCPMO45Omn
|
||||
) -> dict[str, NestedTensors]:
|
||||
from vllm.model_executor.models.minicpmv import MiniCPMVMultiModalProcessor
|
||||
|
||||
return MiniCPMVMultiModalProcessor._call_hf_processor_on_prompts(
|
||||
self,
|
||||
prompts,
|
||||
mm_data,
|
||||
{key: value for key, value in mm_kwargs.items() if key != "use_tts"},
|
||||
out_keys=out_keys,
|
||||
)
|
||||
mm_kwargs = {key: value for key, value in mm_kwargs.items() if key != "use_tts"}
|
||||
call_prompts = getattr(MiniCPMVMultiModalProcessor, "_call_hf_processor_on_prompts", None)
|
||||
if call_prompts is not None:
|
||||
return call_prompts(self, prompts, mm_data, mm_kwargs, out_keys=out_keys)
|
||||
|
||||
# vLLM 0.28 predates this MiniCPMV helper. Keep the same per-modality
|
||||
# processing contract while using its available HF processor context.
|
||||
if self.info.get_model_version() in {(2, 6), (4, 0), (4, 5), (4, 6)}:
|
||||
inputs = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**mm_kwargs),
|
||||
dict(text=prompts, **mm_data),
|
||||
mm_kwargs,
|
||||
)
|
||||
else:
|
||||
inputs = defaultdict(list)
|
||||
for i, prompt in enumerate(prompts):
|
||||
inputs_one = self.info.ctx.call_hf_processor(
|
||||
self.info.get_hf_processor(**mm_kwargs),
|
||||
dict(text=prompt, **{key: value[i] for key, value in mm_data.items()}),
|
||||
mm_kwargs,
|
||||
)
|
||||
for key, value in inputs_one.items():
|
||||
if len(value) != 1:
|
||||
raise ValueError(f"Expected one {key} item per MiniCPM-o prompt, got {len(value)}")
|
||||
inputs[key].append(value[0])
|
||||
return {key: inputs[key] for key in out_keys}
|
||||
|
||||
def _get_prompt_updates(
|
||||
self,
|
||||
|
||||
@@ -536,6 +536,12 @@ _patch_inductor_factorable_divisibility()
|
||||
# (not a slot or C extension). If CuMemAllocator is rewritten in C/Cython,
|
||||
# this monkey-patch will silently become a no-op.
|
||||
def _patch_cumem_free_callback_cuda() -> None:
|
||||
if sys.platform == "win32":
|
||||
# The native Windows vLLM build may not load libcudart before this
|
||||
# module is imported. This Linux CUDA atexit workaround is not needed
|
||||
# to import Omni on that platform.
|
||||
_PATCH_LOGGER.debug("[cumem-cuda] native Windows; skipping patch")
|
||||
return
|
||||
try:
|
||||
import torch
|
||||
|
||||
|
||||
Reference in new issue
Block a user