Repair MiniCPM-o and InternVLA scoped E2E paths
This commit is contained in:
tzhouam committed
1 parent
2d8b1d4583
commit
c0deaa76ba
36 files changed
+3498
-83
No files matched your search
@@ -0,0 +1,125 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Real-checkpoint InternVLA action smoke with synthetic observations.
|
||||
|
||||
This probes loading and one full policy forward. Synthetic images and state do
|
||||
not establish robot-task quality, physical units, or control safety.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--model-dir", type=Path, required=True)
|
||||
parser.add_argument("--cosmos-dir", type=Path, required=True)
|
||||
parser.add_argument("--processor-dir", type=Path, required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--device", default="cuda")
|
||||
args = parser.parse_args()
|
||||
|
||||
os.environ["INTERNVLA_A1_COSMOS_DIR"] = str(args.cosmos_dir.resolve())
|
||||
os.environ["INTERNVLA_A1_PROCESSOR_DIR"] = str(args.processor_dir.resolve())
|
||||
os.environ["HF_HUB_OFFLINE"] = "1"
|
||||
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.registry import initialize_model
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
model_dir = args.model_dir.resolve()
|
||||
if not (model_dir / "model.safetensors").is_file():
|
||||
raise FileNotFoundError(f"Missing real InternVLA checkpoint: {model_dir / 'model.safetensors'}")
|
||||
|
||||
config = OmniDiffusionConfig(
|
||||
model=str(model_dir),
|
||||
model_class_name="InternVLAA1Pipeline",
|
||||
dtype=torch.bfloat16,
|
||||
custom_pipeline_args={
|
||||
"device": args.device,
|
||||
"dtype": "bfloat16",
|
||||
"compile_model": False,
|
||||
"enable_regional_compile": False,
|
||||
"enable_warmup": False,
|
||||
"strict_load": True,
|
||||
"processor_model_name": str(args.processor_dir.resolve()),
|
||||
},
|
||||
)
|
||||
started = time.perf_counter()
|
||||
pipeline = initialize_model(config)
|
||||
if pipeline.runtime_mode() != "real_checkpoint_loaded":
|
||||
raise RuntimeError(f"Unexpected InternVLA runtime mode: {pipeline.runtime_mode()}")
|
||||
startup_s = time.perf_counter() - started
|
||||
|
||||
batch_inputs = pipeline._build_fake_batch_inputs()
|
||||
noise = torch.zeros(
|
||||
(1, pipeline.config.chunk_size, pipeline.config.max_action_dim),
|
||||
device=args.device,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
if args.device.startswith("cuda"):
|
||||
torch.accelerator.synchronize()
|
||||
torch.accelerator.reset_peak_memory_stats()
|
||||
started = time.perf_counter()
|
||||
output = pipeline.forward(
|
||||
DiffusionRequestBatch(
|
||||
requests=[
|
||||
OmniDiffusionRequest(
|
||||
prompt="",
|
||||
sampling_params=OmniDiffusionSamplingParams(
|
||||
extra_args={"batch_inputs": batch_inputs, "noise": noise, "decode_image": False}
|
||||
),
|
||||
request_id="internvla-real-weight-synthetic-observation",
|
||||
)
|
||||
]
|
||||
)
|
||||
)
|
||||
if args.device.startswith("cuda"):
|
||||
torch.accelerator.synchronize()
|
||||
forward_s = time.perf_counter() - started
|
||||
if output.error:
|
||||
raise RuntimeError(output.error)
|
||||
result = output.output
|
||||
actions = result.get("payload", {}).get("actions") if isinstance(result, dict) else None
|
||||
if not isinstance(actions, torch.Tensor) or actions.shape != (
|
||||
1,
|
||||
pipeline.config.chunk_size,
|
||||
pipeline.config.max_action_dim,
|
||||
):
|
||||
raise RuntimeError(f"Unexpected action output: {type(actions).__name__}")
|
||||
if not torch.isfinite(actions).all():
|
||||
raise RuntimeError("InternVLA returned non-finite actions")
|
||||
|
||||
action_bytes = actions.detach().float().cpu().contiguous().numpy().tobytes()
|
||||
report = {
|
||||
"scope": "real checkpoint, synthetic zero observations and noise; no task-quality claim",
|
||||
"model_dir": str(model_dir),
|
||||
"runtime_mode": pipeline.runtime_mode(),
|
||||
"device": args.device,
|
||||
"dtype": str(actions.dtype),
|
||||
"action_shape": list(actions.shape),
|
||||
"finite": True,
|
||||
"action_sha256_float32": hashlib.sha256(action_bytes).hexdigest(),
|
||||
"first_action_prefix": actions[0, 0, :8].detach().float().cpu().tolist(),
|
||||
"startup_s": startup_s,
|
||||
"forward_s": forward_s,
|
||||
"cuda_peak_allocated_bytes": (
|
||||
torch.accelerator.max_memory_allocated() if args.device.startswith("cuda") else None
|
||||
),
|
||||
}
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps(report, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,31 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run the MiniCPM-o thinker alone to isolate text quality from speech stages.
|
||||
|
||||
This temporarily narrows the registered pipeline in this process. It is a
|
||||
diagnostic route, not a replacement for the shipped three-stage topology.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import replace
|
||||
|
||||
from vllm_omni.config.pipeline_registry import OMNI_PIPELINES
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.pipeline import MINICPMO_4_5_PIPELINE
|
||||
|
||||
|
||||
def main() -> None:
|
||||
OMNI_PIPELINES["minicpmo_4_5"] = replace(
|
||||
MINICPMO_4_5_PIPELINE,
|
||||
stages=(MINICPMO_4_5_PIPELINE.stages[0],),
|
||||
duplex_runtime_extension=None,
|
||||
duplex_serving_adapter=None,
|
||||
duplex_control_enabled=False,
|
||||
)
|
||||
from examples.offline_inference.minicpmo.end2end import main as run_example
|
||||
from examples.offline_inference.minicpmo.end2end import parse_args
|
||||
|
||||
run_example(parse_args())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -6,15 +6,23 @@ This is a follow-up to the [60-cell audit](../e2e_profiling_20260922/README.md).
|
||||
|---|---|---|
|
||||
| WSL CPU / Qwen3-TTS 0.6B CustomVoice, revision `85e237c12c027371202489a0ec509ded67b5e4b5` | Two-frame real-weight PCM probe passed (1,920 nonzero finite samples). Full short-utterance streaming completed four requests, including a slow consumer and a request after cancellation. | Measured short request: 3.04 s audio, 17.02 s wall time, RTF 5.60 and four simulated playback underruns. No ASR, speaker, listening or sustained thermal qualification. |
|
||||
| WSL RTX 5090 Laptop / Qwen3.8-27B NVFP4, local revision `dbb8f445b3145f8a4c18ddc769f032d57d32867c` | Public Omni text request completed 16 tokens. Public Omni image-to-text request with a synthetic red 96×96 image answered `Red`. | One text and one image case only; video, long context, quality suites, power and sustained workload unqualified. |
|
||||
| WSL RTX 5090 Laptop / InternVLA-A1-3B Place_Markpen fine-tune, revision `0a557c6a503e6545b369bff2f34c76a410bbd190` | Strict real-weight load and one full policy forward with synthetic zero observations/noise passed: finite `[1, 50, 32]` action tensor; measured forward 1.222 s. | No Place_Markpen observation dataset/reference actions, units, timestamps or robot-task quality check; single-run timing is not a latency distribution. |
|
||||
| WSL RTX 5090 Laptop / MiniCPM-o 4.5, revision `503e754207c94da6bb26850b4469f367c9ea3582` | Under an explicit one-session, 2,048-token thinker / 512-token talker / 5 GiB CPU-offload profile, the actual three-stage Omni pipeline completed one text-to-text+speech request. The text was a coherent introduction and the 24 kHz WAV was nonzero. | Functional pass for this one text-to-speech request only. Speech intelligibility, text/audio alignment, other input modalities, interruption, sustained load, power and quality suites remain unqualified. |
|
||||
|
||||
Evidence: [text request](evidence/qwen_text.json), [image request](evidence/qwen_image.json), [two-frame audio](evidence/tts_two_frame.json), [stream report](evidence/tts_stream_report.json), [individual stream requests](evidence/tts_stream_requests.jsonl). The image input and exact prompt are reproducible with [the image probe](../../probe_qwen35_image.py). The streaming run used [profile_local_tts.py](../../profile_local_tts.py) with `--length-band short --concurrency 1 --repeats 1 --sustained-seconds 0`; its fixed slow-consumer and cancellation probes also ran. The CPU vLLM wheel was 0.28.0, Omni source reported 0.29.0rc2, and the installed PyTorch CPU build was 2.13.0. Qwen used the installed vLLM 0.29.0 CUDA wheel, WSL kernel `6.18.33.2-microsoft-standard-WSL2`, RTX 5090 Laptop (24,463 MiB), driver 610.71, BF16 compute, eager mode, 2 GiB explicit host offload, one sequence and 512-token context. All values are single-run observations; they are not medians.
|
||||
|
||||
The code changes make vLLM-native `qwen3_5` a one-stage Omni pipeline, project explicit `language_model_only` and `cpu_offload_gb` to the owning stage, skip vLLM's `-1` unmapped-slot sentinel in both prefix-cache write paths, and accept both vLLM 0.28 three-field and 0.29 four-field min-token state. The image probe first refused a 1,024-token context because the available 0.14 GiB KV cache was below the 0.38 GiB requirement. The successful run explicitly used a 512-token context and 0.85 GPU memory utilization. No context or precision was changed silently.
|
||||
|
||||
The missing primary MiniCPM-o 4.5 and InternVLA-A1 checkpoints are being acquired at pinned revisions; no E2E result is claimed until their transfers finish and real checkpoint runs pass. The required InternVLA Cosmos encoder/decoder and Qwen3-VL processor files have been downloaded separately. Existing Spark-X2.5 4B and 1.7B INT8, Qwen3-TTS, Qwen3.8 FP8 and Qwen3.8 NVFP4 artifacts already covered those model families. Additional official Spark 1.7B BF16 and Qwen3.8-27B BF16 transfers were paused when the Hugging Face data path slowed; they are distinct artifact variants, not replacements for the present weights. InternVLA source checkpoints carry a CC BY-NC-SA 4.0 license; the current download is for local research validation.
|
||||
The missing primary MiniCPM-o 4.5 and InternVLA-A1 checkpoints are now fully downloaded at pinned revisions. Every safetensors file was SHA-256 checked against the corresponding Hugging Face metadata: [MiniCPM-o](evidence/weights/minicpmo.json), [InternVLA base](evidence/weights/internvla_base.json), and [InternVLA Place_Markpen fine-tune](evidence/weights/internvla_place_markpen.json). The required InternVLA Cosmos encoder/decoder and Qwen3-VL processor files were downloaded separately. Existing Spark-X2.5 4B and 1.7B INT8, Qwen3-TTS, Qwen3.8 FP8 and Qwen3.8 NVFP4 artifacts covered those model families. Optional official Spark 1.7B BF16 and Qwen3.8-27B BF16 transfers remain paused; they are additional artifact variants, not replacements for the present family weights. InternVLA source checkpoints carry a CC BY-NC-SA 4.0 license; the current download is for local research validation.
|
||||
|
||||
The pinned MiniCPM-o checkpoint (`openbmb/MiniCPM-o-4_5`, `503e754207c94da6bb26850b4469f367c9ea3582`) has four weight shards totaling 18.74 GB (decimal) within a 20.05 GB repository. InternVLA base (`InternRobotics/InternVLA-A1-3B`, `08cf716d413aa8208a34f346cd8824e6ff901205`) and Place_Markpen fine-tune (`Jia-Zeng/InternVLA-A1-3B-FineTuned-Place_Markpen`, `0a557c6a503e6545b369bff2f34c76a410bbd190`) each have a 6.72 GB weight file. Their partial Xet files are **not** complete artifacts. MiniCPM-o's small configuration and processor files did finish; offline `AutoConfig` and `AutoProcessor` construction passed against the local path. This is a dependency preflight, not a weight load or E2E run. The large weight transfers have been throttled by the Hugging Face data path and remain in progress.
|
||||
The pinned MiniCPM-o checkpoint (`openbmb/MiniCPM-o-4_5`, `503e754207c94da6bb26850b4469f367c9ea3582`) has four weight shards totaling 18.74 GB (decimal). InternVLA base (`InternRobotics/InternVLA-A1-3B`, `08cf716d413aa8208a34f346cd8824e6ff901205`) and Place_Markpen fine-tune (`Jia-Zeng/InternVLA-A1-3B-FineTuned-Place_Markpen`, `0a557c6a503e6545b369bff2f34c76a410bbd190`) each have a 6.72 GB weight file. Checkpoint integrity is a prerequisite, not pipeline qualification.
|
||||
|
||||
InternVLA's real-weight synthetic probe uses [probe_internvla_actions.py](../../probe_internvla_actions.py) and the public `DiffusionRequestBatch` boundary with `strict_load=True`. The raw [JSON](evidence/internvla/internvla_finetune_synthetic.json) and [log](evidence/internvla/internvla_finetune_synthetic.log) record a 36.394 s startup, 1.222 s synchronized forward and 6,683,068,416 bytes of PyTorch CUDA peak allocation during the measured forward. This uses Transformers 5.14.1, vLLM 0.29.0, PyTorch 2.13.0+cu130 and BF16 policy weights on WSL/RTX 5090 Laptop (driver 610.71). Output actions are float32. The inputs are synthetic zeros, so neither task behavior nor action semantics are established.
|
||||
|
||||
MiniCPM-o used the actual three-stage Omni pipeline, real checkpoint and [constrained deploy profile](evidence/minicpmo/minicpmo_24gb_offline.yaml) on the same WSL/RTX 5090 Laptop, with vLLM 0.29.0, Transformers 5.14.1, BF16, eager execution and `VLLM_USE_FLASHINFER_SAMPLER=0` (no CUDA toolkit is present for FlashInfer JIT). The profile limits model length, multimodal inputs and concurrency, and explicitly offloads 5 GiB of thinker weights to host RAM; it does not qualify the stock full-capacity profile. The [raw log](evidence/minicpmo/mini_fixed_e2e.log), [text and token IDs](evidence/minicpmo/0_2c17288b-c543-4aaa-9a6a-e661ab14c97b.txt) and [WAV](evidence/minicpmo/output_0_2c17288b-c543-4aaa-9a6a-e661ab14c97b.wav) record 182.47 s engine startup and 20.53 s for the single measured prompt. The WAV has 124,800 mono PCM16 samples at 24 kHz (5.2 s), 124,579 nonzero samples, peak absolute amplitude 18,852 and RMS 3,430.39. There is no ASR or listening check, so matching speech content is unverified.
|
||||
|
||||
The earlier [three-stage run](evidence/minicpmo/mini_template.log) and [text-only run](evidence/minicpmo/mini_text_only.log) each emitted repeated punctuation. A [one-stage thinker probe](../../probe_minicpmo_thinker.py) isolated zero logits during real request prefill: the Omni runner passed MiniCPM-o its allocated embedding buffer before populating it, and MiniCPM-o treated that buffer as precomputed input. The runner now marks only actually populated embeddings; MiniCPM-o computes token embeddings otherwise. The [repaired thinker log](evidence/minicpmo/mini_thinker_fixed.log) and the three-stage result above show coherent text. The default profile's cache-admission failure and intermediate YAML/import/parameter failures remain in the other `evidence/minicpmo/mini_*.log` files; the fixed result does not erase those constraints.
|
||||
|
||||
The other rows still need actual local devices, a matching complete backend/artifact, memory admission, and model-specific quality and profiling. In particular, no ADB device is attached, native Windows CPU attention operations are missing from the installed extension, AMD iGPU/NPU routes lack a verified stateful whole-model deployment, and there is no qualified joint CPU+iGPU+NPU execution plan. AI Hub component results remain component evidence.
|
||||
|
||||
For interrupted transfers, [download_pending_weights.py](../../download_pending_weights.py) resumes each pinned repository and verifies the SHA-256 of every safetensors file against Hugging Face metadata. Its per-checkpoint JSON status and logs live under `models/.weight_download_status/` outside Git. A status of `verified` means only that the weights are complete; it does not qualify any inference path. Detached local resume jobs were started for MiniCPM-o and both InternVLA checkpoints while their interactive transfers continue.
|
||||
For interrupted transfers, [download_pending_weights.py](../../download_pending_weights.py) resumes each pinned repository and verifies the SHA-256 of every safetensors file against Hugging Face metadata. Its local logs live under `models/.weight_download_status/` outside Git; the verified manifests are copied into `evidence/weights/`. A status of `verified` means only that the weights are complete; it does not qualify any inference path.
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"scope": "real checkpoint, synthetic zero observations and noise; no task-quality claim",
|
||||
"model_dir": "/home/zhout/project/edge_infer/models/InternVLA-A1-3B-FT-Place_Markpen",
|
||||
"runtime_mode": "real_checkpoint_loaded",
|
||||
"device": "cuda",
|
||||
"dtype": "torch.float32",
|
||||
"action_shape": [
|
||||
1,
|
||||
50,
|
||||
32
|
||||
],
|
||||
"finite": true,
|
||||
"action_sha256_float32": "71ddd577e166f417639dd09f8cdaeef5a1bc91a6152c53672542504328675dd6",
|
||||
"first_action_prefix": [
|
||||
0.02230919525027275,
|
||||
-0.033485047519207,
|
||||
0.005473435390740633,
|
||||
0.02993217296898365,
|
||||
-0.00445316918194294,
|
||||
-0.01604151912033558,
|
||||
0.018104737624526024,
|
||||
-0.09112925827503204
|
||||
],
|
||||
"startup_s": 36.394273725978564,
|
||||
"forward_s": 1.222088642010931,
|
||||
"cuda_peak_allocated_bytes": 6683068416
|
||||
}
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
INFO 09-23 14:32:07 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 14:32:07 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 14:32:07 [patch.py:583] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
|
||||
INFO 09-23 14:32:08 [pipeline_internvla_a1.py:164] Loading InternVLA-A1 weights from /home/zhout/project/edge_infer/models/InternVLA-A1-3B-FT-Place_Markpen
|
||||
{
|
||||
"scope": "real checkpoint, synthetic zero observations and noise; no task-quality claim",
|
||||
"model_dir": "/home/zhout/project/edge_infer/models/InternVLA-A1-3B-FT-Place_Markpen",
|
||||
"runtime_mode": "real_checkpoint_loaded",
|
||||
"device": "cuda",
|
||||
"dtype": "torch.float32",
|
||||
"action_shape": [
|
||||
1,
|
||||
50,
|
||||
32
|
||||
],
|
||||
"finite": true,
|
||||
"action_sha256_float32": "71ddd577e166f417639dd09f8cdaeef5a1bc91a6152c53672542504328675dd6",
|
||||
"first_action_prefix": [
|
||||
0.02230919525027275,
|
||||
-0.033485047519207,
|
||||
0.005473435390740633,
|
||||
0.02993217296898365,
|
||||
-0.00445316918194294,
|
||||
-0.01604151912033558,
|
||||
0.018104737624526024,
|
||||
-0.09112925827503204
|
||||
],
|
||||
"startup_s": 36.394273725978564,
|
||||
"forward_s": 1.222088642010931,
|
||||
"cuda_peak_allocated_bytes": 6683068416
|
||||
}
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
Prompt:
|
||||
<|im_start|>system
|
||||
You are MiniCPM-o, a helpful multimodal assistant that can understand images, audio and video, and respond in text and speech.<|im_end|>
|
||||
<|im_start|>user
|
||||
Say hello, then introduce vLLM-Omni in one sentence.<|im_end|>
|
||||
<|im_start|>assistant
|
||||
<think>
|
||||
|
||||
</think>
|
||||
|
||||
<|tts_bos|>
|
||||
vllm_text_output:
|
||||
Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video.
|
||||
vllm_token_ids:
|
||||
[9707, 11, 358, 1079, 348, 4086, 44, 23620, 76, 7751, 11, 264, 3460, 79049, 57597, 1614, 12875, 315, 8660, 323, 23163, 1467, 11, 5335, 11, 7699, 11, 323, 2766, 13, 151704, 151645]
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
Prompt:
|
||||
<|im_start|>system
|
||||
You are MiniCPM-o, a helpful multimodal assistant that can understand images, audio and video, and respond in text and speech.<|im_end|>
|
||||
<|im_start|>user
|
||||
Say hello, then introduce vLLM-Omni in one sentence.<|im_end|>
|
||||
<|im_start|>assistant
|
||||
<think>
|
||||
|
||||
</think>
|
||||
|
||||
<|tts_bos|>
|
||||
vllm_text_output:
|
||||
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
Prompt:
|
||||
<|im_start|>system
|
||||
You are MiniCPM-o, a helpful multimodal assistant that can understand images, audio and video, and respond in text and speech.<|im_end|>
|
||||
<|im_start|>user
|
||||
Say hello, then introduce vLLM-Omni in one sentence.<|im_end|>
|
||||
<|im_start|>assistant
|
||||
<think>
|
||||
|
||||
</think>
|
||||
|
||||
|
||||
vllm_text_output:
|
||||
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
||||
+591
@@ -0,0 +1,591 @@
|
||||
INFO 09-23 14:09:18 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 14:09:18 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 14:09:18 [patch.py:583] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
|
||||
====================
|
||||
vllm version: 0.29.0
|
||||
====================
|
||||
INFO 09-23 14:09:19 [omni_base.py:216] [Omni] Initializing with model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5
|
||||
INFO 09-23 14:09:19 [async_omni_engine.py:172] [AsyncOmniEngine] Initializing with model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5
|
||||
INFO 09-23 14:09:19 [async_omni_engine.py:290] [AsyncOmniEngine] Launching Orchestrator thread with 3 stages
|
||||
INFO 09-23 14:09:19 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
INFO 09-23 14:09:19 [initialization.py:356] Loaded OmniTransferConfig with 1 connector configurations
|
||||
WARNING 09-23 14:09:19 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 14:09:19 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
INFO 09-23 14:09:19 [model.py:684] Resolved architecture: MiniCPMO45OmniForConditionalGeneration
|
||||
INFO 09-23 14:09:19 [model.py:2021] Using max model len 2048
|
||||
WARNING 09-23 14:09:19 [model.py:981] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False.
|
||||
INFO 09-23 14:09:19 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode.
|
||||
INFO 09-23 14:09:19 [model.py:887] Disabled mm_prefix attention mode because multimodal inputs are configuration-disabled. Attention backends without mm_prefix support may now be selected.
|
||||
INFO 09-23 14:09:19 [scheduler.py:277] Chunked prefill is enabled with max_num_batched_tokens=2048.
|
||||
WARNING 09-23 14:09:19 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
WARNING 09-23 14:09:19 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
INFO 09-23 14:09:19 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-23 14:09:19 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
INFO 09-23 14:09:19 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
WARNING 09-23 14:09:19 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 14:09:19 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
INFO 09-23 14:09:20 [model.py:684] Resolved architecture: MiniCPMO45OmniForConditionalGeneration
|
||||
INFO 09-23 14:09:20 [model.py:2021] Using max model len 512
|
||||
WARNING 09-23 14:09:20 [model.py:981] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False.
|
||||
INFO 09-23 14:09:20 [model.py:2021] Using max model len 512
|
||||
INFO 09-23 14:09:20 [scheduler.py:277] Chunked prefill is enabled with max_num_batched_tokens=512.
|
||||
INFO 09-23 14:09:20 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
WARNING 09-23 14:09:20 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 14:09:20 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
INFO 09-23 14:09:20 [model.py:684] Resolved architecture: MiniCPMO45Code2Wav
|
||||
INFO 09-23 14:09:20 [model.py:2021] Using max model len 4096
|
||||
WARNING 09-23 14:09:20 [arg_utils.py:2780] This model does not officially support disabling chunked prefill. Disabling this manually may cause the engine to crash or produce incorrect outputs.
|
||||
INFO 09-23 14:09:20 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-23 14:09:20 [stage_runtime.py:853] [stage_init] Stage-0 set runtime devices: 0
|
||||
INFO 09-23 14:09:28 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 14:09:28 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 14:09:28 [patch.py:583] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) INFO 09-23 14:09:29 [core.py:123] Initializing a V1 LLM engine (v0.29.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=2048, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 0, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) INFO 09-23 14:09:30 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) INFO 09-23 14:09:30 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_08cd396f437643dab40d35da8325dbc9 backend=nccl
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) INFO 09-23 14:09:30 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) WARNING 09-23 14:09:30 [gpu_ar_worker.py:120] OMNI GPUARWorker forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) INFO 09-23 14:09:30 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) INFO 09-23 14:09:31 [base.py:138] Offloader set to UVAOffloader
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) WARNING 09-23 14:09:31 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) WARNING 09-23 14:09:31 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) INFO 09-23 14:09:31 [gpu_model_runner.py:5379] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5...
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) WARNING 09-23 14:09:31 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) WARNING 09-23 14:09:31 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) INFO 09-23 14:09:31 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) INFO 09-23 14:09:31 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) INFO 09-23 14:09:31 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) INFO 09-23 14:09:31 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) INFO 09-23 14:09:31 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) INFO 09-23 14:09:32 [cuda.py:492] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) INFO 09-23 14:09:32 [flash_attn.py:897] Using FlashAttention version 2
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) INFO 09-23 14:09:38 [uva.py:65] Total CPU offloaded parameters: 5.03
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) INFO 09-23 14:09:38 [weight_utils.py:863] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 22.31 GiB.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) INFO 09-23 14:09:38 [weight_utils.py:886] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) ␍Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:00<00:00, 100.89it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) INFO 09-23 14:09:49 [default_loader.py:430] Loading weights took 10.80 seconds
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) INFO 09-23 14:09:54 [gpu_model_runner.py:5471] Model loading took 10.84 GiB memory and 21.551466 seconds
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) INFO 09-23 14:09:54 [utils.py:306] Using LBNHC KV cache layout.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) INFO 09-23 14:09:57 [base.py:148] Available KV cache memory: 2.36 GiB (device-level profiling)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) INFO 09-23 14:09:57 [kv_cache_utils.py:2032] GPU KV cache size: 17,152 tokens, Maximum concurrency for 2,048 tokens per request: 8.38x
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) WARNING 09-23 14:09:57 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) INFO 09-23 14:09:57 [kernel_warmup.py:124] JIT kernel warmup starting.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) ␍JIT kernel warmup (1 compile keys): 0%| | 0/1 [00:00<?, ?kernel/s]␍JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 2.80kernel/s, ComputeSlotMappingKernel (1 keys)]␍JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 2.80kernel/s, ComputeSlotMappingKernel (1 keys)]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) INFO 09-23 14:09:57 [kernel_warmup.py:134] JIT kernel warmup finished in 0.36s.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) WARNING 09-23 14:09:58 [import_utils.py:408] Module vllm.third_party.deep_gemm was found but failed to import
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) WARNING 09-23 14:09:58 [import_utils.py:408] Traceback (most recent call last):
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) WARNING 09-23 14:09:58 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/utils/import_utils.py", line 406, in _has_module
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) WARNING 09-23 14:09:58 [import_utils.py:408] importlib.import_module(module_name)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) WARNING 09-23 14:09:58 [import_utils.py:408] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/importlib/__init__.py", line 90, in import_module
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) WARNING 09-23 14:09:58 [import_utils.py:408] return _bootstrap._gcd_import(name[level:], package, level)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) WARNING 09-23 14:09:58 [import_utils.py:408] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) WARNING 09-23 14:09:58 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1387, in _gcd_import
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) WARNING 09-23 14:09:58 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1360, in _find_and_load
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) WARNING 09-23 14:09:58 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1331, in _find_and_load_unlocked
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) WARNING 09-23 14:09:58 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 935, in _load_unlocked
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) WARNING 09-23 14:09:58 [import_utils.py:408] File "<frozen importlib._bootstrap_external>", line 999, in exec_module
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) WARNING 09-23 14:09:58 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 488, in _call_with_frames_removed
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) WARNING 09-23 14:09:58 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/third_party/deep_gemm/__init__.py", line 126, in <module>
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) WARNING 09-23 14:09:58 [import_utils.py:408] _find_cuda_home() # CUDA home
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) WARNING 09-23 14:09:58 [import_utils.py:408] ^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) WARNING 09-23 14:09:58 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/third_party/deep_gemm/__init__.py", line 120, in _find_cuda_home
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) WARNING 09-23 14:09:58 [import_utils.py:408] assert cuda_home is not None
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) WARNING 09-23 14:09:58 [import_utils.py:408] ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) WARNING 09-23 14:09:58 [import_utils.py:408] AssertionError
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) INFO 09-23 14:09:58 [gpu_worker.py:860] Free memory on device (22.58/23.89 GiB) on startup. Desired GPU memory utilization is (0.58, 13.86 GiB). Actual usage is 11.01 GiB for consumed memory (weights + non-torch), 0.58 GiB for peak activation, and 0.0 GiB for CUDAGraph memory. Replace gpu_memory_utilization config with `--kv-cache-memory=2281984123` (2.13 GiB) to fit into requested memory, or `--kv-cache-memory=11646516224` (10.85 GiB) to fully utilize gpu memory. Current kv cache memory in use is 2.36 GiB.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) INFO 09-23 14:10:00 [jit_monitor.py:85] Kernel JIT monitor activated; monitored JIT compilations during inference will use mode=warn.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) INFO 09-23 14:10:00 [torch_utils.py:277] Reducing Torch threads from 12 to 1 for serving. Set OMP_NUM_THREADS in the external environment to override.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) INFO 09-23 14:10:00 [core.py:368] init engine (profile, create kv cache, warmup model) took 6.27 s
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) WARNING 09-23 14:10:00 [scheduler.py:213] Using custom scheduler class vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) WARNING 09-23 14:10:00 [omni_scheduler_mixin.py:118] OMNI scheduler forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) INFO 09-23 14:10:00 [chunk_transfer_adapter.py:142] Bounded active-stream window enabled: K=1. Multi-replica deployments require sticky per-stream routing across Stage 1 replicas (each replica owns an independent active-set; without sticky routing, a stream can be active on one replica and non-active on another and both will race to evict it).
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) INFO 09-23 14:10:00 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4150071) INFO 09-23 14:10:00 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-23 14:10:00 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed
|
||||
INFO 09-23 14:10:00 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-0 [rep-0] initializing EngineCore
|
||||
INFO 09-23 14:10:00 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-0 [rep-0] EngineCore running
|
||||
INFO 09-23 14:10:00 [stage_runtime.py:958] [StageRuntime] Stage 0 initialized
|
||||
INFO 09-23 14:10:00 [stage_runtime.py:853] [stage_init] Stage-1 set runtime devices: 0
|
||||
INFO 09-23 14:10:13 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 14:10:13 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 14:10:13 [patch.py:583] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) INFO 09-23 14:10:14 [core.py:123] Initializing a V1 LLM engine (v0.29.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=512, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [512], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 0, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) INFO 09-23 14:10:17 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_c63bd1bf3bee408dbec7aa955dc791f4 backend=nccl
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) INFO 09-23 14:10:17 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) WARNING 09-23 14:10:17 [gpu_ar_worker.py:120] OMNI GPUARWorker forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) INFO 09-23 14:10:17 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) [transformers] `MiniCPMOProcessor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) [transformers] Requested torchvision backend is not available. Falling back to pil backend.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) [transformers] `MiniCPMOProcessor` defines `feature_extractor_class = 'WhisperFeatureExtractor'`, which is deprecated. Register the correct mapping in `AutoFeatureExtractor` instead.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) [transformers] Image processor MiniCPMVImageProcessor: kwargs ['max_slice_nums'] were applied for backward compatibility. To avoid this warning, add them to valid_kwargs: create a custom TypedDict extending ImagesKwargs with these keys and set it as the `valid_kwargs` class attribute.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) INFO 09-23 14:10:29 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) WARNING 09-23 14:10:29 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) WARNING 09-23 14:10:29 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) INFO 09-23 14:10:29 [gpu_model_runner.py:5379] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5...
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) WARNING 09-23 14:10:29 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) WARNING 09-23 14:10:29 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) INFO 09-23 14:10:29 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) INFO 09-23 14:10:29 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) INFO 09-23 14:10:29 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) INFO 09-23 14:10:29 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) INFO 09-23 14:10:30 [cuda.py:492] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) INFO 09-23 14:10:30 [flash_attn.py:897] Using FlashAttention version 2
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) INFO 09-23 14:10:30 [weight_utils.py:863] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 20.56 GiB.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) INFO 09-23 14:10:30 [weight_utils.py:886] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) ␍Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:00<00:00, 116.36it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) INFO 09-23 14:10:32 [default_loader.py:430] Loading weights took 1.94 seconds
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) INFO 09-23 14:10:33 [gpu_model_runner.py:5471] Model loading took 0.65 GiB memory and 2.867099 seconds
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) INFO 09-23 14:10:34 [utils.py:306] Using LBNHC KV cache layout.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) INFO 09-23 14:10:34 [gpu_model_runner.py:6519] Skipping memory profiling for multimodal encoder and encoder cache.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) INFO 09-23 14:10:35 [kv_cache_utils.py:2032] GPU KV cache size: 4,368 tokens, Maximum concurrency for 512 tokens per request: 8.53x
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) WARNING 09-23 14:10:35 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) INFO 09-23 14:10:35 [kernel_warmup.py:124] JIT kernel warmup starting.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) ␍JIT kernel warmup (1 compile keys): 0%| | 0/1 [00:00<?, ?kernel/s]␍JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 2.61kernel/s, ComputeSlotMappingKernel (1 keys)]␍JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 2.60kernel/s, ComputeSlotMappingKernel (1 keys)]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) INFO 09-23 14:10:35 [kernel_warmup.py:134] JIT kernel warmup finished in 0.39s.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) WARNING 09-23 14:10:36 [import_utils.py:408] Module vllm.third_party.deep_gemm was found but failed to import
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) WARNING 09-23 14:10:36 [import_utils.py:408] Traceback (most recent call last):
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) WARNING 09-23 14:10:36 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/utils/import_utils.py", line 406, in _has_module
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) WARNING 09-23 14:10:36 [import_utils.py:408] importlib.import_module(module_name)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) WARNING 09-23 14:10:36 [import_utils.py:408] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/importlib/__init__.py", line 90, in import_module
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) WARNING 09-23 14:10:36 [import_utils.py:408] return _bootstrap._gcd_import(name[level:], package, level)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) WARNING 09-23 14:10:36 [import_utils.py:408] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) WARNING 09-23 14:10:36 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1387, in _gcd_import
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) WARNING 09-23 14:10:36 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1360, in _find_and_load
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) WARNING 09-23 14:10:36 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1331, in _find_and_load_unlocked
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) WARNING 09-23 14:10:36 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 935, in _load_unlocked
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) WARNING 09-23 14:10:36 [import_utils.py:408] File "<frozen importlib._bootstrap_external>", line 999, in exec_module
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) WARNING 09-23 14:10:36 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 488, in _call_with_frames_removed
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) WARNING 09-23 14:10:36 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/third_party/deep_gemm/__init__.py", line 126, in <module>
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) WARNING 09-23 14:10:36 [import_utils.py:408] _find_cuda_home() # CUDA home
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) WARNING 09-23 14:10:36 [import_utils.py:408] ^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) WARNING 09-23 14:10:36 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/third_party/deep_gemm/__init__.py", line 120, in _find_cuda_home
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) WARNING 09-23 14:10:36 [import_utils.py:408] assert cuda_home is not None
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) WARNING 09-23 14:10:36 [import_utils.py:408] ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) WARNING 09-23 14:10:36 [import_utils.py:408] AssertionError
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) INFO 09-23 14:10:36 [jit_monitor.py:85] Kernel JIT monitor activated; monitored JIT compilations during inference will use mode=warn.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) INFO 09-23 14:10:37 [torch_utils.py:277] Reducing Torch threads from 12 to 1 for serving. Set OMP_NUM_THREADS in the external environment to override.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) INFO 09-23 14:10:37 [core.py:368] init engine (profile, create kv cache, warmup model) took 3.32 s
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) WARNING 09-23 14:10:37 [scheduler.py:213] Using custom scheduler class vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) WARNING 09-23 14:10:37 [omni_scheduler_mixin.py:118] OMNI scheduler forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) INFO 09-23 14:10:37 [chunk_transfer_adapter.py:142] Bounded active-stream window enabled: K=1. Multi-replica deployments require sticky per-stream routing across Stage 1 replicas (each replica owns an independent active-set; without sticky routing, a stream can be active on one replica and non-active on another and both will race to evict it).
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) INFO 09-23 14:10:37 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4150287) INFO 09-23 14:10:37 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-23 14:10:37 [stage_runtime.py:943] [StageRuntime] Stage 1 engine startup completed
|
||||
INFO 09-23 14:10:37 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-1 [rep-0] initializing EngineCore
|
||||
INFO 09-23 14:10:37 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-1 [rep-0] EngineCore running
|
||||
INFO 09-23 14:10:37 [stage_runtime.py:958] [StageRuntime] Stage 1 initialized
|
||||
INFO 09-23 14:10:37 [stage_runtime.py:853] [stage_init] Stage-2 set runtime devices: 0
|
||||
INFO 09-23 14:10:47 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 14:10:47 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 14:10:47 [patch.py:583] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) INFO 09-23 14:10:48 [core.py:123] Initializing a V1 LLM engine (v0.29.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=4096, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=False, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [4096], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 0, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) INFO 09-23 14:10:48 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_2341b06c0ccd4c07977ac69ed1b79b60 backend=nccl
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) INFO 09-23 14:10:48 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) WARNING 09-23 14:10:49 [gpu_generation_worker.py:103] OMNI GPUGenerationWorker forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) INFO 09-23 14:10:49 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) INFO 09-23 14:10:49 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) WARNING 09-23 14:10:49 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) WARNING 09-23 14:10:49 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) INFO 09-23 14:10:49 [gpu_model_runner.py:5379] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5...
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) INFO 09-23 14:10:49 [weight_utils.py:863] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 19.26 GiB.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) INFO 09-23 14:10:49 [weight_utils.py:893] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre) and the checkpoint size (17.46 GiB) exceeds 90% of available RAM (19.26 GiB).
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ␍Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:00<00:00, 37.30it/s]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:00<00:00, 37.21it/s]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) INFO 09-23 14:10:50 [step_audio2_token2wav.py:163] Loading Token2Wav models from: /home/zhout/project/edge_infer/models/MiniCPM-o-4_5/assets/token2wav
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] StageEngineCoreProc failed to start.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] Traceback (most recent call last):
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_core_proc.py", line 189, in run_stage_core
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] engine_core = StageEngineCoreProc(
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] return func(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/v1/engine/core.py", line 1093, in __init__
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] super().__init__(
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/v1/engine/core.py", line 134, in __init__
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] self.model_executor = executor_class(vllm_config)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] return func(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/v1/executor/abstract.py", line 110, in __init__
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] self._init_executor()
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/v1/executor/uniproc_executor.py", line 74, in _init_executor
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] self.driver_worker.load_model()
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/worker/base.py", line 41, in load_model
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] res = super().load_model(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/v1/worker/gpu_worker.py", line 495, in load_model
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] self.model_runner.load_model(load_dummy_weights=load_dummy_weights)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] return func(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/init_timeline.py", line 343, in wrapper
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] return fn(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/worker/gpu_model_runner.py", line 188, in load_model
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] super().load_model(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] return func(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/v1/worker/gpu_model_runner.py", line 5395, in load_model
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] self.model = model_loader.load_model(
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] return func(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/model_executor/model_loader/base_loader.py", line 64, in load_model
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] self.load_weights(model, model_config)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] return func(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/model_executor/model_loader/default_loader.py", line 427, in load_weights
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] loaded_weights = model.load_weights(self.get_all_weights(model_config, model))
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py", line 823, in load_weights
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] self._build_backend()
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py", line 864, in _build_backend
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] token2wav = Token2wav(
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] ^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_token2wav.py", line 79, in __init__
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] self._core._ensure_models_loaded()
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/step_audio2/step_audio2_token2wav.py", line 177, in _ensure_models_loaded
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] configs = load_hyperpyyaml(f)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/hyperpyyaml/core.py", line 189, in load_hyperpyyaml
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] hparams = yaml.load(yaml_stream, Loader=loader)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/yaml/__init__.py", line 81, in load
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] return loader.get_single_data()
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/ruamel/yaml/constructor.py", line 119, in get_single_data
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] return self.construct_document(node)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/ruamel/yaml/constructor.py", line 123, in construct_document
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] data = self.construct_object(node)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/ruamel/yaml/constructor.py", line 150, in construct_object
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] data = self.construct_non_recursive_object(node)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/ruamel/yaml/constructor.py", line 191, in construct_non_recursive_object
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] for _dummy in generator:
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] ^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/ruamel/yaml/constructor.py", line 629, in construct_yaml_map
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] value = self.construct_mapping(node)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/ruamel/yaml/constructor.py", line 425, in construct_mapping
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] return BaseConstructor.construct_mapping(self, node, deep=deep)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/ruamel/yaml/constructor.py", line 247, in construct_mapping
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] value = self.construct_object(value_node, deep=deep)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/ruamel/yaml/constructor.py", line 150, in construct_object
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] data = self.construct_non_recursive_object(node)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/ruamel/yaml/constructor.py", line 186, in construct_non_recursive_object
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] data = constructor(self, tag_suffix, node)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/hyperpyyaml/core.py", line 473, in _construct_object
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] raise ImportError("There is no such class as %s" % callable_string)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ERROR 09-23 14:10:54 [stage_engine_core_proc.py:242] ImportError: There is no such class as cosyvoice2.flow.flow.CausalMaskedDiffWithXvec
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) Process StageEngineCoreProc_stage2_replica0:
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) Traceback (most recent call last):
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/multiprocessing/process.py", line 314, in _bootstrap
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) self.run()
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/multiprocessing/process.py", line 108, in run
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) self._target(*self._args, **self._kwargs)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_core_proc.py", line 189, in run_stage_core
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) engine_core = StageEngineCoreProc(
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) return func(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/v1/engine/core.py", line 1093, in __init__
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) super().__init__(
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/v1/engine/core.py", line 134, in __init__
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) self.model_executor = executor_class(vllm_config)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) return func(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/v1/executor/abstract.py", line 110, in __init__
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) self._init_executor()
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/v1/executor/uniproc_executor.py", line 74, in _init_executor
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) self.driver_worker.load_model()
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/worker/base.py", line 41, in load_model
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) res = super().load_model(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/v1/worker/gpu_worker.py", line 495, in load_model
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) self.model_runner.load_model(load_dummy_weights=load_dummy_weights)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) return func(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/init_timeline.py", line 343, in wrapper
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) return fn(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/worker/gpu_model_runner.py", line 188, in load_model
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) super().load_model(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) return func(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/v1/worker/gpu_model_runner.py", line 5395, in load_model
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) self.model = model_loader.load_model(
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) return func(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/model_executor/model_loader/base_loader.py", line 64, in load_model
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) self.load_weights(model, model_config)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) return func(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/model_executor/model_loader/default_loader.py", line 427, in load_weights
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) loaded_weights = model.load_weights(self.get_all_weights(model_config, model))
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py", line 823, in load_weights
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) self._build_backend()
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_code2wav.py", line 864, in _build_backend
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) token2wav = Token2wav(
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/minicpmo_4_5/minicpmo_4_5_token2wav.py", line 79, in __init__
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) self._core._ensure_models_loaded()
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/model_executor/models/step_audio2/step_audio2_token2wav.py", line 177, in _ensure_models_loaded
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) configs = load_hyperpyyaml(f)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/hyperpyyaml/core.py", line 189, in load_hyperpyyaml
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) hparams = yaml.load(yaml_stream, Loader=loader)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/yaml/__init__.py", line 81, in load
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) return loader.get_single_data()
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/ruamel/yaml/constructor.py", line 119, in get_single_data
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) return self.construct_document(node)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/ruamel/yaml/constructor.py", line 123, in construct_document
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) data = self.construct_object(node)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/ruamel/yaml/constructor.py", line 150, in construct_object
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) data = self.construct_non_recursive_object(node)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/ruamel/yaml/constructor.py", line 191, in construct_non_recursive_object
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) for _dummy in generator:
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/ruamel/yaml/constructor.py", line 629, in construct_yaml_map
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) value = self.construct_mapping(node)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/ruamel/yaml/constructor.py", line 425, in construct_mapping
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) return BaseConstructor.construct_mapping(self, node, deep=deep)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/ruamel/yaml/constructor.py", line 247, in construct_mapping
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) value = self.construct_object(value_node, deep=deep)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/ruamel/yaml/constructor.py", line 150, in construct_object
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) data = self.construct_non_recursive_object(node)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/ruamel/yaml/constructor.py", line 186, in construct_non_recursive_object
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) data = constructor(self, tag_suffix, node)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/hyperpyyaml/core.py", line 473, in _construct_object
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) raise ImportError("There is no such class as %s" % callable_string)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4150641) ImportError: There is no such class as cosyvoice2.flow.flow.CausalMaskedDiffWithXvec
|
||||
[rank0]:[W923 14:10:55.591483439 ProcessGroupNCCL.cpp:1624] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
|
||||
INFO 09-23 14:10:56 [utils.py:620] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage2_replica0
|
||||
ERROR 09-23 14:10:56 [stage_runtime.py:265] [StageRuntime] Stage initialization failed; shutting down 2 initialized client(s)
|
||||
ERROR 09-23 14:10:56 [stage_runtime.py:265] Traceback (most recent call last):
|
||||
ERROR 09-23 14:10:56 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 252, in initialize
|
||||
ERROR 09-23 14:10:56 [stage_runtime.py:265] initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout)
|
||||
ERROR 09-23 14:10:56 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 14:10:56 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 589, in _initialize_stage_replicas
|
||||
ERROR 09-23 14:10:56 [stage_runtime.py:265] raise primary_exc
|
||||
ERROR 09-23 14:10:56 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 550, in _init_group
|
||||
ERROR 09-23 14:10:56 [stage_runtime.py:265] client = self._initialize_replica(
|
||||
ERROR 09-23 14:10:56 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 14:10:56 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 828, in _initialize_replica
|
||||
ERROR 09-23 14:10:56 [stage_runtime.py:265] return self._initialize_local_llm_replica(plan, stage_init_timeout)
|
||||
ERROR 09-23 14:10:56 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 14:10:56 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 934, in _initialize_local_llm_replica
|
||||
ERROR 09-23 14:10:56 [stage_runtime.py:265] launch_stack.close()
|
||||
ERROR 09-23 14:10:56 [stage_runtime.py:265] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 618, in close
|
||||
ERROR 09-23 14:10:56 [stage_runtime.py:265] self.__exit__(None, None, None)
|
||||
ERROR 09-23 14:10:56 [stage_runtime.py:265] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 610, in __exit__
|
||||
ERROR 09-23 14:10:56 [stage_runtime.py:265] raise exc_details[1]
|
||||
ERROR 09-23 14:10:56 [stage_runtime.py:265] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 595, in __exit__
|
||||
ERROR 09-23 14:10:56 [stage_runtime.py:265] if cb(*exc_details):
|
||||
ERROR 09-23 14:10:56 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 14:10:56 [stage_runtime.py:265] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 144, in __exit__
|
||||
ERROR 09-23 14:10:56 [stage_runtime.py:265] next(self.gen)
|
||||
ERROR 09-23 14:10:56 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_startup.py", line 1211, in launch_stage_replica
|
||||
ERROR 09-23 14:10:56 [stage_runtime.py:265] wait_for_engine_startup(
|
||||
ERROR 09-23 14:10:56 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/v1/engine/utils.py", line 1320, in wait_for_engine_startup
|
||||
ERROR 09-23 14:10:56 [stage_runtime.py:265] raise RuntimeError(
|
||||
ERROR 09-23 14:10:56 [stage_runtime.py:265] RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {}
|
||||
INFO 09-23 14:10:56 [core_client.py:689] [shutdown] MPClient: start timeout=default
|
||||
INFO 09-23 14:10:56 [core_client.py:691] [shutdown] MPClient: stopping engine manager
|
||||
INFO 09-23 14:10:56 [utils.py:620] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage1_replica0
|
||||
INFO 09-23 14:11:01 [core_client.py:693] [shutdown] MPClient: engine manager stopped
|
||||
INFO 09-23 14:11:01 [core_client.py:694] [shutdown] MPClient: cleaning up background resources
|
||||
INFO 09-23 14:11:01 [core_client.py:696] [shutdown] MPClient: complete
|
||||
INFO 09-23 14:11:01 [core_client.py:689] [shutdown] MPClient: start timeout=default
|
||||
INFO 09-23 14:11:01 [utils.py:620] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage0_replica0
|
||||
ERROR 09-23 14:11:06 [async_omni_engine.py:474] [AsyncOmniEngine] Orchestrator thread crashed
|
||||
ERROR 09-23 14:11:06 [async_omni_engine.py:474] Traceback (most recent call last):
|
||||
ERROR 09-23 14:11:06 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 468, in _bootstrap_orchestrator
|
||||
ERROR 09-23 14:11:06 [async_omni_engine.py:474] loop.run_until_complete(_run_orchestrator())
|
||||
ERROR 09-23 14:11:06 [async_omni_engine.py:474] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/base_events.py", line 691, in run_until_complete
|
||||
ERROR 09-23 14:11:06 [async_omni_engine.py:474] return future.result()
|
||||
ERROR 09-23 14:11:06 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 14:11:06 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 424, in _run_orchestrator
|
||||
ERROR 09-23 14:11:06 [async_omni_engine.py:474] self._initialize_stages(stage_init_timeout)
|
||||
ERROR 09-23 14:11:06 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 350, in _initialize_stages
|
||||
ERROR 09-23 14:11:06 [async_omni_engine.py:474] self._initialize_stages_impl(stage_init_timeout)
|
||||
ERROR 09-23 14:11:06 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 371, in _initialize_stages_impl
|
||||
ERROR 09-23 14:11:06 [async_omni_engine.py:474] self._runtime.initialize()
|
||||
ERROR 09-23 14:11:06 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 278, in initialize
|
||||
ERROR 09-23 14:11:06 [async_omni_engine.py:474] raise exc
|
||||
ERROR 09-23 14:11:06 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 252, in initialize
|
||||
ERROR 09-23 14:11:06 [async_omni_engine.py:474] initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout)
|
||||
ERROR 09-23 14:11:06 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 14:11:06 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 589, in _initialize_stage_replicas
|
||||
ERROR 09-23 14:11:06 [async_omni_engine.py:474] raise primary_exc
|
||||
ERROR 09-23 14:11:06 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 550, in _init_group
|
||||
ERROR 09-23 14:11:06 [async_omni_engine.py:474] client = self._initialize_replica(
|
||||
ERROR 09-23 14:11:06 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 14:11:06 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 828, in _initialize_replica
|
||||
ERROR 09-23 14:11:06 [async_omni_engine.py:474] return self._initialize_local_llm_replica(plan, stage_init_timeout)
|
||||
ERROR 09-23 14:11:06 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 14:11:06 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 934, in _initialize_local_llm_replica
|
||||
ERROR 09-23 14:11:06 [async_omni_engine.py:474] launch_stack.close()
|
||||
ERROR 09-23 14:11:06 [async_omni_engine.py:474] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 618, in close
|
||||
ERROR 09-23 14:11:06 [async_omni_engine.py:474] self.__exit__(None, None, None)
|
||||
ERROR 09-23 14:11:06 [async_omni_engine.py:474] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 610, in __exit__
|
||||
ERROR 09-23 14:11:06 [async_omni_engine.py:474] raise exc_details[1]
|
||||
ERROR 09-23 14:11:06 [async_omni_engine.py:474] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 595, in __exit__
|
||||
ERROR 09-23 14:11:06 [async_omni_engine.py:474] if cb(*exc_details):
|
||||
ERROR 09-23 14:11:06 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 14:11:06 [async_omni_engine.py:474] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 144, in __exit__
|
||||
ERROR 09-23 14:11:06 [async_omni_engine.py:474] next(self.gen)
|
||||
ERROR 09-23 14:11:06 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_startup.py", line 1211, in launch_stage_replica
|
||||
ERROR 09-23 14:11:06 [async_omni_engine.py:474] wait_for_engine_startup(
|
||||
ERROR 09-23 14:11:06 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/v1/engine/utils.py", line 1320, in wait_for_engine_startup
|
||||
ERROR 09-23 14:11:06 [async_omni_engine.py:474] raise RuntimeError(
|
||||
ERROR 09-23 14:11:06 [async_omni_engine.py:474] RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {}
|
||||
INFO 09-23 14:11:06 [async_omni_engine.py:1807] [AsyncOmniEngine] Shutting down Orchestrator
|
||||
Exception in thread orchestrator:
|
||||
Traceback (most recent call last):
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
|
||||
self.run()
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/threading.py", line 1012, in run
|
||||
self._target(*self._args, **self._kwargs)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 468, in _bootstrap_orchestrator
|
||||
loop.run_until_complete(_run_orchestrator())
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/base_events.py", line 691, in run_until_complete
|
||||
return future.result()
|
||||
^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 424, in _run_orchestrator
|
||||
self._initialize_stages(stage_init_timeout)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 350, in _initialize_stages
|
||||
self._initialize_stages_impl(stage_init_timeout)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 371, in _initialize_stages_impl
|
||||
self._runtime.initialize()
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 278, in initialize
|
||||
raise exc
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 252, in initialize
|
||||
initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 589, in _initialize_stage_replicas
|
||||
raise primary_exc
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 550, in _init_group
|
||||
client = self._initialize_replica(
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 828, in _initialize_replica
|
||||
return self._initialize_local_llm_replica(plan, stage_init_timeout)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 934, in _initialize_local_llm_replica
|
||||
launch_stack.close()
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 618, in close
|
||||
self.__exit__(None, None, None)
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 610, in __exit__
|
||||
raise exc_details[1]
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 595, in __exit__
|
||||
if cb(*exc_details):
|
||||
^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 144, in __exit__
|
||||
next(self.gen)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_startup.py", line 1211, in launch_stage_replica
|
||||
wait_for_engine_startup(
|
||||
File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/v1/engine/utils.py", line 1320, in wait_for_engine_startup
|
||||
raise RuntimeError(
|
||||
RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {}
|
||||
Traceback (most recent call last):
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
|
||||
self.run()
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/threading.py", line 1012, in run
|
||||
self._target(*self._args, **self._kwargs)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 468, in _bootstrap_orchestrator
|
||||
loop.run_until_complete(_run_orchestrator())
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/base_events.py", line 691, in run_until_complete
|
||||
return future.result()
|
||||
^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 424, in _run_orchestrator
|
||||
self._initialize_stages(stage_init_timeout)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 350, in _initialize_stages
|
||||
self._initialize_stages_impl(stage_init_timeout)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 371, in _initialize_stages_impl
|
||||
self._runtime.initialize()
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 278, in initialize
|
||||
raise exc
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 252, in initialize
|
||||
initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 589, in _initialize_stage_replicas
|
||||
raise primary_exc
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 550, in _init_group
|
||||
client = self._initialize_replica(
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 828, in _initialize_replica
|
||||
return self._initialize_local_llm_replica(plan, stage_init_timeout)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 934, in _initialize_local_llm_replica
|
||||
launch_stack.close()
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 618, in close
|
||||
self.__exit__(None, None, None)
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 610, in __exit__
|
||||
raise exc_details[1]
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 595, in __exit__
|
||||
if cb(*exc_details):
|
||||
^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 144, in __exit__
|
||||
next(self.gen)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_startup.py", line 1211, in launch_stage_replica
|
||||
wait_for_engine_startup(
|
||||
File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/v1/engine/utils.py", line 1320, in wait_for_engine_startup
|
||||
raise RuntimeError(
|
||||
RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {}
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
|
||||
Traceback (most recent call last):
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/examples/offline_inference/minicpmo/end2end.py", line 537, in <module>
|
||||
main(args)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/examples/offline_inference/minicpmo/end2end.py", line 286, in main
|
||||
omni = Omni(**omni_kwargs)
|
||||
^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/entrypoints/omni_base.py", line 223, in __init__
|
||||
self.engine = AsyncOmniEngine(
|
||||
^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 305, in __init__
|
||||
self._wait_for_orchestrator_init(startup_future, startup_timeout)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 539, in _wait_for_orchestrator_init
|
||||
startup_future.result(
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/concurrent/futures/_base.py", line 456, in result
|
||||
return self.__get_result()
|
||||
^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/concurrent/futures/_base.py", line 401, in __get_result
|
||||
raise self._exception
|
||||
RuntimeError: Orchestrator initialization failed: Engine core initialization failed. See root cause above. Failed core proc(s): {}
|
||||
+282
@@ -0,0 +1,282 @@
|
||||
INFO 09-23 14:13:16 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 14:13:16 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 14:13:16 [patch.py:583] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
|
||||
====================
|
||||
vllm version: 0.29.0
|
||||
====================
|
||||
INFO 09-23 14:13:17 [omni_base.py:216] [Omni] Initializing with model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5
|
||||
INFO 09-23 14:13:17 [async_omni_engine.py:172] [AsyncOmniEngine] Initializing with model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5
|
||||
INFO 09-23 14:13:17 [async_omni_engine.py:290] [AsyncOmniEngine] Launching Orchestrator thread with 3 stages
|
||||
INFO 09-23 14:13:17 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
INFO 09-23 14:13:17 [initialization.py:356] Loaded OmniTransferConfig with 1 connector configurations
|
||||
WARNING 09-23 14:13:17 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 14:13:17 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
INFO 09-23 14:13:17 [model.py:684] Resolved architecture: MiniCPMO45OmniForConditionalGeneration
|
||||
INFO 09-23 14:13:17 [model.py:2021] Using max model len 2048
|
||||
WARNING 09-23 14:13:17 [model.py:981] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False.
|
||||
INFO 09-23 14:13:17 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode.
|
||||
INFO 09-23 14:13:17 [model.py:887] Disabled mm_prefix attention mode because multimodal inputs are configuration-disabled. Attention backends without mm_prefix support may now be selected.
|
||||
INFO 09-23 14:13:17 [scheduler.py:277] Chunked prefill is enabled with max_num_batched_tokens=2048.
|
||||
WARNING 09-23 14:13:17 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
WARNING 09-23 14:13:17 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
INFO 09-23 14:13:17 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-23 14:13:17 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
INFO 09-23 14:13:18 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
WARNING 09-23 14:13:18 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 14:13:18 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
INFO 09-23 14:13:18 [model.py:684] Resolved architecture: MiniCPMO45OmniForConditionalGeneration
|
||||
INFO 09-23 14:13:18 [model.py:2021] Using max model len 512
|
||||
WARNING 09-23 14:13:18 [model.py:981] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False.
|
||||
INFO 09-23 14:13:18 [model.py:2021] Using max model len 512
|
||||
INFO 09-23 14:13:18 [scheduler.py:277] Chunked prefill is enabled with max_num_batched_tokens=512.
|
||||
INFO 09-23 14:13:18 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
WARNING 09-23 14:13:18 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 14:13:18 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
INFO 09-23 14:13:18 [model.py:684] Resolved architecture: MiniCPMO45Code2Wav
|
||||
INFO 09-23 14:13:18 [model.py:2021] Using max model len 4096
|
||||
WARNING 09-23 14:13:18 [arg_utils.py:2780] This model does not officially support disabling chunked prefill. Disabling this manually may cause the engine to crash or produce incorrect outputs.
|
||||
INFO 09-23 14:13:18 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-23 14:13:18 [stage_runtime.py:853] [stage_init] Stage-0 set runtime devices: 0
|
||||
INFO 09-23 14:13:26 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 14:13:26 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 14:13:26 [patch.py:583] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) INFO 09-23 14:13:27 [core.py:123] Initializing a V1 LLM engine (v0.29.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=2048, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 0, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) INFO 09-23 14:13:28 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) INFO 09-23 14:13:28 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_f09fe6fcf4b94a49b32f8078cee56d0c backend=nccl
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) INFO 09-23 14:13:28 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) WARNING 09-23 14:13:28 [gpu_ar_worker.py:120] OMNI GPUARWorker forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) INFO 09-23 14:13:28 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) INFO 09-23 14:13:28 [base.py:138] Offloader set to UVAOffloader
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) WARNING 09-23 14:13:28 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) WARNING 09-23 14:13:28 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) INFO 09-23 14:13:28 [gpu_model_runner.py:5379] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5...
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) WARNING 09-23 14:13:29 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) WARNING 09-23 14:13:29 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) INFO 09-23 14:13:29 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) INFO 09-23 14:13:29 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) INFO 09-23 14:13:29 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) INFO 09-23 14:13:29 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) INFO 09-23 14:13:29 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) INFO 09-23 14:13:30 [cuda.py:492] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) INFO 09-23 14:13:30 [flash_attn.py:897] Using FlashAttention version 2
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) INFO 09-23 14:13:34 [uva.py:65] Total CPU offloaded parameters: 5.03
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) INFO 09-23 14:13:34 [weight_utils.py:863] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 22.37 GiB.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) INFO 09-23 14:13:34 [weight_utils.py:886] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) ␍Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:00<00:00, 135.73it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) INFO 09-23 14:13:42 [default_loader.py:430] Loading weights took 7.93 seconds
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) INFO 09-23 14:13:47 [gpu_model_runner.py:5471] Model loading took 10.84 GiB memory and 16.669961 seconds
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) INFO 09-23 14:13:47 [utils.py:306] Using LBNHC KV cache layout.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) INFO 09-23 14:13:51 [base.py:148] Available KV cache memory: 2.36 GiB (device-level profiling)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) INFO 09-23 14:13:51 [kv_cache_utils.py:2032] GPU KV cache size: 17,152 tokens, Maximum concurrency for 2,048 tokens per request: 8.38x
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) WARNING 09-23 14:13:51 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) INFO 09-23 14:13:51 [kernel_warmup.py:124] JIT kernel warmup starting.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) ␍JIT kernel warmup (1 compile keys): 0%| | 0/1 [00:00<?, ?kernel/s]␍JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 2.82kernel/s, ComputeSlotMappingKernel (1 keys)]␍JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 2.82kernel/s, ComputeSlotMappingKernel (1 keys)]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) INFO 09-23 14:13:51 [kernel_warmup.py:134] JIT kernel warmup finished in 0.36s.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) WARNING 09-23 14:13:52 [import_utils.py:408] Module vllm.third_party.deep_gemm was found but failed to import
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) WARNING 09-23 14:13:52 [import_utils.py:408] Traceback (most recent call last):
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) WARNING 09-23 14:13:52 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/utils/import_utils.py", line 406, in _has_module
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) WARNING 09-23 14:13:52 [import_utils.py:408] importlib.import_module(module_name)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) WARNING 09-23 14:13:52 [import_utils.py:408] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/importlib/__init__.py", line 90, in import_module
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) WARNING 09-23 14:13:52 [import_utils.py:408] return _bootstrap._gcd_import(name[level:], package, level)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) WARNING 09-23 14:13:52 [import_utils.py:408] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) WARNING 09-23 14:13:52 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1387, in _gcd_import
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) WARNING 09-23 14:13:52 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1360, in _find_and_load
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) WARNING 09-23 14:13:52 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1331, in _find_and_load_unlocked
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) WARNING 09-23 14:13:52 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 935, in _load_unlocked
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) WARNING 09-23 14:13:52 [import_utils.py:408] File "<frozen importlib._bootstrap_external>", line 999, in exec_module
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) WARNING 09-23 14:13:52 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 488, in _call_with_frames_removed
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) WARNING 09-23 14:13:52 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/third_party/deep_gemm/__init__.py", line 126, in <module>
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) WARNING 09-23 14:13:52 [import_utils.py:408] _find_cuda_home() # CUDA home
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) WARNING 09-23 14:13:52 [import_utils.py:408] ^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) WARNING 09-23 14:13:52 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/third_party/deep_gemm/__init__.py", line 120, in _find_cuda_home
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) WARNING 09-23 14:13:52 [import_utils.py:408] assert cuda_home is not None
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) WARNING 09-23 14:13:52 [import_utils.py:408] ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) WARNING 09-23 14:13:52 [import_utils.py:408] AssertionError
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) INFO 09-23 14:13:52 [gpu_worker.py:860] Free memory on device (22.58/23.89 GiB) on startup. Desired GPU memory utilization is (0.58, 13.86 GiB). Actual usage is 11.01 GiB for consumed memory (weights + non-torch), 0.58 GiB for peak activation, and 0.0 GiB for CUDAGraph memory. Replace gpu_memory_utilization config with `--kv-cache-memory=2281984123` (2.13 GiB) to fit into requested memory, or `--kv-cache-memory=11646516224` (10.85 GiB) to fully utilize gpu memory. Current kv cache memory in use is 2.36 GiB.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) INFO 09-23 14:13:53 [jit_monitor.py:85] Kernel JIT monitor activated; monitored JIT compilations during inference will use mode=warn.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) INFO 09-23 14:13:53 [torch_utils.py:277] Reducing Torch threads from 12 to 1 for serving. Set OMP_NUM_THREADS in the external environment to override.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) INFO 09-23 14:13:53 [core.py:368] init engine (profile, create kv cache, warmup model) took 6.27 s
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) WARNING 09-23 14:13:53 [scheduler.py:213] Using custom scheduler class vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) WARNING 09-23 14:13:53 [omni_scheduler_mixin.py:118] OMNI scheduler forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) INFO 09-23 14:13:53 [chunk_transfer_adapter.py:142] Bounded active-stream window enabled: K=1. Multi-replica deployments require sticky per-stream routing across Stage 1 replicas (each replica owns an independent active-set; without sticky routing, a stream can be active on one replica and non-active on another and both will race to evict it).
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) INFO 09-23 14:13:53 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4151285) INFO 09-23 14:13:53 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-23 14:13:53 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed
|
||||
INFO 09-23 14:13:53 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-0 [rep-0] initializing EngineCore
|
||||
INFO 09-23 14:13:54 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-0 [rep-0] EngineCore running
|
||||
INFO 09-23 14:13:54 [stage_runtime.py:958] [StageRuntime] Stage 0 initialized
|
||||
INFO 09-23 14:13:54 [stage_runtime.py:853] [stage_init] Stage-1 set runtime devices: 0
|
||||
INFO 09-23 14:14:02 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 14:14:02 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 14:14:02 [patch.py:583] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) INFO 09-23 14:14:03 [core.py:123] Initializing a V1 LLM engine (v0.29.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=512, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [512], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 0, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) INFO 09-23 14:14:04 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_d7d9ead30b1849fab3acd3b7df633d7f backend=nccl
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) INFO 09-23 14:14:04 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) WARNING 09-23 14:14:04 [gpu_ar_worker.py:120] OMNI GPUARWorker forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) INFO 09-23 14:14:04 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) [transformers] `MiniCPMOProcessor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) [transformers] Requested torchvision backend is not available. Falling back to pil backend.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) [transformers] `MiniCPMOProcessor` defines `feature_extractor_class = 'WhisperFeatureExtractor'`, which is deprecated. Register the correct mapping in `AutoFeatureExtractor` instead.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) [transformers] Image processor MiniCPMVImageProcessor: kwargs ['max_slice_nums'] were applied for backward compatibility. To avoid this warning, add them to valid_kwargs: create a custom TypedDict extending ImagesKwargs with these keys and set it as the `valid_kwargs` class attribute.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) INFO 09-23 14:14:14 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) WARNING 09-23 14:14:14 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) WARNING 09-23 14:14:14 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) INFO 09-23 14:14:14 [gpu_model_runner.py:5379] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5...
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) WARNING 09-23 14:14:15 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) WARNING 09-23 14:14:15 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) INFO 09-23 14:14:15 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) INFO 09-23 14:14:15 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) INFO 09-23 14:14:15 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) INFO 09-23 14:14:15 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) INFO 09-23 14:14:16 [cuda.py:492] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) INFO 09-23 14:14:16 [flash_attn.py:897] Using FlashAttention version 2
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) INFO 09-23 14:14:16 [weight_utils.py:863] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 20.69 GiB.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) INFO 09-23 14:14:16 [weight_utils.py:886] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) ␍Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:00<00:00, 115.94it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) INFO 09-23 14:14:17 [default_loader.py:430] Loading weights took 1.11 seconds
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) INFO 09-23 14:14:18 [gpu_model_runner.py:5471] Model loading took 0.65 GiB memory and 1.919989 seconds
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) INFO 09-23 14:14:18 [utils.py:306] Using LBNHC KV cache layout.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) INFO 09-23 14:14:18 [gpu_model_runner.py:6519] Skipping memory profiling for multimodal encoder and encoder cache.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) INFO 09-23 14:14:19 [kv_cache_utils.py:2032] GPU KV cache size: 4,368 tokens, Maximum concurrency for 512 tokens per request: 8.53x
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) WARNING 09-23 14:14:19 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) INFO 09-23 14:14:19 [kernel_warmup.py:124] JIT kernel warmup starting.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) ␍JIT kernel warmup (1 compile keys): 0%| | 0/1 [00:00<?, ?kernel/s]␍JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 3.59kernel/s, ComputeSlotMappingKernel (1 keys)]␍JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 3.58kernel/s, ComputeSlotMappingKernel (1 keys)]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) INFO 09-23 14:14:20 [kernel_warmup.py:134] JIT kernel warmup finished in 0.28s.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) WARNING 09-23 14:14:20 [import_utils.py:408] Module vllm.third_party.deep_gemm was found but failed to import
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) WARNING 09-23 14:14:20 [import_utils.py:408] Traceback (most recent call last):
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) WARNING 09-23 14:14:20 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/utils/import_utils.py", line 406, in _has_module
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) WARNING 09-23 14:14:20 [import_utils.py:408] importlib.import_module(module_name)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) WARNING 09-23 14:14:20 [import_utils.py:408] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/importlib/__init__.py", line 90, in import_module
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) WARNING 09-23 14:14:20 [import_utils.py:408] return _bootstrap._gcd_import(name[level:], package, level)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) WARNING 09-23 14:14:20 [import_utils.py:408] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) WARNING 09-23 14:14:20 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1387, in _gcd_import
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) WARNING 09-23 14:14:20 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1360, in _find_and_load
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) WARNING 09-23 14:14:20 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1331, in _find_and_load_unlocked
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) WARNING 09-23 14:14:20 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 935, in _load_unlocked
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) WARNING 09-23 14:14:20 [import_utils.py:408] File "<frozen importlib._bootstrap_external>", line 999, in exec_module
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) WARNING 09-23 14:14:20 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 488, in _call_with_frames_removed
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) WARNING 09-23 14:14:20 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/third_party/deep_gemm/__init__.py", line 126, in <module>
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) WARNING 09-23 14:14:20 [import_utils.py:408] _find_cuda_home() # CUDA home
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) WARNING 09-23 14:14:20 [import_utils.py:408] ^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) WARNING 09-23 14:14:20 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/third_party/deep_gemm/__init__.py", line 120, in _find_cuda_home
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) WARNING 09-23 14:14:20 [import_utils.py:408] assert cuda_home is not None
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) WARNING 09-23 14:14:20 [import_utils.py:408] ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) WARNING 09-23 14:14:20 [import_utils.py:408] AssertionError
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) INFO 09-23 14:14:21 [jit_monitor.py:85] Kernel JIT monitor activated; monitored JIT compilations during inference will use mode=warn.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) INFO 09-23 14:14:21 [torch_utils.py:277] Reducing Torch threads from 12 to 1 for serving. Set OMP_NUM_THREADS in the external environment to override.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) INFO 09-23 14:14:21 [core.py:368] init engine (profile, create kv cache, warmup model) took 2.89 s
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) WARNING 09-23 14:14:21 [scheduler.py:213] Using custom scheduler class vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) WARNING 09-23 14:14:21 [omni_scheduler_mixin.py:118] OMNI scheduler forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) INFO 09-23 14:14:21 [chunk_transfer_adapter.py:142] Bounded active-stream window enabled: K=1. Multi-replica deployments require sticky per-stream routing across Stage 1 replicas (each replica owns an independent active-set; without sticky routing, a stream can be active on one replica and non-active on another and both will race to evict it).
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) INFO 09-23 14:14:21 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4151683) INFO 09-23 14:14:21 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-23 14:14:21 [stage_runtime.py:943] [StageRuntime] Stage 1 engine startup completed
|
||||
INFO 09-23 14:14:21 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-1 [rep-0] initializing EngineCore
|
||||
INFO 09-23 14:14:21 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-1 [rep-0] EngineCore running
|
||||
INFO 09-23 14:14:21 [stage_runtime.py:958] [StageRuntime] Stage 1 initialized
|
||||
INFO 09-23 14:14:21 [stage_runtime.py:853] [stage_init] Stage-2 set runtime devices: 0
|
||||
INFO 09-23 14:14:30 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 14:14:31 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 14:14:31 [patch.py:583] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) INFO 09-23 14:14:31 [core.py:123] Initializing a V1 LLM engine (v0.29.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=4096, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=False, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [4096], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 0, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) INFO 09-23 14:14:32 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_ac6da76b61ed4f85b86393c443498491 backend=nccl
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) INFO 09-23 14:14:32 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) WARNING 09-23 14:14:32 [gpu_generation_worker.py:103] OMNI GPUGenerationWorker forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) INFO 09-23 14:14:32 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) INFO 09-23 14:14:32 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) WARNING 09-23 14:14:32 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) WARNING 09-23 14:14:32 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) INFO 09-23 14:14:32 [gpu_model_runner.py:5379] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5...
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) INFO 09-23 14:14:33 [weight_utils.py:863] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 19.02 GiB.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) INFO 09-23 14:14:33 [weight_utils.py:893] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre) and the checkpoint size (17.46 GiB) exceeds 90% of available RAM (19.02 GiB).
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) ␍Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:00<00:00, 64.03it/s]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) INFO 09-23 14:14:33 [step_audio2_token2wav.py:163] Loading Token2Wav models from: /home/zhout/project/edge_infer/models/MiniCPM-o-4_5/assets/token2wav
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) INFO 09-23 14:14:43 [step_audio2_token2wav.py:201] Token2Wav models loaded successfully
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) INFO 09-23 14:14:45 [batched_token2wav.py:227] Enabling HiFT CUDA Graph with batch sizes [1]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) INFO 09-23 14:14:45 [cuda_graph_wrapper.py:98] Captured HiFT CUDA Graph for shape (1, 50, 0)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) INFO 09-23 14:14:46 [cuda_graph_wrapper.py:98] Captured HiFT CUDA Graph for shape (1, 58, 3840)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) INFO 09-23 14:14:46 [batched_token2wav.py:235] HiFT CUDA Graph captured successfully
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) INFO 09-23 14:14:46 [batched_token2wav.py:246] CFM CUDA Graph enabled (max_graphs=32)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) INFO 09-23 14:14:46 [default_loader.py:430] Loading weights took 12.64 seconds
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) INFO 09-23 14:14:46 [gpu_model_runner.py:5471] Model loading took 3.36 GiB memory and 12.657647 seconds
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) INFO 09-23 14:14:47 [cuda.py:492] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) INFO 09-23 14:14:47 [utils.py:306] Using LBNHC KV cache layout.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) WARNING 09-23 14:14:47 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) INFO 09-23 14:14:47 [kernel_warmup.py:124] JIT kernel warmup starting.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) ␍JIT kernel warmup (1 compile keys): 0%| | 0/1 [00:00<?, ?kernel/s]␍JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 505.28kernel/s, ComputeSlotMappingKernel (1 keys)]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) INFO 09-23 14:14:47 [kernel_warmup.py:134] JIT kernel warmup finished in 0.00s.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) WARNING 09-23 14:14:48 [import_utils.py:408] Module vllm.third_party.deep_gemm was found but failed to import
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) WARNING 09-23 14:14:48 [import_utils.py:408] Traceback (most recent call last):
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) WARNING 09-23 14:14:48 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/utils/import_utils.py", line 406, in _has_module
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) WARNING 09-23 14:14:48 [import_utils.py:408] importlib.import_module(module_name)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) WARNING 09-23 14:14:48 [import_utils.py:408] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/importlib/__init__.py", line 90, in import_module
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) WARNING 09-23 14:14:48 [import_utils.py:408] return _bootstrap._gcd_import(name[level:], package, level)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) WARNING 09-23 14:14:48 [import_utils.py:408] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) WARNING 09-23 14:14:48 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1387, in _gcd_import
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) WARNING 09-23 14:14:48 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1360, in _find_and_load
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) WARNING 09-23 14:14:48 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1331, in _find_and_load_unlocked
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) WARNING 09-23 14:14:48 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 935, in _load_unlocked
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) WARNING 09-23 14:14:48 [import_utils.py:408] File "<frozen importlib._bootstrap_external>", line 999, in exec_module
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) WARNING 09-23 14:14:48 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 488, in _call_with_frames_removed
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) WARNING 09-23 14:14:48 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/third_party/deep_gemm/__init__.py", line 126, in <module>
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) WARNING 09-23 14:14:48 [import_utils.py:408] _find_cuda_home() # CUDA home
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) WARNING 09-23 14:14:48 [import_utils.py:408] ^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) WARNING 09-23 14:14:48 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/third_party/deep_gemm/__init__.py", line 120, in _find_cuda_home
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) WARNING 09-23 14:14:48 [import_utils.py:408] assert cuda_home is not None
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) WARNING 09-23 14:14:48 [import_utils.py:408] ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) WARNING 09-23 14:14:48 [import_utils.py:408] AssertionError
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) WARNING 09-23 14:14:48 [gpu_generation_model_runner.py:558] Dummy sampler run is not implemented for generation model
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) INFO 09-23 14:14:49 [jit_monitor.py:85] Kernel JIT monitor activated; monitored JIT compilations during inference will use mode=warn.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) INFO 09-23 14:14:49 [torch_utils.py:277] Reducing Torch threads from 12 to 1 for serving. Set OMP_NUM_THREADS in the external environment to override.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) INFO 09-23 14:14:49 [core.py:368] init engine (profile, create kv cache, warmup model) took 2.34 s
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) WARNING 09-23 14:14:50 [scheduler.py:213] Using custom scheduler class vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) WARNING 09-23 14:14:50 [omni_scheduler_mixin.py:118] OMNI scheduler forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) INFO 09-23 14:14:50 [chunk_transfer_adapter.py:142] Bounded active-stream window enabled: K=1. Multi-replica deployments require sticky per-stream routing across Stage 1 replicas (each replica owns an independent active-set; without sticky routing, a stream can be active on one replica and non-active on another and both will race to evict it).
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) INFO 09-23 14:14:50 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) WARNING 09-23 14:14:50 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) WARNING 09-23 14:14:50 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
INFO 09-23 14:14:50 [stage_runtime.py:943] [StageRuntime] Stage 2 engine startup completed
|
||||
INFO 09-23 14:14:50 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-2 [rep-0] initializing EngineCore
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) INFO 09-23 14:14:50 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) INFO 09-23 14:14:50 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4151929) INFO 09-23 14:14:50 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
INFO 09-23 14:14:50 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-2 [rep-0] EngineCore running
|
||||
INFO 09-23 14:14:50 [stage_runtime.py:958] [StageRuntime] Stage 2 initialized
|
||||
INFO 09-23 14:14:52 [orchestrator.py:604] [Orchestrator] Starting event loop
|
||||
INFO 09-23 14:14:52 [async_omni_engine.py:323] [AsyncOmniEngine] Orchestrator ready with 3 stages
|
||||
INFO 09-23 14:14:52 [omni_base.py:235] [Omni] AsyncOmniEngine initialized in 94.69 seconds
|
||||
INFO 09-23 14:14:52 [omni_base.py:255] [Omni] Initialized with 3 stages for model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5
|
||||
Traceback (most recent call last):
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/examples/offline_inference/minicpmo/end2end.py", line 537, in <module>
|
||||
main(args)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/examples/offline_inference/minicpmo/end2end.py", line 328, in main
|
||||
omni_generator = omni.generate(prompts, sampling_params_list, py_generator=args.py_generator)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/entrypoints/omni.py", line 86, in generate
|
||||
sampling_params_list = self.resolve_sampling_params_list(sampling_params_list)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/entrypoints/omni_base.py", line 365, in resolve_sampling_params_list
|
||||
raise ValueError(f"Expected {self.num_stages} sampling params, got {len(normalized)}")
|
||||
ValueError: Expected 3 sampling params, got 2
|
||||
INFO 09-23 14:14:52 [async_omni_engine.py:1807] [AsyncOmniEngine] Shutting down Orchestrator
|
||||
INFO 09-23 14:14:52 [orchestrator.py:705] [Orchestrator] Received shutdown signal
|
||||
INFO 09-23 14:14:52 [orchestrator.py:3092] [Orchestrator] Shutting down all 3 client(s)
|
||||
INFO 09-23 14:14:52 [core_client.py:689] [shutdown] MPClient: start timeout=default
|
||||
INFO 09-23 14:14:52 [core_client.py:691] [shutdown] MPClient: stopping engine manager
|
||||
INFO 09-23 14:14:52 [utils.py:620] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage0_replica0
|
||||
INFO 09-23 14:14:56 [core_client.py:693] [shutdown] MPClient: engine manager stopped
|
||||
INFO 09-23 14:14:56 [core_client.py:694] [shutdown] MPClient: cleaning up background resources
|
||||
INFO 09-23 14:14:56 [core_client.py:696] [shutdown] MPClient: complete
|
||||
INFO 09-23 14:14:56 [stage_pool.py:1354] [StagePool] Stage 0 replica 0 shut down
|
||||
INFO 09-23 14:14:56 [core_client.py:689] [shutdown] MPClient: start timeout=default
|
||||
INFO 09-23 14:14:56 [utils.py:620] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage1_replica0
|
||||
INFO 09-23 14:15:00 [stage_pool.py:1354] [StagePool] Stage 1 replica 0 shut down
|
||||
INFO 09-23 14:15:00 [core_client.py:689] [shutdown] MPClient: start timeout=default
|
||||
INFO 09-23 14:15:00 [utils.py:620] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage2_replica0
|
||||
INFO 09-23 14:15:04 [stage_pool.py:1354] [StagePool] Stage 2 replica 0 shut down
|
||||
+289
@@ -0,0 +1,289 @@
|
||||
INFO 09-23 14:15:50 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 14:15:50 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 14:15:50 [patch.py:583] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
|
||||
====================
|
||||
vllm version: 0.29.0
|
||||
====================
|
||||
INFO 09-23 14:15:51 [omni_base.py:216] [Omni] Initializing with model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5
|
||||
INFO 09-23 14:15:51 [async_omni_engine.py:172] [AsyncOmniEngine] Initializing with model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5
|
||||
INFO 09-23 14:15:51 [async_omni_engine.py:290] [AsyncOmniEngine] Launching Orchestrator thread with 3 stages
|
||||
INFO 09-23 14:15:51 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
INFO 09-23 14:15:51 [initialization.py:356] Loaded OmniTransferConfig with 1 connector configurations
|
||||
WARNING 09-23 14:15:51 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 14:15:51 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
INFO 09-23 14:15:51 [model.py:684] Resolved architecture: MiniCPMO45OmniForConditionalGeneration
|
||||
INFO 09-23 14:15:51 [model.py:2021] Using max model len 2048
|
||||
WARNING 09-23 14:15:51 [model.py:981] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False.
|
||||
INFO 09-23 14:15:52 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode.
|
||||
INFO 09-23 14:15:52 [model.py:887] Disabled mm_prefix attention mode because multimodal inputs are configuration-disabled. Attention backends without mm_prefix support may now be selected.
|
||||
INFO 09-23 14:15:52 [scheduler.py:277] Chunked prefill is enabled with max_num_batched_tokens=2048.
|
||||
WARNING 09-23 14:15:52 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
WARNING 09-23 14:15:52 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
INFO 09-23 14:15:52 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-23 14:15:52 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
INFO 09-23 14:15:52 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
WARNING 09-23 14:15:52 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 14:15:52 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
INFO 09-23 14:15:52 [model.py:684] Resolved architecture: MiniCPMO45OmniForConditionalGeneration
|
||||
INFO 09-23 14:15:52 [model.py:2021] Using max model len 512
|
||||
WARNING 09-23 14:15:52 [model.py:981] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False.
|
||||
INFO 09-23 14:15:52 [model.py:2021] Using max model len 512
|
||||
INFO 09-23 14:15:52 [scheduler.py:277] Chunked prefill is enabled with max_num_batched_tokens=512.
|
||||
INFO 09-23 14:15:52 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
WARNING 09-23 14:15:52 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 14:15:52 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
INFO 09-23 14:15:52 [model.py:684] Resolved architecture: MiniCPMO45Code2Wav
|
||||
INFO 09-23 14:15:52 [model.py:2021] Using max model len 4096
|
||||
WARNING 09-23 14:15:52 [arg_utils.py:2780] This model does not officially support disabling chunked prefill. Disabling this manually may cause the engine to crash or produce incorrect outputs.
|
||||
INFO 09-23 14:15:52 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-23 14:15:52 [stage_runtime.py:853] [stage_init] Stage-0 set runtime devices: 0
|
||||
INFO 09-23 14:15:59 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 14:15:59 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 14:15:59 [patch.py:583] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) INFO 09-23 14:16:00 [core.py:123] Initializing a V1 LLM engine (v0.29.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=2048, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 0, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) INFO 09-23 14:16:01 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) INFO 09-23 14:16:01 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_fac8d3d9be8c43dda9c20b025e5d2604 backend=nccl
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) INFO 09-23 14:16:01 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) WARNING 09-23 14:16:01 [gpu_ar_worker.py:120] OMNI GPUARWorker forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) INFO 09-23 14:16:01 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) INFO 09-23 14:16:01 [base.py:138] Offloader set to UVAOffloader
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) WARNING 09-23 14:16:01 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) WARNING 09-23 14:16:01 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) INFO 09-23 14:16:01 [gpu_model_runner.py:5379] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5...
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) WARNING 09-23 14:16:02 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) WARNING 09-23 14:16:02 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) INFO 09-23 14:16:02 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) INFO 09-23 14:16:02 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) INFO 09-23 14:16:02 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) INFO 09-23 14:16:02 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) INFO 09-23 14:16:02 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) INFO 09-23 14:16:02 [cuda.py:492] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) INFO 09-23 14:16:02 [flash_attn.py:897] Using FlashAttention version 2
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) INFO 09-23 14:16:07 [uva.py:65] Total CPU offloaded parameters: 5.03
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) INFO 09-23 14:16:07 [weight_utils.py:863] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 22.46 GiB.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) INFO 09-23 14:16:07 [weight_utils.py:886] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) ␍Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) ␍Loading safetensors checkpoint shards: 25% Completed | 1/4 [00:00<00:01, 1.76it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:00<00:00, 6.12it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) INFO 09-23 14:16:14 [default_loader.py:430] Loading weights took 7.42 seconds
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) INFO 09-23 14:16:18 [gpu_model_runner.py:5471] Model loading took 10.84 GiB memory and 15.231180 seconds
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) INFO 09-23 14:16:18 [utils.py:306] Using LBNHC KV cache layout.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) INFO 09-23 14:16:21 [base.py:148] Available KV cache memory: 2.36 GiB (device-level profiling)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) INFO 09-23 14:16:21 [kv_cache_utils.py:2032] GPU KV cache size: 17,152 tokens, Maximum concurrency for 2,048 tokens per request: 8.38x
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) WARNING 09-23 14:16:21 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) INFO 09-23 14:16:22 [kernel_warmup.py:124] JIT kernel warmup starting.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) ␍JIT kernel warmup (1 compile keys): 0%| | 0/1 [00:00<?, ?kernel/s]␍JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 3.70kernel/s, ComputeSlotMappingKernel (1 keys)]␍JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 3.69kernel/s, ComputeSlotMappingKernel (1 keys)]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) INFO 09-23 14:16:22 [kernel_warmup.py:134] JIT kernel warmup finished in 0.27s.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) WARNING 09-23 14:16:22 [import_utils.py:408] Module vllm.third_party.deep_gemm was found but failed to import
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) WARNING 09-23 14:16:22 [import_utils.py:408] Traceback (most recent call last):
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) WARNING 09-23 14:16:22 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/utils/import_utils.py", line 406, in _has_module
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) WARNING 09-23 14:16:22 [import_utils.py:408] importlib.import_module(module_name)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) WARNING 09-23 14:16:22 [import_utils.py:408] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/importlib/__init__.py", line 90, in import_module
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) WARNING 09-23 14:16:22 [import_utils.py:408] return _bootstrap._gcd_import(name[level:], package, level)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) WARNING 09-23 14:16:22 [import_utils.py:408] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) WARNING 09-23 14:16:22 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1387, in _gcd_import
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) WARNING 09-23 14:16:22 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1360, in _find_and_load
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) WARNING 09-23 14:16:22 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1331, in _find_and_load_unlocked
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) WARNING 09-23 14:16:22 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 935, in _load_unlocked
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) WARNING 09-23 14:16:22 [import_utils.py:408] File "<frozen importlib._bootstrap_external>", line 999, in exec_module
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) WARNING 09-23 14:16:22 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 488, in _call_with_frames_removed
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) WARNING 09-23 14:16:22 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/third_party/deep_gemm/__init__.py", line 126, in <module>
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) WARNING 09-23 14:16:22 [import_utils.py:408] _find_cuda_home() # CUDA home
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) WARNING 09-23 14:16:22 [import_utils.py:408] ^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) WARNING 09-23 14:16:22 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/third_party/deep_gemm/__init__.py", line 120, in _find_cuda_home
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) WARNING 09-23 14:16:22 [import_utils.py:408] assert cuda_home is not None
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) WARNING 09-23 14:16:22 [import_utils.py:408] ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) WARNING 09-23 14:16:22 [import_utils.py:408] AssertionError
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) INFO 09-23 14:16:22 [gpu_worker.py:860] Free memory on device (22.58/23.89 GiB) on startup. Desired GPU memory utilization is (0.58, 13.86 GiB). Actual usage is 11.01 GiB for consumed memory (weights + non-torch), 0.58 GiB for peak activation, and 0.0 GiB for CUDAGraph memory. Replace gpu_memory_utilization config with `--kv-cache-memory=2281984123` (2.13 GiB) to fit into requested memory, or `--kv-cache-memory=11646516224` (10.85 GiB) to fully utilize gpu memory. Current kv cache memory in use is 2.36 GiB.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) INFO 09-23 14:16:23 [jit_monitor.py:85] Kernel JIT monitor activated; monitored JIT compilations during inference will use mode=warn.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) INFO 09-23 14:16:24 [torch_utils.py:277] Reducing Torch threads from 12 to 1 for serving. Set OMP_NUM_THREADS in the external environment to override.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) INFO 09-23 14:16:24 [core.py:368] init engine (profile, create kv cache, warmup model) took 5.63 s
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) WARNING 09-23 14:16:24 [scheduler.py:213] Using custom scheduler class vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) WARNING 09-23 14:16:24 [omni_scheduler_mixin.py:118] OMNI scheduler forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) INFO 09-23 14:16:24 [chunk_transfer_adapter.py:142] Bounded active-stream window enabled: K=1. Multi-replica deployments require sticky per-stream routing across Stage 1 replicas (each replica owns an independent active-set; without sticky routing, a stream can be active on one replica and non-active on another and both will race to evict it).
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) INFO 09-23 14:16:24 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4152525) INFO 09-23 14:16:24 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-23 14:16:24 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed
|
||||
INFO 09-23 14:16:24 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-0 [rep-0] initializing EngineCore
|
||||
INFO 09-23 14:16:24 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-0 [rep-0] EngineCore running
|
||||
INFO 09-23 14:16:24 [stage_runtime.py:958] [StageRuntime] Stage 0 initialized
|
||||
INFO 09-23 14:16:24 [stage_runtime.py:853] [stage_init] Stage-1 set runtime devices: 0
|
||||
INFO 09-23 14:16:31 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 14:16:31 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 14:16:31 [patch.py:583] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) INFO 09-23 14:16:32 [core.py:123] Initializing a V1 LLM engine (v0.29.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=512, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [512], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 0, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) INFO 09-23 14:16:34 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_5fabba368db344b7979baf05e10c877f backend=nccl
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) INFO 09-23 14:16:34 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) WARNING 09-23 14:16:34 [gpu_ar_worker.py:120] OMNI GPUARWorker forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) INFO 09-23 14:16:34 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) [transformers] `MiniCPMOProcessor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) [transformers] Requested torchvision backend is not available. Falling back to pil backend.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) [transformers] `MiniCPMOProcessor` defines `feature_extractor_class = 'WhisperFeatureExtractor'`, which is deprecated. Register the correct mapping in `AutoFeatureExtractor` instead.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) [transformers] Image processor MiniCPMVImageProcessor: kwargs ['max_slice_nums'] were applied for backward compatibility. To avoid this warning, add them to valid_kwargs: create a custom TypedDict extending ImagesKwargs with these keys and set it as the `valid_kwargs` class attribute.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) INFO 09-23 14:16:44 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) WARNING 09-23 14:16:44 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) WARNING 09-23 14:16:44 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) INFO 09-23 14:16:44 [gpu_model_runner.py:5379] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5...
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) WARNING 09-23 14:16:45 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) WARNING 09-23 14:16:45 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) INFO 09-23 14:16:45 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) INFO 09-23 14:16:45 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) INFO 09-23 14:16:45 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) INFO 09-23 14:16:45 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) INFO 09-23 14:16:46 [cuda.py:492] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) INFO 09-23 14:16:46 [flash_attn.py:897] Using FlashAttention version 2
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) INFO 09-23 14:16:46 [weight_utils.py:863] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 20.74 GiB.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) INFO 09-23 14:16:46 [weight_utils.py:886] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) ␍Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:00<00:00, 81.25it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) INFO 09-23 14:16:46 [default_loader.py:430] Loading weights took 0.30 seconds
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) INFO 09-23 14:16:47 [gpu_model_runner.py:5471] Model loading took 0.65 GiB memory and 1.143747 seconds
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) INFO 09-23 14:16:47 [utils.py:306] Using LBNHC KV cache layout.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) INFO 09-23 14:16:47 [gpu_model_runner.py:6519] Skipping memory profiling for multimodal encoder and encoder cache.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) INFO 09-23 14:16:48 [kv_cache_utils.py:2032] GPU KV cache size: 4,368 tokens, Maximum concurrency for 512 tokens per request: 8.53x
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) WARNING 09-23 14:16:48 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) INFO 09-23 14:16:49 [kernel_warmup.py:124] JIT kernel warmup starting.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) ␍JIT kernel warmup (1 compile keys): 0%| | 0/1 [00:00<?, ?kernel/s]␍JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 2.41kernel/s, ComputeSlotMappingKernel (1 keys)]␍JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 2.41kernel/s, ComputeSlotMappingKernel (1 keys)]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) INFO 09-23 14:16:49 [kernel_warmup.py:134] JIT kernel warmup finished in 0.42s.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) WARNING 09-23 14:16:49 [import_utils.py:408] Module vllm.third_party.deep_gemm was found but failed to import
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) WARNING 09-23 14:16:49 [import_utils.py:408] Traceback (most recent call last):
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) WARNING 09-23 14:16:49 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/utils/import_utils.py", line 406, in _has_module
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) WARNING 09-23 14:16:49 [import_utils.py:408] importlib.import_module(module_name)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) WARNING 09-23 14:16:49 [import_utils.py:408] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/importlib/__init__.py", line 90, in import_module
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) WARNING 09-23 14:16:49 [import_utils.py:408] return _bootstrap._gcd_import(name[level:], package, level)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) WARNING 09-23 14:16:49 [import_utils.py:408] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) WARNING 09-23 14:16:49 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1387, in _gcd_import
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) WARNING 09-23 14:16:49 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1360, in _find_and_load
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) WARNING 09-23 14:16:49 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1331, in _find_and_load_unlocked
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) WARNING 09-23 14:16:49 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 935, in _load_unlocked
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) WARNING 09-23 14:16:49 [import_utils.py:408] File "<frozen importlib._bootstrap_external>", line 999, in exec_module
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) WARNING 09-23 14:16:49 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 488, in _call_with_frames_removed
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) WARNING 09-23 14:16:49 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/third_party/deep_gemm/__init__.py", line 126, in <module>
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) WARNING 09-23 14:16:49 [import_utils.py:408] _find_cuda_home() # CUDA home
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) WARNING 09-23 14:16:49 [import_utils.py:408] ^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) WARNING 09-23 14:16:49 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/third_party/deep_gemm/__init__.py", line 120, in _find_cuda_home
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) WARNING 09-23 14:16:49 [import_utils.py:408] assert cuda_home is not None
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) WARNING 09-23 14:16:49 [import_utils.py:408] ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) WARNING 09-23 14:16:49 [import_utils.py:408] AssertionError
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) INFO 09-23 14:16:50 [jit_monitor.py:85] Kernel JIT monitor activated; monitored JIT compilations during inference will use mode=warn.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) INFO 09-23 14:16:50 [torch_utils.py:277] Reducing Torch threads from 12 to 1 for serving. Set OMP_NUM_THREADS in the external environment to override.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) INFO 09-23 14:16:50 [core.py:368] init engine (profile, create kv cache, warmup model) took 2.83 s
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) WARNING 09-23 14:16:50 [scheduler.py:213] Using custom scheduler class vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) WARNING 09-23 14:16:50 [omni_scheduler_mixin.py:118] OMNI scheduler forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) INFO 09-23 14:16:50 [chunk_transfer_adapter.py:142] Bounded active-stream window enabled: K=1. Multi-replica deployments require sticky per-stream routing across Stage 1 replicas (each replica owns an independent active-set; without sticky routing, a stream can be active on one replica and non-active on another and both will race to evict it).
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) INFO 09-23 14:16:50 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4152819) INFO 09-23 14:16:50 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-23 14:16:50 [stage_runtime.py:943] [StageRuntime] Stage 1 engine startup completed
|
||||
INFO 09-23 14:16:50 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-1 [rep-0] initializing EngineCore
|
||||
INFO 09-23 14:16:51 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-1 [rep-0] EngineCore running
|
||||
INFO 09-23 14:16:51 [stage_runtime.py:958] [StageRuntime] Stage 1 initialized
|
||||
INFO 09-23 14:16:51 [stage_runtime.py:853] [stage_init] Stage-2 set runtime devices: 0
|
||||
INFO 09-23 14:17:00 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 14:17:00 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 14:17:00 [patch.py:583] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) INFO 09-23 14:17:01 [core.py:123] Initializing a V1 LLM engine (v0.29.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=4096, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=False, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [4096], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 0, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) INFO 09-23 14:17:01 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_ee4ef51aa7a04a1ea64dcd3bc362a480 backend=nccl
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) INFO 09-23 14:17:01 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) WARNING 09-23 14:17:01 [gpu_generation_worker.py:103] OMNI GPUGenerationWorker forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) INFO 09-23 14:17:01 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) INFO 09-23 14:17:01 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) WARNING 09-23 14:17:01 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) WARNING 09-23 14:17:01 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) INFO 09-23 14:17:01 [gpu_model_runner.py:5379] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5...
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) INFO 09-23 14:17:02 [weight_utils.py:863] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 19.30 GiB.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) INFO 09-23 14:17:02 [weight_utils.py:893] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre) and the checkpoint size (17.46 GiB) exceeds 90% of available RAM (19.30 GiB).
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) ␍Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:00<00:00, 287.86it/s]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) INFO 09-23 14:17:02 [step_audio2_token2wav.py:163] Loading Token2Wav models from: /home/zhout/project/edge_infer/models/MiniCPM-o-4_5/assets/token2wav
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) INFO 09-23 14:17:10 [step_audio2_token2wav.py:201] Token2Wav models loaded successfully
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) INFO 09-23 14:17:11 [batched_token2wav.py:227] Enabling HiFT CUDA Graph with batch sizes [1]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) INFO 09-23 14:17:11 [cuda_graph_wrapper.py:98] Captured HiFT CUDA Graph for shape (1, 50, 0)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) INFO 09-23 14:17:12 [cuda_graph_wrapper.py:98] Captured HiFT CUDA Graph for shape (1, 58, 3840)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) INFO 09-23 14:17:12 [batched_token2wav.py:235] HiFT CUDA Graph captured successfully
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) INFO 09-23 14:17:12 [batched_token2wav.py:246] CFM CUDA Graph enabled (max_graphs=32)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) INFO 09-23 14:17:12 [default_loader.py:430] Loading weights took 9.69 seconds
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) INFO 09-23 14:17:12 [gpu_model_runner.py:5471] Model loading took 3.36 GiB memory and 9.704718 seconds
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) INFO 09-23 14:17:14 [cuda.py:492] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) INFO 09-23 14:17:14 [utils.py:306] Using LBNHC KV cache layout.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) WARNING 09-23 14:17:14 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) INFO 09-23 14:17:14 [kernel_warmup.py:124] JIT kernel warmup starting.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) ␍JIT kernel warmup (1 compile keys): 0%| | 0/1 [00:00<?, ?kernel/s]␍JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 380.61kernel/s, ComputeSlotMappingKernel (1 keys)]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) INFO 09-23 14:17:14 [kernel_warmup.py:134] JIT kernel warmup finished in 0.00s.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) WARNING 09-23 14:17:14 [import_utils.py:408] Module vllm.third_party.deep_gemm was found but failed to import
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) WARNING 09-23 14:17:14 [import_utils.py:408] Traceback (most recent call last):
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) WARNING 09-23 14:17:14 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/utils/import_utils.py", line 406, in _has_module
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) WARNING 09-23 14:17:14 [import_utils.py:408] importlib.import_module(module_name)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) WARNING 09-23 14:17:14 [import_utils.py:408] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/importlib/__init__.py", line 90, in import_module
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) WARNING 09-23 14:17:14 [import_utils.py:408] return _bootstrap._gcd_import(name[level:], package, level)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) WARNING 09-23 14:17:14 [import_utils.py:408] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) WARNING 09-23 14:17:14 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1387, in _gcd_import
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) WARNING 09-23 14:17:14 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1360, in _find_and_load
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) WARNING 09-23 14:17:14 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1331, in _find_and_load_unlocked
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) WARNING 09-23 14:17:14 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 935, in _load_unlocked
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) WARNING 09-23 14:17:14 [import_utils.py:408] File "<frozen importlib._bootstrap_external>", line 999, in exec_module
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) WARNING 09-23 14:17:14 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 488, in _call_with_frames_removed
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) WARNING 09-23 14:17:14 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/third_party/deep_gemm/__init__.py", line 126, in <module>
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) WARNING 09-23 14:17:14 [import_utils.py:408] _find_cuda_home() # CUDA home
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) WARNING 09-23 14:17:14 [import_utils.py:408] ^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) WARNING 09-23 14:17:14 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/third_party/deep_gemm/__init__.py", line 120, in _find_cuda_home
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) WARNING 09-23 14:17:14 [import_utils.py:408] assert cuda_home is not None
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) WARNING 09-23 14:17:14 [import_utils.py:408] ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) WARNING 09-23 14:17:14 [import_utils.py:408] AssertionError
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) WARNING 09-23 14:17:14 [gpu_generation_model_runner.py:558] Dummy sampler run is not implemented for generation model
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) INFO 09-23 14:17:15 [jit_monitor.py:85] Kernel JIT monitor activated; monitored JIT compilations during inference will use mode=warn.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) INFO 09-23 14:17:15 [torch_utils.py:277] Reducing Torch threads from 12 to 1 for serving. Set OMP_NUM_THREADS in the external environment to override.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) INFO 09-23 14:17:15 [core.py:368] init engine (profile, create kv cache, warmup model) took 2.51 s
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) WARNING 09-23 14:17:16 [scheduler.py:213] Using custom scheduler class vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) WARNING 09-23 14:17:16 [omni_scheduler_mixin.py:118] OMNI scheduler forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) INFO 09-23 14:17:16 [chunk_transfer_adapter.py:142] Bounded active-stream window enabled: K=1. Multi-replica deployments require sticky per-stream routing across Stage 1 replicas (each replica owns an independent active-set; without sticky routing, a stream can be active on one replica and non-active on another and both will race to evict it).
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) INFO 09-23 14:17:16 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) WARNING 09-23 14:17:16 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) WARNING 09-23 14:17:16 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) INFO 09-23 14:17:16 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) INFO 09-23 14:17:16 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
INFO 09-23 14:17:16 [stage_runtime.py:943] [StageRuntime] Stage 2 engine startup completed
|
||||
INFO 09-23 14:17:16 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-2 [rep-0] initializing EngineCore
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) INFO 09-23 14:17:16 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
INFO 09-23 14:17:16 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-2 [rep-0] EngineCore running
|
||||
INFO 09-23 14:17:16 [stage_runtime.py:958] [StageRuntime] Stage 2 initialized
|
||||
INFO 09-23 14:17:18 [orchestrator.py:604] [Orchestrator] Starting event loop
|
||||
INFO 09-23 14:17:18 [async_omni_engine.py:323] [AsyncOmniEngine] Orchestrator ready with 3 stages
|
||||
INFO 09-23 14:17:18 [omni_base.py:235] [Omni] AsyncOmniEngine initialized in 86.49 seconds
|
||||
INFO 09-23 14:17:18 [omni_base.py:255] [Omni] Initialized with 3 stages for model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5
|
||||
WARNING 09-23 14:17:18 [input_processor.py:315] Passing raw prompts to InputProcessor is deprecated and will be removed in the future. You should instead pass the outputs of Renderer.render_cmpl() or Renderer.render_chat().
|
||||
␍Processed prompts: 0%| | 0/1 [00:00<?, ?it/s](StageEngineCoreProc_stage0_replica0 pid=4152525) WARNING 09-23 14:17:18 [gpu_ar_model_runner.py:1461] prefer_model_sampler model MiniCPMO45OmniForConditionalGeneration returned None from sample(); falling back to the default sampler.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) INFO 09-23 14:18:21 [cuda_graph_wrapper.py:300] Captured CFM CUDA Graph for shape ('estimator_step', ((2, 320, 302), 'torch.float32', 'cuda:0'), ((2, 1, 512), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 0, 128), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 302, 128), 'torch.float32', 'cuda:0')) (cache=1/32, stats={'calls': 1, 'hits': 0, 'captures': 1, 'flushes': 0, 'eager': 0, 'cache_size': 0}) [allocated 3633.9 MiB (+32.2), reserved 5464.0 MiB (+22.0)]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) INFO 09-23 14:18:22 [cuda_graph_wrapper.py:300] Captured CFM CUDA Graph for shape ('estimator_step', ((2, 320, 50), 'torch.float32', 'cuda:0'), ((2, 1, 512), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 302, 128), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 352, 128), 'torch.float32', 'cuda:0')) (cache=2/32, stats={'calls': 11, 'hits': 9, 'captures': 2, 'flushes': 0, 'eager': 0, 'cache_size': 1}) [allocated 4539.7 MiB (+32.0), reserved 6378.0 MiB (+28.0)]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) INFO 09-23 14:18:22 [cuda_graph_wrapper.py:300] Captured CFM CUDA Graph for shape ('estimator_step', ((2, 320, 50), 'torch.float32', 'cuda:0'), ((2, 1, 512), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 352, 128), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 402, 128), 'torch.float32', 'cuda:0')) (cache=3/32, stats={'calls': 21, 'hits': 18, 'captures': 3, 'flushes': 0, 'eager': 0, 'cache_size': 2}) [allocated 4803.9 MiB (+32.0), reserved 6956.0 MiB (+-352.0)]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) INFO 09-23 14:18:22 [cuda_graph_wrapper.py:300] Captured CFM CUDA Graph for shape ('estimator_step', ((2, 320, 50), 'torch.float32', 'cuda:0'), ((2, 1, 512), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 402, 128), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 452, 128), 'torch.float32', 'cuda:0')) (cache=4/32, stats={'calls': 31, 'hits': 27, 'captures': 4, 'flushes': 0, 'eager': 0, 'cache_size': 3}) [allocated 5079.1 MiB (+32.0), reserved 7096.0 MiB (+-876.0)]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) INFO 09-23 14:18:23 [cuda_graph_wrapper.py:300] Captured CFM CUDA Graph for shape ('estimator_step', ((2, 320, 10), 'torch.float32', 'cuda:0'), ((2, 1, 512), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 402, 128), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 412, 128), 'torch.float32', 'cuda:0')) (cache=5/32, stats={'calls': 61, 'hits': 56, 'captures': 5, 'flushes': 0, 'eager': 0, 'cache_size': 4}) [allocated 5206.6 MiB (+32.0), reserved 7190.0 MiB (+-1550.0)]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) INFO 09-23 14:18:23 [cuda_graph_wrapper.py:120] Lazily capturing HiFT CUDA Graph for shape (1, 18, 3840)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) WARNING 09-23 14:18:23 [jit_monitor.py:141] Triton kernel JIT compilation during inference: SnakeBeta._init_triton.<locals>._kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4153030) INFO 09-23 14:18:23 [cuda_graph_wrapper.py:98] Captured HiFT CUDA Graph for shape (1, 18, 3840)
|
||||
␍Processed prompts: 100%|██████████| 1/1 [01:05<00:00, 65.81s/it]␍Processed prompts: 100%|██████████| 1/1 [01:05<00:00, 65.81s/it]
|
||||
query type: text
|
||||
use_tts (tts_bos): True
|
||||
Request ID: 0_ebbebf11-419e-4980-9dca-1006ac3ccd5a, Text saved to /home/zhout/project/edge_infer/analysis/experiments/e2e_recovery_20260923/mini_output_constrained/0_ebbebf11-419e-4980-9dca-1006ac3ccd5a.txt
|
||||
Request ID: 0_ebbebf11-419e-4980-9dca-1006ac3ccd5a, Saved audio to /home/zhout/project/edge_infer/analysis/experiments/e2e_recovery_20260923/mini_output_constrained/output_0_ebbebf11-419e-4980-9dca-1006ac3ccd5a.wav
|
||||
INFO 09-23 14:18:23 [omni_base.py:781] [Omni] Shutting down
|
||||
INFO 09-23 14:18:23 [async_omni_engine.py:1807] [AsyncOmniEngine] Shutting down Orchestrator
|
||||
INFO 09-23 14:18:23 [orchestrator.py:705] [Orchestrator] Received shutdown signal
|
||||
INFO 09-23 14:18:23 [orchestrator.py:3092] [Orchestrator] Shutting down all 3 client(s)
|
||||
INFO 09-23 14:18:23 [core_client.py:689] [shutdown] MPClient: start timeout=default
|
||||
INFO 09-23 14:18:23 [core_client.py:691] [shutdown] MPClient: stopping engine manager
|
||||
INFO 09-23 14:18:23 [utils.py:620] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage0_replica0
|
||||
INFO 09-23 14:18:29 [core_client.py:693] [shutdown] MPClient: engine manager stopped
|
||||
INFO 09-23 14:18:29 [core_client.py:694] [shutdown] MPClient: cleaning up background resources
|
||||
INFO 09-23 14:18:29 [core_client.py:696] [shutdown] MPClient: complete
|
||||
INFO 09-23 14:18:29 [stage_pool.py:1354] [StagePool] Stage 0 replica 0 shut down
|
||||
INFO 09-23 14:18:29 [core_client.py:689] [shutdown] MPClient: start timeout=default
|
||||
INFO 09-23 14:18:29 [utils.py:620] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage1_replica0
|
||||
INFO 09-23 14:18:33 [stage_pool.py:1354] [StagePool] Stage 1 replica 0 shut down
|
||||
INFO 09-23 14:18:33 [core_client.py:689] [shutdown] MPClient: start timeout=default
|
||||
INFO 09-23 14:18:33 [utils.py:620] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage2_replica0
|
||||
INFO 09-23 14:18:37 [stage_pool.py:1354] [StagePool] Stage 2 replica 0 shut down
|
||||
/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/multiprocessing/resource_tracker.py:279: UserWarning: resource_tracker: There appear to be 1 leaked shared_memory objects to clean up at shutdown
|
||||
warnings.warn('resource_tracker: There appear to be %d '
|
||||
+315
@@ -0,0 +1,315 @@
|
||||
INFO 09-23 14:06:30 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 14:06:30 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 14:06:30 [patch.py:583] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
|
||||
====================
|
||||
vllm version: 0.29.0
|
||||
====================
|
||||
INFO 09-23 14:06:31 [omni_base.py:216] [Omni] Initializing with model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5
|
||||
INFO 09-23 14:06:31 [async_omni_engine.py:172] [AsyncOmniEngine] Initializing with model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5
|
||||
INFO 09-23 14:06:31 [async_omni_engine.py:290] [AsyncOmniEngine] Launching Orchestrator thread with 3 stages
|
||||
INFO 09-23 14:06:31 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
INFO 09-23 14:06:31 [initialization.py:356] Loaded OmniTransferConfig with 1 connector configurations
|
||||
WARNING 09-23 14:06:31 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 14:06:31 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
INFO 09-23 14:06:31 [model.py:684] Resolved architecture: MiniCPMO45OmniForConditionalGeneration
|
||||
INFO 09-23 14:06:31 [model.py:2021] Using max model len 40960
|
||||
WARNING 09-23 14:06:31 [model.py:981] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False.
|
||||
INFO 09-23 14:06:32 [scheduler.py:277] Chunked prefill is enabled with max_num_batched_tokens=16384.
|
||||
INFO 09-23 14:06:32 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
WARNING 09-23 14:06:32 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 14:06:32 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
INFO 09-23 14:06:32 [model.py:684] Resolved architecture: MiniCPMO45OmniForConditionalGeneration
|
||||
INFO 09-23 14:06:32 [model.py:2021] Using max model len 4096
|
||||
WARNING 09-23 14:06:32 [model.py:981] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False.
|
||||
INFO 09-23 14:06:32 [model.py:2021] Using max model len 4096
|
||||
INFO 09-23 14:06:32 [scheduler.py:277] Chunked prefill is enabled with max_num_batched_tokens=8192.
|
||||
INFO 09-23 14:06:32 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
WARNING 09-23 14:06:32 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 14:06:32 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
INFO 09-23 14:06:32 [model.py:684] Resolved architecture: MiniCPMO45Code2Wav
|
||||
WARNING 09-23 14:06:32 [model.py:2515] User-specified max_model_len (65536) is greater than the derived max_model_len (max_position_embeddings=40960.0 or model_max_length=None in model's config.json). VLLM_ALLOW_LONG_MAX_MODEL_LEN must be used with extreme caution. If the model uses relative position encoding (RoPE), positions exceeding derived_max_model_len lead to nan. If the model uses absolute position encoding, positions exceeding derived_max_model_len will cause a CUDA array out-of-bounds error.
|
||||
INFO 09-23 14:06:32 [model.py:2021] Using max model len 65536
|
||||
WARNING 09-23 14:06:32 [arg_utils.py:2780] This model does not officially support disabling chunked prefill. Disabling this manually may cause the engine to crash or produce incorrect outputs.
|
||||
WARNING 09-23 14:06:32 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
WARNING 09-23 14:06:32 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
INFO 09-23 14:06:32 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-23 14:06:32 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
INFO 09-23 14:06:32 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
INFO 09-23 14:06:32 [stage_runtime.py:853] [stage_init] Stage-0 set runtime devices: 0
|
||||
INFO 09-23 14:06:40 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 14:06:40 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 14:06:40 [patch.py:583] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) INFO 09-23 14:06:41 [core.py:123] Initializing a V1 LLM engine (v0.29.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=40960, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::qwen4_exp_compute_ple_ngram_ids', 'vllm::qwen4_exp_ple_short_conv', 'vllm::qwen4_exp_qsa_with_output', 'vllm::linear_attention', 'vllm::qwen_gdn_attention_core', 'vllm::qwen_gdn_attention_core_fused_norm_packed', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::hpc_rope_norm_forward', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [16384], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 8, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) INFO 09-23 14:06:42 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_8a215ee3ae104eaa974483b32508c610 backend=nccl
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) INFO 09-23 14:06:42 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) WARNING 09-23 14:06:43 [gpu_ar_worker.py:120] OMNI GPUARWorker forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) INFO 09-23 14:06:43 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) [transformers] `MiniCPMOProcessor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) [transformers] Requested torchvision backend is not available. Falling back to pil backend.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) [transformers] `MiniCPMOProcessor` defines `feature_extractor_class = 'WhisperFeatureExtractor'`, which is deprecated. Register the correct mapping in `AutoFeatureExtractor` instead.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) [transformers] Image processor MiniCPMVImageProcessor: kwargs ['max_slice_nums'] were applied for backward compatibility. To avoid this warning, add them to valid_kwargs: create a custom TypedDict extending ImagesKwargs with these keys and set it as the `valid_kwargs` class attribute.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) WARNING 09-23 14:06:53 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) WARNING 09-23 14:06:53 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) INFO 09-23 14:06:53 [gpu_model_runner.py:5379] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5...
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) INFO 09-23 14:06:54 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) INFO 09-23 14:06:54 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) INFO 09-23 14:06:54 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) INFO 09-23 14:06:55 [cuda.py:492] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) INFO 09-23 14:06:55 [flash_attn.py:897] Using FlashAttention version 2
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) INFO 09-23 14:06:55 [weight_utils.py:863] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 26.66 GiB.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) INFO 09-23 14:06:55 [weight_utils.py:886] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) ␍Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:00<00:00, 151.44it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) INFO 09-23 14:07:10 [default_loader.py:430] Loading weights took 14.83 seconds
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) INFO 09-23 14:07:11 [gpu_model_runner.py:5471] Model loading took 16.83 GiB memory and 16.500706 seconds
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) INFO 09-23 14:07:12 [utils.py:306] Using LBNHC KV cache layout.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) INFO 09-23 14:07:12 [gpu_model_runner.py:6546] Encoder cache will be initialized with a budget of 16384 tokens, and profiled with 4 video items of the maximum feature size.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) [transformers] `use_return_dict` is deprecated! Use `return_dict` instead!
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) INFO 09-23 14:08:05 [backends.py:1094] Using cache directory: /home/zhout/.cache/vllm/torch_compile_cache/8ba29dd641/rank_0_0/backbone for vLLM's torch.compile
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) INFO 09-23 14:08:05 [backends.py:1155] Dynamo bytecode transform time: 5.13 s
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) INFO 09-23 14:08:22 [backends.py:393] Compiling a graph for compile range (1, 16384) takes 16.74 s
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) INFO 09-23 14:08:25 [backends.py:920] collected artifacts: 37 entries, 4 artifacts, 8252760 bytes total
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) INFO 09-23 14:08:25 [decorators.py:719] saved AOT compiled function to /home/zhout/.cache/vllm/torch_compile_cache/torch_aot_compile/2c72d07c99cd7298c67b04082700dca8c0cec20054928968738abdc67e470976/rank_0_0/model
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) INFO 09-23 14:08:25 [monitor.py:53] torch.compile took 25.11 s in total
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) INFO 09-23 14:08:25 [monitor.py:81] Initial profiling/warmup run took 0.01 s
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) INFO 09-23 14:08:30 [base.py:148] Available KV cache memory: 0.0 GiB (device-level profiling)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) ERROR 09-23 14:08:30 [stage_engine_core_proc.py:242] StageEngineCoreProc failed to start.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) ERROR 09-23 14:08:30 [stage_engine_core_proc.py:242] Traceback (most recent call last):
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) ERROR 09-23 14:08:30 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_core_proc.py", line 189, in run_stage_core
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) ERROR 09-23 14:08:30 [stage_engine_core_proc.py:242] engine_core = StageEngineCoreProc(
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) ERROR 09-23 14:08:30 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) ERROR 09-23 14:08:30 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) ERROR 09-23 14:08:30 [stage_engine_core_proc.py:242] return func(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) ERROR 09-23 14:08:30 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) ERROR 09-23 14:08:30 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/v1/engine/core.py", line 1093, in __init__
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) ERROR 09-23 14:08:30 [stage_engine_core_proc.py:242] super().__init__(
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) ERROR 09-23 14:08:30 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/v1/engine/core.py", line 145, in __init__
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) ERROR 09-23 14:08:30 [stage_engine_core_proc.py:242] kv_cache_config = self._initialize_kv_caches(vllm_config)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) ERROR 09-23 14:08:30 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) ERROR 09-23 14:08:30 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) ERROR 09-23 14:08:30 [stage_engine_core_proc.py:242] return func(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) ERROR 09-23 14:08:30 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) ERROR 09-23 14:08:30 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/v1/engine/core.py", line 319, in _initialize_kv_caches
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) ERROR 09-23 14:08:30 [stage_engine_core_proc.py:242] kv_cache_configs = get_kv_cache_configs(
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) ERROR 09-23 14:08:30 [stage_engine_core_proc.py:242] ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) ERROR 09-23 14:08:30 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/v1/core/kv_cache_utils.py", line 2331, in get_kv_cache_configs
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) ERROR 09-23 14:08:30 [stage_engine_core_proc.py:242] _check_enough_kv_cache_memory(
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) ERROR 09-23 14:08:30 [stage_engine_core_proc.py:242] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/v1/core/kv_cache_utils.py", line 859, in _check_enough_kv_cache_memory
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) ERROR 09-23 14:08:30 [stage_engine_core_proc.py:242] raise ValueError(
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) ERROR 09-23 14:08:30 [stage_engine_core_proc.py:242] ValueError: No available memory for the cache blocks. Try increasing `gpu_memory_utilization` when initializing the engine (this flag also controls CPU memory reservation on the CPU backend, despite its name). See https://docs.vllm.ai/en/latest/configuration/conserving_memory/ for more details.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) Process StageEngineCoreProc_stage0_replica0:
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) Traceback (most recent call last):
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/multiprocessing/process.py", line 314, in _bootstrap
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) self.run()
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/multiprocessing/process.py", line 108, in run
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) self._target(*self._args, **self._kwargs)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_core_proc.py", line 189, in run_stage_core
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) engine_core = StageEngineCoreProc(
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) ^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) return func(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/v1/engine/core.py", line 1093, in __init__
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) super().__init__(
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/v1/engine/core.py", line 145, in __init__
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) kv_cache_config = self._initialize_kv_caches(vllm_config)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/tracing/otel.py", line 178, in sync_wrapper
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) return func(*args, **kwargs)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/v1/engine/core.py", line 319, in _initialize_kv_caches
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) kv_cache_configs = get_kv_cache_configs(
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/v1/core/kv_cache_utils.py", line 2331, in get_kv_cache_configs
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) _check_enough_kv_cache_memory(
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/v1/core/kv_cache_utils.py", line 859, in _check_enough_kv_cache_memory
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) raise ValueError(
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4149560) ValueError: No available memory for the cache blocks. Try increasing `gpu_memory_utilization` when initializing the engine (this flag also controls CPU memory reservation on the CPU backend, despite its name). See https://docs.vllm.ai/en/latest/configuration/conserving_memory/ for more details.
|
||||
[rank0]:[W923 14:08:32.490792482 ProcessGroupNCCL.cpp:1624] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator())
|
||||
INFO 09-23 14:08:34 [utils.py:620] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage0_replica0
|
||||
ERROR 09-23 14:08:34 [stage_runtime.py:265] [StageRuntime] Stage initialization failed; shutting down 0 initialized client(s)
|
||||
ERROR 09-23 14:08:34 [stage_runtime.py:265] Traceback (most recent call last):
|
||||
ERROR 09-23 14:08:34 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 252, in initialize
|
||||
ERROR 09-23 14:08:34 [stage_runtime.py:265] initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout)
|
||||
ERROR 09-23 14:08:34 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 14:08:34 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 589, in _initialize_stage_replicas
|
||||
ERROR 09-23 14:08:34 [stage_runtime.py:265] raise primary_exc
|
||||
ERROR 09-23 14:08:34 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 550, in _init_group
|
||||
ERROR 09-23 14:08:34 [stage_runtime.py:265] client = self._initialize_replica(
|
||||
ERROR 09-23 14:08:34 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 14:08:34 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 828, in _initialize_replica
|
||||
ERROR 09-23 14:08:34 [stage_runtime.py:265] return self._initialize_local_llm_replica(plan, stage_init_timeout)
|
||||
ERROR 09-23 14:08:34 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 14:08:34 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 934, in _initialize_local_llm_replica
|
||||
ERROR 09-23 14:08:34 [stage_runtime.py:265] launch_stack.close()
|
||||
ERROR 09-23 14:08:34 [stage_runtime.py:265] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 618, in close
|
||||
ERROR 09-23 14:08:34 [stage_runtime.py:265] self.__exit__(None, None, None)
|
||||
ERROR 09-23 14:08:34 [stage_runtime.py:265] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 610, in __exit__
|
||||
ERROR 09-23 14:08:34 [stage_runtime.py:265] raise exc_details[1]
|
||||
ERROR 09-23 14:08:34 [stage_runtime.py:265] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 595, in __exit__
|
||||
ERROR 09-23 14:08:34 [stage_runtime.py:265] if cb(*exc_details):
|
||||
ERROR 09-23 14:08:34 [stage_runtime.py:265] ^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 14:08:34 [stage_runtime.py:265] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 144, in __exit__
|
||||
ERROR 09-23 14:08:34 [stage_runtime.py:265] next(self.gen)
|
||||
ERROR 09-23 14:08:34 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_startup.py", line 1211, in launch_stage_replica
|
||||
ERROR 09-23 14:08:34 [stage_runtime.py:265] wait_for_engine_startup(
|
||||
ERROR 09-23 14:08:34 [stage_runtime.py:265] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/v1/engine/utils.py", line 1320, in wait_for_engine_startup
|
||||
ERROR 09-23 14:08:34 [stage_runtime.py:265] raise RuntimeError(
|
||||
ERROR 09-23 14:08:34 [stage_runtime.py:265] RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {}
|
||||
ERROR 09-23 14:08:34 [async_omni_engine.py:474] [AsyncOmniEngine] Orchestrator thread crashed
|
||||
ERROR 09-23 14:08:34 [async_omni_engine.py:474] Traceback (most recent call last):
|
||||
ERROR 09-23 14:08:34 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 468, in _bootstrap_orchestrator
|
||||
ERROR 09-23 14:08:34 [async_omni_engine.py:474] loop.run_until_complete(_run_orchestrator())
|
||||
ERROR 09-23 14:08:34 [async_omni_engine.py:474] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/base_events.py", line 691, in run_until_complete
|
||||
ERROR 09-23 14:08:34 [async_omni_engine.py:474] return future.result()
|
||||
ERROR 09-23 14:08:34 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 14:08:34 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 424, in _run_orchestrator
|
||||
ERROR 09-23 14:08:34 [async_omni_engine.py:474] self._initialize_stages(stage_init_timeout)
|
||||
ERROR 09-23 14:08:34 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 350, in _initialize_stages
|
||||
ERROR 09-23 14:08:34 [async_omni_engine.py:474] self._initialize_stages_impl(stage_init_timeout)
|
||||
ERROR 09-23 14:08:34 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 371, in _initialize_stages_impl
|
||||
ERROR 09-23 14:08:34 [async_omni_engine.py:474] self._runtime.initialize()
|
||||
ERROR 09-23 14:08:34 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 278, in initialize
|
||||
ERROR 09-23 14:08:34 [async_omni_engine.py:474] raise exc
|
||||
ERROR 09-23 14:08:34 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 252, in initialize
|
||||
ERROR 09-23 14:08:34 [async_omni_engine.py:474] initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout)
|
||||
ERROR 09-23 14:08:34 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 14:08:34 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 589, in _initialize_stage_replicas
|
||||
ERROR 09-23 14:08:34 [async_omni_engine.py:474] raise primary_exc
|
||||
ERROR 09-23 14:08:34 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 550, in _init_group
|
||||
ERROR 09-23 14:08:34 [async_omni_engine.py:474] client = self._initialize_replica(
|
||||
ERROR 09-23 14:08:34 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 14:08:34 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 828, in _initialize_replica
|
||||
ERROR 09-23 14:08:34 [async_omni_engine.py:474] return self._initialize_local_llm_replica(plan, stage_init_timeout)
|
||||
ERROR 09-23 14:08:34 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 14:08:34 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 934, in _initialize_local_llm_replica
|
||||
ERROR 09-23 14:08:34 [async_omni_engine.py:474] launch_stack.close()
|
||||
ERROR 09-23 14:08:34 [async_omni_engine.py:474] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 618, in close
|
||||
ERROR 09-23 14:08:34 [async_omni_engine.py:474] self.__exit__(None, None, None)
|
||||
ERROR 09-23 14:08:34 [async_omni_engine.py:474] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 610, in __exit__
|
||||
ERROR 09-23 14:08:34 [async_omni_engine.py:474] raise exc_details[1]
|
||||
ERROR 09-23 14:08:34 [async_omni_engine.py:474] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 595, in __exit__
|
||||
ERROR 09-23 14:08:34 [async_omni_engine.py:474] if cb(*exc_details):
|
||||
ERROR 09-23 14:08:34 [async_omni_engine.py:474] ^^^^^^^^^^^^^^^^
|
||||
ERROR 09-23 14:08:34 [async_omni_engine.py:474] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 144, in __exit__
|
||||
ERROR 09-23 14:08:34 [async_omni_engine.py:474] next(self.gen)
|
||||
ERROR 09-23 14:08:34 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_startup.py", line 1211, in launch_stage_replica
|
||||
ERROR 09-23 14:08:34 [async_omni_engine.py:474] wait_for_engine_startup(
|
||||
ERROR 09-23 14:08:34 [async_omni_engine.py:474] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/v1/engine/utils.py", line 1320, in wait_for_engine_startup
|
||||
ERROR 09-23 14:08:34 [async_omni_engine.py:474] raise RuntimeError(
|
||||
ERROR 09-23 14:08:34 [async_omni_engine.py:474] RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {}
|
||||
INFO 09-23 14:08:34 [async_omni_engine.py:1807] [AsyncOmniEngine] Shutting down Orchestrator
|
||||
Exception in thread orchestrator:
|
||||
Traceback (most recent call last):
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
|
||||
self.run()
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/threading.py", line 1012, in run
|
||||
self._target(*self._args, **self._kwargs)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 468, in _bootstrap_orchestrator
|
||||
loop.run_until_complete(_run_orchestrator())
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/base_events.py", line 691, in run_until_complete
|
||||
return future.result()
|
||||
^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 424, in _run_orchestrator
|
||||
self._initialize_stages(stage_init_timeout)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 350, in _initialize_stages
|
||||
self._initialize_stages_impl(stage_init_timeout)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 371, in _initialize_stages_impl
|
||||
self._runtime.initialize()
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 278, in initialize
|
||||
raise exc
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 252, in initialize
|
||||
initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 589, in _initialize_stage_replicas
|
||||
raise primary_exc
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 550, in _init_group
|
||||
client = self._initialize_replica(
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 828, in _initialize_replica
|
||||
return self._initialize_local_llm_replica(plan, stage_init_timeout)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 934, in _initialize_local_llm_replica
|
||||
launch_stack.close()
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 618, in close
|
||||
self.__exit__(None, None, None)
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 610, in __exit__
|
||||
raise exc_details[1]
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 595, in __exit__
|
||||
if cb(*exc_details):
|
||||
^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 144, in __exit__
|
||||
next(self.gen)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_startup.py", line 1211, in launch_stage_replica
|
||||
wait_for_engine_startup(
|
||||
File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/v1/engine/utils.py", line 1320, in wait_for_engine_startup
|
||||
raise RuntimeError(
|
||||
RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {}
|
||||
Traceback (most recent call last):
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/threading.py", line 1075, in _bootstrap_inner
|
||||
self.run()
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/threading.py", line 1012, in run
|
||||
self._target(*self._args, **self._kwargs)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 468, in _bootstrap_orchestrator
|
||||
loop.run_until_complete(_run_orchestrator())
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/asyncio/base_events.py", line 691, in run_until_complete
|
||||
return future.result()
|
||||
^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 424, in _run_orchestrator
|
||||
self._initialize_stages(stage_init_timeout)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 350, in _initialize_stages
|
||||
self._initialize_stages_impl(stage_init_timeout)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 371, in _initialize_stages_impl
|
||||
self._runtime.initialize()
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 278, in initialize
|
||||
raise exc
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 252, in initialize
|
||||
initialized_clients = self._initialize_stage_replicas(stage_plans, self._stage_init_timeout)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 589, in _initialize_stage_replicas
|
||||
raise primary_exc
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 550, in _init_group
|
||||
client = self._initialize_replica(
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 828, in _initialize_replica
|
||||
return self._initialize_local_llm_replica(plan, stage_init_timeout)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_runtime.py", line 934, in _initialize_local_llm_replica
|
||||
launch_stack.close()
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 618, in close
|
||||
self.__exit__(None, None, None)
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 610, in __exit__
|
||||
raise exc_details[1]
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 595, in __exit__
|
||||
if cb(*exc_details):
|
||||
^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/contextlib.py", line 144, in __exit__
|
||||
next(self.gen)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/stage_engine_startup.py", line 1211, in launch_stage_replica
|
||||
wait_for_engine_startup(
|
||||
File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/v1/engine/utils.py", line 1320, in wait_for_engine_startup
|
||||
raise RuntimeError(
|
||||
RuntimeError: Engine core initialization failed. See root cause above. Failed core proc(s): {}
|
||||
|
||||
The above exception was the direct cause of the following exception:
|
||||
|
||||
Traceback (most recent call last):
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/examples/offline_inference/minicpmo/end2end.py", line 537, in <module>
|
||||
main(args)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/examples/offline_inference/minicpmo/end2end.py", line 286, in main
|
||||
omni = Omni(**omni_kwargs)
|
||||
^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/entrypoints/omni_base.py", line 223, in __init__
|
||||
self.engine = AsyncOmniEngine(
|
||||
^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 305, in __init__
|
||||
self._wait_for_orchestrator_init(startup_future, startup_timeout)
|
||||
File "/home/zhout/project/edge_infer/vllm-omni-edge/vllm_omni/engine/async_omni_engine.py", line 539, in _wait_for_orchestrator_init
|
||||
startup_future.result(
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/concurrent/futures/_base.py", line 456, in result
|
||||
return self.__get_result()
|
||||
^^^^^^^^^^^^^^^^^^^
|
||||
File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/concurrent/futures/_base.py", line 401, in __get_result
|
||||
raise self._exception
|
||||
RuntimeError: Orchestrator initialization failed: Engine core initialization failed. See root cause above. Failed core proc(s): {}
|
||||
+290
@@ -0,0 +1,290 @@
|
||||
INFO 09-23 15:04:34 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 15:04:34 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 15:04:34 [patch.py:583] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
|
||||
====================
|
||||
vllm version: 0.29.0
|
||||
====================
|
||||
INFO 09-23 15:04:35 [omni_base.py:216] [Omni] Initializing with model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5
|
||||
INFO 09-23 15:04:35 [async_omni_engine.py:172] [AsyncOmniEngine] Initializing with model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5
|
||||
INFO 09-23 15:04:35 [async_omni_engine.py:290] [AsyncOmniEngine] Launching Orchestrator thread with 3 stages
|
||||
INFO 09-23 15:04:35 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
INFO 09-23 15:04:35 [initialization.py:356] Loaded OmniTransferConfig with 1 connector configurations
|
||||
WARNING 09-23 15:04:35 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 15:04:35 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
INFO 09-23 15:04:45 [model.py:684] Resolved architecture: MiniCPMO45OmniForConditionalGeneration
|
||||
INFO 09-23 15:04:45 [model.py:2021] Using max model len 2048
|
||||
WARNING 09-23 15:04:45 [model.py:981] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False.
|
||||
INFO 09-23 15:04:45 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode.
|
||||
INFO 09-23 15:04:45 [model.py:887] Disabled mm_prefix attention mode because multimodal inputs are configuration-disabled. Attention backends without mm_prefix support may now be selected.
|
||||
INFO 09-23 15:04:45 [scheduler.py:277] Chunked prefill is enabled with max_num_batched_tokens=2048.
|
||||
WARNING 09-23 15:04:45 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
WARNING 09-23 15:04:45 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
INFO 09-23 15:04:45 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-23 15:04:45 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
INFO 09-23 15:04:45 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
WARNING 09-23 15:04:45 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 15:04:45 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
INFO 09-23 15:04:46 [model.py:684] Resolved architecture: MiniCPMO45OmniForConditionalGeneration
|
||||
INFO 09-23 15:04:46 [model.py:2021] Using max model len 512
|
||||
WARNING 09-23 15:04:46 [model.py:981] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False.
|
||||
INFO 09-23 15:04:46 [model.py:2021] Using max model len 512
|
||||
INFO 09-23 15:04:46 [scheduler.py:277] Chunked prefill is enabled with max_num_batched_tokens=512.
|
||||
INFO 09-23 15:04:46 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
WARNING 09-23 15:04:46 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 15:04:46 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
INFO 09-23 15:04:46 [model.py:684] Resolved architecture: MiniCPMO45Code2Wav
|
||||
INFO 09-23 15:04:46 [model.py:2021] Using max model len 4096
|
||||
WARNING 09-23 15:04:46 [arg_utils.py:2780] This model does not officially support disabling chunked prefill. Disabling this manually may cause the engine to crash or produce incorrect outputs.
|
||||
INFO 09-23 15:04:46 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-23 15:04:46 [stage_runtime.py:853] [stage_init] Stage-0 set runtime devices: 0
|
||||
INFO 09-23 15:04:52 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 15:04:52 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 15:04:52 [patch.py:583] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) INFO 09-23 15:04:53 [core.py:123] Initializing a V1 LLM engine (v0.29.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=2048, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 0, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) INFO 09-23 15:04:54 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) INFO 09-23 15:04:54 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_b4a47065e7bb48f0921b8c642f25a6a3 backend=nccl
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) INFO 09-23 15:04:54 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) WARNING 09-23 15:04:55 [gpu_ar_worker.py:120] OMNI GPUARWorker forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) INFO 09-23 15:04:55 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) INFO 09-23 15:04:55 [base.py:138] Offloader set to UVAOffloader
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) WARNING 09-23 15:04:55 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) WARNING 09-23 15:04:55 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) INFO 09-23 15:04:55 [gpu_model_runner.py:5379] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5...
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) WARNING 09-23 15:04:55 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) WARNING 09-23 15:04:55 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) INFO 09-23 15:04:55 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) INFO 09-23 15:04:55 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) INFO 09-23 15:04:55 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) INFO 09-23 15:04:55 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) INFO 09-23 15:04:55 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) INFO 09-23 15:04:56 [cuda.py:492] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) INFO 09-23 15:04:56 [flash_attn.py:897] Using FlashAttention version 2
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) INFO 09-23 15:05:11 [uva.py:65] Total CPU offloaded parameters: 5.03
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) INFO 09-23 15:05:12 [weight_utils.py:863] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 20.96 GiB.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) INFO 09-23 15:05:12 [weight_utils.py:886] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) ␍Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:00<00:00, 89.79it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) INFO 09-23 15:05:30 [default_loader.py:430] Loading weights took 18.75 seconds
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) INFO 09-23 15:05:31 [gpu_model_runner.py:5471] Model loading took 10.84 GiB memory and 35.121139 seconds
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) INFO 09-23 15:05:32 [utils.py:306] Using LBNHC KV cache layout.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) INFO 09-23 15:05:45 [base.py:148] Available KV cache memory: 2.72 GiB (device-level profiling)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) INFO 09-23 15:05:45 [kv_cache_utils.py:2032] GPU KV cache size: 19,776 tokens, Maximum concurrency for 2,048 tokens per request: 9.66x
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) WARNING 09-23 15:05:45 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) INFO 09-23 15:05:45 [kernel_warmup.py:124] JIT kernel warmup starting.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) ␍JIT kernel warmup (1 compile keys): 0%| | 0/1 [00:00<?, ?kernel/s]␍JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:01<00:00, 1.76s/kernel, ComputeSlotMappingKernel (1 keys)]␍JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:01<00:00, 1.76s/kernel, ComputeSlotMappingKernel (1 keys)]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) INFO 09-23 15:05:47 [kernel_warmup.py:134] JIT kernel warmup finished in 1.76s.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) WARNING 09-23 15:05:48 [import_utils.py:408] Module vllm.third_party.deep_gemm was found but failed to import
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) WARNING 09-23 15:05:48 [import_utils.py:408] Traceback (most recent call last):
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) WARNING 09-23 15:05:48 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/utils/import_utils.py", line 406, in _has_module
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) WARNING 09-23 15:05:48 [import_utils.py:408] importlib.import_module(module_name)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) WARNING 09-23 15:05:48 [import_utils.py:408] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/importlib/__init__.py", line 90, in import_module
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) WARNING 09-23 15:05:48 [import_utils.py:408] return _bootstrap._gcd_import(name[level:], package, level)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) WARNING 09-23 15:05:48 [import_utils.py:408] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) WARNING 09-23 15:05:48 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1387, in _gcd_import
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) WARNING 09-23 15:05:48 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1360, in _find_and_load
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) WARNING 09-23 15:05:48 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1331, in _find_and_load_unlocked
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) WARNING 09-23 15:05:48 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 935, in _load_unlocked
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) WARNING 09-23 15:05:48 [import_utils.py:408] File "<frozen importlib._bootstrap_external>", line 999, in exec_module
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) WARNING 09-23 15:05:48 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 488, in _call_with_frames_removed
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) WARNING 09-23 15:05:48 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/third_party/deep_gemm/__init__.py", line 126, in <module>
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) WARNING 09-23 15:05:48 [import_utils.py:408] _find_cuda_home() # CUDA home
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) WARNING 09-23 15:05:48 [import_utils.py:408] ^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) WARNING 09-23 15:05:48 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/third_party/deep_gemm/__init__.py", line 120, in _find_cuda_home
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) WARNING 09-23 15:05:48 [import_utils.py:408] assert cuda_home is not None
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) WARNING 09-23 15:05:48 [import_utils.py:408] ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) WARNING 09-23 15:05:48 [import_utils.py:408] AssertionError
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) INFO 09-23 15:05:48 [gpu_worker.py:860] Free memory on device (22.58/23.89 GiB) on startup. Desired GPU memory utilization is (0.58, 13.86 GiB). Actual usage is 11.01 GiB for consumed memory (weights + non-torch), 0.22 GiB for peak activation, and 0.0 GiB for CUDAGraph memory. Replace gpu_memory_utilization config with `--kv-cache-memory=2667877499` (2.48 GiB) to fit into requested memory, or `--kv-cache-memory=12032409600` (11.21 GiB) to fully utilize gpu memory. Current kv cache memory in use is 2.72 GiB.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) INFO 09-23 15:05:49 [jit_monitor.py:85] Kernel JIT monitor activated; monitored JIT compilations during inference will use mode=warn.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) INFO 09-23 15:05:50 [torch_utils.py:277] Reducing Torch threads from 12 to 1 for serving. Set OMP_NUM_THREADS in the external environment to override.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) INFO 09-23 15:05:50 [core.py:368] init engine (profile, create kv cache, warmup model) took 18.03 s
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) WARNING 09-23 15:05:50 [scheduler.py:213] Using custom scheduler class vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) WARNING 09-23 15:05:50 [omni_scheduler_mixin.py:118] OMNI scheduler forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) INFO 09-23 15:05:50 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163945) INFO 09-23 15:05:50 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-23 15:05:50 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed
|
||||
INFO 09-23 15:05:50 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-0 [rep-0] initializing EngineCore
|
||||
INFO 09-23 15:05:50 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-0 [rep-0] EngineCore running
|
||||
INFO 09-23 15:05:50 [stage_runtime.py:958] [StageRuntime] Stage 0 initialized
|
||||
INFO 09-23 15:05:50 [stage_runtime.py:853] [stage_init] Stage-1 set runtime devices: 0
|
||||
INFO 09-23 15:06:11 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 15:06:11 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 15:06:11 [patch.py:583] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) INFO 09-23 15:06:13 [core.py:123] Initializing a V1 LLM engine (v0.29.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=512, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [512], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 0, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) INFO 09-23 15:06:16 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_30ae85cbfd864d449a54426162c1ef3a backend=nccl
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) INFO 09-23 15:06:16 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) WARNING 09-23 15:06:17 [gpu_ar_worker.py:120] OMNI GPUARWorker forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) INFO 09-23 15:06:17 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) [transformers] `MiniCPMOProcessor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) [transformers] Requested torchvision backend is not available. Falling back to pil backend.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) [transformers] `MiniCPMOProcessor` defines `feature_extractor_class = 'WhisperFeatureExtractor'`, which is deprecated. Register the correct mapping in `AutoFeatureExtractor` instead.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) [transformers] Image processor MiniCPMVImageProcessor: kwargs ['max_slice_nums'] were applied for backward compatibility. To avoid this warning, add them to valid_kwargs: create a custom TypedDict extending ImagesKwargs with these keys and set it as the `valid_kwargs` class attribute.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) INFO 09-23 15:06:33 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) WARNING 09-23 15:06:33 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) WARNING 09-23 15:06:33 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) INFO 09-23 15:06:33 [gpu_model_runner.py:5379] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5...
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) WARNING 09-23 15:06:34 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) WARNING 09-23 15:06:34 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) INFO 09-23 15:06:34 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) INFO 09-23 15:06:34 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) INFO 09-23 15:06:34 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) INFO 09-23 15:06:34 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) INFO 09-23 15:06:35 [cuda.py:492] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) INFO 09-23 15:06:35 [flash_attn.py:897] Using FlashAttention version 2
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) INFO 09-23 15:06:35 [weight_utils.py:863] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 19.13 GiB.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) INFO 09-23 15:06:35 [weight_utils.py:893] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre) and the checkpoint size (17.46 GiB) exceeds 90% of available RAM (19.13 GiB).
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) ␍Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) ␍Loading safetensors checkpoint shards: 25% Completed | 1/4 [00:00<00:00, 3.34it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) ␍Loading safetensors checkpoint shards: 75% Completed | 3/4 [00:00<00:00, 5.65it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:02<00:00, 1.55it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:02<00:00, 1.91it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) INFO 09-23 15:06:40 [default_loader.py:430] Loading weights took 5.22 seconds
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) INFO 09-23 15:06:42 [gpu_model_runner.py:5471] Model loading took 0.65 GiB memory and 6.501276 seconds
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) INFO 09-23 15:06:42 [utils.py:306] Using LBNHC KV cache layout.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) INFO 09-23 15:06:42 [gpu_model_runner.py:6519] Skipping memory profiling for multimodal encoder and encoder cache.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) INFO 09-23 15:06:46 [kv_cache_utils.py:2032] GPU KV cache size: 4,368 tokens, Maximum concurrency for 512 tokens per request: 8.53x
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) WARNING 09-23 15:06:46 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) INFO 09-23 15:06:46 [kernel_warmup.py:124] JIT kernel warmup starting.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) ␍JIT kernel warmup (1 compile keys): 0%| | 0/1 [00:00<?, ?kernel/s]␍JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 1.72kernel/s, ComputeSlotMappingKernel (1 keys)]␍JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 1.72kernel/s, ComputeSlotMappingKernel (1 keys)]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) INFO 09-23 15:06:46 [kernel_warmup.py:134] JIT kernel warmup finished in 0.58s.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) WARNING 09-23 15:06:47 [import_utils.py:408] Module vllm.third_party.deep_gemm was found but failed to import
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) WARNING 09-23 15:06:47 [import_utils.py:408] Traceback (most recent call last):
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) WARNING 09-23 15:06:47 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/utils/import_utils.py", line 406, in _has_module
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) WARNING 09-23 15:06:47 [import_utils.py:408] importlib.import_module(module_name)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) WARNING 09-23 15:06:47 [import_utils.py:408] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/importlib/__init__.py", line 90, in import_module
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) WARNING 09-23 15:06:47 [import_utils.py:408] return _bootstrap._gcd_import(name[level:], package, level)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) WARNING 09-23 15:06:47 [import_utils.py:408] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) WARNING 09-23 15:06:47 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1387, in _gcd_import
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) WARNING 09-23 15:06:47 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1360, in _find_and_load
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) WARNING 09-23 15:06:47 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1331, in _find_and_load_unlocked
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) WARNING 09-23 15:06:47 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 935, in _load_unlocked
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) WARNING 09-23 15:06:47 [import_utils.py:408] File "<frozen importlib._bootstrap_external>", line 999, in exec_module
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) WARNING 09-23 15:06:47 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 488, in _call_with_frames_removed
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) WARNING 09-23 15:06:47 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/third_party/deep_gemm/__init__.py", line 126, in <module>
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) WARNING 09-23 15:06:47 [import_utils.py:408] _find_cuda_home() # CUDA home
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) WARNING 09-23 15:06:47 [import_utils.py:408] ^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) WARNING 09-23 15:06:47 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/third_party/deep_gemm/__init__.py", line 120, in _find_cuda_home
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) WARNING 09-23 15:06:47 [import_utils.py:408] assert cuda_home is not None
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) WARNING 09-23 15:06:47 [import_utils.py:408] ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) WARNING 09-23 15:06:47 [import_utils.py:408] AssertionError
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) INFO 09-23 15:06:48 [jit_monitor.py:85] Kernel JIT monitor activated; monitored JIT compilations during inference will use mode=warn.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) INFO 09-23 15:06:49 [torch_utils.py:277] Reducing Torch threads from 12 to 1 for serving. Set OMP_NUM_THREADS in the external environment to override.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) INFO 09-23 15:06:49 [core.py:368] init engine (profile, create kv cache, warmup model) took 6.53 s
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) WARNING 09-23 15:06:49 [scheduler.py:213] Using custom scheduler class vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) WARNING 09-23 15:06:49 [omni_scheduler_mixin.py:118] OMNI scheduler forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) INFO 09-23 15:06:49 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4164226) INFO 09-23 15:06:49 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-23 15:06:49 [stage_runtime.py:943] [StageRuntime] Stage 1 engine startup completed
|
||||
INFO 09-23 15:06:49 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-1 [rep-0] initializing EngineCore
|
||||
INFO 09-23 15:06:49 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-1 [rep-0] EngineCore running
|
||||
INFO 09-23 15:06:49 [stage_runtime.py:958] [StageRuntime] Stage 1 initialized
|
||||
INFO 09-23 15:06:49 [stage_runtime.py:853] [stage_init] Stage-2 set runtime devices: 0
|
||||
INFO 09-23 15:07:00 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 15:07:00 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 15:07:00 [patch.py:583] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) INFO 09-23 15:07:01 [core.py:123] Initializing a V1 LLM engine (v0.29.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=4096, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=False, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [4096], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 0, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) INFO 09-23 15:07:02 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_1e3158f14a274b6b81deeec51b9a4da1 backend=nccl
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) INFO 09-23 15:07:02 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) WARNING 09-23 15:07:03 [gpu_generation_worker.py:103] OMNI GPUGenerationWorker forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) INFO 09-23 15:07:03 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) INFO 09-23 15:07:03 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) WARNING 09-23 15:07:03 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) WARNING 09-23 15:07:03 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) INFO 09-23 15:07:03 [gpu_model_runner.py:5379] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5...
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) INFO 09-23 15:07:03 [weight_utils.py:863] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 17.09 GiB.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) INFO 09-23 15:07:03 [weight_utils.py:893] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre) and the checkpoint size (17.46 GiB) exceeds 90% of available RAM (17.09 GiB).
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) ␍Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:00<00:00, 77.13it/s]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) INFO 09-23 15:07:04 [step_audio2_token2wav.py:163] Loading Token2Wav models from: /home/zhout/project/edge_infer/models/MiniCPM-o-4_5/assets/token2wav
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) INFO 09-23 15:07:28 [step_audio2_token2wav.py:201] Token2Wav models loaded successfully
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) INFO 09-23 15:07:30 [batched_token2wav.py:227] Enabling HiFT CUDA Graph with batch sizes [1]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) INFO 09-23 15:07:30 [cuda_graph_wrapper.py:98] Captured HiFT CUDA Graph for shape (1, 50, 0)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) INFO 09-23 15:07:31 [cuda_graph_wrapper.py:98] Captured HiFT CUDA Graph for shape (1, 58, 3840)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) INFO 09-23 15:07:31 [batched_token2wav.py:235] HiFT CUDA Graph captured successfully
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) INFO 09-23 15:07:31 [batched_token2wav.py:246] CFM CUDA Graph enabled (max_graphs=32)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) INFO 09-23 15:07:31 [default_loader.py:430] Loading weights took 27.47 seconds
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) INFO 09-23 15:07:32 [gpu_model_runner.py:5471] Model loading took 3.36 GiB memory and 27.490452 seconds
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) INFO 09-23 15:07:33 [cuda.py:492] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) INFO 09-23 15:07:33 [utils.py:306] Using LBNHC KV cache layout.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) WARNING 09-23 15:07:33 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) INFO 09-23 15:07:33 [kernel_warmup.py:124] JIT kernel warmup starting.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) ␍JIT kernel warmup (1 compile keys): 0%| | 0/1 [00:00<?, ?kernel/s]␍JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 272.34kernel/s, ComputeSlotMappingKernel (1 keys)]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) INFO 09-23 15:07:33 [kernel_warmup.py:134] JIT kernel warmup finished in 0.00s.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) WARNING 09-23 15:07:33 [import_utils.py:408] Module vllm.third_party.deep_gemm was found but failed to import
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) WARNING 09-23 15:07:33 [import_utils.py:408] Traceback (most recent call last):
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) WARNING 09-23 15:07:33 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/utils/import_utils.py", line 406, in _has_module
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) WARNING 09-23 15:07:33 [import_utils.py:408] importlib.import_module(module_name)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) WARNING 09-23 15:07:33 [import_utils.py:408] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/importlib/__init__.py", line 90, in import_module
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) WARNING 09-23 15:07:33 [import_utils.py:408] return _bootstrap._gcd_import(name[level:], package, level)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) WARNING 09-23 15:07:33 [import_utils.py:408] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) WARNING 09-23 15:07:33 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1387, in _gcd_import
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) WARNING 09-23 15:07:33 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1360, in _find_and_load
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) WARNING 09-23 15:07:33 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1331, in _find_and_load_unlocked
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) WARNING 09-23 15:07:33 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 935, in _load_unlocked
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) WARNING 09-23 15:07:33 [import_utils.py:408] File "<frozen importlib._bootstrap_external>", line 999, in exec_module
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) WARNING 09-23 15:07:33 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 488, in _call_with_frames_removed
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) WARNING 09-23 15:07:33 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/third_party/deep_gemm/__init__.py", line 126, in <module>
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) WARNING 09-23 15:07:33 [import_utils.py:408] _find_cuda_home() # CUDA home
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) WARNING 09-23 15:07:33 [import_utils.py:408] ^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) WARNING 09-23 15:07:33 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/third_party/deep_gemm/__init__.py", line 120, in _find_cuda_home
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) WARNING 09-23 15:07:33 [import_utils.py:408] assert cuda_home is not None
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) WARNING 09-23 15:07:33 [import_utils.py:408] ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) WARNING 09-23 15:07:33 [import_utils.py:408] AssertionError
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) WARNING 09-23 15:07:33 [gpu_generation_model_runner.py:558] Dummy sampler run is not implemented for generation model
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) INFO 09-23 15:07:34 [jit_monitor.py:85] Kernel JIT monitor activated; monitored JIT compilations during inference will use mode=warn.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) INFO 09-23 15:07:35 [torch_utils.py:277] Reducing Torch threads from 12 to 1 for serving. Set OMP_NUM_THREADS in the external environment to override.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) INFO 09-23 15:07:35 [core.py:368] init engine (profile, create kv cache, warmup model) took 2.76 s
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) WARNING 09-23 15:07:35 [scheduler.py:213] Using custom scheduler class vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) WARNING 09-23 15:07:35 [omni_scheduler_mixin.py:118] OMNI scheduler forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) INFO 09-23 15:07:35 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) WARNING 09-23 15:07:35 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) WARNING 09-23 15:07:35 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) INFO 09-23 15:07:35 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) INFO 09-23 15:07:35 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
INFO 09-23 15:07:35 [stage_runtime.py:943] [StageRuntime] Stage 2 engine startup completed
|
||||
INFO 09-23 15:07:35 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-2 [rep-0] initializing EngineCore
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) INFO 09-23 15:07:35 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
INFO 09-23 15:07:35 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-2 [rep-0] EngineCore running
|
||||
INFO 09-23 15:07:35 [stage_runtime.py:958] [StageRuntime] Stage 2 initialized
|
||||
INFO 09-23 15:07:37 [orchestrator.py:604] [Orchestrator] Starting event loop
|
||||
INFO 09-23 15:07:37 [async_omni_engine.py:323] [AsyncOmniEngine] Orchestrator ready with 3 stages
|
||||
INFO 09-23 15:07:37 [omni_base.py:235] [Omni] AsyncOmniEngine initialized in 182.47 seconds
|
||||
INFO 09-23 15:07:37 [omni_base.py:255] [Omni] Initialized with 3 stages for model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5
|
||||
WARNING 09-23 15:07:37 [input_processor.py:315] Passing raw prompts to InputProcessor is deprecated and will be removed in the future. You should instead pass the outputs of Renderer.render_cmpl() or Renderer.render_chat().
|
||||
␍Processed prompts: 0%| | 0/1 [00:00<?, ?it/s](StageEngineCoreProc_stage0_replica0 pid=4163945) WARNING 09-23 15:07:38 [gpu_ar_model_runner.py:1464] prefer_model_sampler model MiniCPMO45OmniForConditionalGeneration returned None from sample(); falling back to the default sampler.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) INFO 09-23 15:07:55 [cuda_graph_wrapper.py:300] Captured CFM CUDA Graph for shape ('estimator_step', ((2, 320, 302), 'torch.float32', 'cuda:0'), ((2, 1, 512), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 0, 128), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 302, 128), 'torch.float32', 'cuda:0')) (cache=1/32, stats={'calls': 1, 'hits': 0, 'captures': 1, 'flushes': 0, 'eager': 0, 'cache_size': 0}) [allocated 3633.9 MiB (+32.2), reserved 5464.0 MiB (+22.0)]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) INFO 09-23 15:07:55 [cuda_graph_wrapper.py:300] Captured CFM CUDA Graph for shape ('estimator_step', ((2, 320, 50), 'torch.float32', 'cuda:0'), ((2, 1, 512), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 302, 128), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 352, 128), 'torch.float32', 'cuda:0')) (cache=2/32, stats={'calls': 11, 'hits': 9, 'captures': 2, 'flushes': 0, 'eager': 0, 'cache_size': 1}) [allocated 4539.7 MiB (+32.0), reserved 6378.0 MiB (+28.0)]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) INFO 09-23 15:07:56 [cuda_graph_wrapper.py:300] Captured CFM CUDA Graph for shape ('estimator_step', ((2, 320, 50), 'torch.float32', 'cuda:0'), ((2, 1, 512), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 352, 128), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 402, 128), 'torch.float32', 'cuda:0')) (cache=3/32, stats={'calls': 21, 'hits': 18, 'captures': 3, 'flushes': 0, 'eager': 0, 'cache_size': 2}) [allocated 4803.9 MiB (+32.0), reserved 6956.0 MiB (+-352.0)]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) INFO 09-23 15:07:57 [cuda_graph_wrapper.py:300] Captured CFM CUDA Graph for shape ('estimator_step', ((2, 320, 50), 'torch.float32', 'cuda:0'), ((2, 1, 512), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 402, 128), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 452, 128), 'torch.float32', 'cuda:0')) (cache=4/32, stats={'calls': 31, 'hits': 27, 'captures': 4, 'flushes': 0, 'eager': 0, 'cache_size': 3}) [allocated 5079.1 MiB (+32.0), reserved 7096.0 MiB (+-876.0)]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) INFO 09-23 15:07:58 [cuda_graph_wrapper.py:300] Captured CFM CUDA Graph for shape ('estimator_step', ((2, 320, 10), 'torch.float32', 'cuda:0'), ((2, 1, 512), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 402, 128), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 412, 128), 'torch.float32', 'cuda:0')) (cache=5/32, stats={'calls': 61, 'hits': 56, 'captures': 5, 'flushes': 0, 'eager': 0, 'cache_size': 4}) [allocated 5206.6 MiB (+32.0), reserved 7190.0 MiB (+-1550.0)]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) INFO 09-23 15:07:58 [cuda_graph_wrapper.py:120] Lazily capturing HiFT CUDA Graph for shape (1, 18, 3840)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) WARNING 09-23 15:07:58 [jit_monitor.py:141] Triton kernel JIT compilation during inference: SnakeBeta._init_triton.<locals>._kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4164450) INFO 09-23 15:07:58 [cuda_graph_wrapper.py:98] Captured HiFT CUDA Graph for shape (1, 18, 3840)
|
||||
␍Processed prompts: 100%|██████████| 1/1 [00:20<00:00, 20.53s/it]␍Processed prompts: 100%|██████████| 1/1 [00:20<00:00, 20.53s/it]
|
||||
query type: text
|
||||
use_tts (tts_bos): True
|
||||
Request ID: 0_2c17288b-c543-4aaa-9a6a-e661ab14c97b, Text saved to /home/zhout/project/edge_infer/analysis/experiments/e2e_recovery_20260923/mini_output_fixed_e2e/0_2c17288b-c543-4aaa-9a6a-e661ab14c97b.txt
|
||||
Request ID: 0_2c17288b-c543-4aaa-9a6a-e661ab14c97b, Saved audio to /home/zhout/project/edge_infer/analysis/experiments/e2e_recovery_20260923/mini_output_fixed_e2e/output_0_2c17288b-c543-4aaa-9a6a-e661ab14c97b.wav
|
||||
INFO 09-23 15:07:58 [omni_base.py:781] [Omni] Shutting down
|
||||
INFO 09-23 15:07:58 [async_omni_engine.py:1807] [AsyncOmniEngine] Shutting down Orchestrator
|
||||
INFO 09-23 15:07:58 [orchestrator.py:705] [Orchestrator] Received shutdown signal
|
||||
INFO 09-23 15:07:58 [orchestrator.py:3092] [Orchestrator] Shutting down all 3 client(s)
|
||||
INFO 09-23 15:07:58 [core_client.py:689] [shutdown] MPClient: start timeout=default
|
||||
INFO 09-23 15:07:58 [core_client.py:691] [shutdown] MPClient: stopping engine manager
|
||||
INFO 09-23 15:07:58 [utils.py:620] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage0_replica0
|
||||
WARNING 09-23 15:08:03 [utils.py:640] [shutdown] Process manager: force killing remaining processes count=1
|
||||
WARNING 09-23 15:08:03 [utils.py:645] [shutdown] Process manager: force killing remaining process StageEngineCoreProc_stage0_replica0 pid 4163945
|
||||
INFO 09-23 15:08:03 [core_client.py:693] [shutdown] MPClient: engine manager stopped
|
||||
INFO 09-23 15:08:03 [core_client.py:694] [shutdown] MPClient: cleaning up background resources
|
||||
INFO 09-23 15:08:03 [core_client.py:696] [shutdown] MPClient: complete
|
||||
INFO 09-23 15:08:03 [stage_pool.py:1354] [StagePool] Stage 0 replica 0 shut down
|
||||
INFO 09-23 15:08:03 [core_client.py:689] [shutdown] MPClient: start timeout=default
|
||||
INFO 09-23 15:08:03 [utils.py:620] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage1_replica0
|
||||
INFO 09-23 15:08:08 [stage_pool.py:1354] [StagePool] Stage 1 replica 0 shut down
|
||||
INFO 09-23 15:08:08 [core_client.py:689] [shutdown] MPClient: start timeout=default
|
||||
INFO 09-23 15:08:08 [utils.py:620] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage2_replica0
|
||||
INFO 09-23 15:08:13 [stage_pool.py:1354] [StagePool] Stage 2 replica 0 shut down
|
||||
/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/multiprocessing/resource_tracker.py:279: UserWarning: resource_tracker: There appear to be 1 leaked shared_memory objects to clean up at shutdown
|
||||
warnings.warn('resource_tracker: There appear to be %d '
|
||||
+196
@@ -0,0 +1,196 @@
|
||||
INFO 09-23 14:41:50 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 14:41:51 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 14:41:51 [patch.py:583] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
|
||||
====================
|
||||
vllm version: 0.29.0
|
||||
====================
|
||||
INFO 09-23 14:41:52 [omni_base.py:216] [Omni] Initializing with model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5
|
||||
INFO 09-23 14:41:52 [async_omni_engine.py:172] [AsyncOmniEngine] Initializing with model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5
|
||||
INFO 09-23 14:41:52 [async_omni_engine.py:290] [AsyncOmniEngine] Launching Orchestrator thread with 3 stages
|
||||
INFO 09-23 14:41:52 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
INFO 09-23 14:41:52 [initialization.py:356] Loaded OmniTransferConfig with 0 connector configurations
|
||||
WARNING 09-23 14:41:52 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 14:41:52 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
WARNING 09-23 14:41:52 [envs.py:2248] Unknown vLLM environment variable detected: VLLM_OMNI_MINICPMO_LOGITS_PROBE
|
||||
INFO 09-23 14:42:01 [model.py:684] Resolved architecture: MiniCPMO45OmniForConditionalGeneration
|
||||
INFO 09-23 14:42:01 [model.py:2021] Using max model len 2048
|
||||
WARNING 09-23 14:42:01 [model.py:981] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False.
|
||||
INFO 09-23 14:42:02 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode.
|
||||
INFO 09-23 14:42:02 [model.py:887] Disabled mm_prefix attention mode because multimodal inputs are configuration-disabled. Attention backends without mm_prefix support may now be selected.
|
||||
INFO 09-23 14:42:02 [scheduler.py:277] Chunked prefill is enabled with max_num_batched_tokens=2048.
|
||||
WARNING 09-23 14:42:02 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
WARNING 09-23 14:42:02 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
INFO 09-23 14:42:02 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-23 14:42:02 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
INFO 09-23 14:42:02 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
WARNING 09-23 14:42:02 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 14:42:02 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
WARNING 09-23 14:42:02 [envs.py:2248] Unknown vLLM environment variable detected: VLLM_OMNI_MINICPMO_LOGITS_PROBE
|
||||
INFO 09-23 14:42:02 [model.py:684] Resolved architecture: MiniCPMO45OmniForConditionalGeneration
|
||||
INFO 09-23 14:42:02 [model.py:2021] Using max model len 40960
|
||||
WARNING 09-23 14:42:02 [model.py:981] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False.
|
||||
INFO 09-23 14:42:02 [model.py:2021] Using max model len 4096
|
||||
INFO 09-23 14:42:02 [scheduler.py:277] Chunked prefill is enabled with max_num_batched_tokens=8192.
|
||||
INFO 09-23 14:42:02 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
WARNING 09-23 14:42:03 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 14:42:03 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
WARNING 09-23 14:42:03 [envs.py:2248] Unknown vLLM environment variable detected: VLLM_OMNI_MINICPMO_LOGITS_PROBE
|
||||
INFO 09-23 14:42:03 [model.py:684] Resolved architecture: MiniCPMO45Code2Wav
|
||||
INFO 09-23 14:42:03 [model.py:2021] Using max model len 40960
|
||||
INFO 09-23 14:42:03 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
WARNING 09-23 14:42:03 [stage_runtime.py:778] [stage_init] Stage-1 replica 0 physical devices None are not integer GPU ids; all serial LLM replicas will share one init group
|
||||
WARNING 09-23 14:42:03 [stage_runtime.py:778] [stage_init] Stage-2 replica 0 physical devices None are not integer GPU ids; all serial LLM replicas will share one init group
|
||||
INFO 09-23 14:42:03 [stage_runtime.py:853] [stage_init] Stage-0 set runtime devices: 0
|
||||
INFO 09-23 14:42:09 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 14:42:09 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 14:42:09 [patch.py:583] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) INFO 09-23 14:42:10 [core.py:123] Initializing a V1 LLM engine (v0.29.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=2048, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 0, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) INFO 09-23 14:42:11 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) INFO 09-23 14:42:11 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_9682e9c24f084fc882f394d72103e18a backend=nccl
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) INFO 09-23 14:42:11 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) WARNING 09-23 14:42:12 [gpu_ar_worker.py:120] OMNI GPUARWorker forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) INFO 09-23 14:42:12 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) WARNING 09-23 14:42:12 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) WARNING 09-23 14:42:12 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) INFO 09-23 14:42:12 [gpu_model_runner.py:5379] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5...
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) WARNING 09-23 14:42:12 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) WARNING 09-23 14:42:12 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) INFO 09-23 14:42:12 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) INFO 09-23 14:42:12 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) INFO 09-23 14:42:12 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) INFO 09-23 14:42:12 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) INFO 09-23 14:42:12 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) INFO 09-23 14:42:13 [cuda.py:492] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) INFO 09-23 14:42:13 [flash_attn.py:897] Using FlashAttention version 2
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) INFO 09-23 14:42:14 [weight_utils.py:863] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 27.47 GiB.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) INFO 09-23 14:42:14 [weight_utils.py:886] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) ␍Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:00<00:00, 22.99it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:00<00:00, 22.96it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) INFO 09-23 14:42:22 [default_loader.py:430] Loading weights took 8.72 seconds
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) INFO 09-23 14:42:23 [gpu_model_runner.py:5471] Model loading took 15.88 GiB memory and 10.051277 seconds
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) INFO 09-23 14:42:24 [utils.py:306] Using LBNHC KV cache layout.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) WARNING 09-23 14:42:25 [minicpmo_4_5_omni.py:648] MiniCPM-o first-token logits: finite=True top_ids=[625, 9342, 16400, 8983, 8645] top_values=[8.6875, 8.4375, 8.3125, 8.3125, 8.3125] head_abs_mean=0.0286865234375
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) INFO 09-23 14:42:27 [base.py:148] Available KV cache memory: 3.15 GiB (device-level profiling)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) INFO 09-23 14:42:27 [kv_cache_utils.py:2032] GPU KV cache size: 22,912 tokens, Maximum concurrency for 2,048 tokens per request: 11.19x
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) WARNING 09-23 14:42:27 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) INFO 09-23 14:42:27 [kernel_warmup.py:124] JIT kernel warmup starting.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) ␍JIT kernel warmup (1 compile keys): 0%| | 0/1 [00:00<?, ?kernel/s]␍JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 3.85kernel/s, ComputeSlotMappingKernel (1 keys)]␍JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 3.85kernel/s, ComputeSlotMappingKernel (1 keys)]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) INFO 09-23 14:42:27 [kernel_warmup.py:134] JIT kernel warmup finished in 0.26s.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) WARNING 09-23 14:42:27 [import_utils.py:408] Module vllm.third_party.deep_gemm was found but failed to import
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) WARNING 09-23 14:42:27 [import_utils.py:408] Traceback (most recent call last):
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) WARNING 09-23 14:42:27 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/utils/import_utils.py", line 406, in _has_module
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) WARNING 09-23 14:42:27 [import_utils.py:408] importlib.import_module(module_name)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) WARNING 09-23 14:42:27 [import_utils.py:408] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/importlib/__init__.py", line 90, in import_module
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) WARNING 09-23 14:42:27 [import_utils.py:408] return _bootstrap._gcd_import(name[level:], package, level)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) WARNING 09-23 14:42:27 [import_utils.py:408] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) WARNING 09-23 14:42:27 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1387, in _gcd_import
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) WARNING 09-23 14:42:27 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1360, in _find_and_load
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) WARNING 09-23 14:42:27 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1331, in _find_and_load_unlocked
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) WARNING 09-23 14:42:27 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 935, in _load_unlocked
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) WARNING 09-23 14:42:27 [import_utils.py:408] File "<frozen importlib._bootstrap_external>", line 999, in exec_module
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) WARNING 09-23 14:42:27 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 488, in _call_with_frames_removed
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) WARNING 09-23 14:42:27 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/third_party/deep_gemm/__init__.py", line 126, in <module>
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) WARNING 09-23 14:42:27 [import_utils.py:408] _find_cuda_home() # CUDA home
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) WARNING 09-23 14:42:27 [import_utils.py:408] ^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) WARNING 09-23 14:42:27 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/third_party/deep_gemm/__init__.py", line 120, in _find_cuda_home
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) WARNING 09-23 14:42:27 [import_utils.py:408] assert cuda_home is not None
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) WARNING 09-23 14:42:27 [import_utils.py:408] ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) WARNING 09-23 14:42:27 [import_utils.py:408] AssertionError
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) INFO 09-23 14:42:27 [gpu_worker.py:860] Free memory on device (22.58/23.89 GiB) on startup. Desired GPU memory utilization is (0.85, 20.31 GiB). Actual usage is 16.04 GiB for consumed memory (weights + non-torch), 1.21 GiB for peak activation, and 0.0 GiB for CUDAGraph memory. Replace gpu_memory_utilization config with `--kv-cache-memory=3131941786` (2.92 GiB) to fit into requested memory, or `--kv-cache-memory=5570742784` (5.19 GiB) to fully utilize gpu memory. Current kv cache memory in use is 3.15 GiB.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) INFO 09-23 14:42:28 [jit_monitor.py:85] Kernel JIT monitor activated; monitored JIT compilations during inference will use mode=warn.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) INFO 09-23 14:42:28 [torch_utils.py:277] Reducing Torch threads from 12 to 1 for serving. Set OMP_NUM_THREADS in the external environment to override.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) INFO 09-23 14:42:28 [core.py:368] init engine (profile, create kv cache, warmup model) took 4.52 s
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) WARNING 09-23 14:42:28 [scheduler.py:213] Using custom scheduler class vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) WARNING 09-23 14:42:28 [omni_scheduler_mixin.py:118] OMNI scheduler forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) INFO 09-23 14:42:28 [chunk_transfer_adapter.py:142] Bounded active-stream window enabled: K=1. Multi-replica deployments require sticky per-stream routing across Stage 1 replicas (each replica owns an independent active-set; without sticky routing, a stream can be active on one replica and non-active on another and both will race to evict it).
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) INFO 09-23 14:42:28 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4158127) INFO 09-23 14:42:28 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-23 14:42:28 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed
|
||||
INFO 09-23 14:42:28 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-0 [rep-0] initializing EngineCore
|
||||
INFO 09-23 14:42:29 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-0 [rep-0] EngineCore running
|
||||
INFO 09-23 14:42:29 [stage_runtime.py:958] [StageRuntime] Stage 0 initialized
|
||||
INFO 09-23 14:42:36 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 14:42:36 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 14:42:36 [patch.py:583] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) INFO 09-23 14:42:37 [core.py:123] Initializing a V1 LLM engine (v0.29.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=4096, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=False, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.VLLM_COMPILE: 3>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['none'], 'ir_enable_torch_wrap': True, 'splitting_ops': ['vllm::unified_attention_with_output', 'vllm::unified_mla_attention_with_output', 'vllm::mamba_mixer2', 'vllm::mamba_mixer', 'vllm::short_conv', 'vllm::qwen4_exp_compute_ple_ngram_ids', 'vllm::qwen4_exp_ple_short_conv', 'vllm::qwen4_exp_qsa_with_output', 'vllm::linear_attention', 'vllm::qwen_gdn_attention_core', 'vllm::qwen_gdn_attention_core_fused_norm_packed', 'vllm::gdn_attention_core_xpu', 'vllm::olmo_hybrid_gdn_full_forward', 'vllm::sparse_attn_indexer', 'vllm::rocm_aiter_sparse_attn_indexer', 'vllm::deepseek_v4_attention', 'vllm::hpc_rope_norm_forward', 'vllm::unified_kv_cache_update', 'vllm::unified_mla_kv_cache_update'], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [8192], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.FULL_AND_PIECEWISE: (2, 1)>, 'cudagraph_num_of_warmups': 1, 'cudagraph_capture_sizes': [1, 2, 4, 8, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': False, 'fuse_act_quant': False, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 512, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) INFO 09-23 14:42:38 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_4a42f865c6a340b08adfa60702af11a6 backend=nccl
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) INFO 09-23 14:42:38 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) WARNING 09-23 14:42:38 [memory_utils.py:39] Free memory on device cuda:0 (21.21/23.89 GiB) on startup is less than desired GPU memory utilization (0.92, 21.98 GiB). Capping requested memory to available free memory (21.21 GiB). This is expected when multiple Omni stages share a GPU; the per-process NVML accounting in determine_available_memory() will compute the correct KV cache budget.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) WARNING 09-23 14:42:38 [gpu_ar_worker.py:120] OMNI GPUARWorker forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) INFO 09-23 14:42:38 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) [transformers] `MiniCPMOProcessor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) [transformers] Requested torchvision backend is not available. Falling back to pil backend.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) [transformers] `MiniCPMOProcessor` defines `feature_extractor_class = 'WhisperFeatureExtractor'`, which is deprecated. Register the correct mapping in `AutoFeatureExtractor` instead.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) [transformers] Image processor MiniCPMVImageProcessor: kwargs ['max_slice_nums'] were applied for backward compatibility. To avoid this warning, add them to valid_kwargs: create a custom TypedDict extending ImagesKwargs with these keys and set it as the `valid_kwargs` class attribute.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) INFO 09-23 14:42:49 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) WARNING 09-23 14:42:49 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) WARNING 09-23 14:42:49 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) INFO 09-23 14:42:49 [gpu_model_runner.py:5379] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5...
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) INFO 09-23 14:42:49 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) INFO 09-23 14:42:49 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) INFO 09-23 14:42:50 [cuda.py:492] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) INFO 09-23 14:42:50 [flash_attn.py:897] Using FlashAttention version 2
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) INFO 09-23 14:42:50 [weight_utils.py:863] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 25.45 GiB.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) INFO 09-23 14:42:50 [weight_utils.py:886] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) ␍Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:00<00:00, 224.27it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) INFO 09-23 14:42:50 [default_loader.py:430] Loading weights took 0.29 seconds
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) INFO 09-23 14:42:51 [gpu_model_runner.py:5471] Model loading took 0.65 GiB memory and 1.256915 seconds
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) INFO 09-23 14:42:52 [utils.py:306] Using LBNHC KV cache layout.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) INFO 09-23 14:42:52 [gpu_model_runner.py:6519] Skipping memory profiling for multimodal encoder and encoder cache.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) INFO 09-23 14:42:56 [backends.py:1094] Using cache directory: /home/zhout/.cache/vllm/torch_compile_cache/bbb0253fe1/rank_0_0/backbone for vLLM's torch.compile
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) INFO 09-23 14:42:56 [backends.py:1155] Dynamo bytecode transform time: 2.46 s
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) INFO 09-23 14:43:00 [backends.py:393] Compiling a graph for compile range (1, 8192) takes 4.51 s
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) INFO 09-23 14:43:02 [backends.py:920] collected artifacts: 21 entries, 4 artifacts, 5117476 bytes total
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) INFO 09-23 14:43:02 [decorators.py:719] saved AOT compiled function to /home/zhout/.cache/vllm/torch_compile_cache/torch_aot_compile/58ecadbab6d63ed955c9fd3da880874d877b5384559edc756342e6f2216afb7a/rank_0_0/model
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) INFO 09-23 14:43:02 [monitor.py:53] torch.compile took 8.35 s in total
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) INFO 09-23 14:43:02 [monitor.py:81] Initial profiling/warmup run took 0.14 s
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) INFO 09-23 14:43:05 [base.py:148] Available KV cache memory: 19.65 GiB (device-level profiling)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) INFO 09-23 14:43:05 [kv_cache_utils.py:2032] GPU KV cache size: 343,488 tokens, Maximum concurrency for 4,096 tokens per request: 83.86x
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) WARNING 09-23 14:43:05 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) INFO 09-23 14:43:08 [kernel_warmup.py:124] JIT kernel warmup starting.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) ␍JIT kernel warmup (1 compile keys): 0%| | 0/1 [00:00<?, ?kernel/s]␍JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 11.65kernel/s, ComputeSlotMappingKernel (1 keys)]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) INFO 09-23 14:43:08 [kernel_warmup.py:134] JIT kernel warmup finished in 0.09s.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) WARNING 09-23 14:43:09 [import_utils.py:408] Module vllm.third_party.deep_gemm was found but failed to import
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) WARNING 09-23 14:43:09 [import_utils.py:408] Traceback (most recent call last):
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) WARNING 09-23 14:43:09 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/utils/import_utils.py", line 406, in _has_module
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) WARNING 09-23 14:43:09 [import_utils.py:408] importlib.import_module(module_name)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) WARNING 09-23 14:43:09 [import_utils.py:408] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/importlib/__init__.py", line 90, in import_module
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) WARNING 09-23 14:43:09 [import_utils.py:408] return _bootstrap._gcd_import(name[level:], package, level)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) WARNING 09-23 14:43:09 [import_utils.py:408] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) WARNING 09-23 14:43:09 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1387, in _gcd_import
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) WARNING 09-23 14:43:09 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1360, in _find_and_load
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) WARNING 09-23 14:43:09 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1331, in _find_and_load_unlocked
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) WARNING 09-23 14:43:09 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 935, in _load_unlocked
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) WARNING 09-23 14:43:09 [import_utils.py:408] File "<frozen importlib._bootstrap_external>", line 999, in exec_module
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) WARNING 09-23 14:43:09 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 488, in _call_with_frames_removed
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) WARNING 09-23 14:43:09 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/third_party/deep_gemm/__init__.py", line 126, in <module>
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) WARNING 09-23 14:43:09 [import_utils.py:408] _find_cuda_home() # CUDA home
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) WARNING 09-23 14:43:09 [import_utils.py:408] ^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) WARNING 09-23 14:43:09 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/third_party/deep_gemm/__init__.py", line 120, in _find_cuda_home
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) WARNING 09-23 14:43:09 [import_utils.py:408] assert cuda_home is not None
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) WARNING 09-23 14:43:09 [import_utils.py:408] ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) WARNING 09-23 14:43:09 [import_utils.py:408] AssertionError
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) INFO 09-23 14:43:09 [gpu_model_runner.py:6955] Rank 0: Torch profiler disabled for CUDA graph capture
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) ␍Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 0%| | 0/51 [00:00<?, ?it/s]␍Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 2%|▏ | 1/51 [00:00<00:46, 1.09it/s]␍Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 4%|▍ | 2/51 [00:01<00:23, 2.11it/s]␍Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 8%|▊ | 4/51 [00:01<00:10, 4.29it/s]␍Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 12%|█▏ | 6/51 [00:01<00:07, 6.41it/s]␍Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 16%|█▌ | 8/51 [00:01<00:05, 8.24it/s]␍Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 20%|█▉ | 10/51 [00:01<00:04, 10.24it/s]␍Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 24%|██▎ | 12/51 [00:01<00:03, 11.95it/s]␍Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 29%|██▉ | 15/51 [00:01<00:02, 15.40it/s]␍Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 33%|███▎ | 17/51 [00:02<00:02, 14.68it/s]␍Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 39%|███▉ | 20/51 [00:02<00:01, 17.23it/s]␍Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 45%|████▌ | 23/51 [00:02<00:01, 19.55it/s]␍Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 51%|█████ | 26/51 [00:02<00:01, 20.35it/s]␍Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 57%|█████▋ | 29/51 [00:02<00:01, 21.66it/s]␍Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 63%|██████▎ | 32/51 [00:02<00:00, 21.48it/s]␍Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 69%|██████▊ | 35/51 [00:02<00:00, 21.59it/s]␍Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 75%|███████▍ | 38/51 [00:02<00:00, 21.16it/s]␍Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 80%|████████ | 41/51 [00:03<00:00, 20.16it/s]␍Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 86%|████████▋ | 44/51 [00:03<00:00, 22.17it/s]␍Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 92%|█████████▏| 47/51 [00:03<00:00, 23.12it/s]␍Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 51/51 [00:03<00:00, 19.17it/s]␍Capturing CUDA graphs (mixed prefill-decode, PIECEWISE): 100%|██████████| 51/51 [00:03<00:00, 14.09it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) ␍Capturing CUDA graphs (decode, FULL): 0%| | 0/35 [00:00<?, ?it/s]␍Capturing CUDA graphs (decode, FULL): 3%|▎ | 1/35 [00:00<00:10, 3.26it/s]␍Capturing CUDA graphs (decode, FULL): 17%|█▋ | 6/35 [00:00<00:01, 17.44it/s]␍Capturing CUDA graphs (decode, FULL): 29%|██▊ | 10/35 [00:00<00:01, 23.74it/s]␍Capturing CUDA graphs (decode, FULL): 40%|████ | 14/35 [00:00<00:00, 26.22it/s]␍Capturing CUDA graphs (decode, FULL): 51%|█████▏ | 18/35 [00:00<00:00, 28.89it/s]␍Capturing CUDA graphs (decode, FULL): 71%|███████▏ | 25/35 [00:00<00:00, 38.66it/s]␍Capturing CUDA graphs (decode, FULL): 89%|████████▊ | 31/35 [00:00<00:00, 43.73it/s]␍Capturing CUDA graphs (decode, FULL): 100%|██████████| 35/35 [00:01<00:00, 32.64it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) INFO 09-23 14:43:38 [gpu_model_runner.py:7001] Graph capturing finished in 29 secs, took 0.00 GiB
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) INFO 09-23 14:43:38 [gpu_worker.py:860] Free memory on device (21.21/23.89 GiB) on startup. Desired GPU memory utilization is (0.92, 21.21 GiB). Actual usage is 1.45 GiB for consumed memory (weights + non-torch), 0.23 GiB for peak activation, and 0.0 GiB for CUDAGraph memory. Replace gpu_memory_utilization config with `--kv-cache-memory=20816297984` (19.39 GiB) to fit into requested memory, or `--kv-cache-memory=20816297984` (19.39 GiB) to fully utilize gpu memory. Current kv cache memory in use is 19.65 GiB.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) INFO 09-23 14:43:39 [jit_monitor.py:85] Kernel JIT monitor activated; monitored JIT compilations during inference will use mode=warn.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) INFO 09-23 14:43:39 [torch_utils.py:277] Reducing Torch threads from 12 to 1 for serving. Set OMP_NUM_THREADS in the external environment to override.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) INFO 09-23 14:43:39 [core.py:361] init engine (profile, create kv cache, warmup model) took 47.52 s (compilation: 8.35 s)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) WARNING 09-23 14:43:39 [scheduler.py:213] Using custom scheduler class vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) WARNING 09-23 14:43:40 [omni_scheduler_mixin.py:118] OMNI scheduler forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) INFO 09-23 14:43:40 [chunk_transfer_adapter.py:142] Bounded active-stream window enabled: K=1. Multi-replica deployments require sticky per-stream routing across Stage 1 replicas (each replica owns an independent active-set; without sticky routing, a stream can be active on one replica and non-active on another and both will race to evict it).
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) INFO 09-23 14:43:40 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4158361) INFO 09-23 14:43:40 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['native'], fused_add_rms_norm=['native'])
|
||||
INFO 09-23 14:43:40 [stage_runtime.py:943] [StageRuntime] Stage 1 engine startup completed
|
||||
INFO 09-23 14:43:40 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-1 [rep-0] initializing EngineCore
|
||||
INFO 09-23 14:43:40 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-1 [rep-0] EngineCore running
|
||||
INFO 09-23 14:43:40 [stage_runtime.py:958] [StageRuntime] Stage 1 initialized
|
||||
INFO 09-23 14:43:52 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 14:43:52 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 14:43:52 [patch.py:583] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
|
||||
+294
@@ -0,0 +1,294 @@
|
||||
INFO 09-23 14:33:14 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 14:33:15 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 14:33:15 [patch.py:583] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
|
||||
====================
|
||||
vllm version: 0.29.0
|
||||
====================
|
||||
INFO 09-23 14:33:15 [omni_base.py:216] [Omni] Initializing with model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5
|
||||
INFO 09-23 14:33:15 [async_omni_engine.py:172] [AsyncOmniEngine] Initializing with model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5
|
||||
INFO 09-23 14:33:16 [async_omni_engine.py:290] [AsyncOmniEngine] Launching Orchestrator thread with 3 stages
|
||||
INFO 09-23 14:33:16 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
INFO 09-23 14:33:16 [initialization.py:356] Loaded OmniTransferConfig with 1 connector configurations
|
||||
WARNING 09-23 14:33:16 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 14:33:16 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
INFO 09-23 14:33:16 [model.py:684] Resolved architecture: MiniCPMO45OmniForConditionalGeneration
|
||||
INFO 09-23 14:33:16 [model.py:2021] Using max model len 2048
|
||||
WARNING 09-23 14:33:16 [model.py:981] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False.
|
||||
INFO 09-23 14:33:17 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode.
|
||||
INFO 09-23 14:33:17 [model.py:887] Disabled mm_prefix attention mode because multimodal inputs are configuration-disabled. Attention backends without mm_prefix support may now be selected.
|
||||
INFO 09-23 14:33:17 [scheduler.py:277] Chunked prefill is enabled with max_num_batched_tokens=2048.
|
||||
WARNING 09-23 14:33:17 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
WARNING 09-23 14:33:17 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
INFO 09-23 14:33:17 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-23 14:33:17 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
INFO 09-23 14:33:17 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
WARNING 09-23 14:33:17 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 14:33:17 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
INFO 09-23 14:33:17 [model.py:684] Resolved architecture: MiniCPMO45OmniForConditionalGeneration
|
||||
INFO 09-23 14:33:17 [model.py:2021] Using max model len 512
|
||||
WARNING 09-23 14:33:17 [model.py:981] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False.
|
||||
INFO 09-23 14:33:17 [model.py:2021] Using max model len 512
|
||||
INFO 09-23 14:33:17 [scheduler.py:277] Chunked prefill is enabled with max_num_batched_tokens=512.
|
||||
INFO 09-23 14:33:17 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
WARNING 09-23 14:33:17 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 14:33:17 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
INFO 09-23 14:33:17 [model.py:684] Resolved architecture: MiniCPMO45Code2Wav
|
||||
INFO 09-23 14:33:17 [model.py:2021] Using max model len 4096
|
||||
WARNING 09-23 14:33:17 [arg_utils.py:2780] This model does not officially support disabling chunked prefill. Disabling this manually may cause the engine to crash or produce incorrect outputs.
|
||||
INFO 09-23 14:33:17 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-23 14:33:17 [stage_runtime.py:853] [stage_init] Stage-0 set runtime devices: 0
|
||||
INFO 09-23 14:33:23 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 14:33:23 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 14:33:23 [patch.py:583] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) INFO 09-23 14:33:24 [core.py:123] Initializing a V1 LLM engine (v0.29.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=2048, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 0, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) INFO 09-23 14:33:25 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) INFO 09-23 14:33:25 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_028c7c75f37a4889a1dffc16bb0237b9 backend=nccl
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) INFO 09-23 14:33:25 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) WARNING 09-23 14:33:25 [gpu_ar_worker.py:120] OMNI GPUARWorker forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) INFO 09-23 14:33:25 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) INFO 09-23 14:33:25 [base.py:138] Offloader set to UVAOffloader
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) WARNING 09-23 14:33:25 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) WARNING 09-23 14:33:25 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) INFO 09-23 14:33:25 [gpu_model_runner.py:5379] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5...
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) WARNING 09-23 14:33:26 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) WARNING 09-23 14:33:26 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) INFO 09-23 14:33:26 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) INFO 09-23 14:33:26 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) INFO 09-23 14:33:26 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) INFO 09-23 14:33:26 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) INFO 09-23 14:33:26 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) INFO 09-23 14:33:27 [cuda.py:492] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) INFO 09-23 14:33:27 [flash_attn.py:897] Using FlashAttention version 2
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) INFO 09-23 14:33:37 [uva.py:65] Total CPU offloaded parameters: 5.03
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) INFO 09-23 14:33:37 [weight_utils.py:863] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 20.71 GiB.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) INFO 09-23 14:33:37 [weight_utils.py:886] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) ␍Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) ␍Loading safetensors checkpoint shards: 50% Completed | 2/4 [00:00<00:00, 3.13it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) ␍Loading safetensors checkpoint shards: 75% Completed | 3/4 [00:00<00:00, 3.22it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:00<00:00, 4.02it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) INFO 09-23 14:34:15 [default_loader.py:430] Loading weights took 37.89 seconds
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) INFO 09-23 14:34:16 [gpu_model_runner.py:5471] Model loading took 10.84 GiB memory and 49.349124 seconds
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) INFO 09-23 14:34:16 [utils.py:306] Using LBNHC KV cache layout.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) INFO 09-23 14:34:40 [base.py:148] Available KV cache memory: 2.72 GiB (device-level profiling)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) INFO 09-23 14:34:40 [kv_cache_utils.py:2032] GPU KV cache size: 19,776 tokens, Maximum concurrency for 2,048 tokens per request: 9.66x
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) WARNING 09-23 14:34:40 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) INFO 09-23 14:34:41 [kernel_warmup.py:124] JIT kernel warmup starting.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) ␍JIT kernel warmup (1 compile keys): 0%| | 0/1 [00:00<?, ?kernel/s]␍JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 1.01kernel/s, ComputeSlotMappingKernel (1 keys)]␍JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 1.01kernel/s, ComputeSlotMappingKernel (1 keys)]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) INFO 09-23 14:34:42 [kernel_warmup.py:134] JIT kernel warmup finished in 0.99s.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) WARNING 09-23 14:34:43 [import_utils.py:408] Module vllm.third_party.deep_gemm was found but failed to import
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) WARNING 09-23 14:34:43 [import_utils.py:408] Traceback (most recent call last):
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) WARNING 09-23 14:34:43 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/utils/import_utils.py", line 406, in _has_module
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) WARNING 09-23 14:34:43 [import_utils.py:408] importlib.import_module(module_name)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) WARNING 09-23 14:34:43 [import_utils.py:408] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/importlib/__init__.py", line 90, in import_module
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) WARNING 09-23 14:34:43 [import_utils.py:408] return _bootstrap._gcd_import(name[level:], package, level)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) WARNING 09-23 14:34:43 [import_utils.py:408] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) WARNING 09-23 14:34:43 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1387, in _gcd_import
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) WARNING 09-23 14:34:43 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1360, in _find_and_load
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) WARNING 09-23 14:34:43 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1331, in _find_and_load_unlocked
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) WARNING 09-23 14:34:43 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 935, in _load_unlocked
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) WARNING 09-23 14:34:43 [import_utils.py:408] File "<frozen importlib._bootstrap_external>", line 999, in exec_module
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) WARNING 09-23 14:34:43 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 488, in _call_with_frames_removed
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) WARNING 09-23 14:34:43 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/third_party/deep_gemm/__init__.py", line 126, in <module>
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) WARNING 09-23 14:34:43 [import_utils.py:408] _find_cuda_home() # CUDA home
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) WARNING 09-23 14:34:43 [import_utils.py:408] ^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) WARNING 09-23 14:34:43 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/third_party/deep_gemm/__init__.py", line 120, in _find_cuda_home
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) WARNING 09-23 14:34:43 [import_utils.py:408] assert cuda_home is not None
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) WARNING 09-23 14:34:43 [import_utils.py:408] ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) WARNING 09-23 14:34:43 [import_utils.py:408] AssertionError
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) INFO 09-23 14:34:43 [gpu_worker.py:860] Free memory on device (22.58/23.89 GiB) on startup. Desired GPU memory utilization is (0.58, 13.86 GiB). Actual usage is 11.01 GiB for consumed memory (weights + non-torch), 0.22 GiB for peak activation, and 0.0 GiB for CUDAGraph memory. Replace gpu_memory_utilization config with `--kv-cache-memory=2667877499` (2.48 GiB) to fit into requested memory, or `--kv-cache-memory=12032409600` (11.21 GiB) to fully utilize gpu memory. Current kv cache memory in use is 2.72 GiB.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) INFO 09-23 14:34:44 [jit_monitor.py:85] Kernel JIT monitor activated; monitored JIT compilations during inference will use mode=warn.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) INFO 09-23 14:34:44 [torch_utils.py:277] Reducing Torch threads from 12 to 1 for serving. Set OMP_NUM_THREADS in the external environment to override.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) INFO 09-23 14:34:44 [core.py:368] init engine (profile, create kv cache, warmup model) took 27.79 s
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) WARNING 09-23 14:34:44 [scheduler.py:213] Using custom scheduler class vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) WARNING 09-23 14:34:44 [omni_scheduler_mixin.py:118] OMNI scheduler forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) INFO 09-23 14:34:44 [chunk_transfer_adapter.py:142] Bounded active-stream window enabled: K=1. Multi-replica deployments require sticky per-stream routing across Stage 1 replicas (each replica owns an independent active-set; without sticky routing, a stream can be active on one replica and non-active on another and both will race to evict it).
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) INFO 09-23 14:34:44 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4155868) INFO 09-23 14:34:44 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-23 14:34:44 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed
|
||||
INFO 09-23 14:34:44 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-0 [rep-0] initializing EngineCore
|
||||
INFO 09-23 14:34:44 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-0 [rep-0] EngineCore running
|
||||
INFO 09-23 14:34:44 [stage_runtime.py:958] [StageRuntime] Stage 0 initialized
|
||||
INFO 09-23 14:34:44 [stage_runtime.py:853] [stage_init] Stage-1 set runtime devices: 0
|
||||
INFO 09-23 14:34:54 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 14:34:54 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 14:34:54 [patch.py:583] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) INFO 09-23 14:34:55 [core.py:123] Initializing a V1 LLM engine (v0.29.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=512, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [512], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 0, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) INFO 09-23 14:34:57 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_47ce84a916794d5aaae4f1d8e8082eff backend=nccl
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) INFO 09-23 14:34:57 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) WARNING 09-23 14:34:57 [gpu_ar_worker.py:120] OMNI GPUARWorker forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) INFO 09-23 14:34:57 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) [transformers] `MiniCPMOProcessor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) [transformers] Requested torchvision backend is not available. Falling back to pil backend.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) [transformers] `MiniCPMOProcessor` defines `feature_extractor_class = 'WhisperFeatureExtractor'`, which is deprecated. Register the correct mapping in `AutoFeatureExtractor` instead.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) [transformers] Image processor MiniCPMVImageProcessor: kwargs ['max_slice_nums'] were applied for backward compatibility. To avoid this warning, add them to valid_kwargs: create a custom TypedDict extending ImagesKwargs with these keys and set it as the `valid_kwargs` class attribute.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) INFO 09-23 14:35:08 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) WARNING 09-23 14:35:08 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) WARNING 09-23 14:35:08 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) INFO 09-23 14:35:08 [gpu_model_runner.py:5379] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5...
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) WARNING 09-23 14:35:08 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) WARNING 09-23 14:35:08 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) INFO 09-23 14:35:08 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) INFO 09-23 14:35:08 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) INFO 09-23 14:35:08 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) INFO 09-23 14:35:08 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) INFO 09-23 14:35:09 [cuda.py:492] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) INFO 09-23 14:35:09 [flash_attn.py:897] Using FlashAttention version 2
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) INFO 09-23 14:35:09 [weight_utils.py:863] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 19.14 GiB.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) INFO 09-23 14:35:09 [weight_utils.py:893] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre) and the checkpoint size (17.46 GiB) exceeds 90% of available RAM (19.14 GiB).
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) ␍Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) ␍Loading safetensors checkpoint shards: 50% Completed | 2/4 [00:00<00:00, 12.72it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:00<00:00, 3.89it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:00<00:00, 4.34it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) INFO 09-23 14:35:11 [default_loader.py:430] Loading weights took 1.31 seconds
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) INFO 09-23 14:35:12 [gpu_model_runner.py:5471] Model loading took 0.65 GiB memory and 2.414316 seconds
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) INFO 09-23 14:35:12 [utils.py:306] Using LBNHC KV cache layout.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) INFO 09-23 14:35:12 [gpu_model_runner.py:6519] Skipping memory profiling for multimodal encoder and encoder cache.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) INFO 09-23 14:35:13 [kv_cache_utils.py:2032] GPU KV cache size: 4,368 tokens, Maximum concurrency for 512 tokens per request: 8.53x
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) WARNING 09-23 14:35:13 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) INFO 09-23 14:35:13 [kernel_warmup.py:124] JIT kernel warmup starting.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) ␍JIT kernel warmup (1 compile keys): 0%| | 0/1 [00:00<?, ?kernel/s]␍JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 2.71kernel/s, ComputeSlotMappingKernel (1 keys)]␍JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 2.71kernel/s, ComputeSlotMappingKernel (1 keys)]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) INFO 09-23 14:35:14 [kernel_warmup.py:134] JIT kernel warmup finished in 0.37s.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) WARNING 09-23 14:35:14 [import_utils.py:408] Module vllm.third_party.deep_gemm was found but failed to import
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) WARNING 09-23 14:35:14 [import_utils.py:408] Traceback (most recent call last):
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) WARNING 09-23 14:35:14 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/utils/import_utils.py", line 406, in _has_module
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) WARNING 09-23 14:35:14 [import_utils.py:408] importlib.import_module(module_name)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) WARNING 09-23 14:35:14 [import_utils.py:408] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/importlib/__init__.py", line 90, in import_module
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) WARNING 09-23 14:35:14 [import_utils.py:408] return _bootstrap._gcd_import(name[level:], package, level)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) WARNING 09-23 14:35:14 [import_utils.py:408] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) WARNING 09-23 14:35:14 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1387, in _gcd_import
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) WARNING 09-23 14:35:14 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1360, in _find_and_load
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) WARNING 09-23 14:35:14 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1331, in _find_and_load_unlocked
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) WARNING 09-23 14:35:14 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 935, in _load_unlocked
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) WARNING 09-23 14:35:14 [import_utils.py:408] File "<frozen importlib._bootstrap_external>", line 999, in exec_module
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) WARNING 09-23 14:35:14 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 488, in _call_with_frames_removed
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) WARNING 09-23 14:35:14 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/third_party/deep_gemm/__init__.py", line 126, in <module>
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) WARNING 09-23 14:35:14 [import_utils.py:408] _find_cuda_home() # CUDA home
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) WARNING 09-23 14:35:14 [import_utils.py:408] ^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) WARNING 09-23 14:35:14 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/third_party/deep_gemm/__init__.py", line 120, in _find_cuda_home
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) WARNING 09-23 14:35:14 [import_utils.py:408] assert cuda_home is not None
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) WARNING 09-23 14:35:14 [import_utils.py:408] ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) WARNING 09-23 14:35:14 [import_utils.py:408] AssertionError
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) INFO 09-23 14:35:15 [jit_monitor.py:85] Kernel JIT monitor activated; monitored JIT compilations during inference will use mode=warn.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) INFO 09-23 14:35:15 [torch_utils.py:277] Reducing Torch threads from 12 to 1 for serving. Set OMP_NUM_THREADS in the external environment to override.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) INFO 09-23 14:35:15 [core.py:368] init engine (profile, create kv cache, warmup model) took 2.74 s
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) WARNING 09-23 14:35:15 [scheduler.py:213] Using custom scheduler class vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) WARNING 09-23 14:35:15 [omni_scheduler_mixin.py:118] OMNI scheduler forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) INFO 09-23 14:35:15 [chunk_transfer_adapter.py:142] Bounded active-stream window enabled: K=1. Multi-replica deployments require sticky per-stream routing across Stage 1 replicas (each replica owns an independent active-set; without sticky routing, a stream can be active on one replica and non-active on another and both will race to evict it).
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) INFO 09-23 14:35:15 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
INFO 09-23 14:35:15 [stage_runtime.py:943] [StageRuntime] Stage 1 engine startup completed
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4156145) INFO 09-23 14:35:15 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-23 14:35:15 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-1 [rep-0] initializing EngineCore
|
||||
INFO 09-23 14:35:16 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-1 [rep-0] EngineCore running
|
||||
INFO 09-23 14:35:16 [stage_runtime.py:958] [StageRuntime] Stage 1 initialized
|
||||
INFO 09-23 14:35:16 [stage_runtime.py:853] [stage_init] Stage-2 set runtime devices: 0
|
||||
INFO 09-23 14:35:24 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 14:35:24 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 14:35:24 [patch.py:583] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) INFO 09-23 14:35:25 [core.py:123] Initializing a V1 LLM engine (v0.29.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=4096, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=False, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [4096], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 0, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) INFO 09-23 14:35:25 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_0f97cdb270804b129ff4c6180e8d4b22 backend=nccl
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) INFO 09-23 14:35:25 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) WARNING 09-23 14:35:26 [gpu_generation_worker.py:103] OMNI GPUGenerationWorker forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) INFO 09-23 14:35:26 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) INFO 09-23 14:35:26 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) WARNING 09-23 14:35:26 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) WARNING 09-23 14:35:26 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) INFO 09-23 14:35:26 [gpu_model_runner.py:5379] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5...
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) INFO 09-23 14:35:27 [weight_utils.py:863] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 17.72 GiB.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) INFO 09-23 14:35:27 [weight_utils.py:893] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre) and the checkpoint size (17.46 GiB) exceeds 90% of available RAM (17.72 GiB).
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) ␍Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:00<00:00, 45.73it/s]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) INFO 09-23 14:35:27 [step_audio2_token2wav.py:163] Loading Token2Wav models from: /home/zhout/project/edge_infer/models/MiniCPM-o-4_5/assets/token2wav
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) INFO 09-23 14:35:34 [step_audio2_token2wav.py:201] Token2Wav models loaded successfully
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) INFO 09-23 14:35:35 [batched_token2wav.py:227] Enabling HiFT CUDA Graph with batch sizes [1]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) INFO 09-23 14:35:37 [cuda_graph_wrapper.py:98] Captured HiFT CUDA Graph for shape (1, 50, 0)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) INFO 09-23 14:35:37 [cuda_graph_wrapper.py:98] Captured HiFT CUDA Graph for shape (1, 58, 3840)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) INFO 09-23 14:35:37 [batched_token2wav.py:235] HiFT CUDA Graph captured successfully
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) INFO 09-23 14:35:37 [batched_token2wav.py:246] CFM CUDA Graph enabled (max_graphs=32)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) INFO 09-23 14:35:37 [default_loader.py:430] Loading weights took 10.18 seconds
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) INFO 09-23 14:35:38 [gpu_model_runner.py:5471] Model loading took 3.36 GiB memory and 10.196961 seconds
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) INFO 09-23 14:35:39 [cuda.py:492] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) INFO 09-23 14:35:39 [utils.py:306] Using LBNHC KV cache layout.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) WARNING 09-23 14:35:39 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) INFO 09-23 14:35:39 [kernel_warmup.py:124] JIT kernel warmup starting.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) ␍JIT kernel warmup (1 compile keys): 0%| | 0/1 [00:00<?, ?kernel/s]␍JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 413.27kernel/s, ComputeSlotMappingKernel (1 keys)]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) INFO 09-23 14:35:39 [kernel_warmup.py:134] JIT kernel warmup finished in 0.00s.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) WARNING 09-23 14:35:39 [import_utils.py:408] Module vllm.third_party.deep_gemm was found but failed to import
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) WARNING 09-23 14:35:39 [import_utils.py:408] Traceback (most recent call last):
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) WARNING 09-23 14:35:39 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/utils/import_utils.py", line 406, in _has_module
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) WARNING 09-23 14:35:39 [import_utils.py:408] importlib.import_module(module_name)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) WARNING 09-23 14:35:39 [import_utils.py:408] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/importlib/__init__.py", line 90, in import_module
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) WARNING 09-23 14:35:39 [import_utils.py:408] return _bootstrap._gcd_import(name[level:], package, level)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) WARNING 09-23 14:35:39 [import_utils.py:408] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) WARNING 09-23 14:35:39 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1387, in _gcd_import
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) WARNING 09-23 14:35:39 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1360, in _find_and_load
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) WARNING 09-23 14:35:39 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1331, in _find_and_load_unlocked
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) WARNING 09-23 14:35:39 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 935, in _load_unlocked
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) WARNING 09-23 14:35:39 [import_utils.py:408] File "<frozen importlib._bootstrap_external>", line 999, in exec_module
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) WARNING 09-23 14:35:39 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 488, in _call_with_frames_removed
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) WARNING 09-23 14:35:39 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/third_party/deep_gemm/__init__.py", line 126, in <module>
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) WARNING 09-23 14:35:39 [import_utils.py:408] _find_cuda_home() # CUDA home
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) WARNING 09-23 14:35:39 [import_utils.py:408] ^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) WARNING 09-23 14:35:39 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/third_party/deep_gemm/__init__.py", line 120, in _find_cuda_home
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) WARNING 09-23 14:35:39 [import_utils.py:408] assert cuda_home is not None
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) WARNING 09-23 14:35:39 [import_utils.py:408] ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) WARNING 09-23 14:35:39 [import_utils.py:408] AssertionError
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) WARNING 09-23 14:35:39 [gpu_generation_model_runner.py:558] Dummy sampler run is not implemented for generation model
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) INFO 09-23 14:35:40 [jit_monitor.py:85] Kernel JIT monitor activated; monitored JIT compilations during inference will use mode=warn.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) INFO 09-23 14:35:40 [torch_utils.py:277] Reducing Torch threads from 12 to 1 for serving. Set OMP_NUM_THREADS in the external environment to override.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) INFO 09-23 14:35:40 [core.py:368] init engine (profile, create kv cache, warmup model) took 2.39 s
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) WARNING 09-23 14:35:41 [scheduler.py:213] Using custom scheduler class vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) WARNING 09-23 14:35:41 [omni_scheduler_mixin.py:118] OMNI scheduler forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) INFO 09-23 14:35:41 [chunk_transfer_adapter.py:142] Bounded active-stream window enabled: K=1. Multi-replica deployments require sticky per-stream routing across Stage 1 replicas (each replica owns an independent active-set; without sticky routing, a stream can be active on one replica and non-active on another and both will race to evict it).
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) INFO 09-23 14:35:41 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) WARNING 09-23 14:35:41 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) WARNING 09-23 14:35:41 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) INFO 09-23 14:35:41 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-23 14:35:41 [stage_runtime.py:943] [StageRuntime] Stage 2 engine startup completed
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) INFO 09-23 14:35:41 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
INFO 09-23 14:35:41 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-2 [rep-0] initializing EngineCore
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) INFO 09-23 14:35:41 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
INFO 09-23 14:35:41 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-2 [rep-0] EngineCore running
|
||||
INFO 09-23 14:35:41 [stage_runtime.py:958] [StageRuntime] Stage 2 initialized
|
||||
INFO 09-23 14:35:43 [orchestrator.py:604] [Orchestrator] Starting event loop
|
||||
INFO 09-23 14:35:43 [async_omni_engine.py:323] [AsyncOmniEngine] Orchestrator ready with 3 stages
|
||||
INFO 09-23 14:35:43 [omni_base.py:235] [Omni] AsyncOmniEngine initialized in 147.07 seconds
|
||||
INFO 09-23 14:35:43 [omni_base.py:255] [Omni] Initialized with 3 stages for model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5
|
||||
WARNING 09-23 14:35:43 [input_processor.py:315] Passing raw prompts to InputProcessor is deprecated and will be removed in the future. You should instead pass the outputs of Renderer.render_cmpl() or Renderer.render_chat().
|
||||
␍Processed prompts: 0%| | 0/1 [00:00<?, ?it/s](StageEngineCoreProc_stage0_replica0 pid=4155868) WARNING 09-23 14:35:43 [gpu_ar_model_runner.py:1461] prefer_model_sampler model MiniCPMO45OmniForConditionalGeneration returned None from sample(); falling back to the default sampler.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) INFO 09-23 14:36:14 [cuda_graph_wrapper.py:300] Captured CFM CUDA Graph for shape ('estimator_step', ((2, 320, 302), 'torch.float32', 'cuda:0'), ((2, 1, 512), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 0, 128), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 302, 128), 'torch.float32', 'cuda:0')) (cache=1/32, stats={'calls': 1, 'hits': 0, 'captures': 1, 'flushes': 0, 'eager': 0, 'cache_size': 0}) [allocated 3633.9 MiB (+32.2), reserved 5464.0 MiB (+22.0)]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) INFO 09-23 14:36:15 [cuda_graph_wrapper.py:300] Captured CFM CUDA Graph for shape ('estimator_step', ((2, 320, 50), 'torch.float32', 'cuda:0'), ((2, 1, 512), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 302, 128), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 352, 128), 'torch.float32', 'cuda:0')) (cache=2/32, stats={'calls': 11, 'hits': 9, 'captures': 2, 'flushes': 0, 'eager': 0, 'cache_size': 1}) [allocated 4539.7 MiB (+32.0), reserved 6378.0 MiB (+28.0)]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) INFO 09-23 14:36:15 [cuda_graph_wrapper.py:300] Captured CFM CUDA Graph for shape ('estimator_step', ((2, 320, 50), 'torch.float32', 'cuda:0'), ((2, 1, 512), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 352, 128), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 402, 128), 'torch.float32', 'cuda:0')) (cache=3/32, stats={'calls': 21, 'hits': 18, 'captures': 3, 'flushes': 0, 'eager': 0, 'cache_size': 2}) [allocated 4803.9 MiB (+32.0), reserved 6956.0 MiB (+-352.0)]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) INFO 09-23 14:36:16 [cuda_graph_wrapper.py:300] Captured CFM CUDA Graph for shape ('estimator_step', ((2, 320, 50), 'torch.float32', 'cuda:0'), ((2, 1, 512), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 402, 128), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 452, 128), 'torch.float32', 'cuda:0')) (cache=4/32, stats={'calls': 31, 'hits': 27, 'captures': 4, 'flushes': 0, 'eager': 0, 'cache_size': 3}) [allocated 5079.1 MiB (+32.0), reserved 7096.0 MiB (+-876.0)]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) INFO 09-23 14:36:17 [cuda_graph_wrapper.py:300] Captured CFM CUDA Graph for shape ('estimator_step', ((2, 320, 10), 'torch.float32', 'cuda:0'), ((2, 1, 512), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 402, 128), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 412, 128), 'torch.float32', 'cuda:0')) (cache=5/32, stats={'calls': 61, 'hits': 56, 'captures': 5, 'flushes': 0, 'eager': 0, 'cache_size': 4}) [allocated 5206.6 MiB (+32.0), reserved 7190.0 MiB (+-1550.0)]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) INFO 09-23 14:36:17 [cuda_graph_wrapper.py:120] Lazily capturing HiFT CUDA Graph for shape (1, 18, 3840)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) WARNING 09-23 14:36:17 [jit_monitor.py:141] Triton kernel JIT compilation during inference: SnakeBeta._init_triton.<locals>._kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4156415) INFO 09-23 14:36:17 [cuda_graph_wrapper.py:98] Captured HiFT CUDA Graph for shape (1, 18, 3840)
|
||||
␍Processed prompts: 100%|██████████| 1/1 [00:34<00:00, 34.81s/it]␍Processed prompts: 100%|██████████| 1/1 [00:34<00:00, 34.81s/it]
|
||||
query type: text
|
||||
use_tts (tts_bos): True
|
||||
Request ID: 0_8a6e5016-03c2-4673-a2e5-c79b90f62ccc, Text saved to /home/zhout/project/edge_infer/analysis/experiments/e2e_recovery_20260923/mini_output_template/0_8a6e5016-03c2-4673-a2e5-c79b90f62ccc.txt
|
||||
Request ID: 0_8a6e5016-03c2-4673-a2e5-c79b90f62ccc, Saved audio to /home/zhout/project/edge_infer/analysis/experiments/e2e_recovery_20260923/mini_output_template/output_0_8a6e5016-03c2-4673-a2e5-c79b90f62ccc.wav
|
||||
INFO 09-23 14:36:17 [omni_base.py:781] [Omni] Shutting down
|
||||
INFO 09-23 14:36:17 [async_omni_engine.py:1807] [AsyncOmniEngine] Shutting down Orchestrator
|
||||
INFO 09-23 14:36:17 [orchestrator.py:705] [Orchestrator] Received shutdown signal
|
||||
INFO 09-23 14:36:17 [orchestrator.py:3092] [Orchestrator] Shutting down all 3 client(s)
|
||||
INFO 09-23 14:36:17 [core_client.py:689] [shutdown] MPClient: start timeout=default
|
||||
INFO 09-23 14:36:17 [core_client.py:691] [shutdown] MPClient: stopping engine manager
|
||||
INFO 09-23 14:36:17 [utils.py:620] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage0_replica0
|
||||
WARNING 09-23 14:36:22 [utils.py:640] [shutdown] Process manager: force killing remaining processes count=1
|
||||
WARNING 09-23 14:36:22 [utils.py:645] [shutdown] Process manager: force killing remaining process StageEngineCoreProc_stage0_replica0 pid 4155868
|
||||
INFO 09-23 14:36:22 [core_client.py:693] [shutdown] MPClient: engine manager stopped
|
||||
INFO 09-23 14:36:22 [core_client.py:694] [shutdown] MPClient: cleaning up background resources
|
||||
INFO 09-23 14:36:22 [core_client.py:696] [shutdown] MPClient: complete
|
||||
INFO 09-23 14:36:22 [stage_pool.py:1354] [StagePool] Stage 0 replica 0 shut down
|
||||
INFO 09-23 14:36:22 [core_client.py:689] [shutdown] MPClient: start timeout=default
|
||||
INFO 09-23 14:36:22 [utils.py:620] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage1_replica0
|
||||
INFO 09-23 14:36:27 [stage_pool.py:1354] [StagePool] Stage 1 replica 0 shut down
|
||||
INFO 09-23 14:36:27 [core_client.py:689] [shutdown] MPClient: start timeout=default
|
||||
INFO 09-23 14:36:27 [utils.py:620] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage2_replica0
|
||||
INFO 09-23 14:36:31 [stage_pool.py:1354] [StagePool] Stage 2 replica 0 shut down
|
||||
/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/multiprocessing/resource_tracker.py:279: UserWarning: resource_tracker: There appear to be 1 leaked shared_memory objects to clean up at shutdown
|
||||
warnings.warn('resource_tracker: There appear to be %d '
|
||||
+279
@@ -0,0 +1,279 @@
|
||||
INFO 09-23 14:37:51 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 14:37:51 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 14:37:51 [patch.py:583] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
|
||||
====================
|
||||
vllm version: 0.29.0
|
||||
====================
|
||||
INFO 09-23 14:37:52 [omni_base.py:216] [Omni] Initializing with model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5
|
||||
INFO 09-23 14:37:52 [async_omni_engine.py:172] [AsyncOmniEngine] Initializing with model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5
|
||||
INFO 09-23 14:37:52 [async_omni_engine.py:290] [AsyncOmniEngine] Launching Orchestrator thread with 3 stages
|
||||
INFO 09-23 14:37:52 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
INFO 09-23 14:37:52 [initialization.py:356] Loaded OmniTransferConfig with 1 connector configurations
|
||||
WARNING 09-23 14:37:52 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 14:37:52 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
INFO 09-23 14:37:52 [model.py:684] Resolved architecture: MiniCPMO45OmniForConditionalGeneration
|
||||
INFO 09-23 14:37:52 [model.py:2021] Using max model len 2048
|
||||
WARNING 09-23 14:37:52 [model.py:981] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False.
|
||||
INFO 09-23 14:37:53 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode.
|
||||
INFO 09-23 14:37:53 [model.py:887] Disabled mm_prefix attention mode because multimodal inputs are configuration-disabled. Attention backends without mm_prefix support may now be selected.
|
||||
INFO 09-23 14:37:53 [scheduler.py:277] Chunked prefill is enabled with max_num_batched_tokens=2048.
|
||||
WARNING 09-23 14:37:53 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
WARNING 09-23 14:37:53 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
INFO 09-23 14:37:53 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-23 14:37:53 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
INFO 09-23 14:37:53 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
WARNING 09-23 14:37:53 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 14:37:53 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
INFO 09-23 14:37:53 [model.py:684] Resolved architecture: MiniCPMO45OmniForConditionalGeneration
|
||||
INFO 09-23 14:37:53 [model.py:2021] Using max model len 512
|
||||
WARNING 09-23 14:37:53 [model.py:981] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False.
|
||||
INFO 09-23 14:37:53 [model.py:2021] Using max model len 512
|
||||
INFO 09-23 14:37:53 [scheduler.py:277] Chunked prefill is enabled with max_num_batched_tokens=512.
|
||||
INFO 09-23 14:37:53 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
WARNING 09-23 14:37:53 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 14:37:53 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
INFO 09-23 14:37:53 [model.py:684] Resolved architecture: MiniCPMO45Code2Wav
|
||||
INFO 09-23 14:37:53 [model.py:2021] Using max model len 4096
|
||||
WARNING 09-23 14:37:53 [arg_utils.py:2780] This model does not officially support disabling chunked prefill. Disabling this manually may cause the engine to crash or produce incorrect outputs.
|
||||
INFO 09-23 14:37:53 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-23 14:37:53 [stage_runtime.py:853] [stage_init] Stage-0 set runtime devices: 0
|
||||
INFO 09-23 14:37:59 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 14:38:00 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 14:38:00 [patch.py:583] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) INFO 09-23 14:38:01 [core.py:123] Initializing a V1 LLM engine (v0.29.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=2048, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 0, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) INFO 09-23 14:38:02 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) INFO 09-23 14:38:02 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_f8ec0c16156044e3bae1107090805fb7 backend=nccl
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) INFO 09-23 14:38:02 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) WARNING 09-23 14:38:02 [gpu_ar_worker.py:120] OMNI GPUARWorker forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) INFO 09-23 14:38:02 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) INFO 09-23 14:38:02 [base.py:138] Offloader set to UVAOffloader
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) WARNING 09-23 14:38:02 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) WARNING 09-23 14:38:02 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) INFO 09-23 14:38:02 [gpu_model_runner.py:5379] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5...
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) WARNING 09-23 14:38:03 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) WARNING 09-23 14:38:03 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) INFO 09-23 14:38:03 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) INFO 09-23 14:38:03 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) INFO 09-23 14:38:03 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) INFO 09-23 14:38:03 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) INFO 09-23 14:38:03 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) INFO 09-23 14:38:04 [cuda.py:492] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) INFO 09-23 14:38:04 [flash_attn.py:897] Using FlashAttention version 2
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) INFO 09-23 14:38:14 [uva.py:65] Total CPU offloaded parameters: 5.03
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) INFO 09-23 14:38:14 [weight_utils.py:863] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 20.78 GiB.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) INFO 09-23 14:38:14 [weight_utils.py:886] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) ␍Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:00<00:00, 240.22it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) INFO 09-23 14:38:23 [default_loader.py:430] Loading weights took 9.10 seconds
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) INFO 09-23 14:38:24 [gpu_model_runner.py:5471] Model loading took 10.84 GiB memory and 20.601335 seconds
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) INFO 09-23 14:38:25 [utils.py:306] Using LBNHC KV cache layout.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) INFO 09-23 14:38:36 [base.py:148] Available KV cache memory: 2.72 GiB (device-level profiling)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) INFO 09-23 14:38:36 [kv_cache_utils.py:2032] GPU KV cache size: 19,776 tokens, Maximum concurrency for 2,048 tokens per request: 9.66x
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) WARNING 09-23 14:38:36 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) INFO 09-23 14:38:36 [kernel_warmup.py:124] JIT kernel warmup starting.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) ␍JIT kernel warmup (1 compile keys): 0%| | 0/1 [00:00<?, ?kernel/s]␍JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 1.45kernel/s, ComputeSlotMappingKernel (1 keys)]␍JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 1.45kernel/s, ComputeSlotMappingKernel (1 keys)]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) INFO 09-23 14:38:37 [kernel_warmup.py:134] JIT kernel warmup finished in 0.69s.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) WARNING 09-23 14:38:37 [import_utils.py:408] Module vllm.third_party.deep_gemm was found but failed to import
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) WARNING 09-23 14:38:37 [import_utils.py:408] Traceback (most recent call last):
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) WARNING 09-23 14:38:37 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/utils/import_utils.py", line 406, in _has_module
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) WARNING 09-23 14:38:37 [import_utils.py:408] importlib.import_module(module_name)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) WARNING 09-23 14:38:37 [import_utils.py:408] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/importlib/__init__.py", line 90, in import_module
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) WARNING 09-23 14:38:37 [import_utils.py:408] return _bootstrap._gcd_import(name[level:], package, level)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) WARNING 09-23 14:38:37 [import_utils.py:408] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) WARNING 09-23 14:38:37 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1387, in _gcd_import
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) WARNING 09-23 14:38:37 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1360, in _find_and_load
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) WARNING 09-23 14:38:37 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1331, in _find_and_load_unlocked
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) WARNING 09-23 14:38:37 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 935, in _load_unlocked
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) WARNING 09-23 14:38:37 [import_utils.py:408] File "<frozen importlib._bootstrap_external>", line 999, in exec_module
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) WARNING 09-23 14:38:37 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 488, in _call_with_frames_removed
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) WARNING 09-23 14:38:37 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/third_party/deep_gemm/__init__.py", line 126, in <module>
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) WARNING 09-23 14:38:37 [import_utils.py:408] _find_cuda_home() # CUDA home
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) WARNING 09-23 14:38:37 [import_utils.py:408] ^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) WARNING 09-23 14:38:37 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/third_party/deep_gemm/__init__.py", line 120, in _find_cuda_home
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) WARNING 09-23 14:38:37 [import_utils.py:408] assert cuda_home is not None
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) WARNING 09-23 14:38:37 [import_utils.py:408] ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) WARNING 09-23 14:38:37 [import_utils.py:408] AssertionError
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) INFO 09-23 14:38:37 [gpu_worker.py:860] Free memory on device (22.58/23.89 GiB) on startup. Desired GPU memory utilization is (0.58, 13.86 GiB). Actual usage is 11.01 GiB for consumed memory (weights + non-torch), 0.22 GiB for peak activation, and 0.0 GiB for CUDAGraph memory. Replace gpu_memory_utilization config with `--kv-cache-memory=2667877499` (2.48 GiB) to fit into requested memory, or `--kv-cache-memory=12032409600` (11.21 GiB) to fully utilize gpu memory. Current kv cache memory in use is 2.72 GiB.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) INFO 09-23 14:38:38 [jit_monitor.py:85] Kernel JIT monitor activated; monitored JIT compilations during inference will use mode=warn.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) INFO 09-23 14:38:38 [torch_utils.py:277] Reducing Torch threads from 12 to 1 for serving. Set OMP_NUM_THREADS in the external environment to override.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) INFO 09-23 14:38:38 [core.py:368] init engine (profile, create kv cache, warmup model) took 13.15 s
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) WARNING 09-23 14:38:38 [scheduler.py:213] Using custom scheduler class vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) WARNING 09-23 14:38:38 [omni_scheduler_mixin.py:118] OMNI scheduler forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) INFO 09-23 14:38:38 [chunk_transfer_adapter.py:142] Bounded active-stream window enabled: K=1. Multi-replica deployments require sticky per-stream routing across Stage 1 replicas (each replica owns an independent active-set; without sticky routing, a stream can be active on one replica and non-active on another and both will race to evict it).
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) INFO 09-23 14:38:38 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4156973) INFO 09-23 14:38:38 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-23 14:38:38 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed
|
||||
INFO 09-23 14:38:38 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-0 [rep-0] initializing EngineCore
|
||||
INFO 09-23 14:38:38 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-0 [rep-0] EngineCore running
|
||||
INFO 09-23 14:38:38 [stage_runtime.py:958] [StageRuntime] Stage 0 initialized
|
||||
INFO 09-23 14:38:38 [stage_runtime.py:853] [stage_init] Stage-1 set runtime devices: 0
|
||||
INFO 09-23 14:38:45 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 14:38:46 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 14:38:46 [patch.py:583] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) INFO 09-23 14:38:47 [core.py:123] Initializing a V1 LLM engine (v0.29.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=512, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [512], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 0, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) INFO 09-23 14:38:48 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_7e1227f0df324162a62f71e91ee2fc27 backend=nccl
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) INFO 09-23 14:38:48 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) WARNING 09-23 14:38:48 [gpu_ar_worker.py:120] OMNI GPUARWorker forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) INFO 09-23 14:38:48 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) [transformers] `MiniCPMOProcessor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) [transformers] Requested torchvision backend is not available. Falling back to pil backend.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) [transformers] `MiniCPMOProcessor` defines `feature_extractor_class = 'WhisperFeatureExtractor'`, which is deprecated. Register the correct mapping in `AutoFeatureExtractor` instead.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) [transformers] Image processor MiniCPMVImageProcessor: kwargs ['max_slice_nums'] were applied for backward compatibility. To avoid this warning, add them to valid_kwargs: create a custom TypedDict extending ImagesKwargs with these keys and set it as the `valid_kwargs` class attribute.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) INFO 09-23 14:38:59 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) WARNING 09-23 14:38:59 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) WARNING 09-23 14:38:59 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) INFO 09-23 14:38:59 [gpu_model_runner.py:5379] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5...
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) WARNING 09-23 14:38:59 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) WARNING 09-23 14:38:59 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) INFO 09-23 14:38:59 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) INFO 09-23 14:38:59 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) INFO 09-23 14:38:59 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) INFO 09-23 14:38:59 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) INFO 09-23 14:39:00 [cuda.py:492] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) INFO 09-23 14:39:00 [flash_attn.py:897] Using FlashAttention version 2
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) INFO 09-23 14:39:00 [weight_utils.py:863] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 19.14 GiB.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) INFO 09-23 14:39:00 [weight_utils.py:893] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre) and the checkpoint size (17.46 GiB) exceeds 90% of available RAM (19.14 GiB).
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) ␍Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:00<00:00, 201.38it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) INFO 09-23 14:39:00 [default_loader.py:430] Loading weights took 0.53 seconds
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) INFO 09-23 14:39:01 [gpu_model_runner.py:5471] Model loading took 0.65 GiB memory and 1.446033 seconds
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) INFO 09-23 14:39:02 [utils.py:306] Using LBNHC KV cache layout.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) INFO 09-23 14:39:02 [gpu_model_runner.py:6519] Skipping memory profiling for multimodal encoder and encoder cache.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) INFO 09-23 14:39:03 [kv_cache_utils.py:2032] GPU KV cache size: 4,368 tokens, Maximum concurrency for 512 tokens per request: 8.53x
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) WARNING 09-23 14:39:03 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) INFO 09-23 14:39:03 [kernel_warmup.py:124] JIT kernel warmup starting.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) ␍JIT kernel warmup (1 compile keys): 0%| | 0/1 [00:00<?, ?kernel/s]␍JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 3.95kernel/s, ComputeSlotMappingKernel (1 keys)]␍JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 3.95kernel/s, ComputeSlotMappingKernel (1 keys)]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) INFO 09-23 14:39:03 [kernel_warmup.py:134] JIT kernel warmup finished in 0.25s.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) WARNING 09-23 14:39:03 [import_utils.py:408] Module vllm.third_party.deep_gemm was found but failed to import
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) WARNING 09-23 14:39:03 [import_utils.py:408] Traceback (most recent call last):
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) WARNING 09-23 14:39:03 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/utils/import_utils.py", line 406, in _has_module
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) WARNING 09-23 14:39:03 [import_utils.py:408] importlib.import_module(module_name)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) WARNING 09-23 14:39:03 [import_utils.py:408] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/importlib/__init__.py", line 90, in import_module
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) WARNING 09-23 14:39:03 [import_utils.py:408] return _bootstrap._gcd_import(name[level:], package, level)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) WARNING 09-23 14:39:03 [import_utils.py:408] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) WARNING 09-23 14:39:03 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1387, in _gcd_import
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) WARNING 09-23 14:39:03 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1360, in _find_and_load
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) WARNING 09-23 14:39:03 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1331, in _find_and_load_unlocked
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) WARNING 09-23 14:39:03 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 935, in _load_unlocked
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) WARNING 09-23 14:39:03 [import_utils.py:408] File "<frozen importlib._bootstrap_external>", line 999, in exec_module
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) WARNING 09-23 14:39:03 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 488, in _call_with_frames_removed
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) WARNING 09-23 14:39:03 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/third_party/deep_gemm/__init__.py", line 126, in <module>
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) WARNING 09-23 14:39:03 [import_utils.py:408] _find_cuda_home() # CUDA home
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) WARNING 09-23 14:39:03 [import_utils.py:408] ^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) WARNING 09-23 14:39:03 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/third_party/deep_gemm/__init__.py", line 120, in _find_cuda_home
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) WARNING 09-23 14:39:03 [import_utils.py:408] assert cuda_home is not None
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) WARNING 09-23 14:39:03 [import_utils.py:408] ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) WARNING 09-23 14:39:03 [import_utils.py:408] AssertionError
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) INFO 09-23 14:39:04 [jit_monitor.py:85] Kernel JIT monitor activated; monitored JIT compilations during inference will use mode=warn.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) INFO 09-23 14:39:04 [torch_utils.py:277] Reducing Torch threads from 12 to 1 for serving. Set OMP_NUM_THREADS in the external environment to override.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) INFO 09-23 14:39:04 [core.py:368] init engine (profile, create kv cache, warmup model) took 2.37 s
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) WARNING 09-23 14:39:04 [scheduler.py:213] Using custom scheduler class vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) WARNING 09-23 14:39:04 [omni_scheduler_mixin.py:118] OMNI scheduler forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) INFO 09-23 14:39:04 [chunk_transfer_adapter.py:142] Bounded active-stream window enabled: K=1. Multi-replica deployments require sticky per-stream routing across Stage 1 replicas (each replica owns an independent active-set; without sticky routing, a stream can be active on one replica and non-active on another and both will race to evict it).
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) INFO 09-23 14:39:04 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage1_replica0 pid=4157223) INFO 09-23 14:39:04 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-23 14:39:04 [stage_runtime.py:943] [StageRuntime] Stage 1 engine startup completed
|
||||
INFO 09-23 14:39:04 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-1 [rep-0] initializing EngineCore
|
||||
INFO 09-23 14:39:04 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-1 [rep-0] EngineCore running
|
||||
INFO 09-23 14:39:04 [stage_runtime.py:958] [StageRuntime] Stage 1 initialized
|
||||
INFO 09-23 14:39:04 [stage_runtime.py:853] [stage_init] Stage-2 set runtime devices: 0
|
||||
INFO 09-23 14:39:11 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 14:39:11 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 14:39:11 [patch.py:583] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) INFO 09-23 14:39:12 [core.py:123] Initializing a V1 LLM engine (v0.29.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=4096, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=False, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [4096], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 0, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) INFO 09-23 14:39:12 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_a8132b1d955648dd9d02b133c7b1fe35 backend=nccl
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) INFO 09-23 14:39:12 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) WARNING 09-23 14:39:13 [gpu_generation_worker.py:103] OMNI GPUGenerationWorker forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) INFO 09-23 14:39:13 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) INFO 09-23 14:39:13 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) WARNING 09-23 14:39:13 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) WARNING 09-23 14:39:13 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) INFO 09-23 14:39:13 [gpu_model_runner.py:5379] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5...
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) INFO 09-23 14:39:13 [weight_utils.py:863] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 17.77 GiB.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) INFO 09-23 14:39:13 [weight_utils.py:893] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre) and the checkpoint size (17.46 GiB) exceeds 90% of available RAM (17.77 GiB).
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) ␍Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:00<00:00, 258.04it/s]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) INFO 09-23 14:39:13 [step_audio2_token2wav.py:163] Loading Token2Wav models from: /home/zhout/project/edge_infer/models/MiniCPM-o-4_5/assets/token2wav
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) INFO 09-23 14:39:22 [step_audio2_token2wav.py:201] Token2Wav models loaded successfully
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) INFO 09-23 14:39:23 [batched_token2wav.py:227] Enabling HiFT CUDA Graph with batch sizes [1]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) INFO 09-23 14:39:24 [cuda_graph_wrapper.py:98] Captured HiFT CUDA Graph for shape (1, 50, 0)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) INFO 09-23 14:39:24 [cuda_graph_wrapper.py:98] Captured HiFT CUDA Graph for shape (1, 58, 3840)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) INFO 09-23 14:39:24 [batched_token2wav.py:235] HiFT CUDA Graph captured successfully
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) INFO 09-23 14:39:24 [batched_token2wav.py:246] CFM CUDA Graph enabled (max_graphs=32)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) INFO 09-23 14:39:24 [default_loader.py:430] Loading weights took 10.90 seconds
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) INFO 09-23 14:39:25 [gpu_model_runner.py:5471] Model loading took 3.36 GiB memory and 10.915807 seconds
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) INFO 09-23 14:39:26 [cuda.py:492] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) INFO 09-23 14:39:26 [utils.py:306] Using LBNHC KV cache layout.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) WARNING 09-23 14:39:26 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) INFO 09-23 14:39:26 [kernel_warmup.py:124] JIT kernel warmup starting.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) ␍JIT kernel warmup (1 compile keys): 0%| | 0/1 [00:00<?, ?kernel/s]␍JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 380.44kernel/s, ComputeSlotMappingKernel (1 keys)]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) INFO 09-23 14:39:26 [kernel_warmup.py:134] JIT kernel warmup finished in 0.00s.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) WARNING 09-23 14:39:26 [import_utils.py:408] Module vllm.third_party.deep_gemm was found but failed to import
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) WARNING 09-23 14:39:26 [import_utils.py:408] Traceback (most recent call last):
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) WARNING 09-23 14:39:26 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/utils/import_utils.py", line 406, in _has_module
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) WARNING 09-23 14:39:26 [import_utils.py:408] importlib.import_module(module_name)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) WARNING 09-23 14:39:26 [import_utils.py:408] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/importlib/__init__.py", line 90, in import_module
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) WARNING 09-23 14:39:26 [import_utils.py:408] return _bootstrap._gcd_import(name[level:], package, level)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) WARNING 09-23 14:39:26 [import_utils.py:408] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) WARNING 09-23 14:39:26 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1387, in _gcd_import
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) WARNING 09-23 14:39:26 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1360, in _find_and_load
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) WARNING 09-23 14:39:26 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1331, in _find_and_load_unlocked
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) WARNING 09-23 14:39:26 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 935, in _load_unlocked
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) WARNING 09-23 14:39:26 [import_utils.py:408] File "<frozen importlib._bootstrap_external>", line 999, in exec_module
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) WARNING 09-23 14:39:26 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 488, in _call_with_frames_removed
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) WARNING 09-23 14:39:26 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/third_party/deep_gemm/__init__.py", line 126, in <module>
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) WARNING 09-23 14:39:26 [import_utils.py:408] _find_cuda_home() # CUDA home
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) WARNING 09-23 14:39:26 [import_utils.py:408] ^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) WARNING 09-23 14:39:26 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/third_party/deep_gemm/__init__.py", line 120, in _find_cuda_home
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) WARNING 09-23 14:39:26 [import_utils.py:408] assert cuda_home is not None
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) WARNING 09-23 14:39:26 [import_utils.py:408] ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) WARNING 09-23 14:39:26 [import_utils.py:408] AssertionError
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) WARNING 09-23 14:39:26 [gpu_generation_model_runner.py:558] Dummy sampler run is not implemented for generation model
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) INFO 09-23 14:39:27 [jit_monitor.py:85] Kernel JIT monitor activated; monitored JIT compilations during inference will use mode=warn.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) INFO 09-23 14:39:28 [torch_utils.py:277] Reducing Torch threads from 12 to 1 for serving. Set OMP_NUM_THREADS in the external environment to override.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) INFO 09-23 14:39:28 [core.py:368] init engine (profile, create kv cache, warmup model) took 2.58 s
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) WARNING 09-23 14:39:28 [scheduler.py:213] Using custom scheduler class vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) WARNING 09-23 14:39:28 [omni_scheduler_mixin.py:118] OMNI scheduler forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) INFO 09-23 14:39:28 [chunk_transfer_adapter.py:142] Bounded active-stream window enabled: K=1. Multi-replica deployments require sticky per-stream routing across Stage 1 replicas (each replica owns an independent active-set; without sticky routing, a stream can be active on one replica and non-active on another and both will race to evict it).
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) INFO 09-23 14:39:28 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) WARNING 09-23 14:39:28 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) WARNING 09-23 14:39:28 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) INFO 09-23 14:39:28 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) INFO 09-23 14:39:28 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
(StageEngineCoreProc_stage2_replica0 pid=4157481) INFO 09-23 14:39:28 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
INFO 09-23 14:39:28 [stage_runtime.py:943] [StageRuntime] Stage 2 engine startup completed
|
||||
INFO 09-23 14:39:28 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-2 [rep-0] initializing EngineCore
|
||||
INFO 09-23 14:39:29 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-2 [rep-0] EngineCore running
|
||||
INFO 09-23 14:39:29 [stage_runtime.py:958] [StageRuntime] Stage 2 initialized
|
||||
INFO 09-23 14:39:31 [orchestrator.py:604] [Orchestrator] Starting event loop
|
||||
INFO 09-23 14:39:31 [async_omni_engine.py:323] [AsyncOmniEngine] Orchestrator ready with 3 stages
|
||||
INFO 09-23 14:39:31 [omni_base.py:235] [Omni] AsyncOmniEngine initialized in 99.15 seconds
|
||||
INFO 09-23 14:39:31 [omni_base.py:255] [Omni] Initialized with 3 stages for model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5
|
||||
WARNING 09-23 14:39:31 [input_processor.py:315] Passing raw prompts to InputProcessor is deprecated and will be removed in the future. You should instead pass the outputs of Renderer.render_cmpl() or Renderer.render_chat().
|
||||
␍Processed prompts: 0%| | 0/1 [00:00<?, ?it/s](StageEngineCoreProc_stage0_replica0 pid=4156973) WARNING 09-23 14:39:31 [gpu_ar_model_runner.py:1461] prefer_model_sampler model MiniCPMO45OmniForConditionalGeneration returned None from sample(); falling back to the default sampler.
|
||||
␍Processed prompts: 100%|██████████| 1/1 [00:30<00:00, 30.12s/it]␍Processed prompts: 100%|██████████| 1/1 [00:30<00:00, 30.12s/it]
|
||||
query type: text
|
||||
use_tts (tts_bos): False
|
||||
Request ID: 0_de28fb29-b5a4-4f2c-b908-a0bcacfe063c, Text saved to /home/zhout/project/edge_infer/analysis/experiments/e2e_recovery_20260923/mini_output_text_only/0_de28fb29-b5a4-4f2c-b908-a0bcacfe063c.txt
|
||||
INFO 09-23 14:40:01 [omni_base.py:781] [Omni] Shutting down
|
||||
INFO 09-23 14:40:01 [async_omni_engine.py:1807] [AsyncOmniEngine] Shutting down Orchestrator
|
||||
INFO 09-23 14:40:01 [orchestrator.py:705] [Orchestrator] Received shutdown signal
|
||||
INFO 09-23 14:40:01 [orchestrator.py:3092] [Orchestrator] Shutting down all 3 client(s)
|
||||
INFO 09-23 14:40:01 [core_client.py:689] [shutdown] MPClient: start timeout=default
|
||||
INFO 09-23 14:40:01 [core_client.py:691] [shutdown] MPClient: stopping engine manager
|
||||
INFO 09-23 14:40:01 [utils.py:620] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage0_replica0
|
||||
INFO 09-23 14:40:07 [core_client.py:693] [shutdown] MPClient: engine manager stopped
|
||||
INFO 09-23 14:40:07 [core_client.py:694] [shutdown] MPClient: cleaning up background resources
|
||||
INFO 09-23 14:40:07 [core_client.py:696] [shutdown] MPClient: complete
|
||||
INFO 09-23 14:40:07 [stage_pool.py:1354] [StagePool] Stage 0 replica 0 shut down
|
||||
INFO 09-23 14:40:07 [core_client.py:689] [shutdown] MPClient: start timeout=default
|
||||
INFO 09-23 14:40:07 [utils.py:620] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage1_replica0
|
||||
INFO 09-23 14:40:11 [stage_pool.py:1354] [StagePool] Stage 1 replica 0 shut down
|
||||
INFO 09-23 14:40:11 [core_client.py:689] [shutdown] MPClient: start timeout=default
|
||||
INFO 09-23 14:40:11 [utils.py:620] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage2_replica0
|
||||
INFO 09-23 14:40:15 [stage_pool.py:1354] [StagePool] Stage 2 replica 0 shut down
|
||||
/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/multiprocessing/resource_tracker.py:279: UserWarning: resource_tracker: There appear to be 1 leaked shared_memory objects to clean up at shutdown
|
||||
warnings.warn('resource_tracker: There appear to be %d '
|
||||
+124
@@ -0,0 +1,124 @@
|
||||
INFO 09-23 15:01:56 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 15:01:56 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 15:01:56 [patch.py:583] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
|
||||
====================
|
||||
vllm version: 0.29.0
|
||||
====================
|
||||
INFO 09-23 15:01:57 [omni_base.py:216] [Omni] Initializing with model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5
|
||||
INFO 09-23 15:01:57 [async_omni_engine.py:172] [AsyncOmniEngine] Initializing with model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5
|
||||
INFO 09-23 15:01:57 [async_omni_engine.py:290] [AsyncOmniEngine] Launching Orchestrator thread with 1 stages
|
||||
INFO 09-23 15:01:57 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
INFO 09-23 15:01:57 [initialization.py:356] Loaded OmniTransferConfig with 0 connector configurations
|
||||
WARNING 09-23 15:01:57 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 15:01:57 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
WARNING 09-23 15:01:57 [envs.py:2248] Unknown vLLM environment variable detected: VLLM_OMNI_MINICPMO_LOGITS_PROBE
|
||||
INFO 09-23 15:02:07 [model.py:684] Resolved architecture: MiniCPMO45OmniForConditionalGeneration
|
||||
INFO 09-23 15:02:07 [model.py:2021] Using max model len 2048
|
||||
WARNING 09-23 15:02:07 [model.py:981] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False.
|
||||
INFO 09-23 15:02:07 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode.
|
||||
INFO 09-23 15:02:07 [model.py:887] Disabled mm_prefix attention mode because multimodal inputs are configuration-disabled. Attention backends without mm_prefix support may now be selected.
|
||||
INFO 09-23 15:02:07 [scheduler.py:277] Chunked prefill is enabled with max_num_batched_tokens=2048.
|
||||
WARNING 09-23 15:02:07 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
WARNING 09-23 15:02:07 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
INFO 09-23 15:02:07 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-23 15:02:07 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
INFO 09-23 15:02:07 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
INFO 09-23 15:02:07 [stage_runtime.py:853] [stage_init] Stage-0 set runtime devices: 0
|
||||
INFO 09-23 15:02:14 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 15:02:14 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 15:02:14 [patch.py:583] [cumem-cuda] CuMemAllocator._python_free_callback patched: asleep guard extended to all platforms.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) INFO 09-23 15:02:15 [core.py:123] Initializing a V1 LLM engine (v0.29.0) with config: model='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', speculative_config=None, tokenizer='/home/zhout/project/edge_infer/models/MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=2048, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=/home/zhout/project/edge_infer/models/MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 0, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) INFO 09-23 15:02:15 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) INFO 09-23 15:02:15 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file:///tmp/vllm_dist_96bb721bd42747ccbd1599c4212953bb backend=nccl
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) INFO 09-23 15:02:15 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) WARNING 09-23 15:02:16 [gpu_ar_worker.py:120] OMNI GPUARWorker forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) INFO 09-23 15:02:16 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) WARNING 09-23 15:02:16 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) WARNING 09-23 15:02:16 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) INFO 09-23 15:02:16 [gpu_model_runner.py:5379] Starting to load model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5...
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) WARNING 09-23 15:02:16 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) WARNING 09-23 15:02:16 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) INFO 09-23 15:02:16 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) INFO 09-23 15:02:16 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) INFO 09-23 15:02:16 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) INFO 09-23 15:02:16 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) INFO 09-23 15:02:16 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) INFO 09-23 15:02:17 [cuda.py:492] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'FLASHINFER', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) INFO 09-23 15:02:17 [flash_attn.py:897] Using FlashAttention version 2
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) INFO 09-23 15:02:17 [weight_utils.py:863] Filesystem type for checkpoints: EXT4. Checkpoint size: 17.46 GiB. Available RAM: 27.27 GiB.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) INFO 09-23 15:02:17 [weight_utils.py:886] Auto-prefetch is disabled because the filesystem (EXT4) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) ␍Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) ␍Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:00<00:00, 121.68it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) INFO 09-23 15:02:35 [default_loader.py:430] Loading weights took 17.72 seconds
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) INFO 09-23 15:02:36 [gpu_model_runner.py:5471] Model loading took 15.88 GiB memory and 18.899084 seconds
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) INFO 09-23 15:02:37 [utils.py:306] Using LBNHC KV cache layout.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) WARNING 09-23 15:02:38 [minicpmo_4_5_omni.py:571] MiniCPM-o forward probe 1: ids_shape=(2048,) ids_prefix=[0, 0, 0, 0, 0] embeds_shape=(2048, 4096) embeds_abs_mean=0.0 hidden_abs_mean=0.0 positions=[0, 0, 0, 0, 0]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) WARNING 09-23 15:02:38 [minicpmo_4_5_omni.py:671] MiniCPM-o logits probe 1: shape=(1, 151748) finite=True top_ids=[625, 9342, 16400, 8983, 8645] top_values=[8.6875, 8.4375, 8.3125, 8.3125, 8.3125] hidden_abs_mean=0.490234375 head_abs_mean=0.0286865234375
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) INFO 09-23 15:02:40 [base.py:148] Available KV cache memory: 3.15 GiB (device-level profiling)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) INFO 09-23 15:02:40 [kv_cache_utils.py:2032] GPU KV cache size: 22,912 tokens, Maximum concurrency for 2,048 tokens per request: 11.19x
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) WARNING 09-23 15:02:40 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) INFO 09-23 15:02:40 [kernel_warmup.py:124] JIT kernel warmup starting.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) ␍JIT kernel warmup (1 compile keys): 0%| | 0/1 [00:00<?, ?kernel/s]␍JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 1.35kernel/s, ComputeSlotMappingKernel (1 keys)]␍JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 1.35kernel/s, ComputeSlotMappingKernel (1 keys)]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) INFO 09-23 15:02:41 [kernel_warmup.py:134] JIT kernel warmup finished in 0.74s.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) WARNING 09-23 15:02:41 [import_utils.py:408] Module vllm.third_party.deep_gemm was found but failed to import
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) WARNING 09-23 15:02:41 [import_utils.py:408] Traceback (most recent call last):
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) WARNING 09-23 15:02:41 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/utils/import_utils.py", line 406, in _has_module
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) WARNING 09-23 15:02:41 [import_utils.py:408] importlib.import_module(module_name)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) WARNING 09-23 15:02:41 [import_utils.py:408] File "/home/zhout/.local/share/uv/python/cpython-3.12.13-linux-x86_64-gnu/lib/python3.12/importlib/__init__.py", line 90, in import_module
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) WARNING 09-23 15:02:41 [import_utils.py:408] return _bootstrap._gcd_import(name[level:], package, level)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) WARNING 09-23 15:02:41 [import_utils.py:408] ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) WARNING 09-23 15:02:41 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1387, in _gcd_import
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) WARNING 09-23 15:02:41 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1360, in _find_and_load
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) WARNING 09-23 15:02:41 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 1331, in _find_and_load_unlocked
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) WARNING 09-23 15:02:41 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 935, in _load_unlocked
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) WARNING 09-23 15:02:41 [import_utils.py:408] File "<frozen importlib._bootstrap_external>", line 999, in exec_module
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) WARNING 09-23 15:02:41 [import_utils.py:408] File "<frozen importlib._bootstrap>", line 488, in _call_with_frames_removed
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) WARNING 09-23 15:02:41 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/third_party/deep_gemm/__init__.py", line 126, in <module>
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) WARNING 09-23 15:02:41 [import_utils.py:408] _find_cuda_home() # CUDA home
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) WARNING 09-23 15:02:41 [import_utils.py:408] ^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) WARNING 09-23 15:02:41 [import_utils.py:408] File "/home/zhout/project/edge_infer/.venvs/omni-cuda-029/lib/python3.12/site-packages/vllm/third_party/deep_gemm/__init__.py", line 120, in _find_cuda_home
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) WARNING 09-23 15:02:41 [import_utils.py:408] assert cuda_home is not None
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) WARNING 09-23 15:02:41 [import_utils.py:408] ^^^^^^^^^^^^^^^^^^^^^
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) WARNING 09-23 15:02:41 [import_utils.py:408] AssertionError
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) INFO 09-23 15:02:41 [gpu_worker.py:860] Free memory on device (22.58/23.89 GiB) on startup. Desired GPU memory utilization is (0.85, 20.31 GiB). Actual usage is 16.04 GiB for consumed memory (weights + non-torch), 1.21 GiB for peak activation, and 0.0 GiB for CUDAGraph memory. Replace gpu_memory_utilization config with `--kv-cache-memory=3131941786` (2.92 GiB) to fit into requested memory, or `--kv-cache-memory=5570742784` (5.19 GiB) to fully utilize gpu memory. Current kv cache memory in use is 3.15 GiB.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) WARNING 09-23 15:02:41 [minicpmo_4_5_omni.py:571] MiniCPM-o forward probe 2: ids_shape=(1,) ids_prefix=[0] embeds_shape=(1, 4096) embeds_abs_mean=0.0 hidden_abs_mean=0.0 positions=[0]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) WARNING 09-23 15:02:42 [minicpmo_4_5_omni.py:671] MiniCPM-o logits probe 2: shape=(1, 151748) finite=True top_ids=[6682, 14952, 1895, 8844, 47739] top_values=[7.90625, 7.875, 7.8125, 7.75, 7.71875] hidden_abs_mean=0.498046875 head_abs_mean=0.0286865234375
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) INFO 09-23 15:02:42 [jit_monitor.py:85] Kernel JIT monitor activated; monitored JIT compilations during inference will use mode=warn.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) INFO 09-23 15:02:43 [torch_utils.py:277] Reducing Torch threads from 12 to 1 for serving. Set OMP_NUM_THREADS in the external environment to override.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) INFO 09-23 15:02:43 [core.py:368] init engine (profile, create kv cache, warmup model) took 6.21 s
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) WARNING 09-23 15:02:43 [scheduler.py:213] Using custom scheduler class vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) WARNING 09-23 15:02:43 [omni_scheduler_mixin.py:118] OMNI scheduler forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) INFO 09-23 15:02:43 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-23 15:02:43 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed
|
||||
INFO 09-23 15:02:43 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-0 [rep-0] initializing EngineCore
|
||||
INFO 09-23 15:02:43 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-0 [rep-0] EngineCore running
|
||||
INFO 09-23 15:02:43 [stage_runtime.py:958] [StageRuntime] Stage 0 initialized
|
||||
INFO 09-23 15:02:45 [orchestrator.py:604] [Orchestrator] Starting event loop
|
||||
INFO 09-23 15:02:45 [async_omni_engine.py:323] [AsyncOmniEngine] Orchestrator ready with 1 stages
|
||||
INFO 09-23 15:02:45 [omni_base.py:235] [Omni] AsyncOmniEngine initialized in 48.07 seconds
|
||||
INFO 09-23 15:02:45 [omni_base.py:255] [Omni] Initialized with 1 stages for model /home/zhout/project/edge_infer/models/MiniCPM-o-4_5
|
||||
WARNING 09-23 15:02:45 [input_processor.py:315] Passing raw prompts to InputProcessor is deprecated and will be removed in the future. You should instead pass the outputs of Renderer.render_cmpl() or Renderer.render_chat().
|
||||
␍Processed prompts: 0%| | 0/1 [00:00<?, ?it/s](StageEngineCoreProc_stage0_replica0 pid=4163207) WARNING 09-23 15:02:45 [minicpmo_4_5_omni.py:571] MiniCPM-o forward probe 3: ids_shape=(60,) ids_prefix=[151644, 8948, 198, 2610, 525] embeds_shape=(60, 4096) embeds_abs_mean=10.25 hidden_abs_mean=1.03125 positions=[0, 1, 2, 3, 4]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) WARNING 09-23 15:02:45 [minicpmo_4_5_omni.py:671] MiniCPM-o logits probe 3: shape=(1, 151748) finite=True top_ids=[9707, 108386, 14990, 13048, 50712] top_values=[24.5, 20.0, 20.0, 17.875, 16.125] hidden_abs_mean=0.8828125 head_abs_mean=0.0286865234375
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) WARNING 09-23 15:02:45 [gpu_ar_model_runner.py:1450] MiniCPM-o before custom bias: top_id=9707 top_value=24.5 sampler=Sampler
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) WARNING 09-23 15:02:45 [gpu_ar_model_runner.py:1489] MiniCPM-o after custom bias: top_id=9707 top_value=24.5
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) WARNING 09-23 15:02:45 [minicpmo_4_5_omni.py:713] MiniCPM-o pre-sampler: top_id=9707 top_value=24.5 allowed_mask_shape=None allowed_count=None temperature=None all_greedy=True
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) WARNING 09-23 15:02:45 [gpu_ar_model_runner.py:1501] prefer_model_sampler model MiniCPMO45OmniForConditionalGeneration returned None from sample(); falling back to the default sampler.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) WARNING 09-23 15:02:45 [minicpmo_4_5_omni.py:571] MiniCPM-o forward probe 4: ids_shape=(1,) ids_prefix=[9707] embeds_shape=(1, 4096) embeds_abs_mean=17.5 hidden_abs_mean=1.2578125 positions=[60]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=4163207) WARNING 09-23 15:02:45 [minicpmo_4_5_omni.py:671] MiniCPM-o logits probe 4: shape=(1, 151748) finite=True top_ids=[0, 11, 13, 6313, 2219] top_values=[38.0, 36.5, 33.0, 31.875, 31.75] hidden_abs_mean=1.2578125 head_abs_mean=0.0286865234375
|
||||
␍Processed prompts: 100%|██████████| 1/1 [00:01<00:00, 1.47s/it]␍Processed prompts: 100%|██████████| 1/1 [00:01<00:00, 1.47s/it]
|
||||
query type: text
|
||||
use_tts (tts_bos): False
|
||||
Request ID: 0_7ef249da-0588-427b-b26d-5fb7906d774c, Text saved to /home/zhout/project/edge_infer/analysis/experiments/e2e_recovery_20260923/mini_output_thinker_fixed/0_7ef249da-0588-427b-b26d-5fb7906d774c.txt
|
||||
INFO 09-23 15:02:47 [omni_base.py:781] [Omni] Shutting down
|
||||
INFO 09-23 15:02:47 [async_omni_engine.py:1807] [AsyncOmniEngine] Shutting down Orchestrator
|
||||
INFO 09-23 15:02:47 [orchestrator.py:705] [Orchestrator] Received shutdown signal
|
||||
INFO 09-23 15:02:47 [orchestrator.py:3092] [Orchestrator] Shutting down all 1 client(s)
|
||||
INFO 09-23 15:02:47 [core_client.py:689] [shutdown] MPClient: start timeout=default
|
||||
INFO 09-23 15:02:47 [core_client.py:691] [shutdown] MPClient: stopping engine manager
|
||||
INFO 09-23 15:02:47 [utils.py:620] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage0_replica0
|
||||
INFO 09-23 15:02:51 [core_client.py:693] [shutdown] MPClient: engine manager stopped
|
||||
INFO 09-23 15:02:51 [core_client.py:694] [shutdown] MPClient: cleaning up background resources
|
||||
INFO 09-23 15:02:51 [core_client.py:696] [shutdown] MPClient: complete
|
||||
INFO 09-23 15:02:51 [stage_pool.py:1354] [StagePool] Stage 0 replica 0 shut down
|
||||
+126
@@ -0,0 +1,126 @@
|
||||
# MiniCPM-o 4.5 offline text-to-text+speech probe on RTX 5090 Laptop.
|
||||
# This limits context, concurrency and multimodal inputs and offloads 5 GiB.
|
||||
# Duplex session controls are omitted for ordinary offline requests.
|
||||
pipeline: minicpmo_4_5
|
||||
async_chunk: true
|
||||
|
||||
trust_remote_code: true
|
||||
enable_prefix_caching: false
|
||||
|
||||
connectors:
|
||||
connector_of_shared_memory:
|
||||
name: SharedMemoryConnector
|
||||
extra:
|
||||
connector_get_sleep_s: 0.01
|
||||
connector_get_max_wait_first_chunk: 3000
|
||||
connector_get_max_wait: 300
|
||||
codec_chunk_frames: 25
|
||||
codec_left_context_frames: 3
|
||||
# HiFT capture buckets are derived from codec_chunk_frames and
|
||||
# codec_left_context_frames. (#5869)
|
||||
# A future initial_codec_chunk_frames shape will use lazy capture unless
|
||||
# HiFTGraphWrapper.derive_capture_bucket_size explicitly includes it.
|
||||
enable_hift_graph: true
|
||||
enable_cfm_graph: true
|
||||
cfm_max_graphs: 32
|
||||
|
||||
stages:
|
||||
- stage_id: 0
|
||||
max_num_seqs: 1
|
||||
# Bound the thinker budget and offload weights for GPU co-residency.
|
||||
gpu_memory_utilization: 0.58
|
||||
cpu_offload_gb: 5
|
||||
enforce_eager: true
|
||||
tensor_parallel_size: 1
|
||||
max_num_batched_tokens: 2048
|
||||
max_model_len: 2048
|
||||
devices: "0"
|
||||
# Present in the parent profile; this probe submits text only.
|
||||
interleave_mm_strings: true
|
||||
media_io_kwargs:
|
||||
video:
|
||||
fps: 1
|
||||
num_frames: 128
|
||||
limit_mm_per_prompt:
|
||||
# No image, audio or video inputs are admitted in this text-to-speech probe.
|
||||
image: 0
|
||||
audio: 0
|
||||
video: 0
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 64
|
||||
seed: 42
|
||||
detokenize: true
|
||||
# 1.0 for MCQ accuracy; 1.1 can bias short A–D answers.
|
||||
repetition_penalty: 1.0
|
||||
|
||||
- stage_id: 1
|
||||
max_num_seqs: 1
|
||||
# One short utterance; this does not qualify the full 4096-token Talker.
|
||||
gpu_memory_utilization: 0.10
|
||||
enforce_eager: true
|
||||
max_num_batched_tokens: 512
|
||||
# The checkpoint supports 4096, but this probe explicitly admits 512.
|
||||
max_model_len: 512
|
||||
devices: "0"
|
||||
output_connectors:
|
||||
to_stage_2: connector_of_shared_memory
|
||||
# Talker is a single-vocab codec LM. These knobs go to vLLM's Sampler
|
||||
# (head_code logits). EOS is tts_config.eos_token_id (num_audio_tokens-1).
|
||||
# Values track upstream utils.TTSSamplingParams (min_p/win_size/tau_r are
|
||||
# declared there but never read, so they stay out). repetition_penalty is
|
||||
# consumed by the Talker, which rescores it over upstream's 16-frame window
|
||||
# instead of the Sampler's whole-stream presence penalty.
|
||||
# min_tokens is upstream's min_new_token.
|
||||
default_sampling_params:
|
||||
temperature: 0.8
|
||||
top_p: 0.85
|
||||
top_k: 25
|
||||
repetition_penalty: 1.05
|
||||
min_tokens: 50
|
||||
max_tokens: 128
|
||||
seed: 42
|
||||
detokenize: false
|
||||
|
||||
- stage_id: 2
|
||||
max_num_seqs: 1
|
||||
gpu_memory_utilization: 0.10
|
||||
enforce_eager: true
|
||||
enable_chunked_prefill: false
|
||||
max_num_batched_tokens: 4096
|
||||
max_model_len: 4096
|
||||
devices: "0"
|
||||
input_connectors:
|
||||
from_stage_1: connector_of_shared_memory
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 4096
|
||||
detokenize: true
|
||||
|
||||
platforms:
|
||||
cuda:
|
||||
stages:
|
||||
- stage_id: 1
|
||||
# The cap must still admit the single 512-token Talker session.
|
||||
kv_cache_memory_bytes: 268435456 # 256 MiB for the 512-token probe
|
||||
|
||||
npu:
|
||||
stages:
|
||||
- stage_id: 0
|
||||
max_num_batched_tokens: 8192
|
||||
compilation_config:
|
||||
cudagraph_mode: PIECEWISE
|
||||
- stage_id: 1
|
||||
max_num_batched_tokens: 8192
|
||||
compilation_config:
|
||||
cudagraph_mode: PIECEWISE
|
||||
- stage_id: 2
|
||||
# Stage 2 stays outer-eager and captures only exact-shape CFM DiT
|
||||
# estimator kernels through the NPU platform patch.
|
||||
additional_config:
|
||||
code2wav_enable_npu_graph: true
|
||||
code2wav_max_npu_graphs: 32
|
||||
+135
@@ -0,0 +1,135 @@
|
||||
# MiniCPM-o 4.5 single-request text-to-text+speech probe on the 24 GiB
|
||||
# RTX 5090 Laptop with 4 GiB occupied by the host display. This explicitly
|
||||
# reduces context, concurrency and multimodal input limits and offloads 5 GiB.
|
||||
pipeline: minicpmo_4_5
|
||||
async_chunk: true
|
||||
# Native full-duplex shares this profile with /v1/chat/completions. Every other
|
||||
# duplex_session knob keeps its runtime default.
|
||||
session_mode: duplex
|
||||
# Keep the active window aligned with the session and stage capacities below.
|
||||
# A smaller window serializes first-packet generation for concurrent chat turns.
|
||||
active_stream_window: 1
|
||||
duplex_session:
|
||||
# Matches Stage 0/1 max_num_seqs below.
|
||||
max_sessions: 1
|
||||
|
||||
trust_remote_code: true
|
||||
enable_prefix_caching: false
|
||||
|
||||
connectors:
|
||||
connector_of_shared_memory:
|
||||
name: SharedMemoryConnector
|
||||
extra:
|
||||
connector_get_sleep_s: 0.01
|
||||
connector_get_max_wait_first_chunk: 3000
|
||||
connector_get_max_wait: 300
|
||||
codec_chunk_frames: 25
|
||||
codec_left_context_frames: 3
|
||||
# HiFT capture buckets are derived from codec_chunk_frames and
|
||||
# codec_left_context_frames. (#5869)
|
||||
# A future initial_codec_chunk_frames shape will use lazy capture unless
|
||||
# HiFTGraphWrapper.derive_capture_bucket_size explicitly includes it.
|
||||
enable_hift_graph: true
|
||||
enable_cfm_graph: true
|
||||
cfm_max_graphs: 32
|
||||
|
||||
stages:
|
||||
- stage_id: 0
|
||||
max_num_seqs: 1
|
||||
# Bound the thinker budget and offload weights for GPU co-residency.
|
||||
gpu_memory_utilization: 0.58
|
||||
cpu_offload_gb: 5
|
||||
enforce_eager: true
|
||||
tensor_parallel_size: 1
|
||||
max_num_batched_tokens: 2048
|
||||
max_model_len: 2048
|
||||
devices: "0"
|
||||
# Present in the parent profile; this probe submits text only.
|
||||
interleave_mm_strings: true
|
||||
media_io_kwargs:
|
||||
video:
|
||||
fps: 1
|
||||
num_frames: 128
|
||||
limit_mm_per_prompt:
|
||||
# No image, audio or video inputs are admitted in this text-to-speech probe.
|
||||
image: 0
|
||||
audio: 0
|
||||
video: 0
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 64
|
||||
seed: 42
|
||||
detokenize: true
|
||||
# 1.0 for MCQ accuracy; 1.1 can bias short A–D answers.
|
||||
repetition_penalty: 1.0
|
||||
|
||||
- stage_id: 1
|
||||
max_num_seqs: 1
|
||||
# One short utterance; this does not qualify the full 4096-token Talker.
|
||||
gpu_memory_utilization: 0.10
|
||||
enforce_eager: true
|
||||
max_num_batched_tokens: 512
|
||||
# The checkpoint supports 4096, but this probe explicitly admits 512.
|
||||
max_model_len: 512
|
||||
devices: "0"
|
||||
output_connectors:
|
||||
to_stage_2: connector_of_shared_memory
|
||||
# Talker is a single-vocab codec LM. These knobs go to vLLM's Sampler
|
||||
# (head_code logits). EOS is tts_config.eos_token_id (num_audio_tokens-1).
|
||||
# Values track upstream utils.TTSSamplingParams (min_p/win_size/tau_r are
|
||||
# declared there but never read, so they stay out). repetition_penalty is
|
||||
# consumed by the Talker, which rescores it over upstream's 16-frame window
|
||||
# instead of the Sampler's whole-stream presence penalty.
|
||||
# min_tokens is upstream's min_new_token.
|
||||
default_sampling_params:
|
||||
temperature: 0.8
|
||||
top_p: 0.85
|
||||
top_k: 25
|
||||
repetition_penalty: 1.05
|
||||
min_tokens: 50
|
||||
max_tokens: 128
|
||||
seed: 42
|
||||
detokenize: false
|
||||
|
||||
- stage_id: 2
|
||||
max_num_seqs: 1
|
||||
gpu_memory_utilization: 0.10
|
||||
enforce_eager: true
|
||||
enable_chunked_prefill: false
|
||||
max_num_batched_tokens: 4096
|
||||
max_model_len: 4096
|
||||
devices: "0"
|
||||
input_connectors:
|
||||
from_stage_1: connector_of_shared_memory
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 4096
|
||||
detokenize: true
|
||||
|
||||
platforms:
|
||||
cuda:
|
||||
stages:
|
||||
- stage_id: 1
|
||||
# The cap must still admit the single 512-token Talker session.
|
||||
kv_cache_memory_bytes: 268435456 # 256 MiB for the 512-token probe
|
||||
|
||||
npu:
|
||||
stages:
|
||||
- stage_id: 0
|
||||
max_num_batched_tokens: 8192
|
||||
compilation_config:
|
||||
cudagraph_mode: PIECEWISE
|
||||
- stage_id: 1
|
||||
max_num_batched_tokens: 8192
|
||||
compilation_config:
|
||||
cudagraph_mode: PIECEWISE
|
||||
- stage_id: 2
|
||||
# Stage 2 stays outer-eager and captures only exact-shape CFM DiT
|
||||
# estimator kernels through the NPU platform patch.
|
||||
additional_config:
|
||||
code2wav_enable_npu_graph: true
|
||||
code2wav_max_npu_graphs: 32
|
||||
+56
@@ -0,0 +1,56 @@
|
||||
# MiniCPM-o 4.5 one-stage thinker diagnostic on the RTX 5090 Laptop.
|
||||
# The probe registers a text-only PipelineConfig before creating Omni.
|
||||
# No thinker weights are offloaded; context, concurrency and input modalities
|
||||
# are bounded. This is a diagnosis, not the three-stage speech configuration.
|
||||
pipeline: minicpmo_4_5
|
||||
trust_remote_code: true
|
||||
enable_prefix_caching: false
|
||||
|
||||
connectors:
|
||||
connector_of_shared_memory:
|
||||
name: SharedMemoryConnector
|
||||
extra:
|
||||
connector_get_sleep_s: 0.01
|
||||
connector_get_max_wait_first_chunk: 3000
|
||||
connector_get_max_wait: 300
|
||||
codec_chunk_frames: 25
|
||||
codec_left_context_frames: 3
|
||||
# HiFT capture buckets are derived from codec_chunk_frames and
|
||||
# codec_left_context_frames. (#5869)
|
||||
# A future initial_codec_chunk_frames shape will use lazy capture unless
|
||||
# HiFTGraphWrapper.derive_capture_bucket_size explicitly includes it.
|
||||
enable_hift_graph: true
|
||||
enable_cfm_graph: true
|
||||
cfm_max_graphs: 32
|
||||
|
||||
stages:
|
||||
- stage_id: 0
|
||||
max_num_seqs: 1
|
||||
# Bound the thinker budget while keeping its weights resident.
|
||||
gpu_memory_utilization: 0.85
|
||||
cpu_offload_gb: 0
|
||||
enforce_eager: true
|
||||
tensor_parallel_size: 1
|
||||
max_num_batched_tokens: 2048
|
||||
max_model_len: 2048
|
||||
devices: "0"
|
||||
# Present in the parent profile; this probe submits text only.
|
||||
interleave_mm_strings: true
|
||||
media_io_kwargs:
|
||||
video:
|
||||
fps: 1
|
||||
num_frames: 128
|
||||
limit_mm_per_prompt:
|
||||
# No image, audio or video inputs are admitted in this text-to-speech probe.
|
||||
image: 0
|
||||
audio: 0
|
||||
video: 0
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 64
|
||||
seed: 42
|
||||
detokenize: true
|
||||
# 1.0 for MCQ accuracy; 1.1 can bias short A–D answers.
|
||||
repetition_penalty: 1.0
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
+12
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"files": [
|
||||
{
|
||||
"bytes": 6721209208,
|
||||
"file": "model.safetensors",
|
||||
"sha256": "41b0fc1e6772257ab5c47f46338e8438cb8c05ef0a6f2c83123b1a614c397f59"
|
||||
}
|
||||
],
|
||||
"repo": "InternRobotics/InternVLA-A1-3B",
|
||||
"revision": "08cf716d413aa8208a34f346cd8824e6ff901205",
|
||||
"status": "verified"
|
||||
}
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"files": [
|
||||
{
|
||||
"bytes": 6721209208,
|
||||
"file": "model.safetensors",
|
||||
"sha256": "586799539888c69c35f84d083e559cc5f0ba887801d000e70e65f58d3f85c313"
|
||||
}
|
||||
],
|
||||
"repo": "Jia-Zeng/InternVLA-A1-3B-FineTuned-Place_Markpen",
|
||||
"revision": "0a557c6a503e6545b369bff2f34c76a410bbd190",
|
||||
"status": "verified"
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"files": [
|
||||
{
|
||||
"bytes": 5273477136,
|
||||
"file": "model-00001-of-00004.safetensors",
|
||||
"sha256": "30c40b9a10386c1bc404568d8829e5aada2e3501d9de6fb46ff80451aff7e077"
|
||||
},
|
||||
{
|
||||
"bytes": 5301855080,
|
||||
"file": "model-00002-of-00004.safetensors",
|
||||
"sha256": "fe0faef420aca8f771bec1f9fcfaae01206c56b8e64e4429a1ff500c962fbf96"
|
||||
},
|
||||
{
|
||||
"bytes": 5301855048,
|
||||
"file": "model-00003-of-00004.safetensors",
|
||||
"sha256": "5d0b20153f9bfa88ebcd1cb6bbd5b7cac4c217b3cafb137a291a1b380d8b7821"
|
||||
},
|
||||
{
|
||||
"bytes": 2866549964,
|
||||
"file": "model-00004-of-00004.safetensors",
|
||||
"sha256": "f61addf4747c94fedcaee059e5d9918ed15543beec494404139a99f2f86c9b31"
|
||||
}
|
||||
],
|
||||
"repo": "openbmb/MiniCPM-o-4_5",
|
||||
"revision": "503e754207c94da6bb26850b4469f367c9ea3582",
|
||||
"status": "verified"
|
||||
}
|
||||
@@ -7,9 +7,11 @@ every device. The [current profiling review](../../benchmarks/edge_harness/resul
|
||||
records measured execution separately from release qualification.
|
||||
|
||||
The [2026-09-23 recovery run](../../benchmarks/edge_harness/results/e2e_recovery_20260923/README.md)
|
||||
adds a functional CPU TTS stream with playback underruns and a scoped RTX
|
||||
Qwen3.8 text/image pass. Its missing-checkpoint transfers and remaining device
|
||||
gates are tracked there; the 2026-09-22 audit remains the historical baseline.
|
||||
adds a functional CPU TTS stream with playback underruns, a scoped RTX
|
||||
Qwen3.8 text/image pass, verified MiniCPM-o and InternVLA weights, a MiniCPM-o
|
||||
three-stage text-to-speech functional pass after correcting request embeddings,
|
||||
and an InternVLA real-weight synthetic action forward. Model-quality and device gates
|
||||
remain open; the 2026-09-22 audit remains the historical baseline.
|
||||
|
||||
## Architecture and prerequisites
|
||||
|
||||
@@ -33,9 +35,9 @@ Before advancing individual cells:
|
||||
|
||||
1. Pin installed runtime builds and checkpoint revisions. Repair the CPU
|
||||
vLLM/Omni mismatch and native Windows CPU attention-operator gap independently.
|
||||
2. Obtain complete MiniCPM-o 4.5 and InternVLA-A1 artifacts; another model with a
|
||||
similar name is not a substitute. Record export, quantization, calibration,
|
||||
compiler, shape buckets, state layout and quality evidence.
|
||||
2. The complete MiniCPM-o 4.5 and InternVLA-A1 source checkpoints are now
|
||||
SHA-256 verified. Record export, quantization, calibration, compiler, shape
|
||||
buckets, state layout and quality evidence before target-specific claims.
|
||||
3. Integrate native and external stages into the shared admission ledger.
|
||||
CPU/iGPU/NPU share physical RAM; Windows availability and WSL quota are
|
||||
separate constraints on that RAM. Budget loading peaks, weights, state,
|
||||
@@ -66,8 +68,8 @@ capability and lifecycle implementation; a state-handle type alone is insufficie
|
||||
| M1 | S25 Spark | Device-local Omni-compatible controller; embedding, prefill, continuous decode, ring/full KV, head and sampler; resident state | Reference checks, at least 128 output tokens, 512-window/1024-bucket transitions, cancellation, resident memory and sustained profile |
|
||||
| M2 | Qwen3-TTS | Resolve desktop playback and shutdown findings and CPU compatibility; then mobile talker/predictor plus GPU vocoder | Complete PCM and tail, history/chunk correctness, interruption/recovery, quality checks, no post-startup underruns and RTF below 1 in the declared workload |
|
||||
| M3 | AMD stages | Connect compatible real encoders or other useful coarse stages through current Omni workers; start from the existing 890M vision evidence | Actual node/device placement, numerical and task quality, complete downstream output, shared-memory and handoff costs; reject unhelpful splits |
|
||||
| M4a | MiniCPM-o 4.5 | Desktop reference for encoders, thinker, speech head, flow/HiFT and persistent state; then target-specific mobile artifacts | Separate text, image, audio-understanding and speech-generation suites, then combined streaming and interruption |
|
||||
| M4b | InternVLA-A1 | Real weights, observation preprocessing, shared prefix state, iterative action head and versioned action metadata | Reference action agreement, units/order/timestamps, observation age, stale-input handling and application-specific deadline reporting |
|
||||
| M4a | MiniCPM-o 4.5 | Three-stage desktop text+WAV path produced coherent text and nonzero WAV under a constrained 24 GiB profile after fixing uninitialized request embeddings; add encoder and persistent-state reference before mobile artifacts | Speech intelligibility/alignment, separate text, image and audio-understanding suites, then combined streaming and interruption |
|
||||
| M4b | InternVLA-A1 | Real weights, observation preprocessing, shared prefix state, iterative action head and versioned action metadata; one strict-load synthetic action forward now passes on RTX 5090 Laptop | Reference action agreement on real observations, units/order/timestamps, observation age, stale-input handling and application-specific deadline reporting |
|
||||
| Independent | Qwen3.8-27B | Public Omni pipeline binding; text, then image/video and longer contexts; same-model CPU/iGPU artifact and explicit offload evaluation | Modality-specific quality, actual loading/runtime memory and measured performance; each precision/context receives its own qualification |
|
||||
|
||||
The initial M0 desktop text acceptance remains scoped to its tested checkpoint,
|
||||
|
||||
@@ -35,6 +35,7 @@ from vllm_omni.diffusion.models.internvla_a1 import ( # noqa: E402
|
||||
from vllm_omni.diffusion.models.internvla_a1.config import OBS_STATE # noqa: E402
|
||||
from vllm_omni.diffusion.registry import initialize_model # noqa: E402
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest # noqa: E402
|
||||
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch # noqa: E402
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams # noqa: E402
|
||||
|
||||
|
||||
@@ -158,23 +159,29 @@ def run_pipeline_forward(
|
||||
request_id: str,
|
||||
) -> torch.Tensor:
|
||||
output = pipeline.forward(
|
||||
OmniDiffusionRequest(
|
||||
prompts=[""],
|
||||
sampling_params=OmniDiffusionSamplingParams(
|
||||
extra_args={
|
||||
"batch_inputs": batch_inputs,
|
||||
"noise": noise,
|
||||
"decode_image": False,
|
||||
}
|
||||
),
|
||||
request_id=request_id,
|
||||
DiffusionRequestBatch(
|
||||
requests=[
|
||||
OmniDiffusionRequest(
|
||||
prompt="",
|
||||
sampling_params=OmniDiffusionSamplingParams(
|
||||
extra_args={
|
||||
"batch_inputs": batch_inputs,
|
||||
"noise": noise,
|
||||
"decode_image": False,
|
||||
}
|
||||
),
|
||||
request_id=request_id,
|
||||
)
|
||||
]
|
||||
)
|
||||
)
|
||||
if output.error:
|
||||
raise RuntimeError(output.error)
|
||||
if output.output is None:
|
||||
raise RuntimeError("InternVLAA1Pipeline.forward returned no output tensor.")
|
||||
return output.output
|
||||
payload = output.output
|
||||
actions = payload.get("payload", {}).get("actions") if isinstance(payload, dict) else None
|
||||
if not isinstance(actions, torch.Tensor):
|
||||
raise RuntimeError("InternVLAA1Pipeline.forward returned no action tensor in output.payload.actions.")
|
||||
return actions
|
||||
|
||||
|
||||
def _synchronize(device: str) -> None:
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Offline end-to-end inference for MiniCPM-o 4.5 (thinker + talker/Token2Wav).
|
||||
Offline end-to-end inference for MiniCPM-o 4.5 (thinker + talker + Code2Wav).
|
||||
|
||||
MiniCPM-o uses placeholders ``(<image>./</image>)``, ``(<audio>./</audio>)``,
|
||||
and ``(<video>./</video>)``. Speech output requires a ``<|tts_bos|>`` suffix on
|
||||
@@ -16,7 +16,6 @@ import numpy as np
|
||||
import soundfile as sf
|
||||
import vllm
|
||||
from PIL import Image
|
||||
from vllm import SamplingParams
|
||||
from vllm.assets.audio import AudioAsset
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.assets.video import VideoAsset, video_to_ndarrays
|
||||
@@ -26,8 +25,6 @@ from vllm.multimodal.media.audio import load_audio
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.utils.tracking_parser import TrackingArgumentParser
|
||||
|
||||
SEED = 42
|
||||
|
||||
default_system = (
|
||||
"You are MiniCPM-o, a helpful multimodal assistant that can "
|
||||
"understand images, audio and video, and respond in text and speech."
|
||||
@@ -40,8 +37,8 @@ class QueryResult(NamedTuple):
|
||||
|
||||
|
||||
def _assistant_prefix(use_tts: bool) -> str:
|
||||
# Matches HF chat_template: assistant header, optional empty <think>, then TTS bos.
|
||||
prefix = "<|im_start|>assistant\n"
|
||||
# Match the checkpoint chat template with enable_thinking=False.
|
||||
prefix = "<|im_start|>assistant\n<think>\n\n</think>\n\n"
|
||||
if use_tts:
|
||||
prefix += "<|tts_bos|>"
|
||||
return prefix
|
||||
@@ -285,27 +282,10 @@ def main(args):
|
||||
omni_kwargs["trust_remote_code"] = True
|
||||
omni = Omni(**omni_kwargs)
|
||||
|
||||
# Stage 0 (thinker): multimodal understanding → text (+ TTS span when enabled).
|
||||
thinker_sampling_params = SamplingParams(
|
||||
temperature=0.0,
|
||||
top_p=1.0,
|
||||
top_k=-1,
|
||||
max_tokens=2048,
|
||||
seed=SEED,
|
||||
detokenize=True,
|
||||
repetition_penalty=1.1,
|
||||
)
|
||||
# Stage 1 (talker + Token2Wav): max_tokens=1 satisfies the scheduler;
|
||||
# waveform is produced in-process by Token2Wav.
|
||||
talker_sampling_params = SamplingParams(
|
||||
temperature=0.0,
|
||||
top_p=1.0,
|
||||
top_k=-1,
|
||||
max_tokens=1,
|
||||
seed=SEED,
|
||||
detokenize=False,
|
||||
)
|
||||
sampling_params_list = [thinker_sampling_params, talker_sampling_params][: omni.num_stages]
|
||||
# The deploy config owns all three stages' sampling parameters. In
|
||||
# particular, the Talker requires min_tokens and the Code2Wav stage has
|
||||
# its own output limit; a two-entry list cannot describe this pipeline.
|
||||
sampling_params_list = None
|
||||
|
||||
if args.txt_prompts is None:
|
||||
prompts = [query_result.inputs for _ in range(args.num_prompts)]
|
||||
@@ -345,6 +325,8 @@ def main(args):
|
||||
str(output.prompt) + "\n",
|
||||
"vllm_text_output:\n",
|
||||
str(text_output).strip() + "\n",
|
||||
"vllm_token_ids:\n",
|
||||
str(output.outputs[0].token_ids) + "\n",
|
||||
]
|
||||
try:
|
||||
with open(out_txt, "w", encoding="utf-8") as f:
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
"""Regression for text requests sharing the Omni preprocess embedding buffer."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
|
||||
from vllm_omni.model_executor.models.minicpmo_4_5.minicpmo_4_5_omni import (
|
||||
MiniCPMO45OmniForConditionalGeneration,
|
||||
)
|
||||
|
||||
|
||||
def test_text_preprocess_replaces_uninitialized_embedding_buffer() -> None:
|
||||
ids = torch.tensor([4, 7])
|
||||
scratch = torch.zeros((2, 3))
|
||||
expected = torch.tensor([[4.0, 4.0, 4.0], [7.0, 7.0, 7.0]])
|
||||
model = SimpleNamespace(model_stage="llm", get_input_embeddings=lambda input_ids: input_ids[:, None].expand(-1, 3).float())
|
||||
|
||||
_, embeds, _ = MiniCPMO45OmniForConditionalGeneration.preprocess(
|
||||
model,
|
||||
ids,
|
||||
input_embeds=scratch,
|
||||
_omni_input_embeds_precomputed=False,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(embeds, expected)
|
||||
|
||||
|
||||
def test_text_preprocess_preserves_precomputed_multimodal_embeddings() -> None:
|
||||
ids = torch.tensor([4, 7])
|
||||
prepared = torch.tensor([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]])
|
||||
model = SimpleNamespace(
|
||||
model_stage="llm",
|
||||
get_input_embeddings=lambda _input_ids: (_ for _ in ()).throw(AssertionError("must reuse embeddings")),
|
||||
)
|
||||
|
||||
_, embeds, _ = MiniCPMO45OmniForConditionalGeneration.preprocess(
|
||||
model,
|
||||
ids,
|
||||
input_embeds=prepared,
|
||||
_omni_input_embeds_precomputed=True,
|
||||
)
|
||||
|
||||
assert embeds is prepared
|
||||
@@ -1756,6 +1756,7 @@ class TestPreferModelSamplerNoneFallback:
|
||||
runner._sampling_metadata_for_model_sampler = lambda smd: smd
|
||||
runner._resolve_duplex_sampling_hook = lambda: None
|
||||
runner.sampler = lambda logits, sampling_metadata: default_result
|
||||
runner.device = torch.device("cpu")
|
||||
return runner
|
||||
|
||||
def test_none_falls_back_to_default_sampler_and_warns_once(self):
|
||||
@@ -1801,6 +1802,27 @@ class TestPreferModelSamplerNoneFallback:
|
||||
|
||||
assert out is model_out
|
||||
|
||||
def test_declining_model_cannot_mutate_default_sampler_logits(self):
|
||||
runner = self._runner(model_sample_result=None, default_result=None)
|
||||
original = torch.tensor([[0.1, 0.2, 0.3]])
|
||||
seen = []
|
||||
|
||||
def decline_after_mutation(logits, _metadata):
|
||||
logits[:, :] = float("-inf")
|
||||
logits[:, 0] = 0.0
|
||||
return None
|
||||
|
||||
def default_sampler(*, logits, sampling_metadata):
|
||||
seen.append(logits.clone())
|
||||
return torch.argmax(logits, dim=-1).item()
|
||||
|
||||
runner.model.sample = decline_after_mutation
|
||||
runner.sampler = default_sampler
|
||||
runner.device = torch.device("cpu")
|
||||
|
||||
assert GPUARModelRunner._sample(runner, original, None) == 2
|
||||
torch.testing.assert_close(seen[0], original)
|
||||
|
||||
def test_declaration_matcher_ignores_mentions_and_opt_outs(self):
|
||||
# Guards the guard: the inventory below is only meaningful if
|
||||
# "declares" means an actual opt-in assignment.
|
||||
@@ -1838,6 +1860,7 @@ class TestPreferModelSamplerNoneFallback:
|
||||
"minicpmo_4_5",
|
||||
"minimax_music3",
|
||||
"nemotron_voicechat",
|
||||
"qwen3_tts",
|
||||
}
|
||||
assert declarers == expected, (
|
||||
"The set of models declaring `prefer_model_sampler` changed:\n"
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from inspect import signature
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from transformers.models.qwen3_vl.modeling_qwen3_vl import (
|
||||
@@ -15,7 +17,6 @@ from transformers.models.qwen3_vl.modeling_qwen3_vl import (
|
||||
Qwen3VLVisionModel,
|
||||
Unpack,
|
||||
apply_rotary_pos_emb,
|
||||
check_model_inputs,
|
||||
create_causal_mask,
|
||||
deprecate_kwarg,
|
||||
eager_attention_forward,
|
||||
@@ -39,6 +40,13 @@ from transformers.models.qwen3_vl.modeling_qwen3_vl import (
|
||||
Qwen3VLTextRMSNorm as HFQwen3VLTextRMSNorm,
|
||||
)
|
||||
|
||||
try:
|
||||
from transformers.models.qwen3_vl.modeling_qwen3_vl import check_model_inputs
|
||||
except ImportError:
|
||||
# Transformers 5 moved this decorator out of the model module. The
|
||||
# compatibility alias keeps the custom decoder's config-default behavior.
|
||||
from transformers.utils.generic import merge_with_config_defaults as check_model_inputs
|
||||
|
||||
|
||||
class Qwen3VLTextRMSNorm(HFQwen3VLTextRMSNorm):
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
@@ -83,8 +91,13 @@ class Qwen3VLTextAttention(HFQwen3VLTextAttention):
|
||||
key_states, value_states, self.layer_idx, cache_kwargs
|
||||
)
|
||||
else:
|
||||
key_states = torch.cat([past_key_values[self.layer_idx][0], key_states], dim=2)
|
||||
value_states = torch.cat([past_key_values[self.layer_idx][1], value_states], dim=2)
|
||||
if hasattr(past_key_values, "layers"):
|
||||
layer_cache = past_key_values.layers[self.layer_idx]
|
||||
cached_keys, cached_values = layer_cache.keys, layer_cache.values
|
||||
else:
|
||||
cached_keys, cached_values = past_key_values[self.layer_idx]
|
||||
key_states = torch.cat([cached_keys, key_states], dim=2)
|
||||
value_states = torch.cat([cached_values, value_states], dim=2)
|
||||
|
||||
attention_interface = eager_attention_forward
|
||||
if self.config._attn_implementation != "eager":
|
||||
@@ -174,14 +187,17 @@ class Qwen3VLTextModel(HFQwen3VLTextModel):
|
||||
else:
|
||||
text_position_ids = position_ids[0]
|
||||
|
||||
attention_mask = create_causal_mask(
|
||||
mask_kwargs = dict(
|
||||
config=self.config,
|
||||
input_embeds=inputs_embeds,
|
||||
attention_mask=attention_mask,
|
||||
cache_position=cache_position,
|
||||
past_key_values=past_key_values,
|
||||
position_ids=text_position_ids,
|
||||
)
|
||||
mask_params = signature(create_causal_mask).parameters
|
||||
mask_kwargs["inputs_embeds" if "inputs_embeds" in mask_params else "input_embeds"] = inputs_embeds
|
||||
if "cache_position" in mask_params:
|
||||
mask_kwargs["cache_position"] = cache_position
|
||||
attention_mask = create_causal_mask(**mask_kwargs)
|
||||
|
||||
hidden_states = inputs_embeds
|
||||
position_embeddings = self.rotary_emb(hidden_states, position_ids)
|
||||
@@ -230,7 +246,7 @@ class Qwen3VLModel(HFQwen3VLModel):
|
||||
|
||||
class Qwen3VLForConditionalGeneration(HFQwen3VLForConditionalGeneration):
|
||||
_checkpoint_conversion_mapping = {}
|
||||
_tied_weights_keys = ["lm_head.weight"]
|
||||
_tied_weights_keys = {"lm_head.weight": "model.language_model.embed_tokens.weight"}
|
||||
accepts_loss_kwargs = False
|
||||
config: Qwen3VLConfig
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ from __future__ import annotations
|
||||
import math
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from inspect import signature
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
|
||||
@@ -163,7 +164,7 @@ def compute_layer_complete(
|
||||
gen_expert: nn.Module,
|
||||
act_expert: nn.Module,
|
||||
) -> list[torch.Tensor]:
|
||||
models = [und_expert.language_model, gen_expert, act_expert]
|
||||
models = [und_expert.model.language_model, gen_expert, act_expert]
|
||||
query_states = []
|
||||
key_states = []
|
||||
value_states = []
|
||||
@@ -199,9 +200,9 @@ def compute_layer_complete(
|
||||
sin,
|
||||
unsqueeze_dim=1,
|
||||
)
|
||||
scaling = und_expert.language_model.layers[layer_idx].self_attn.scaling
|
||||
scaling = und_expert.model.language_model.layers[layer_idx].self_attn.scaling
|
||||
att_output, _ = eager_attention_forward(
|
||||
und_expert.language_model.layers[layer_idx].self_attn,
|
||||
und_expert.model.language_model.layers[layer_idx].self_attn,
|
||||
query_states,
|
||||
key_states,
|
||||
value_states,
|
||||
@@ -209,8 +210,8 @@ def compute_layer_complete(
|
||||
scaling,
|
||||
)
|
||||
|
||||
head_dim = und_expert.language_model.layers[layer_idx].self_attn.head_dim
|
||||
num_attention_heads = und_expert.language_model.layers[layer_idx].self_attn.config.num_attention_heads
|
||||
head_dim = und_expert.model.language_model.layers[layer_idx].self_attn.head_dim
|
||||
num_attention_heads = und_expert.model.language_model.layers[layer_idx].self_attn.config.num_attention_heads
|
||||
batch_size = query_states.shape[0]
|
||||
att_output = att_output.reshape(batch_size, -1, num_attention_heads * head_dim)
|
||||
|
||||
@@ -313,7 +314,8 @@ class Qwen3VLWithExpertModel(nn.Module):
|
||||
vlm_config_hf.text_config.num_hidden_layers = vlm_config.num_hidden_layers
|
||||
vlm_config_hf.text_config.num_key_value_heads = vlm_config.num_key_value_heads
|
||||
vlm_config_hf.text_config.max_position_embeddings = 262144
|
||||
vlm_config_hf.text_config.rope_scaling = {
|
||||
vlm_config_hf.text_config.rope_parameters = {
|
||||
**vlm_config_hf.text_config.rope_parameters,
|
||||
"mrope_interleaved": True,
|
||||
"mrope_section": [24, 20, 20],
|
||||
"rope_type": "default",
|
||||
@@ -336,7 +338,7 @@ class Qwen3VLWithExpertModel(nn.Module):
|
||||
gen_expert_config_hf.num_hidden_layers = action_expert_config.num_hidden_layers
|
||||
gen_expert_config_hf.num_key_value_heads = action_expert_config.num_key_value_heads
|
||||
gen_expert_config_hf.max_position_embeddings = self.und_expert.config.text_config.max_position_embeddings
|
||||
gen_expert_config_hf.rope_scaling = self.und_expert.config.text_config.rope_scaling
|
||||
gen_expert_config_hf.rope_parameters = self.und_expert.config.text_config.rope_parameters.copy()
|
||||
self.gen_expert = Qwen3VLTextModel(config=gen_expert_config_hf)
|
||||
self.gen_expert.embed_tokens = None
|
||||
self.gen_expert.lm_head = None
|
||||
@@ -349,7 +351,7 @@ class Qwen3VLWithExpertModel(nn.Module):
|
||||
act_expert_config_hf.num_hidden_layers = action_expert_config.num_hidden_layers
|
||||
act_expert_config_hf.num_key_value_heads = action_expert_config.num_key_value_heads
|
||||
act_expert_config_hf.max_position_embeddings = self.und_expert.config.text_config.max_position_embeddings
|
||||
act_expert_config_hf.rope_scaling = self.und_expert.config.text_config.rope_scaling
|
||||
act_expert_config_hf.rope_parameters = self.und_expert.config.text_config.rope_parameters.copy()
|
||||
self.act_expert = Qwen3VLTextModel(config=act_expert_config_hf)
|
||||
self.act_expert.embed_tokens = None
|
||||
self.act_expert.lm_head = None
|
||||
@@ -383,7 +385,7 @@ class Qwen3VLWithExpertModel(nn.Module):
|
||||
use_cache: bool,
|
||||
) -> tuple[list[torch.Tensor | None], Any]:
|
||||
if inputs_embeds[1] is None and inputs_embeds[2] is None:
|
||||
prefix_output = self.und_expert.language_model.forward(
|
||||
prefix_output = self.und_expert.model.language_model.forward(
|
||||
inputs_embeds=inputs_embeds[0],
|
||||
attention_mask=attention_mask,
|
||||
position_ids=position_ids,
|
||||
@@ -412,7 +414,7 @@ class Qwen3VLWithExpertModel(nn.Module):
|
||||
)
|
||||
return [None, None, suffix_output.last_hidden_state], None
|
||||
|
||||
models = [self.und_expert.language_model, self.gen_expert, self.act_expert]
|
||||
models = [self.und_expert.model.language_model, self.gen_expert, self.act_expert]
|
||||
stacked_inputs = [inputs_embeds[0], inputs_embeds[1], inputs_embeds[2]]
|
||||
for layer_idx in range(self.und_expert.config.text_config.num_hidden_layers):
|
||||
stacked_inputs = compute_layer_complete(
|
||||
@@ -481,7 +483,7 @@ class InternVLAA1(nn.Module):
|
||||
def set_attention_implementation(self, attn_implementation: str) -> None:
|
||||
self.config.attn_implementation = attn_implementation
|
||||
self.qwen3_vl_with_expert.und_expert.config.text_config._attn_implementation = attn_implementation
|
||||
self.qwen3_vl_with_expert.und_expert.language_model.config._attn_implementation = attn_implementation
|
||||
self.qwen3_vl_with_expert.und_expert.model.language_model.config._attn_implementation = attn_implementation
|
||||
self.qwen3_vl_with_expert.gen_expert.config._attn_implementation = attn_implementation
|
||||
self.qwen3_vl_with_expert.act_expert.config._attn_implementation = attn_implementation
|
||||
|
||||
@@ -515,7 +517,12 @@ class InternVLAA1(nn.Module):
|
||||
image_token_id = self.qwen3_vl_with_expert.und_expert.config.image_token_id
|
||||
pixel_values = pixel_values.view(-1, pixel_values.shape[-1])
|
||||
image_grid_thw = image_grid_thw.view(-1, 3)
|
||||
image_embs, _ = self.qwen3_vl_with_expert.und_expert.visual(pixel_values, image_grid_thw)
|
||||
vision_output = self.qwen3_vl_with_expert.und_expert.model.visual(pixel_values, image_grid_thw)
|
||||
image_embs = (
|
||||
vision_output.pooler_output
|
||||
if hasattr(vision_output, "pooler_output")
|
||||
else vision_output[0]
|
||||
)
|
||||
|
||||
embs = self.qwen3_vl_with_expert.und_expert.get_input_embeddings()(lang_tokens)
|
||||
batch_size, seq_len, hidden_dim = embs.shape
|
||||
@@ -642,11 +649,16 @@ class InternVLAA1(nn.Module):
|
||||
attention_mask = pad_masks.to(lang_tokens)
|
||||
if image_grid_thw is not None:
|
||||
image_grid_thw = image_grid_thw.view(-1, 3)
|
||||
return self.qwen3_vl_with_expert.und_expert.model.get_rope_index(
|
||||
padded_lang_tokens,
|
||||
image_grid_thw,
|
||||
attention_mask=attention_mask,
|
||||
)
|
||||
get_rope_index = self.qwen3_vl_with_expert.und_expert.model.get_rope_index
|
||||
if "mm_token_type_ids" in signature(get_rope_index).parameters:
|
||||
token_types = (padded_lang_tokens == self.qwen3_vl_with_expert.und_expert.config.image_token_id).int()
|
||||
return get_rope_index(
|
||||
padded_lang_tokens,
|
||||
mm_token_type_ids=token_types,
|
||||
image_grid_thw=image_grid_thw,
|
||||
attention_mask=attention_mask,
|
||||
)
|
||||
return get_rope_index(padded_lang_tokens, image_grid_thw, attention_mask=attention_mask)
|
||||
|
||||
@torch.no_grad()
|
||||
def sample_actions(
|
||||
@@ -860,12 +872,24 @@ class InternVLAA1Policy(nn.Module):
|
||||
loaded = loader.load_weights((name, f.get_tensor(name)) for name in file_keys)
|
||||
|
||||
if strict:
|
||||
state = instance.state_dict()
|
||||
expected = {
|
||||
name
|
||||
for name, _ in instance.state_dict().items()
|
||||
for name in state
|
||||
if not any(name.startswith(prefix) for prefix in cls._AUTO_WEIGHTS_IGNORE_PREFIXES)
|
||||
}
|
||||
missing = sorted(expected - loaded)
|
||||
# The checkpoint stores lm_head only; tied embed_tokens shares its
|
||||
# storage after post_init. Loading either name initializes both.
|
||||
loaded_storage = {
|
||||
(state[name].data_ptr(), state[name].storage_offset())
|
||||
for name in loaded
|
||||
if name in state
|
||||
}
|
||||
missing = sorted(
|
||||
name
|
||||
for name in expected - loaded
|
||||
if (state[name].data_ptr(), state[name].storage_offset()) not in loaded_storage
|
||||
)
|
||||
if missing:
|
||||
preview = ", ".join(missing[:10])
|
||||
suffix = " ..." if len(missing) > 10 else ""
|
||||
|
||||
@@ -301,12 +301,20 @@ class MiniCPMO45OmniForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
|
||||
if self.model_stage == "tts":
|
||||
return self.talker.preprocess(input_ids=input_ids, input_embeds=input_embeds, **kwargs)
|
||||
if self.model_stage != "llm":
|
||||
embeds = input_embeds if input_embeds is not None else self.get_input_embeddings(input_ids)
|
||||
embeds = (
|
||||
input_embeds
|
||||
if kwargs.get("_omni_input_embeds_precomputed") and input_embeds is not None
|
||||
else self.get_input_embeddings(input_ids)
|
||||
)
|
||||
return input_ids, embeds, {}
|
||||
|
||||
duplex = kwargs.get("duplex")
|
||||
if not isinstance(duplex, dict) or duplex.get("data_plane") is not True:
|
||||
embeds = input_embeds if input_embeds is not None else self.get_input_embeddings(input_ids)
|
||||
embeds = (
|
||||
input_embeds
|
||||
if kwargs.get("_omni_input_embeds_precomputed") and input_embeds is not None
|
||||
else self.get_input_embeddings(input_ids)
|
||||
)
|
||||
return input_ids, embeds, {}
|
||||
|
||||
prompt_len_meta = kwargs.get("duplex_prompt_len")
|
||||
|
||||
@@ -174,7 +174,13 @@ class StepAudio2Token2WavCore(nn.Module):
|
||||
)
|
||||
|
||||
with open(f"{self.model_path}/flow.yaml") as f:
|
||||
configs = load_hyperpyyaml(f)
|
||||
flow_yaml = f.read()
|
||||
# MiniCPM-o's flow.yaml names the standalone ``cosyvoice2`` package,
|
||||
# while the shared in-tree vocoder uses its implementation shipped in
|
||||
# ``stepaudio2-minicpmo``. Resolve those YAML tags locally instead of
|
||||
# relying on the external Token2wav constructor's sys.modules aliases.
|
||||
flow_yaml = flow_yaml.replace("!new:cosyvoice2.", "!new:stepaudio2.cosyvoice2.")
|
||||
configs = load_hyperpyyaml(io.StringIO(flow_yaml))
|
||||
self._flow = configs["flow"]
|
||||
if self.float16:
|
||||
self._flow.half()
|
||||
|
||||
@@ -1439,19 +1439,22 @@ class GPUARModelRunner(OmniGPUModelRunner, OmniConnectorModelRunnerMixin, Duplex
|
||||
with _stats.timed("model.sample_async_ids_ms"):
|
||||
self.input_batch.update_async_output_token_ids()
|
||||
if logits is not None and callable(model_sample) and getattr(self.model, "prefer_model_sampler", False):
|
||||
# Custom samplers may decline a row. Keep the default sampler's
|
||||
# logits intact: it applies its own bias and penalties below.
|
||||
model_logits = logits.clone()
|
||||
# Apply logit bias (min_tokens, allowed_token_ids) before
|
||||
# the custom model sampler — the standard GPU sampler does
|
||||
# this internally, but prefer_model_sampler bypasses it.
|
||||
if hasattr(self.sampler, "logit_bias_state"):
|
||||
self.sampler.logit_bias_state.apply_logit_bias(
|
||||
logits,
|
||||
model_logits,
|
||||
self.input_batch.expanded_idx_mapping,
|
||||
self.input_batch.idx_mapping_np,
|
||||
self.input_batch.positions[self.input_batch.logits_indices],
|
||||
)
|
||||
prepared_sampling_metadata = self._sampling_metadata_for_model_sampler(sampling_metadata)
|
||||
self._apply_duplex_sampling(logits, prepared_sampling_metadata)
|
||||
sampler_output = model_sample(logits, prepared_sampling_metadata)
|
||||
self._apply_duplex_sampling(model_logits, prepared_sampling_metadata)
|
||||
sampler_output = model_sample(model_logits, prepared_sampling_metadata)
|
||||
if sampler_output is not None:
|
||||
return sampler_output
|
||||
# Contract: None => fall back to the default sampler (see
|
||||
|
||||
@@ -1790,6 +1790,16 @@ class OmniGPUModelRunner(GPUModelRunner):
|
||||
req_infos["_omni_prompt_len"] = prompt_len
|
||||
req_infos["_omni_num_computed_tokens"] = num_computed_tokens
|
||||
req_infos["_omni_is_prefill"] = is_prefill
|
||||
# The has_preprocess branch offers a reusable embedding buffer
|
||||
# that is not initialized for ordinary text requests. Models
|
||||
# may reuse its contents only when this runner filled them.
|
||||
req_infos["_omni_input_embeds_precomputed"] = bool(
|
||||
is_first_rank
|
||||
and (
|
||||
(self.supports_mm_inputs and not is_encoder_decoder)
|
||||
or self.enable_prompt_embeds
|
||||
)
|
||||
)
|
||||
# Output-token cap, so a model that must ship a payload on the
|
||||
# request's final step can tell which step that is. A finished
|
||||
# request drops out of req_ids_output_copy, and downstream
|
||||
|
||||
Reference in new issue
Block a user