Validate MiniCPM-o speech pipeline on native Windows GPU
This commit is contained in:
tzhouam committed
1 parent
2f5d0edcf7
commit
8a4559c46c
13 files changed
+2215
-21
No files matched your search
@@ -25,6 +25,7 @@ from vllm_omni.entrypoints.omni import Omni
|
||||
class MemorySampler:
|
||||
def __init__(self, interval_s: float = 0.5):
|
||||
self.interval_s = interval_s
|
||||
self.scope = "wsl_guest" if "microsoft" in platform.release().lower() else "native_os"
|
||||
self.stop = threading.Event()
|
||||
self.samples: list[dict[str, int | float]] = []
|
||||
self.thread = threading.Thread(target=self._run, name="memory-sampler", daemon=True)
|
||||
@@ -43,8 +44,8 @@ class MemorySampler:
|
||||
self.samples.append(
|
||||
{
|
||||
"monotonic_s": time.perf_counter(),
|
||||
"wsl_used_bytes": memory.used,
|
||||
"wsl_available_bytes": memory.available,
|
||||
"system_used_bytes": memory.used,
|
||||
"system_available_bytes": memory.available,
|
||||
"swap_used_bytes": swap.used,
|
||||
"process_tree_rss_sum_bytes": rss_sum,
|
||||
}
|
||||
@@ -101,6 +102,18 @@ def _run_request(omni: Omni, prompt: dict, memory_sampler: MemorySampler) -> dic
|
||||
if not text or not token_ids or not audio_samples or audio_rms == 0:
|
||||
raise RuntimeError("MiniCPM-o did not produce both nonempty text and nonzero audio")
|
||||
samples = [sample for sample in memory_sampler.samples if started <= sample["monotonic_s"] <= completed]
|
||||
sampled_memory = {
|
||||
"scope": memory_sampler.scope,
|
||||
"n": len(samples),
|
||||
"max_system_used_bytes": max((s["system_used_bytes"] for s in samples), default=None),
|
||||
"min_system_available_bytes": min((s["system_available_bytes"] for s in samples), default=None),
|
||||
"max_swap_used_bytes": max((s["swap_used_bytes"] for s in samples), default=None),
|
||||
# Sum of RSS can double-count shared pages; it is not unique RAM.
|
||||
"max_process_tree_rss_sum_bytes": max((s["process_tree_rss_sum_bytes"] for s in samples), default=None),
|
||||
}
|
||||
if memory_sampler.scope == "wsl_guest":
|
||||
sampled_memory["max_wsl_used_bytes"] = sampled_memory["max_system_used_bytes"]
|
||||
sampled_memory["min_wsl_available_bytes"] = sampled_memory["min_system_available_bytes"]
|
||||
return {
|
||||
"request_ids": sorted(request_ids),
|
||||
"wall_s": completed - started,
|
||||
@@ -111,14 +124,7 @@ def _run_request(omni: Omni, prompt: dict, memory_sampler: MemorySampler) -> dic
|
||||
"audio_samples": audio_samples,
|
||||
"audio_duration_s_at_24khz": audio_samples / 24000,
|
||||
"audio_rms_float": audio_rms,
|
||||
"sampled_memory": {
|
||||
"n": len(samples),
|
||||
"max_wsl_used_bytes": max((s["wsl_used_bytes"] for s in samples), default=None),
|
||||
"min_wsl_available_bytes": min((s["wsl_available_bytes"] for s in samples), default=None),
|
||||
"max_swap_used_bytes": max((s["swap_used_bytes"] for s in samples), default=None),
|
||||
# Sum of RSS can double-count shared pages; it is not unique RAM.
|
||||
"max_process_tree_rss_sum_bytes": max((s["process_tree_rss_sum_bytes"] for s in samples), default=None),
|
||||
},
|
||||
"sampled_memory": sampled_memory,
|
||||
}
|
||||
|
||||
|
||||
@@ -129,6 +135,8 @@ def main() -> None:
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--warmups", type=int, default=1)
|
||||
parser.add_argument("--repeats", type=int, default=20)
|
||||
parser.add_argument("--init-timeout", type=int, default=600)
|
||||
parser.add_argument("--stage-init-timeout", type=int, default=300)
|
||||
args = parser.parse_args()
|
||||
if args.warmups < 0 or args.repeats < 1:
|
||||
parser.error("warmups must be nonnegative and repeats must be positive")
|
||||
@@ -141,6 +149,8 @@ def main() -> None:
|
||||
model=str(args.model.resolve()),
|
||||
deploy_config=str(args.deploy_config.resolve()),
|
||||
trust_remote_code=True,
|
||||
init_timeout=args.init_timeout,
|
||||
stage_init_timeout=args.stage_init_timeout,
|
||||
)
|
||||
startup_s = time.perf_counter() - started
|
||||
try:
|
||||
@@ -168,10 +178,21 @@ def main() -> None:
|
||||
"torch": torch.__version__,
|
||||
"vllm": vllm.__version__,
|
||||
"torch_cuda_available": torch.cuda.is_available(),
|
||||
"memory_scope": sampler.scope,
|
||||
"environment": {
|
||||
name: os.environ.get(name)
|
||||
for name in ("VLLM_TARGET_DEVICE", "VLLM_CPU_KVCACHE_SPACE", "OMP_NUM_THREADS", "CUDA_VISIBLE_DEVICES")
|
||||
for name in (
|
||||
"VLLM_TARGET_DEVICE",
|
||||
"VLLM_CPU_KVCACHE_SPACE",
|
||||
"OMP_NUM_THREADS",
|
||||
"MKL_NUM_THREADS",
|
||||
"CUDA_VISIBLE_DEVICES",
|
||||
"VLLM_USE_FLASHINFER_SAMPLER",
|
||||
"VLLM_ENABLE_V1_MULTIPROCESSING",
|
||||
)
|
||||
},
|
||||
"init_timeout_s": args.init_timeout,
|
||||
"stage_init_timeout_s": args.stage_init_timeout,
|
||||
"startup_s": startup_s,
|
||||
"warmups": args.warmups,
|
||||
"repeats": args.repeats,
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
# 2026-09-23 WSL CPU and native Windows expansion
|
||||
|
||||
This extends the [60-cell device × model matrix](../e2e_profiling_20260922/evidence/summary/README.md) with previously unrun WSL CPU and native Windows paths. The September 22 timing tables remain historical. MiniCPM-o now has a separate 20-request serial text-to-speech timing run; each InternVLA timing remains one observation. None is a sustained or full-modality profile. The [checkpoint manifests](../e2e_recovery_20260923/README.md) pin the real weights. No model, precision or context was silently substituted.
|
||||
This extends the [60-cell device × model matrix](../e2e_profiling_20260922/evidence/summary/README.md) with previously unrun WSL CPU and native Windows paths. The September 22 timing tables remain historical. MiniCPM-o now has separate 20-request serial text-to-speech timing runs on WSL CPU and native Windows GPU; each InternVLA timing remains one observation. None is a sustained or full-modality profile. The [checkpoint manifests](../e2e_recovery_20260923/README.md) pin the real weights. No model, precision or context was silently substituted.
|
||||
|
||||
| Device / model | Verified depth | Observed output and time | Remaining gate |
|
||||
|---|---|---|---|
|
||||
| WSL CPU / MiniCPM-o 4.5, revision `503e754207c94da6bb26850b4469f367c9ea3582` | Scoped complete three-stage text-to-text+speech path (P for this input only) | Coherent text and a 5.2 s mono 24 kHz WAV. First request 27.09 s after 73.65 s startup; separate 1-warmup/20-measured serial run: startup 73.89 s, request wall p50 21.86 s/p95 22.97 s. | Speech intelligibility/alignment, image/audio/video inputs, concurrency, interruption, power and thermal behavior. |
|
||||
| Native Windows RTX 5090 Laptop / MiniCPM-o 4.5, same revision | Scoped complete three-stage text-to-text+speech path (P for this input only) | Coherent text with the same 32 thinker token IDs as WSL CUDA, and a nonzero 5.2 s mono 24 kHz WAV. First engine startup 297.20 s and request 78.75 s, including first-use graph/JIT work; separate 1-warmup/20-measured serial run: startup 239.05 s, request wall p50 18.38 s/p95 18.73 s. | Speech intelligibility/alignment, image/audio/video inputs, concurrency, interruption, loading peak and sustained behavior. |
|
||||
| WSL CPU / InternVLA-A1-3B Place_Markpen, revision `0a557c6a503e6545b369bff2f34c76a410bbd190` | Strict real-weight load and full policy forward on **synthetic zero** observations/noise | Finite float32 actions `[1, 50, 32]`; CPU-only PyTorch startup 31.78 s and synchronized forward 3.25 s. Compared with a same-checkpoint CUDA run, relative L2 difference 1.3025% and cosine similarity 0.999917. | Real Place_Markpen observations and reference actions, task tolerance, units/timestamps, repeated latency, memory and deadlines. |
|
||||
| Native Windows CPU / InternVLA-A1-3B Place_Markpen, same revision | Strict real-weight load and full policy forward on **synthetic zero** observations/noise; policy and actions checked on CPU | Finite float32 actions `[1, 50, 32]`; placement-checked startup 61.66 s and forward 3.36 s. Relative L2 difference from WSL CPU 1.0824%; cosine 0.999942. | Real observations/reference actions and quality tolerance; repeated latency, memory, units/timestamps and deadlines. This is the direct policy CPU path, not native vLLM CPU attention. |
|
||||
| Native Windows RTX 5090 Laptop / InternVLA-A1-3B Place_Markpen, same revision | Strict real-weight load and full policy forward on **synthetic zero** observations/noise; policy and actions checked on CUDA | Finite float32 actions `[1, 50, 32]`; placement-checked startup 75.54 s and synchronized forward 1.12 s. Relative L2 difference from WSL CUDA 0.8315%; cosine 0.999966. | Real observations/reference actions and quality tolerance; repeated latency, memory, units/timestamps and deadlines. |
|
||||
@@ -13,6 +14,10 @@ Both runs used Ryzen AI 9 HX 370 on Ubuntu 26.04/WSL2 kernel `6.18.33.2-microsof
|
||||
|
||||
The separate [20-request profile](evidence/minicpmo/minicpmo_cpu_profile_20.json) records every measured output and sampled memory; the [JSONL](evidence/minicpmo/minicpmo_cpu_profile_20.jsonl) survives interruption, and the [run log](evidence/minicpmo/minicpmo_cpu_profile_20.log) retains worker details. One warmup was excluded. All 20 measured requests returned the same thinker token sequence, nonzero 124,800-sample audio, and request wall times 20.95–23.33 s; nearest-rank p50/p95 were 21.86/22.97 s (real-time factor 4.20/4.42 for 5.2 s of audio). Sampled WSL available RAM reached a minimum of 3.80 GB and swap use a maximum of 5.75 GB; summed process-tree RSS reached 29.04 GB but double-counts shared pages. These are 0.5 s samples during requests, not load-peak or unique-resident-memory measurements. Audio and text were returned together at request completion, so the profile cannot establish playable-stream latency, and no concurrency or 30-minute thermal run was performed.
|
||||
|
||||
Native Windows MiniCPM-o used the same [constrained CUDA deployment config](../e2e_recovery_20260923/evidence/minicpmo/minicpmo_24gb_offline.yaml) as the WSL CUDA run: BF16, eager vLLM 0.29.0, 2,048-token thinker / 512-token talker limits, text input only, one request and 5 GiB thinker host offload. The local Windows 11 build 26200 host has 63.12 GiB visible RAM, an RTX 5090 Laptop GPU (driver 610.71), Python 3.12.10, PyTorch 2.13.0+cu130, Transformers 5.14.1, `stepaudio2-minicpmo` 0.1.1 and `pyarrow` 25.0.1. The checkpoint was read through a local WSL UNC path. The example's then-default 300 s overall startup timeout [expired](evidence/minicpmo_windows/minicpmo_native_gpu.log) after stage initialization had nearly completed; rerunning with `--init-timeout 900 --stage-init-timeout 600` [completed](evidence/minicpmo_windows/minicpmo_native_gpu_timeout900.log) in 297.20 s startup and one 78.75 s request. The example default has since been raised to 600 s, matching Omni's default. The text [and token IDs](evidence/minicpmo_windows/0_b4835d8d-f4f8-48b9-85d2-bb7ef06d9249.txt) exactly match the WSL CUDA result. The [WAV](evidence/minicpmo_windows/output_0_b4835d8d-f4f8-48b9-85d2-bb7ef06d9249.wav) has 124,800 mono PCM16 frames at 24 kHz; 124,560 samples are nonzero. The [artifact report](evidence/minicpmo_windows/minicpmo_windows_artifact_report.json) records its checksum and signal statistics. One first-use request is not a latency distribution or speech-quality assessment; the distinct Windows timing must not be transferred to WSL or CPU.
|
||||
|
||||
The separate native Windows [20-request profile](evidence/minicpmo_windows/minicpmo_windows_gpu_profile20.json), [per-request JSONL](evidence/minicpmo_windows/minicpmo_windows_gpu_profile20.jsonl), and [raw log](evidence/minicpmo_windows/minicpmo_windows_gpu_profile20.log) used one warmup followed by 20 measured serial requests in one engine. All measured requests returned one consistent thinker token sequence and 124,800-sample audio with nonzero RMS. Nearest-rank request wall p50/p95 were 18.38/18.73 s (real-time factor 3.53/3.60 for 5.2 s of audio), with observed range 18.22–19.79 s. Engine startup was 239.05 s on this later, warm-filesystem run. During measured requests, 0.5 s samples found native host available RAM as low as 0.78 GB and pagefile use as high as 11.17 GB; summed process-tree RSS reached 21.08 GB but double-counts shared pages. These are host-wide samples, not model-only RAM or loading peak. Neither a playable-stream timing nor speech intelligibility, concurrency, interruption, power or a 30-minute thermal run was measured.
|
||||
|
||||
The CPU-only InternVLA run used the same BF16 policy checkpoint, synthetic six-camera history/state input and zero diffusion noise as the CUDA reference. The raw [CPU JSON](evidence/internvla/internvla_pure_cpu_synthetic.json), [CPU log](evidence/internvla/internvla_pure_cpu_synthetic.log), [CPU actions](evidence/internvla/internvla_pure_cpu_actions.npy), [CUDA JSON](evidence/internvla/internvla_cuda_synthetic_with_actions.json), [CUDA actions](evidence/internvla/internvla_cuda_actions.npy) and [numerical comparison](evidence/internvla/internvla_pure_cpu_cuda_parity.json) are retained. The CUDA reference used the WSL RTX 5090 Laptop and PyTorch 2.13.0+cu130; its one forward took 1.33 s. The action difference has maximum absolute error 0.00993 and mean absolute error 0.00117. No reference-task tolerance is available, so this comparison is evidence of numerical proximity, not an accuracy pass.
|
||||
|
||||
The native Windows runs used Windows 11 build 26200, NVIDIA driver 610.71 for the GPU path, Python 3.12.10, PyTorch 2.13.0+cu130, vLLM 0.29.0, Transformers 5.14.1, and the current Omni source. The CPU run explicitly selected the CPU policy despite the CUDA-capable wheel. The placement-checked reruns assert that all policy parameters and the output action tensor remain on the requested device. Both loaded the same checkpoint from the local WSL filesystem through a UNC path; Windows CPU and GPU runs used the same synthetic input and zero diffusion noise as the WSL reference. The native [CPU JSON](evidence/internvla/internvla_windows_cpu_synthetic.json), [GPU JSON](evidence/internvla/internvla_windows_gpu_synthetic.json), [CPU log](evidence/internvla/internvla_windows_cpu_synthetic.log), [GPU log](evidence/internvla/internvla_windows_gpu_synthetic.log), [CPU actions](evidence/internvla/internvla_windows_cpu_actions.npy), [GPU actions](evidence/internvla/internvla_windows_gpu_actions.npy), [CPU/WSL comparison](evidence/internvla/internvla_windows_cpu_wsl_cpu_parity.json), and [GPU/WSL comparison](evidence/internvla/internvla_windows_gpu_wsl_cuda_parity.json) are retained. Native vLLM import initially failed on a Linux-specific CuMem allocator patch; the added Windows guard resolved the import. Native default GBK source decoding also required `PYTHONUTF8=1`. Both platform differences are setup requirements, not model-quality evidence. Power mode and thermal state were not captured.
|
||||
@@ -42,3 +47,21 @@ export CUDA_VISIBLE_DEVICES= VLLM_TARGET_DEVICE=cpu VLLM_CPU_KVCACHE_SPACE=1 VLL
|
||||
```
|
||||
|
||||
For native Windows InternVLA, set `PYTHONUTF8=1`, put this checkout on `PYTHONPATH`, and run `benchmarks/edge_harness/probe_internvla_actions.py` with the same model/processor paths as above but Windows-accessible UNC paths; use `--device cpu` or `--device cuda` and separate output paths.
|
||||
|
||||
The native Windows MiniCPM-o profile used this PowerShell command (run the example `end2end.py` instead for the single WAV-producing probe):
|
||||
|
||||
```powershell
|
||||
$root = '\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer'
|
||||
$env:PYTHONUTF8 = '1'
|
||||
$env:PYTHONPATH = "$root\vllm-omni-edge"
|
||||
$env:OMP_NUM_THREADS = '8'
|
||||
$env:MKL_NUM_THREADS = '8'
|
||||
$env:TOKENIZERS_PARALLELISM = 'false'
|
||||
$env:VLLM_USE_FLASHINFER_SAMPLER = '0'
|
||||
$env:VLLM_ENABLE_V1_MULTIPROCESSING = '0'
|
||||
& 'C:\Users\zhout\w2\omni029venv\Scripts\python.exe' "$root\vllm-omni-edge\benchmarks\edge_harness\profile_minicpmo_text_speech.py" `
|
||||
--model "$root\models\MiniCPM-o-4_5" `
|
||||
--deploy-config "$root\vllm-omni-edge\benchmarks\edge_harness\results\e2e_recovery_20260923\evidence\minicpmo\minicpmo_24gb_offline.yaml" `
|
||||
--warmups 1 --repeats 20 --init-timeout 900 --stage-init-timeout 600 `
|
||||
--output 'C:\Users\zhout\w2\minicpmo_windows_gpu_profile20.json'
|
||||
```
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
Prompt:
|
||||
<|im_start|>system
|
||||
You are MiniCPM-o, a helpful multimodal assistant that can understand images, audio and video, and respond in text and speech.<|im_end|>
|
||||
<|im_start|>user
|
||||
Say hello, then introduce vLLM-Omni in one sentence.<|im_end|>
|
||||
<|im_start|>assistant
|
||||
<think>
|
||||
|
||||
</think>
|
||||
|
||||
<|tts_bos|>
|
||||
vllm_text_output:
|
||||
Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video.
|
||||
vllm_token_ids:
|
||||
[9707, 11, 358, 1079, 348, 4086, 44, 23620, 76, 7751, 11, 264, 3460, 79049, 57597, 1614, 12875, 315, 8660, 323, 23163, 1467, 11, 5335, 11, 7699, 11, 323, 2766, 13, 151704, 151645]
|
||||
+261
@@ -0,0 +1,261 @@
|
||||
INFO 09-23 16:24:18 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 16:24:18 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
====================
|
||||
vllm version: 0.29.0
|
||||
====================
|
||||
INFO 09-23 16:24:21 [omni_base.py:216] [Omni] Initializing with model \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\MiniCPM-o-4_5
|
||||
INFO 09-23 16:24:21 [async_omni_engine.py:172] [AsyncOmniEngine] Initializing with model \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\MiniCPM-o-4_5
|
||||
INFO 09-23 16:24:22 [async_omni_engine.py:290] [AsyncOmniEngine] Launching Orchestrator thread with 3 stages
|
||||
INFO 09-23 16:24:22 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
INFO 09-23 16:24:23 [initialization.py:356] Loaded OmniTransferConfig with 1 connector configurations
|
||||
WARNING 09-23 16:24:23 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 16:24:23 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
WARNING 09-23 16:24:23 [arg_utils.py:1801] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM.
|
||||
INFO 09-23 16:24:38 [model.py:684] Resolved architecture: MiniCPMO45OmniForConditionalGeneration
|
||||
INFO 09-23 16:24:38 [model.py:2021] Using max model len 2048
|
||||
WARNING 09-23 16:24:38 [model.py:981] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False.
|
||||
INFO 09-23 16:24:39 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode.
|
||||
INFO 09-23 16:24:39 [model.py:887] Disabled mm_prefix attention mode because multimodal inputs are configuration-disabled. Attention backends without mm_prefix support may now be selected.
|
||||
INFO 09-23 16:24:39 [scheduler.py:277] Chunked prefill is enabled with max_num_batched_tokens=2048.
|
||||
WARNING 09-23 16:24:39 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
WARNING 09-23 16:24:39 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
INFO 09-23 16:24:39 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-23 16:24:39 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
INFO 09-23 16:24:39 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
WARNING 09-23 16:24:39 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 16:24:39 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
WARNING 09-23 16:24:40 [arg_utils.py:1801] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM.
|
||||
INFO 09-23 16:24:40 [model.py:684] Resolved architecture: MiniCPMO45OmniForConditionalGeneration
|
||||
INFO 09-23 16:24:40 [model.py:2021] Using max model len 512
|
||||
WARNING 09-23 16:24:40 [model.py:981] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False.
|
||||
INFO 09-23 16:24:40 [model.py:2021] Using max model len 512
|
||||
INFO 09-23 16:24:41 [scheduler.py:277] Chunked prefill is enabled with max_num_batched_tokens=512.
|
||||
INFO 09-23 16:24:41 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
WARNING 09-23 16:24:41 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 16:24:41 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
WARNING 09-23 16:24:41 [arg_utils.py:1801] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM.
|
||||
INFO 09-23 16:24:56 [model.py:684] Resolved architecture: MiniCPMO45Code2Wav
|
||||
INFO 09-23 16:24:56 [model.py:2021] Using max model len 4096
|
||||
WARNING 09-23 16:24:56 [arg_utils.py:2780] This model does not officially support disabling chunked prefill. Disabling this manually may cause the engine to crash or produce incorrect outputs.
|
||||
INFO 09-23 16:24:56 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-23 16:24:56 [stage_runtime.py:853] [stage_init] Stage-0 set runtime devices: 0
|
||||
INFO 09-23 16:25:08 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 16:25:08 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840) INFO 09-23 16:25:12 [core.py:123] Initializing a V1 LLM engine (v0.29.0) with config: model='\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\MiniCPM-o-4_5', speculative_config=None, tokenizer='\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=2048, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 0, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840) INFO 09-23 16:25:14 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840) INFO 09-23 16:25:14 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file://C:\Users\zhout\AppData\Local\Temp/vllm_dist_2d36997f7ac0457d963dd5c17559f334 backend=nccl
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840) WARNING 09-23 16:25:14 [parallel_state.py:1788] Distributed backend nccl is not available; falling back to gloo.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840) INFO 09-23 16:25:14 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840) WARNING 09-23 16:25:15 [gpu_ar_worker.py:120] OMNI GPUARWorker forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840) INFO 09-23 16:25:15 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840) INFO 09-23 16:25:15 [base.py:138] Offloader set to UVAOffloader
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840) WARNING 09-23 16:25:15 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840) WARNING 09-23 16:25:15 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840) INFO 09-23 16:25:15 [gpu_model_runner.py:5379] Starting to load model \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\MiniCPM-o-4_5...
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840) WARNING 09-23 16:25:15 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840) WARNING 09-23 16:25:15 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840) INFO 09-23 16:25:15 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840) INFO 09-23 16:25:15 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840) INFO 09-23 16:25:15 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840) INFO 09-23 16:25:15 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840) INFO 09-23 16:25:15 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840) INFO 09-23 16:25:15 [cuda.py:492] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840) INFO 09-23 16:25:15 [flash_attn.py:897] Using FlashAttention version 2
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840) INFO 09-23 16:25:17 [uva.py:65] Total CPU offloaded parameters: 5.03
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840) INFO 09-23 16:25:17 [weight_utils.py:863] Filesystem type for checkpoints: unknown. Checkpoint size: 17.46 GiB. Available RAM: 21.93 GiB.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840) INFO 09-23 16:25:17 [weight_utils.py:886] Auto-prefetch is disabled because the filesystem (unknown) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840)
|
||||
Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840)
|
||||
Loading safetensors checkpoint shards: 25% Completed | 1/4 [00:00<00:00, 4.65it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840)
|
||||
Loading safetensors checkpoint shards: 50% Completed | 2/4 [00:00<00:00, 4.58it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840)
|
||||
Loading safetensors checkpoint shards: 75% Completed | 3/4 [00:00<00:00, 3.51it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:01<00:00, 3.14it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:01<00:00, 3.42it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840) INFO 09-23 16:27:10 [default_loader.py:430] Loading weights took 112.69 seconds
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840) INFO 09-23 16:27:12 [gpu_model_runner.py:5471] Model loading took 10.84 GiB memory and 115.113511 seconds
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840) INFO 09-23 16:27:12 [utils.py:306] Using LBNHC KV cache layout.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840) INFO 09-23 16:27:49 [base.py:148] Available KV cache memory: 2.71 GiB (device-level profiling)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840) INFO 09-23 16:27:49 [kv_cache_utils.py:2032] GPU KV cache size: 19,760 tokens, Maximum concurrency for 2,048 tokens per request: 9.65x
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840) WARNING 09-23 16:27:49 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840) INFO 09-23 16:27:49 [kernel_warmup.py:124] JIT kernel warmup starting.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840)
|
||||
JIT kernel warmup (1 compile keys): 0%| | 0/1 [00:00<?, ?kernel/s]
|
||||
JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 1.50kernel/s, ComputeSlotMappingKernel (1 keys)]
|
||||
JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 1.50kernel/s, ComputeSlotMappingKernel (1 keys)]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840) INFO 09-23 16:27:49 [kernel_warmup.py:134] JIT kernel warmup finished in 0.67s.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840) INFO 09-23 16:27:50 [ll_bf16.py:30] cuteDSL (CUTLASS Python) not available, ll_bf16_gemm disabled
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840) INFO 09-23 16:27:50 [gpu_worker.py:860] Free memory on device (22.61/23.89 GiB) on startup. Desired GPU memory utilization is (0.58, 13.86 GiB). Actual usage is 11.01 GiB for consumed memory (weights + non-torch), 0.22 GiB for peak activation, and 0.0 GiB for CUDAGraph memory. Replace gpu_memory_utilization config with `--kv-cache-memory=2665780347` (2.48 GiB) to fit into requested memory, or `--kv-cache-memory=12063866880` (11.24 GiB) to fully utilize gpu memory. Current kv cache memory in use is 2.71 GiB.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840) INFO 09-23 16:27:51 [jit_monitor.py:85] Kernel JIT monitor activated; monitored JIT compilations during inference will use mode=warn.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840) WARNING 09-23 16:27:51 [torch_utils.py:265] OMP_NUM_THREADS=8 is set; leaving Torch threads at 8 for serving. Multi-threaded torch CPU ops during serving can degrade performance through spin-wait contention and cgroup CPU-quota throttling.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840) INFO 09-23 16:27:51 [core.py:368] init engine (profile, create kv cache, warmup model) took 39.02 s
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840) WARNING 09-23 16:27:51 [scheduler.py:213] Using custom scheduler class vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840) WARNING 09-23 16:27:51 [omni_scheduler_mixin.py:118] OMNI scheduler forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840) INFO 09-23 16:27:51 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage0_replica0 pid=8840) INFO 09-23 16:27:51 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-23 16:27:51 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed
|
||||
INFO 09-23 16:27:51 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-0 [rep-0] initializing EngineCore
|
||||
INFO 09-23 16:27:52 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-0 [rep-0] EngineCore running
|
||||
INFO 09-23 16:27:52 [stage_runtime.py:958] [StageRuntime] Stage 0 initialized
|
||||
INFO 09-23 16:27:52 [stage_runtime.py:853] [stage_init] Stage-1 set runtime devices: 0
|
||||
INFO 09-23 16:28:11 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 16:28:11 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896) INFO 09-23 16:28:15 [core.py:123] Initializing a V1 LLM engine (v0.29.0) with config: model='\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\MiniCPM-o-4_5', speculative_config=None, tokenizer='\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=512, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [512], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 0, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896) INFO 09-23 16:28:18 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file://C:\Users\zhout\AppData\Local\Temp/vllm_dist_c46fc169d2b24ce5bf1909728c14d88d backend=nccl
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896) WARNING 09-23 16:28:18 [parallel_state.py:1788] Distributed backend nccl is not available; falling back to gloo.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896) INFO 09-23 16:28:18 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896) WARNING 09-23 16:28:19 [gpu_ar_worker.py:120] OMNI GPUARWorker forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896) INFO 09-23 16:28:19 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896) [transformers] `MiniCPMOProcessor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896) [transformers] Requested torchvision backend is not available. Falling back to pil backend.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896) [transformers] `MiniCPMOProcessor` defines `feature_extractor_class = 'WhisperFeatureExtractor'`, which is deprecated. Register the correct mapping in `AutoFeatureExtractor` instead.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896) [transformers] Image processor MiniCPMVImageProcessor: kwargs ['max_slice_nums'] were applied for backward compatibility. To avoid this warning, add them to valid_kwargs: create a custom TypedDict extending ImagesKwargs with these keys and set it as the `valid_kwargs` class attribute.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896) INFO 09-23 16:28:36 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896) WARNING 09-23 16:28:36 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896) WARNING 09-23 16:28:36 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896) INFO 09-23 16:28:36 [gpu_model_runner.py:5379] Starting to load model \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\MiniCPM-o-4_5...
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896) WARNING 09-23 16:28:36 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896) WARNING 09-23 16:28:36 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896) INFO 09-23 16:28:36 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896) INFO 09-23 16:28:36 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896) INFO 09-23 16:28:36 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896) INFO 09-23 16:28:36 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896) INFO 09-23 16:28:37 [cuda.py:492] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896) INFO 09-23 16:28:37 [flash_attn.py:897] Using FlashAttention version 2
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896) INFO 09-23 16:28:37 [weight_utils.py:863] Filesystem type for checkpoints: unknown. Checkpoint size: 17.46 GiB. Available RAM: 15.41 GiB.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896) INFO 09-23 16:28:37 [weight_utils.py:893] Auto-prefetch is disabled because the filesystem (unknown) is not a recognized network FS (NFS/Lustre) and the checkpoint size (17.46 GiB) exceeds 90% of available RAM (15.41 GiB).
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896)
|
||||
Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896)
|
||||
Loading safetensors checkpoint shards: 25% Completed | 1/4 [00:00<00:00, 4.54it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896)
|
||||
Loading safetensors checkpoint shards: 50% Completed | 2/4 [00:00<00:00, 2.55it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896)
|
||||
Loading safetensors checkpoint shards: 75% Completed | 3/4 [00:01<00:00, 2.77it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:01<00:00, 2.37it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:01<00:00, 2.54it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896) INFO 09-23 16:28:43 [default_loader.py:430] Loading weights took 6.31 seconds
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896) INFO 09-23 16:28:44 [gpu_model_runner.py:5471] Model loading took 0.65 GiB memory and 6.940160 seconds
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896) INFO 09-23 16:28:45 [utils.py:306] Using LBNHC KV cache layout.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896) INFO 09-23 16:28:45 [gpu_model_runner.py:6519] Skipping memory profiling for multimodal encoder and encoder cache.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896) INFO 09-23 16:28:45 [kv_cache_utils.py:2032] GPU KV cache size: 4,368 tokens, Maximum concurrency for 512 tokens per request: 8.53x
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896) WARNING 09-23 16:28:45 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896) INFO 09-23 16:28:45 [kernel_warmup.py:124] JIT kernel warmup starting.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896)
|
||||
JIT kernel warmup (1 compile keys): 0%| | 0/1 [00:00<?, ?kernel/s]
|
||||
JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 6.14kernel/s, ComputeSlotMappingKernel (1 keys)]
|
||||
JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 6.14kernel/s, ComputeSlotMappingKernel (1 keys)]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896) INFO 09-23 16:28:46 [kernel_warmup.py:134] JIT kernel warmup finished in 0.17s.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896) INFO 09-23 16:28:46 [ll_bf16.py:30] cuteDSL (CUTLASS Python) not available, ll_bf16_gemm disabled
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896) INFO 09-23 16:28:46 [jit_monitor.py:85] Kernel JIT monitor activated; monitored JIT compilations during inference will use mode=warn.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896) WARNING 09-23 16:28:47 [torch_utils.py:265] OMP_NUM_THREADS=8 is set; leaving Torch threads at 8 for serving. Multi-threaded torch CPU ops during serving can degrade performance through spin-wait contention and cgroup CPU-quota throttling.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896) INFO 09-23 16:28:47 [core.py:368] init engine (profile, create kv cache, warmup model) took 2.02 s
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896) WARNING 09-23 16:28:47 [scheduler.py:213] Using custom scheduler class vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896) WARNING 09-23 16:28:47 [omni_scheduler_mixin.py:118] OMNI scheduler forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896) INFO 09-23 16:28:47 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage1_replica0 pid=57896) INFO 09-23 16:28:47 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-23 16:28:47 [stage_runtime.py:943] [StageRuntime] Stage 1 engine startup completed
|
||||
INFO 09-23 16:28:47 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-1 [rep-0] initializing EngineCore
|
||||
INFO 09-23 16:28:47 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-1 [rep-0] EngineCore running
|
||||
INFO 09-23 16:28:47 [stage_runtime.py:958] [StageRuntime] Stage 1 initialized
|
||||
INFO 09-23 16:28:47 [stage_runtime.py:853] [stage_init] Stage-2 set runtime devices: 0
|
||||
INFO 09-23 16:29:03 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 16:29:03 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=6400) INFO 09-23 16:29:08 [core.py:123] Initializing a V1 LLM engine (v0.29.0) with config: model='\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\MiniCPM-o-4_5', speculative_config=None, tokenizer='\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=4096, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=False, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [4096], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 0, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
(StageEngineCoreProc_stage2_replica0 pid=6400) INFO 09-23 16:29:09 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file://C:\Users\zhout\AppData\Local\Temp/vllm_dist_e5dbe2b9f92549ba9f9566b6a436c49d backend=nccl
|
||||
(StageEngineCoreProc_stage2_replica0 pid=6400) WARNING 09-23 16:29:09 [parallel_state.py:1788] Distributed backend nccl is not available; falling back to gloo.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=6400) INFO 09-23 16:29:09 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage2_replica0 pid=6400) WARNING 09-23 16:29:09 [gpu_generation_worker.py:103] OMNI GPUGenerationWorker forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=6400) INFO 09-23 16:29:09 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=6400) INFO 09-23 16:29:09 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage2_replica0 pid=6400) WARNING 09-23 16:29:09 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=6400) WARNING 09-23 16:29:09 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=6400) INFO 09-23 16:29:09 [gpu_model_runner.py:5379] Starting to load model \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\MiniCPM-o-4_5...
|
||||
(StageEngineCoreProc_stage2_replica0 pid=6400) INFO 09-23 16:29:10 [weight_utils.py:863] Filesystem type for checkpoints: unknown. Checkpoint size: 17.46 GiB. Available RAM: 13.21 GiB.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=6400) INFO 09-23 16:29:10 [weight_utils.py:893] Auto-prefetch is disabled because the filesystem (unknown) is not a recognized network FS (NFS/Lustre) and the checkpoint size (17.46 GiB) exceeds 90% of available RAM (13.21 GiB).
|
||||
(StageEngineCoreProc_stage2_replica0 pid=6400)
|
||||
Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=6400)
|
||||
Loading safetensors checkpoint shards: 25% Completed | 1/4 [00:00<00:00, 5.95it/s]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=6400)
|
||||
Loading safetensors checkpoint shards: 50% Completed | 2/4 [00:00<00:00, 7.62it/s]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=6400)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:00<00:00, 6.15it/s]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=6400)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:00<00:00, 6.28it/s]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=6400)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=6400) INFO 09-23 16:29:11 [step_audio2_token2wav.py:163] Loading Token2Wav models from: \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\MiniCPM-o-4_5\assets\token2wav
|
||||
WARNING 09-23 16:29:22 [async_omni_engine.py:528] [AsyncOmniEngine] Orchestrator startup timed out after 300s. Multi-stage deployments that initialize stages sequentially on one device or load checkpoints from slow storage may need larger --init-timeout and --stage-init-timeout values.
|
||||
INFO 09-23 16:29:22 [async_omni_engine.py:1807] [AsyncOmniEngine] Shutting down Orchestrator
|
||||
(StageEngineCoreProc_stage2_replica0 pid=6400) INFO 09-23 16:29:27 [step_audio2_token2wav.py:201] Token2Wav models loaded successfully
|
||||
remark: snake_activation.py:48:0: 3282 instructions in function
|
||||
(StageEngineCoreProc_stage2_replica0 pid=6400) INFO 09-23 16:29:30 [batched_token2wav.py:227] Enabling HiFT CUDA Graph with batch sizes [1]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=6400) INFO 09-23 16:29:31 [cuda_graph_wrapper.py:98] Captured HiFT CUDA Graph for shape (1, 50, 0)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=6400) INFO 09-23 16:29:31 [cuda_graph_wrapper.py:98] Captured HiFT CUDA Graph for shape (1, 58, 3840)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=6400) INFO 09-23 16:29:31 [batched_token2wav.py:235] HiFT CUDA Graph captured successfully
|
||||
(StageEngineCoreProc_stage2_replica0 pid=6400) INFO 09-23 16:29:31 [batched_token2wav.py:246] CFM CUDA Graph enabled (max_graphs=32)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=6400) INFO 09-23 16:29:31 [default_loader.py:430] Loading weights took 21.09 seconds
|
||||
(StageEngineCoreProc_stage2_replica0 pid=6400) INFO 09-23 16:29:32 [gpu_model_runner.py:5471] Model loading took 3.36 GiB memory and 21.537910 seconds
|
||||
(StageEngineCoreProc_stage2_replica0 pid=6400) INFO 09-23 16:29:33 [cuda.py:492] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(StageEngineCoreProc_stage2_replica0 pid=6400) INFO 09-23 16:29:33 [utils.py:306] Using LBNHC KV cache layout.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=6400) WARNING 09-23 16:29:33 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=6400) INFO 09-23 16:29:33 [kernel_warmup.py:124] JIT kernel warmup starting.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=6400)
|
||||
JIT kernel warmup (1 compile keys): 0%| | 0/1 [00:00<?, ?kernel/s]
|
||||
JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 111.11kernel/s, ComputeSlotMappingKernel (1 keys)]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=6400) INFO 09-23 16:29:33 [kernel_warmup.py:134] JIT kernel warmup finished in 0.01s.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=6400) INFO 09-23 16:29:33 [ll_bf16.py:30] cuteDSL (CUTLASS Python) not available, ll_bf16_gemm disabled
|
||||
(StageEngineCoreProc_stage2_replica0 pid=6400) WARNING 09-23 16:29:33 [gpu_generation_model_runner.py:558] Dummy sampler run is not implemented for generation model
|
||||
(StageEngineCoreProc_stage2_replica0 pid=6400) INFO 09-23 16:29:33 [jit_monitor.py:85] Kernel JIT monitor activated; monitored JIT compilations during inference will use mode=warn.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=6400) WARNING 09-23 16:29:34 [torch_utils.py:265] OMP_NUM_THREADS=8 is set; leaving Torch threads at 8 for serving. Multi-threaded torch CPU ops during serving can degrade performance through spin-wait contention and cgroup CPU-quota throttling.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=6400) INFO 09-23 16:29:34 [core.py:368] init engine (profile, create kv cache, warmup model) took 0.98 s
|
||||
(StageEngineCoreProc_stage2_replica0 pid=6400) WARNING 09-23 16:29:35 [scheduler.py:213] Using custom scheduler class vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=6400) WARNING 09-23 16:29:35 [omni_scheduler_mixin.py:118] OMNI scheduler forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=6400) INFO 09-23 16:29:35 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage2_replica0 pid=6400) WARNING 09-23 16:29:35 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
(StageEngineCoreProc_stage2_replica0 pid=6400) WARNING 09-23 16:29:35 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=6400) INFO 09-23 16:29:35 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage2_replica0 pid=6400) INFO 09-23 16:29:35 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
INFO 09-23 16:29:35 [stage_runtime.py:943] [StageRuntime] Stage 2 engine startup completed
|
||||
INFO 09-23 16:29:35 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-2 [rep-0] initializing EngineCore
|
||||
(StageEngineCoreProc_stage2_replica0 pid=6400) INFO 09-23 16:29:35 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
INFO 09-23 16:29:36 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-2 [rep-0] EngineCore running
|
||||
INFO 09-23 16:29:36 [stage_runtime.py:958] [StageRuntime] Stage 2 initialized
|
||||
INFO 09-23 16:29:39 [orchestrator.py:604] [Orchestrator] Starting event loop
|
||||
INFO 09-23 16:29:39 [orchestrator.py:705] [Orchestrator] Received shutdown signal
|
||||
INFO 09-23 16:29:39 [orchestrator.py:3092] [Orchestrator] Shutting down all 3 client(s)
|
||||
INFO 09-23 16:29:39 [core_client.py:689] [shutdown] MPClient: start timeout=default
|
||||
INFO 09-23 16:29:39 [core_client.py:691] [shutdown] MPClient: stopping engine manager
|
||||
INFO 09-23 16:29:39 [utils.py:626] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage0_replica0
|
||||
INFO 09-23 16:29:39 [core_client.py:693] [shutdown] MPClient: engine manager stopped
|
||||
INFO 09-23 16:29:39 [core_client.py:694] [shutdown] MPClient: cleaning up background resources
|
||||
INFO 09-23 16:29:39 [core_client.py:696] [shutdown] MPClient: complete
|
||||
INFO 09-23 16:29:39 [stage_pool.py:1354] [StagePool] Stage 0 replica 0 shut down
|
||||
INFO 09-23 16:29:39 [core_client.py:689] [shutdown] MPClient: start timeout=default
|
||||
INFO 09-23 16:29:39 [utils.py:626] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage1_replica0
|
||||
INFO 09-23 16:29:40 [stage_pool.py:1354] [StagePool] Stage 1 replica 0 shut down
|
||||
INFO 09-23 16:29:40 [core_client.py:689] [shutdown] MPClient: start timeout=default
|
||||
INFO 09-23 16:29:40 [utils.py:626] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage2_replica0
|
||||
INFO 09-23 16:29:40 [stage_pool.py:1354] [StagePool] Stage 2 replica 0 shut down
|
||||
Traceback (most recent call last):
|
||||
File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\examples\offline_inference\minicpmo\end2end.py", line 519, in <module>
|
||||
main(args)
|
||||
File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\examples\offline_inference\minicpmo\end2end.py", line 283, in main
|
||||
omni = Omni(**omni_kwargs)
|
||||
^^^^^^^^^^^^^^^^^^^
|
||||
File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\entrypoints\omni_base.py", line 223, in __init__
|
||||
self.engine = AsyncOmniEngine(
|
||||
^^^^^^^^^^^^^^^^
|
||||
File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 305, in __init__
|
||||
self._wait_for_orchestrator_init(startup_future, startup_timeout)
|
||||
File "\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\vllm-omni-edge\vllm_omni\engine\async_omni_engine.py", line 537, in _wait_for_orchestrator_init
|
||||
raise TimeoutError(f"Orchestrator did not become ready within {startup_timeout}s")
|
||||
TimeoutError: Orchestrator did not become ready within 300s
|
||||
+267
@@ -0,0 +1,267 @@
|
||||
INFO 09-23 16:30:42 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 16:30:42 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
====================
|
||||
vllm version: 0.29.0
|
||||
====================
|
||||
INFO 09-23 16:30:47 [omni_base.py:216] [Omni] Initializing with model \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\MiniCPM-o-4_5
|
||||
INFO 09-23 16:30:47 [async_omni_engine.py:172] [AsyncOmniEngine] Initializing with model \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\MiniCPM-o-4_5
|
||||
INFO 09-23 16:30:48 [async_omni_engine.py:290] [AsyncOmniEngine] Launching Orchestrator thread with 3 stages
|
||||
INFO 09-23 16:30:48 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
INFO 09-23 16:30:48 [initialization.py:356] Loaded OmniTransferConfig with 1 connector configurations
|
||||
WARNING 09-23 16:30:48 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 16:30:48 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
WARNING 09-23 16:30:48 [arg_utils.py:1801] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM.
|
||||
INFO 09-23 16:30:48 [model.py:684] Resolved architecture: MiniCPMO45OmniForConditionalGeneration
|
||||
INFO 09-23 16:30:48 [model.py:2021] Using max model len 2048
|
||||
WARNING 09-23 16:30:48 [model.py:981] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False.
|
||||
INFO 09-23 16:30:51 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode.
|
||||
INFO 09-23 16:30:51 [model.py:887] Disabled mm_prefix attention mode because multimodal inputs are configuration-disabled. Attention backends without mm_prefix support may now be selected.
|
||||
INFO 09-23 16:30:51 [scheduler.py:277] Chunked prefill is enabled with max_num_batched_tokens=2048.
|
||||
WARNING 09-23 16:30:51 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
WARNING 09-23 16:30:51 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
INFO 09-23 16:30:51 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-23 16:30:51 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
INFO 09-23 16:30:51 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
WARNING 09-23 16:30:51 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 16:30:51 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
WARNING 09-23 16:30:52 [arg_utils.py:1801] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM.
|
||||
INFO 09-23 16:30:52 [model.py:684] Resolved architecture: MiniCPMO45OmniForConditionalGeneration
|
||||
INFO 09-23 16:30:52 [model.py:2021] Using max model len 512
|
||||
WARNING 09-23 16:30:52 [model.py:981] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False.
|
||||
INFO 09-23 16:30:52 [model.py:2021] Using max model len 512
|
||||
INFO 09-23 16:30:52 [scheduler.py:277] Chunked prefill is enabled with max_num_batched_tokens=512.
|
||||
INFO 09-23 16:30:52 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
WARNING 09-23 16:30:52 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 16:30:52 [utils.py:501] Dropping unknown OmniEngineArgs field 'shm_threshold_bytes' (not declared on the dataclass)
|
||||
WARNING 09-23 16:30:52 [arg_utils.py:1801] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM.
|
||||
INFO 09-23 16:30:52 [model.py:684] Resolved architecture: MiniCPMO45Code2Wav
|
||||
INFO 09-23 16:30:52 [model.py:2021] Using max model len 4096
|
||||
WARNING 09-23 16:30:52 [arg_utils.py:2780] This model does not officially support disabling chunked prefill. Disabling this manually may cause the engine to crash or produce incorrect outputs.
|
||||
INFO 09-23 16:30:53 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-23 16:30:53 [stage_runtime.py:853] [stage_init] Stage-0 set runtime devices: 0
|
||||
INFO 09-23 16:31:08 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 16:31:08 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176) INFO 09-23 16:31:12 [core.py:123] Initializing a V1 LLM engine (v0.29.0) with config: model='\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\MiniCPM-o-4_5', speculative_config=None, tokenizer='\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=2048, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 0, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176) INFO 09-23 16:31:15 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176) INFO 09-23 16:31:15 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file://C:\Users\zhout\AppData\Local\Temp/vllm_dist_53967da026804fc7a1199a7d00af3066 backend=nccl
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176) WARNING 09-23 16:31:15 [parallel_state.py:1788] Distributed backend nccl is not available; falling back to gloo.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176) INFO 09-23 16:31:15 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176) WARNING 09-23 16:31:16 [gpu_ar_worker.py:120] OMNI GPUARWorker forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176) INFO 09-23 16:31:16 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176) INFO 09-23 16:31:16 [base.py:138] Offloader set to UVAOffloader
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176) WARNING 09-23 16:31:16 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176) WARNING 09-23 16:31:16 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176) INFO 09-23 16:31:16 [gpu_model_runner.py:5379] Starting to load model \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\MiniCPM-o-4_5...
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176) WARNING 09-23 16:31:16 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176) WARNING 09-23 16:31:16 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176) INFO 09-23 16:31:16 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176) INFO 09-23 16:31:16 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176) INFO 09-23 16:31:16 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176) INFO 09-23 16:31:16 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176) INFO 09-23 16:31:16 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176) INFO 09-23 16:31:17 [cuda.py:492] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176) INFO 09-23 16:31:17 [flash_attn.py:897] Using FlashAttention version 2
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176) INFO 09-23 16:31:20 [uva.py:65] Total CPU offloaded parameters: 5.03
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176) INFO 09-23 16:31:20 [weight_utils.py:863] Filesystem type for checkpoints: unknown. Checkpoint size: 17.46 GiB. Available RAM: 27.79 GiB.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176) INFO 09-23 16:31:20 [weight_utils.py:886] Auto-prefetch is disabled because the filesystem (unknown) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176)
|
||||
Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176)
|
||||
Loading safetensors checkpoint shards: 25% Completed | 1/4 [00:00<00:00, 3.03it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176)
|
||||
Loading safetensors checkpoint shards: 50% Completed | 2/4 [00:00<00:00, 2.29it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176)
|
||||
Loading safetensors checkpoint shards: 75% Completed | 3/4 [00:01<00:00, 2.39it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:01<00:00, 2.13it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:01<00:00, 2.23it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176) INFO 09-23 16:33:15 [default_loader.py:430] Loading weights took 114.86 seconds
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176) INFO 09-23 16:33:17 [gpu_model_runner.py:5471] Model loading took 10.84 GiB memory and 119.227534 seconds
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176) INFO 09-23 16:33:17 [utils.py:306] Using LBNHC KV cache layout.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176) INFO 09-23 16:33:54 [base.py:148] Available KV cache memory: 2.71 GiB (device-level profiling)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176) INFO 09-23 16:33:54 [kv_cache_utils.py:2032] GPU KV cache size: 19,760 tokens, Maximum concurrency for 2,048 tokens per request: 9.65x
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176) WARNING 09-23 16:33:54 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176) INFO 09-23 16:33:54 [kernel_warmup.py:124] JIT kernel warmup starting.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176)
|
||||
JIT kernel warmup (1 compile keys): 0%| | 0/1 [00:00<?, ?kernel/s]
|
||||
JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 5.51kernel/s, ComputeSlotMappingKernel (1 keys)]
|
||||
JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 5.48kernel/s, ComputeSlotMappingKernel (1 keys)]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176) INFO 09-23 16:33:55 [kernel_warmup.py:134] JIT kernel warmup finished in 0.19s.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176) INFO 09-23 16:33:55 [ll_bf16.py:30] cuteDSL (CUTLASS Python) not available, ll_bf16_gemm disabled
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176) INFO 09-23 16:33:55 [gpu_worker.py:860] Free memory on device (22.61/23.89 GiB) on startup. Desired GPU memory utilization is (0.58, 13.86 GiB). Actual usage is 11.01 GiB for consumed memory (weights + non-torch), 0.22 GiB for peak activation, and 0.0 GiB for CUDAGraph memory. Replace gpu_memory_utilization config with `--kv-cache-memory=2665780347` (2.48 GiB) to fit into requested memory, or `--kv-cache-memory=12063866880` (11.24 GiB) to fully utilize gpu memory. Current kv cache memory in use is 2.71 GiB.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176) INFO 09-23 16:33:56 [jit_monitor.py:85] Kernel JIT monitor activated; monitored JIT compilations during inference will use mode=warn.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176) WARNING 09-23 16:33:56 [torch_utils.py:265] OMP_NUM_THREADS=8 is set; leaving Torch threads at 8 for serving. Multi-threaded torch CPU ops during serving can degrade performance through spin-wait contention and cgroup CPU-quota throttling.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176) INFO 09-23 16:33:56 [core.py:368] init engine (profile, create kv cache, warmup model) took 38.94 s
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176) WARNING 09-23 16:33:56 [scheduler.py:213] Using custom scheduler class vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176) WARNING 09-23 16:33:56 [omni_scheduler_mixin.py:118] OMNI scheduler forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176) INFO 09-23 16:33:56 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176) INFO 09-23 16:33:56 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-23 16:33:56 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed
|
||||
INFO 09-23 16:33:56 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-0 [rep-0] initializing EngineCore
|
||||
INFO 09-23 16:33:57 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-0 [rep-0] EngineCore running
|
||||
INFO 09-23 16:33:57 [stage_runtime.py:958] [StageRuntime] Stage 0 initialized
|
||||
INFO 09-23 16:33:57 [stage_runtime.py:853] [stage_init] Stage-1 set runtime devices: 0
|
||||
INFO 09-23 16:34:18 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 16:34:18 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484) INFO 09-23 16:34:23 [core.py:123] Initializing a V1 LLM engine (v0.29.0) with config: model='\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\MiniCPM-o-4_5', speculative_config=None, tokenizer='\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=512, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [512], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 0, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484) INFO 09-23 16:34:26 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file://C:\Users\zhout\AppData\Local\Temp/vllm_dist_a346e62d6aed4005983a9bf545a9460f backend=nccl
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484) WARNING 09-23 16:34:26 [parallel_state.py:1788] Distributed backend nccl is not available; falling back to gloo.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484) INFO 09-23 16:34:26 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484) WARNING 09-23 16:34:27 [gpu_ar_worker.py:120] OMNI GPUARWorker forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484) INFO 09-23 16:34:27 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484) [transformers] `MiniCPMOProcessor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484) [transformers] Requested torchvision backend is not available. Falling back to pil backend.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484) [transformers] `MiniCPMOProcessor` defines `feature_extractor_class = 'WhisperFeatureExtractor'`, which is deprecated. Register the correct mapping in `AutoFeatureExtractor` instead.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484) [transformers] Image processor MiniCPMVImageProcessor: kwargs ['max_slice_nums'] were applied for backward compatibility. To avoid this warning, add them to valid_kwargs: create a custom TypedDict extending ImagesKwargs with these keys and set it as the `valid_kwargs` class attribute.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484) INFO 09-23 16:34:43 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484) WARNING 09-23 16:34:43 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484) WARNING 09-23 16:34:43 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484) INFO 09-23 16:34:43 [gpu_model_runner.py:5379] Starting to load model \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\MiniCPM-o-4_5...
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484) WARNING 09-23 16:34:43 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484) WARNING 09-23 16:34:43 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484) INFO 09-23 16:34:43 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484) INFO 09-23 16:34:43 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484) INFO 09-23 16:34:44 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484) INFO 09-23 16:34:44 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484) INFO 09-23 16:34:44 [cuda.py:492] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484) INFO 09-23 16:34:44 [flash_attn.py:897] Using FlashAttention version 2
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484) INFO 09-23 16:34:44 [weight_utils.py:863] Filesystem type for checkpoints: unknown. Checkpoint size: 17.46 GiB. Available RAM: 26.39 GiB.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484) INFO 09-23 16:34:44 [weight_utils.py:886] Auto-prefetch is disabled because the filesystem (unknown) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484)
|
||||
Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484)
|
||||
Loading safetensors checkpoint shards: 25% Completed | 1/4 [00:00<00:00, 3.75it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484)
|
||||
Loading safetensors checkpoint shards: 50% Completed | 2/4 [00:00<00:00, 4.83it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484)
|
||||
Loading safetensors checkpoint shards: 75% Completed | 3/4 [00:00<00:00, 5.65it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:01<00:00, 3.56it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:01<00:00, 3.95it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484) INFO 09-23 16:34:50 [default_loader.py:430] Loading weights took 6.47 seconds
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484) INFO 09-23 16:34:51 [gpu_model_runner.py:5471] Model loading took 0.65 GiB memory and 7.013238 seconds
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484) INFO 09-23 16:34:52 [utils.py:306] Using LBNHC KV cache layout.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484) INFO 09-23 16:34:52 [gpu_model_runner.py:6519] Skipping memory profiling for multimodal encoder and encoder cache.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484) INFO 09-23 16:34:53 [kv_cache_utils.py:2032] GPU KV cache size: 4,368 tokens, Maximum concurrency for 512 tokens per request: 8.53x
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484) WARNING 09-23 16:34:53 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484) INFO 09-23 16:34:53 [kernel_warmup.py:124] JIT kernel warmup starting.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484)
|
||||
JIT kernel warmup (1 compile keys): 0%| | 0/1 [00:00<?, ?kernel/s]
|
||||
JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 5.67kernel/s, ComputeSlotMappingKernel (1 keys)]
|
||||
JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 5.67kernel/s, ComputeSlotMappingKernel (1 keys)]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484) INFO 09-23 16:34:53 [kernel_warmup.py:134] JIT kernel warmup finished in 0.18s.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484) INFO 09-23 16:34:53 [ll_bf16.py:30] cuteDSL (CUTLASS Python) not available, ll_bf16_gemm disabled
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484) INFO 09-23 16:34:53 [jit_monitor.py:85] Kernel JIT monitor activated; monitored JIT compilations during inference will use mode=warn.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484) WARNING 09-23 16:34:54 [torch_utils.py:265] OMP_NUM_THREADS=8 is set; leaving Torch threads at 8 for serving. Multi-threaded torch CPU ops during serving can degrade performance through spin-wait contention and cgroup CPU-quota throttling.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484) INFO 09-23 16:34:54 [core.py:368] init engine (profile, create kv cache, warmup model) took 2.00 s
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484) WARNING 09-23 16:34:54 [scheduler.py:213] Using custom scheduler class vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484) WARNING 09-23 16:34:54 [omni_scheduler_mixin.py:118] OMNI scheduler forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484) INFO 09-23 16:34:54 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
INFO 09-23 16:34:54 [stage_runtime.py:943] [StageRuntime] Stage 1 engine startup completed
|
||||
INFO 09-23 16:34:54 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-1 [rep-0] initializing EngineCore
|
||||
(StageEngineCoreProc_stage1_replica0 pid=58484) INFO 09-23 16:34:54 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-23 16:34:55 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-1 [rep-0] EngineCore running
|
||||
INFO 09-23 16:34:55 [stage_runtime.py:958] [StageRuntime] Stage 1 initialized
|
||||
INFO 09-23 16:34:55 [stage_runtime.py:853] [stage_init] Stage-2 set runtime devices: 0
|
||||
INFO 09-23 16:35:11 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 16:35:11 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) INFO 09-23 16:35:15 [core.py:123] Initializing a V1 LLM engine (v0.29.0) with config: model='\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\MiniCPM-o-4_5', speculative_config=None, tokenizer='\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=4096, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=False, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [4096], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 0, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) INFO 09-23 16:35:17 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file://C:\Users\zhout\AppData\Local\Temp/vllm_dist_136def56607542418cb1ca141c7c28fb backend=nccl
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) WARNING 09-23 16:35:17 [parallel_state.py:1788] Distributed backend nccl is not available; falling back to gloo.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) INFO 09-23 16:35:17 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) WARNING 09-23 16:35:17 [gpu_generation_worker.py:103] OMNI GPUGenerationWorker forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) INFO 09-23 16:35:17 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) INFO 09-23 16:35:17 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) WARNING 09-23 16:35:17 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) WARNING 09-23 16:35:17 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) INFO 09-23 16:35:17 [gpu_model_runner.py:5379] Starting to load model \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\MiniCPM-o-4_5...
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) INFO 09-23 16:35:18 [weight_utils.py:863] Filesystem type for checkpoints: unknown. Checkpoint size: 17.46 GiB. Available RAM: 23.92 GiB.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) INFO 09-23 16:35:18 [weight_utils.py:886] Auto-prefetch is disabled because the filesystem (unknown) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372)
|
||||
Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372)
|
||||
Loading safetensors checkpoint shards: 25% Completed | 1/4 [00:00<00:00, 6.24it/s]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372)
|
||||
Loading safetensors checkpoint shards: 75% Completed | 3/4 [00:00<00:00, 9.46it/s]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:00<00:00, 6.07it/s]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:00<00:00, 6.56it/s]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) INFO 09-23 16:35:19 [step_audio2_token2wav.py:163] Loading Token2Wav models from: \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\MiniCPM-o-4_5\assets\token2wav
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) INFO 09-23 16:35:35 [step_audio2_token2wav.py:201] Token2Wav models loaded successfully
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) INFO 09-23 16:35:36 [batched_token2wav.py:227] Enabling HiFT CUDA Graph with batch sizes [1]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) INFO 09-23 16:35:37 [cuda_graph_wrapper.py:98] Captured HiFT CUDA Graph for shape (1, 50, 0)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) INFO 09-23 16:35:37 [cuda_graph_wrapper.py:98] Captured HiFT CUDA Graph for shape (1, 58, 3840)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) INFO 09-23 16:35:37 [batched_token2wav.py:235] HiFT CUDA Graph captured successfully
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) INFO 09-23 16:35:37 [batched_token2wav.py:246] CFM CUDA Graph enabled (max_graphs=32)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) INFO 09-23 16:35:37 [default_loader.py:430] Loading weights took 18.78 seconds
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) INFO 09-23 16:35:38 [gpu_model_runner.py:5471] Model loading took 3.36 GiB memory and 19.256970 seconds
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) INFO 09-23 16:35:39 [cuda.py:492] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) INFO 09-23 16:35:39 [utils.py:306] Using LBNHC KV cache layout.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) WARNING 09-23 16:35:39 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) INFO 09-23 16:35:39 [kernel_warmup.py:124] JIT kernel warmup starting.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372)
|
||||
JIT kernel warmup (1 compile keys): 0%| | 0/1 [00:00<?, ?kernel/s]
|
||||
JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 142.93kernel/s, ComputeSlotMappingKernel (1 keys)]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) INFO 09-23 16:35:39 [kernel_warmup.py:134] JIT kernel warmup finished in 0.01s.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) INFO 09-23 16:35:39 [ll_bf16.py:30] cuteDSL (CUTLASS Python) not available, ll_bf16_gemm disabled
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) WARNING 09-23 16:35:39 [gpu_generation_model_runner.py:558] Dummy sampler run is not implemented for generation model
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) INFO 09-23 16:35:39 [jit_monitor.py:85] Kernel JIT monitor activated; monitored JIT compilations during inference will use mode=warn.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) WARNING 09-23 16:35:40 [torch_utils.py:265] OMP_NUM_THREADS=8 is set; leaving Torch threads at 8 for serving. Multi-threaded torch CPU ops during serving can degrade performance through spin-wait contention and cgroup CPU-quota throttling.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) INFO 09-23 16:35:40 [core.py:368] init engine (profile, create kv cache, warmup model) took 0.88 s
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) WARNING 09-23 16:35:41 [scheduler.py:213] Using custom scheduler class vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) WARNING 09-23 16:35:41 [omni_scheduler_mixin.py:118] OMNI scheduler forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) INFO 09-23 16:35:41 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) WARNING 09-23 16:35:41 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) WARNING 09-23 16:35:41 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) INFO 09-23 16:35:41 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) INFO 09-23 16:35:41 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
INFO 09-23 16:35:41 [stage_runtime.py:943] [StageRuntime] Stage 2 engine startup completed
|
||||
INFO 09-23 16:35:41 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-2 [rep-0] initializing EngineCore
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) INFO 09-23 16:35:41 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
INFO 09-23 16:35:41 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-2 [rep-0] EngineCore running
|
||||
INFO 09-23 16:35:41 [stage_runtime.py:958] [StageRuntime] Stage 2 initialized
|
||||
INFO 09-23 16:35:44 [orchestrator.py:604] [Orchestrator] Starting event loop
|
||||
INFO 09-23 16:35:44 [async_omni_engine.py:323] [AsyncOmniEngine] Orchestrator ready with 3 stages
|
||||
INFO 09-23 16:35:44 [omni_base.py:235] [Omni] AsyncOmniEngine initialized in 297.20 seconds
|
||||
INFO 09-23 16:35:44 [omni_base.py:255] [Omni] Initialized with 3 stages for model \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\MiniCPM-o-4_5
|
||||
WARNING 09-23 16:35:44 [input_processor.py:315] Passing raw prompts to InputProcessor is deprecated and will be removed in the future. You should instead pass the outputs of Renderer.render_cmpl() or Renderer.render_chat().
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=62176) WARNING 09-23 16:35:44 [gpu_ar_model_runner.py:1464] prefer_model_sampler model MiniCPMO45OmniForConditionalGeneration returned None from sample(); falling back to the default sampler.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) INFO 09-23 16:36:53 [cuda_graph_wrapper.py:300] Captured CFM CUDA Graph for shape ('estimator_step', ((2, 320, 302), 'torch.float32', 'cuda:0'), ((2, 1, 512), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 0, 128), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 302, 128), 'torch.float32', 'cuda:0')) (cache=1/32, stats={'calls': 1, 'hits': 0, 'captures': 1, 'flushes': 0, 'eager': 0, 'cache_size': 0}) [allocated 3633.9 MiB (+32.2), reserved 5464.0 MiB (+22.0)]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) INFO 09-23 16:36:57 [cuda_graph_wrapper.py:300] Captured CFM CUDA Graph for shape ('estimator_step', ((2, 320, 50), 'torch.float32', 'cuda:0'), ((2, 1, 512), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 302, 128), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 352, 128), 'torch.float32', 'cuda:0')) (cache=2/32, stats={'calls': 11, 'hits': 9, 'captures': 2, 'flushes': 0, 'eager': 0, 'cache_size': 1}) [allocated 4539.7 MiB (+32.0), reserved 6378.0 MiB (+28.0)]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) INFO 09-23 16:36:58 [cuda_graph_wrapper.py:300] Captured CFM CUDA Graph for shape ('estimator_step', ((2, 320, 50), 'torch.float32', 'cuda:0'), ((2, 1, 512), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 352, 128), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 402, 128), 'torch.float32', 'cuda:0')) (cache=3/32, stats={'calls': 21, 'hits': 18, 'captures': 3, 'flushes': 0, 'eager': 0, 'cache_size': 2}) [allocated 4803.9 MiB (+32.0), reserved 6956.0 MiB (+-352.0)]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) INFO 09-23 16:36:59 [cuda_graph_wrapper.py:300] Captured CFM CUDA Graph for shape ('estimator_step', ((2, 320, 50), 'torch.float32', 'cuda:0'), ((2, 1, 512), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 402, 128), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 452, 128), 'torch.float32', 'cuda:0')) (cache=4/32, stats={'calls': 31, 'hits': 27, 'captures': 4, 'flushes': 0, 'eager': 0, 'cache_size': 3}) [allocated 5079.1 MiB (+32.0), reserved 7096.0 MiB (+-876.0)]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) INFO 09-23 16:37:02 [cuda_graph_wrapper.py:300] Captured CFM CUDA Graph for shape ('estimator_step', ((2, 320, 10), 'torch.float32', 'cuda:0'), ((2, 1, 512), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 402, 128), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 412, 128), 'torch.float32', 'cuda:0')) (cache=5/32, stats={'calls': 61, 'hits': 56, 'captures': 5, 'flushes': 0, 'eager': 0, 'cache_size': 4}) [allocated 5206.6 MiB (+32.0), reserved 7190.0 MiB (+-1550.0)]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) INFO 09-23 16:37:02 [cuda_graph_wrapper.py:120] Lazily capturing HiFT CUDA Graph for shape (1, 18, 3840)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) WARNING 09-23 16:37:02 [jit_monitor.py:141] Triton kernel JIT compilation during inference: SnakeBeta._init_triton.<locals>._kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=59372) INFO 09-23 16:37:03 [cuda_graph_wrapper.py:98] Captured HiFT CUDA Graph for shape (1, 18, 3840)
|
||||
Processed prompts: 0%| | 0/1 [00:00<?, ?it/s]
|
||||
Processed prompts: 100%|██████████| 1/1 [01:18<00:00, 78.75s/it]
|
||||
Processed prompts: 100%|██████████| 1/1 [01:18<00:00, 78.75s/it]
|
||||
query type: text
|
||||
use_tts (tts_bos): True
|
||||
Request ID: 0_b4835d8d-f4f8-48b9-85d2-bb7ef06d9249, Text saved to C:\Users\zhout\w2\minicpmo_native_gpu\0_b4835d8d-f4f8-48b9-85d2-bb7ef06d9249.txt
|
||||
Request ID: 0_b4835d8d-f4f8-48b9-85d2-bb7ef06d9249, Saved audio to C:\Users\zhout\w2\minicpmo_native_gpu\output_0_b4835d8d-f4f8-48b9-85d2-bb7ef06d9249.wav
|
||||
INFO 09-23 16:37:03 [omni_base.py:781] [Omni] Shutting down
|
||||
INFO 09-23 16:37:03 [async_omni_engine.py:1807] [AsyncOmniEngine] Shutting down Orchestrator
|
||||
INFO 09-23 16:37:03 [orchestrator.py:705] [Orchestrator] Received shutdown signal
|
||||
INFO 09-23 16:37:03 [orchestrator.py:3092] [Orchestrator] Shutting down all 3 client(s)
|
||||
INFO 09-23 16:37:03 [core_client.py:689] [shutdown] MPClient: start timeout=default
|
||||
INFO 09-23 16:37:03 [core_client.py:691] [shutdown] MPClient: stopping engine manager
|
||||
INFO 09-23 16:37:03 [utils.py:626] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage0_replica0
|
||||
INFO 09-23 16:37:03 [core_client.py:693] [shutdown] MPClient: engine manager stopped
|
||||
INFO 09-23 16:37:03 [core_client.py:694] [shutdown] MPClient: cleaning up background resources
|
||||
INFO 09-23 16:37:03 [core_client.py:696] [shutdown] MPClient: complete
|
||||
INFO 09-23 16:37:03 [stage_pool.py:1354] [StagePool] Stage 0 replica 0 shut down
|
||||
INFO 09-23 16:37:03 [core_client.py:689] [shutdown] MPClient: start timeout=default
|
||||
INFO 09-23 16:37:03 [utils.py:626] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage1_replica0
|
||||
INFO 09-23 16:37:04 [stage_pool.py:1354] [StagePool] Stage 1 replica 0 shut down
|
||||
INFO 09-23 16:37:04 [core_client.py:689] [shutdown] MPClient: start timeout=default
|
||||
INFO 09-23 16:37:04 [utils.py:626] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage2_replica0
|
||||
INFO 09-23 16:37:04 [stage_pool.py:1354] [StagePool] Stage 2 replica 0 shut down
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"scope": "native Windows RTX 5090 Laptop, real checkpoint, one text-to-text+speech request; no speech quality claim",
|
||||
"text": "Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video.",
|
||||
"token_count": 32,
|
||||
"token_ids_match_wsl_cuda": true,
|
||||
"text_match_wsl_cuda": true,
|
||||
"wav": {
|
||||
"sample_rate_hz": 24000,
|
||||
"frames": 124800,
|
||||
"channels": 1,
|
||||
"duration_s": 5.2,
|
||||
"nonzero_samples": 124560,
|
||||
"peak_abs_float": 0.577606201171875,
|
||||
"rms_float": 0.10467266091641916,
|
||||
"sha256": "279d1ca9bb637b4da7d38872b10d5425bca22e2ab09f8091cf1f2539e220657f"
|
||||
}
|
||||
}
|
||||
+1205
File diff suppressed because it is too large.
Load diff
+21
@@ -0,0 +1,21 @@
|
||||
{"request_ids": ["0_70278de0-179d-43a4-95cd-6edf445459a3"], "wall_s": 22.788874699966982, "first_text_s": 22.786775399930775, "first_audio_s": 22.786779499961995, "text": "Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video.", "token_ids": [9707, 11, 358, 1079, 348, 4086, 44, 23620, 76, 7751, 11, 264, 3460, 79049, 57597, 1614, 12875, 315, 8660, 323, 23163, 1467, 11, 5335, 11, 7699, 11, 323, 2766, 13, 151704, 151645], "audio_samples": 124800, "audio_duration_s_at_24khz": 5.2, "audio_rms_float": 0.10467265753187106, "sampled_memory": {"scope": "native_os", "n": 43, "max_system_used_bytes": 62498856960, "min_system_available_bytes": 5274652672, "max_swap_used_bytes": 8699838464, "max_process_tree_rss_sum_bytes": 16430612480}, "index": 0, "warmup": true}
|
||||
{"request_ids": ["0_701e501d-c847-42d4-a482-ba33839e5fd6"], "wall_s": 19.793629699968733, "first_text_s": 19.79311460000463, "first_audio_s": 19.79311820003204, "text": "Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video.", "token_ids": [9707, 11, 358, 1079, 348, 4086, 44, 23620, 76, 7751, 11, 264, 3460, 79049, 57597, 1614, 12875, 315, 8660, 323, 23163, 1467, 11, 5335, 11, 7699, 11, 323, 2766, 13, 151704, 151645], "audio_samples": 124800, "audio_duration_s_at_24khz": 5.2, "audio_rms_float": 0.10466819862292, "sampled_memory": {"scope": "native_os", "n": 38, "max_system_used_bytes": 66618683392, "min_system_available_bytes": 1154826240, "max_swap_used_bytes": 8595886080, "max_process_tree_rss_sum_bytes": 20498448384}, "index": 1, "warmup": false}
|
||||
{"request_ids": ["0_6099e01d-73ab-4588-b11e-b8b296620eaa"], "wall_s": 18.726146699977107, "first_text_s": 18.725689699989744, "first_audio_s": 18.725691400002688, "text": "Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video.", "token_ids": [9707, 11, 358, 1079, 348, 4086, 44, 23620, 76, 7751, 11, 264, 3460, 79049, 57597, 1614, 12875, 315, 8660, 323, 23163, 1467, 11, 5335, 11, 7699, 11, 323, 2766, 13, 151704, 151645], "audio_samples": 124800, "audio_duration_s_at_24khz": 5.2, "audio_rms_float": 0.10466938958454462, "sampled_memory": {"scope": "native_os", "n": 36, "max_system_used_bytes": 66980012032, "min_system_available_bytes": 793497600, "max_swap_used_bytes": 8584896512, "max_process_tree_rss_sum_bytes": 21013266432}, "index": 2, "warmup": false}
|
||||
{"request_ids": ["0_dcf0a6a7-3f41-45ad-bb8d-dcda9fcabf17"], "wall_s": 18.394545300048776, "first_text_s": 18.394298400031403, "first_audio_s": 18.394300000043586, "text": "Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video.", "token_ids": [9707, 11, 358, 1079, 348, 4086, 44, 23620, 76, 7751, 11, 264, 3460, 79049, 57597, 1614, 12875, 315, 8660, 323, 23163, 1467, 11, 5335, 11, 7699, 11, 323, 2766, 13, 151704, 151645], "audio_samples": 124800, "audio_duration_s_at_24khz": 5.2, "audio_rms_float": 0.10466382640758563, "sampled_memory": {"scope": "native_os", "n": 35, "max_system_used_bytes": 66948526080, "min_system_available_bytes": 824983552, "max_swap_used_bytes": 8584847360, "max_process_tree_rss_sum_bytes": 21074481152}, "index": 3, "warmup": false}
|
||||
{"request_ids": ["0_b68e2c88-64b3-437b-8606-b85b02da5684"], "wall_s": 18.615840599988587, "first_text_s": 18.615528600057587, "first_audio_s": 18.615530400071293, "text": "Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video.", "token_ids": [9707, 11, 358, 1079, 348, 4086, 44, 23620, 76, 7751, 11, 264, 3460, 79049, 57597, 1614, 12875, 315, 8660, 323, 23163, 1467, 11, 5335, 11, 7699, 11, 323, 2766, 13, 151704, 151645], "audio_samples": 124800, "audio_duration_s_at_24khz": 5.2, "audio_rms_float": 0.10466829307595575, "sampled_memory": {"scope": "native_os", "n": 35, "max_system_used_bytes": 66998358016, "min_system_available_bytes": 775151616, "max_swap_used_bytes": 9450921984, "max_process_tree_rss_sum_bytes": 21083639808}, "index": 4, "warmup": false}
|
||||
{"request_ids": ["0_00e07b1b-23f9-492a-95dc-b442fed9d75f"], "wall_s": 18.38139640004374, "first_text_s": 18.381037300103344, "first_audio_s": 18.381039100000635, "text": "Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video.", "token_ids": [9707, 11, 358, 1079, 348, 4086, 44, 23620, 76, 7751, 11, 264, 3460, 79049, 57597, 1614, 12875, 315, 8660, 323, 23163, 1467, 11, 5335, 11, 7699, 11, 323, 2766, 13, 151704, 151645], "audio_samples": 124800, "audio_duration_s_at_24khz": 5.2, "audio_rms_float": 0.10467388138893752, "sampled_memory": {"scope": "native_os", "n": 35, "max_system_used_bytes": 61600636928, "min_system_available_bytes": 6172872704, "max_swap_used_bytes": 11146727424, "max_process_tree_rss_sum_bytes": 17739722752}, "index": 5, "warmup": false}
|
||||
{"request_ids": ["0_990d8d9a-6074-42c6-9209-bfebf78ca469"], "wall_s": 18.53084919997491, "first_text_s": 18.530396799906157, "first_audio_s": 18.530399299925193, "text": "Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video.", "token_ids": [9707, 11, 358, 1079, 348, 4086, 44, 23620, 76, 7751, 11, 264, 3460, 79049, 57597, 1614, 12875, 315, 8660, 323, 23163, 1467, 11, 5335, 11, 7699, 11, 323, 2766, 13, 151704, 151645], "audio_samples": 124800, "audio_duration_s_at_24khz": 5.2, "audio_rms_float": 0.10466815512997239, "sampled_memory": {"scope": "native_os", "n": 35, "max_system_used_bytes": 61453783040, "min_system_available_bytes": 6319726592, "max_swap_used_bytes": 11173052416, "max_process_tree_rss_sum_bytes": 18005409792}, "index": 6, "warmup": false}
|
||||
{"request_ids": ["0_ef06d207-ac8a-4056-a6fa-e050e0c99171"], "wall_s": 18.48581220000051, "first_text_s": 18.485562800080515, "first_audio_s": 18.485564300091937, "text": "Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video.", "token_ids": [9707, 11, 358, 1079, 348, 4086, 44, 23620, 76, 7751, 11, 264, 3460, 79049, 57597, 1614, 12875, 315, 8660, 323, 23163, 1467, 11, 5335, 11, 7699, 11, 323, 2766, 13, 151704, 151645], "audio_samples": 124800, "audio_duration_s_at_24khz": 5.2, "audio_rms_float": 0.10466932563844575, "sampled_memory": {"scope": "native_os", "n": 36, "max_system_used_bytes": 61566926848, "min_system_available_bytes": 6206582784, "max_swap_used_bytes": 11172913152, "max_process_tree_rss_sum_bytes": 18043736064}, "index": 7, "warmup": false}
|
||||
{"request_ids": ["0_a28dface-a68f-4bba-86fd-89f7c48a0580"], "wall_s": 18.31971340009477, "first_text_s": 18.319422600092366, "first_audio_s": 18.319424100103788, "text": "Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video.", "token_ids": [9707, 11, 358, 1079, 348, 4086, 44, 23620, 76, 7751, 11, 264, 3460, 79049, 57597, 1614, 12875, 315, 8660, 323, 23163, 1467, 11, 5335, 11, 7699, 11, 323, 2766, 13, 151704, 151645], "audio_samples": 124800, "audio_duration_s_at_24khz": 5.2, "audio_rms_float": 0.10466680439707206, "sampled_memory": {"scope": "native_os", "n": 34, "max_system_used_bytes": 61515964416, "min_system_available_bytes": 6257545216, "max_swap_used_bytes": 11066531840, "max_process_tree_rss_sum_bytes": 18045808640}, "index": 8, "warmup": false}
|
||||
{"request_ids": ["0_fa29a415-44d3-4788-bd23-b2b1ee5879eb"], "wall_s": 18.507400500006042, "first_text_s": 18.50691050000023, "first_audio_s": 18.50691330002155, "text": "Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video.", "token_ids": [9707, 11, 358, 1079, 348, 4086, 44, 23620, 76, 7751, 11, 264, 3460, 79049, 57597, 1614, 12875, 315, 8660, 323, 23163, 1467, 11, 5335, 11, 7699, 11, 323, 2766, 13, 151704, 151645], "audio_samples": 124800, "audio_duration_s_at_24khz": 5.2, "audio_rms_float": 0.1046625739599199, "sampled_memory": {"scope": "native_os", "n": 36, "max_system_used_bytes": 61658324992, "min_system_available_bytes": 6115184640, "max_swap_used_bytes": 11062472704, "max_process_tree_rss_sum_bytes": 18105544704}, "index": 9, "warmup": false}
|
||||
{"request_ids": ["0_00b10b06-16e7-4930-a445-205633ed6a8e"], "wall_s": 18.278693299973384, "first_text_s": 18.278446500073187, "first_audio_s": 18.278447899967432, "text": "Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video.", "token_ids": [9707, 11, 358, 1079, 348, 4086, 44, 23620, 76, 7751, 11, 264, 3460, 79049, 57597, 1614, 12875, 315, 8660, 323, 23163, 1467, 11, 5335, 11, 7699, 11, 323, 2766, 13, 151704, 151645], "audio_samples": 124800, "audio_duration_s_at_24khz": 5.2, "audio_rms_float": 0.1046701869388329, "sampled_memory": {"scope": "native_os", "n": 35, "max_system_used_bytes": 61773553664, "min_system_available_bytes": 5999955968, "max_swap_used_bytes": 11061014528, "max_process_tree_rss_sum_bytes": 18244509696}, "index": 10, "warmup": false}
|
||||
{"request_ids": ["0_37bbde93-93c8-487a-b547-111c06abce24"], "wall_s": 18.26734599994961, "first_text_s": 18.267101099947467, "first_audio_s": 18.26710269995965, "text": "Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video.", "token_ids": [9707, 11, 358, 1079, 348, 4086, 44, 23620, 76, 7751, 11, 264, 3460, 79049, 57597, 1614, 12875, 315, 8660, 323, 23163, 1467, 11, 5335, 11, 7699, 11, 323, 2766, 13, 151704, 151645], "audio_samples": 124800, "audio_duration_s_at_24khz": 5.2, "audio_rms_float": 0.10466189606698147, "sampled_memory": {"scope": "native_os", "n": 34, "max_system_used_bytes": 61546921984, "min_system_available_bytes": 6226587648, "max_swap_used_bytes": 11060998144, "max_process_tree_rss_sum_bytes": 18239041536}, "index": 11, "warmup": false}
|
||||
{"request_ids": ["0_09451eda-ec4d-4066-b3d1-c24fc65bc1f7"], "wall_s": 18.22269750002306, "first_text_s": 18.222456899937242, "first_audio_s": 18.22245869995095, "text": "Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video.", "token_ids": [9707, 11, 358, 1079, 348, 4086, 44, 23620, 76, 7751, 11, 264, 3460, 79049, 57597, 1614, 12875, 315, 8660, 323, 23163, 1467, 11, 5335, 11, 7699, 11, 323, 2766, 13, 151704, 151645], "audio_samples": 124800, "audio_duration_s_at_24khz": 5.2, "audio_rms_float": 0.10466721712576618, "sampled_memory": {"scope": "native_os", "n": 35, "max_system_used_bytes": 61385695232, "min_system_available_bytes": 6387814400, "max_swap_used_bytes": 11060297728, "max_process_tree_rss_sum_bytes": 18052141056}, "index": 12, "warmup": false}
|
||||
{"request_ids": ["0_6ab743b9-ca85-4bba-bd49-cb279cf8ff1a"], "wall_s": 18.294015699997544, "first_text_s": 18.293660400086083, "first_audio_s": 18.293662100099027, "text": "Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video.", "token_ids": [9707, 11, 358, 1079, 348, 4086, 44, 23620, 76, 7751, 11, 264, 3460, 79049, 57597, 1614, 12875, 315, 8660, 323, 23163, 1467, 11, 5335, 11, 7699, 11, 323, 2766, 13, 151704, 151645], "audio_samples": 124800, "audio_duration_s_at_24khz": 5.2, "audio_rms_float": 0.1046654124080557, "sampled_memory": {"scope": "native_os", "n": 35, "max_system_used_bytes": 61566234624, "min_system_available_bytes": 6207275008, "max_swap_used_bytes": 11060248576, "max_process_tree_rss_sum_bytes": 18120994816}, "index": 13, "warmup": false}
|
||||
{"request_ids": ["0_6ae60d4a-6518-4740-ac4c-c8aa0e12be6e"], "wall_s": 18.241102600004524, "first_text_s": 18.240880000055768, "first_audio_s": 18.240881499950774, "text": "Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video.", "token_ids": [9707, 11, 358, 1079, 348, 4086, 44, 23620, 76, 7751, 11, 264, 3460, 79049, 57597, 1614, 12875, 315, 8660, 323, 23163, 1467, 11, 5335, 11, 7699, 11, 323, 2766, 13, 151704, 151645], "audio_samples": 124800, "audio_duration_s_at_24khz": 5.2, "audio_rms_float": 0.10467125038087612, "sampled_memory": {"scope": "native_os", "n": 35, "max_system_used_bytes": 61577641984, "min_system_available_bytes": 6195867648, "max_swap_used_bytes": 11057086464, "max_process_tree_rss_sum_bytes": 18069032960}, "index": 14, "warmup": false}
|
||||
{"request_ids": ["0_5db7f55f-2067-42a2-b064-1b83bbc596a8"], "wall_s": 18.361572599969804, "first_text_s": 18.361045999918133, "first_audio_s": 18.361048699938692, "text": "Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video.", "token_ids": [9707, 11, 358, 1079, 348, 4086, 44, 23620, 76, 7751, 11, 264, 3460, 79049, 57597, 1614, 12875, 315, 8660, 323, 23163, 1467, 11, 5335, 11, 7699, 11, 323, 2766, 13, 151704, 151645], "audio_samples": 124800, "audio_duration_s_at_24khz": 5.2, "audio_rms_float": 0.1046722734473641, "sampled_memory": {"scope": "native_os", "n": 35, "max_system_used_bytes": 61761769472, "min_system_available_bytes": 6011740160, "max_swap_used_bytes": 11051286528, "max_process_tree_rss_sum_bytes": 18112528384}, "index": 15, "warmup": false}
|
||||
{"request_ids": ["0_2556d238-308a-44eb-b401-e66bf42c0a0a"], "wall_s": 18.404275199980475, "first_text_s": 18.404027500073425, "first_audio_s": 18.40402920008637, "text": "Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video.", "token_ids": [9707, 11, 358, 1079, 348, 4086, 44, 23620, 76, 7751, 11, 264, 3460, 79049, 57597, 1614, 12875, 315, 8660, 323, 23163, 1467, 11, 5335, 11, 7699, 11, 323, 2766, 13, 151704, 151645], "audio_samples": 124800, "audio_duration_s_at_24khz": 5.2, "audio_rms_float": 0.10466386893342523, "sampled_memory": {"scope": "native_os", "n": 35, "max_system_used_bytes": 61959766016, "min_system_available_bytes": 5813743616, "max_swap_used_bytes": 11046825984, "max_process_tree_rss_sum_bytes": 18114121728}, "index": 16, "warmup": false}
|
||||
{"request_ids": ["0_1f33a496-051d-46d6-8ca3-1d088a0109e2"], "wall_s": 18.31900940008927, "first_text_s": 18.318775900057517, "first_audio_s": 18.318777300068177, "text": "Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video.", "token_ids": [9707, 11, 358, 1079, 348, 4086, 44, 23620, 76, 7751, 11, 264, 3460, 79049, 57597, 1614, 12875, 315, 8660, 323, 23163, 1467, 11, 5335, 11, 7699, 11, 323, 2766, 13, 151704, 151645], "audio_samples": 124800, "audio_duration_s_at_24khz": 5.2, "audio_rms_float": 0.10466608086224949, "sampled_memory": {"scope": "native_os", "n": 35, "max_system_used_bytes": 61935316992, "min_system_available_bytes": 5838192640, "max_swap_used_bytes": 11046154240, "max_process_tree_rss_sum_bytes": 18114887680}, "index": 17, "warmup": false}
|
||||
{"request_ids": ["0_ac2828af-3eec-454b-942d-eea337a58d8c"], "wall_s": 18.361741599976085, "first_text_s": 18.36128319997806, "first_audio_s": 18.361284899991006, "text": "Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video.", "token_ids": [9707, 11, 358, 1079, 348, 4086, 44, 23620, 76, 7751, 11, 264, 3460, 79049, 57597, 1614, 12875, 315, 8660, 323, 23163, 1467, 11, 5335, 11, 7699, 11, 323, 2766, 13, 151704, 151645], "audio_samples": 124800, "audio_duration_s_at_24khz": 5.2, "audio_rms_float": 0.10466168358281563, "sampled_memory": {"scope": "native_os", "n": 35, "max_system_used_bytes": 62047383552, "min_system_available_bytes": 5726126080, "max_swap_used_bytes": 11043921920, "max_process_tree_rss_sum_bytes": 18125848576}, "index": 18, "warmup": false}
|
||||
{"request_ids": ["0_0dbb7125-99f9-49a2-9f2b-cbf2033998c4"], "wall_s": 18.41667849989608, "first_text_s": 18.416337699978612, "first_audio_s": 18.41633949999232, "text": "Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video.", "token_ids": [9707, 11, 358, 1079, 348, 4086, 44, 23620, 76, 7751, 11, 264, 3460, 79049, 57597, 1614, 12875, 315, 8660, 323, 23163, 1467, 11, 5335, 11, 7699, 11, 323, 2766, 13, 151704, 151645], "audio_samples": 124800, "audio_duration_s_at_24khz": 5.2, "audio_rms_float": 0.10467158776491159, "sampled_memory": {"scope": "native_os", "n": 35, "max_system_used_bytes": 62368432128, "min_system_available_bytes": 5405077504, "max_swap_used_bytes": 11043516416, "max_process_tree_rss_sum_bytes": 18259714048}, "index": 19, "warmup": false}
|
||||
{"request_ids": ["0_9eca9083-db7c-4821-a4b2-10ae7990c8a2"], "wall_s": 18.625242199981585, "first_text_s": 18.624819200020283, "first_audio_s": 18.624822200043127, "text": "Hello, I am vLLM-Omni, a large multimodal model capable of understanding and generating text, images, audio, and video.", "token_ids": [9707, 11, 358, 1079, 348, 4086, 44, 23620, 76, 7751, 11, 264, 3460, 79049, 57597, 1614, 12875, 315, 8660, 323, 23163, 1467, 11, 5335, 11, 7699, 11, 323, 2766, 13, 151704, 151645], "audio_samples": 124800, "audio_duration_s_at_24khz": 5.2, "audio_rms_float": 0.10467051067278418, "sampled_memory": {"scope": "native_os", "n": 35, "max_system_used_bytes": 62423117824, "min_system_available_bytes": 5350391808, "max_swap_used_bytes": 11043033088, "max_process_tree_rss_sum_bytes": 18261417984}, "index": 20, "warmup": false}
|
||||
+359
@@ -0,0 +1,359 @@
|
||||
INFO 09-23 16:39:18 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 16:39:18 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
INFO 09-23 16:39:21 [omni_base.py:216] [Omni] Initializing with model \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\MiniCPM-o-4_5
|
||||
INFO 09-23 16:39:22 [async_omni_engine.py:172] [AsyncOmniEngine] Initializing with model \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\MiniCPM-o-4_5
|
||||
INFO 09-23 16:39:22 [async_omni_engine.py:290] [AsyncOmniEngine] Launching Orchestrator thread with 3 stages
|
||||
INFO 09-23 16:39:23 [stage_init_utils.py:768] [stage_init] Set VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
INFO 09-23 16:39:23 [initialization.py:356] Loaded OmniTransferConfig with 1 connector configurations
|
||||
WARNING 09-23 16:39:23 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 16:39:23 [arg_utils.py:1801] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM.
|
||||
INFO 09-23 16:39:23 [model.py:684] Resolved architecture: MiniCPMO45OmniForConditionalGeneration
|
||||
INFO 09-23 16:39:23 [model.py:2021] Using max model len 2048
|
||||
WARNING 09-23 16:39:23 [model.py:981] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False.
|
||||
INFO 09-23 16:39:24 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode.
|
||||
INFO 09-23 16:39:24 [model.py:887] Disabled mm_prefix attention mode because multimodal inputs are configuration-disabled. Attention backends without mm_prefix support may now be selected.
|
||||
INFO 09-23 16:39:24 [scheduler.py:277] Chunked prefill is enabled with max_num_batched_tokens=2048.
|
||||
WARNING 09-23 16:39:24 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
WARNING 09-23 16:39:24 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
INFO 09-23 16:39:24 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-23 16:39:24 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
INFO 09-23 16:39:24 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
WARNING 09-23 16:39:24 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 16:39:25 [arg_utils.py:1801] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM.
|
||||
INFO 09-23 16:39:25 [model.py:684] Resolved architecture: MiniCPMO45OmniForConditionalGeneration
|
||||
INFO 09-23 16:39:25 [model.py:2021] Using max model len 512
|
||||
WARNING 09-23 16:39:25 [model.py:981] Model does not support mm_device_do_normalize, forcing mm_device_do_normalize = False.
|
||||
INFO 09-23 16:39:25 [model.py:2021] Using max model len 512
|
||||
INFO 09-23 16:39:25 [scheduler.py:277] Chunked prefill is enabled with max_num_batched_tokens=512.
|
||||
INFO 09-23 16:39:25 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
WARNING 09-23 16:39:25 [utils.py:501] Dropping unknown OmniEngineArgs field 'inline_diffusion' (not declared on the dataclass)
|
||||
WARNING 09-23 16:39:25 [arg_utils.py:1801] The global random seed is set to 0. Since VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may affect the random state of the Python process that launched vLLM.
|
||||
INFO 09-23 16:39:26 [model.py:684] Resolved architecture: MiniCPMO45Code2Wav
|
||||
INFO 09-23 16:39:26 [model.py:2021] Using max model len 4096
|
||||
WARNING 09-23 16:39:26 [arg_utils.py:2780] This model does not officially support disabling chunked prefill. Disabling this manually may cause the engine to crash or produce incorrect outputs.
|
||||
INFO 09-23 16:39:26 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-23 16:39:26 [stage_runtime.py:853] [stage_init] Stage-0 set runtime devices: 0
|
||||
INFO 09-23 16:39:37 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 16:39:38 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364) INFO 09-23 16:39:41 [core.py:123] Initializing a V1 LLM engine (v0.29.0) with config: model='\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\MiniCPM-o-4_5', speculative_config=None, tokenizer='\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=2048, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [2048], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 0, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364) INFO 09-23 16:39:43 [registry.py:141] All limits of multimodal modalities supported by the model are set to 0, running in text-only mode.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364) INFO 09-23 16:39:43 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file://C:\Users\zhout\AppData\Local\Temp/vllm_dist_f3f857b1fafb42afad38d136cada03d8 backend=nccl
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364) WARNING 09-23 16:39:43 [parallel_state.py:1788] Distributed backend nccl is not available; falling back to gloo.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364) INFO 09-23 16:39:43 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364) WARNING 09-23 16:39:44 [gpu_ar_worker.py:120] OMNI GPUARWorker forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364) INFO 09-23 16:39:44 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364) INFO 09-23 16:39:44 [base.py:138] Offloader set to UVAOffloader
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364) WARNING 09-23 16:39:44 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364) WARNING 09-23 16:39:44 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364) INFO 09-23 16:39:44 [gpu_model_runner.py:5379] Starting to load model \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\MiniCPM-o-4_5...
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364) WARNING 09-23 16:39:44 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364) WARNING 09-23 16:39:44 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364) INFO 09-23 16:39:44 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364) INFO 09-23 16:39:44 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364) INFO 09-23 16:39:44 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364) INFO 09-23 16:39:44 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364) INFO 09-23 16:39:44 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364) INFO 09-23 16:39:45 [cuda.py:492] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364) INFO 09-23 16:39:45 [flash_attn.py:897] Using FlashAttention version 2
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364) INFO 09-23 16:39:46 [uva.py:65] Total CPU offloaded parameters: 5.03
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364) INFO 09-23 16:39:47 [weight_utils.py:863] Filesystem type for checkpoints: unknown. Checkpoint size: 17.46 GiB. Available RAM: 23.26 GiB.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364) INFO 09-23 16:39:47 [weight_utils.py:886] Auto-prefetch is disabled because the filesystem (unknown) is not a recognized network FS (NFS/Lustre). If you want to force prefetching, start vLLM with --safetensors-load-strategy=prefetch.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364)
|
||||
Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364)
|
||||
Loading safetensors checkpoint shards: 25% Completed | 1/4 [00:00<00:00, 5.65it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364)
|
||||
Loading safetensors checkpoint shards: 50% Completed | 2/4 [00:00<00:00, 3.20it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364)
|
||||
Loading safetensors checkpoint shards: 75% Completed | 3/4 [00:00<00:00, 3.67it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:01<00:00, 3.63it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:01<00:00, 3.67it/s]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364) INFO 09-23 16:41:40 [default_loader.py:430] Loading weights took 113.81 seconds
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364) INFO 09-23 16:41:42 [gpu_model_runner.py:5471] Model loading took 10.84 GiB memory and 116.234515 seconds
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364) INFO 09-23 16:41:42 [utils.py:306] Using LBNHC KV cache layout.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364) INFO 09-23 16:41:53 [base.py:148] Available KV cache memory: 2.71 GiB (device-level profiling)
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364) INFO 09-23 16:41:53 [kv_cache_utils.py:2032] GPU KV cache size: 19,760 tokens, Maximum concurrency for 2,048 tokens per request: 9.65x
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364) WARNING 09-23 16:41:53 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364) INFO 09-23 16:41:53 [kernel_warmup.py:124] JIT kernel warmup starting.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364)
|
||||
JIT kernel warmup (1 compile keys): 0%| | 0/1 [00:00<?, ?kernel/s]
|
||||
JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 7.39kernel/s, ComputeSlotMappingKernel (1 keys)]
|
||||
JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 7.39kernel/s, ComputeSlotMappingKernel (1 keys)]
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364) INFO 09-23 16:41:53 [kernel_warmup.py:134] JIT kernel warmup finished in 0.14s.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364) INFO 09-23 16:41:53 [ll_bf16.py:30] cuteDSL (CUTLASS Python) not available, ll_bf16_gemm disabled
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364) INFO 09-23 16:41:53 [gpu_worker.py:860] Free memory on device (22.61/23.89 GiB) on startup. Desired GPU memory utilization is (0.58, 13.86 GiB). Actual usage is 11.01 GiB for consumed memory (weights + non-torch), 0.22 GiB for peak activation, and 0.0 GiB for CUDAGraph memory. Replace gpu_memory_utilization config with `--kv-cache-memory=2665780347` (2.48 GiB) to fit into requested memory, or `--kv-cache-memory=12063866880` (11.24 GiB) to fully utilize gpu memory. Current kv cache memory in use is 2.71 GiB.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364) INFO 09-23 16:41:53 [jit_monitor.py:85] Kernel JIT monitor activated; monitored JIT compilations during inference will use mode=warn.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364) WARNING 09-23 16:41:54 [torch_utils.py:265] OMP_NUM_THREADS=8 is set; leaving Torch threads at 8 for serving. Multi-threaded torch CPU ops during serving can degrade performance through spin-wait contention and cgroup CPU-quota throttling.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364) INFO 09-23 16:41:54 [core.py:368] init engine (profile, create kv cache, warmup model) took 11.89 s
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364) WARNING 09-23 16:41:54 [scheduler.py:213] Using custom scheduler class vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364) WARNING 09-23 16:41:54 [omni_scheduler_mixin.py:118] OMNI scheduler forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364) INFO 09-23 16:41:54 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364) INFO 09-23 16:41:54 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-23 16:41:54 [stage_runtime.py:943] [StageRuntime] Stage 0 engine startup completed
|
||||
INFO 09-23 16:41:54 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-0 [rep-0] initializing EngineCore
|
||||
INFO 09-23 16:41:54 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-0 [rep-0] EngineCore running
|
||||
INFO 09-23 16:41:54 [stage_runtime.py:958] [StageRuntime] Stage 0 initialized
|
||||
INFO 09-23 16:41:54 [stage_runtime.py:853] [stage_init] Stage-1 set runtime devices: 0
|
||||
INFO 09-23 16:42:09 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 16:42:09 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376) INFO 09-23 16:42:13 [core.py:123] Initializing a V1 LLM engine (v0.29.0) with config: model='\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\MiniCPM-o-4_5', speculative_config=None, tokenizer='\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=512, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=True, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [512], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 0, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376) INFO 09-23 16:42:16 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file://C:\Users\zhout\AppData\Local\Temp/vllm_dist_3dcd7151d4db4756a5d71f8cc6beb2e4 backend=nccl
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376) WARNING 09-23 16:42:16 [parallel_state.py:1788] Distributed backend nccl is not available; falling back to gloo.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376) INFO 09-23 16:42:16 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376) WARNING 09-23 16:42:16 [gpu_ar_worker.py:120] OMNI GPUARWorker forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376) INFO 09-23 16:42:16 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376) [transformers] `MiniCPMOProcessor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376) [transformers] Requested torchvision backend is not available. Falling back to pil backend.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376) [transformers] `MiniCPMOProcessor` defines `feature_extractor_class = 'WhisperFeatureExtractor'`, which is deprecated. Register the correct mapping in `AutoFeatureExtractor` instead.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376) [transformers] Image processor MiniCPMVImageProcessor: kwargs ['max_slice_nums'] were applied for backward compatibility. To avoid this warning, add them to valid_kwargs: create a custom TypedDict extending ImagesKwargs with these keys and set it as the `valid_kwargs` class attribute.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376) INFO 09-23 16:42:28 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376) WARNING 09-23 16:42:29 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376) WARNING 09-23 16:42:29 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376) INFO 09-23 16:42:29 [gpu_model_runner.py:5379] Starting to load model \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\MiniCPM-o-4_5...
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376) WARNING 09-23 16:42:29 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376) WARNING 09-23 16:42:29 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376) INFO 09-23 16:42:29 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376) INFO 09-23 16:42:29 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376) INFO 09-23 16:42:29 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376) INFO 09-23 16:42:29 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376) INFO 09-23 16:42:29 [cuda.py:492] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376) INFO 09-23 16:42:29 [flash_attn.py:897] Using FlashAttention version 2
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376) INFO 09-23 16:42:29 [weight_utils.py:863] Filesystem type for checkpoints: unknown. Checkpoint size: 17.46 GiB. Available RAM: 14.27 GiB.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376) INFO 09-23 16:42:29 [weight_utils.py:893] Auto-prefetch is disabled because the filesystem (unknown) is not a recognized network FS (NFS/Lustre) and the checkpoint size (17.46 GiB) exceeds 90% of available RAM (14.27 GiB).
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376)
|
||||
Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376)
|
||||
Loading safetensors checkpoint shards: 25% Completed | 1/4 [00:00<00:00, 4.82it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376)
|
||||
Loading safetensors checkpoint shards: 50% Completed | 2/4 [00:00<00:01, 1.93it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376)
|
||||
Loading safetensors checkpoint shards: 75% Completed | 3/4 [00:01<00:00, 2.24it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:01<00:00, 2.14it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:01<00:00, 2.22it/s]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376)
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376) INFO 09-23 16:42:38 [default_loader.py:430] Loading weights took 8.21 seconds
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376) INFO 09-23 16:42:38 [gpu_model_runner.py:5471] Model loading took 0.65 GiB memory and 8.665011 seconds
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376) INFO 09-23 16:42:39 [utils.py:306] Using LBNHC KV cache layout.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376) INFO 09-23 16:42:39 [gpu_model_runner.py:6519] Skipping memory profiling for multimodal encoder and encoder cache.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376) INFO 09-23 16:42:39 [kv_cache_utils.py:2032] GPU KV cache size: 4,368 tokens, Maximum concurrency for 512 tokens per request: 8.53x
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376) WARNING 09-23 16:42:39 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376) INFO 09-23 16:42:39 [kernel_warmup.py:124] JIT kernel warmup starting.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376)
|
||||
JIT kernel warmup (1 compile keys): 0%| | 0/1 [00:00<?, ?kernel/s]
|
||||
JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 9.34kernel/s, ComputeSlotMappingKernel (1 keys)]
|
||||
JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 9.34kernel/s, ComputeSlotMappingKernel (1 keys)]
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376) INFO 09-23 16:42:39 [kernel_warmup.py:134] JIT kernel warmup finished in 0.11s.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376) INFO 09-23 16:42:40 [ll_bf16.py:30] cuteDSL (CUTLASS Python) not available, ll_bf16_gemm disabled
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376) INFO 09-23 16:42:40 [jit_monitor.py:85] Kernel JIT monitor activated; monitored JIT compilations during inference will use mode=warn.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376) WARNING 09-23 16:42:40 [torch_utils.py:265] OMP_NUM_THREADS=8 is set; leaving Torch threads at 8 for serving. Multi-threaded torch CPU ops during serving can degrade performance through spin-wait contention and cgroup CPU-quota throttling.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376) INFO 09-23 16:42:40 [core.py:368] init engine (profile, create kv cache, warmup model) took 1.52 s
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376) WARNING 09-23 16:42:40 [scheduler.py:213] Using custom scheduler class vllm_omni.core.sched.omni_ar_scheduler.OmniARAsyncScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376) WARNING 09-23 16:42:40 [omni_scheduler_mixin.py:118] OMNI scheduler forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376) INFO 09-23 16:42:40 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage1_replica0 pid=42376) INFO 09-23 16:42:40 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
INFO 09-23 16:42:40 [stage_runtime.py:943] [StageRuntime] Stage 1 engine startup completed
|
||||
INFO 09-23 16:42:40 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-1 [rep-0] initializing EngineCore
|
||||
INFO 09-23 16:42:41 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-1 [rep-0] EngineCore running
|
||||
INFO 09-23 16:42:41 [stage_runtime.py:958] [StageRuntime] Stage 1 initialized
|
||||
INFO 09-23 16:42:41 [stage_runtime.py:853] [stage_init] Stage-2 set runtime devices: 0
|
||||
INFO 09-23 16:42:53 [patch.py:259] NVFP4 W4A4 weight_scale NaN-clamp: installed.
|
||||
INFO 09-23 16:42:53 [patch.py:510] inductor factorable-divisibility patch: installed.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) INFO 09-23 16:42:57 [core.py:123] Initializing a V1 LLM engine (v0.29.0) with config: model='\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\MiniCPM-o-4_5', speculative_config=None, tokenizer='\\\\wsl.localhost\\Ubuntu\\home\\zhout\\project\\edge_infer\\models\\MiniCPM-o-4_5', skip_tokenizer_init=False, tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.bfloat16, max_seq_len=4096, download_dir=None, load_format=auto, tensor_parallel_size=1, pipeline_parallel_size=1, data_parallel_size=1, decode_context_parallel_size=1, dcp_comm_backend=ag_rs, disable_custom_all_reduce=False, quantization=None, quantization_config=None, enforce_eager=True, enable_return_routed_experts=False, kv_cache_dtype=auto, device_config=cuda, structured_outputs_config=StructuredOutputsConfig(backend='auto', disable_any_whitespace=False, disable_additional_properties=False, reasoning_parser='', reasoning_parser_plugin='', enable_in_reasoning=False), observability_config=ObservabilityConfig(show_hidden_metrics_for_version=None, otlp_traces_endpoint=None, collect_detailed_traces=None, per_request_spec_decode_metrics='none', kv_cache_metrics=False, kv_cache_metrics_sample=0.01, cudagraph_metrics=False, enable_layerwise_nvtx_tracing=False, enable_mfu_metrics=False, enable_mm_processor_stats=False, enable_logging_iteration_details=False, jit_monitor_mode='warn', jit_monitor_verbose=False), seed=0, served_model_name=\\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\MiniCPM-o-4_5, enable_prefix_caching=False, enable_chunked_prefill=False, pooler_config=None, compilation_config={'mode': <CompilationMode.NONE: 0>, 'debug_dump_path': None, 'cache_dir': '', 'compile_cache_save_format': 'binary', 'backend': 'inductor', 'custom_ops': ['all'], 'ir_enable_torch_wrap': False, 'splitting_ops': [], 'compile_mm_encoder': False, 'cudagraph_mm_encoder': False, 'encoder_cudagraph_token_budgets': [], 'encoder_cudagraph_max_vision_items_per_batch': 0, 'encoder_cudagraph_max_frames_per_batch': None, 'compile_sizes': [], 'compile_ranges_endpoints': [4096], 'inductor_compile_config': {'enable_auto_functionalized_v2': False, 'combo_kernels': True, 'benchmark_combo_kernel': True}, 'inductor_passes': {}, 'cudagraph_mode': <CUDAGraphMode.NONE: 0>, 'cudagraph_num_of_warmups': 0, 'cudagraph_capture_sizes': [], 'cudagraph_copy_inputs': False, 'cudagraph_specialize_lora': True, 'use_inductor_graph_partition': False, 'pass_config': {'fuse_norm_quant': True, 'fuse_act_quant': True, 'fuse_attn_quant': False, 'enable_sp': False, 'fuse_gemm_comms': False, 'fuse_allreduce_rms': False, 'enable_qk_norm_rope_fusion': False, 'fuse_rope_kvcache_cat_mla': False, 'fuse_act_padding': False, 'fuse_qk_norm_rope_kvcache': False}, 'max_cudagraph_capture_size': 0, 'dynamic_shapes_config': {'type': <DynamicShapesType.BACKED: 'backed'>, 'evaluate_guards': False, 'assume_32_bit_indexing': False}, 'local_cache_dir': None, 'fast_moe_cold_start': False, 'static_all_moe_layers': []}, kernel_config=KernelConfig(ir_op_priority=IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native']), enable_flashinfer_autotune=True, enable_cutedsl_warmup=True, enable_jit_warmup=True, enable_bf16x3_router_gemm=False, moe_backend='auto', linear_backend='auto')
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) INFO 09-23 16:42:58 [parallel_state.py:1775] world_size=1 rank=0 local_rank=0 distributed_init_method=file://C:\Users\zhout\AppData\Local\Temp/vllm_dist_6fe666f458a54616af06bcd6b9a314b2 backend=nccl
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) WARNING 09-23 16:42:58 [parallel_state.py:1788] Distributed backend nccl is not available; falling back to gloo.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) INFO 09-23 16:42:58 [parallel_state.py:2119] rank 0 in world size 1 is assigned as DP rank 0, PP rank 0, PCP rank 0, TP rank 0, EP rank N/A, EPLB rank N/A
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) WARNING 09-23 16:42:58 [gpu_generation_worker.py:103] OMNI GPUGenerationWorker forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) INFO 09-23 16:42:58 [topk_topp_sampler.py:46] FlashInfer top-p/top-k sampling disabled via VLLM_USE_FLASHINFER_SAMPLER=0.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) INFO 09-23 16:42:58 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) WARNING 09-23 16:42:58 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) WARNING 09-23 16:42:58 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) INFO 09-23 16:42:58 [gpu_model_runner.py:5379] Starting to load model \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\MiniCPM-o-4_5...
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) INFO 09-23 16:42:59 [weight_utils.py:863] Filesystem type for checkpoints: unknown. Checkpoint size: 17.46 GiB. Available RAM: 12.14 GiB.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) INFO 09-23 16:42:59 [weight_utils.py:893] Auto-prefetch is disabled because the filesystem (unknown) is not a recognized network FS (NFS/Lustre) and the checkpoint size (17.46 GiB) exceeds 90% of available RAM (12.14 GiB).
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720)
|
||||
Loading safetensors checkpoint shards: 0% Completed | 0/4 [00:00<?, ?it/s]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720)
|
||||
Loading safetensors checkpoint shards: 25% Completed | 1/4 [00:00<00:00, 4.89it/s]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720)
|
||||
Loading safetensors checkpoint shards: 75% Completed | 3/4 [00:00<00:00, 8.52it/s]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:00<00:00, 5.76it/s]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720)
|
||||
Loading safetensors checkpoint shards: 100% Completed | 4/4 [00:00<00:00, 6.08it/s]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) INFO 09-23 16:43:00 [step_audio2_token2wav.py:163] Loading Token2Wav models from: \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\MiniCPM-o-4_5\assets\token2wav
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) INFO 09-23 16:43:14 [step_audio2_token2wav.py:201] Token2Wav models loaded successfully
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) INFO 09-23 16:43:14 [batched_token2wav.py:227] Enabling HiFT CUDA Graph with batch sizes [1]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) INFO 09-23 16:43:15 [cuda_graph_wrapper.py:98] Captured HiFT CUDA Graph for shape (1, 50, 0)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) INFO 09-23 16:43:15 [cuda_graph_wrapper.py:98] Captured HiFT CUDA Graph for shape (1, 58, 3840)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) INFO 09-23 16:43:15 [batched_token2wav.py:235] HiFT CUDA Graph captured successfully
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) INFO 09-23 16:43:15 [batched_token2wav.py:246] CFM CUDA Graph enabled (max_graphs=32)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) INFO 09-23 16:43:15 [default_loader.py:430] Loading weights took 16.20 seconds
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) INFO 09-23 16:43:16 [gpu_model_runner.py:5471] Model loading took 3.36 GiB memory and 16.660434 seconds
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) INFO 09-23 16:43:17 [cuda.py:492] Using FLASH_ATTN attention backend out of potential backends: ['FLASH_ATTN', 'TRITON_ATTN', 'FLEX_ATTENTION'].
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) INFO 09-23 16:43:17 [utils.py:306] Using LBNHC KV cache layout.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) WARNING 09-23 16:43:17 [base.py:173] [LLM Worker 0] Sleep Mode DISABLED.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) INFO 09-23 16:43:17 [kernel_warmup.py:124] JIT kernel warmup starting.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720)
|
||||
JIT kernel warmup (1 compile keys): 0%| | 0/1 [00:00<?, ?kernel/s]
|
||||
JIT kernel warmup (1 compile keys): 100%|██████████| 1/1 [00:00<00:00, 249.75kernel/s, ComputeSlotMappingKernel (1 keys)]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) INFO 09-23 16:43:17 [kernel_warmup.py:134] JIT kernel warmup finished in 0.01s.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) INFO 09-23 16:43:17 [ll_bf16.py:30] cuteDSL (CUTLASS Python) not available, ll_bf16_gemm disabled
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) WARNING 09-23 16:43:17 [gpu_generation_model_runner.py:558] Dummy sampler run is not implemented for generation model
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) INFO 09-23 16:43:17 [jit_monitor.py:85] Kernel JIT monitor activated; monitored JIT compilations during inference will use mode=warn.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) WARNING 09-23 16:43:17 [torch_utils.py:265] OMP_NUM_THREADS=8 is set; leaving Torch threads at 8 for serving. Multi-threaded torch CPU ops during serving can degrade performance through spin-wait contention and cgroup CPU-quota throttling.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) INFO 09-23 16:43:17 [core.py:368] init engine (profile, create kv cache, warmup model) took 0.68 s
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) WARNING 09-23 16:43:18 [scheduler.py:213] Using custom scheduler class vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler. This scheduler interface is not public and compatibility may not be maintained. If you have subclassed Scheduler instead of AsyncScheduler, you will see degraded performance due to async scheduling being disabled.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) WARNING 09-23 16:43:18 [omni_scheduler_mixin.py:118] OMNI scheduler forces v1 model runner for omni hooks.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) INFO 09-23 16:43:18 [factory.py:46] Created connector: SharedMemoryConnector
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) WARNING 09-23 16:43:18 [vllm.py:1371] Enforce eager set, disabling torch.compile and CUDAGraphs. This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) WARNING 09-23 16:43:18 [vllm.py:1406] Inductor compilation was disabled by user settings, optimizations settings that are only active during inductor compilation will be ignored.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) INFO 09-23 16:43:18 [kernel.py:369] Final IR op priority after setting platform defaults: IrOpPriorityConfig(rms_norm=['vllm_c', 'native'], fused_add_rms_norm=['vllm_c', 'native'])
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) INFO 09-23 16:43:18 [vllm.py:1585] Cudagraph is disabled under eager mode
|
||||
INFO 09-23 16:43:18 [stage_runtime.py:943] [StageRuntime] Stage 2 engine startup completed
|
||||
INFO 09-23 16:43:18 [stage_engine_core_client.py:159] [StageEngineCoreClient] stage-2 [rep-0] initializing EngineCore
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) INFO 09-23 16:43:18 [compilation.py:329] Enabled custom fusions: norm_quant, act_quant
|
||||
INFO 09-23 16:43:19 [stage_engine_core_client.py:215] [StageEngineCoreClient] stage-2 [rep-0] EngineCore running
|
||||
INFO 09-23 16:43:19 [stage_runtime.py:958] [StageRuntime] Stage 2 initialized
|
||||
INFO 09-23 16:43:20 [orchestrator.py:604] [Orchestrator] Starting event loop
|
||||
INFO 09-23 16:43:21 [async_omni_engine.py:323] [AsyncOmniEngine] Orchestrator ready with 3 stages
|
||||
INFO 09-23 16:43:21 [omni_base.py:235] [Omni] AsyncOmniEngine initialized in 239.00 seconds
|
||||
INFO 09-23 16:43:21 [omni_base.py:255] [Omni] Initialized with 3 stages for model \\wsl.localhost\Ubuntu\home\zhout\project\edge_infer\models\MiniCPM-o-4_5
|
||||
WARNING 09-23 16:43:21 [input_processor.py:315] Passing raw prompts to InputProcessor is deprecated and will be removed in the future. You should instead pass the outputs of Renderer.render_cmpl() or Renderer.render_chat().
|
||||
|
||||
(StageEngineCoreProc_stage0_replica0 pid=51364) WARNING 09-23 16:43:21 [gpu_ar_model_runner.py:1464] prefer_model_sampler model MiniCPMO45OmniForConditionalGeneration returned None from sample(); falling back to the default sampler.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) INFO 09-23 16:43:40 [cuda_graph_wrapper.py:300] Captured CFM CUDA Graph for shape ('estimator_step', ((2, 320, 302), 'torch.float32', 'cuda:0'), ((2, 1, 512), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 0, 128), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 302, 128), 'torch.float32', 'cuda:0')) (cache=1/32, stats={'calls': 1, 'hits': 0, 'captures': 1, 'flushes': 0, 'eager': 0, 'cache_size': 0}) [allocated 3633.9 MiB (+32.2), reserved 5464.0 MiB (+22.0)]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) INFO 09-23 16:43:41 [cuda_graph_wrapper.py:300] Captured CFM CUDA Graph for shape ('estimator_step', ((2, 320, 50), 'torch.float32', 'cuda:0'), ((2, 1, 512), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 302, 128), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 352, 128), 'torch.float32', 'cuda:0')) (cache=2/32, stats={'calls': 11, 'hits': 9, 'captures': 2, 'flushes': 0, 'eager': 0, 'cache_size': 1}) [allocated 4539.7 MiB (+32.0), reserved 6378.0 MiB (+28.0)]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) INFO 09-23 16:43:42 [cuda_graph_wrapper.py:300] Captured CFM CUDA Graph for shape ('estimator_step', ((2, 320, 50), 'torch.float32', 'cuda:0'), ((2, 1, 512), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 352, 128), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 402, 128), 'torch.float32', 'cuda:0')) (cache=3/32, stats={'calls': 21, 'hits': 18, 'captures': 3, 'flushes': 0, 'eager': 0, 'cache_size': 2}) [allocated 4803.9 MiB (+32.0), reserved 6956.0 MiB (+-352.0)]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) INFO 09-23 16:43:42 [cuda_graph_wrapper.py:300] Captured CFM CUDA Graph for shape ('estimator_step', ((2, 320, 50), 'torch.float32', 'cuda:0'), ((2, 1, 512), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 402, 128), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 452, 128), 'torch.float32', 'cuda:0')) (cache=4/32, stats={'calls': 31, 'hits': 27, 'captures': 4, 'flushes': 0, 'eager': 0, 'cache_size': 3}) [allocated 5079.1 MiB (+32.0), reserved 7096.0 MiB (+-876.0)]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) INFO 09-23 16:43:43 [cuda_graph_wrapper.py:300] Captured CFM CUDA Graph for shape ('estimator_step', ((2, 320, 10), 'torch.float32', 'cuda:0'), ((2, 1, 512), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 402, 128), 'torch.float32', 'cuda:0'), ((16, 2, 1024, 2), 'torch.float32', 'cuda:0'), ((16, 2, 8, 412, 128), 'torch.float32', 'cuda:0')) (cache=5/32, stats={'calls': 61, 'hits': 56, 'captures': 5, 'flushes': 0, 'eager': 0, 'cache_size': 4}) [allocated 5206.6 MiB (+32.0), reserved 7190.0 MiB (+-1550.0)]
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) INFO 09-23 16:43:43 [cuda_graph_wrapper.py:120] Lazily capturing HiFT CUDA Graph for shape (1, 18, 3840)
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) WARNING 09-23 16:43:43 [jit_monitor.py:141] Triton kernel JIT compilation during inference: SnakeBeta._init_triton.<locals>._kernel. This causes a latency spike; consider extending warmup to cover this shape/config.
|
||||
(StageEngineCoreProc_stage2_replica0 pid=55720) INFO 09-23 16:43:43 [cuda_graph_wrapper.py:98] Captured HiFT CUDA Graph for shape (1, 18, 3840)
|
||||
Processed prompts: 0%| | 0/1 [00:00<?, ?it/s]
|
||||
Processed prompts: 100%|██████████| 1/1 [00:22<00:00, 22.75s/it]
|
||||
Processed prompts: 100%|██████████| 1/1 [00:22<00:00, 22.75s/it]
|
||||
request=0 warmup=True wall_s=22.789
|
||||
|
||||
Processed prompts: 0%| | 0/1 [00:00<?, ?it/s]
|
||||
Processed prompts: 100%|██████████| 1/1 [00:19<00:00, 19.79s/it]
|
||||
Processed prompts: 100%|██████████| 1/1 [00:19<00:00, 19.79s/it]
|
||||
request=1 warmup=False wall_s=19.794
|
||||
|
||||
Processed prompts: 0%| | 0/1 [00:00<?, ?it/s]
|
||||
Processed prompts: 100%|██████████| 1/1 [00:18<00:00, 18.72s/it]
|
||||
Processed prompts: 100%|██████████| 1/1 [00:18<00:00, 18.72s/it]
|
||||
request=2 warmup=False wall_s=18.726
|
||||
|
||||
Processed prompts: 0%| | 0/1 [00:00<?, ?it/s]
|
||||
Processed prompts: 100%|██████████| 1/1 [00:18<00:00, 18.39s/it]
|
||||
Processed prompts: 100%|██████████| 1/1 [00:18<00:00, 18.39s/it]
|
||||
request=3 warmup=False wall_s=18.395
|
||||
|
||||
Processed prompts: 0%| | 0/1 [00:00<?, ?it/s]
|
||||
Processed prompts: 100%|██████████| 1/1 [00:18<00:00, 18.61s/it]
|
||||
Processed prompts: 100%|██████████| 1/1 [00:18<00:00, 18.61s/it]
|
||||
request=4 warmup=False wall_s=18.616
|
||||
|
||||
Processed prompts: 0%| | 0/1 [00:00<?, ?it/s]
|
||||
Processed prompts: 100%|██████████| 1/1 [00:18<00:00, 18.38s/it]
|
||||
Processed prompts: 100%|██████████| 1/1 [00:18<00:00, 18.38s/it]
|
||||
request=5 warmup=False wall_s=18.381
|
||||
|
||||
Processed prompts: 0%| | 0/1 [00:00<?, ?it/s]
|
||||
Processed prompts: 100%|██████████| 1/1 [00:18<00:00, 18.53s/it]
|
||||
Processed prompts: 100%|██████████| 1/1 [00:18<00:00, 18.53s/it]
|
||||
request=6 warmup=False wall_s=18.531
|
||||
|
||||
Processed prompts: 0%| | 0/1 [00:00<?, ?it/s]
|
||||
Processed prompts: 100%|██████████| 1/1 [00:18<00:00, 18.48s/it]
|
||||
Processed prompts: 100%|██████████| 1/1 [00:18<00:00, 18.48s/it]
|
||||
request=7 warmup=False wall_s=18.486
|
||||
|
||||
Processed prompts: 0%| | 0/1 [00:00<?, ?it/s]
|
||||
Processed prompts: 100%|██████████| 1/1 [00:18<00:00, 18.32s/it]
|
||||
Processed prompts: 100%|██████████| 1/1 [00:18<00:00, 18.32s/it]
|
||||
request=8 warmup=False wall_s=18.320
|
||||
|
||||
Processed prompts: 0%| | 0/1 [00:00<?, ?it/s]
|
||||
Processed prompts: 100%|██████████| 1/1 [00:18<00:00, 18.51s/it]
|
||||
Processed prompts: 100%|██████████| 1/1 [00:18<00:00, 18.51s/it]
|
||||
request=9 warmup=False wall_s=18.507
|
||||
|
||||
Processed prompts: 0%| | 0/1 [00:00<?, ?it/s]
|
||||
Processed prompts: 100%|██████████| 1/1 [00:18<00:00, 18.28s/it]
|
||||
Processed prompts: 100%|██████████| 1/1 [00:18<00:00, 18.28s/it]
|
||||
request=10 warmup=False wall_s=18.279
|
||||
|
||||
Processed prompts: 0%| | 0/1 [00:00<?, ?it/s]
|
||||
Processed prompts: 100%|██████████| 1/1 [00:18<00:00, 18.27s/it]
|
||||
Processed prompts: 100%|██████████| 1/1 [00:18<00:00, 18.27s/it]
|
||||
request=11 warmup=False wall_s=18.267
|
||||
|
||||
Processed prompts: 0%| | 0/1 [00:00<?, ?it/s]
|
||||
Processed prompts: 100%|██████████| 1/1 [00:18<00:00, 18.22s/it]
|
||||
Processed prompts: 100%|██████████| 1/1 [00:18<00:00, 18.22s/it]
|
||||
request=12 warmup=False wall_s=18.223
|
||||
|
||||
Processed prompts: 0%| | 0/1 [00:00<?, ?it/s]
|
||||
Processed prompts: 100%|██████████| 1/1 [00:18<00:00, 18.29s/it]
|
||||
Processed prompts: 100%|██████████| 1/1 [00:18<00:00, 18.29s/it]
|
||||
request=13 warmup=False wall_s=18.294
|
||||
|
||||
Processed prompts: 0%| | 0/1 [00:00<?, ?it/s]
|
||||
Processed prompts: 100%|██████████| 1/1 [00:18<00:00, 18.24s/it]
|
||||
Processed prompts: 100%|██████████| 1/1 [00:18<00:00, 18.24s/it]
|
||||
request=14 warmup=False wall_s=18.241
|
||||
|
||||
Processed prompts: 0%| | 0/1 [00:00<?, ?it/s]
|
||||
Processed prompts: 100%|██████████| 1/1 [00:18<00:00, 18.36s/it]
|
||||
Processed prompts: 100%|██████████| 1/1 [00:18<00:00, 18.36s/it]
|
||||
request=15 warmup=False wall_s=18.362
|
||||
|
||||
Processed prompts: 0%| | 0/1 [00:00<?, ?it/s]
|
||||
Processed prompts: 100%|██████████| 1/1 [00:18<00:00, 18.40s/it]
|
||||
Processed prompts: 100%|██████████| 1/1 [00:18<00:00, 18.40s/it]
|
||||
request=16 warmup=False wall_s=18.404
|
||||
|
||||
Processed prompts: 0%| | 0/1 [00:00<?, ?it/s]
|
||||
Processed prompts: 100%|██████████| 1/1 [00:18<00:00, 18.32s/it]
|
||||
Processed prompts: 100%|██████████| 1/1 [00:18<00:00, 18.32s/it]
|
||||
request=17 warmup=False wall_s=18.319
|
||||
|
||||
Processed prompts: 0%| | 0/1 [00:00<?, ?it/s]
|
||||
Processed prompts: 100%|██████████| 1/1 [00:18<00:00, 18.36s/it]
|
||||
Processed prompts: 100%|██████████| 1/1 [00:18<00:00, 18.36s/it]
|
||||
request=18 warmup=False wall_s=18.362
|
||||
|
||||
Processed prompts: 0%| | 0/1 [00:00<?, ?it/s]
|
||||
Processed prompts: 100%|██████████| 1/1 [00:18<00:00, 18.41s/it]
|
||||
Processed prompts: 100%|██████████| 1/1 [00:18<00:00, 18.41s/it]
|
||||
request=19 warmup=False wall_s=18.417
|
||||
|
||||
Processed prompts: 0%| | 0/1 [00:00<?, ?it/s]
|
||||
Processed prompts: 100%|██████████| 1/1 [00:18<00:00, 18.62s/it]
|
||||
Processed prompts: 100%|██████████| 1/1 [00:18<00:00, 18.62s/it]
|
||||
request=20 warmup=False wall_s=18.625
|
||||
INFO 09-23 16:49:53 [omni_base.py:781] [Omni] Shutting down
|
||||
INFO 09-23 16:49:53 [async_omni_engine.py:1807] [AsyncOmniEngine] Shutting down Orchestrator
|
||||
INFO 09-23 16:49:53 [orchestrator.py:705] [Orchestrator] Received shutdown signal
|
||||
INFO 09-23 16:49:53 [orchestrator.py:3092] [Orchestrator] Shutting down all 3 client(s)
|
||||
INFO 09-23 16:49:53 [core_client.py:689] [shutdown] MPClient: start timeout=default
|
||||
INFO 09-23 16:49:53 [core_client.py:691] [shutdown] MPClient: stopping engine manager
|
||||
INFO 09-23 16:49:53 [utils.py:626] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage0_replica0
|
||||
INFO 09-23 16:49:54 [core_client.py:693] [shutdown] MPClient: engine manager stopped
|
||||
INFO 09-23 16:49:54 [core_client.py:694] [shutdown] MPClient: cleaning up background resources
|
||||
INFO 09-23 16:49:54 [core_client.py:696] [shutdown] MPClient: complete
|
||||
INFO 09-23 16:49:54 [stage_pool.py:1354] [StagePool] Stage 0 replica 0 shut down
|
||||
INFO 09-23 16:49:54 [core_client.py:689] [shutdown] MPClient: start timeout=default
|
||||
INFO 09-23 16:49:54 [utils.py:626] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage1_replica0
|
||||
INFO 09-23 16:49:54 [stage_pool.py:1354] [StagePool] Stage 1 replica 0 shut down
|
||||
INFO 09-23 16:49:54 [core_client.py:689] [shutdown] MPClient: start timeout=default
|
||||
INFO 09-23 16:49:54 [utils.py:626] [shutdown] Process manager: send sigterm to process StageEngineCoreProc_stage2_replica0
|
||||
INFO 09-23 16:49:55 [stage_pool.py:1354] [StagePool] Stage 2 replica 0 shut down
|
||||
Wrote C:\Users\zhout\w2\minicpmo_windows_gpu_profile20.json
|
||||
BIN
Binary file not shown.
@@ -1,6 +1,6 @@
|
||||
# E2E check and profiling
|
||||
|
||||
All 60 pairings have a disposition. The timing tables below remain the 2026-09-22 measurements. The configuration matrix and affected per-cell reasons incorporate the [2026-09-23 scoped recovery runs](../../../e2e_recovery_20260923/README.md) and [CPU expansion](../../../e2e_expansion_20260923/README.md). The CPU expansion adds a separate 20-request serial MiniCPM-o text-to-speech profile; other new paths are single-request or short-stream runs. Profiling completion does not imply full release qualification.
|
||||
All 60 pairings have a disposition. The timing tables below remain the 2026-09-22 measurements. The configuration matrix and affected per-cell reasons incorporate the [2026-09-23 scoped recovery runs](../../../e2e_recovery_20260923/README.md) and [WSL CPU/native Windows expansion](../../../e2e_expansion_20260923/README.md). The expansion adds separate 20-request serial MiniCPM-o text-to-speech profiles on WSL CPU and native Windows GPU; other new paths are single-request or short-stream runs. Profiling completion does not imply full release qualification.
|
||||
|
||||
## Configuration × model
|
||||
|
||||
@@ -11,7 +11,7 @@ NOT E2E means no complete device/model workload was verified. “Scoped E2E” i
|
||||
| PC HX370: CPU / WSL | profile completed | scoped E2E: short stream | NOT E2E | scoped E2E: text → speech | synthetic policy pass |
|
||||
| PC HX370: CPU / native Windows | NOT E2E | NOT E2E | NOT E2E | NOT E2E | synthetic policy pass |
|
||||
| PC HX370 + RTX 5090 Laptop / WSL | profile completed | profile completed | scoped E2E: text + image | scoped E2E: text → speech | synthetic policy pass |
|
||||
| PC HX370 + RTX 5090 Laptop / Windows | profile completed | profile completed | NOT E2E | NOT E2E | synthetic policy pass |
|
||||
| PC HX370 + RTX 5090 Laptop / Windows | profile completed | profile completed | NOT E2E | scoped E2E: text → speech | synthetic policy pass |
|
||||
| PC HX370 + Radeon 890M / Windows worker | NOT E2E | NOT E2E | NOT E2E | NOT E2E | NOT E2E |
|
||||
| PC HX370 + AMD NPU / Windows worker | NOT E2E | NOT E2E | NOT E2E | NOT E2E | NOT E2E |
|
||||
| PC CPU+iGPU+NPU +/- discrete GPU, joint model execution | NOT E2E | NOT E2E | NOT E2E | NOT E2E | NOT E2E |
|
||||
@@ -43,7 +43,7 @@ NOT E2E means no complete device/model workload was verified. “Scoped E2E” i
|
||||
| PC HX370 + RTX 5090 Laptop / Windows | Spark-X2.5 | PROFILE_COMPLETED | not fully qualified: reference-quality and remaining reliability/performance gates must be reviewed | Broaden only with a named workload/device/precision qualification. |
|
||||
| PC HX370 + RTX 5090 Laptop / Windows | Qwen3-TTS 0.6B CustomVoice | PROFILE_COMPLETED | not fully qualified: reference-quality and remaining reliability/performance gates must be reviewed | Resolve and measure playback stalls; run quality, interruption and thermal tests. |
|
||||
| PC HX370 + RTX 5090 Laptop / Windows | Qwen3.8-27B | NOT_E2E_PROFILEABLE | The Omni pipeline binding now exists and passed scoped text/image cases in WSL, but no native Windows Omni run has verified the NVFP4 checkpoint, memory admission or modality behavior. | Run the same checkpoint and explicit context on native Windows, then broaden quality and profiling; [WSL-only evidence](../../../e2e_recovery_20260923/README.md). |
|
||||
| PC HX370 + RTX 5090 Laptop / Windows | MiniCPM-o 4.5 | NOT_E2E_PROFILEABLE | The exact checkpoint is verified and one constrained three-stage text/speech request passed in WSL; no native Windows execution, memory admission or speech-quality check exists. | Repeat the real three-stage pipeline on native Windows and verify speech and sustained behavior; [WSL-only evidence](../../../e2e_recovery_20260923/README.md). |
|
||||
| PC HX370 + RTX 5090 Laptop / Windows | MiniCPM-o 4.5 | SCOPED_E2E | The verified checkpoint completed native Windows three-stage BF16 text-to-text+speech requests with 5 GiB thinker host offload and explicit 2,048/512-token limits. Text and all 32 thinker token IDs matched WSL CUDA; a 5.2 s WAV was nonzero. After a 300 s default startup timeout on the first attempt, a 900/600 s limit allowed a functional pass. A separate 1-warmup/20-measured serial run had request wall p50 18.38 s/p95 18.73 s; sampled host available RAM fell to 0.78 GB and pagefile use reached 11.17 GB. Speech meaning remains unverified. | Check speech intelligibility/alignment, other input modalities, concurrency, interruption, loading-peak admission and sustained thermal/power behavior before broader qualification; [native evidence](../../../e2e_expansion_20260923/README.md). |
|
||||
| PC HX370 + RTX 5090 Laptop / Windows | InternVLA-A1 | SYNTHETIC_POLICY_PASS | The Place_Markpen checkpoint strict-loaded from a UNC path and produced finite `[1, 50, 32]` actions on synthetic zero observations/noise. Policy/output CUDA placement was checked; one native forward took 1.12 s, with 6.68 GB peak allocated. Relative L2 difference from WSL CUDA was 0.83% (no task tolerance). | Validate real observations/reference actions, units/timestamps, memory and repeated latency before task qualification; [native evidence](../../../e2e_expansion_20260923/README.md). |
|
||||
| PC HX370 + Radeon 890M / Windows worker | Spark-X2.5 | NOT_E2E_PROFILEABLE | No placement- and quality-verified whole-model graph bundle/stateful adapter for this model on this AMD route. Small backend graphs do not establish model support. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
| PC HX370 + Radeon 890M / Windows worker | Qwen3-TTS 0.6B CustomVoice | NOT_E2E_PROFILEABLE | No placement- and quality-verified whole-model graph bundle/stateful adapter for this model on this AMD route. Small backend graphs do not establish model support. | Supply and validate the missing matching artifact/backend, then run the complete specified workload. |
|
||||
|
||||
@@ -18,9 +18,14 @@ adds constrained three-stage MiniCPM-o text-to-speech requests and a CPU-only
|
||||
InternVLA synthetic policy forward. A subsequent 20-request serial MiniCPM-o
|
||||
profile measured p50 21.86 s/p95 22.97 s request wall time after one warmup, with
|
||||
swap use under the 30.91 GiB WSL limit. It does not qualify speech quality,
|
||||
concurrency, streaming, loading peak or sustained behavior. Native Windows
|
||||
InternVLA direct-policy runs separately passed on CPU and RTX 5090 Laptop with
|
||||
synthetic inputs; AMD and mobile cells do not inherit these passes.
|
||||
concurrency, streaming, loading peak or sustained behavior. A separate native
|
||||
Windows RTX 5090 Laptop MiniCPM-o three-stage text-to-speech request passed
|
||||
with explicit 900/600 s startup limits after the default 300 s overall timeout
|
||||
had expired. A later native Windows 20-request serial run measured p50 18.38 s
|
||||
and p95 18.73 s, but sampled host available RAM fell to 0.78 GB and pagefile
|
||||
use reached 11.17 GB. Native Windows InternVLA direct-policy runs separately
|
||||
passed on CPU and RTX 5090 Laptop with synthetic inputs; AMD and mobile cells
|
||||
do not inherit these passes.
|
||||
|
||||
## Architecture and prerequisites
|
||||
|
||||
@@ -77,7 +82,7 @@ capability and lifecycle implementation; a state-handle type alone is insufficie
|
||||
| M1 | S25 Spark | Device-local Omni-compatible controller; embedding, prefill, continuous decode, ring/full KV, head and sampler; resident state | Reference checks, at least 128 output tokens, 512-window/1024-bucket transitions, cancellation, resident memory and sustained profile |
|
||||
| M2 | Qwen3-TTS | Resolve desktop playback and shutdown findings and CPU compatibility; then mobile talker/predictor plus GPU vocoder | Complete PCM and tail, history/chunk correctness, interruption/recovery, quality checks, no post-startup underruns and RTF below 1 in the declared workload |
|
||||
| M3 | AMD stages | Connect compatible real encoders or other useful coarse stages through current Omni workers; start from the existing 890M vision evidence | Actual node/device placement, numerical and task quality, complete downstream output, shared-memory and handoff costs; reject unhelpful splits |
|
||||
| M4a | MiniCPM-o 4.5 | Three-stage desktop text+WAV produced coherent text and nonzero WAV on RTX 5090 Laptop and WSL CPU under separate constrained plans. The CPU plan needed vLLM 0.28/0.29 processor compatibility and used swap; 20 serial CPU requests measured p50 21.86 s/p95 22.97 s. Add encoder and persistent-state reference before mobile artifacts | Speech intelligibility/alignment, CPU loading-peak admission, concurrency, separate text, image and audio-understanding suites, then combined streaming and interruption |
|
||||
| M4a | MiniCPM-o 4.5 | Three-stage desktop text+WAV produced coherent text and nonzero WAV on RTX 5090 Laptop under WSL and native Windows, and WSL CPU, using separate constrained plans. The CPU plan needed vLLM 0.28/0.29 processor compatibility and used swap; 20 serial CPU requests measured p50 21.86 s/p95 22.97 s. Native Windows needed longer startup limits; 20 serial requests measured p50 18.38 s/p95 18.73 s with host RAM/pagefile pressure. Add encoder and persistent-state reference before mobile artifacts | Speech intelligibility/alignment, loading-peak admission, concurrency, separate text, image and audio-understanding suites, then combined streaming and interruption |
|
||||
| M4b | InternVLA-A1 | Real weights, observation preprocessing, shared prefix state, iterative action head and versioned action metadata; strict-load synthetic policy forwards now pass on WSL and native Windows, each on CPU and RTX 5090 Laptop | Reference action agreement on real observations, units/order/timestamps, observation age, stale-input handling and application-specific deadline reporting |
|
||||
| Independent | Qwen3.8-27B | Public Omni pipeline binding; text, then image/video and longer contexts; same-model CPU/iGPU artifact and explicit offload evaluation | Modality-specific quality, actual loading/runtime memory and measured performance; each precision/context receives its own qualification |
|
||||
|
||||
|
||||
@@ -402,8 +402,8 @@ def parse_args():
|
||||
parser.add_argument(
|
||||
"--init-timeout",
|
||||
type=int,
|
||||
default=300,
|
||||
help="Timeout for initializing stages in seconds (default: 300)",
|
||||
default=600,
|
||||
help="Timeout for initializing stages in seconds (default: 600)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--shm-threshold-bytes",
|
||||
|
||||
Reference in new issue
Block a user