diff --git a/examples/online_serving/minicpmo/README.md b/examples/online_serving/minicpmo/README.md index 134ddd62c..e565a3de3 100644 --- a/examples/online_serving/minicpmo/README.md +++ b/examples/online_serving/minicpmo/README.md @@ -25,15 +25,14 @@ The deploy config auto-loads via `--omni`. The default `vllm_omni/deploy/minicpmo_4_5.yaml` keeps all three stages on logical device 0 with memory budgets of 55%, 22%, and 22%. The profile admits at most four sequences per stage and bounds startup video profiling to 32 -frames per video. For throughput,`minicpmo_4_5_batching.yaml` gives the Thinker +frames per video. For throughput,`minicpmo_4_5_2gpu.yaml` gives the Thinker GPU 0 (90%) and colocates the Talker (55%) and Code2Wav (35%) on GPU 1. That profile admits at most four concurrent sequences per stage. | deploy config | GPUs | Notes | |---|---|---| | `minicpmo_4_5.yaml` (default) | 1 | Memory-constrained compatibility layout. | -| `minicpmo_4_5_batching.yaml` | 2 | Recommended continuous-batching layout; Talker and Code2Wav share GPU 1. | -| `minicpmo_4_5_2gpu.yaml` | 2 | Backward-compatible alias for `minicpmo_4_5_batching.yaml`. | +| `minicpmo_4_5_2gpu.yaml` | 2 | Recommended continuous-batching layout; Talker and Code2Wav share GPU 1. | | `minicpmo_4_5_3gpu.yaml` | 3 | One GPU per stage. | | `minicpmo_4_5_8x4090.yaml` | 8 | Full 8x4090 layout. | | `minicpmo_4_5_duplex.yaml` | 1 | Experimental native full-duplex overlay. | diff --git a/recipes/OpenBMB/MiniCPM-o-4_5.md b/recipes/OpenBMB/MiniCPM-o-4_5.md index 97b3c6d6d..87f68af5d 100644 --- a/recipes/OpenBMB/MiniCPM-o-4_5.md +++ b/recipes/OpenBMB/MiniCPM-o-4_5.md @@ -30,8 +30,6 @@ also provided. `hf_config.version="4.5"`): - Default single-GPU compatibility layout (auto-loaded): [`vllm_omni/deploy/minicpmo_4_5.yaml`](../../vllm_omni/deploy/minicpmo_4_5.yaml) - - Recommended 2-GPU continuous-batching layout: - [`vllm_omni/deploy/minicpmo_4_5_batching.yaml`](../../vllm_omni/deploy/minicpmo_4_5_batching.yaml) - 2-GPU and 3-GPU layouts: [`vllm_omni/deploy/minicpmo_4_5_2gpu.yaml`](../../vllm_omni/deploy/minicpmo_4_5_2gpu.yaml), [`vllm_omni/deploy/minicpmo_4_5_3gpu.yaml`](../../vllm_omni/deploy/minicpmo_4_5_3gpu.yaml) @@ -68,7 +66,7 @@ Code2Wav consumes them through a shared-memory async connector. MiniCPM-o 4.5 now requires the three-stage topology: the Talker owns request-local codec generation and Code2Wav owns waveform state and reference-voice prompt features. `minicpmo_4_5.yaml` remains the stable -single-GPU entry point; `minicpmo_4_5_batching.yaml` is the recommended +single-GPU entry point; `minicpmo_4_5_2gpu.yaml` is the recommended two-GPU profile. The removed fused two-stage implementation is not retained as a fallback because it would duplicate state machines and correctness paths. @@ -105,7 +103,7 @@ The deploy config is auto-loaded by the model registry — no For the recommended two-GPU layout, add: ```bash ---deploy-config vllm_omni/deploy/minicpmo_4_5_batching.yaml +--deploy-config vllm_omni/deploy/minicpmo_4_5_2gpu.yaml ``` #### Performance comparison diff --git a/tests/dfx/reliability/invalid_param_test/test_invalid_minicpmo_4_5_omni.py b/tests/dfx/reliability/invalid_param_test/test_invalid_minicpmo_4_5_omni.py index 2082be10d..9c6880d68 100644 --- a/tests/dfx/reliability/invalid_param_test/test_invalid_minicpmo_4_5_omni.py +++ b/tests/dfx/reliability/invalid_param_test/test_invalid_minicpmo_4_5_omni.py @@ -14,15 +14,17 @@ from tests.helpers.stage_config import get_deploy_config_path pytestmark = [pytest.mark.slow, pytest.mark.omni, pytest.mark.full_model] -_MINICPMO_DEPLOY = get_deploy_config_path("minicpmo_4_5_batching.yaml") +_MINICPMO_DEPLOY = get_deploy_config_path("minicpmo_4_5.yaml") _MINICPMO_SERVER = [ pytest.param( OmniServerParams( model="openbmb/MiniCPM-o-4_5", stage_config_path=_MINICPMO_DEPLOY, - use_stage_cli=True, - server_args=["--trust-remote-code"], + use_stage_cli=False, + server_args=[ + "--trust-remote-code", + ], ), id="minicpmo_4_5", ) @@ -48,7 +50,7 @@ def _prompt() -> str: return "What is the capital of China? Answer in 20 words." -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1) @pytest.mark.parametrize("omni_server", _MINICPMO_SERVER, indirect=True) def test_invalid_audio_format_rejected(omni_server: OmniServerParams, openai_client: OpenAIClientHandler) -> None: """ diff --git a/tests/e2e/offline_inference/test_minicpmo_4_5.py b/tests/e2e/offline_inference/test_minicpmo_4_5.py index a8855e04b..12f24e3dc 100644 --- a/tests/e2e/offline_inference/test_minicpmo_4_5.py +++ b/tests/e2e/offline_inference/test_minicpmo_4_5.py @@ -14,7 +14,7 @@ from tests.helpers.stage_config import get_deploy_config_path models = ["openbmb/MiniCPM-o-4_5"] -_CI_DEPLOY = get_deploy_config_path("minicpmo_4_5_batching.yaml") +_CI_DEPLOY = get_deploy_config_path("minicpmo_4_5.yaml") test_params = [(model, None, {"deploy_config": _CI_DEPLOY, "trust_remote_code": True}) for model in models] @@ -31,11 +31,10 @@ def get_question(prompt_type: str = "text") -> str: return prompts.get(prompt_type, prompts["text"]) -@pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/5437") @pytest.mark.core_model @pytest.mark.advanced_model @pytest.mark.omni -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1) @pytest.mark.parametrize("omni_runner", test_params, indirect=True) def test_text_to_text(omni_runner, omni_runner_handler) -> None: """Test processing text, generating text output.""" @@ -45,7 +44,7 @@ def test_text_to_text(omni_runner, omni_runner_handler) -> None: @pytest.mark.full_model @pytest.mark.omni -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1) @pytest.mark.parametrize("omni_runner", test_params, indirect=True) def test_audio_to_text(omni_runner, omni_runner_handler) -> None: """Test processing audio, generating text output.""" @@ -58,7 +57,7 @@ def test_audio_to_text(omni_runner, omni_runner_handler) -> None: @pytest.mark.full_model @pytest.mark.omni -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1) @pytest.mark.parametrize("omni_runner", test_params, indirect=True) def test_image_to_text(omni_runner, omni_runner_handler) -> None: """Test processing image, generating text output.""" @@ -69,7 +68,7 @@ def test_image_to_text(omni_runner, omni_runner_handler) -> None: @pytest.mark.full_model @pytest.mark.omni -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1) @pytest.mark.parametrize("omni_runner", test_params, indirect=True) def test_video_to_text(omni_runner, omni_runner_handler) -> None: """Test processing video, generating text output.""" @@ -80,7 +79,7 @@ def test_video_to_text(omni_runner, omni_runner_handler) -> None: @pytest.mark.full_model @pytest.mark.omni -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1) @pytest.mark.parametrize("omni_runner", test_params, indirect=True) def test_text_to_audio(omni_runner, omni_runner_handler) -> None: """Test processing text and generating audio through Talker and Code2Wav.""" @@ -88,11 +87,10 @@ def test_text_to_audio(omni_runner, omni_runner_handler) -> None: omni_runner_handler.send_omni_request(request_config) -@pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/5437") @pytest.mark.core_model @pytest.mark.advanced_model @pytest.mark.omni -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1) @pytest.mark.parametrize("omni_runner", test_params, indirect=True) def test_mix_to_audio(omni_runner, omni_runner_handler) -> None: """Test processing mixed modalities (image + audio), generating audio output.""" @@ -111,7 +109,7 @@ def test_mix_to_audio(omni_runner, omni_runner_handler) -> None: @pytest.mark.full_model @pytest.mark.omni -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1) @pytest.mark.parametrize("omni_runner", test_params, indirect=True) def test_video_to_audio(omni_runner, omni_runner_handler) -> None: """Test processing video, generating audio output.""" diff --git a/tests/e2e/online_serving/test_minicpmo_4_5.py b/tests/e2e/online_serving/test_minicpmo_4_5.py index a4fefec98..7e77de2e1 100644 --- a/tests/e2e/online_serving/test_minicpmo_4_5.py +++ b/tests/e2e/online_serving/test_minicpmo_4_5.py @@ -1,7 +1,10 @@ -"""E2E online tests for MiniCPM-o 4.5 multimodal input and audio/text output. +""" +E2E Online tests for MiniCPM-o 4.5 model with multimodal input and audio / text output. -The batching deployment uses async chunk transfer across separate Thinker, -Talker, and Code2Wav stages. +MiniCPM-o 4.5 is a three-stage pipeline: Stage 0 Thinker (text), Stage 1 Talker +(TTS codec), Stage 2 Code2Wav (vocoder). The talker -> Code2Wav bridge supports +both async_chunk (streaming) and full-payload (``--no-async-chunk``) modes, so the +tests below exercise the sync path as well. """ import os @@ -16,15 +19,17 @@ from tests.helpers.stage_config import get_deploy_config_path os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" _MODEL = "openbmb/MiniCPM-o-4_5" -_CI_DEPLOY = get_deploy_config_path("minicpmo_4_5_batching.yaml") +_CI_DEPLOY = get_deploy_config_path("minicpmo_4_5.yaml") test_params = [ pytest.param( OmniServerParams( model=_MODEL, stage_config_path=_CI_DEPLOY, - use_stage_cli=True, - server_args=["--trust-remote-code"], + use_stage_cli=False, + server_args=[ + "--trust-remote-code", + ], ), id="default", ) @@ -60,11 +65,10 @@ def get_max_batch_size(size_type="few"): return batch_sizes.get(size_type, 5) -@pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/5437") @pytest.mark.core_model @pytest.mark.advanced_model @pytest.mark.omni -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_text_to_text_001(omni_server, openai_client) -> None: """ @@ -89,12 +93,12 @@ def test_text_to_text_001(omni_server, openai_client) -> None: @pytest.mark.full_model @pytest.mark.omni -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_text_to_audio_001(omni_server, openai_client) -> None: """ Test text-only input generating text + audio output via OpenAI API. - This exercises Talker TTS region detection and the Code2Wav stage. + This exercises the talker TTS region detection and in-process token2wav vocoder. Deploy Setting: default 2GPU Input Modal: text Output Modal: text + audio @@ -114,7 +118,7 @@ def test_text_to_audio_001(omni_server, openai_client) -> None: @pytest.mark.full_model @pytest.mark.omni -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_audio_to_text_audio_001(omni_server, openai_client) -> None: """ @@ -128,14 +132,13 @@ def test_audio_to_text_audio_001(omni_server, openai_client) -> None: messages = dummy_messages_from_mix_data( system_prompt=get_system_prompt(), audio_data_url=audio_data_url, - content_text=get_prompt(), + content_text=get_prompt("mix"), ) request_config = { "model": omni_server.model, "messages": messages, "stream": True, - "key_words": {"text": ["Beijing"]}, } openai_client.send_omni_request(request_config, request_num=get_max_batch_size()) @@ -143,7 +146,7 @@ def test_audio_to_text_audio_001(omni_server, openai_client) -> None: @pytest.mark.full_model @pytest.mark.omni -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_image_to_text_audio_001(omni_server, openai_client) -> None: """ @@ -171,7 +174,7 @@ def test_image_to_text_audio_001(omni_server, openai_client) -> None: @pytest.mark.full_model @pytest.mark.omni -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_video_to_text_audio_001(omni_server, openai_client) -> None: """ @@ -197,11 +200,10 @@ def test_video_to_text_audio_001(omni_server, openai_client) -> None: openai_client.send_omni_request(request_config, request_num=get_max_batch_size()) -@pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/5437") @pytest.mark.core_model @pytest.mark.advanced_model @pytest.mark.omni -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_mix_to_text_audio_001(omni_server, openai_client) -> None: """ diff --git a/tests/e2e/online_serving/test_minicpmo_4_5_expansion.py b/tests/e2e/online_serving/test_minicpmo_4_5_expansion.py index 068502903..089a304a7 100644 --- a/tests/e2e/online_serving/test_minicpmo_4_5_expansion.py +++ b/tests/e2e/online_serving/test_minicpmo_4_5_expansion.py @@ -18,14 +18,14 @@ pytestmark = [pytest.mark.full_model, pytest.mark.omni] os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" _MODEL = "openbmb/MiniCPM-o-4_5" -_CI_DEPLOY = get_deploy_config_path("minicpmo_4_5_batching.yaml") +_CI_DEPLOY = get_deploy_config_path("minicpmo_4_5.yaml") test_params = [ pytest.param( OmniServerParams( model=_MODEL, stage_config_path=_CI_DEPLOY, - use_stage_cli=True, + use_stage_cli=False, server_args=[ "--trust-remote-code", ], @@ -63,7 +63,7 @@ def get_max_batch_size(size_type="few"): return batch_sizes.get(size_type, 5) -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_text_video_to_text_001(omni_server, openai_client) -> None: """ @@ -90,7 +90,7 @@ def test_text_video_to_text_001(omni_server, openai_client) -> None: openai_client.send_omni_request(request_config, request_num=get_max_batch_size()) -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_sequential_requests_independent(omni_server, openai_client) -> None: """ @@ -131,7 +131,7 @@ def test_sequential_requests_independent(omni_server, openai_client) -> None: ) -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_text_to_audio_long_output_001(omni_server, openai_client) -> None: """ @@ -157,7 +157,7 @@ def test_text_to_audio_long_output_001(omni_server, openai_client) -> None: openai_client.send_omni_request(request_config, request_num=get_max_batch_size()) -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=1) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_chinese_text_to_audio(omni_server, openai_client) -> None: """ diff --git a/tests/model_executor/models/minicpmo_4_5/test_pipeline.py b/tests/model_executor/models/minicpmo_4_5/test_pipeline.py index 2442cad44..b6f11ff5d 100644 --- a/tests/model_executor/models/minicpmo_4_5/test_pipeline.py +++ b/tests/model_executor/models/minicpmo_4_5/test_pipeline.py @@ -36,7 +36,6 @@ _PIPELINE_KEY = "minicpmo_4_5" _DEPLOY_DIR = Path(__file__).resolve().parents[4] / "vllm_omni" / "deploy" _DEPLOY_LAYOUTS = { "minicpmo_4_5.yaml": ["0", "0", "0"], - "minicpmo_4_5_batching.yaml": ["0", "1", "1"], "minicpmo_4_5_2gpu.yaml": ["0", "1", "1"], "minicpmo_4_5_3gpu.yaml": ["0", "1", "2"], "minicpmo_4_5_8x4090.yaml": ["0,1,2,3", "4", "5"], @@ -170,7 +169,7 @@ class TestDeployTopology: 0.22, ] assert stages[0].yaml_engine_args["limit_mm_per_prompt"] == {"video": {"count": 1, "num_frames": 32}} - elif filename in {"minicpmo_4_5_batching.yaml", "minicpmo_4_5_2gpu.yaml"}: + elif filename in {"minicpmo_4_5_2gpu.yaml"}: assert [stage.yaml_engine_args["gpu_memory_utilization"] for stage in stages] == [ 0.9, 0.55, diff --git a/vllm_omni/deploy/minicpmo_4_5_2gpu.yaml b/vllm_omni/deploy/minicpmo_4_5_2gpu.yaml index eddfd0877..75693feb7 100644 --- a/vllm_omni/deploy/minicpmo_4_5_2gpu.yaml +++ b/vllm_omni/deploy/minicpmo_4_5_2gpu.yaml @@ -1,2 +1,88 @@ -# Backward-compatible name for the canonical two-GPU batching profile. -base_config: minicpmo_4_5_batching.yaml +# MiniCPM-o 4.5 two-GPU continuous-batching layout. +# +# Thinker owns GPU 0. Talker and Code2Wav share GPU 1, keeping text decoding +# isolated from codec/vocoder contention while preserving stage-2 batching. +pipeline: minicpmo_4_5 +async_chunk: true + +trust_remote_code: true +enable_prefix_caching: false + +connectors: + connector_of_shared_memory: + name: SharedMemoryConnector + extra: + connector_get_sleep_s: 0.01 + connector_get_max_wait_first_chunk: 3000 + connector_get_max_wait: 300 + codec_chunk_frames: 25 + codec_left_context_frames: 3 + +stages: + - stage_id: 0 + max_num_seqs: 4 + gpu_memory_utilization: 0.9 + enforce_eager: false + tensor_parallel_size: 1 + max_num_batched_tokens: 16384 + devices: "0" + limit_mm_per_prompt: + video: 1 + default_sampling_params: + temperature: 0.0 + top_p: 1.0 + top_k: -1 + max_tokens: 2048 + seed: 42 + detokenize: true + repetition_penalty: 1.1 + + - stage_id: 1 + max_num_seqs: 4 + gpu_memory_utilization: 0.55 + enforce_eager: false + max_num_batched_tokens: 8192 + devices: "1" + output_connectors: + to_stage_2: connector_of_shared_memory + # Codec sampling is model-owned by the checkpoint's tts_config (defaults: + # T=0.8, top-k=25, top-p=0.85, repetition penalty=1.05, seed=42). + # These parameters only govern vLLM's binary continue/stop token. + default_sampling_params: + temperature: 0.0 + top_p: 1.0 + top_k: -1 + min_tokens: 50 + max_tokens: 2048 + stop_token_ids: [1] + seed: 42 + detokenize: false + + - stage_id: 2 + max_num_seqs: 4 + gpu_memory_utilization: 0.35 + enforce_eager: true + enable_chunked_prefill: false + max_num_batched_tokens: 65536 + max_model_len: 65536 + devices: "1" + input_connectors: + from_stage_1: connector_of_shared_memory + default_sampling_params: + temperature: 0.0 + top_p: 1.0 + top_k: -1 + max_tokens: 65536 + detokenize: true + +platforms: + npu: + stages: + - stage_id: 0 + max_num_batched_tokens: 8192 + compilation_config: + cudagraph_mode: PIECEWISE + - stage_id: 1 + max_num_batched_tokens: 8192 + compilation_config: + cudagraph_mode: PIECEWISE diff --git a/vllm_omni/deploy/minicpmo_4_5_batching.yaml b/vllm_omni/deploy/minicpmo_4_5_batching.yaml deleted file mode 100644 index 7e6b1fb63..000000000 --- a/vllm_omni/deploy/minicpmo_4_5_batching.yaml +++ /dev/null @@ -1,89 +0,0 @@ -# MiniCPM-o 4.5 two-GPU continuous-batching layout. -# -# Thinker owns GPU 0. Talker and Code2Wav share GPU 1, keeping text decoding -# isolated from codec/vocoder contention while preserving stage-2 batching. -pipeline: minicpmo_4_5 -async_chunk: true - -trust_remote_code: true -enable_prefix_caching: false - -connectors: - connector_of_shared_memory: - name: SharedMemoryConnector - extra: - connector_get_sleep_s: 0.01 - connector_get_max_wait_first_chunk: 3000 - connector_get_max_wait: 300 - codec_chunk_frames: 25 - codec_left_context_frames: 3 - -stages: - - stage_id: 0 - max_num_seqs: 4 - gpu_memory_utilization: 0.9 - enforce_eager: false - tensor_parallel_size: 1 - max_num_batched_tokens: 16384 - devices: "0" - limit_mm_per_prompt: - video: 1 - default_sampling_params: - temperature: 0.0 - top_p: 1.0 - top_k: -1 - max_tokens: 2048 - seed: 42 - detokenize: true - repetition_penalty: 1.1 - - - stage_id: 1 - max_num_seqs: 4 - gpu_memory_utilization: 0.55 - enforce_eager: false - max_num_batched_tokens: 8192 - devices: "1" - output_connectors: - to_stage_2: connector_of_shared_memory - # Codec sampling is model-owned by the checkpoint's tts_config (defaults: - # T=0.8, top-k=25, top-p=0.85, repetition penalty=1.05, seed=42). - # These parameters only govern vLLM's binary continue/stop token. - default_sampling_params: - temperature: 0.0 - top_p: 1.0 - top_k: -1 - min_tokens: 50 - max_tokens: 2048 - stop_token_ids: [1] - seed: 42 - detokenize: false - - - stage_id: 2 - max_num_seqs: 4 - gpu_memory_utilization: 0.35 - enforce_eager: true - enable_chunked_prefill: false - max_num_batched_tokens: 65536 - max_model_len: 65536 - devices: "1" - input_connectors: - from_stage_1: connector_of_shared_memory - default_sampling_params: - temperature: 0.0 - top_p: 1.0 - top_k: -1 - max_tokens: 65536 - detokenize: true - -platforms: - npu: - stages: - - stage_id: 0 - max_num_batched_tokens: 8192 - compilation_config: - cudagraph_mode: PIECEWISE - - stage_id: 1 - max_num_batched_tokens: 8192 - gpu_memory_utilization: 0.4 - compilation_config: - cudagraph_mode: PIECEWISE