diff --git a/.buildkite/npu/test-npu-nightly.yml b/.buildkite/npu/test-npu-nightly.yml index 36b9c4055..8590fdc6f 100644 --- a/.buildkite/npu/test-npu-nightly.yml +++ b/.buildkite/npu/test-npu-nightly.yml @@ -48,3 +48,23 @@ steps: HF_TOKEN: "${HF_TOKEN}" commands: - pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5_expansion.py -m 'full_model' --run-level 'full_model' + + # NPU has no merge pipeline; run advanced duplex coverage on nightly + # (CUDA covers these via test-merge.yml). + - label: ":robot: Omni · MiniCPM-o 4.5 Duplex Test" + mirror_hardwares: a3_npu_2 + env: + VLLM_WORKER_MULTIPROC_METHOD: spawn + VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" + HF_TOKEN: "${HF_TOKEN}" + commands: + - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'advanced_model and npu' --run-level 'advanced_model' + + - label: ":robot: Omni · MiniCPM-o 4.5 Duplex Expansion Test" + mirror_hardwares: a3_npu_2 + env: + VLLM_WORKER_MULTIPROC_METHOD: spawn + VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" + HF_TOKEN: "${HF_TOKEN}" + commands: + - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py -m 'full_model and npu and A3 and omni' --run-level 'full_model' diff --git a/.buildkite/npu/test-npu-ready.yml b/.buildkite/npu/test-npu-ready.yml index e27fcfa1f..5ba914ae5 100644 --- a/.buildkite/npu/test-npu-ready.yml +++ b/.buildkite/npu/test-npu-ready.yml @@ -12,6 +12,15 @@ steps: commands: - pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py -m 'core_model' --run-level 'core_model' + - label: ":robot: Omni · MiniCPM-o 4.5 Duplex Test" + mirror_hardwares: a3_npu_2 + env: + VLLM_WORKER_MULTIPROC_METHOD: spawn + VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" + HF_TOKEN: "${HF_TOKEN}" + commands: + - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'core_model and npu' --run-level 'core_model' + # - group: ":card_index_dividers: TTS Model Test" # key: ready-tts-group # depends_on: upload-ready-pipeline diff --git a/docker/Dockerfile.npu.ci b/docker/Dockerfile.npu.ci index c0b4af9b6..341ed5965 100644 --- a/docker/Dockerfile.npu.ci +++ b/docker/Dockerfile.npu.ci @@ -32,6 +32,7 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ "pytest-cov>=4.0.0" \ "pytest-mock>=3.10.0" \ "openpyxl>=3.0.0" \ + "opencc==1.4.1" \ "psutil>=7.2.0" \ "soundfile>=0.13.1" \ "pyttsx3>=2.99" \ diff --git a/docker/Dockerfile.npu.ci.a3 b/docker/Dockerfile.npu.ci.a3 index b86d7c0f8..714e6b43c 100644 --- a/docker/Dockerfile.npu.ci.a3 +++ b/docker/Dockerfile.npu.ci.a3 @@ -32,6 +32,7 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ "pytest-cov>=4.0.0" \ "pytest-mock>=3.10.0" \ "openpyxl>=3.0.0" \ + "opencc==1.4.1" \ "psutil>=7.2.0" \ "soundfile>=0.13.1" \ "pyttsx3>=2.99" \ diff --git a/tests/e2e/offline_inference/test_minicpmo_4_5.py b/tests/e2e/offline_inference/test_minicpmo_4_5.py index 3d5c42517..6725ce2aa 100644 --- a/tests/e2e/offline_inference/test_minicpmo_4_5.py +++ b/tests/e2e/offline_inference/test_minicpmo_4_5.py @@ -14,7 +14,7 @@ from tests.helpers.stage_config import get_deploy_config_path models = ["openbmb/MiniCPM-o-4_5"] -_CI_DEPLOY = get_deploy_config_path("minicpmo_4_5_batching.yaml") +_CI_DEPLOY = get_deploy_config_path("minicpmo_4_5.yaml") test_params = [(model, None, {"deploy_config": _CI_DEPLOY, "trust_remote_code": True}) for model in models] @@ -34,7 +34,7 @@ def get_question(prompt_type: str = "text") -> str: @pytest.mark.core_model @pytest.mark.advanced_model @pytest.mark.omni -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) @pytest.mark.parametrize("omni_runner", test_params, indirect=True) def test_text_to_text(omni_runner, omni_runner_handler) -> None: """Test processing text, generating text output.""" @@ -44,7 +44,7 @@ def test_text_to_text(omni_runner, omni_runner_handler) -> None: @pytest.mark.full_model @pytest.mark.omni -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) @pytest.mark.parametrize("omni_runner", test_params, indirect=True) def test_audio_to_text(omni_runner, omni_runner_handler) -> None: """Test processing audio, generating text output.""" @@ -57,7 +57,7 @@ def test_audio_to_text(omni_runner, omni_runner_handler) -> None: @pytest.mark.full_model @pytest.mark.omni -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) @pytest.mark.parametrize("omni_runner", test_params, indirect=True) def test_image_to_text(omni_runner, omni_runner_handler) -> None: """Test processing image, generating text output.""" @@ -68,7 +68,7 @@ def test_image_to_text(omni_runner, omni_runner_handler) -> None: @pytest.mark.full_model @pytest.mark.omni -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) @pytest.mark.parametrize("omni_runner", test_params, indirect=True) def test_video_to_text(omni_runner, omni_runner_handler) -> None: """Test processing video, generating text output.""" @@ -79,7 +79,7 @@ def test_video_to_text(omni_runner, omni_runner_handler) -> None: @pytest.mark.full_model @pytest.mark.omni -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) @pytest.mark.parametrize("omni_runner", test_params, indirect=True) def test_text_to_audio(omni_runner, omni_runner_handler) -> None: """Test processing text and generating audio through Talker and Code2Wav.""" @@ -90,7 +90,7 @@ def test_text_to_audio(omni_runner, omni_runner_handler) -> None: @pytest.mark.core_model @pytest.mark.advanced_model @pytest.mark.omni -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) @pytest.mark.parametrize("omni_runner", test_params, indirect=True) def test_mix_to_audio(omni_runner, omni_runner_handler) -> None: """Test processing mixed modalities (image + audio), generating audio output.""" @@ -109,7 +109,7 @@ def test_mix_to_audio(omni_runner, omni_runner_handler) -> None: @pytest.mark.full_model @pytest.mark.omni -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) @pytest.mark.parametrize("omni_runner", test_params, indirect=True) def test_video_to_audio(omni_runner, omni_runner_handler) -> None: """Test processing video, generating audio output.""" diff --git a/tests/e2e/online_serving/test_minicpmo_4_5.py b/tests/e2e/online_serving/test_minicpmo_4_5.py index f1b5ae178..81356cddd 100644 --- a/tests/e2e/online_serving/test_minicpmo_4_5.py +++ b/tests/e2e/online_serving/test_minicpmo_4_5.py @@ -1,6 +1,6 @@ """E2E online tests for MiniCPM-o 4.5 multimodal input and audio/text output. -The batching deployment uses async chunk transfer across separate Thinker, +Exercises async chunk streaming (``--async-chunk``) across separate Thinker, Talker, and Code2Wav stages. """ @@ -11,23 +11,28 @@ import pytest from tests.helpers.mark import hardware_test from tests.helpers.media import generate_synthetic_audio, generate_synthetic_image, generate_synthetic_video from tests.helpers.runtime import OmniServerParams, dummy_messages_from_mix_data -from tests.helpers.stage_config import get_deploy_config_path +from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" _MODEL = "openbmb/MiniCPM-o-4_5" -_CI_DEPLOY = get_deploy_config_path("minicpmo_4_5_batching.yaml") +_CI_DEPLOY = modify_stage_config( + get_deploy_config_path("minicpmo_4_5.yaml"), + updates={ + "stages": {0: {"default_sampling_params.max_tokens": 128}, 1: {"default_sampling_params.max_tokens": 512}} + }, +) test_params = [ pytest.param( OmniServerParams( model=_MODEL, stage_config_path=_CI_DEPLOY, - use_stage_cli=True, - server_args=["--trust-remote-code"], + use_stage_cli=False, + server_args=["--trust-remote-code", "--async-chunk"], ), - id="default", - ) + id="async_chunk", + ), ] @@ -63,7 +68,7 @@ def get_max_batch_size(size_type="few"): @pytest.mark.core_model @pytest.mark.advanced_model @pytest.mark.omni -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_text_to_text_001(omni_server, openai_client) -> None: """ @@ -88,7 +93,7 @@ def test_text_to_text_001(omni_server, openai_client) -> None: @pytest.mark.full_model @pytest.mark.omni -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_text_to_audio_001(omni_server, openai_client) -> None: """ @@ -113,7 +118,7 @@ def test_text_to_audio_001(omni_server, openai_client) -> None: @pytest.mark.full_model @pytest.mark.omni -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_audio_to_text_audio_001(omni_server, openai_client) -> None: """ @@ -142,7 +147,7 @@ def test_audio_to_text_audio_001(omni_server, openai_client) -> None: @pytest.mark.full_model @pytest.mark.omni -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_image_to_text_audio_001(omni_server, openai_client) -> None: """ @@ -170,7 +175,7 @@ def test_image_to_text_audio_001(omni_server, openai_client) -> None: @pytest.mark.full_model @pytest.mark.omni -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_video_to_text_audio_001(omni_server, openai_client) -> None: """ @@ -199,7 +204,7 @@ def test_video_to_text_audio_001(omni_server, openai_client) -> None: @pytest.mark.core_model @pytest.mark.advanced_model @pytest.mark.omni -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_mix_to_text_audio_001(omni_server, openai_client) -> None: """ diff --git a/tests/e2e/online_serving/test_minicpmo_4_5_duplex.py b/tests/e2e/online_serving/test_minicpmo_4_5_duplex.py index 2e409f689..23fa0edc6 100644 --- a/tests/e2e/online_serving/test_minicpmo_4_5_duplex.py +++ b/tests/e2e/online_serving/test_minicpmo_4_5_duplex.py @@ -19,7 +19,6 @@ from tests.e2e.online_serving.run_minicpmo_realtime_duplex_multi_session import ) from tests.helpers.mark import hardware_test from tests.helpers.minicpmo_4_5_duplex import ( - CORE_SERVER_PARAMS, SERVER_PARAMS, demo_args, multi_session_args, @@ -83,8 +82,8 @@ async def _run_protocol_smoke(*, url: str, model: str, ref_audio: Path) -> list[ @pytest.mark.core_model -@hardware_test(res={"cuda": "H100"}, num_cards=2) -@pytest.mark.parametrize("omni_server", CORE_SERVER_PARAMS, indirect=True) +@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) +@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True) def test_duplex_websocket_protocol_smoke(omni_server, model_prefix: str) -> None: ref_audio = resolve_ref_audio(model_prefix) events = asyncio.run( @@ -101,7 +100,7 @@ def test_duplex_websocket_protocol_smoke(omni_server, model_prefix: str) -> None @pytest.mark.advanced_model -@hardware_test(res={"cuda": "H100"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) @pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True) def test_duplex_single_session_response_required(omni_server, model_prefix: str, tmp_path: Path) -> None: result = asyncio.run( @@ -123,7 +122,7 @@ def test_duplex_single_session_response_required(omni_server, model_prefix: str, @pytest.mark.advanced_model -@hardware_test(res={"cuda": "H100"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) @pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True) def test_duplex_two_sessions_resume_and_takeover(omni_server, model_prefix: str, tmp_path: Path) -> None: result = asyncio.run( diff --git a/tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py b/tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py index 69c266456..acec7faec 100644 --- a/tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py +++ b/tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py @@ -29,7 +29,7 @@ from tests.helpers.minicpmo_4_5_duplex import ( pytestmark = [pytest.mark.full_model, pytest.mark.omni] -@hardware_test(res={"cuda": "H100"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) @pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True) def test_duplex_admission_and_expiry_reaper(omni_server, model_prefix: str, tmp_path: Path) -> None: args = multi_session_args( @@ -53,7 +53,7 @@ def test_duplex_admission_and_expiry_reaper(omni_server, model_prefix: str, tmp_ assert result["admission"]["overflow_error_code"] == "resource_exhausted" -@hardware_test(res={"cuda": "H100"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) @pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True) def test_duplex_soft_interrupt(omni_server, model_prefix: str, tmp_path: Path) -> None: input_wav = validated_soft_interrupt_wav() diff --git a/tests/e2e/online_serving/test_minicpmo_4_5_expansion.py b/tests/e2e/online_serving/test_minicpmo_4_5_expansion.py index 068502903..9572c55b6 100644 --- a/tests/e2e/online_serving/test_minicpmo_4_5_expansion.py +++ b/tests/e2e/online_serving/test_minicpmo_4_5_expansion.py @@ -2,6 +2,9 @@ These cover modality combinations, separate Code2Wav behavior, sequential request isolation, longer audio output, and Chinese speech. + +Like Qwen3-Omni expansion, ``default`` exercises ``--no-async-chunk`` and +``async_chunk`` exercises ``--async-chunk``. """ import os @@ -18,20 +21,29 @@ pytestmark = [pytest.mark.full_model, pytest.mark.omni] os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" _MODEL = "openbmb/MiniCPM-o-4_5" -_CI_DEPLOY = get_deploy_config_path("minicpmo_4_5_batching.yaml") +_CI_DEPLOY = get_deploy_config_path("minicpmo_4_5.yaml") +# Deploy YAML defaults to ``async_chunk: true``; keep sync and async as +# separate parametrizations so both handoff paths are exercised. test_params = [ pytest.param( OmniServerParams( model=_MODEL, stage_config_path=_CI_DEPLOY, use_stage_cli=True, - server_args=[ - "--trust-remote-code", - ], + server_args=["--trust-remote-code", "--no-async-chunk"], ), id="default", - ) + ), + pytest.param( + OmniServerParams( + model=_MODEL, + stage_config_path=_CI_DEPLOY, + use_stage_cli=True, + server_args=["--trust-remote-code", "--async-chunk"], + ), + id="async_chunk", + ), ] @@ -63,7 +75,7 @@ def get_max_batch_size(size_type="few"): return batch_sizes.get(size_type, 5) -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_text_video_to_text_001(omni_server, openai_client) -> None: """ @@ -90,7 +102,7 @@ def test_text_video_to_text_001(omni_server, openai_client) -> None: openai_client.send_omni_request(request_config, request_num=get_max_batch_size()) -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_sequential_requests_independent(omni_server, openai_client) -> None: """ @@ -131,7 +143,7 @@ def test_sequential_requests_independent(omni_server, openai_client) -> None: ) -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_text_to_audio_long_output_001(omni_server, openai_client) -> None: """ @@ -157,7 +169,7 @@ def test_text_to_audio_long_output_001(omni_server, openai_client) -> None: openai_client.send_omni_request(request_config, request_num=get_max_batch_size()) -@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2) +@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_chinese_text_to_audio(omni_server, openai_client) -> None: """ diff --git a/tests/helpers/minicpmo_4_5_duplex.py b/tests/helpers/minicpmo_4_5_duplex.py index dd32c0dcf..e95706de6 100644 --- a/tests/helpers/minicpmo_4_5_duplex.py +++ b/tests/helpers/minicpmo_4_5_duplex.py @@ -19,20 +19,6 @@ DEPLOY_CONFIG = modify_stage_config( get_deploy_config_path("minicpmo_4_5_duplex.yaml"), updates={ "base_config": get_deploy_config_path("minicpmo_4_5.yaml"), - "stages": { - 0: {"kv_cache_memory_bytes": 6 * 1024 * 1024 * 1024}, - 1: {"kv_cache_memory_bytes": 512 * 1024 * 1024}, - 2: {"kv_cache_memory_bytes": 256 * 1024 * 1024}, - }, - }, -) -CORE_DEPLOY_CONFIG = modify_stage_config( - DEPLOY_CONFIG, - updates={ - "stages": { - 0: {"enforce_eager": True}, - 1: {"enforce_eager": True}, - } }, ) ASSET_DIR = Path(__file__).resolve().parents[1] / "assets" / "minicpmo_4_5" @@ -47,18 +33,7 @@ SERVER_PARAMS = [ OmniServerParams( model=MODEL, stage_config_path=DEPLOY_CONFIG, - use_stage_cli=True, - server_args=["--trust-remote-code"], - ), - id="three-stage-single-gpu", - ) -] -CORE_SERVER_PARAMS = [ - pytest.param( - OmniServerParams( - model=MODEL, - stage_config_path=CORE_DEPLOY_CONFIG, - use_stage_cli=True, + use_stage_cli=False, server_args=["--trust-remote-code"], ), id="three-stage-single-gpu", diff --git a/tests/model_executor/models/minicpmo_4_5/test_pipeline.py b/tests/model_executor/models/minicpmo_4_5/test_pipeline.py index 2442cad44..feb60df31 100644 --- a/tests/model_executor/models/minicpmo_4_5/test_pipeline.py +++ b/tests/model_executor/models/minicpmo_4_5/test_pipeline.py @@ -163,11 +163,11 @@ class TestDeployTopology: expected_processor = "tts2code2wav_async_chunk" if deploy.async_chunk else "tts2code2wav_full_payload" assert stages[1].yaml_engine_args["custom_process_next_stage_input_func"].endswith(expected_processor) if filename == "minicpmo_4_5.yaml": - assert [stage.yaml_engine_args["max_num_seqs"] for stage in stages] == [4, 4, 4] + assert [stage.yaml_engine_args["max_num_seqs"] for stage in stages] == [16, 16, 16] assert [stage.yaml_engine_args["gpu_memory_utilization"] for stage in stages] == [ - 0.55, - 0.22, - 0.22, + 0.6, + 0.1, + 0.1, ] assert stages[0].yaml_engine_args["limit_mm_per_prompt"] == {"video": {"count": 1, "num_frames": 32}} elif filename in {"minicpmo_4_5_batching.yaml", "minicpmo_4_5_2gpu.yaml"}: diff --git a/vllm_omni/deploy/minicpmo_4_5.yaml b/vllm_omni/deploy/minicpmo_4_5.yaml index ea1fdaf2f..7e74c4081 100644 --- a/vllm_omni/deploy/minicpmo_4_5.yaml +++ b/vllm_omni/deploy/minicpmo_4_5.yaml @@ -17,11 +17,12 @@ connectors: stages: - stage_id: 0 - max_num_seqs: 4 - gpu_memory_utilization: 0.55 + max_num_seqs: 16 + gpu_memory_utilization: 0.6 enforce_eager: false tensor_parallel_size: 1 max_num_batched_tokens: 16384 + max_model_len: 32768 devices: "0" limit_mm_per_prompt: video: @@ -37,10 +38,11 @@ stages: repetition_penalty: 1.1 - stage_id: 1 - max_num_seqs: 4 - gpu_memory_utilization: 0.22 + max_num_seqs: 16 + gpu_memory_utilization: 0.1 enforce_eager: false max_num_batched_tokens: 8192 + max_model_len: 32768 devices: "0" output_connectors: to_stage_2: connector_of_shared_memory @@ -57,8 +59,8 @@ stages: detokenize: false - stage_id: 2 - max_num_seqs: 4 - gpu_memory_utilization: 0.22 + max_num_seqs: 16 + gpu_memory_utilization: 0.1 enforce_eager: true enable_chunked_prefill: false max_num_batched_tokens: 65536