[Config Refactor] Migrate Ming and Ming-TTS deploy/pipline configs (#3154)
Signed-off-by: Yuanheng Zhao <jonathan.zhaoyh@gmail.com> Signed-off-by: yuanheng <jonathan.zhaoyh@gmail.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
6a837849dd
commit
d49356e4ca
25 files changed
+518
-398
No files matched your search
@@ -8,8 +8,8 @@ Please refer to [README.md](../../../README.md)
|
||||
|
||||
| Mode | Launch command | Output |
|
||||
|------|---------------|--------|
|
||||
| Thinker only (multimodal understanding) | `vllm serve ... --omni` | Text |
|
||||
| Thinker + Talker (omni-speech) | `vllm serve ... --omni --stage-configs-path ming_flash_omni.yaml` | Text + Audio |
|
||||
| Thinker + Talker (omni-speech, default) | `vllm serve ... --omni` | Text + Audio |
|
||||
| Thinker only (multimodal understanding) | `vllm serve ... --omni --deploy-config vllm_omni/deploy/ming_flash_omni_thinker_only.yaml` | Text |
|
||||
|
||||
For standalone TTS (talker only), see [`examples/online_serving/ming_flash_omni_tts/`](../ming_flash_omni_tts/).
|
||||
|
||||
@@ -17,18 +17,20 @@ For standalone TTS (talker only), see [`examples/online_serving/ming_flash_omni_
|
||||
|
||||
### Launch the Server
|
||||
|
||||
**Thinker only (text output):**
|
||||
**Thinker + Talker (omni-speech, text + audio output):**
|
||||
```bash
|
||||
vllm serve Jonathan1909/Ming-flash-omni-2.0 --omni --port 8091
|
||||
```
|
||||
|
||||
**Thinker + Talker (omni-speech, text + audio output):**
|
||||
The model registry auto-loads corresponding deploy yaml.
|
||||
|
||||
**Thinker-only (text output):**
|
||||
```bash
|
||||
vllm serve Jonathan1909/Ming-flash-omni-2.0 --omni --port 8091 \
|
||||
--stage-configs-path vllm_omni/model_executor/stage_configs/ming_flash_omni.yaml
|
||||
--deploy-config vllm_omni/deploy/ming_flash_omni_thinker_only.yaml
|
||||
```
|
||||
|
||||
Pass `--stage-configs-path /path/to/your_config.yaml` to use a custom stage
|
||||
Pass `--deploy-config /path/to/your_deploy.yaml` to use a custom deploy
|
||||
config.
|
||||
|
||||
### Send Multi-modal Request
|
||||
|
||||
@@ -18,17 +18,6 @@ if [[ ! "$QUERY_TYPE" =~ ^(text|use_audio|use_image|use_video|use_mixed_modaliti
|
||||
exit 1
|
||||
fi
|
||||
|
||||
thinker_sampling_params='{
|
||||
"temperature": 0.4,
|
||||
"top_p": 0.9,
|
||||
"top_k": -1,
|
||||
"max_tokens": 16384,
|
||||
"seed": 42,
|
||||
"detokenize": true,
|
||||
"repetition_penalty": 1.05
|
||||
}'
|
||||
# Above is optional, it has a default setting in stage_configs of the corresponding model.
|
||||
|
||||
# Define URLs for assets
|
||||
MARY_HAD_LAMB_AUDIO_URL="https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/mary_had_lamb.ogg"
|
||||
CHERRY_BLOSSOM_IMAGE_URL="https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/cherry_blossom.jpg"
|
||||
@@ -114,9 +103,6 @@ echo ""
|
||||
request_body=$(cat <<EOF
|
||||
{
|
||||
"model": "Jonathan1909/Ming-flash-omni-2.0",
|
||||
"sampling_params_list": [
|
||||
$thinker_sampling_params
|
||||
],
|
||||
"modalities": ["text"],
|
||||
"messages": [
|
||||
{
|
||||
|
||||
@@ -5,7 +5,7 @@ This directory contains online e2e examples for **Ming-flash-omni-2.0 standalone
|
||||
Server uses:
|
||||
|
||||
- `model`: `Jonathan1909/Ming-flash-omni-2.0`
|
||||
- `stage config`: `vllm_omni/model_executor/stage_configs/ming_flash_omni_tts.yaml`
|
||||
- `deploy config`: `vllm_omni/deploy/ming_flash_omni_tts.yaml`
|
||||
|
||||
## Launch the Server
|
||||
|
||||
@@ -18,7 +18,7 @@ Equivalent manual command:
|
||||
|
||||
```bash
|
||||
vllm serve Jonathan1909/Ming-flash-omni-2.0 \
|
||||
--stage-configs-path vllm_omni/model_executor/stage_configs/ming_flash_omni_tts.yaml \
|
||||
--deploy-config vllm_omni/deploy/ming_flash_omni_tts.yaml \
|
||||
--host 0.0.0.0 \
|
||||
--port 8091 \
|
||||
--trust-remote-code \
|
||||
|
||||
@@ -12,13 +12,13 @@ set -e
|
||||
MODEL="${MODEL:-Jonathan1909/Ming-flash-omni-2.0}"
|
||||
HOST="${HOST:-0.0.0.0}"
|
||||
PORT="${PORT:-8091}"
|
||||
STAGE_CONFIG="${STAGE_CONFIG:-vllm_omni/model_executor/stage_configs/ming_flash_omni_tts.yaml}"
|
||||
DEPLOY_CONFIG="${DEPLOY_CONFIG:-vllm_omni/deploy/ming_flash_omni_tts.yaml}"
|
||||
|
||||
echo "Starting Ming standalone TTS server with model: $MODEL"
|
||||
echo "Stage config: $STAGE_CONFIG"
|
||||
echo "Deploy config: $DEPLOY_CONFIG"
|
||||
|
||||
vllm serve "$MODEL" \
|
||||
--stage-configs-path "$STAGE_CONFIG" \
|
||||
--deploy-config "$DEPLOY_CONFIG" \
|
||||
--host "$HOST" \
|
||||
--port "$PORT" \
|
||||
--trust-remote-code \
|
||||
|
||||
Reference in new issue
Block a user