[CI/Build][MiniCPM-o] Add MiniCPM-o 4.5 accuracy and performance nightly coverage (#5710)
Signed-off-by: psv666 <2693925048@qq.com> Signed-off-by: psv666 <150513104+psv666@users.noreply.github.com> Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com> Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
1 parent
a5eb2a9a87
commit
4ce5fcb16c
15 files changed
+428
-108
No files matched your search
@@ -8,8 +8,7 @@
|
|||||||
# NPU naming: {queue}_{resource_class}
|
# NPU naming: {queue}_{resource_class}
|
||||||
# a2b3_npu_1 — ascend-a2b3, resource_class npu-1 (A2/B3 CI image)
|
# a2b3_npu_1 — ascend-a2b3, resource_class npu-1 (A2/B3 CI image)
|
||||||
# a2b3_npu_4 — ascend-a2b3, resource_class npu-4
|
# a2b3_npu_4 — ascend-a2b3, resource_class npu-4
|
||||||
# a3_npu_1 — ascend-a3, resource_class npu-1 (A3 CI image tag …:$BUILDKITE_COMMIT-a3)
|
# a3_npu_2 — ascend-a3, resource_class npu-2 (A3 CI image tag …:$BUILDKITE_COMMIT-a3)
|
||||||
# a3_npu_2 — ascend-a3, resource_class npu-2
|
|
||||||
|
|
||||||
x-npu-k8s-plugins: &npu_k8s_plugins
|
x-npu-k8s-plugins: &npu_k8s_plugins
|
||||||
- kubernetes:
|
- kubernetes:
|
||||||
@@ -211,13 +210,6 @@ mirror_hardwares:
|
|||||||
image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}"
|
image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}"
|
||||||
plugins: *npu_k8s_plugins
|
plugins: *npu_k8s_plugins
|
||||||
|
|
||||||
a3_npu_1:
|
|
||||||
agents:
|
|
||||||
queue: "ascend-a3"
|
|
||||||
resource_class: "npu-1"
|
|
||||||
image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a3"
|
|
||||||
plugins: *npu_k8s_plugins
|
|
||||||
|
|
||||||
a3_npu_2:
|
a3_npu_2:
|
||||||
agents:
|
agents:
|
||||||
queue: "ascend-a3"
|
queue: "ascend-a3"
|
||||||
|
|||||||
@@ -41,7 +41,7 @@ steps:
|
|||||||
if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "omni-test"
|
if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "omni-test"
|
||||||
steps:
|
steps:
|
||||||
- label: ":full_moon: Omni · MiniCPM-o 4.5 Test"
|
- label: ":full_moon: Omni · MiniCPM-o 4.5 Test"
|
||||||
mirror_hardwares: a3_npu_1
|
mirror_hardwares: a3_npu_2
|
||||||
env:
|
env:
|
||||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||||
@@ -50,21 +50,27 @@ steps:
|
|||||||
- pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5_expansion.py -m 'full_model' --run-level 'full_model'
|
- pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5_expansion.py -m 'full_model' --run-level 'full_model'
|
||||||
- label: ":full_moon: Omni · MiniCPM-o 4.5 · Duplex Test"
|
- label: ":full_moon: Omni · MiniCPM-o 4.5 · Duplex Test"
|
||||||
timeout_in_minutes: 180
|
timeout_in_minutes: 180
|
||||||
mirror_hardwares: a3_npu_1
|
mirror_hardwares: a3_npu_2
|
||||||
env:
|
env:
|
||||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||||
HF_TOKEN: "${HF_TOKEN}"
|
HF_TOKEN: "${HF_TOKEN}"
|
||||||
commands:
|
commands:
|
||||||
|
# NPU has no merge-level pipeline, so advanced_model duplex coverage
|
||||||
|
# rides on nightly alongside full_model.
|
||||||
|
#
|
||||||
|
# test_duplex_soft_interrupt is CUDA-only and carries no npu mark, so
|
||||||
|
# the marker filter below drops it: it needs real-time duplex
|
||||||
|
# throughput that the current NPU stack does not reach.
|
||||||
- >-
|
- >-
|
||||||
pytest -s -v
|
pytest -s -v
|
||||||
tests/e2e/online_serving/test_minicpmo_4_5_duplex.py
|
tests/e2e/online_serving/test_minicpmo_4_5_duplex.py
|
||||||
tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
|
tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
|
||||||
-m "npu and A3 and omni"
|
-m "(full_model or advanced_model) and npu and A3 and omni"
|
||||||
--run-level full_model
|
--run-level full_model
|
||||||
- label: ":full_moon: Omni · MiniCPM-o 4.5 · Accuracy Test"
|
- label: ":full_moon: Omni · MiniCPM-o 4.5 · Accuracy Test"
|
||||||
timeout_in_minutes: 180
|
timeout_in_minutes: 180
|
||||||
mirror_hardwares: a3_npu_1
|
mirror_hardwares: a3_npu_2
|
||||||
env:
|
env:
|
||||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||||
@@ -72,7 +78,39 @@ steps:
|
|||||||
commands:
|
commands:
|
||||||
- |
|
- |
|
||||||
set +e
|
set +e
|
||||||
pytest -s -v tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py -m "full_model and npu" --run-level full_model
|
pytest -s -v tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py -m "full_model and npu and A3" --run-level full_model
|
||||||
EXIT=$$?
|
EXIT=$$?
|
||||||
buildkite-agent artifact upload "tests/e2e/accuracy/minicpmo_4_5/results/*.json"
|
buildkite-agent artifact upload "tests/e2e/accuracy/minicpmo_4_5/results/*.json"
|
||||||
exit $$EXIT
|
exit $$EXIT
|
||||||
|
- label: ":full_moon: Omni · MiniCPM-o 4.5 · Perf Test"
|
||||||
|
key: nightly-omni-performance-minicpmo-4-5-npu
|
||||||
|
timeout_in_minutes: 180
|
||||||
|
mirror_hardwares: a3_npu_2
|
||||||
|
env:
|
||||||
|
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||||
|
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||||
|
HF_TOKEN: "${HF_TOKEN}"
|
||||||
|
commands:
|
||||||
|
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||||
|
- |
|
||||||
|
set +e
|
||||||
|
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m "npu and A3" --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5.json
|
||||||
|
EXIT=$$?
|
||||||
|
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
||||||
|
exit $$EXIT
|
||||||
|
- label: ":full_moon: Omni · MiniCPM-o 4.5 · Duplex Seed-TTS Perf Test"
|
||||||
|
key: nightly-omni-performance-minicpmo-4-5-duplex-seed-tts-npu
|
||||||
|
timeout_in_minutes: 180
|
||||||
|
mirror_hardwares: a3_npu_2
|
||||||
|
env:
|
||||||
|
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||||
|
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||||
|
HF_TOKEN: "${HF_TOKEN}"
|
||||||
|
commands:
|
||||||
|
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||||
|
- |
|
||||||
|
set +e
|
||||||
|
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m "npu and A3" --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json
|
||||||
|
EXIT=$$?
|
||||||
|
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
||||||
|
exit $$EXIT
|
||||||
@@ -4,7 +4,7 @@ steps:
|
|||||||
depends_on: upload-ready-pipeline
|
depends_on: upload-ready-pipeline
|
||||||
steps:
|
steps:
|
||||||
- label: "Omni · MiniCPM-o 4.5 Test"
|
- label: "Omni · MiniCPM-o 4.5 Test"
|
||||||
mirror_hardwares: a3_npu_1
|
mirror_hardwares: a3_npu_2
|
||||||
env:
|
env:
|
||||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||||
@@ -12,7 +12,7 @@ steps:
|
|||||||
commands:
|
commands:
|
||||||
- pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py -m 'core_model and npu and A3 and omni' --run-level 'core_model'
|
- pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py -m 'core_model and npu and A3 and omni' --run-level 'core_model'
|
||||||
- label: "Omni · MiniCPM-o 4.5 Duplex Test"
|
- label: "Omni · MiniCPM-o 4.5 Duplex Test"
|
||||||
mirror_hardwares: a3_npu_1
|
mirror_hardwares: a3_npu_2
|
||||||
env:
|
env:
|
||||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||||
|
|||||||
@@ -41,6 +41,9 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
|
|||||||
"hyperpyyaml>=1.2.0" \
|
"hyperpyyaml>=1.2.0" \
|
||||||
"onnx>=1.16.0" \
|
"onnx>=1.16.0" \
|
||||||
"librosa>=0.10.0" \
|
"librosa>=0.10.0" \
|
||||||
|
"jiwer>=4.0.0" \
|
||||||
|
"zhon>=2.1.1" \
|
||||||
|
"zhconv>=1.4.3" \
|
||||||
--index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
|
--index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
|
||||||
--trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
|
--trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
|
||||||
--default-timeout=60 && \
|
--default-timeout=60 && \
|
||||||
|
|||||||
@@ -41,6 +41,9 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
|
|||||||
"hyperpyyaml>=1.2.0" \
|
"hyperpyyaml>=1.2.0" \
|
||||||
"onnx>=1.16.0" \
|
"onnx>=1.16.0" \
|
||||||
"librosa>=0.10.0" \
|
"librosa>=0.10.0" \
|
||||||
|
"jiwer>=4.0.0" \
|
||||||
|
"zhon>=2.1.1" \
|
||||||
|
"zhconv>=1.4.3" \
|
||||||
--index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
|
--index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
|
||||||
--trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
|
--trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
|
||||||
--default-timeout=60 && \
|
--default-timeout=60 && \
|
||||||
|
|||||||
@@ -0,0 +1,92 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
"""Offline Daily-Omni sources: local mirrors must not fall back to the Hub id."""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import tarfile
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from vllm_omni.benchmarks.data_modules.daily_omni_dataset import (
|
||||||
|
daily_omni_local_qa_json,
|
||||||
|
daily_omni_local_videos_dir,
|
||||||
|
resolve_daily_omni_local_root,
|
||||||
|
)
|
||||||
|
|
||||||
|
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.benchmark]
|
||||||
|
|
||||||
|
|
||||||
|
def _write_qa_json(root: Path) -> Path:
|
||||||
|
qa = root / "qa.json"
|
||||||
|
qa.write_text(json.dumps([{"Question": "q", "Answer": "A", "video_id": "vid0"}]), encoding="utf-8")
|
||||||
|
return qa
|
||||||
|
|
||||||
|
|
||||||
|
def test_resolve_local_root_returns_none_for_hub_id() -> None:
|
||||||
|
assert resolve_daily_omni_local_root("liarliar/Daily-Omni") is None
|
||||||
|
assert resolve_daily_omni_local_root("") is None
|
||||||
|
assert resolve_daily_omni_local_root(None) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_plain_local_mirror(tmp_path: Path) -> None:
|
||||||
|
_write_qa_json(tmp_path)
|
||||||
|
(tmp_path / "Videos" / "vid0").mkdir(parents=True)
|
||||||
|
|
||||||
|
root = resolve_daily_omni_local_root(str(tmp_path))
|
||||||
|
assert root == tmp_path.resolve()
|
||||||
|
assert daily_omni_local_qa_json(root) == (tmp_path / "qa.json").resolve()
|
||||||
|
assert daily_omni_local_videos_dir(root) == root / "Videos"
|
||||||
|
|
||||||
|
|
||||||
|
def test_hf_cache_dir_resolves_to_snapshot(tmp_path: Path) -> None:
|
||||||
|
cache_dir = tmp_path / "datasets--liarliar--Daily-Omni"
|
||||||
|
snapshot = cache_dir / "snapshots" / "deadbeef"
|
||||||
|
snapshot.mkdir(parents=True)
|
||||||
|
_write_qa_json(snapshot)
|
||||||
|
(cache_dir / "refs").mkdir()
|
||||||
|
(cache_dir / "refs" / "main").write_text("deadbeef", encoding="utf-8")
|
||||||
|
|
||||||
|
root = resolve_daily_omni_local_root(str(cache_dir))
|
||||||
|
assert root == snapshot.resolve()
|
||||||
|
assert daily_omni_local_qa_json(root) == (snapshot / "qa.json").resolve()
|
||||||
|
|
||||||
|
|
||||||
|
def test_hf_cache_dir_without_refs_picks_newest_snapshot(tmp_path: Path) -> None:
|
||||||
|
cache_dir = tmp_path / "datasets--liarliar--Daily-Omni"
|
||||||
|
older = cache_dir / "snapshots" / "aaa"
|
||||||
|
newer = cache_dir / "snapshots" / "bbb"
|
||||||
|
older.mkdir(parents=True)
|
||||||
|
newer.mkdir(parents=True)
|
||||||
|
os.utime(older, (1, 1))
|
||||||
|
os.utime(newer, (10, 10))
|
||||||
|
|
||||||
|
assert resolve_daily_omni_local_root(str(cache_dir)) == newer.resolve()
|
||||||
|
|
||||||
|
|
||||||
|
def test_videos_tar_is_extracted(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
monkeypatch.setenv("HF_HOME", str(tmp_path / "hf_home"))
|
||||||
|
root = tmp_path / "mirror"
|
||||||
|
(root / "Videos" / "vid0").mkdir(parents=True)
|
||||||
|
(root / "Videos" / "vid0" / "vid0_video.mp4").write_bytes(b"fake")
|
||||||
|
with tarfile.open(root / "Videos.tar", "w") as tf:
|
||||||
|
tf.add(root / "Videos", arcname="Videos")
|
||||||
|
# Only the tarball is left in the mirror.
|
||||||
|
for child in (root / "Videos").rglob("*"):
|
||||||
|
if child.is_file():
|
||||||
|
child.unlink()
|
||||||
|
(root / "Videos" / "vid0").rmdir()
|
||||||
|
(root / "Videos").rmdir()
|
||||||
|
|
||||||
|
videos_dir = daily_omni_local_videos_dir(root)
|
||||||
|
assert videos_dir is not None
|
||||||
|
assert (videos_dir / "vid0" / "vid0_video.mp4").is_file()
|
||||||
|
|
||||||
|
# Second call reuses the extraction instead of redoing it.
|
||||||
|
assert daily_omni_local_videos_dir(root) == videos_dir
|
||||||
|
|
||||||
|
|
||||||
|
def test_missing_qa_json_reports_none(tmp_path: Path) -> None:
|
||||||
|
root = resolve_daily_omni_local_root(str(tmp_path))
|
||||||
|
assert daily_omni_local_qa_json(root) is None
|
||||||
|
assert daily_omni_local_videos_dir(root) is None
|
||||||
@@ -5,7 +5,8 @@
|
|||||||
{
|
{
|
||||||
"hardware_marks": {
|
"hardware_marks": {
|
||||||
"res": {
|
"res": {
|
||||||
"cuda": "H100"
|
"cuda": "H100",
|
||||||
|
"npu": "A3"
|
||||||
},
|
},
|
||||||
"num_cards": 1
|
"num_cards": 1
|
||||||
}
|
}
|
||||||
@@ -30,14 +31,12 @@
|
|||||||
"num_prompts": [
|
"num_prompts": [
|
||||||
32,
|
32,
|
||||||
64,
|
64,
|
||||||
128,
|
128
|
||||||
256
|
|
||||||
],
|
],
|
||||||
"max_concurrency": [
|
"max_concurrency": [
|
||||||
1,
|
1,
|
||||||
4,
|
4,
|
||||||
8,
|
8
|
||||||
16
|
|
||||||
],
|
],
|
||||||
"no_oversample": true,
|
"no_oversample": true,
|
||||||
"trust_remote_code": true,
|
"trust_remote_code": true,
|
||||||
@@ -55,34 +54,29 @@
|
|||||||
"percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration",
|
"percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration",
|
||||||
"baseline": {
|
"baseline": {
|
||||||
"request_throughput": [
|
"request_throughput": [
|
||||||
0.7803,
|
0.5383,
|
||||||
1.1008,
|
0.6042,
|
||||||
1.2027,
|
0.7547
|
||||||
1.3254
|
|
||||||
],
|
],
|
||||||
"mean_ttft_ms": [
|
"mean_ttft_ms": [
|
||||||
166.9764,
|
333.2633,
|
||||||
307.7133,
|
533.8656,
|
||||||
323.584,
|
547.3388
|
||||||
356.6077
|
|
||||||
],
|
],
|
||||||
"mean_e2el_ms": [
|
"mean_e2el_ms": [
|
||||||
1361.1682,
|
1857.2154,
|
||||||
3800.6332,
|
6600.5101,
|
||||||
6961.8636,
|
10450.6811
|
||||||
12609.4295
|
|
||||||
],
|
],
|
||||||
"mean_audio_ttfp_ms": [
|
"mean_audio_ttfp_ms": [
|
||||||
635.8099,
|
986.4666,
|
||||||
1615.6591,
|
3411.0754,
|
||||||
2888.9831,
|
3352.7515
|
||||||
8320.5422
|
|
||||||
],
|
],
|
||||||
"mean_audio_rtf": [
|
"mean_audio_rtf": [
|
||||||
0.3151,
|
0.4423,
|
||||||
0.8568,
|
1.5734,
|
||||||
1.5425,
|
2.3024
|
||||||
2.8224
|
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,7 +5,8 @@
|
|||||||
{
|
{
|
||||||
"hardware_marks": {
|
"hardware_marks": {
|
||||||
"res": {
|
"res": {
|
||||||
"cuda": "H100"
|
"cuda": "H100",
|
||||||
|
"npu": "A3"
|
||||||
},
|
},
|
||||||
"num_cards": 1
|
"num_cards": 1
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -39,10 +39,7 @@ _MAX_SEED_TTS_MEAN_WER = 0.05
|
|||||||
# Match the validated Daily-Omni client body from daily_omni_bench.sh.
|
# Match the validated Daily-Omni client body from daily_omni_bench.sh.
|
||||||
_DAILY_EXTRA_BODY = {
|
_DAILY_EXTRA_BODY = {
|
||||||
"modalities": ["text"],
|
"modalities": ["text"],
|
||||||
"chat_template_kwargs": {
|
"chat_template_kwargs": {"enable_thinking": False},
|
||||||
"enable_thinking": False,
|
|
||||||
"use_tts_template": True,
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
_SEED_EXTRA_BODY = {
|
_SEED_EXTRA_BODY = {
|
||||||
"modalities": ["text", "audio"],
|
"modalities": ["text", "audio"],
|
||||||
@@ -151,6 +148,7 @@ def test_minicpmo_4_5_seed_tts_wer_bench(omni_server) -> None:
|
|||||||
omni_server,
|
omni_server,
|
||||||
skip_seed=False,
|
skip_seed=False,
|
||||||
skip_daily=True,
|
skip_daily=True,
|
||||||
|
max_concurrency=4,
|
||||||
)
|
)
|
||||||
argv.extend(
|
argv.extend(
|
||||||
[
|
[
|
||||||
|
|||||||
@@ -963,12 +963,23 @@ async def _send_clean_turn(
|
|||||||
if outcome == "listen":
|
if outcome == "listen":
|
||||||
return None, "listen"
|
return None, "listen"
|
||||||
else:
|
else:
|
||||||
await _wait_for(
|
try:
|
||||||
state,
|
await _wait_for(
|
||||||
lambda: state.count("response.created") > before_created,
|
state,
|
||||||
timeout_s=timeout_s,
|
lambda: state.count("response.created") > before_created,
|
||||||
label=f"{transcript} response.created",
|
timeout_s=timeout_s,
|
||||||
)
|
label=f"{transcript} response.created",
|
||||||
|
)
|
||||||
|
except TimeoutError:
|
||||||
|
# ``auto_response`` sessions only get ``response.created`` once the model
|
||||||
|
# decides to speak, so report the listen decisions that took its place.
|
||||||
|
raise TimeoutError(
|
||||||
|
f"Timed out waiting for {transcript} response.created after "
|
||||||
|
f"{state.count('input_audio_buffer.committed')} committed input turns; "
|
||||||
|
f"model_listen={state.model_listen_count - before_model_listen} "
|
||||||
|
f"buffering_listen={state.buffering_listen_count} "
|
||||||
|
f"errors={state.count('error')}"
|
||||||
|
) from None
|
||||||
await _wait_for(
|
await _wait_for(
|
||||||
state,
|
state,
|
||||||
lambda: eligible_response_id() is not None,
|
lambda: eligible_response_id() is not None,
|
||||||
|
|||||||
@@ -133,6 +133,10 @@ def test_duplex_single_session_response_required(omni_server, model_prefix: str,
|
|||||||
output_dir=tmp_path / "single_session",
|
output_dir=tmp_path / "single_session",
|
||||||
)
|
)
|
||||||
args.turns = 2
|
args.turns = 2
|
||||||
|
# Every turn replays the same active-speech window as the first one. The default
|
||||||
|
# shorter follow-up window is a different mid-utterance slice, which the native
|
||||||
|
# duplex model may legitimately answer with "listen" instead of a response.
|
||||||
|
args.turn_duration_ms = [args.first_turn_ms] * args.turns
|
||||||
result = asyncio.run(run_demo(args))
|
result = asyncio.run(run_demo(args))
|
||||||
assert result["ok"] is True
|
assert result["ok"] is True
|
||||||
assert result["audio_delta_count"] > 0
|
assert result["audio_delta_count"] > 0
|
||||||
|
|||||||
@@ -53,7 +53,11 @@ def test_duplex_admission_and_expiry_reaper(omni_server, model_prefix: str, tmp_
|
|||||||
assert result["admission"]["overflow_error_code"] == "resource_exhausted"
|
assert result["admission"]["overflow_error_code"] == "resource_exhausted"
|
||||||
|
|
||||||
|
|
||||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
# CUDA-only for now: this contract asserts the model speaks while the user is
|
||||||
|
# still talking, which only holds when the duplex pipeline sustains real-time
|
||||||
|
# throughput. The current NPU stack runs several times slower than real time,
|
||||||
|
# so it never reaches a mid-stream decision point.
|
||||||
|
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||||
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
|
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
|
||||||
def test_duplex_soft_interrupt(omni_server, model_prefix: str, tmp_path: Path) -> None:
|
def test_duplex_soft_interrupt(omni_server, model_prefix: str, tmp_path: Path) -> None:
|
||||||
input_wav = validated_soft_interrupt_wav()
|
input_wav = validated_soft_interrupt_wav()
|
||||||
|
|||||||
@@ -74,9 +74,11 @@ def validated_soft_interrupt_wav() -> Path:
|
|||||||
|
|
||||||
|
|
||||||
def resolve_ref_audio(model_prefix: str) -> Path:
|
def resolve_ref_audio(model_prefix: str) -> Path:
|
||||||
if model_prefix:
|
# ``MODEL`` may be a local checkpoint directory instead of a Hub repo id
|
||||||
model_root = Path(model_prefix) / MODEL
|
# (``Path("/prefix") / "/abs/path"`` collapses to the absolute path), so only
|
||||||
else:
|
# fall back to the Hub cache when no such directory exists on disk.
|
||||||
|
model_root = Path(model_prefix) / MODEL if model_prefix else Path(MODEL)
|
||||||
|
if not model_root.is_dir():
|
||||||
model_root = Path(snapshot_download(MODEL, local_files_only=True))
|
model_root = Path(snapshot_download(MODEL, local_files_only=True))
|
||||||
ref_audio = model_root / REF_AUDIO_RELATIVE_PATH
|
ref_audio = model_root / REF_AUDIO_RELATIVE_PATH
|
||||||
if not ref_audio.is_file():
|
if not ref_audio.is_file():
|
||||||
|
|||||||
@@ -110,47 +110,17 @@ def _daily_omni_find_videos_root_in_extract(tmp: Path) -> Path:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def ensure_daily_omni_hub_videos_dir(repo_id: str) -> Path:
|
def _extract_daily_omni_videos_tar(tar_path: Path, cache_key: str) -> Path:
|
||||||
"""Download ``Videos.tar`` from the Hugging Face dataset repo and return the ``Videos`` root.
|
"""Extract ``Videos.tar`` once into the shared media cache and return the ``Videos`` root.
|
||||||
|
|
||||||
The returned path matches ``--daily-omni-video-dir`` (directory containing ``{{video_id}}/``).
|
Cached under ``HF_HOME`` / ``vllm_omni/daily_omni_media/<cache_key>``. Reuses a previous
|
||||||
|
extraction when the tarball fingerprint matches.
|
||||||
Cached under ``HF_HOME`` / ``vllm_omni/daily_omni_media/<repo>``. Reuses extraction when the
|
|
||||||
tarball fingerprint matches.
|
|
||||||
|
|
||||||
Raises:
|
|
||||||
ImportError: if ``huggingface_hub`` is not installed.
|
|
||||||
FileNotFoundError / RuntimeError: if the archive is missing or malformed.
|
|
||||||
"""
|
"""
|
||||||
rid = (repo_id or "").strip()
|
safe = cache_key.replace("/", "__").replace("\\", "_")
|
||||||
if not rid:
|
|
||||||
raise ValueError("repo_id is required to download Daily-Omni Videos.tar")
|
|
||||||
|
|
||||||
try:
|
|
||||||
from huggingface_hub import hf_hub_download
|
|
||||||
except ImportError as e:
|
|
||||||
raise ImportError(
|
|
||||||
"Daily-Omni Hub media download requires huggingface_hub. "
|
|
||||||
"Install it (e.g. with vLLM) or provide --daily-omni-video-dir with a local extract."
|
|
||||||
) from e
|
|
||||||
|
|
||||||
safe = rid.replace("/", "__").replace("\\", "_")
|
|
||||||
staging_root = _daily_omni_hf_cache_root() / "vllm_omni" / "daily_omni_media" / safe
|
staging_root = _daily_omni_hf_cache_root() / "vllm_omni" / "daily_omni_media" / safe
|
||||||
videos_dir = staging_root / "Videos"
|
videos_dir = staging_root / "Videos"
|
||||||
marker = staging_root / ".videos_extracted"
|
marker = staging_root / ".videos_extracted"
|
||||||
|
|
||||||
tar_path: Path | None = None
|
|
||||||
for fname in ("Videos.tar", "videos.tar"):
|
|
||||||
try:
|
|
||||||
tar_path = Path(hf_hub_download(repo_id=rid, filename=fname, repo_type="dataset"))
|
|
||||||
break
|
|
||||||
except Exception:
|
|
||||||
continue
|
|
||||||
if tar_path is None or not tar_path.is_file():
|
|
||||||
raise FileNotFoundError(
|
|
||||||
f"Could not download Videos.tar from Hugging Face dataset {rid!r} (tried Videos.tar / videos.tar)."
|
|
||||||
)
|
|
||||||
|
|
||||||
fp = _daily_omni_tar_fingerprint(tar_path)
|
fp = _daily_omni_tar_fingerprint(tar_path)
|
||||||
if marker.is_file() and videos_dir.is_dir():
|
if marker.is_file() and videos_dir.is_dir():
|
||||||
try:
|
try:
|
||||||
@@ -167,7 +137,7 @@ def ensure_daily_omni_hub_videos_dir(repo_id: str) -> Path:
|
|||||||
shutil.rmtree(work, ignore_errors=True)
|
shutil.rmtree(work, ignore_errors=True)
|
||||||
work.mkdir(parents=True)
|
work.mkdir(parents=True)
|
||||||
try:
|
try:
|
||||||
logger.info("Extracting Daily-Omni Videos.tar from %s (repo=%s)", tar_path, rid)
|
logger.info("Extracting Daily-Omni Videos.tar from %s (cache_key=%s)", tar_path, cache_key)
|
||||||
with tarfile.open(tar_path, "r:*") as tf:
|
with tarfile.open(tar_path, "r:*") as tf:
|
||||||
tf.extractall(path=work, filter="data")
|
tf.extractall(path=work, filter="data")
|
||||||
found = _daily_omni_find_videos_root_in_extract(work)
|
found = _daily_omni_find_videos_root_in_extract(work)
|
||||||
@@ -178,10 +148,106 @@ def ensure_daily_omni_hub_videos_dir(repo_id: str) -> Path:
|
|||||||
shutil.rmtree(work, ignore_errors=True)
|
shutil.rmtree(work, ignore_errors=True)
|
||||||
|
|
||||||
marker.write_text(fp, encoding="utf-8")
|
marker.write_text(fp, encoding="utf-8")
|
||||||
logger.info("Daily-Omni Hub media ready at %s", videos_dir)
|
logger.info("Daily-Omni media ready at %s", videos_dir)
|
||||||
return videos_dir
|
return videos_dir
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_daily_omni_hub_videos_dir(repo_id: str) -> Path:
|
||||||
|
"""Download ``Videos.tar`` from the Hugging Face dataset repo and return the ``Videos`` root.
|
||||||
|
|
||||||
|
The returned path matches ``--daily-omni-video-dir`` (directory containing ``{{video_id}}/``).
|
||||||
|
|
||||||
|
Raises:
|
||||||
|
ImportError: if ``huggingface_hub`` is not installed.
|
||||||
|
FileNotFoundError / RuntimeError: if the archive is missing or malformed.
|
||||||
|
"""
|
||||||
|
rid = (repo_id or "").strip()
|
||||||
|
if not rid:
|
||||||
|
raise ValueError("repo_id is required to download Daily-Omni Videos.tar")
|
||||||
|
|
||||||
|
try:
|
||||||
|
from huggingface_hub import hf_hub_download
|
||||||
|
except ImportError as e:
|
||||||
|
raise ImportError(
|
||||||
|
"Daily-Omni Hub media download requires huggingface_hub. "
|
||||||
|
"Install it (e.g. with vLLM) or provide --daily-omni-video-dir with a local extract."
|
||||||
|
) from e
|
||||||
|
|
||||||
|
tar_path: Path | None = None
|
||||||
|
for fname in ("Videos.tar", "videos.tar"):
|
||||||
|
try:
|
||||||
|
tar_path = Path(hf_hub_download(repo_id=rid, filename=fname, repo_type="dataset"))
|
||||||
|
break
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
if tar_path is None or not tar_path.is_file():
|
||||||
|
raise FileNotFoundError(
|
||||||
|
f"Could not download Videos.tar from Hugging Face dataset {rid!r} (tried Videos.tar / videos.tar)."
|
||||||
|
)
|
||||||
|
|
||||||
|
return _extract_daily_omni_videos_tar(tar_path, rid)
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_hf_cache_snapshot(path: Path) -> Path:
|
||||||
|
"""Map a HF hub cache dir (``datasets--org--name``) to its checked-out snapshot dir.
|
||||||
|
|
||||||
|
Any other directory is returned unchanged.
|
||||||
|
"""
|
||||||
|
snapshots = path / "snapshots"
|
||||||
|
if not snapshots.is_dir():
|
||||||
|
return path
|
||||||
|
ref = path / "refs" / "main"
|
||||||
|
if ref.is_file():
|
||||||
|
try:
|
||||||
|
target = snapshots / ref.read_text(encoding="utf-8").strip()
|
||||||
|
except OSError:
|
||||||
|
target = snapshots
|
||||||
|
if target.is_dir():
|
||||||
|
return target
|
||||||
|
revisions = [p for p in snapshots.iterdir() if p.is_dir()]
|
||||||
|
if revisions:
|
||||||
|
return max(revisions, key=lambda p: p.stat().st_mtime)
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_daily_omni_local_root(dataset_path: str | None) -> Path | None:
|
||||||
|
"""Return the local Daily-Omni mirror root for ``dataset_path``, or ``None`` if not local.
|
||||||
|
|
||||||
|
Accepts a plain directory (``qa.json`` + ``Videos/`` or ``Videos.tar``, i.e. the layout of the
|
||||||
|
Hub repo) as well as a HuggingFace hub cache directory (``datasets--liarliar--Daily-Omni``),
|
||||||
|
which is resolved to its snapshot. Hub ids such as ``liarliar/Daily-Omni`` return ``None``.
|
||||||
|
"""
|
||||||
|
raw = (dataset_path or "").strip()
|
||||||
|
if not raw:
|
||||||
|
return None
|
||||||
|
path = Path(raw).expanduser()
|
||||||
|
if not path.is_dir():
|
||||||
|
return None
|
||||||
|
return _resolve_hf_cache_snapshot(path.resolve())
|
||||||
|
|
||||||
|
|
||||||
|
def daily_omni_local_qa_json(root: Path) -> Path | None:
|
||||||
|
"""Locate ``qa.json`` inside a local Daily-Omni mirror (``None`` when absent)."""
|
||||||
|
for name in ("qa.json", "QA.json"):
|
||||||
|
candidate = root / name
|
||||||
|
if candidate.is_file():
|
||||||
|
return candidate
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def daily_omni_local_videos_dir(root: Path) -> Path | None:
|
||||||
|
"""Return the ``Videos`` root of a local mirror, extracting ``Videos.tar`` when needed."""
|
||||||
|
for name in ("Videos", "videos"):
|
||||||
|
candidate = root / name
|
||||||
|
if candidate.is_dir():
|
||||||
|
return candidate
|
||||||
|
for name in ("Videos.tar", "videos.tar"):
|
||||||
|
tar_path = root / name
|
||||||
|
if tar_path.is_file():
|
||||||
|
return _extract_daily_omni_videos_tar(tar_path, f"local__{root.name}")
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
class _ListDatasetIterator:
|
class _ListDatasetIterator:
|
||||||
"""Simple iterator wrapper around a list to mimic HuggingFace streaming dataset behavior."""
|
"""Simple iterator wrapper around a list to mimic HuggingFace streaming dataset behavior."""
|
||||||
|
|
||||||
@@ -243,6 +309,73 @@ def _uniform_sample_indices(n: int, k: int) -> list[int]:
|
|||||||
return [int(round(i * (n - 1) / (k - 1))) for i in range(k)]
|
return [int(round(i * (n - 1) / (k - 1))) for i in range(k)]
|
||||||
|
|
||||||
|
|
||||||
|
def _probe_video_frames_and_fps(video_path: Path) -> tuple[int, float]:
|
||||||
|
"""Frame count and average FPS, replacing ``len(vr)`` / ``vr.get_avg_fps()``.
|
||||||
|
|
||||||
|
PyAV is used instead of ``decord`` because ``decord`` ships x86_64-only Linux wheels
|
||||||
|
and is unmaintained, while ``av`` is already a first-class requirement and has
|
||||||
|
aarch64 wheels (Ascend hosts).
|
||||||
|
"""
|
||||||
|
import av
|
||||||
|
|
||||||
|
with av.open(str(video_path)) as container:
|
||||||
|
stream = container.streams.video[0]
|
||||||
|
avg_fps = float(stream.average_rate) if stream.average_rate else 0.0
|
||||||
|
num_frames = stream.frames or 0
|
||||||
|
if num_frames <= 0:
|
||||||
|
# Containers such as raw webm leave ``frames`` at 0; derive it from duration.
|
||||||
|
duration: float | None = None
|
||||||
|
if stream.duration is not None and stream.time_base is not None:
|
||||||
|
duration = float(stream.duration * stream.time_base)
|
||||||
|
elif container.duration is not None:
|
||||||
|
duration = container.duration / av.time_base
|
||||||
|
if duration and avg_fps:
|
||||||
|
num_frames = int(math.floor(duration * avg_fps))
|
||||||
|
if num_frames <= 0:
|
||||||
|
# Last resort for streams with no usable metadata at all.
|
||||||
|
stream.thread_type = "AUTO"
|
||||||
|
num_frames = sum(1 for _ in container.decode(stream))
|
||||||
|
if num_frames <= 0:
|
||||||
|
raise ValueError(f"No decodable video frames in {video_path}")
|
||||||
|
return num_frames, avg_fps or 1.0
|
||||||
|
|
||||||
|
|
||||||
|
def _decode_video_frames_rgb(video_path: Path, frame_idx: list[int]) -> list[Any]:
|
||||||
|
"""Decode presentation-order ``frame_idx`` into RGB ``PIL.Image``s.
|
||||||
|
|
||||||
|
Mirrors ``decord.VideoReader.get_batch(frame_idx)``: one image per requested index,
|
||||||
|
in the requested order, duplicates repeated. Decoding is a single forward pass
|
||||||
|
(no seeking) so the mapping from index to picture matches decord's indexing.
|
||||||
|
"""
|
||||||
|
import av
|
||||||
|
|
||||||
|
wanted = sorted({int(i) for i in frame_idx})
|
||||||
|
if not wanted:
|
||||||
|
return []
|
||||||
|
|
||||||
|
decoded: dict[int, Any] = {}
|
||||||
|
with av.open(str(video_path)) as container:
|
||||||
|
stream = container.streams.video[0]
|
||||||
|
stream.thread_type = "AUTO"
|
||||||
|
cursor = 0
|
||||||
|
for position, frame in enumerate(container.decode(stream)):
|
||||||
|
if wanted[cursor] > position:
|
||||||
|
continue
|
||||||
|
image = frame.to_image()
|
||||||
|
# ``<=`` also absorbs indices that overshoot the real frame count.
|
||||||
|
while cursor < len(wanted) and wanted[cursor] <= position:
|
||||||
|
decoded[wanted[cursor]] = image
|
||||||
|
cursor += 1
|
||||||
|
if cursor >= len(wanted):
|
||||||
|
break
|
||||||
|
|
||||||
|
if not decoded:
|
||||||
|
raise ValueError(f"Decoded no frames from {video_path} for indices {wanted[:8]}")
|
||||||
|
# Metadata may promise more frames than the stream actually decodes; clamp to the last one.
|
||||||
|
last_image = decoded[max(decoded)]
|
||||||
|
return [decoded.get(int(i), last_image) for i in frame_idx]
|
||||||
|
|
||||||
|
|
||||||
def _numpy_to_wav_bytes(audio_np: Any, sample_rate: int = _MINICPM_AUDIO_SR) -> bytes:
|
def _numpy_to_wav_bytes(audio_np: Any, sample_rate: int = _MINICPM_AUDIO_SR) -> bytes:
|
||||||
"""Encode a mono float/int numpy waveform as 16-bit PCM WAV bytes."""
|
"""Encode a mono float/int numpy waveform as 16-bit PCM WAV bytes."""
|
||||||
import numpy as np
|
import numpy as np
|
||||||
@@ -1029,28 +1162,24 @@ class DailyOmniDataset(BenchmarkDataset):
|
|||||||
) -> tuple[list[Any], list[Any]]:
|
) -> tuple[list[Any], list[Any]]:
|
||||||
"""Port of MiniCPM ``get_video_frame_audio_segments`` (stack_frames=1, 1fps)."""
|
"""Port of MiniCPM ``get_video_frame_audio_segments`` (stack_frames=1, 1fps)."""
|
||||||
import numpy as np
|
import numpy as np
|
||||||
from decord import VideoReader, cpu
|
|
||||||
from PIL import Image
|
|
||||||
from vllm.multimodal.media.audio import load_audio
|
from vllm.multimodal.media.audio import load_audio
|
||||||
|
|
||||||
vr = VideoReader(str(video_path), ctx=cpu(0))
|
num_video_frames, avg_fps = _probe_video_frames_and_fps(video_path)
|
||||||
avg_fps = float(vr.get_avg_fps()) or 1.0
|
duration = num_video_frames / avg_fps
|
||||||
duration = len(vr) / avg_fps
|
|
||||||
num_seconds = max(1, int(math.ceil(duration)))
|
num_seconds = max(1, int(math.ceil(duration)))
|
||||||
second_timestamps = list(range(num_seconds))
|
second_timestamps = list(range(num_seconds))
|
||||||
|
|
||||||
if duration > max_num_frames:
|
if duration > max_num_frames:
|
||||||
timestamps = [round(i * 0.1, 1) for i in range(int(duration / 0.1))]
|
timestamps = [round(i * 0.1, 1) for i in range(int(duration / 0.1))]
|
||||||
frame_idx = [min(int(ts * avg_fps), len(vr) - 1) for ts in timestamps]
|
frame_idx = [min(int(ts * avg_fps), num_video_frames - 1) for ts in timestamps]
|
||||||
pick = _uniform_sample_indices(len(frame_idx), max_num_frames)
|
pick = _uniform_sample_indices(len(frame_idx), max_num_frames)
|
||||||
frame_idx = [frame_idx[i] for i in pick]
|
frame_idx = [frame_idx[i] for i in pick]
|
||||||
timestamps = [timestamps[i] for i in pick]
|
timestamps = [timestamps[i] for i in pick]
|
||||||
else:
|
else:
|
||||||
frame_idx = [min(int(i * avg_fps), len(vr) - 1) for i in range(num_seconds)]
|
frame_idx = [min(int(i * avg_fps), num_video_frames - 1) for i in range(num_seconds)]
|
||||||
timestamps = second_timestamps
|
timestamps = second_timestamps
|
||||||
|
|
||||||
video = vr.get_batch(frame_idx).asnumpy()
|
frames = _decode_video_frames_rgb(video_path, frame_idx)
|
||||||
frames = [Image.fromarray(v.astype("uint8")).convert("RGB") for v in video]
|
|
||||||
|
|
||||||
audio_segments: list[Any] = []
|
audio_segments: list[Any] = []
|
||||||
if include_audio:
|
if include_audio:
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ import wave
|
|||||||
from collections.abc import Iterable
|
from collections.abc import Iterable
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
|
from pathlib import Path
|
||||||
from typing import Any, Literal
|
from typing import Any, Literal
|
||||||
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
||||||
|
|
||||||
@@ -38,7 +39,13 @@ from vllm.logger import init_logger
|
|||||||
from vllm.tokenizers import TokenizerLike
|
from vllm.tokenizers import TokenizerLike
|
||||||
|
|
||||||
from vllm_omni.benchmarks.audio_continuity import compute_continuity_stats
|
from vllm_omni.benchmarks.audio_continuity import compute_continuity_stats
|
||||||
from vllm_omni.benchmarks.data_modules.daily_omni_dataset import DailyOmniDataset, DailyOmniSampleRequest
|
from vllm_omni.benchmarks.data_modules.daily_omni_dataset import (
|
||||||
|
DailyOmniDataset,
|
||||||
|
DailyOmniSampleRequest,
|
||||||
|
daily_omni_local_qa_json,
|
||||||
|
daily_omni_local_videos_dir,
|
||||||
|
resolve_daily_omni_local_root,
|
||||||
|
)
|
||||||
from vllm_omni.benchmarks.data_modules.random_multi_modal_dataset import OmniRandomMultiModalDataset
|
from vllm_omni.benchmarks.data_modules.random_multi_modal_dataset import OmniRandomMultiModalDataset
|
||||||
from vllm_omni.benchmarks.data_modules.seed_tts_dataset import (
|
from vllm_omni.benchmarks.data_modules.seed_tts_dataset import (
|
||||||
SEED_TTS_DEFAULT_OMNI_SYSTEM_PROMPT,
|
SEED_TTS_DEFAULT_OMNI_SYSTEM_PROMPT,
|
||||||
@@ -282,6 +289,26 @@ def get_samples(args, tokenizer):
|
|||||||
if isinstance(qa_json, str):
|
if isinstance(qa_json, str):
|
||||||
qa_json = qa_json.strip() or None
|
qa_json = qa_json.strip() or None
|
||||||
|
|
||||||
|
# A local mirror of the dataset repo (plain directory or HF hub cache dir) is used
|
||||||
|
# directly: no Hub round-trip, so offline / air-gapped runs work without extra flags.
|
||||||
|
local_root: Path | None = None
|
||||||
|
if qa_json is None:
|
||||||
|
local_root = resolve_daily_omni_local_root(getattr(args, "dataset_path", None)) or (
|
||||||
|
resolve_daily_omni_local_root(getattr(args, "hf_name", None))
|
||||||
|
)
|
||||||
|
if local_root is not None:
|
||||||
|
local_qa = daily_omni_local_qa_json(local_root)
|
||||||
|
if local_qa is not None:
|
||||||
|
qa_json = str(local_qa)
|
||||||
|
if video_dir is None:
|
||||||
|
video_dir = daily_omni_local_videos_dir(local_root)
|
||||||
|
logger.info("Using local Daily-Omni mirror: root=%s", local_root)
|
||||||
|
else:
|
||||||
|
logger.info(
|
||||||
|
"Local Daily-Omni path %s has no qa.json; loading it with `datasets` instead",
|
||||||
|
local_root,
|
||||||
|
)
|
||||||
|
|
||||||
if qa_json is not None:
|
if qa_json is not None:
|
||||||
logger.info(
|
logger.info(
|
||||||
"Loading Daily-Omni dataset: qa_json=%s, video_dir=%s (Hub not used for QA)",
|
"Loading Daily-Omni dataset: qa_json=%s, video_dir=%s (Hub not used for QA)",
|
||||||
@@ -304,7 +331,8 @@ def get_samples(args, tokenizer):
|
|||||||
repo_id = _daily_omni_repo_from_args(args)
|
repo_id = _daily_omni_repo_from_args(args)
|
||||||
if args.dataset_name == "daily-omni":
|
if args.dataset_name == "daily-omni":
|
||||||
if repo_id is None:
|
if repo_id is None:
|
||||||
repo_id = _DEFAULT_DAILY_OMNI_REPO
|
# Prefer an on-disk copy over the Hub id so offline runs keep working.
|
||||||
|
repo_id = str(local_root) if local_root is not None else _DEFAULT_DAILY_OMNI_REPO
|
||||||
elif repo_id is None:
|
elif repo_id is None:
|
||||||
raise ValueError(
|
raise ValueError(
|
||||||
"Daily-Omni with --dataset-name hf requires "
|
"Daily-Omni with --dataset-name hf requires "
|
||||||
@@ -700,6 +728,29 @@ def _image_generation_ms_from_content(content: Any) -> float:
|
|||||||
return 0.0
|
return 0.0
|
||||||
|
|
||||||
|
|
||||||
|
#: Non-200 responses are otherwise silent (only ``reason`` was kept), which turns a fully failed
|
||||||
|
#: run into "Successful requests: 0" with no cause. Log the first few bodies, keep all of them on
|
||||||
|
#: ``output.error`` for ``--save-detailed``.
|
||||||
|
_HTTP_ERROR_LOG_BUDGET = 5
|
||||||
|
_http_error_logged = 0
|
||||||
|
|
||||||
|
|
||||||
|
async def _record_http_error(output: "MixRequestFuncOutput", response: aiohttp.ClientResponse, context: str) -> None:
|
||||||
|
"""Store status + response body on ``output.error`` and log the first few occurrences."""
|
||||||
|
global _http_error_logged
|
||||||
|
|
||||||
|
try:
|
||||||
|
body = (await response.text())[:512]
|
||||||
|
except Exception:
|
||||||
|
body = ""
|
||||||
|
detail = f"HTTP {response.status} {response.reason or ''}".strip()
|
||||||
|
output.error = f"{detail}: {body}" if body else detail
|
||||||
|
output.success = False
|
||||||
|
if _http_error_logged < _HTTP_ERROR_LOG_BUDGET:
|
||||||
|
_http_error_logged += 1
|
||||||
|
logger.warning("%s request failed: %s", context, output.error)
|
||||||
|
|
||||||
|
|
||||||
async def async_request_openai_chat_omni_completions(
|
async def async_request_openai_chat_omni_completions(
|
||||||
request_func_input: RequestFuncInput,
|
request_func_input: RequestFuncInput,
|
||||||
session: aiohttp.ClientSession,
|
session: aiohttp.ClientSession,
|
||||||
@@ -997,8 +1048,7 @@ async def async_request_openai_chat_omni_completions(
|
|||||||
logger.warning("seed_tts WER PCM export failed: %s", ex)
|
logger.warning("seed_tts WER PCM export failed: %s", ex)
|
||||||
output.success = True
|
output.success = True
|
||||||
else:
|
else:
|
||||||
output.error = response.reason or ""
|
await _record_http_error(output, response, "openai-chat-omni")
|
||||||
output.success = False
|
|
||||||
break
|
break
|
||||||
except aiohttp.ClientError as e:
|
except aiohttp.ClientError as e:
|
||||||
# transient transport error: may retry
|
# transient transport error: may retry
|
||||||
@@ -1272,8 +1322,7 @@ async def async_request_openai_audio_speech(
|
|||||||
)
|
)
|
||||||
output.success = True
|
output.success = True
|
||||||
else:
|
else:
|
||||||
output.error = response.reason or ""
|
await _record_http_error(output, response, "openai-audio-speech")
|
||||||
output.success = False
|
|
||||||
except Exception:
|
except Exception:
|
||||||
output.success = False
|
output.success = False
|
||||||
output.error = traceback.format_exc()
|
output.error = traceback.format_exc()
|
||||||
|
|||||||
Reference in new issue
Block a user