[BugFix] Fix local media access for diffusion speech (#6622)

Signed-off-by: zhuhh97 <zhuhonghan@huawei.com>
This commit is contained in:
Honghan Zhu authored and GitHub committed 2026-08-29 02:01:24 +00:00
1 parent 034af1f657
commit f584729f24
4 files changed
+121 -15

No files matched your search

@@ -739,7 +739,11 @@ async def test_pure_diffusion_app_state_key_snapshot(monkeypatch) -> None:
)
state = State()
await api_server.omni_init_app_state(engine, state, _minimal_args())
await api_server.omni_init_app_state(
engine,
state,
_minimal_args(),
)
_assert_app_state_snapshot(
state,
@@ -750,6 +754,55 @@ async def test_pure_diffusion_app_state_key_snapshot(monkeypatch) -> None:
assert state.diffusion_engine is engine
@pytest.mark.asyncio
async def test_pure_diffusion_speech_forwards_media_access_args(monkeypatch) -> None:
stage = SimpleNamespace(engine_args={})
engine = _FakeEngineClient(stage_configs=[stage])
speech_kwargs = {}
def _for_diffusion_factory(label: str):
@classmethod
def _factory(cls, *args, **kwargs):
return _marker(label)
return _factory
@classmethod
def _speech_factory(cls, *args, **kwargs):
speech_kwargs.update(kwargs)
return _marker("speech")
monkeypatch.setattr(api_server, "get_stage_type", lambda _cfg: "diffusion")
monkeypatch.setattr(api_server.OmniOpenAIServingChat, "for_diffusion", _for_diffusion_factory("chat"))
monkeypatch.setattr(api_server.OmniOpenAIServingChatBatch, "for_diffusion", _for_diffusion_factory("chat_batch"))
monkeypatch.setattr(
api_server.OmniOpenAIServingAudioGenerate,
"for_diffusion",
_for_diffusion_factory("audio_generate"),
)
monkeypatch.setattr(api_server.OmniOpenAIServingVideo, "for_diffusion", _for_diffusion_factory("video"))
monkeypatch.setattr(api_server.OmniStreamingVideoOutputHandler, "__init__", lambda self, *a, **k: None)
monkeypatch.setattr(api_server.OmniOpenAIServingSpeech, "for_diffusion", _speech_factory)
monkeypatch.setattr(
api_server.ServingRealtimeRobotOpenPI,
"create_policy_server",
classmethod(lambda cls, *a, **k: _marker("openpi")),
)
state = State()
await api_server.omni_init_app_state(
engine,
state,
_minimal_args(
allowed_local_media_path="/allowed/media",
allowed_media_domains=["media.example.com"],
),
)
assert speech_kwargs["allowed_local_media_path"] == "/allowed/media"
assert speech_kwargs["allowed_media_domains"] == ["media.example.com"]
@pytest.mark.asyncio
async def test_multistage_app_state_key_snapshot(monkeypatch) -> None:
"""Lock multi-stage ``app.state`` keys after init, including live vs None.
@@ -1245,6 +1245,46 @@ class TestTTSMethods:
cache_key
) == speech_server._make_ref_audio_artifact_cache_key(np.asarray(first[0], dtype=np.float32), 24000)
@pytest.mark.asyncio
async def test_diffusion_ref_audio_uses_media_access_config(
self,
mocker: MockerFixture,
monkeypatch,
tmp_path,
):
monkeypatch.setenv("SPEAKER_SAMPLES_DIR", str(tmp_path / "speakers"))
connector = mocker.MagicMock()
connector.fetch_audio_async = mocker.AsyncMock(return_value=(np.zeros(24000, dtype=np.float32), 24000))
connector_cls = mocker.patch.object(
serving_speech_module,
"MediaConnector",
return_value=connector,
)
media_dir = tmp_path / "media"
media_dir.mkdir()
ref_audio = media_dir / "reference.wav"
ref_audio.write_bytes(b"v1")
server = OmniOpenAIServingSpeech.for_diffusion(
diffusion_engine=mocker.MagicMock(),
model_name="test-model",
allowed_local_media_path=str(media_dir),
allowed_media_domains=["media.example.com"],
)
try:
first = await server._resolve_ref_audio(ref_audio.as_uri())
ref_audio.write_bytes(b"v2-with-different-size")
second = await server._resolve_ref_audio(ref_audio.as_uri())
finally:
server.shutdown()
connector_cls.assert_called_once_with(
allowed_local_media_path=str(media_dir),
allowed_media_domains=["media.example.com"],
)
assert first[2] != second[2]
assert connector.fetch_audio_async.await_count == 2
# ── ref-audio cache key tests (static helper) ──
def test_local_file_cache_key_invalidation(self):
@@ -825,6 +825,8 @@ async def omni_init_app_state(
diffusion_engine=engine_client,
model_name=model_name,
stage_configs=diffusion_stage_configs,
allowed_local_media_path=getattr(args, "allowed_local_media_path", ""),
allowed_media_domains=getattr(args, "allowed_media_domains", None),
)
state.openai_serving_duplex = None
state.openai_streaming_speech = None
+25 -14
View File
@@ -1,3 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import asyncio
import base64
import hashlib
@@ -275,6 +278,8 @@ def _conditioning_cache_salt(request, tts_params: dict | None = None) -> str:
class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
_diffusion_mode: bool = False
_media_connector: MediaConnector | None = None
_allowed_local_media_path: str = ""
_tts_executor: ThreadPoolExecutor | None = None
def _init_speaker_storage(self) -> None:
@@ -391,6 +396,8 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
diffusion_engine: "Any",
model_name: str,
stage_configs: "list[Any] | None" = None,
allowed_local_media_path: str = "",
allowed_media_domains: list[str] | None = None,
) -> "OmniOpenAIServingSpeech":
"""Create a speech serving instance for pure diffusion TTS models.
@@ -402,6 +409,11 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
instance._diffusion_engine = diffusion_engine
instance._diffusion_model_name = model_name
instance._diffusion_stage_configs = stage_configs
instance._allowed_local_media_path = allowed_local_media_path
instance._media_connector = MediaConnector(
allowed_local_media_path=allowed_local_media_path,
allowed_media_domains=allowed_media_domains,
)
instance._tts_model_type = "omnivoice"
instance._is_tts = False
# Diffusion-only instances don't have a TTS stage; set None so any
@@ -412,6 +424,8 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
return instance
def __init__(self, *args, **kwargs):
self._media_connector = None
self._allowed_local_media_path = ""
self.model_name = kwargs.pop("model_name", None)
# True when the server was launched with --forced-aligner (a pooling
# aligner stage is appended to the pipeline). Gates word_timestamps.
@@ -1860,15 +1874,16 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
"""
# Pass the allowed-local-media-path so the stat is restricted to
# paths the server operator has explicitly permitted.
allowed_path = None
if not self._diffusion_mode:
allowed_path: str | None
if self._diffusion_mode:
allowed_path = self._allowed_local_media_path
else:
allowed_path = getattr(self.model_config, "allowed_local_media_path", None)
wav_list: list[float] | None = None
sr = 0
artifact_key = ""
post_key = ""
connector: MediaConnector | None = None
for attempt in range(_REF_AUDIO_METADATA_FETCH_ATTEMPTS):
cache_key = await self._ref_audio_cache_key(ref_audio_str, allowed_path)
cached = self._ref_audio_resolve_cache.get(cache_key)
@@ -1883,19 +1898,15 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
)
return wav_list, sr, cache_key
if connector is None:
# In diffusion mode, model_config may not be available
if self._diffusion_mode:
connector = MediaConnector()
else:
model_config = self.model_config
connector = MediaConnector(
allowed_local_media_path=model_config.allowed_local_media_path,
allowed_media_domains=model_config.allowed_media_domains,
)
if self._media_connector is None:
model_config = self.model_config
self._media_connector = MediaConnector(
allowed_local_media_path=model_config.allowed_local_media_path,
allowed_media_domains=model_config.allowed_media_domains,
)
fetch_start_s = time.perf_counter()
wav_np, fetched_sr = await connector.fetch_audio_async(ref_audio_str)
wav_np, fetched_sr = await self._media_connector.fetch_audio_async(ref_audio_str)
fetch_decode_ms = (time.perf_counter() - fetch_start_s) * 1000.0
tolist_start_s = time.perf_counter()
wav_list, sr, artifact_key, duration = self._finalize_fetched_ref_audio(wav_np, fetched_sr)