[Bugfix]: modify diffusion pipeline profiler result in videos (#2647)
Signed-off-by: bjf-frz <frz123db@gmail.com>
This commit is contained in:
1 parent
c2ae58bb84
commit
fbb5dd5794
6 files changed
+158
-30
No files matched your search
@@ -2009,6 +2009,8 @@ async def _run_video_generation_job(
|
||||
"file_name": file_name,
|
||||
"completed_at": int(time.time()),
|
||||
"inference_time_s": time.perf_counter() - started_at,
|
||||
"stage_durations": response.stage_durations,
|
||||
"peak_memory_mb": response.peak_memory_mb,
|
||||
},
|
||||
)
|
||||
except Exception as exc:
|
||||
@@ -2182,7 +2184,7 @@ async def create_video_sync(
|
||||
request_id = f"video_sync-{random_uuid()}"
|
||||
started_at = time.perf_counter()
|
||||
try:
|
||||
video_bytes = await asyncio.wait_for(
|
||||
video_bytes, stage_durations, peak_memory_mb = await asyncio.wait_for(
|
||||
handler.generate_video_bytes(request, request_id, reference_image=reference_image),
|
||||
timeout=VIDEO_SYNC_TIMEOUT_S,
|
||||
)
|
||||
@@ -2208,6 +2210,8 @@ async def create_video_sync(
|
||||
"X-Request-Id": request_id,
|
||||
"X-Model": effective_model_name,
|
||||
"X-Inference-Time-S": f"{inference_time_s:.3f}",
|
||||
"X-Stage-Durations": json.dumps(stage_durations, separators=(",", ":")),
|
||||
"X-Peak-Memory-MB": f"{peak_memory_mb:.3f}",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@@ -201,6 +201,14 @@ class VideoGenerationResponse(BaseModel):
|
||||
|
||||
created: int = Field(..., description="Unix timestamp of when the generation completed")
|
||||
data: list[VideoData] = Field(..., description="Array of generated videos")
|
||||
stage_durations: dict[str, float] = Field(
|
||||
default_factory=dict,
|
||||
description="Profiler stage durations reported by the diffusion pipeline.",
|
||||
)
|
||||
peak_memory_mb: float = Field(
|
||||
default=0.0,
|
||||
description="Peak device memory usage in MB reported by the diffusion pipeline.",
|
||||
)
|
||||
|
||||
|
||||
class VideoError(BaseModel):
|
||||
@@ -250,6 +258,14 @@ class VideoResponse(BaseModel):
|
||||
description="Filename of the saved output video files for this job.",
|
||||
)
|
||||
inference_time_s: float | None = Field(default=None, description="End-to-end inference time in seconds.")
|
||||
stage_durations: dict[str, float] = Field(
|
||||
default_factory=dict,
|
||||
description="Profiler stage durations reported by the diffusion pipeline.",
|
||||
)
|
||||
peak_memory_mb: float = Field(
|
||||
default=0.0,
|
||||
description="Peak device memory usage in MB reported by the diffusion pipeline.",
|
||||
)
|
||||
|
||||
@property
|
||||
def file_extension(self) -> str:
|
||||
|
||||
@@ -33,6 +33,18 @@ class ReferenceImage:
|
||||
data: Image.Image
|
||||
|
||||
|
||||
@dataclass
|
||||
class VideoGenerationArtifacts:
|
||||
"""Normalized outputs and profiler metadata extracted from one request."""
|
||||
|
||||
videos: list[Any]
|
||||
audios: list[Any | None]
|
||||
audio_sample_rate: int
|
||||
output_fps: int
|
||||
stage_durations: dict[str, float]
|
||||
peak_memory_mb: float
|
||||
|
||||
|
||||
class OmniOpenAIServingVideo:
|
||||
"""OpenAI-style video generation handler for omni diffusion models."""
|
||||
|
||||
@@ -77,12 +89,8 @@ class OmniOpenAIServingVideo:
|
||||
reference_id: str,
|
||||
*,
|
||||
reference_image: ReferenceImage | None = None,
|
||||
) -> tuple[list[Any], list[Any | None], int, int]:
|
||||
"""Run the generation pipeline and extract video/audio outputs.
|
||||
|
||||
Returns:
|
||||
Tuple of (videos, audios, audio_sample_rate, output_fps).
|
||||
"""
|
||||
) -> VideoGenerationArtifacts:
|
||||
"""Run the generation pipeline and extract video/audio/profiler outputs."""
|
||||
prompt: OmniTextPrompt = OmniTextPrompt(prompt=request.prompt)
|
||||
if request.negative_prompt is not None:
|
||||
prompt["negative_prompt"] = request.negative_prompt
|
||||
@@ -153,7 +161,14 @@ class OmniOpenAIServingVideo:
|
||||
audios = self._extract_audio_outputs(result, expected_count=len(videos))
|
||||
audio_sample_rate = self._resolve_audio_sample_rate(result)
|
||||
output_fps = vp.fps or self._resolve_fps(result) or 24
|
||||
return videos, audios, audio_sample_rate, output_fps
|
||||
return VideoGenerationArtifacts(
|
||||
videos=videos,
|
||||
audios=audios,
|
||||
audio_sample_rate=audio_sample_rate,
|
||||
output_fps=output_fps,
|
||||
stage_durations=self._extract_stage_durations(result),
|
||||
peak_memory_mb=self._extract_peak_memory_mb(result),
|
||||
)
|
||||
|
||||
async def generate_videos(
|
||||
self,
|
||||
@@ -162,28 +177,31 @@ class OmniOpenAIServingVideo:
|
||||
*,
|
||||
reference_image: ReferenceImage | None = None,
|
||||
) -> VideoGenerationResponse:
|
||||
videos, audios, audio_sample_rate, output_fps = await self._run_and_extract(
|
||||
request, reference_id, reference_image=reference_image
|
||||
)
|
||||
artifacts = await self._run_and_extract(request, reference_id, reference_image=reference_image)
|
||||
_t_encode_start = time.perf_counter()
|
||||
video_data = [
|
||||
VideoData(
|
||||
b64_json=(
|
||||
encode_video_base64(video, fps=output_fps)
|
||||
if audios[idx] is None
|
||||
encode_video_base64(video, fps=artifacts.output_fps)
|
||||
if artifacts.audios[idx] is None
|
||||
else encode_video_base64(
|
||||
video,
|
||||
fps=output_fps,
|
||||
audio=audios[idx],
|
||||
audio_sample_rate=audio_sample_rate,
|
||||
fps=artifacts.output_fps,
|
||||
audio=artifacts.audios[idx],
|
||||
audio_sample_rate=artifacts.audio_sample_rate,
|
||||
)
|
||||
)
|
||||
)
|
||||
for idx, video in enumerate(videos)
|
||||
for idx, video in enumerate(artifacts.videos)
|
||||
]
|
||||
_t_encode_ms = (time.perf_counter() - _t_encode_start) * 1000
|
||||
logger.info("Video response encoding (MP4+base64): %.2f ms", _t_encode_ms)
|
||||
return VideoGenerationResponse(created=int(time.time()), data=video_data)
|
||||
return VideoGenerationResponse(
|
||||
created=int(time.time()),
|
||||
data=video_data,
|
||||
stage_durations=artifacts.stage_durations,
|
||||
peak_memory_mb=artifacts.peak_memory_mb,
|
||||
)
|
||||
|
||||
async def generate_video_bytes(
|
||||
self,
|
||||
@@ -191,25 +209,25 @@ class OmniOpenAIServingVideo:
|
||||
reference_id: str,
|
||||
*,
|
||||
reference_image: ReferenceImage | None = None,
|
||||
) -> bytes:
|
||||
) -> tuple[bytes, dict[str, float], float]:
|
||||
"""Generate a video and return raw MP4 bytes, bypassing base64 encoding."""
|
||||
videos, audios, audio_sample_rate, output_fps = await self._run_and_extract(
|
||||
request, reference_id, reference_image=reference_image
|
||||
)
|
||||
if len(videos) > 1:
|
||||
artifacts = await self._run_and_extract(request, reference_id, reference_image=reference_image)
|
||||
if len(artifacts.videos) > 1:
|
||||
logger.warning(
|
||||
"Video request %s generated %d outputs; returning only the first.", reference_id, len(videos)
|
||||
"Video request %s generated %d outputs; returning only the first.",
|
||||
reference_id,
|
||||
len(artifacts.videos),
|
||||
)
|
||||
audio = audios[0]
|
||||
audio = artifacts.audios[0]
|
||||
_t_encode_start = time.perf_counter()
|
||||
video_bytes = _encode_video_bytes(
|
||||
videos[0],
|
||||
fps=output_fps,
|
||||
**({"audio": audio, "audio_sample_rate": audio_sample_rate} if audio is not None else {}),
|
||||
artifacts.videos[0],
|
||||
fps=artifacts.output_fps,
|
||||
**({"audio": audio, "audio_sample_rate": artifacts.audio_sample_rate} if audio is not None else {}),
|
||||
)
|
||||
_t_encode_ms = (time.perf_counter() - _t_encode_start) * 1000
|
||||
logger.info("Video response encoding (MP4 bytes): %.2f ms", _t_encode_ms)
|
||||
return video_bytes
|
||||
return video_bytes, artifacts.stage_durations, artifacts.peak_memory_mb
|
||||
|
||||
@staticmethod
|
||||
def _apply_lora(lora_body: Any, gen_params: OmniDiffusionSamplingParams) -> None:
|
||||
@@ -483,3 +501,16 @@ class OmniOpenAIServingVideo:
|
||||
return None
|
||||
|
||||
return sample_rate if sample_rate > 0 else None
|
||||
|
||||
@staticmethod
|
||||
def _extract_stage_durations(result: Any) -> dict[str, float]:
|
||||
stage_durations = getattr(result, "stage_durations", None)
|
||||
return stage_durations if isinstance(stage_durations, dict) else {}
|
||||
|
||||
@staticmethod
|
||||
def _extract_peak_memory_mb(result: Any) -> float:
|
||||
peak_memory_mb = getattr(result, "peak_memory_mb", 0.0)
|
||||
try:
|
||||
return float(peak_memory_mb or 0.0)
|
||||
except (TypeError, ValueError):
|
||||
return 0.0
|
||||
Reference in new issue
Block a user