vllm-omni/vllm_omni/utils/forced_aligner.py

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Forced-aligner config + word-timestamp decoding for TTS.

The aligner runs as a pooling stage appended to the pipeline by
``--forced-aligner`` (see :func:`inject_forced_aligner_stage`). Qwen-specific
word segmentation / prompt building / marker repair live in
:mod:`vllm_omni.utils.qwen3_force_align_processor`.
"""

from __future__ import annotations

import logging
from dataclasses import dataclass, replace
from pathlib import Path
from typing import TYPE_CHECKING, Any

import numpy as np
import yaml

from vllm_omni.utils import qwen3_force_align_processor as _processor

if TYPE_CHECKING:
    from vllm_omni.config.stage_config import DeployConfig, PipelineConfig

logger = logging.getLogger(__name__)

_DEFAULT_CONFIG_PATH = Path(__file__).resolve().parents[1] / "deploy" / "qwen3_tts_forced_aligner.yaml"


@dataclass(frozen=True, slots=True)
class WordTimestamp:
    """Internal alignment record. Serialized to a plain JSON object
    (``{"word", "start_ms", "end_ms"}``) at the WebSocket boundary.
    """

    word: str
    start_ms: int
    end_ms: int


@dataclass(frozen=True, slots=True)
class ForcedAlignerConfig:
    """Plain-data config for the forced-aligner pipeline stage; fields are
    consumed by :func:`inject_forced_aligner_stage`."""

    model: str
    runner: str | None = None
    architecture: str | None = None
    pooling_task: str | None = None
    gpu_memory_utilization: float | None = None
    dtype: str | None = None
    max_model_len: int | None = None
    trust_remote_code: bool | None = None


def _load_forced_aligner_yaml(path: str | Path) -> dict[str, Any]:
    with Path(path).open(encoding="utf-8") as f:
        raw = yaml.safe_load(f) or {}
    return dict(raw.get("forced_aligner", raw))


def build_forced_aligner_config(model: str | None, config_path: str | None = None) -> ForcedAlignerConfig | None:
    """Build a config from the CLI values, or ``None`` when the feature is off.

    Precedence (lowest to highest): the packaged default YAML
    (:data:`_DEFAULT_CONFIG_PATH`, Qwen deploy defaults) -> a user YAML passed
    via ``--forced-aligner-config`` -> the ``--forced-aligner`` model path. The
    feature is off (returns ``None``) unless a model resolves from this chain.
    Per-field overrides such as ``gpu_memory_utilization`` live in the YAML.
    """
    config_data: dict[str, Any] = {}
    if config_path or model:
        config_data.update(_load_forced_aligner_yaml(_DEFAULT_CONFIG_PATH))
    if config_path:
        config_data.update(_load_forced_aligner_yaml(config_path))
    if model:
        config_data["model"] = str(model)
    if not config_data.get("model"):
        return None
    allowed = set(ForcedAlignerConfig.__dataclass_fields__)
    return ForcedAlignerConfig(**{k: v for k, v in config_data.items() if k in allowed})


def inject_forced_aligner_stage(
    pipeline: PipelineConfig,
    deploy: DeployConfig,
    cli_overrides: dict[str, Any],
) -> tuple[PipelineConfig, DeployConfig]:
    """Append a forced-aligner pooling stage to the pipeline tail when
    ``--forced-aligner`` or ``--forced-aligner-config`` resolves an aligner
    model; no-op otherwise. The stage runs the aligner model with
    ``runner="pooling"``, consuming the previous stage's audio and emitting a
    terminal word-timestamps side-output.
    """
    if not (cli_overrides.get("forced_aligner") or cli_overrides.get("forced_aligner_config")):
        return pipeline, deploy

    from vllm_omni.config.stage_config import (
        StageDeployConfig,
        StageExecutionType,
        StagePipelineConfig,
    )
    from vllm_omni.model_executor.stage_input_processors.forced_aligner import (
        POOLING_OUTPUT_DECODER_PATH,
        TIMESTAMPS_MODALITY,
    )

    fa = build_forced_aligner_config(
        cli_overrides.get("forced_aligner"),
        cli_overrides.get("forced_aligner_config"),
    )
    if fa is None:
        return pipeline, deploy

    _FA = "vllm_omni.model_executor.stage_input_processors.forced_aligner"
    new_id = len(pipeline.stages)
    aligner_ps = StagePipelineConfig(
        stage_id=new_id,
        model_stage="forced_aligner",
        # Pooling stage = LLM_AR stage run with runner="pooling" (vLLM derives
        # is_pooling_model from runner_type).
        execution_type=StageExecutionType.LLM_AR,
        input_sources=(new_id - 1,),
        final_output=True,
        final_output_type=TIMESTAMPS_MODALITY,
        owns_tokenizer=True,
        requires_multimodal_data=True,
        model_arch=fa.architecture,
        custom_process_input_func=f"{_FA}.code2wav2aligner",
    )
    extended = replace(pipeline, stages=pipeline.stages + (aligner_ps,))

    engine_extras: dict[str, Any] = {
        "model": fa.model,
        "runner": fa.runner or "pooling",
        # Worker-side decoder hook: pooler logits -> word-timestamps tensor.
        "pooling_output_decoder": POOLING_OUTPUT_DECODER_PATH,
    }
    if fa.architecture:
        engine_extras["hf_overrides"] = {"architectures": [fa.architecture]}
    if fa.trust_remote_code is not None:
        engine_extras["trust_remote_code"] = fa.trust_remote_code
    if fa.dtype is not None:
        engine_extras["dtype"] = fa.dtype
    aligner_ds = StageDeployConfig(
        stage_id=new_id,
        devices=cli_overrides.get("forced_aligner_device"),
        gpu_memory_utilization=fa.gpu_memory_utilization,
        max_model_len=fa.max_model_len,
        engine_extras=engine_extras,
        default_pooling_params={"task": fa.pooling_task or "token_classify"},
        # Pooling is one-shot -> sync AR scheduler (the async one assumes
        # token-by-token generation).
        async_scheduling=False,
    )
    deploy = replace(deploy, stages=list(deploy.stages) + [aligner_ds])
    logger.info("[stage_init] Injected forced-aligner stage %d (model=%s)", new_id, fa.model)
    return extended, deploy


def _decode_timestamps(
    *,
    logits: Any,
    words: list[str],
    timestamp_positions: list[int],
    classify_num: int,
    audio_duration_ms: float,
    timestamp_segment_time_ms: float | None = None,
) -> list[WordTimestamp]:
    """Translate ``[n_token, classify_num]`` logits into word timestamps.

    ``words`` must be the exact segmentation used to build the prompt (see
    :func:`qwen3_force_align_processor.segment_words`); each word owns two
    consecutive markers.
    """
    arr = logits.detach().cpu().numpy() if hasattr(logits, "detach") else np.asarray(logits)
    if arr.ndim != 2:
        raise ValueError(f"Expected 2D logits [n_token, classify_num]; got shape {arr.shape}")
    if arr.shape[1] != classify_num:
        raise ValueError(
            f"Logits last dim {arr.shape[1]} != classify_num {classify_num}; "
            "model config and prompt template may be out of sync."
        )

    expected = len(words) * 2
    if len(timestamp_positions) != expected:
        logger.warning(
            "Got %d timestamp positions but text has %d words (expected %d start/end markers); "
            "returning empty alignment.",
            len(timestamp_positions),
            len(words),
            expected,
        )
        return []

    marker_logits = arr[timestamp_positions, :]
    bin_idx = marker_logits.argmax(axis=-1)
    bin_size_ms = (
        float(timestamp_segment_time_ms)
        if timestamp_segment_time_ms is not None
        else (audio_duration_ms / classify_num if classify_num > 0 else 0.0)
    )

    # Repair non-monotonic bins across the whole start/end sequence before
    # pairing, so each (start, end) pair stays ordered and one bad bin can't
    # corrupt its neighbours.
    marker_ms = _processor.fix_timestamp([int(round(int(b) * bin_size_ms)) for b in bin_idx])

    # Pair markers into per-word intervals, enforcing the wire contract here so
    # callers don't have to: monotonic and non-overlapping across words, with
    # start <= end and nothing past the audio length.
    duration_ms = int(round(audio_duration_ms))
    out: list[WordTimestamp] = []
    prev_end_ms = 0
    for i, word in enumerate(words):
        start_ms = min(max(marker_ms[i * 2], prev_end_ms), duration_ms)
        end_ms = min(max(marker_ms[i * 2 + 1], start_ms), duration_ms)
        out.append(WordTimestamp(word=word, start_ms=start_ms, end_ms=end_ms))
        prev_end_ms = end_ms
    return out


def extract_word_timestamps(res: Any, text: str, language: str | None = None) -> list[dict] | None:
    """Read a forced-aligner stage's engine output into per-word timestamps.

    Returns ``[{word, start_ms, end_ms}, ...]`` from the stage's pooling output
    (``res.outputs.data`` is an int32 ``[n_words, 2]`` tensor). Words come from
    re-segmenting ``text``; any mismatch or non-aligner input returns ``None``.
    """
    if not text:
        return None
    try:
        outs = getattr(res, "outputs", None)
        if outs is None:
            return None
        # The aligner stage's res.outputs is a single PoolingOutput (not a list).
        # Its .data is a [n_words, 2] int32 tensor of [start_ms, end_ms] per word.
        out0 = outs[0] if isinstance(outs, (list, tuple)) else outs
        ms = getattr(out0, "data", None)
        if ms is None or not hasattr(ms, "tolist"):
            return None
        ms_list = ms.tolist()
        if not ms_list or not isinstance(ms_list[0], (list, tuple)):
            return None

        words = _processor.segment_words(text, language)
        if len(words) != len(ms_list):
            # Segmentation drifted from the engine side; pairing positionally would mislabel every word.
            logger.warning(
                "word/timestamp count mismatch (%d words vs %d intervals); returning no timestamps",
                len(words),
                len(ms_list),
            )
            return None

        return [{"word": w, "start_ms": int(p[0]), "end_ms": int(p[1])} for w, p in zip(words, ms_list)]
    except Exception:
        logger.debug("failed to extract word timestamps from aligner output", exc_info=True)
        return None