5d28b6a40c
Signed-off-by: Sy03 <1370724210@qq.com> Signed-off-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com> Signed-off-by: linyueqian <linyueqian@outlook.com> Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com> Co-authored-by: linyueqian <linyueqian@outlook.com>
121 lines
3.9 KiB
Python
121 lines
3.9 KiB
Python
"""Multi-language sentence boundary detector for streaming TTS input.
|
||
|
||
Buffers incoming text and splits at sentence boundaries (English and CJK),
|
||
yielding complete sentences for audio generation.
|
||
"""
|
||
|
||
import re
|
||
from re import Pattern
|
||
|
||
# Maximum buffer size (in characters) to prevent unbounded memory growth.
|
||
_MAX_BUFFER_SIZE = 100_000 # ~100 KB of text
|
||
|
||
# Sentence-level: .!? + CJK sentence-ending 。！？
|
||
# NOTE: English requires trailing whitespace to confirm a boundary —
|
||
# end-of-string is NOT treated as a boundary (that is what flush() is for).
|
||
SPLIT_SENTENCE = re.compile(
|
||
r"(?<=[.!?])\s+"
|
||
r"|(?<=[。！？])"
|
||
)
|
||
|
||
# Clause-level: adds CJK commas ， and semicolons ；
|
||
SPLIT_CLAUSE = re.compile(
|
||
r"(?<=[.!?])\s+"
|
||
r"|(?<=[。！？，；])"
|
||
)
|
||
|
||
# Default alias
|
||
_SENTENCE_BOUNDARY_RE = SPLIT_SENTENCE
|
||
|
||
|
||
class SentenceSplitter:
|
||
"""Incremental sentence splitter for streaming text input.
|
||
|
||
Buffers text and yields complete sentences when boundaries are detected.
|
||
Designed for TTS pipelines where text arrives incrementally (e.g., from STT).
|
||
|
||
Args:
|
||
min_sentence_length: Minimum character length for a sentence.
|
||
Sentences shorter than this are kept in the buffer to avoid
|
||
splitting on abbreviations like "Dr." or "U.S.".
|
||
boundary_re: Custom compiled regex for sentence boundaries.
|
||
Use ``SPLIT_SENTENCE`` (default) for sentence-level splitting,
|
||
``SPLIT_CLAUSE`` for finer-grained clause-level splitting,
|
||
or pass your own ``re.Pattern``.
|
||
"""
|
||
|
||
def __init__(
|
||
self,
|
||
min_sentence_length: int = 2,
|
||
boundary_re: Pattern[str] | None = None,
|
||
) -> None:
|
||
self._buffer: str = ""
|
||
self._min_sentence_length = min_sentence_length
|
||
self._boundary_re = boundary_re or _SENTENCE_BOUNDARY_RE
|
||
|
||
@property
|
||
def buffer(self) -> str:
|
||
"""Current buffered text."""
|
||
return self._buffer
|
||
|
||
def add_text(self, text: str) -> list[str]:
|
||
"""Add text to the buffer and return any complete sentences.
|
||
|
||
Args:
|
||
text: Incoming text chunk.
|
||
|
||
Returns:
|
||
List of complete sentences extracted from the buffer.
|
||
May be empty if no sentence boundary was found.
|
||
|
||
Raises:
|
||
ValueError: If the buffer exceeds the maximum size.
|
||
"""
|
||
if not text:
|
||
return []
|
||
|
||
self._buffer += text
|
||
if len(self._buffer) > _MAX_BUFFER_SIZE:
|
||
raise ValueError(
|
||
f"Text buffer exceeded maximum size ({_MAX_BUFFER_SIZE} chars). "
|
||
"Consider adding sentence-ending punctuation to your input."
|
||
)
|
||
return self._extract_sentences()
|
||
|
||
def flush(self) -> str | None:
|
||
"""Flush remaining buffered text as a final sentence.
|
||
|
||
Returns:
|
||
The remaining buffered text (stripped), or None if buffer is empty.
|
||
"""
|
||
remaining = self._buffer.strip()
|
||
self._buffer = ""
|
||
return remaining if remaining else None
|
||
|
||
def _extract_sentences(self) -> list[str]:
|
||
"""Split buffer at sentence boundaries, keeping incomplete text buffered."""
|
||
parts = self._boundary_re.split(self._buffer)
|
||
|
||
if len(parts) <= 1:
|
||
# No boundary found — keep everything in buffer
|
||
return []
|
||
|
||
sentences: list[str] = []
|
||
carry = ""
|
||
# All parts except the last are complete sentences
|
||
for i in range(len(parts) - 1):
|
||
text = carry + parts[i]
|
||
carry = ""
|
||
stripped = text.strip()
|
||
if len(stripped) >= self._min_sentence_length:
|
||
sentences.append(stripped)
|
||
elif stripped:
|
||
# Too short (e.g. "Dr.") — carry forward to next part
|
||
carry = text
|
||
# else: empty, skip
|
||
|
||
# Last part stays in buffer (may be incomplete)
|
||
self._buffer = carry + parts[-1]
|
||
|
||
return sentences
|