From f99e8b2d3ce00bf2c45358a29563953d1cecd317 Mon Sep 17 00:00:00 2001 From: Yueqian Lin <70319226+linyueqian@users.noreply.github.com> Date: Wed, 20 May 2026 01:08:56 -0400 Subject: [PATCH] [Bench] Add audio-streaming continuity metric for TTS (#3618) Signed-off-by: Yueqian Lin Co-authored-by: Hongsheng Liu --- benchmarks/tts/README.md | 24 ++++- benchmarks/tts/bench_tts.py | 2 +- tests/benchmarks/metrics/test_metrics.py | 32 ++++++ tests/benchmarks/test_audio_continuity.py | 123 ++++++++++++++++++++++ vllm_omni/benchmarks/audio_continuity.py | 91 ++++++++++++++++ vllm_omni/benchmarks/metrics/metrics.py | 26 ++++- vllm_omni/benchmarks/patch/patch.py | 62 ++++++++++- 7 files changed, 350 insertions(+), 10 deletions(-) create mode 100644 tests/benchmarks/test_audio_continuity.py create mode 100644 vllm_omni/benchmarks/audio_continuity.py diff --git a/benchmarks/tts/README.md b/benchmarks/tts/README.md index 9e2fd35b1..d8841f382 100644 --- a/benchmarks/tts/README.md +++ b/benchmarks/tts/README.md @@ -40,7 +40,7 @@ vllm bench serve --omni \ --num-prompts 20 --num-warmups 2 \ --extra-body '{"task_type":"Base"}' \ --max-concurrency 1 --request-rate inf \ - --percentile-metrics ttft,e2el,audio_rtf,audio_ttfp,audio_duration \ + --percentile-metrics ttft,e2el,audio_rtf,audio_ttfp,audio_duration,audio_underrun \ --save-result --result-dir ./results ``` @@ -58,7 +58,7 @@ vllm bench serve --omni \ --num-prompts 20 --num-warmups 2 \ --extra-body '{"voice":"Vivian","language":"English","task_type":"CustomVoice"}' \ --max-concurrency 1 --request-rate inf \ - --percentile-metrics ttft,e2el,audio_rtf,audio_ttfp,audio_duration \ + --percentile-metrics ttft,e2el,audio_rtf,audio_ttfp,audio_duration,audio_underrun \ --save-result --result-dir ./results ``` @@ -76,10 +76,28 @@ vllm bench serve --omni \ --num-prompts 20 --num-warmups 2 \ --extra-body '{"task_type":"VoiceDesign","language":"English"}' \ --max-concurrency 1 --request-rate inf \ - --percentile-metrics ttft,e2el,audio_rtf,audio_ttfp,audio_duration \ + --percentile-metrics ttft,e2el,audio_rtf,audio_ttfp,audio_duration,audio_underrun \ --save-result --result-dir ./results ``` +#### Streaming continuity (`audio_underrun`) + +`audio_underrun` (seconds) is the per-request worst-case buffer deficit +under a realtime-rate player simulation. The audio-speech backend records +chunk arrival times during the SSE stream and surfaces: + +- `Mean / Median / P{50,99} AUDIO_UNDERRUN (s)` per the standard percentile + output - any positive value means at least one inter-chunk gap was longer + than the chunk's audio duration. +- `Streaming continuity OK rate` - fraction of requests whose worst-case + underrun stayed under the threshold (default 100 ms, the commonly cited + "audible gap" budget). Override the threshold with + `VLLM_OMNI_BENCH_AUDIO_CONTINUITY_THRESHOLD_S=`. + +This captures the failure mode where `RTF_p50 < 1` (server keeps up in +aggregate) but per-stream chunk arrival is bursty enough that listeners +still hear gaps - common at high concurrency on streaming TTS pipelines. + #### Add WER / SIM / UTMOS to any of the above Append `--seed-tts-wer-eval` (and optionally `SEED_TTS_EVAL_DEVICE=cuda:0` diff --git a/benchmarks/tts/bench_tts.py b/benchmarks/tts/bench_tts.py index ba82b1c9b..8d2eb99c8 100644 --- a/benchmarks/tts/bench_tts.py +++ b/benchmarks/tts/bench_tts.py @@ -118,7 +118,7 @@ def build_bench_args( "--num-warmups", "2", "--percentile-metrics", - "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", + "ttft,e2el,audio_rtf,audio_ttfp,audio_duration,audio_underrun", ] if resolved_dataset_path: diff --git a/tests/benchmarks/metrics/test_metrics.py b/tests/benchmarks/metrics/test_metrics.py index 788e1227d..2688b224f 100644 --- a/tests/benchmarks/metrics/test_metrics.py +++ b/tests/benchmarks/metrics/test_metrics.py @@ -63,6 +63,38 @@ def test_total_input_aggregated_from_output_prompt_len(): ) +def test_audio_continuity_aggregation(): + """Continuity rate and underrun percentile must aggregate from per-output fields.""" + bad = _make_output(100) + bad.audio_underrun_s = 0.5 + bad.audio_continuity_ok = False + good_a = _make_output(100) + good_a.audio_underrun_s = 0.02 + good_a.audio_continuity_ok = True + good_b = _make_output(100) + good_b.audio_underrun_s = 0.0 + good_b.audio_continuity_ok = True + + metrics, _ = calculate_metrics( + input_requests=[], + outputs=[bad, good_a, good_b], + dur_s=10.0, + tokenizer=None, + selected_percentiles=[50.0, 99.0], + goodput_config_dict={}, + task_type=TaskType.GENERATION, + selected_percentile_metrics=["audio_underrun"], + max_concurrency=None, + request_rate=float("inf"), + benchmark_duration=10.0, + ) + + assert metrics.audio_continuity_ok_rate == pytest.approx(2 / 3, abs=1e-6) + # p99 of [0.5, 0.02, 0.0] is dominated by the 0.5 outlier. + p99 = dict(metrics.percentiles_audio_underrun_s).get(99.0) + assert p99 is not None and p99 > 0.4 + + # ============================================================================ # TTFT suppression for pure-audio (TTS) benchmarks # ============================================================================ diff --git a/tests/benchmarks/test_audio_continuity.py b/tests/benchmarks/test_audio_continuity.py new file mode 100644 index 000000000..7737986f0 --- /dev/null +++ b/tests/benchmarks/test_audio_continuity.py @@ -0,0 +1,123 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Unit tests for the audio-streaming continuity helper. + +The helper is intentionally vllm-free: it operates on a flat timeline of +``(arrival_time_s, num_bytes)`` chunks plus PCM format params. These tests +exercise the underrun arithmetic against hand-crafted timelines so future +refactors of the streaming backend cannot silently regress the metric. +""" + +from __future__ import annotations + +import pytest + +from vllm_omni.benchmarks.audio_continuity import ( + ContinuityStats, + compute_continuity_stats, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.benchmark, pytest.mark.cpu] + + +# PCM s16le mono at 24 kHz -> 48 000 bytes/sec, the Qwen3-TTS / VoxCPM2 default. +_SR = 24_000 +_BPS = _SR * 2 # 48 000 + + +def test_empty_timeline_is_continuous() -> None: + stats = compute_continuity_stats([], [], sample_rate=_SR) + assert stats == ContinuityStats( + max_underrun_s=0.0, + underrun_event_count=0, + is_continuous=True, + ) + + +def test_single_chunk_has_no_underrun() -> None: + # One large chunk, no later arrivals to check against. + stats = compute_continuity_stats( + chunk_arrival_times_s=[0.05], + chunk_bytes=[_BPS], # one second of audio in a single chunk + sample_rate=_SR, + ) + assert stats.max_underrun_s == 0.0 + assert stats.underrun_event_count == 0 + assert stats.is_continuous is True + + +def test_chunks_faster_than_realtime_have_no_underrun() -> None: + # 100ms of audio per chunk, but arrive every 50ms — buffer always builds. + stats = compute_continuity_stats( + chunk_arrival_times_s=[0.0, 0.05, 0.10, 0.15], + chunk_bytes=[_BPS // 10] * 4, + sample_rate=_SR, + ) + assert stats.max_underrun_s == 0.0 + assert stats.underrun_event_count == 0 + assert stats.is_continuous is True + + +def test_single_gap_above_threshold_breaks_continuity() -> None: + # First chunk carries 100ms of audio at t=0. Next chunk arrives at t=0.5s, + # so the player ran dry from t=0.1 to t=0.5 -> 400ms underrun. + stats = compute_continuity_stats( + chunk_arrival_times_s=[0.0, 0.5], + chunk_bytes=[_BPS // 10, _BPS // 10], # 100ms each + sample_rate=_SR, + threshold_s=0.1, + ) + assert stats.max_underrun_s == pytest.approx(0.4, abs=1e-6) + assert stats.underrun_event_count == 1 + assert stats.is_continuous is False + + +def test_underrun_is_max_across_gaps_not_sum() -> None: + # Two gaps: a 200ms shortfall and a 50ms shortfall. max_underrun_s + # must surface the 200ms one, not the sum. + stats = compute_continuity_stats( + chunk_arrival_times_s=[0.0, 0.3, 0.45], + chunk_bytes=[_BPS // 10, _BPS // 10, _BPS // 10], # 100ms chunks + sample_rate=_SR, + threshold_s=0.1, + ) + # At t=0.3 the player consumed 300ms but only 100ms received -> 200ms deficit + # At t=0.45 player consumed 450ms, received 200ms -> 250ms deficit. + assert stats.max_underrun_s == pytest.approx(0.25, abs=1e-6) + assert stats.underrun_event_count == 2 + assert stats.is_continuous is False + + +def test_continuity_threshold_boundary_inclusive() -> None: + # First chunk = 100ms audio at t=0. Next arrives at t=0.2 -> 100ms deficit + # exactly. With threshold=0.1, continuity_ok stays True (boundary). + stats = compute_continuity_stats( + chunk_arrival_times_s=[0.0, 0.2], + chunk_bytes=[_BPS // 10, _BPS // 10], + sample_rate=_SR, + threshold_s=0.1, + ) + assert stats.max_underrun_s == pytest.approx(0.1, abs=1e-6) + assert stats.is_continuous is True + + +def test_mismatched_lengths_returns_zero_stats() -> None: + # Defensive: never crash on malformed timeline. + stats = compute_continuity_stats( + chunk_arrival_times_s=[0.0, 0.1], + chunk_bytes=[100], + sample_rate=_SR, + ) + assert stats == ContinuityStats(0.0, 0, True) + + +def test_threshold_is_configurable() -> None: + # 200ms underrun: violates 100ms threshold, OK against 500ms threshold. + arrivals = [0.0, 0.3] + bytes_list = [_BPS // 10, _BPS // 10] + strict = compute_continuity_stats(arrivals, bytes_list, sample_rate=_SR, threshold_s=0.1) + lenient = compute_continuity_stats(arrivals, bytes_list, sample_rate=_SR, threshold_s=0.5) + assert strict.is_continuous is False + assert lenient.is_continuous is True + assert strict.max_underrun_s == lenient.max_underrun_s diff --git a/vllm_omni/benchmarks/audio_continuity.py b/vllm_omni/benchmarks/audio_continuity.py new file mode 100644 index 000000000..e92ed9a0a --- /dev/null +++ b/vllm_omni/benchmarks/audio_continuity.py @@ -0,0 +1,91 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""Streaming-audio continuity metrics for the TTS bench. + +Captures the failure mode where ``RTF_p50 < 1`` (server keeps up with realtime +in aggregate) but per-stream chunk arrival is bursty enough that listeners +hear audible gaps. Companion to ``audio_ttfp`` / ``audio_rtf``. + +The math is purposefully simple so the helper has no ``vllm`` dependency and +can be unit-tested in isolation: given the wall-clock arrival timeline of +audio chunks and the PCM format, simulate a player that consumes at realtime +rate from the first chunk onward, and surface the worst-case deficit. + +A deficit > ``threshold_s`` at any point counts as an audible underrun. +""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class ContinuityStats: + """Result of a single-request continuity analysis. + + Attributes: + max_underrun_s: Worst-case wall-clock seconds the player was starved. + underrun_event_count: Inter-chunk intervals during which the buffer + went negative (one per gap, not per starved millisecond). + is_continuous: ``max_underrun_s <= threshold_s`` for the request. + """ + + max_underrun_s: float + underrun_event_count: int + is_continuous: bool + + +def compute_continuity_stats( + chunk_arrival_times_s: list[float], + chunk_bytes: list[int], + sample_rate: int, + sample_width: int = 2, + channels: int = 1, + threshold_s: float = 0.1, +) -> ContinuityStats: + """Compute the continuity stats for one streaming response. + + Args: + chunk_arrival_times_s: Wall-clock seconds (since request start) at + which each non-empty audio chunk's *last byte* arrived. Must be + monotonically non-decreasing. + chunk_bytes: Byte count of each chunk in the same order. + sample_rate: PCM sample rate (Hz). 24 000 for Qwen3-TTS / VoxCPM2. + sample_width: Bytes per sample (2 = s16le). + channels: PCM channel count (1 = mono). + threshold_s: Underrun budget. The default 0.1 s matches the + commonly-cited "audible gap" threshold for streaming TTS. + + Returns: + A :class:`ContinuityStats` summarising the worst-case deficit and + whether it stayed under the threshold. + """ + n = len(chunk_arrival_times_s) + if n == 0 or n != len(chunk_bytes): + return ContinuityStats(0.0, 0, True) + + bytes_per_s = sample_rate * sample_width * channels + if bytes_per_s <= 0: + return ContinuityStats(0.0, 0, True) + + t0 = chunk_arrival_times_s[0] + received_before = 0 + max_underrun_s = 0.0 + event_count = 0 + for i in range(n): + if i > 0: + played_bytes = (chunk_arrival_times_s[i] - t0) * bytes_per_s + deficit_bytes = played_bytes - received_before + if deficit_bytes > 0: + deficit_s = deficit_bytes / bytes_per_s + if deficit_s > max_underrun_s: + max_underrun_s = deficit_s + event_count += 1 + received_before += chunk_bytes[i] + + return ContinuityStats( + max_underrun_s=max_underrun_s, + underrun_event_count=event_count, + is_continuous=max_underrun_s <= threshold_s, + ) diff --git a/vllm_omni/benchmarks/metrics/metrics.py b/vllm_omni/benchmarks/metrics/metrics.py index 4b32662c5..56ebb20ea 100644 --- a/vllm_omni/benchmarks/metrics/metrics.py +++ b/vllm_omni/benchmarks/metrics/metrics.py @@ -25,6 +25,12 @@ class MultiModalsBenchmarkMetrics(BenchmarkMetrics): median_audio_duration_s: float = 0.0 std_audio_duration_s: float = 0.0 percentiles_audio_duration_s: list[tuple[float, float]] = None + # Streaming-audio continuity (see vllm_omni/benchmarks/audio_continuity.py). + mean_audio_underrun_s: float = 0.0 + median_audio_underrun_s: float = 0.0 + std_audio_underrun_s: float = 0.0 + percentiles_audio_underrun_s: list[tuple[float, float]] = None + audio_continuity_ok_rate: float = 1.0 def print_metrics( @@ -83,6 +89,12 @@ def print_audio_metrics(selected_percentile_metrics, metrics: MultiModalsBenchma print("{:<40} {:<10.2f}".format("Total audio duration generated(s):", metrics.total_audio_duration_s)) print("{:<40} {:<10}".format("Total audio frames generated:", metrics.total_audio_frames)) print("{:<40} {:<10.2f}".format("Audio throughput(audio duration/s):", metrics.audio_throughput)) + print( + "{:<40} {:<10.2%}".format( + "Streaming continuity OK rate:", + metrics.audio_continuity_ok_rate, + ) + ) for metric in selected_percentile_metrics: if metric.startswith("audio"): process_one_metric(metric, metrics) @@ -100,17 +112,18 @@ def process_one_metric( "audio_ttfp": "Time to First Packet", "audio_rtf": "Real Time Factor", "audio_duration": "Audio Duration", + "audio_underrun": "Streaming Audio Underrun", } header = metric_header_map.get(metric_attribute_name, metric_attribute_name) print("{s:{c}^{n}}".format(s=header, n=50, c="-")) is_audio_rtf = metric_attribute_name == "audio_rtf" - is_audio_duration = metric_attribute_name == "audio_duration" + is_audio_duration_or_underrun = metric_attribute_name in ("audio_duration", "audio_underrun") suffix = "_ms" unit_suffix = " (ms)" - if is_audio_duration: + if is_audio_duration_or_underrun: suffix = "_s" unit_suffix = " (s)" elif is_audio_rtf: @@ -172,6 +185,8 @@ def calculate_metrics( audio_rtfs: list[float] = [] audio_duration: list[float] = [] audio_frames: list[int] = [] + audio_underruns: list[float] = [] + audio_continuity_ok: list[bool] = [] input_audio_duration = 0.0 for i in range(len(outputs)): if outputs[i].success: @@ -205,6 +220,8 @@ def calculate_metrics( audio_rtfs.append(getattr(outputs[i], "audio_rtf", 0.0)) audio_duration.append(getattr(outputs[i], "audio_duration", 0.0)) audio_frames.append(getattr(outputs[i], "audio_frames", 0.0)) + audio_underruns.append(getattr(outputs[i], "audio_underrun_s", 0.0)) + audio_continuity_ok.append(bool(getattr(outputs[i], "audio_continuity_ok", True))) e2els.append(outputs[i].latency) input_audio_duration += outputs[i].input_audio_duration completed += 1 @@ -332,6 +349,11 @@ def calculate_metrics( std_audio_rtf=np.std(audio_rtfs or 0), median_audio_rtf=np.median(audio_rtfs or 0), percentiles_audio_rtf=[(p, np.percentile(audio_rtfs or 0, p)) for p in selected_percentiles], + mean_audio_underrun_s=np.mean(audio_underruns or 0), + std_audio_underrun_s=np.std(audio_underruns or 0), + median_audio_underrun_s=np.median(audio_underruns or 0), + percentiles_audio_underrun_s=[(p, np.percentile(audio_underruns or 0, p)) for p in selected_percentiles], + audio_continuity_ok_rate=(sum(audio_continuity_ok) / len(audio_continuity_ok)) if audio_continuity_ok else 1.0, mean_tpot_ms=np.mean(tpots or 0) * 1000, std_tpot_ms=np.std(tpots or 0) * 1000, median_tpot_ms=np.median(tpots or 0) * 1000, diff --git a/vllm_omni/benchmarks/patch/patch.py b/vllm_omni/benchmarks/patch/patch.py index ea3b6b4c9..12aeef7e4 100644 --- a/vllm_omni/benchmarks/patch/patch.py +++ b/vllm_omni/benchmarks/patch/patch.py @@ -36,6 +36,7 @@ from vllm.tokenizers import TokenizerLike logger = init_logger(__name__) +from vllm_omni.benchmarks.audio_continuity import compute_continuity_stats from vllm_omni.benchmarks.data_modules.daily_omni_dataset import DailyOmniDataset, DailyOmniSampleRequest from vllm_omni.benchmarks.data_modules.random_multi_modal_dataset import OmniRandomMultiModalDataset from vllm_omni.benchmarks.data_modules.seed_tts_dataset import ( @@ -46,6 +47,33 @@ from vllm_omni.benchmarks.data_modules.seed_tts_dataset import ( SeedTTSTextDataset, ) +_AUDIO_CONTINUITY_THRESHOLD_ENV = "VLLM_OMNI_BENCH_AUDIO_CONTINUITY_THRESHOLD_S" +_DEFAULT_AUDIO_CONTINUITY_THRESHOLD_S = 0.1 + + +def _audio_continuity_threshold_s() -> float: + """Return the per-request underrun budget (s). + + Read from ``VLLM_OMNI_BENCH_AUDIO_CONTINUITY_THRESHOLD_S`` so users can + re-aim the SLO without rebuilding. Defaults to 100 ms - the standard + "audible gap" budget for streaming TTS. + """ + raw = os.environ.get(_AUDIO_CONTINUITY_THRESHOLD_ENV) + if not raw: + return _DEFAULT_AUDIO_CONTINUITY_THRESHOLD_S + try: + value = float(raw) + except ValueError: + logger.warning( + "Invalid %s=%r; using default %.3fs", + _AUDIO_CONTINUITY_THRESHOLD_ENV, + raw, + _DEFAULT_AUDIO_CONTINUITY_THRESHOLD_S, + ) + return _DEFAULT_AUDIO_CONTINUITY_THRESHOLD_S + return max(value, 0.0) + + get_samples_old = datasets.get_samples _DEFAULT_DAILY_OMNI_REPO = "liarliar/Daily-Omni" @@ -315,6 +343,16 @@ class MixRequestFuncOutput(RequestFuncOutput): audio_frames: int = 0 audio_rtf: float = 0.0 text_latency: float = 0.0 + #: Worst-case streaming-audio underrun (wall-clock seconds the player + #: would have been starved). Populated by the audio-speech backend; ``0.0`` + #: for backends that do not run continuity analysis. + audio_underrun_s: float = 0.0 + #: Whether the request stayed under the continuity threshold (default + #: 100 ms). Mirrors ``audio_underrun_s <= threshold``. + audio_continuity_ok: bool = True + #: Number of inter-chunk intervals during which the player buffer went + #: negative. + audio_underrun_event_count: int = 0 #: Raw PCM s16le mono at 24 kHz for Seed-TTS WER: from ``/v1/audio/speech`` stream or #: resampled export after ``openai-chat-omni`` audio deltas. tts_output_pcm_bytes: bytes | None = None @@ -605,6 +643,8 @@ async def async_request_openai_audio_speech( total_pcm_bytes = 0 capture_wer_pcm = _seed_tts_capture_pcm_for_wer() and getattr(request_func_input, "seed_tts_row", False) pcm_capture = bytearray() if capture_wer_pcm else None + chunk_arrival_times_s: list[float] = [] + chunk_sizes: list[int] = [] try: async with session.post(url=api_url, json=payload, headers=headers) as response: if response.status == 200: @@ -617,6 +657,8 @@ async def async_request_openai_audio_speech( # not defined here; only audio TTFP is meaningful. output.audio_ttfp = timestamp - st total_pcm_bytes += len(chunk) + chunk_arrival_times_s.append(timestamp - st) + chunk_sizes.append(len(chunk)) if pcm_capture is not None: pcm_capture.extend(chunk) @@ -631,6 +673,18 @@ async def async_request_openai_audio_speech( else: output.audio_rtf = 0 logger.warning("Audio duration is zero") + + continuity = compute_continuity_stats( + chunk_arrival_times_s=chunk_arrival_times_s, + chunk_bytes=chunk_sizes, + sample_rate=sample_rate, + sample_width=sample_width, + channels=channels, + threshold_s=_audio_continuity_threshold_s(), + ) + output.audio_underrun_s = continuity.max_underrun_s + output.audio_continuity_ok = continuity.is_continuous + output.audio_underrun_event_count = continuity.underrun_event_count if pcm_capture is not None and pcm_capture: output.tts_output_pcm_bytes = bytes(pcm_capture) elif capture_wer_pcm: @@ -1020,14 +1074,14 @@ async def benchmark( return # No text tokens generated (e.g. pure TTS speech endpoint): per-token # latency metrics (ttft/tpot/itl) are undefined, so skip them. - is_text_token_metric = metric_attribute_name not in ("e2el", "audio_ttfp", "audio_rtf", "audio_duration") + is_text_token_metric = not (metric_attribute_name == "e2el" or metric_attribute_name.startswith("audio")) if is_text_token_metric and getattr(metrics, "total_output", 0) == 0: return is_audio_rtf = metric_attribute_name == "audio_rtf" - is_audio_duration = metric_attribute_name == "audio_duration" + is_audio_duration_or_underrun = metric_attribute_name in ("audio_duration", "audio_underrun") suffix = "_ms" - if is_audio_duration: + if is_audio_duration_or_underrun: suffix = "_s" elif is_audio_rtf: suffix = "" @@ -1038,7 +1092,7 @@ async def benchmark( median_attr_name = f"median_{metric_attribute_name}{suffix}" median_value = getattr(metrics, median_attr_name, 0.0) result[median_attr_name] = median_value - for p, value in getattr(metrics, f"percentiles_{metric_attribute_name}{suffix}"): + for p, value in getattr(metrics, f"percentiles_{metric_attribute_name}{suffix}", None) or []: p_word = str(int(p)) if int(p) == p else str(p) result[f"p{p_word}_{metric_attribute_name}{suffix}"] = value