[BugFix] Fix the issue of incorrect generated tokens (#4461)
Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
1 parent
b46c5af06f
commit
d897852258
4 files changed
+304
-24
No files matched your search
@@ -14,6 +14,37 @@ from vllm_omni.benchmarks.patch.patch import MixRequestFuncOutput
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.benchmark, pytest.mark.cpu]
|
||||
|
||||
|
||||
def test_tpot_matches_mean_itl_per_request():
|
||||
"""TPOT and pooled ITL should agree when output_len tracks ITL samples."""
|
||||
output = MixRequestFuncOutput()
|
||||
output.success = True
|
||||
output.prompt_len = 100
|
||||
# Simulate server reporting more tokens than SSE chunks (bundled tokens).
|
||||
# len(itl)=2 → 2 inter-chunk intervals, but server generated 10 tokens.
|
||||
output.output_tokens = 10
|
||||
output.generated_text = "hello world"
|
||||
output.ttft = 0.05
|
||||
output.text_latency = 0.25
|
||||
output.latency = 0.30
|
||||
output.itl = [0.10, 0.10]
|
||||
|
||||
metrics, _ = calculate_metrics(
|
||||
input_requests=[],
|
||||
outputs=[output],
|
||||
dur_s=1.0,
|
||||
tokenizer=None,
|
||||
selected_percentiles=[50.0],
|
||||
goodput_config_dict={},
|
||||
task_type=TaskType.GENERATION,
|
||||
selected_percentile_metrics=["tpot", "itl"],
|
||||
max_concurrency=None,
|
||||
request_rate=float("inf"),
|
||||
benchmark_duration=1.0,
|
||||
)
|
||||
|
||||
assert metrics.mean_tpot_ms == pytest.approx(metrics.mean_itl_ms, rel=1e-6, abs=1e-6)
|
||||
|
||||
|
||||
def _make_output(prompt_len: int, output_tokens: int = 10) -> MixRequestFuncOutput:
|
||||
"""Build a minimal successful MixRequestFuncOutput for metrics aggregation."""
|
||||
output = MixRequestFuncOutput()
|
||||
|
||||
@@ -133,7 +133,9 @@ async def test_output_tokens_not_assigned_without_metrics(mocker: MockerFixture)
|
||||
|
||||
# Assert
|
||||
assert output.success is True
|
||||
assert output.output_tokens == 0, "output_tokens should default to 0 when no metrics"
|
||||
# Without server metrics we no longer infer count from SSE chunks because
|
||||
# one chunk may carry multiple tokens. output_tokens stays 0 here.
|
||||
assert output.output_tokens == 0, "output_tokens should be 0 when no server metrics are present"
|
||||
assert output.generated_text == "Hello world"
|
||||
|
||||
|
||||
@@ -277,7 +279,7 @@ async def test_output_tokens_with_missing_num_tokens_out(mocker: MockerFixture):
|
||||
|
||||
# Assert
|
||||
assert output.success is True
|
||||
assert output.output_tokens == 0, "output_tokens should default to 0 when num_tokens_out is missing"
|
||||
assert output.output_tokens == 0, "output_tokens should be 0 when num_tokens_out is missing"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -579,6 +581,167 @@ class TestTextLatencyAttribute:
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_skips_empty_role_chunk_for_ttft_and_itl(mocker: MockerFixture):
|
||||
"""Role/empty text chunks must not count toward TTFT or ITL."""
|
||||
request_input = RequestFuncInput(
|
||||
model="test-model",
|
||||
model_name="test-model",
|
||||
prompt="test prompt",
|
||||
api_url="http://test.com/v1/chat/completions",
|
||||
prompt_len=10,
|
||||
output_len=20,
|
||||
)
|
||||
|
||||
chunks = [
|
||||
create_sse_chunk(
|
||||
{
|
||||
"choices": [{"delta": {"role": "assistant", "content": ""}}],
|
||||
"modality": "text",
|
||||
}
|
||||
),
|
||||
create_sse_chunk(
|
||||
{
|
||||
"choices": [{"delta": {"content": "Hello"}}],
|
||||
"modality": "text",
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 1, "total_tokens": 11},
|
||||
}
|
||||
),
|
||||
create_sse_chunk(
|
||||
{
|
||||
"choices": [{"delta": {"content": " world"}}],
|
||||
"modality": "text",
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 2, "total_tokens": 12},
|
||||
}
|
||||
),
|
||||
b"data: [DONE]\n\n",
|
||||
]
|
||||
|
||||
mock_response = MockResponse(200, chunks, delay_between_chunks=0.01)
|
||||
mock_session = mocker.AsyncMock()
|
||||
mock_session.post = mocker.MagicMock(return_value=mock_response)
|
||||
|
||||
output = await async_request_openai_chat_omni_completions(request_input, mock_session)
|
||||
|
||||
assert output.success is True
|
||||
assert output.output_tokens == 2
|
||||
assert len(output.itl) == 1
|
||||
assert output.ttft > 0
|
||||
assert output.text_latency - output.ttft == pytest.approx(sum(output.itl), rel=1e-6, abs=1e-6)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bundled_tokens_split_itl_from_usage(mocker: MockerFixture):
|
||||
"""Multiple tokens in one SSE chunk should expand ITL via usage deltas."""
|
||||
request_input = RequestFuncInput(
|
||||
model="test-model",
|
||||
model_name="test-model",
|
||||
prompt="test prompt",
|
||||
api_url="http://test.com/v1/chat/completions",
|
||||
prompt_len=10,
|
||||
output_len=20,
|
||||
)
|
||||
|
||||
chunks = [
|
||||
create_sse_chunk(
|
||||
{
|
||||
"choices": [{"delta": {"content": "A"}}],
|
||||
"modality": "text",
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 1, "total_tokens": 11},
|
||||
}
|
||||
),
|
||||
create_sse_chunk(
|
||||
{
|
||||
"choices": [{"delta": {"content": "BCD"}}],
|
||||
"modality": "text",
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 4, "total_tokens": 14},
|
||||
}
|
||||
),
|
||||
b"data: [DONE]\n\n",
|
||||
]
|
||||
|
||||
mock_response = MockResponse(200, chunks, delay_between_chunks=0.02)
|
||||
mock_session = mocker.AsyncMock()
|
||||
mock_session.post = mocker.MagicMock(return_value=mock_response)
|
||||
|
||||
output = await async_request_openai_chat_omni_completions(request_input, mock_session)
|
||||
|
||||
assert output.success is True
|
||||
assert output.output_tokens == 4
|
||||
assert len(output.itl) == 3
|
||||
assert output.text_latency - output.ttft == pytest.approx(sum(output.itl), rel=1e-6, abs=1e-6)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_output_tokens_prefers_usage_over_metrics(mocker: MockerFixture):
|
||||
request_input = RequestFuncInput(
|
||||
model="test-model",
|
||||
model_name="test-model",
|
||||
prompt="test prompt",
|
||||
api_url="http://test.com/v1/chat/completions",
|
||||
prompt_len=10,
|
||||
output_len=20,
|
||||
)
|
||||
|
||||
chunks = [
|
||||
create_sse_chunk(
|
||||
{
|
||||
"choices": [{"delta": {"content": "Hello"}}],
|
||||
"modality": "text",
|
||||
"metrics": {"num_tokens_out": 5},
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 8, "total_tokens": 18},
|
||||
}
|
||||
),
|
||||
b"data: [DONE]\n\n",
|
||||
]
|
||||
|
||||
mock_response = MockResponse(200, chunks)
|
||||
mock_session = mocker.AsyncMock()
|
||||
mock_session.post = mocker.MagicMock(return_value=mock_response)
|
||||
|
||||
output = await async_request_openai_chat_omni_completions(request_input, mock_session)
|
||||
|
||||
assert output.success is True
|
||||
assert output.output_tokens == 8
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_metrics_only_chunk_updates_output_tokens(mocker: MockerFixture):
|
||||
request_input = RequestFuncInput(
|
||||
model="test-model",
|
||||
model_name="test-model",
|
||||
prompt="test prompt",
|
||||
api_url="http://test.com/v1/chat/completions",
|
||||
prompt_len=10,
|
||||
output_len=20,
|
||||
)
|
||||
|
||||
chunks = [
|
||||
create_sse_chunk(
|
||||
{
|
||||
"choices": [{"delta": {"content": "Text response"}}],
|
||||
"modality": "text",
|
||||
}
|
||||
),
|
||||
create_sse_chunk(
|
||||
{
|
||||
"modality": "text",
|
||||
"metrics": {"num_tokens_out": 25, "num_tokens_in": 10},
|
||||
}
|
||||
),
|
||||
b"data: [DONE]\n\n",
|
||||
]
|
||||
|
||||
mock_response = MockResponse(200, chunks)
|
||||
mock_session = mocker.AsyncMock()
|
||||
mock_session.post = mocker.MagicMock(return_value=mock_response)
|
||||
|
||||
output = await async_request_openai_chat_omni_completions(request_input, mock_session)
|
||||
|
||||
assert output.success is True
|
||||
assert output.output_tokens == 25
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prompt_len_assigned_from_usage(mocker: MockerFixture):
|
||||
# Arrange: request claims prompt_len=100, but server reports 4992 (multimodal).
|
||||
|
||||
Reference in new issue
Block a user