[Core / Bugfix] Add Mechanism for Endpoint Rejection (#4762)
Signed-off-by: Alex Brooks <albrooks@redhat.com>
This commit is contained in:
1 parent
7d4decc1d8
commit
b7d08fe5f6
14 files changed
+508
-158
No files matched your search
@@ -226,3 +226,27 @@ def test_thinker_prefix_caching_audio_output(omni_server, openai_client) -> None
|
||||
}
|
||||
|
||||
_run_prefix_cache_check(openai_client, request_config)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||
def test_completions_rejected_for_thinker_talker(omni_server, openai_client) -> None:
|
||||
"""Ensure Thinker-talker models reject /v1/completions; we do this because the
|
||||
thinker-talker handoff implementations currently use ChatML <|im_start|> and
|
||||
<|im_end|> markers to segment the input sequence; when we don't have them,
|
||||
the talker does not get any embeddings, which breaks the server.
|
||||
"""
|
||||
responses = openai_client.send_completions_http_request(
|
||||
{
|
||||
"json": {
|
||||
"model": omni_server.model,
|
||||
"prompt": "Hello, how are you?",
|
||||
"max_tokens": 10,
|
||||
},
|
||||
},
|
||||
err_code=400,
|
||||
)
|
||||
assert not responses[0].success
|
||||
Reference in new issue
Block a user