[Frontend] Implement Batched Chat Completions (#5317)

Signed-off-by: Alex Brooks <albrooks@redhat.com>
This commit is contained in:
Alex Brooks authored and GitHub committed 2026-08-07 16:12:09 -06:00
1 parent 18bf7e0ff9
commit b15d4e6a64
7 files changed
+531 -39

No files matched your search

+22
View File
@@ -1252,6 +1252,28 @@ class OpenAIClientHandler:
)
return [resp]
def send_batched_chat_completions_http_request(
self,
request_config: dict[str, Any],
*,
err_code: int | tuple[int, ...] | list[int] | None = None,
err_message: str | tuple[str, ...] | list[str] | None = None,
) -> list[HttpResponse]:
"""Post to batched chat completions."""
cfg = _merge_http_expectation_kwargs(
request_config,
err_code=err_code,
err_message=err_message,
)
r = self._post_json_endpoint("/v1/chat/completions/batch", cfg, default_timeout=120.0)
resp = self._http_response_from_requests(r)
assert_http_error(
resp,
err_code=cfg.get("err_code"),
err_message=cfg.get("err_message"),
)
return [resp]
def send_completions_http_request(
self,
request_config: dict[str, Any],