143 lines
4.7 KiB
Python
143 lines
4.7 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
|
|
import re
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
import yaml
|
|
|
|
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
|
|
|
ROOT_DIR = Path(__file__).parents[2]
|
|
COVERAGE_PATH = ROOT_DIR / "docs/models/recipe_coverage.yaml"
|
|
SUPPORTED_MODELS_PATH = ROOT_DIR / "docs/models/supported_models.md"
|
|
VALID_STATUSES = {"published", "repository", "no_validated_recipe_yet"}
|
|
VALID_HARDWARE = {"nvidia", "amd", "ascend", "intel"}
|
|
HARDWARE_COLUMNS = {
|
|
"nvidia": 4,
|
|
"amd": 5,
|
|
"ascend": 6,
|
|
"intel": 7,
|
|
}
|
|
|
|
|
|
def load_entries() -> list[dict]:
|
|
data = yaml.safe_load(COVERAGE_PATH.read_text(encoding="utf-8"))
|
|
assert data["version"] == 1
|
|
assert isinstance(data["entries"], list)
|
|
return data["entries"]
|
|
|
|
|
|
def supported_model_ids() -> set[str]:
|
|
model_ids = set()
|
|
for line in SUPPORTED_MODELS_PATH.read_text(encoding="utf-8").splitlines():
|
|
if not line.startswith("|"):
|
|
continue
|
|
columns = line.split("|")
|
|
if len(columns) <= 3:
|
|
continue
|
|
model_ids.update(re.findall(r"`([^`]+/[^`]+)`", columns[3]))
|
|
return model_ids
|
|
|
|
|
|
def supported_model_recipe_cells() -> dict[str, str]:
|
|
recipe_cells = {}
|
|
in_table = False
|
|
for line in SUPPORTED_MODELS_PATH.read_text(encoding="utf-8").splitlines():
|
|
if line.startswith("| Architecture |"):
|
|
in_table = True
|
|
continue
|
|
if in_table and not line.startswith("|"):
|
|
break
|
|
if not in_table:
|
|
continue
|
|
|
|
columns = line.split("|")
|
|
if len(columns) <= 8 or columns[1].strip().startswith("-"):
|
|
continue
|
|
for model_id in re.findall(r"`([^`]+/[^`]+)`", columns[3]):
|
|
recipe_cells[model_id] = columns[8].strip()
|
|
return recipe_cells
|
|
|
|
|
|
def expected_recipe_link(entry: dict) -> str:
|
|
recipe = entry["recipe"]
|
|
if recipe["status"] == "published":
|
|
return recipe["published_url"]
|
|
if recipe["status"] == "repository":
|
|
return f"https://github.com/vllm-project/vllm-omni/blob/main/{recipe['repository_path']}"
|
|
return "No validated recipe yet"
|
|
|
|
|
|
def supported_model_hardware_cells() -> dict[str, list[str]]:
|
|
hardware_cells = {}
|
|
in_table = False
|
|
for line in SUPPORTED_MODELS_PATH.read_text(encoding="utf-8").splitlines():
|
|
if line.startswith("| Architecture |"):
|
|
in_table = True
|
|
continue
|
|
if in_table and not line.startswith("|"):
|
|
break
|
|
if not in_table:
|
|
continue
|
|
|
|
columns = line.split("|")
|
|
if len(columns) <= 8 or columns[1].strip().startswith("-"):
|
|
continue
|
|
cells = [columns[index].strip() for index in HARDWARE_COLUMNS.values()]
|
|
for model_id in re.findall(r"`([^`]+/[^`]+)`", columns[3]):
|
|
hardware_cells[model_id] = cells
|
|
return hardware_cells
|
|
|
|
|
|
def test_registry_keys_are_exact_supported_checkpoints():
|
|
entries = load_entries()
|
|
keys = [(entry["model"], entry["task"]) for entry in entries]
|
|
|
|
assert len(keys) == len(set(keys))
|
|
assert all(entry["model"] in supported_model_ids() for entry in entries)
|
|
|
|
|
|
def test_supported_models_render_registry_recipe_links():
|
|
recipe_cells = supported_model_recipe_cells()
|
|
|
|
for entry in load_entries():
|
|
assert expected_recipe_link(entry) in recipe_cells[entry["model"]]
|
|
|
|
|
|
def test_registry_entries_have_explicit_recipe_precedence():
|
|
for entry in load_entries():
|
|
assert set(entry) >= {"model", "task", "examples", "recipe"}
|
|
assert isinstance(entry["examples"], list)
|
|
|
|
recipe = entry["recipe"]
|
|
status = recipe["status"]
|
|
assert status in VALID_STATUSES
|
|
assert set(recipe.get("hardware", [])) <= VALID_HARDWARE
|
|
|
|
if status == "published":
|
|
assert recipe["published_url"].startswith("https://recipes.vllm.ai/")
|
|
elif status == "repository":
|
|
assert recipe["repository_path"].startswith("recipes/")
|
|
assert "published_url" not in recipe
|
|
else:
|
|
assert "published_url" not in recipe
|
|
assert "repository_path" not in recipe
|
|
|
|
paths = [*entry["examples"], recipe.get("repository_path")]
|
|
for relative_path in filter(None, paths):
|
|
path = Path(relative_path)
|
|
assert not path.is_absolute()
|
|
assert (ROOT_DIR / path).is_file(), relative_path
|
|
|
|
|
|
def test_supported_models_render_recipe_hardware():
|
|
hardware_cells = supported_model_hardware_cells()
|
|
|
|
for entry in load_entries():
|
|
recipe_hardware = set(entry["recipe"].get("hardware", []))
|
|
cells = hardware_cells[entry["model"]]
|
|
for hardware, cell in zip(HARDWARE_COLUMNS, cells, strict=True):
|
|
assert (cell == "✅︎") == (hardware in recipe_hardware)
|