docs: preserve quantization URLs and recipe coverage
Signed-off-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
c588208cc6
commit
d10bf78c60
6 files changed
+499
-90
No files matched your search
@@ -3,8 +3,10 @@
|
||||
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
@@ -43,3 +45,113 @@ def test_model_display_names_require_string_mapping(tmp_path, monkeypatch):
|
||||
|
||||
with pytest.raises(ValueError, match="MODEL_DISPLAY_NAMES_FILE"):
|
||||
generate_examples.load_model_display_names()
|
||||
|
||||
|
||||
def test_quantization_examples_keep_source_urls_under_feature_navigation(tmp_path, monkeypatch):
|
||||
nav_file = tmp_path / "docs" / ".nav.yml"
|
||||
nav_file.parent.mkdir(parents=True)
|
||||
nav_file.write_text(
|
||||
yaml.safe_dump(
|
||||
{
|
||||
"nav": [
|
||||
{
|
||||
"User Guide": [
|
||||
{
|
||||
"Examples": [
|
||||
"examples/README.md",
|
||||
{"Offline Inference": [{"Old": "user_guide/examples/offline_inference/old.md"}]},
|
||||
]
|
||||
},
|
||||
{"Features": [{"Quantization": [{"Overview": "user_guide/quantization/overview.md"}]}]},
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
sort_keys=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(generate_examples, "ROOT_DIR", tmp_path)
|
||||
monkeypatch.setattr(generate_examples, "NAV_FILE", nav_file)
|
||||
monkeypatch.setattr(generate_examples, "EXAMPLE_DOC_DIR", tmp_path / "docs/user_guide/examples")
|
||||
|
||||
quantization_example = SimpleNamespace(
|
||||
category="quantization",
|
||||
path=tmp_path / "examples/quantization/check_modelopt_fp8_export.py",
|
||||
nav_title="Check Modelopt FP8 Export",
|
||||
)
|
||||
offline_example = SimpleNamespace(
|
||||
category="offline_inference",
|
||||
path=tmp_path / "examples/offline_inference/demo.py",
|
||||
nav_title="Demo",
|
||||
)
|
||||
|
||||
generate_examples.update_nav_file([quantization_example, offline_example])
|
||||
generate_examples.update_nav_file([quantization_example, offline_example])
|
||||
|
||||
nav_data = yaml.safe_load(nav_file.read_text(encoding="utf-8"))
|
||||
user_guide = nav_data["nav"][0]["User Guide"]
|
||||
examples = next(item["Examples"] for item in user_guide if "Examples" in item)
|
||||
features = next(item["Features"] for item in user_guide if "Features" in item)
|
||||
quantization = next(item["Quantization"] for item in features if "Quantization" in item)
|
||||
|
||||
assert not any(isinstance(item, dict) and "Quantization" in item for item in examples)
|
||||
generated_examples = [
|
||||
item[generate_examples.GENERATED_QUANTIZATION_EXAMPLES_TITLE]
|
||||
for item in quantization
|
||||
if generate_examples.GENERATED_QUANTIZATION_EXAMPLES_TITLE in item
|
||||
]
|
||||
assert len(generated_examples) == 1
|
||||
assert generated_examples[0] == [
|
||||
{"Check Modelopt FP8 Export": "user_guide/examples/quantization/check_modelopt_fp8_export.md"}
|
||||
]
|
||||
|
||||
# A later build without quantization examples must not leave a stale
|
||||
# generated group in the hand-authored Quantization section.
|
||||
generate_examples.update_nav_file([offline_example])
|
||||
nav_data = yaml.safe_load(nav_file.read_text(encoding="utf-8"))
|
||||
user_guide = nav_data["nav"][0]["User Guide"]
|
||||
features = next(item["Features"] for item in user_guide if "Features" in item)
|
||||
quantization = next(item["Quantization"] for item in features if "Quantization" in item)
|
||||
assert not any(
|
||||
isinstance(item, dict) and generate_examples.GENERATED_QUANTIZATION_EXAMPLES_TITLE in item
|
||||
for item in quantization
|
||||
)
|
||||
|
||||
|
||||
def test_quantization_examples_are_not_added_to_examples_without_feature_target(tmp_path, monkeypatch, caplog):
|
||||
nav_file = tmp_path / "docs" / ".nav.yml"
|
||||
nav_file.parent.mkdir(parents=True)
|
||||
nav_file.write_text(
|
||||
yaml.safe_dump(
|
||||
{
|
||||
"nav": [
|
||||
{
|
||||
"User Guide": [
|
||||
{"Examples": ["examples/README.md"]},
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
sort_keys=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(generate_examples, "ROOT_DIR", tmp_path)
|
||||
monkeypatch.setattr(generate_examples, "NAV_FILE", nav_file)
|
||||
monkeypatch.setattr(generate_examples, "EXAMPLE_DOC_DIR", tmp_path / "docs/user_guide/examples")
|
||||
|
||||
quantization_example = SimpleNamespace(
|
||||
category="quantization",
|
||||
path=tmp_path / "examples/quantization/check_modelopt_fp8_export.py",
|
||||
nav_title="Check Modelopt FP8 Export",
|
||||
)
|
||||
|
||||
generate_examples.update_nav_file([quantization_example])
|
||||
|
||||
nav_data = yaml.safe_load(nav_file.read_text(encoding="utf-8"))
|
||||
examples = nav_data["nav"][0]["User Guide"][0]["Examples"]
|
||||
assert not any(isinstance(item, dict) and "Quantization" in item for item in examples)
|
||||
assert "omitting generated quantization examples from navigation" in caplog.text
|
||||
@@ -0,0 +1,142 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
ROOT_DIR = Path(__file__).parents[2]
|
||||
COVERAGE_PATH = ROOT_DIR / "docs/models/recipe_coverage.yaml"
|
||||
SUPPORTED_MODELS_PATH = ROOT_DIR / "docs/models/supported_models.md"
|
||||
VALID_STATUSES = {"published", "repository", "no_validated_recipe_yet"}
|
||||
VALID_HARDWARE = {"nvidia", "amd", "ascend", "intel"}
|
||||
HARDWARE_COLUMNS = {
|
||||
"nvidia": 4,
|
||||
"amd": 5,
|
||||
"ascend": 6,
|
||||
"intel": 7,
|
||||
}
|
||||
|
||||
|
||||
def load_entries() -> list[dict]:
|
||||
data = yaml.safe_load(COVERAGE_PATH.read_text(encoding="utf-8"))
|
||||
assert data["version"] == 1
|
||||
assert isinstance(data["entries"], list)
|
||||
return data["entries"]
|
||||
|
||||
|
||||
def supported_model_ids() -> set[str]:
|
||||
model_ids = set()
|
||||
for line in SUPPORTED_MODELS_PATH.read_text(encoding="utf-8").splitlines():
|
||||
if not line.startswith("|"):
|
||||
continue
|
||||
columns = line.split("|")
|
||||
if len(columns) <= 3:
|
||||
continue
|
||||
model_ids.update(re.findall(r"`([^`]+/[^`]+)`", columns[3]))
|
||||
return model_ids
|
||||
|
||||
|
||||
def supported_model_recipe_cells() -> dict[str, str]:
|
||||
recipe_cells = {}
|
||||
in_table = False
|
||||
for line in SUPPORTED_MODELS_PATH.read_text(encoding="utf-8").splitlines():
|
||||
if line.startswith("| Architecture |"):
|
||||
in_table = True
|
||||
continue
|
||||
if in_table and not line.startswith("|"):
|
||||
break
|
||||
if not in_table:
|
||||
continue
|
||||
|
||||
columns = line.split("|")
|
||||
if len(columns) <= 8 or columns[1].strip().startswith("-"):
|
||||
continue
|
||||
for model_id in re.findall(r"`([^`]+/[^`]+)`", columns[3]):
|
||||
recipe_cells[model_id] = columns[8].strip()
|
||||
return recipe_cells
|
||||
|
||||
|
||||
def expected_recipe_link(entry: dict) -> str:
|
||||
recipe = entry["recipe"]
|
||||
if recipe["status"] == "published":
|
||||
return recipe["published_url"]
|
||||
if recipe["status"] == "repository":
|
||||
return f"https://github.com/vllm-project/vllm-omni/blob/main/{recipe['repository_path']}"
|
||||
return "No validated recipe yet"
|
||||
|
||||
|
||||
def supported_model_hardware_cells() -> dict[str, list[str]]:
|
||||
hardware_cells = {}
|
||||
in_table = False
|
||||
for line in SUPPORTED_MODELS_PATH.read_text(encoding="utf-8").splitlines():
|
||||
if line.startswith("| Architecture |"):
|
||||
in_table = True
|
||||
continue
|
||||
if in_table and not line.startswith("|"):
|
||||
break
|
||||
if not in_table:
|
||||
continue
|
||||
|
||||
columns = line.split("|")
|
||||
if len(columns) <= 8 or columns[1].strip().startswith("-"):
|
||||
continue
|
||||
cells = [columns[index].strip() for index in HARDWARE_COLUMNS.values()]
|
||||
for model_id in re.findall(r"`([^`]+/[^`]+)`", columns[3]):
|
||||
hardware_cells[model_id] = cells
|
||||
return hardware_cells
|
||||
|
||||
|
||||
def test_registry_keys_are_exact_supported_checkpoints():
|
||||
entries = load_entries()
|
||||
keys = [(entry["model"], entry["task"]) for entry in entries]
|
||||
|
||||
assert len(keys) == len(set(keys))
|
||||
assert all(entry["model"] in supported_model_ids() for entry in entries)
|
||||
|
||||
|
||||
def test_supported_models_render_registry_recipe_links():
|
||||
recipe_cells = supported_model_recipe_cells()
|
||||
|
||||
for entry in load_entries():
|
||||
assert expected_recipe_link(entry) in recipe_cells[entry["model"]]
|
||||
|
||||
|
||||
def test_registry_entries_have_explicit_recipe_precedence():
|
||||
for entry in load_entries():
|
||||
assert set(entry) >= {"model", "task", "examples", "recipe"}
|
||||
assert isinstance(entry["examples"], list)
|
||||
|
||||
recipe = entry["recipe"]
|
||||
status = recipe["status"]
|
||||
assert status in VALID_STATUSES
|
||||
assert set(recipe.get("hardware", [])) <= VALID_HARDWARE
|
||||
|
||||
if status == "published":
|
||||
assert recipe["published_url"].startswith("https://recipes.vllm.ai/")
|
||||
elif status == "repository":
|
||||
assert recipe["repository_path"].startswith("recipes/")
|
||||
assert "published_url" not in recipe
|
||||
else:
|
||||
assert "published_url" not in recipe
|
||||
assert "repository_path" not in recipe
|
||||
|
||||
paths = [*entry["examples"], recipe.get("repository_path")]
|
||||
for relative_path in filter(None, paths):
|
||||
path = Path(relative_path)
|
||||
assert not path.is_absolute()
|
||||
assert (ROOT_DIR / path).is_file(), relative_path
|
||||
|
||||
|
||||
def test_supported_models_render_recipe_hardware():
|
||||
hardware_cells = supported_model_hardware_cells()
|
||||
|
||||
for entry in load_entries():
|
||||
recipe_hardware = set(entry["recipe"].get("hardware", []))
|
||||
cells = hardware_cells[entry["model"]]
|
||||
for hardware, cell in zip(HARDWARE_COLUMNS, cells, strict=True):
|
||||
assert (cell == "✅︎") == (hardware in recipe_hardware)
|
||||
Reference in new issue
Block a user