Extend Cosmos and TTS component checks to S24 and RB3

This commit is contained in:
tzhouam committed 2026-09-24 09:51:08 +08:00
1 parent 2ff0a783d4
commit 05210fb2db
25 files changed
+468 -11

No files matched your search

@@ -0,0 +1,87 @@
#!/usr/bin/env python3
"""Join a complete sustained TTS request run with device-wide GPU telemetry."""
from __future__ import annotations
import argparse
import json
from collections import Counter
from pathlib import Path
from analyze_tts_sustained import WINDOWS_MIN, nearest_rank, summarize
METRICS = (
"gpu_temperature_c",
"gpu_power_mw",
"gpu_clock_sm_mhz",
"gpu_clock_memory_mhz",
"gpu_utilization_pct",
"gpu_memory_used_bytes",
"host_available_bytes",
"host_cpu_utilization_pct",
)
def analyze(report_path: Path, request_path: Path, telemetry_path: Path) -> dict:
report = json.loads(report_path.read_text(encoding="utf-8"))
if report.get("status") != "completed" or report.get("sustained_wall_s", 0) < 1800:
raise ValueError("A terminal 30-minute sustained report is required")
if report.get("gpu_telemetry", {}).get("error"):
raise ValueError("GPU telemetry reported an error")
if report.get("gpu_telemetry", {}).get("samples", 0) < 1500:
raise ValueError("Too few device telemetry samples for a 30-minute run")
start = report["sustained_start_unix"]
base = summarize(request_path)
telemetry = [json.loads(line) for line in telemetry_path.open(encoding="utf-8")]
if len(telemetry) != report["gpu_telemetry"]["samples"]:
raise ValueError("Telemetry sample count differs from the final report")
for (lo, hi), window in zip(WINDOWS_MIN, base["windows"], strict=True):
samples = [row for row in telemetry if lo * 60 <= row["unix"] - start < hi * 60]
window["device_telemetry_samples"] = len(samples)
window["device_telemetry"] = {}
for name in METRICS:
values = [row[name] for row in samples if row.get(name) is not None]
window["device_telemetry"][name] = {
"p50": nearest_rank(values, 0.5),
"p95": nearest_rank(values, 0.95),
"min": min(values) if values else None,
"max": max(values) if values else None,
"available_samples": len(values),
}
window["gpu_throttle_reason_counts"] = dict(sorted(Counter(
str(row["gpu_throttle_reasons"]) for row in samples
).items()))
return {
"scope": "complete-model Qwen3-TTS stream; simulated playback; telemetry covers whole GPU/host, not just this process",
"report": str(report_path),
"requests": str(request_path),
"telemetry": str(telemetry_path),
"loaded_omni_path": report.get("loaded_omni_path"),
"cwd": report.get("cwd"),
"start_unix": start,
"sustained_wall_s": report["sustained_wall_s"],
"windows": base["windows"],
}
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--run-dir", required=True, type=Path)
parser.add_argument("--out", type=Path)
args = parser.parse_args()
result = analyze(
args.run_dir / "report.json",
args.run_dir / "requests.jsonl",
args.run_dir / "gpu_telemetry.jsonl",
)
output = json.dumps(result, indent=2) + "\n"
if args.out:
args.out.parent.mkdir(parents=True, exist_ok=True)
args.out.write_text(output, encoding="utf-8")
else:
print(output, end="")
if __name__ == "__main__":
main()
@@ -33,6 +33,8 @@ def main() -> None:
parser.add_argument(f"--{name}", type=Path, required=True)
parser.add_argument("--device-name", default="Samsung Galaxy S25")
parser.add_argument("--profile-report", type=Path)
parser.add_argument("--expected-compile-options", default=COMPILE_OPTIONS)
parser.add_argument("--expected-run-options", default="--compute_unit npu")
args = parser.parse_args()
compiled = json.loads(args.compile_report.read_text(encoding="utf-8"))
@@ -40,7 +42,8 @@ def main() -> None:
inferred = json.loads(args.inference_report.read_text(encoding="utf-8"))
if compiled.get("status") != "SUCCESS" or inferred.get("status") != "SUCCESS":
raise ValueError("compile and inference must be terminal successes")
if compiled.get("source_model_id") != "mn4okj3rq" or compiled.get("options") != COMPILE_OPTIONS:
if (compiled.get("source_model_id") != "mn4okj3rq"
or compiled.get("options") != args.expected_compile_options):
raise ValueError("compile used a different source or options")
if inferred.get("model_id") != compiled.get("target_model_id"):
raise ValueError("inference used a different target model")
@@ -48,8 +51,8 @@ def main() -> None:
raise ValueError("compile/inference device metadata differs")
if inferred.get("device", {}).get("name") != args.device_name:
raise ValueError("inference did not use the requested exact device")
if inferred.get("options") != "--compute_unit npu":
raise ValueError("inference did not request NPU")
if inferred.get("options") != args.expected_run_options:
raise ValueError("inference did not request the expected compute unit")
if (inferred.get("job_id") != submitted.get("job_id")
or inferred.get("input_dataset_id") != submitted.get("input_dataset_id")):
raise ValueError("inference used a different job or input dataset")
@@ -121,7 +124,7 @@ def main() -> None:
if (profiled.get("status") != "SUCCESS"
or profiled.get("model_id") != compiled["target_model_id"]
or profiled.get("device") != inferred["device"]
or profiled.get("options") != "--compute_unit npu"):
or profiled.get("options") != args.expected_run_options):
raise ValueError("profile did not execute the same target on the exact device")
profile = profiled["profile"]
summary = profile["execution_summary"]
@@ -47,9 +47,12 @@ def collect(evidence_dir: Path, kind: str, client: hub.Client) -> str:
"options": job.options,
"hub_version": str(job.hub_version),
"device": device_record(job.device),
"model_id": job.model.model_id,
}
if kind == "inference":
if kind == "compile":
report["source_model_id"] = job.model.model_id
report["target_model_id"] = job.get_target_model().model_id if status.code == "SUCCESS" else None
elif kind == "inference":
report["model_id"] = job.model.model_id
report["input_dataset_id"] = job.inputs.dataset_id
if status.code == "SUCCESS":
output_path = evidence_dir / "device_output.npz"
@@ -58,6 +61,7 @@ def collect(evidence_dir: Path, kind: str, client: hub.Client) -> str:
np.savez_compressed(output_path, **arrays)
report["outputs"] = output_path.name
elif kind == "profile":
report["model_id"] = job.model.model_id
report["shapes"] = job.shapes
if status.code == "SUCCESS":
report["profile"] = job.download_profile()
@@ -70,7 +74,7 @@ def collect(evidence_dir: Path, kind: str, client: hub.Client) -> str:
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("evidence_dirs", nargs="+", type=Path)
parser.add_argument("--kinds", nargs="+", choices=("inference", "profile"), default=("inference", "profile"))
parser.add_argument("--kinds", nargs="+", choices=("compile", "inference", "profile"), default=("inference", "profile"))
args = parser.parse_args()
client = hub.Client()
for directory in args.evidence_dirs: