Extend Cosmos and TTS component checks to S24 and RB3
This commit is contained in:
1 parent
2ff0a783d4
commit
05210fb2db
25 files changed
+468
-11
No files matched your search
@@ -0,0 +1,87 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Join a complete sustained TTS request run with device-wide GPU telemetry."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
from analyze_tts_sustained import WINDOWS_MIN, nearest_rank, summarize
|
||||
|
||||
|
||||
METRICS = (
|
||||
"gpu_temperature_c",
|
||||
"gpu_power_mw",
|
||||
"gpu_clock_sm_mhz",
|
||||
"gpu_clock_memory_mhz",
|
||||
"gpu_utilization_pct",
|
||||
"gpu_memory_used_bytes",
|
||||
"host_available_bytes",
|
||||
"host_cpu_utilization_pct",
|
||||
)
|
||||
|
||||
|
||||
def analyze(report_path: Path, request_path: Path, telemetry_path: Path) -> dict:
|
||||
report = json.loads(report_path.read_text(encoding="utf-8"))
|
||||
if report.get("status") != "completed" or report.get("sustained_wall_s", 0) < 1800:
|
||||
raise ValueError("A terminal 30-minute sustained report is required")
|
||||
if report.get("gpu_telemetry", {}).get("error"):
|
||||
raise ValueError("GPU telemetry reported an error")
|
||||
if report.get("gpu_telemetry", {}).get("samples", 0) < 1500:
|
||||
raise ValueError("Too few device telemetry samples for a 30-minute run")
|
||||
start = report["sustained_start_unix"]
|
||||
base = summarize(request_path)
|
||||
telemetry = [json.loads(line) for line in telemetry_path.open(encoding="utf-8")]
|
||||
if len(telemetry) != report["gpu_telemetry"]["samples"]:
|
||||
raise ValueError("Telemetry sample count differs from the final report")
|
||||
for (lo, hi), window in zip(WINDOWS_MIN, base["windows"], strict=True):
|
||||
samples = [row for row in telemetry if lo * 60 <= row["unix"] - start < hi * 60]
|
||||
window["device_telemetry_samples"] = len(samples)
|
||||
window["device_telemetry"] = {}
|
||||
for name in METRICS:
|
||||
values = [row[name] for row in samples if row.get(name) is not None]
|
||||
window["device_telemetry"][name] = {
|
||||
"p50": nearest_rank(values, 0.5),
|
||||
"p95": nearest_rank(values, 0.95),
|
||||
"min": min(values) if values else None,
|
||||
"max": max(values) if values else None,
|
||||
"available_samples": len(values),
|
||||
}
|
||||
window["gpu_throttle_reason_counts"] = dict(sorted(Counter(
|
||||
str(row["gpu_throttle_reasons"]) for row in samples
|
||||
).items()))
|
||||
return {
|
||||
"scope": "complete-model Qwen3-TTS stream; simulated playback; telemetry covers whole GPU/host, not just this process",
|
||||
"report": str(report_path),
|
||||
"requests": str(request_path),
|
||||
"telemetry": str(telemetry_path),
|
||||
"loaded_omni_path": report.get("loaded_omni_path"),
|
||||
"cwd": report.get("cwd"),
|
||||
"start_unix": start,
|
||||
"sustained_wall_s": report["sustained_wall_s"],
|
||||
"windows": base["windows"],
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--run-dir", required=True, type=Path)
|
||||
parser.add_argument("--out", type=Path)
|
||||
args = parser.parse_args()
|
||||
result = analyze(
|
||||
args.run_dir / "report.json",
|
||||
args.run_dir / "requests.jsonl",
|
||||
args.run_dir / "gpu_telemetry.jsonl",
|
||||
)
|
||||
output = json.dumps(result, indent=2) + "\n"
|
||||
if args.out:
|
||||
args.out.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.out.write_text(output, encoding="utf-8")
|
||||
else:
|
||||
print(output, end="")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -33,6 +33,8 @@ def main() -> None:
|
||||
parser.add_argument(f"--{name}", type=Path, required=True)
|
||||
parser.add_argument("--device-name", default="Samsung Galaxy S25")
|
||||
parser.add_argument("--profile-report", type=Path)
|
||||
parser.add_argument("--expected-compile-options", default=COMPILE_OPTIONS)
|
||||
parser.add_argument("--expected-run-options", default="--compute_unit npu")
|
||||
args = parser.parse_args()
|
||||
|
||||
compiled = json.loads(args.compile_report.read_text(encoding="utf-8"))
|
||||
@@ -40,7 +42,8 @@ def main() -> None:
|
||||
inferred = json.loads(args.inference_report.read_text(encoding="utf-8"))
|
||||
if compiled.get("status") != "SUCCESS" or inferred.get("status") != "SUCCESS":
|
||||
raise ValueError("compile and inference must be terminal successes")
|
||||
if compiled.get("source_model_id") != "mn4okj3rq" or compiled.get("options") != COMPILE_OPTIONS:
|
||||
if (compiled.get("source_model_id") != "mn4okj3rq"
|
||||
or compiled.get("options") != args.expected_compile_options):
|
||||
raise ValueError("compile used a different source or options")
|
||||
if inferred.get("model_id") != compiled.get("target_model_id"):
|
||||
raise ValueError("inference used a different target model")
|
||||
@@ -48,8 +51,8 @@ def main() -> None:
|
||||
raise ValueError("compile/inference device metadata differs")
|
||||
if inferred.get("device", {}).get("name") != args.device_name:
|
||||
raise ValueError("inference did not use the requested exact device")
|
||||
if inferred.get("options") != "--compute_unit npu":
|
||||
raise ValueError("inference did not request NPU")
|
||||
if inferred.get("options") != args.expected_run_options:
|
||||
raise ValueError("inference did not request the expected compute unit")
|
||||
if (inferred.get("job_id") != submitted.get("job_id")
|
||||
or inferred.get("input_dataset_id") != submitted.get("input_dataset_id")):
|
||||
raise ValueError("inference used a different job or input dataset")
|
||||
@@ -121,7 +124,7 @@ def main() -> None:
|
||||
if (profiled.get("status") != "SUCCESS"
|
||||
or profiled.get("model_id") != compiled["target_model_id"]
|
||||
or profiled.get("device") != inferred["device"]
|
||||
or profiled.get("options") != "--compute_unit npu"):
|
||||
or profiled.get("options") != args.expected_run_options):
|
||||
raise ValueError("profile did not execute the same target on the exact device")
|
||||
profile = profiled["profile"]
|
||||
summary = profile["execution_summary"]
|
||||
|
||||
@@ -47,9 +47,12 @@ def collect(evidence_dir: Path, kind: str, client: hub.Client) -> str:
|
||||
"options": job.options,
|
||||
"hub_version": str(job.hub_version),
|
||||
"device": device_record(job.device),
|
||||
"model_id": job.model.model_id,
|
||||
}
|
||||
if kind == "inference":
|
||||
if kind == "compile":
|
||||
report["source_model_id"] = job.model.model_id
|
||||
report["target_model_id"] = job.get_target_model().model_id if status.code == "SUCCESS" else None
|
||||
elif kind == "inference":
|
||||
report["model_id"] = job.model.model_id
|
||||
report["input_dataset_id"] = job.inputs.dataset_id
|
||||
if status.code == "SUCCESS":
|
||||
output_path = evidence_dir / "device_output.npz"
|
||||
@@ -58,6 +61,7 @@ def collect(evidence_dir: Path, kind: str, client: hub.Client) -> str:
|
||||
np.savez_compressed(output_path, **arrays)
|
||||
report["outputs"] = output_path.name
|
||||
elif kind == "profile":
|
||||
report["model_id"] = job.model.model_id
|
||||
report["shapes"] = job.shapes
|
||||
if status.code == "SUCCESS":
|
||||
report["profile"] = job.download_profile()
|
||||
@@ -70,7 +74,7 @@ def collect(evidence_dir: Path, kind: str, client: hub.Client) -> str:
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("evidence_dirs", nargs="+", type=Path)
|
||||
parser.add_argument("--kinds", nargs="+", choices=("inference", "profile"), default=("inference", "profile"))
|
||||
parser.add_argument("--kinds", nargs="+", choices=("compile", "inference", "profile"), default=("inference", "profile"))
|
||||
args = parser.parse_args()
|
||||
client = hub.Client()
|
||||
for directory in args.evidence_dirs:
|
||||
|
||||
Reference in new issue
Block a user