62 lines
2.0 KiB
Python
62 lines
2.0 KiB
Python
"""Stability pytest hooks and fixtures."""
|
|
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
|
|
import pytest
|
|
|
|
from tests.dfx.stability.helpers import (
|
|
finalize_resource_monitor,
|
|
report_latest_gpu_samples,
|
|
start_resource_monitor,
|
|
wait_for_run_dir,
|
|
)
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def stability_resource_monitor_per_test(request: pytest.FixtureRequest):
|
|
"""
|
|
For each test under this directory: start GPU monitor before the test,
|
|
then finalize after the test so this case gets its own report.html.
|
|
"""
|
|
proc = start_resource_monitor()
|
|
stop_event = threading.Event()
|
|
reporter: threading.Thread | None = None
|
|
|
|
if proc is not None:
|
|
reporter = threading.Thread(
|
|
target=report_latest_gpu_samples,
|
|
args=(stop_event,),
|
|
name="stability-resource-monitor-reporter",
|
|
daemon=True,
|
|
)
|
|
reporter.start()
|
|
run_dir = wait_for_run_dir(timeout_sec=5)
|
|
node_name = request.node.name
|
|
if run_dir is not None:
|
|
sys.stderr.write(f"[Stability] Resource monitor started for test: {node_name} | run dir: {run_dir}\n")
|
|
else:
|
|
sys.stderr.write(f"[Stability] Resource monitor started for test: {node_name} (run dir not ready yet)\n")
|
|
|
|
yield
|
|
|
|
# Teardown: stop reporter, stop monitor, finalize → one HTML per test
|
|
if proc is not None:
|
|
stop_event.set()
|
|
if reporter is not None and reporter.is_alive():
|
|
reporter.join(timeout=2)
|
|
if proc.poll() is None:
|
|
proc.terminate()
|
|
try:
|
|
proc.wait(timeout=10)
|
|
except subprocess.TimeoutExpired:
|
|
proc.kill()
|
|
proc.wait()
|
|
bundle_dir = finalize_resource_monitor()
|
|
node_name = request.node.name
|
|
if bundle_dir:
|
|
sys.stderr.write(f"[Stability] Report for test «{node_name}»: {bundle_dir}/report.html\n")
|
|
else:
|
|
sys.stderr.write(f"[Stability] Finalize skipped or failed for test «{node_name}»\n")
|