[CI] Add scripts for bechmark collection and email distribution. (#1307)
Signed-off-by: Alicia <115451386+congw729@users.noreply.github.com>
This commit is contained in:
1 parent
3b2b5012eb
commit
e7f2c88701
5 files changed
+704
-13
No files matched your search
@@ -1,5 +1,5 @@
|
||||
steps:
|
||||
- label: "Omni Model Test with H100"
|
||||
- label: ":full_moon: Omni Model Test with H100"
|
||||
timeout_in_minutes: 90
|
||||
depends_on: upload-nightly-pipeline
|
||||
if: build.env("NIGHTLY") == "1"
|
||||
@@ -41,7 +41,7 @@ steps:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: "Omni Model Test"
|
||||
- label: ":full_moon: Omni Model Test"
|
||||
timeout_in_minutes: 60
|
||||
depends_on: upload-nightly-pipeline
|
||||
if: build.env("NIGHTLY") == "1"
|
||||
@@ -56,18 +56,22 @@ steps:
|
||||
always-pull: true
|
||||
shm-size: "8gb"
|
||||
propagate-environment: true
|
||||
shm-size: "8gb"
|
||||
environment:
|
||||
- "HF_HOME=/fsx/hf_cache"
|
||||
volumes:
|
||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||
|
||||
- label: "Omni Model Perf Test"
|
||||
timeout_in_minutes: 120
|
||||
- label: ":full_moon: Omni Model Perf Test with H100"
|
||||
key: nightly-performance
|
||||
timeout_in_minutes: 180
|
||||
depends_on: upload-nightly-pipeline
|
||||
if: build.env("NIGHTLY") == "1"
|
||||
commands:
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- export BENCHMARK_DIR=tests
|
||||
- pytest -s -v tests/perf/scripts/run_benchmark.py
|
||||
- buildkite-agent artifact upload "tests/*.json"
|
||||
agents:
|
||||
queue: "mithril-h100-pool"
|
||||
plugins:
|
||||
@@ -96,3 +100,18 @@ steps:
|
||||
hostPath:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: ":email: Nightly Perf Collection & Email"
|
||||
key: nightly-perf-distribution
|
||||
depends_on: nightly-performance
|
||||
if: build.env("NIGHTLY") == "1"
|
||||
commands:
|
||||
- pip install openpyxl
|
||||
- export DEFAULT_INPUT_DIR=tests
|
||||
- export DEFAULT_OUTPUT_DIR=tests
|
||||
- buildkite-agent artifact download "tests/*.json" . --step nightly-performance
|
||||
- python tools/nightly/generate_nightly_perf_excel.py
|
||||
- python tools/nightly/send_nightly_perf_email.py
|
||||
- buildkite-agent artifact upload "tests/*.xlsx"
|
||||
agents:
|
||||
queue: "cpu_queue_premerge"
|
||||
Reference in new issue
Block a user