admin
e778d21f91
quantize_dit_int8wo.py builds the offline W8A16 artifact (per-channel symmetric int8, pack-quantized via compressed-tensors' own packer, per-component config wrapper so encoder/VAE stay BF16, digest manifest); serve.sh/probe.py/gate.py/psnr.py adapt the issue-2 kit for the structure-gated capacity matrix (1-4 refs x seeds 2301-2304).
34 lines
1.2 KiB
Bash
34 lines
1.2 KiB
Bash
#!/usr/bin/env bash
|
|
# Issue 5 probe: serve the int8 weight-only compressed-tensors artifact on
|
|
# the RTX 3090 with one labeled flag set. VRAM is polled once per second for
|
|
# the whole server lifetime, matching issue 2's instrument (nvidia-smi csv
|
|
# series). The default flag set is cpu-offload + eager; pass extras after
|
|
# the label to override/extend (they are appended, vLLM last-one-wins).
|
|
#
|
|
# Usage: serve.sh LABEL [extra vllm flags...]
|
|
set -uo pipefail
|
|
|
|
BASE=$HOME/Projects/vllm-omni-val
|
|
W=$BASE/issue5
|
|
LABEL=$1; shift
|
|
RUN=$W/runs/$LABEL
|
|
mkdir -p "$RUN"
|
|
|
|
STATE=$HOME/.local/state/multi-model-helper-server/state/image-service
|
|
ARTIFACT=${ARTIFACT:-$STATE/artifacts/qwen-image-2.1-dit-int8wo}
|
|
export HF_HOME=$STATE/weights
|
|
export HF_HUB_OFFLINE=1
|
|
|
|
( while sleep 1; do nvidia-smi --query-gpu=memory.used,utilization.gpu --format=csv,noheader; done ) > "$RUN/vram.log" 2>&1 &
|
|
POLL=$!
|
|
T0=$(date +%s)
|
|
echo "model: $ARTIFACT" > "$RUN/flags.txt"
|
|
echo "flags: $*" >> "$RUN/flags.txt"
|
|
|
|
$BASE/venv/bin/vllm serve "$ARTIFACT" --omni --port 8091 \
|
|
--served-model-name Qwen/Qwen-Image-2.1 \
|
|
--enforce-eager --enable-cpu-offload "$@" > "$RUN/serve.log" 2>&1
|
|
RC=$?
|
|
kill $POLL 2>/dev/null
|
|
echo "serve_rc=$RC wall=$(( $(date +%s) - T0 ))s" >> "$RUN/serve.log"
|