From e778d21f9125300eb2a33e69fd18f7d62a207169 Mon Sep 17 00:00:00 2001 From: Kyle Merritt Date: Thu, 8 Oct 2026 07:00:16 -0400 Subject: [PATCH] probe: int8 weight-only compressed-tensors DiT artifact kit (issue 5) quantize_dit_int8wo.py builds the offline W8A16 artifact (per-channel symmetric int8, pack-quantized via compressed-tensors' own packer, per-component config wrapper so encoder/VAE stay BF16, digest manifest); serve.sh/probe.py/gate.py/psnr.py adapt the issue-2 kit for the structure-gated capacity matrix (1-4 refs x seeds 2301-2304). --- probe/issue-5/gate.py | 94 ++++++++++ probe/issue-5/probe.py | 181 ++++++++++++++++++++ probe/issue-5/psnr.py | 98 +++++++++++ probe/issue-5/quantize_dit_int8wo.py | 208 +++++++++++++++++++++++ probe/issue-5/serve.sh | 33 ++++ tests/quantization/test_quant_factory.py | 11 +- 6 files changed, 622 insertions(+), 3 deletions(-) create mode 100644 probe/issue-5/gate.py create mode 100644 probe/issue-5/probe.py create mode 100644 probe/issue-5/psnr.py create mode 100644 probe/issue-5/quantize_dit_int8wo.py create mode 100644 probe/issue-5/serve.sh diff --git a/probe/issue-5/gate.py b/probe/issue-5/gate.py new file mode 100644 index 000000000..e4d4d106d --- /dev/null +++ b/probe/issue-5/gate.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python +"""Issue 5 structure gate + capacity summarizer. + +Applies the gate rule from issue 1's FP8 misread to every recorded request +and folds a run directory into the capacity matrix row: per reference +count, wall time, peak VRAM, and the server-reported peak. + +Gate rule (a run only counts when structurally sound): + opaque_fraction >= 0.99 AND mean(rgb_std) >= 10 + +Broken FP8 output measured opaque 0.008-0.035 with flat grey RGB (std of a +few); the BF16 path is 100% opaque with healthy channel spread. + +Usage: gate.py RUN_DIR [RUN_DIR ...] # reads matrix.json / matrix-*.json +""" + +from __future__ import annotations + +import argparse +import json +import pathlib +import re + + +def peak_vram_mb(vram_log: pathlib.Path) -> int | None: + if not vram_log.exists(): + return None + peak = 0 + for line in vram_log.read_text().splitlines(): + match = re.match(r"\s*(\d+)\s+MiB\s*,", line) or re.match(r"\s*(\d+),", line) + if match: + peak = max(peak, int(match.group(1))) + return peak or None + + +def server_peak_mb(serve_log: pathlib.Path) -> int | None: + if not serve_log.exists(): + return None + peaks = [int(m) for m in re.findall(r"peak.*?(\d[\d,]+)\s*MB", serve_log.read_text())] + if not peaks: + return None + return max(peaks) + + +def gate(structure: dict) -> bool: + return ( + structure["opaque_fraction"] >= 0.99 + and sum(structure["rgb_std"]) / len(structure["rgb_std"]) >= 10 + ) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("runs", nargs="+", type=pathlib.Path) + args = parser.parse_args() + + for run_dir in args.runs: + records: list[dict] = [] + for candidate in sorted(run_dir.glob("matrix*.json")): + records.extend(json.loads(candidate.read_text())) + by_refs: dict[int, list[dict]] = {} + for rec in records: + by_refs.setdefault(rec.get("n_references", 0), []).append(rec) + + print(f"== {run_dir.name} ==") + all_pass = bool(records) + for n_refs in sorted(by_refs): + rows = by_refs[n_refs] + walls = [r["wall_s"] for r in rows if "wall_s" in r] + gates = [gate(r["structure"]) for r in rows if "structure" in r] + row_pass = bool(rows) and all(gates) and len(gates) == len(rows) + all_pass = all_pass and row_pass + print( + f" {n_refs}-ref: n={len(rows)} gate={'PASS' if row_pass else 'FAIL'} " + f"wall {min(walls):.1f}-{max(walls):.1f}s" + + (f" (mean {sum(walls) / len(walls):.1f}s)" if walls else "") + ) + for rec in rows: + s = rec.get("structure") + tag = "" if s is None else (" ok" if gate(s) else " BROKEN") + err = f" error={rec['error'][:80]}" if rec.get("error") else "" + print( + f" {rec.get('fixture_id')} s{rec.get('seed')}: " + f"{rec.get('wall_s')}s {json.dumps(s) if s else 'no-image'}{tag}{err}" + ) + peak = peak_vram_mb(run_dir / "vram.log") + server_peak = server_peak_mb(run_dir / "serve.log") + print(f" nvidia-smi peak: {peak} MiB; server peak: {server_peak} MB") + print(f" STRUCTURE GATE: {'PASS' if all_pass else 'FAIL'}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/probe/issue-5/probe.py b/probe/issue-5/probe.py new file mode 100644 index 000000000..44e8e773c --- /dev/null +++ b/probe/issue-5/probe.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python +"""Issue 5 probe client: reference-conditioned requests against the int8 +weight-only compressed-tensors artifact, with the structure gate applied to +every response. + +Matrix pass (the issue's capacity matrix): one-, two-, three-, and +four-reference fixtures at seeds 2301-2304 — wall time and per-image +structure metrics land in the record for every request. Every request +mirrors the sealed Diffusers baseline attempt 2026-10-07T170729Z-q3-4r +(same frozen references, same rendered prompt, 20 steps, true_cfg_scale +1.0, fixture resolution, no negative prompt), submitted one at a time +through POST /v1/images/edits. + +The structure gate (the lesson from issue 1's FP8 misread): a run only +counts when the output is structurally sound — fully opaque RGBA and +non-flat RGB. Broken FP8 output measured ~96.5-97% transparent with +near-flat grey RGB; the BF16 path is 100% opaque. The gate numbers are +recorded on every request regardless; `gate.py` summarizes them. + +Usage: probe.py --out RUN_DIR [--matrix] [--host H:P] +""" + +from __future__ import annotations + +import argparse +import base64 +import hashlib +import io +import json +import pathlib +import time + +import numpy as np +import requests +from PIL import Image + +W = pathlib.Path.home() / "Projects/vllm-omni-val/issue5" +SUITE = ( + pathlib.Path.home() + / ".local/share/multi-model-helper-server/data/evaluation/media/image/references-suite-2" +) +STEPS = 20 +CFG = 1.0 +MATRIX_FIXTURES = ( + "edit-expression-ysolde", # 1 reference + "edit-compose-pair", # 2 references + "edit-compose-three", # 3 references (derived; see fixtures.json) + "edit-compose-four", # 4 references +) +MATRIX_SEEDS = (2301, 2302, 2303, 2304) + + +def structure_metrics(png: bytes) -> dict: + """Opacity + non-flatness numbers for the structure gate.""" + image = Image.open(io.BytesIO(png)) + array = np.asarray(image.convert("RGBA")) + alpha = array[..., 3] + rgb = array[..., :3].astype(np.float64) + return { + "opaque_fraction": round(float((alpha == 255).mean()), 4), + "alpha_mean": round(float(alpha.mean()), 1), + "rgb_std": [round(float(rgb[..., i].std()), 1) for i in range(3)], + } + + +def submit(host: str, fixture: dict, seed: int) -> tuple[dict, bytes | None]: + files = [] + handles = [] + for ref in fixture["references"]: + path = SUITE / f"{ref}.png" + handle = open(path, "rb") + handles.append(handle) + files.append(("image[]", (path.name, handle, "image/png"))) + data = { + "model": "Qwen/Qwen-Image-2.1", + "prompt": fixture["prompt"], + "seed": str(seed), + "num_inference_steps": str(STEPS), + "true_cfg_scale": str(CFG), + "size": f"{fixture['width']}x{fixture['height']}", + "n": "1", + "response_format": "b64_json", + "output_format": "png", + "return_stage_metrics": "true", + } + rec: dict = { + "fixture_id": None, + "seed": seed, + "n_references": len(fixture["references"]), + "steps": STEPS, + "true_cfg_scale": CFG, + "size": f"{fixture['width']}x{fixture['height']}", + } + t0 = time.time() + try: + response = requests.post( + f"http://{host}/v1/images/edits", data=data, files=files, timeout=1800 + ) + rec["wall_s"] = round(time.time() - t0, 2) + rec["http"] = response.status_code + if response.status_code == 200: + body = response.json() + png = base64.b64decode(body["data"][0]["b64_json"]) + rec["bytes"] = len(png) + rec["sha256"] = hashlib.sha256(png).hexdigest() + rec["structure"] = structure_metrics(png) + rec["usage"] = body.get("usage") + for key in ("stage_metrics", "metrics", "timings"): + if body.get(key): + rec[key] = body[key] + return rec, png + rec["error"] = response.text[:2000] + return rec, None + except Exception as exc: # noqa: BLE001 - recorded as the probe result + rec["wall_s"] = round(time.time() - t0, 2) + rec["error"] = f"{type(exc).__name__}: {exc}" + return rec, None + finally: + for handle in handles: + handle.close() + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--out", required=True, type=pathlib.Path) + parser.add_argument("--host", default="127.0.0.1:8091") + parser.add_argument("--matrix", action="store_true") + parser.add_argument( + "--fixture", + default="edit-compose-four", + help="single fixture id for --single", + ) + parser.add_argument("--single", action="store_true", help="one fixture, all seeds") + parser.add_argument( + "--seeds", + type=lambda s: tuple(int(x) for x in s.split(",")), + default=MATRIX_SEEDS, + ) + args = parser.parse_args() + + fixtures = json.loads((W / "fixtures.json").read_text()) + args.out.mkdir(parents=True, exist_ok=True) + + plan: list[tuple[str, pathlib.Path, int]] = [] + if args.matrix: + for fixture_id in MATRIX_FIXTURES: + for seed in args.seeds: + plan.append( + ( + fixture_id, + args.out / "matrix" / f"{fixture_id}.s{seed}.png", + seed, + ) + ) + if args.single: + for seed in args.seeds: + plan.append( + (args.fixture, args.out / "matrix" / f"{args.fixture}.s{seed}.png", seed) + ) + + records = [] + for fixture_id, out_path, seed in plan: + rec, png = submit(args.host, fixtures[fixture_id], seed) + rec["fixture_id"] = fixture_id + records.append(rec) + print(json.dumps(rec), flush=True) + if png is not None: + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_bytes(png) + elif rec.get("http") == 200: + print(f"warning: 200 but no image for {fixture_id} seed {seed}", flush=True) + + suffix = "matrix" if args.matrix else f"single-{args.fixture}" + (args.out / f"{suffix}.json").write_text(json.dumps(records, indent=1) + "\n") + ok = all(r.get("http") == 200 and "sha256" in r for r in records) + print(("PASS" if ok else "FAIL") + f" {len(records)} requests", flush=True) + return 0 if ok else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/probe/issue-5/psnr.py b/probe/issue-5/psnr.py new file mode 100644 index 000000000..7e68c3fba --- /dev/null +++ b/probe/issue-5/psnr.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python +"""Issue 5 probe: PSNR of the int8 artifact's vLLM outputs against the +sealed Diffusers baseline attempt 2026-10-07T170729Z-q3-4r. + +Usage: psnr.py --fixture FIXTURE_ID RUN_DIR [RUN_DIR ...] + +Compares each RUN_DIR/matrix/FIXTURE_ID.sSEED.png against the baseline +sample with the same fixture and seed. RGBA outputs are composited over +white, matching issue 1's comparator. The sealed baseline covers the 1-, +2-, and 4-reference fixtures; the derived 3-reference fixture has no +sealed row (gate.py + visual check cover it). +""" + +from __future__ import annotations + +import argparse +import json +import math +import pathlib + +import numpy as np +from PIL import Image + +ATTEMPT = ( + pathlib.Path.home() + / ".local/share/multi-model-helper-server/data/evaluation/media/image/attempts/2026-10-07T170729Z-q3-4r" +) + + +def to_rgb(array: np.ndarray) -> np.ndarray: + if array.shape[2] == 4: + alpha = array[..., 3:4].astype(np.float64) / 255.0 + bg = np.full_like(array[..., :3], 255, dtype=np.float64) + return np.round( + array[..., :3].astype(np.float64) * alpha + bg * (1 - alpha) + ).clip(0, 255).astype(np.uint8) + return array[..., :3] + + +def psnr(a: np.ndarray, b: np.ndarray) -> float: + mse = float(np.mean((a.astype(np.float64) - b.astype(np.float64)) ** 2)) + return float("inf") if mse == 0 else 10 * math.log10(255.0**2 / mse) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--fixture", default="edit-compose-four") + parser.add_argument("runs", nargs="+", type=pathlib.Path) + args = parser.parse_args() + fixture = args.fixture + + samples = json.loads((ATTEMPT / "samples.json").read_text())["samples"] + baseline = { + s["seed"]: ATTEMPT / s["file"] + for s in samples + if s["fixture_id"] == fixture and s["status"] == "succeeded" + } + summary = {} + for run_dir in args.runs: + rows = [] + for seed, base_path in sorted(baseline.items()): + ours = run_dir / "matrix" / f"{fixture}.s{seed}.png" + if not ours.exists(): + rows.append({"seed": seed, "error": "missing"}) + continue + bm, om = Image.open(base_path), Image.open(ours) + ba, oa = np.asarray(bm), np.asarray(om) + if ba.shape[:2] != oa.shape[:2]: + rows.append( + {"seed": seed, "error": f"size {oa.shape[:2]} vs {ba.shape[:2]}"} + ) + continue + use_rgba = bm.mode == "RGBA" and om.mode == "RGBA" + a, b = (ba, oa) if use_rgba else (to_rgb(ba), to_rgb(oa)) + rows.append( + { + "seed": seed, + "psnr_db": round(psnr(a, b), 2), + "modes": f"{bm.mode}/{om.mode}", + } + ) + vals = [r["psnr_db"] for r in rows if "psnr_db" in r] + if vals: + summary[run_dir.name] = { + "rows": rows, + "mean_db": round(sum(vals) / len(vals), 2), + "min_db": min(vals), + "max_db": max(vals), + "n": len(vals), + } + else: + summary[run_dir.name] = {"rows": rows} + print(json.dumps(summary, indent=1)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/probe/issue-5/quantize_dit_int8wo.py b/probe/issue-5/quantize_dit_int8wo.py new file mode 100644 index 000000000..6b29f96f7 --- /dev/null +++ b/probe/issue-5/quantize_dit_int8wo.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python +"""Issue 5: quantize the Qwen-Image-2.1 DiT to int8 weight-only (W8A16) and +save it as a compressed-tensors pack-quantized checkpoint. + +Quantizes exactly the linears the vllm-omni transformer constructs with +``quant_config`` — per-block attention (to_q/to_k/to_v/to_out.0) and +img_mlp (proj/gate_layer/out) — and copies every boundary layer (img_in, +txt_in, time_text_embed, modulation, norm_out, proj_out) plus the RMSNorm +q/k scales through in BF16. Per-output-channel symmetric int8, scales in +fp32, weights packed 4x uint8 (biased by 128) per int32 via +compressed-tensors' own ``pack_to_int32`` so the artifact matches the +llm-compressor storage convention bit for bit. + +The artifact is a full diffusers-layout model directory: the quantized +``transformer/`` is real, the remaining components are symlinks into the +sealed snapshot. ``transformer/config.json`` carries the quantization +metadata wrapped per-component (``{"transformer": {...}}``) so vLLM-Omni's +auto-detection quantizes the DiT only — the text encoder and VAE stay BF16. + +A MANIFEST.json (inputs, versions, per-file sha256, quantization stats) is +written beside the weights. Nothing here enters Git; the artifact lives +under the helper-system state root. + +Usage: + quantize_dit_int8wo.py --snapshot SNAPSHOT_DIR --out ARTIFACT_DIR +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import hashlib +import json +import platform +import re +import sys +from pathlib import Path + +import torch +from compressed_tensors.compressors.pack_quantized.helpers import pack_to_int32 +from compressed_tensors.config import CompressionFormat +from compressed_tensors.quantization import ( + QuantizationArgs, + QuantizationConfig, + QuantizationScheme, + QuantizationStatus, + QuantizationStrategy, + QuantizationType, +) +from safetensors.torch import load_file, save_file + +SNAPSHOT_COMMIT = "790c92633540aa0cb11d9abf19eb46d861714758" +# Boundary layers the vllm-omni model pins to quant_config=None (BF16), plus +# the non-linear tensors; everything else in a block is quantized. +IGNORED_TARGETS = ["img_in", "txt_in", "time_text_embed", "modulation", "norm_out", "proj_out"] +QUANTIZED_KEY = re.compile( + r"^transformer_blocks\.\d+\." + r"(attn\.(to_q|to_k|to_v|to_out\.0)|img_mlp\.(proj|gate_layer|out))\.weight$" +) +SYMLINKED_COMPONENTS = ("scheduler", "text_encoder", "vae", "processor") +SYMLINKED_FILES = ("model_index.json",) + + +def build_quantization_dict() -> dict: + """Serialized compressed-tensors int8 weight-only metadata (llm-compressor schema).""" + weights = QuantizationArgs( + num_bits=8, + type=QuantizationType.INT, + symmetric=True, + strategy=QuantizationStrategy.CHANNEL, + dynamic=False, + observer="minmax", + ) + scheme = QuantizationScheme(targets=["Linear"], weights=weights) + config = QuantizationConfig( + config_groups={"group_0": scheme}, + format=CompressionFormat.pack_quantized, + ignore=list(IGNORED_TARGETS), + quant_method="compressed-tensors", + quantization_status=QuantizationStatus.FROZEN, + ) + return config.to_dict() + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def quantize_weight(weight: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor, dict]: + """Per-output-channel symmetric int8: q = round(W/s), s = rowmax(|W|)/127. + + Returns (weight_packed int32 [out, in/4], weight_scale fp32 [out, 1], + stats) using compressed-tensors' packer (uint8 biased by 128, 4 per int32). + """ + w = weight.detach().to(torch.float32) + scale = w.abs().amax(dim=1, keepdim=True).clamp_min(1e-12) / 127.0 + q = torch.round(w / scale).clamp(-127, 127).to(torch.int8) + packed = pack_to_int32(q.contiguous(), 8).contiguous() + dequant = q.to(torch.float32) * scale + err = (w - dequant).norm() / w.norm().clamp_min(1e-12) + stats = {"rel_l2": float(err), "out": int(w.shape[0]), "in": int(w.shape[1])} + return packed, scale.to(torch.float32).contiguous(), stats + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--snapshot", type=Path, required=True, help="sealed snapshot dir") + parser.add_argument("--out", type=Path, required=True, help="artifact dir to create") + args = parser.parse_args() + + snapshot = args.snapshot.resolve() + if not (snapshot / "transformer" / "config.json").exists(): + sys.exit(f"snapshot transformer config not found under {snapshot}") + if args.out.exists(): + sys.exit(f"refusing to overwrite existing {args.out}") + artifact = args.out + (artifact / "transformer").mkdir(parents=True) + + index = json.loads( + (snapshot / "transformer" / "diffusion_pytorch_model.safetensors.index.json").read_text() + ) + shards = sorted(set(index["weight_map"].values())) + state: dict[str, torch.Tensor] = {} + quant_stats: dict[str, dict] = {} + quant_params = bf16_params = 0 + for shard in shards: + tensors = load_file(str(snapshot / "transformer" / shard)) + for name, tensor in tensors.items(): + if QUANTIZED_KEY.match(name): + packed, scale, stats = quantize_weight(tensor) + state[f"{name}_packed"] = packed + state[f"{name}_scale"] = scale + state[f"{name}_shape"] = torch.tensor(tensor.shape, dtype=torch.int64) + quant_stats[name] = stats + quant_params += tensor.numel() + print(f"q {name} {tuple(tensor.shape)} rel_l2={stats['rel_l2']:.5f}", flush=True) + else: + state[name] = tensor + bf16_params += tensor.numel() + del tensors + + weights_path = artifact / "transformer" / "diffusion_pytorch_model.safetensors" + save_file(state, str(weights_path), metadata={"format": "pt"}) + total_bytes = weights_path.stat().st_size + + config = json.loads((snapshot / "transformer" / "config.json").read_text()) + config["quantization_config"] = {"transformer": build_quantization_dict()} + (artifact / "transformer" / "config.json").write_text(json.dumps(config, indent=2) + "\n") + + for name in SYMLINKED_FILES: + target = snapshot / name + (artifact / name).symlink_to(target) + for name in SYMLINKED_COMPONENTS: + target = snapshot / name + (artifact / name).symlink_to(target, target_is_directory=True) + + files = [ + {"path": "transformer/config.json", "bytes": (artifact / "transformer" / "config.json").stat().st_size}, + {"path": "transformer/diffusion_pytorch_model.safetensors", "bytes": total_bytes}, + ] + for entry in files: + entry["sha256"] = sha256_file(artifact / entry["path"]) + tree_json = snapshot.parent.parent / "trees" / f"{SNAPSHOT_COMMIT}.json" + manifest = { + "schema": "issue5-int8wo-v1", + "created_utc": dt.datetime.now(dt.UTC).replace(microsecond=0).isoformat(), + "description": "Qwen-Image-2.1 DiT int8 weight-only (W8A16), compressed-tensors pack-quantized", + "inputs": { + "snapshot": str(snapshot), + "revision": SNAPSHOT_COMMIT, + "tree_manifest": str(tree_json), + "tree_sha256": sha256_file(tree_json) if tree_json.exists() else None, + }, + "versions": { + "python": platform.python_version(), + "torch": torch.__version__, + "compressed_tensors": __import__("compressed_tensors").__version__, + }, + "quantization": { + "method": "compressed-tensors", + "format": "pack-quantized", + "weights": "int8 symmetric per-output-channel, static, minmax", + "config": build_quantization_dict(), + "ignored_layers": list(IGNORED_TARGETS), + "note": "ignore list mirrors the vllm-omni model's quant_config=None boundary layers", + }, + "stats": { + "quantized_tensors": len(quant_stats), + "quantized_params": quant_params, + "bf16_params": bf16_params, + "mean_rel_l2": sum(s["rel_l2"] for s in quant_stats.values()) / max(len(quant_stats), 1), + "max_rel_l2": max((s["rel_l2"] for s in quant_stats.values()), default=0.0), + "weights_file_bytes": total_bytes, + }, + "files": files, + } + (artifact / "MANIFEST.json").write_text(json.dumps(manifest, indent=1) + "\n") + print(json.dumps(manifest["stats"], indent=1)) + print(f"artifact: {artifact}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/probe/issue-5/serve.sh b/probe/issue-5/serve.sh new file mode 100644 index 000000000..f4af7d286 --- /dev/null +++ b/probe/issue-5/serve.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Issue 5 probe: serve the int8 weight-only compressed-tensors artifact on +# the RTX 3090 with one labeled flag set. VRAM is polled once per second for +# the whole server lifetime, matching issue 2's instrument (nvidia-smi csv +# series). The default flag set is cpu-offload + eager; pass extras after +# the label to override/extend (they are appended, vLLM last-one-wins). +# +# Usage: serve.sh LABEL [extra vllm flags...] +set -uo pipefail + +BASE=$HOME/Projects/vllm-omni-val +W=$BASE/issue5 +LABEL=$1; shift +RUN=$W/runs/$LABEL +mkdir -p "$RUN" + +STATE=$HOME/.local/state/multi-model-helper-server/state/image-service +ARTIFACT=${ARTIFACT:-$STATE/artifacts/qwen-image-2.1-dit-int8wo} +export HF_HOME=$STATE/weights +export HF_HUB_OFFLINE=1 + +( while sleep 1; do nvidia-smi --query-gpu=memory.used,utilization.gpu --format=csv,noheader; done ) > "$RUN/vram.log" 2>&1 & +POLL=$! +T0=$(date +%s) +echo "model: $ARTIFACT" > "$RUN/flags.txt" +echo "flags: $*" >> "$RUN/flags.txt" + +$BASE/venv/bin/vllm serve "$ARTIFACT" --omni --port 8091 \ + --served-model-name Qwen/Qwen-Image-2.1 \ + --enforce-eager --enable-cpu-offload "$@" > "$RUN/serve.log" 2>&1 +RC=$? +kill $POLL 2>/dev/null +echo "serve_rc=$RC wall=$(( $(date +%s) - T0 ))s" >> "$RUN/serve.log" diff --git a/tests/quantization/test_quant_factory.py b/tests/quantization/test_quant_factory.py index 2978d7b14..09921acd5 100644 --- a/tests/quantization/test_quant_factory.py +++ b/tests/quantization/test_quant_factory.py @@ -36,10 +36,15 @@ def _int8_weight_only_disk_config() -> dict: } -def test_hyphenated_registry_method_builds_from_str() -> None: - config = _build_single("compressed-tensors") +def test_hyphenated_registry_method_builds_from_disk_schema() -> None: + # A bare str spec cannot construct compressed-tensors (a serialized-format + # method with no parameterless default); the serialized schema must build. + with pytest.raises(TypeError, match="Cannot instantiate CompressedTensorsConfig"): + _build_single("compressed-tensors") + disk = _int8_weight_only_disk_config() + disk.pop("quant_method") + config = _build_single("compressed-tensors", **disk) assert isinstance(config, CompressedTensorsConfig) - assert config.get_name() == "compressed-tensors" @pytest.mark.parametrize("spelling", ["compressed-tensors", "compressed_tensors"])