From 4f5f99d8c652b0dbc9f2e74650e22ace2298f578 Mon Sep 17 00:00:00 2001 From: Kyle Merritt Date: Thu, 8 Oct 2026 06:27:36 -0400 Subject: [PATCH] probe(review): fp8 serve row + 3-ref fixture + fp8 quality-collapse diagnosis MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-up on PR #4: run --quantization fp8 through the served path on the 3090 (the flag issue 1 validated for capacity), add the untested 3-reference fixture, and measure all counts. Capacity: confirmed — all of 1/2/3/4 references fit and run (4-ref 44.1-44.4s, server peak 23,522MB, nvidia-smi max 23,983MiB; Marlin W8A16 is faster than BF16, 3.3x the Diffusers nocache baseline). Quality: every fp8 output is structurally broken on this commit and card (~97% transparent RGBA, grey RGB, PSNR 7-9dB) — on the edit path at every reference count and on T2I alike (issue 1's fp8 artifacts reinspected: same collapse; that run validated rc+image, not output structure). img_mlp exclusion does not fix it; transformer-only fp8 also collapses, pinning the culprit on the Marlin-quantized DiT. 3090 verdict stays not-built; upstream fix order is fp8 output correctness before any capacity role for it. --- probe/issue-2/FINDINGS.md | 84 +- .../evidence/rtx3090/psnr-fp8-rows.json | 112 +++ .../evidence/rtx3090/runs-summary.json | 884 +++++++++++++++++- probe/issue-2/fixtures.json | 13 + probe/issue-2/offline_fp8v.py | 24 +- probe/issue-2/probe.py | 1 + 6 files changed, 1102 insertions(+), 16 deletions(-) create mode 100644 probe/issue-2/evidence/rtx3090/psnr-fp8-rows.json diff --git a/probe/issue-2/FINDINGS.md b/probe/issue-2/FINDINGS.md index 8e58cf6f9..5940da2bd 100644 --- a/probe/issue-2/FINDINGS.md +++ b/probe/issue-2/FINDINGS.md @@ -16,13 +16,21 @@ same seeds 2301–2304, same per-fixture resolutions. **RTX 3090 (24 GB): not-built.** No host-DRAM spill path for the conditioning KV exists anywhere in the pipeline (verified by code read and -by exhausting every reachable knob). The served reference ceiling is two -condition images at `--enable-cpu-offload` — with the conditioning cache -ON, at 2.2× the Diffusers nocache baseline's speed. Four references OOM -inside the DiT forward under every flag combination, and the binding -constraint is the resident BF16 working set plus a fixed ~490 MiB image-MLP -activation, not the conditioning KV: the per-request cache contributes only -roughly 0.25–0.3 GiB per reference (see "KV size" below). +by exhausting every reachable knob). At BF16 the served reference ceiling +is two condition images at `--enable-cpu-offload` — with the conditioning +cache ON, at 2.2× the Diffusers nocache baseline's speed. Four references +OOM inside the DiT forward under every BF16 flag combination, and the +binding constraint is the resident BF16 working set plus a fixed ~490 MiB +image-MLP activation, not the conditioning KV: the per-request cache +contributes only roughly 0.25–0.3 GiB per reference (see "KV size" below). +FP8 weight quantization (`--quantization fp8`) does free the ~6 GiB the +arithmetic calls for — all of one, two, three, and four references then +fit and run (4-ref: 44.1–44.4 s, server peak 23,522 MB, nvidia-smi series +max 23,983 MiB) — **but on this commit and card the FP8 path produces +structurally broken output** (~96.5–97% transparent RGBA, grey noise RGB, +PSNR 7–9 dB), for the edit path at every reference count and for T2I +alike (see "FP8 quality collapse"). So the verdict stands: no usable path +to four references, and none that spills the KV. **RTX 5080 (16 GB): not-built, stronger — the server cannot initialize.** Six init attempts (default, cpu-offload, cpu+layerwise offload, fp8, @@ -99,6 +107,18 @@ All runs: `vllm serve --omni --port 8091 --enforce-eager | cpuoff-layerwise | `+ --enable-layerwise-offload` | ok | **OOM** | **OOM** | **OOM** | 24,137 MiB | | cpuoff-fp8v (yaml) | `+ --deploy-config deploy-fp8v.yaml` | ok | OK 34.3 s | OK 36.8 s | **OOM** | 24,015 MiB | | cpuoff-expandable | `+ PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True` | ok | OK 33.8 s | OK 36.7 s | **OOM** | 24,095 MiB | +| fp8 | `--enable-cpu-offload --quantization fp8` | ok | OK 29.9 s | OK 34.1 s (+3-ref OK 38.9 s) | OK 44.3 s — **output broken** | 23,983 MiB | + +The `fp8` row (review follow-up; #1 validated this flag's capacity +offline) also covered the previously untested three-reference count via +`edit-compose-three` — derived from `edit-compose-four` minus +`ref-shop-hesper` and its Subject line, four→three wording, same +instrument (recorded in `fixtures.json`). Measured under fp8, seeds +2301–2304: four 44.06–44.35 s (peak 22,918 MB), pair 34.62–35.05 s +(peak 21,504 MB), three 39.44–39.69 s (peak 23,532 MB) — every request +succeeds capacity-wise, and Marlin W8A16 is *faster* than BF16 +(4-ref 44.3 s vs Diffusers' 145.8 s = 3.3×). The quality outcome is in +the next section. - 4-ref OOM signature (identical ask across configs): `Tried to allocate 490.00 MiB` inside the image-branch MLP forward @@ -113,6 +133,37 @@ All runs: `vllm serve --omni --port 8091 --enforce-eager `45fac0b8…` pair, `3c4eb2ef…` single) across cpuoff, cpuoff-fp8v, cpuoff-expandable — the frozen-seed instrument is sound. +### FP8 quality collapse (and a correction to issue 1) + +Every FP8-weight output this probe produced is structurally broken, not +merely noisier: + +- Signature (identical everywhere): RGBA output ~96.5–97% transparent + (opaque fraction 0.008–0.035, alpha mean ≈115), near-flat grey RGB + (mean ≈120–130), PSNR vs the Diffusers baseline 6.6–9.0 dB. The BF16 + path on the same requests is 100% opaque and 26–29 dB. +- Scope: the serve path at 1, 2, 3, and 4 references (all seeds measured), + the offline path with `fp8` minus `img_mlp`, and the offline path with + **transformer-only** FP8 (`quantization_config={"transformer": + {"method": "fp8"}}` — the text encoder stays BF16). Transformer-only + still collapsing pins the culprit on the Marlin-quantized DiT, not the + encoder and not the image-MLP. An encoder-only isolator could not + initialize (the encoder's bf16→fp8 conversion transient lands on the GPU + on top of the resident BF16 DiT and exceeds the card). +- **Correction to issue 1**: #1's offline FP8 validation + (`qwen_image_21_fp8_all_rtx3090.png`, `…_fp8_imgmlp_…`) shows the same + collapse on reinspection (opaque fraction 0.035 both) — that run + validated rc=0 and image production, not output structure, so "FP8 + weight-only works on this card" from #1 overstated it. #1's 18.4 GB + capacity measurement stands; its quality implication does not. +- Upstream relevance: the recipe's H200 FP8 numbers quote PSNR only for + text-encoder FP8 (32.1 dB T2I); DiT-FP8 output quality appears + unmeasured upstream, and this probe adds the Ampere/Marlin cell: + online FP8 DiT weights yield near-noise latents for Qwen-Image-2.1. + Until that is fixed (or validated as Ampere-specific), FP8 cannot serve + as the capacity unlock on this card — which is why the 3090 verdict + remains `not-built` despite the fp8 capacity row passing. + ### Offline knob runs (3090, `Omni(...)` engine kwargs) The one knob that reaches the cache — `extras.prefix_kv_cache_dtype` — @@ -148,10 +199,16 @@ KV cache off, wall 81.16 s): samplers/kernels and the Diffusers side's int8 DiT. The four-reference PSNR cell the issue nominally asks for is -**capacity-blocked on every served flag set** (and offline with fp8_v); -the pair row is the highest-reference-count comparison this card can -produce. `edit-compose-four` single-ref walls for reference: -Diffusers 145.8 s (nocache); vLLM would-be number unobtainable. +**capacity-blocked at BF16** (every served flag set, and offline with +fp8_v). Under the fp8 serve row it becomes obtainable but meaningless as +quality: measured seeds 2301–2304 give 7.03–7.21 dB (mean 7.11) against +the sealed 4r baseline because the outputs are the collapsed ~97% +transparent renders of the FP8 quality section — recorded in +`evidence/rtx3090/psnr-fp8-rows.json` as the broken-output datum, not as +a usable comparison. The pair row above (27.57 dB at BF16) remains the +highest-reference-count usable comparison this card can produce. +`edit-compose-four` wall for reference: Diffusers 145.8 s (nocache); +vLLM 44.3 s under fp8 (fast but broken) — no valid BF16 number exists. ## Run matrix — RTX 5080 @@ -215,8 +272,9 @@ summaries in `evidence/rtx3090/runs-summary.json` and pressure of ~0.3 GiB/reference means concurrent multi-reference sessions do not accumulate Diffusers-style 2.8 GiB/reference retention. Candidacy holds for ≤2-reference profiles on 24 GB; 4-reference profiles remain - Diffusers-only until the resident-working-set/init path shrinks - (quantized-load init) or a KV spill lands. + Diffusers-only: FP8 unlocks the capacity (all four counts fit, 3.3× + baseline speed) but its output is broken on this commit, so the unlock + order is upstream FP8-output correctness first, then serving. - **#164 (topology input)**: the 5080 (16 GB) cannot host this stack at all — init-time parameter skeleton exceeds the card before any flag acts (`diffusion/models/utils.py:159-168`). Any topology plan pairing diff --git a/probe/issue-2/evidence/rtx3090/psnr-fp8-rows.json b/probe/issue-2/evidence/rtx3090/psnr-fp8-rows.json new file mode 100644 index 000000000..929a56e87 --- /dev/null +++ b/probe/issue-2/evidence/rtx3090/psnr-fp8-rows.json @@ -0,0 +1,112 @@ +{ + "edit-compose-four": { + "cpuoff-fp8": { + "rows": [ + { + "seed": 2301, + "psnr_db": 7.03, + "modes": "RGBA/RGBA" + }, + { + "seed": 2302, + "psnr_db": 7.11, + "modes": "RGBA/RGBA" + }, + { + "seed": 2303, + "psnr_db": 7.1, + "modes": "RGBA/RGBA" + }, + { + "seed": 2304, + "psnr_db": 7.21, + "modes": "RGBA/RGBA" + } + ], + "mean_db": 7.11, + "min_db": 7.03, + "max_db": 7.21, + "n": 4 + }, + "offline-fp8-imgmlp": { + "rows": [ + { + "seed": 2301, + "psnr_db": 7.27, + "modes": "RGBA/RGBA" + }, + { + "seed": 2302, + "error": "missing" + }, + { + "seed": 2303, + "error": "missing" + }, + { + "seed": 2304, + "error": "missing" + } + ], + "mean_db": 7.27, + "min_db": 7.27, + "max_db": 7.27, + "n": 1 + }, + "offline-fp8-dit": { + "rows": [ + { + "seed": 2301, + "psnr_db": 6.69, + "modes": "RGBA/RGBA" + }, + { + "seed": 2302, + "error": "missing" + }, + { + "seed": 2303, + "error": "missing" + }, + { + "seed": 2304, + "error": "missing" + } + ], + "mean_db": 6.69, + "min_db": 6.69, + "max_db": 6.69, + "n": 1 + } + }, + "edit-compose-pair": { + "cpuoff-fp8": { + "rows": [ + { + "seed": 2301, + "psnr_db": 6.57, + "modes": "RGBA/RGBA" + }, + { + "seed": 2302, + "psnr_db": 7.88, + "modes": "RGBA/RGBA" + }, + { + "seed": 2303, + "psnr_db": 9.02, + "modes": "RGBA/RGBA" + }, + { + "seed": 2304, + "psnr_db": 8.95, + "modes": "RGBA/RGBA" + } + ], + "mean_db": 8.11, + "min_db": 6.57, + "max_db": 9.02, + "n": 4 + } + } +} diff --git a/probe/issue-2/evidence/rtx3090/runs-summary.json b/probe/issue-2/evidence/rtx3090/runs-summary.json index bc937f179..f325dec63 100644 --- a/probe/issue-2/evidence/rtx3090/runs-summary.json +++ b/probe/issue-2/evidence/rtx3090/runs-summary.json @@ -452,6 +452,848 @@ } ] }, + "cpuoff-fp8": { + "functional.json": [ + { + "fixture_id": "edit-expression-ysolde", + "seed": 2301, + "n_references": 1, + "steps": 20, + "true_cfg_scale": 1.0, + "size": "864x1152", + "wall_s": 29.88, + "http": 200, + "bytes": 3983827, + "sha256": "eb6b1de5b4762f6c5d8d5dce5ced5ab037e17fef9d909acc047defddef09e76e", + "usage": null, + "metrics": { + "stage_metrics": { + "0": { + "stage_id": 0, + "final_output_type": "image", + "finish_reason": null, + "num_tokens_in": 0, + "num_tokens_out": 0, + "stage_gen_time_ms": 29772.65954017639, + "postprocess_time_ms": 0.0, + "audio_frames": 0, + "audio_sample_rate": 0, + "audio_duration_s": 0.0, + "image_pixels": 995328, + "denoise_step_latency_ms": 1488.6329770088196, + "output_unit_type": "image", + "output_unit_count": 1, + "serving_time_to_first_output_ms": 29772.92799949646, + "image_time_to_first_output_ms": 29772.64428138733, + "time_per_output_unit_ms": 0.0, + "inter_output_latency_ms": 0.0, + "inter_output_latencies_ms": [], + "vllm_ttft_ms": 0.0, + "vllm_tpot_ms": 0.0, + "vllm_itl_ms": 0.0, + "vllm_itls_ms": [], + "stage_name": "diffusion" + } + }, + "stage_id": 0, + "final_output_type": "image", + "num_tokens_in": 0, + "num_tokens_out": 0, + "stage_durations": { + "queue_wait_ms": 0.24512799427611753, + "stage_0_gen_ms": 29772.65954017639 + }, + "peak_memory_mb": 17848.0 + } + }, + { + "fixture_id": "edit-compose-pair", + "seed": 2301, + "n_references": 2, + "steps": 20, + "true_cfg_scale": 1.0, + "size": "1248x832", + "wall_s": 34.13, + "http": 200, + "bytes": 4155590, + "sha256": "769f62341e67ca7d23c0967db348c47063f3a8e6f30588b44f523baadbe6cbd9", + "usage": null, + "metrics": { + "stage_metrics": { + "0": { + "stage_id": 0, + "final_output_type": "image", + "finish_reason": null, + "num_tokens_in": 0, + "num_tokens_out": 0, + "stage_gen_time_ms": 34044.72541809082, + "postprocess_time_ms": 0.0, + "audio_frames": 0, + "audio_sample_rate": 0, + "audio_duration_s": 0.0, + "image_pixels": 1038336, + "denoise_step_latency_ms": 1702.236270904541, + "output_unit_type": "image", + "output_unit_count": 1, + "serving_time_to_first_output_ms": 34044.8694229126, + "image_time_to_first_output_ms": 34044.692277908325, + "time_per_output_unit_ms": 0.0, + "inter_output_latency_ms": 0.0, + "inter_output_latencies_ms": [], + "vllm_ttft_ms": 0.0, + "vllm_tpot_ms": 0.0, + "vllm_itl_ms": 0.0, + "vllm_itls_ms": [], + "stage_name": "diffusion" + } + }, + "stage_id": 0, + "final_output_type": "image", + "num_tokens_in": 0, + "num_tokens_out": 0, + "stage_durations": { + "queue_wait_ms": 0.13858800230082124, + "stage_0_gen_ms": 34044.72541809082 + }, + "peak_memory_mb": 21502.0 + } + }, + { + "fixture_id": "edit-compose-three", + "seed": 2301, + "n_references": 3, + "steps": 20, + "true_cfg_scale": 1.0, + "size": "1344x768", + "wall_s": 38.92, + "http": 200, + "bytes": 4130915, + "sha256": "0b95e888b27405009462bbc25877f8f30a361b170cecbe68f0d698c8af534411", + "usage": null, + "metrics": { + "stage_metrics": { + "0": { + "stage_id": 0, + "final_output_type": "image", + "finish_reason": null, + "num_tokens_in": 0, + "num_tokens_out": 0, + "stage_gen_time_ms": 38829.46538925171, + "postprocess_time_ms": 0.0, + "audio_frames": 0, + "audio_sample_rate": 0, + "audio_duration_s": 0.0, + "image_pixels": 1032192, + "denoise_step_latency_ms": 1941.4732694625854, + "output_unit_type": "image", + "output_unit_count": 1, + "serving_time_to_first_output_ms": 38829.567193984985, + "image_time_to_first_output_ms": 38829.437017440796, + "time_per_output_unit_ms": 0.0, + "inter_output_latency_ms": 0.0, + "inter_output_latencies_ms": [], + "vllm_ttft_ms": 0.0, + "vllm_tpot_ms": 0.0, + "vllm_itl_ms": 0.0, + "vllm_itls_ms": [], + "stage_name": "diffusion" + } + }, + "stage_id": 0, + "final_output_type": "image", + "num_tokens_in": 0, + "num_tokens_out": 0, + "stage_durations": { + "queue_wait_ms": 0.10217700037173927, + "stage_0_gen_ms": 38829.46538925171 + }, + "peak_memory_mb": 23522.0 + } + }, + { + "fixture_id": "edit-compose-four", + "seed": 2301, + "n_references": 4, + "steps": 20, + "true_cfg_scale": 1.0, + "size": "1344x768", + "wall_s": 44.25, + "http": 200, + "bytes": 4130915, + "sha256": "92c6238b762c6a598be5605db18daa2cb1a76f43e95e065238f126b8c22c1caa", + "usage": null, + "metrics": { + "stage_metrics": { + "0": { + "stage_id": 0, + "final_output_type": "image", + "finish_reason": null, + "num_tokens_in": 0, + "num_tokens_out": 0, + "stage_gen_time_ms": 44154.873847961426, + "postprocess_time_ms": 0.0, + "audio_frames": 0, + "audio_sample_rate": 0, + "audio_duration_s": 0.0, + "image_pixels": 1032192, + "denoise_step_latency_ms": 2207.7436923980713, + "output_unit_type": "image", + "output_unit_count": 1, + "serving_time_to_first_output_ms": 44154.97422218323, + "image_time_to_first_output_ms": 44154.84166145325, + "time_per_output_unit_ms": 0.0, + "inter_output_latency_ms": 0.0, + "inter_output_latencies_ms": [], + "vllm_ttft_ms": 0.0, + "vllm_tpot_ms": 0.0, + "vllm_itl_ms": 0.0, + "vllm_itls_ms": [], + "stage_name": "diffusion" + } + }, + "stage_id": 0, + "final_output_type": "image", + "num_tokens_in": 0, + "num_tokens_out": 0, + "stage_durations": { + "queue_wait_ms": 0.10436499724164605, + "stage_0_gen_ms": 44154.873847961426 + }, + "peak_memory_mb": 23522.0 + } + } + ], + "measured-edit-compose-four.json": [ + { + "fixture_id": "edit-compose-four", + "seed": 2301, + "n_references": 4, + "steps": 20, + "true_cfg_scale": 1.0, + "size": "1344x768", + "wall_s": 44.06, + "http": 200, + "bytes": 4130915, + "sha256": "f294847e9d617db27faa62549a9ebc4ec7a9931a19f23c79f57a525f2a69536f", + "usage": null, + "metrics": { + "stage_metrics": { + "0": { + "stage_id": 0, + "final_output_type": "image", + "finish_reason": null, + "num_tokens_in": 0, + "num_tokens_out": 0, + "stage_gen_time_ms": 43964.03455734253, + "postprocess_time_ms": 0.0, + "audio_frames": 0, + "audio_sample_rate": 0, + "audio_duration_s": 0.0, + "image_pixels": 1032192, + "denoise_step_latency_ms": 2198.2017278671265, + "output_unit_type": "image", + "output_unit_count": 1, + "serving_time_to_first_output_ms": 43964.13779258728, + "image_time_to_first_output_ms": 43963.993072509766, + "time_per_output_unit_ms": 0.0, + "inter_output_latency_ms": 0.0, + "inter_output_latencies_ms": [], + "vllm_ttft_ms": 0.0, + "vllm_tpot_ms": 0.0, + "vllm_itl_ms": 0.0, + "vllm_itls_ms": [], + "stage_name": "diffusion" + } + }, + "stage_id": 0, + "final_output_type": "image", + "num_tokens_in": 0, + "num_tokens_out": 0, + "stage_durations": { + "queue_wait_ms": 0.11695300054270774, + "stage_0_gen_ms": 43964.03455734253 + }, + "peak_memory_mb": 22918.0 + } + }, + { + "fixture_id": "edit-compose-four", + "seed": 2302, + "n_references": 4, + "steps": 20, + "true_cfg_scale": 1.0, + "size": "1344x768", + "wall_s": 44.17, + "http": 200, + "bytes": 4130915, + "sha256": "63fb0c92172213e6a2c2ead499fbb375a0975875d350fecf0eb14cc9893e8141", + "usage": null, + "metrics": { + "stage_metrics": { + "0": { + "stage_id": 0, + "final_output_type": "image", + "finish_reason": null, + "num_tokens_in": 0, + "num_tokens_out": 0, + "stage_gen_time_ms": 44082.07941055298, + "postprocess_time_ms": 0.0, + "audio_frames": 0, + "audio_sample_rate": 0, + "audio_duration_s": 0.0, + "image_pixels": 1032192, + "denoise_step_latency_ms": 2204.103970527649, + "output_unit_type": "image", + "output_unit_count": 1, + "serving_time_to_first_output_ms": 44082.17525482178, + "image_time_to_first_output_ms": 44082.03935623169, + "time_per_output_unit_ms": 0.0, + "inter_output_latency_ms": 0.0, + "inter_output_latencies_ms": [], + "vllm_ttft_ms": 0.0, + "vllm_tpot_ms": 0.0, + "vllm_itl_ms": 0.0, + "vllm_itls_ms": [], + "stage_name": "diffusion" + } + }, + "stage_id": 0, + "final_output_type": "image", + "num_tokens_in": 0, + "num_tokens_out": 0, + "stage_durations": { + "queue_wait_ms": 0.10842100164154544, + "stage_0_gen_ms": 44082.07941055298 + }, + "peak_memory_mb": 22918.0 + } + }, + { + "fixture_id": "edit-compose-four", + "seed": 2303, + "n_references": 4, + "steps": 20, + "true_cfg_scale": 1.0, + "size": "1344x768", + "wall_s": 44.27, + "http": 200, + "bytes": 4130915, + "sha256": "fef674b78949ba9ca82b900a53a41c31aac34979709158e20640ebbe9382a1ad", + "usage": null, + "metrics": { + "stage_metrics": { + "0": { + "stage_id": 0, + "final_output_type": "image", + "finish_reason": null, + "num_tokens_in": 0, + "num_tokens_out": 0, + "stage_gen_time_ms": 44182.273387908936, + "postprocess_time_ms": 0.0, + "audio_frames": 0, + "audio_sample_rate": 0, + "audio_duration_s": 0.0, + "image_pixels": 1032192, + "denoise_step_latency_ms": 2209.113669395447, + "output_unit_type": "image", + "output_unit_count": 1, + "serving_time_to_first_output_ms": 44182.36970901489, + "image_time_to_first_output_ms": 44182.23810195923, + "time_per_output_unit_ms": 0.0, + "inter_output_latency_ms": 0.0, + "inter_output_latencies_ms": [], + "vllm_ttft_ms": 0.0, + "vllm_tpot_ms": 0.0, + "vllm_itl_ms": 0.0, + "vllm_itls_ms": [], + "stage_name": "diffusion" + } + }, + "stage_id": 0, + "final_output_type": "image", + "num_tokens_in": 0, + "num_tokens_out": 0, + "stage_durations": { + "queue_wait_ms": 0.10433499846840277, + "stage_0_gen_ms": 44182.273387908936 + }, + "peak_memory_mb": 22918.0 + } + }, + { + "fixture_id": "edit-compose-four", + "seed": 2304, + "n_references": 4, + "steps": 20, + "true_cfg_scale": 1.0, + "size": "1344x768", + "wall_s": 44.35, + "http": 200, + "bytes": 4130915, + "sha256": "52da93d760d80f3658a04741f4646a200df397bf92859e9aeb81b38ab192cdab", + "usage": null, + "metrics": { + "stage_metrics": { + "0": { + "stage_id": 0, + "final_output_type": "image", + "finish_reason": null, + "num_tokens_in": 0, + "num_tokens_out": 0, + "stage_gen_time_ms": 44257.06672668457, + "postprocess_time_ms": 0.0, + "audio_frames": 0, + "audio_sample_rate": 0, + "audio_duration_s": 0.0, + "image_pixels": 1032192, + "denoise_step_latency_ms": 2212.8533363342285, + "output_unit_type": "image", + "output_unit_count": 1, + "serving_time_to_first_output_ms": 44257.17282295227, + "image_time_to_first_output_ms": 44257.03144073486, + "time_per_output_unit_ms": 0.0, + "inter_output_latency_ms": 0.0, + "inter_output_latencies_ms": [], + "vllm_ttft_ms": 0.0, + "vllm_tpot_ms": 0.0, + "vllm_itl_ms": 0.0, + "vllm_itls_ms": [], + "stage_name": "diffusion" + } + }, + "stage_id": 0, + "final_output_type": "image", + "num_tokens_in": 0, + "num_tokens_out": 0, + "stage_durations": { + "queue_wait_ms": 0.1152460026787594, + "stage_0_gen_ms": 44257.06672668457 + }, + "peak_memory_mb": 22918.0 + } + } + ], + "measured-edit-compose-pair.json": [ + { + "fixture_id": "edit-compose-pair", + "seed": 2301, + "n_references": 2, + "steps": 20, + "true_cfg_scale": 1.0, + "size": "1248x832", + "wall_s": 34.62, + "http": 200, + "bytes": 4155590, + "sha256": "cb982597556e6557b5f848bd5536f973113892194e944ddbf520df1352c3edc8", + "usage": null, + "metrics": { + "stage_metrics": { + "0": { + "stage_id": 0, + "final_output_type": "image", + "finish_reason": null, + "num_tokens_in": 0, + "num_tokens_out": 0, + "stage_gen_time_ms": 34530.22766113281, + "postprocess_time_ms": 0.0, + "audio_frames": 0, + "audio_sample_rate": 0, + "audio_duration_s": 0.0, + "image_pixels": 1038336, + "denoise_step_latency_ms": 1726.5113830566406, + "output_unit_type": "image", + "output_unit_count": 1, + "serving_time_to_first_output_ms": 34530.335664749146, + "image_time_to_first_output_ms": 34530.192852020264, + "time_per_output_unit_ms": 0.0, + "inter_output_latency_ms": 0.0, + "inter_output_latencies_ms": [], + "vllm_ttft_ms": 0.0, + "vllm_tpot_ms": 0.0, + "vllm_itl_ms": 0.0, + "vllm_itls_ms": [], + "stage_name": "diffusion" + } + }, + "stage_id": 0, + "final_output_type": "image", + "num_tokens_in": 0, + "num_tokens_out": 0, + "stage_durations": { + "queue_wait_ms": 0.11738399916794151, + "stage_0_gen_ms": 34530.22766113281 + }, + "peak_memory_mb": 21506.0 + } + }, + { + "fixture_id": "edit-compose-pair", + "seed": 2302, + "n_references": 2, + "steps": 20, + "true_cfg_scale": 1.0, + "size": "1248x832", + "wall_s": 34.99, + "http": 200, + "bytes": 4155590, + "sha256": "f4e6a1e96ef05c38d9da422b64ce725ce6b815bdb4b8b99be130fe89b51c63b9", + "usage": null, + "metrics": { + "stage_metrics": { + "0": { + "stage_id": 0, + "final_output_type": "image", + "finish_reason": null, + "num_tokens_in": 0, + "num_tokens_out": 0, + "stage_gen_time_ms": 34913.61594200134, + "postprocess_time_ms": 0.0, + "audio_frames": 0, + "audio_sample_rate": 0, + "audio_duration_s": 0.0, + "image_pixels": 1038336, + "denoise_step_latency_ms": 1745.6807971000671, + "output_unit_type": "image", + "output_unit_count": 1, + "serving_time_to_first_output_ms": 34913.718938827515, + "image_time_to_first_output_ms": 34913.589000701904, + "time_per_output_unit_ms": 0.0, + "inter_output_latency_ms": 0.0, + "inter_output_latencies_ms": [], + "vllm_ttft_ms": 0.0, + "vllm_tpot_ms": 0.0, + "vllm_itl_ms": 0.0, + "vllm_itls_ms": [], + "stage_name": "diffusion" + } + }, + "stage_id": 0, + "final_output_type": "image", + "num_tokens_in": 0, + "num_tokens_out": 0, + "stage_durations": { + "queue_wait_ms": 0.1037600013660267, + "stage_0_gen_ms": 34913.61594200134 + }, + "peak_memory_mb": 21506.0 + } + }, + { + "fixture_id": "edit-compose-pair", + "seed": 2303, + "n_references": 2, + "steps": 20, + "true_cfg_scale": 1.0, + "size": "1248x832", + "wall_s": 35.0, + "http": 200, + "bytes": 4155590, + "sha256": "537d898198d7ac292790791ba6b59b001169811d58792c0bff05acaaceae5ff8", + "usage": null, + "metrics": { + "stage_metrics": { + "0": { + "stage_id": 0, + "final_output_type": "image", + "finish_reason": null, + "num_tokens_in": 0, + "num_tokens_out": 0, + "stage_gen_time_ms": 34938.143491744995, + "postprocess_time_ms": 0.0, + "audio_frames": 0, + "audio_sample_rate": 0, + "audio_duration_s": 0.0, + "image_pixels": 1038336, + "denoise_step_latency_ms": 1746.9071745872498, + "output_unit_type": "image", + "output_unit_count": 1, + "serving_time_to_first_output_ms": 34938.24124336243, + "image_time_to_first_output_ms": 34938.11655044556, + "time_per_output_unit_ms": 0.0, + "inter_output_latency_ms": 0.0, + "inter_output_latencies_ms": [], + "vllm_ttft_ms": 0.0, + "vllm_tpot_ms": 0.0, + "vllm_itl_ms": 0.0, + "vllm_itls_ms": [], + "stage_name": "diffusion" + } + }, + "stage_id": 0, + "final_output_type": "image", + "num_tokens_in": 0, + "num_tokens_out": 0, + "stage_durations": { + "queue_wait_ms": 0.10082900553243235, + "stage_0_gen_ms": 34938.143491744995 + }, + "peak_memory_mb": 21504.0 + } + }, + { + "fixture_id": "edit-compose-pair", + "seed": 2304, + "n_references": 2, + "steps": 20, + "true_cfg_scale": 1.0, + "size": "1248x832", + "wall_s": 35.05, + "http": 200, + "bytes": 4155590, + "sha256": "9d9c0ef46ec3e8bf1b3bdb5af9dc59e7443d224233e83b1098699618c3b62de2", + "usage": null, + "metrics": { + "stage_metrics": { + "0": { + "stage_id": 0, + "final_output_type": "image", + "finish_reason": null, + "num_tokens_in": 0, + "num_tokens_out": 0, + "stage_gen_time_ms": 34981.534481048584, + "postprocess_time_ms": 0.0, + "audio_frames": 0, + "audio_sample_rate": 0, + "audio_duration_s": 0.0, + "image_pixels": 1038336, + "denoise_step_latency_ms": 1749.0767240524292, + "output_unit_type": "image", + "output_unit_count": 1, + "serving_time_to_first_output_ms": 34981.63151741028, + "image_time_to_first_output_ms": 34981.50610923767, + "time_per_output_unit_ms": 0.0, + "inter_output_latency_ms": 0.0, + "inter_output_latencies_ms": [], + "vllm_ttft_ms": 0.0, + "vllm_tpot_ms": 0.0, + "vllm_itl_ms": 0.0, + "vllm_itls_ms": [], + "stage_name": "diffusion" + } + }, + "stage_id": 0, + "final_output_type": "image", + "num_tokens_in": 0, + "num_tokens_out": 0, + "stage_durations": { + "queue_wait_ms": 0.09891400259220973, + "stage_0_gen_ms": 34981.534481048584 + }, + "peak_memory_mb": 21504.0 + } + } + ], + "measured-edit-compose-three.json": [ + { + "fixture_id": "edit-compose-three", + "seed": 2301, + "n_references": 3, + "steps": 20, + "true_cfg_scale": 1.0, + "size": "1344x768", + "wall_s": 39.44, + "http": 200, + "bytes": 4130915, + "sha256": "3d6901f66a80003aee3e730582540e134bd403faa415730462baf1a5fcc532a2", + "usage": null, + "metrics": { + "stage_metrics": { + "0": { + "stage_id": 0, + "final_output_type": "image", + "finish_reason": null, + "num_tokens_in": 0, + "num_tokens_out": 0, + "stage_gen_time_ms": 39349.570751190186, + "postprocess_time_ms": 0.0, + "audio_frames": 0, + "audio_sample_rate": 0, + "audio_duration_s": 0.0, + "image_pixels": 1032192, + "denoise_step_latency_ms": 1967.4785375595093, + "output_unit_type": "image", + "output_unit_count": 1, + "serving_time_to_first_output_ms": 39349.64394569397, + "image_time_to_first_output_ms": 39349.5192527771, + "time_per_output_unit_ms": 0.0, + "inter_output_latency_ms": 0.0, + "inter_output_latencies_ms": [], + "vllm_ttft_ms": 0.0, + "vllm_tpot_ms": 0.0, + "vllm_itl_ms": 0.0, + "vllm_itls_ms": [], + "stage_name": "diffusion" + } + }, + "stage_id": 0, + "final_output_type": "image", + "num_tokens_in": 0, + "num_tokens_out": 0, + "stage_durations": { + "queue_wait_ms": 0.09929099906003103, + "stage_0_gen_ms": 39349.570751190186 + }, + "peak_memory_mb": 23522.0 + } + }, + { + "fixture_id": "edit-compose-three", + "seed": 2302, + "n_references": 3, + "steps": 20, + "true_cfg_scale": 1.0, + "size": "1344x768", + "wall_s": 39.6, + "http": 200, + "bytes": 4130915, + "sha256": "07fb4f5a3936107c234bcfcbf20b5fa5e427d5f0de44dc240591808033388473", + "usage": null, + "metrics": { + "stage_metrics": { + "0": { + "stage_id": 0, + "final_output_type": "image", + "finish_reason": null, + "num_tokens_in": 0, + "num_tokens_out": 0, + "stage_gen_time_ms": 39527.87637710571, + "postprocess_time_ms": 0.0, + "audio_frames": 0, + "audio_sample_rate": 0, + "audio_duration_s": 0.0, + "image_pixels": 1032192, + "denoise_step_latency_ms": 1976.3938188552856, + "output_unit_type": "image", + "output_unit_count": 1, + "serving_time_to_first_output_ms": 39527.98533439636, + "image_time_to_first_output_ms": 39527.84466743469, + "time_per_output_unit_ms": 0.0, + "inter_output_latency_ms": 0.0, + "inter_output_latencies_ms": [], + "vllm_ttft_ms": 0.0, + "vllm_tpot_ms": 0.0, + "vllm_itl_ms": 0.0, + "vllm_itls_ms": [], + "stage_name": "diffusion" + } + }, + "stage_id": 0, + "final_output_type": "image", + "num_tokens_in": 0, + "num_tokens_out": 0, + "stage_durations": { + "queue_wait_ms": 0.11385999823687598, + "stage_0_gen_ms": 39527.87637710571 + }, + "peak_memory_mb": 23522.0 + } + }, + { + "fixture_id": "edit-compose-three", + "seed": 2303, + "n_references": 3, + "steps": 20, + "true_cfg_scale": 1.0, + "size": "1344x768", + "wall_s": 39.65, + "http": 200, + "bytes": 4130915, + "sha256": "cd02c3b4736d03a0630fb583d0e4d7ed6629e26b4a5627ed718ddff72c28510a", + "usage": null, + "metrics": { + "stage_metrics": { + "0": { + "stage_id": 0, + "final_output_type": "image", + "finish_reason": null, + "num_tokens_in": 0, + "num_tokens_out": 0, + "stage_gen_time_ms": 39579.60629463196, + "postprocess_time_ms": 0.0, + "audio_frames": 0, + "audio_sample_rate": 0, + "audio_duration_s": 0.0, + "image_pixels": 1032192, + "denoise_step_latency_ms": 1978.980314731598, + "output_unit_type": "image", + "output_unit_count": 1, + "serving_time_to_first_output_ms": 39579.7061920166, + "image_time_to_first_output_ms": 39579.572916030884, + "time_per_output_unit_ms": 0.0, + "inter_output_latency_ms": 0.0, + "inter_output_latencies_ms": [], + "vllm_ttft_ms": 0.0, + "vllm_tpot_ms": 0.0, + "vllm_itl_ms": 0.0, + "vllm_itls_ms": [], + "stage_name": "diffusion" + } + }, + "stage_id": 0, + "final_output_type": "image", + "num_tokens_in": 0, + "num_tokens_out": 0, + "stage_durations": { + "queue_wait_ms": 0.10619599925121292, + "stage_0_gen_ms": 39579.60629463196 + }, + "peak_memory_mb": 23532.0 + } + }, + { + "fixture_id": "edit-compose-three", + "seed": 2304, + "n_references": 3, + "steps": 20, + "true_cfg_scale": 1.0, + "size": "1344x768", + "wall_s": 39.69, + "http": 200, + "bytes": 4130915, + "sha256": "73695678f972fb1b1a8f3ac3717fbcf8bb66160b0848e509a5d11c24692d7bba", + "usage": null, + "metrics": { + "stage_metrics": { + "0": { + "stage_id": 0, + "final_output_type": "image", + "finish_reason": null, + "num_tokens_in": 0, + "num_tokens_out": 0, + "stage_gen_time_ms": 39606.837034225464, + "postprocess_time_ms": 0.0, + "audio_frames": 0, + "audio_sample_rate": 0, + "audio_duration_s": 0.0, + "image_pixels": 1032192, + "denoise_step_latency_ms": 1980.3418517112732, + "output_unit_type": "image", + "output_unit_count": 1, + "serving_time_to_first_output_ms": 39606.96840286255, + "image_time_to_first_output_ms": 39606.805086135864, + "time_per_output_unit_ms": 0.0, + "inter_output_latency_ms": 0.0, + "inter_output_latencies_ms": [], + "vllm_ttft_ms": 0.0, + "vllm_tpot_ms": 0.0, + "vllm_itl_ms": 0.0, + "vllm_itls_ms": [], + "stage_name": "diffusion" + } + }, + "stage_id": 0, + "final_output_type": "image", + "num_tokens_in": 0, + "num_tokens_out": 0, + "stage_durations": { + "queue_wait_ms": 0.1375499996356666, + "stage_0_gen_ms": 39606.837034225464 + }, + "peak_memory_mb": 23532.0 + } + } + ] + }, "cpuoff-fp8v": { "flags": "--enable-cpu-offload --deploy-config /home/anodyine/Projects/vllm-omni-val/issue2/deploy-fp8v.yaml", "vram_peak_mib": 24015, @@ -618,6 +1460,46 @@ "serve_wall_s": 14, "vram_peak_mib": 23939 }, + "fp8": { + "flags": "--enable-cpu-offload --quantization fp8", + "vram_peak_mib": 23983 + }, + "fp8-imgmlp": { + "flags": "--enable-cpu-offload --quantization fp8 --ignored-layers img_mlp", + "serve_rc": 2, + "serve_wall_s": 10, + "vram_peak_mib": 48 + }, + "offline-fp8-dit": { + "offline-edit-compose-four.json": [ + { + "fixture_id": "edit-compose-four", + "seed": 2301, + "n_references": 4, + "kv_dtype": "fp8_v", + "quant": "fp8-dit", + "wall_s": 46.0, + "peak_gpu_mb": 20997.9, + "bytes": 3630647, + "sha256": "56bc638cf0876d74ef8464c2bf944e7b3441ae72d06a67e4de30819f6d4d22a0" + } + ] + }, + "offline-fp8-imgmlp": { + "offline-edit-compose-four.json": [ + { + "fixture_id": "edit-compose-four", + "seed": 2301, + "n_references": 4, + "kv_dtype": "fp8_v", + "quant": "fp8-imgmlp", + "wall_s": 44.63, + "peak_gpu_mb": 19028.5, + "bytes": 3661881, + "sha256": "6e532e505b69f1bde6019a8fad1cd79b9f8c9fe8a8d4966016ca7fe98bd5404d" + } + ] + }, "offline-fp8v": { "offline-edit-compose-four.json": [ { @@ -643,4 +1525,4 @@ ] } } -} +} \ No newline at end of file diff --git a/probe/issue-2/fixtures.json b/probe/issue-2/fixtures.json index 3c1cdcd2c..46e777353 100644 --- a/probe/issue-2/fixtures.json +++ b/probe/issue-2/fixtures.json @@ -32,5 +32,18 @@ "height": 768, "prompt": "Bring the four characters from the references together in one scene; keep every identity exactly. Result: All four gather around the apothecary counter. Emotional tone: conspiratorial calm.\nSubject: Ysolde Varn, an adult woman in her early thirties, with deep brown skin, close-cropped black hair with one silver-beaded braid behind the left ear, and amber eyes; wearing a teal oilskin coat with brass buttons over a cream linen shirt; a thin pale scar through the right eyebrow and a brass compass on a cord at her throat. Unfolding a chart on the counter.\nSubject: Corvin Ashdown, an adult man in his mid-fifties, with pale, heavily freckled skin, short receding red hair and a grey-streaked red beard, and pale blue eyes; wearing a dark green harbormaster's greatcoat with a silver chain of office; a black leather patch over the left eye. Leaning on the doorframe.\nSubject: Tamsin Rook, an adult androgynous person in their late twenties, with warm olive skin, straight shoulder-length silver-white hair, and dark brown eyes; wearing a crimson hooded mantle over fitted grey leather armour; three thin black geometric lines tattooed down the left cheek. Sniffing a stoppered bottle.\nSubject: Hesper Quill, an adult woman in her early seventies, with light brown, deeply weathered skin, long white hair in one thick plait, and grey-green eyes; wearing a lavender dress under a stained canvas apothecary's apron; round wire-rimmed spectacles and a silver ring on every finger of the right hand. Behind the counter, weighing herbs on a brass scale.\nSetting: Inside the apothecary: shelves of bottled remedies, drying herbs hanging from the beams, a long counter.\nComposition: wide shot; from behind the counter's end, all four faces visible; lighting: warm lamplight with green light filtering through bottle glass\nStyle: Anime illustration: clean line art, cel shading, flat vivid colour, expressive faces.", "negative_prompt": null + }, + "edit-compose-three": { + "mode": "edit", + "references": [ + "ref-portrait-ysolde", + "ref-storm-corvin", + "ref-cutout-tamsin" + ], + "width": 1344, + "height": 768, + "prompt": "Bring the three characters from the references together in one scene; keep every identity exactly. Result: All three gather around the apothecary counter. Emotional tone: conspiratorial calm.\nSubject: Ysolde Varn, an adult woman in her early thirties, with deep brown skin, close-cropped black hair with one silver-beaded braid behind the left ear, and amber eyes; wearing a teal oilskin coat with brass buttons over a cream linen shirt; a thin pale scar through the right eyebrow and a brass compass on a cord at her throat. Unfolding a chart on the counter.\nSubject: Corvin Ashdown, an adult man in his mid-fifties, with pale, heavily freckled skin, short receding red hair and a grey-streaked red beard, and pale blue eyes; wearing a dark green harbormaster's greatcoat with a silver chain of office; a black leather patch over the left eye. Leaning on the doorframe.\nSubject: Tamsin Rook, an adult androgynous person in their late twenties, with warm olive skin, straight shoulder-length silver-white hair, and dark brown eyes; wearing a crimson hooded mantle over fitted grey leather armour; three thin black geometric lines tattooed down the left cheek. Sniffing a stoppered bottle.\nSetting: Inside the apothecary: shelves of bottled remedies, drying herbs hanging from the beams, a long counter.\nComposition: wide shot; from behind the counter's end, all four faces visible; lighting: warm lamplight with green light filtering through bottle glass\nStyle: Anime illustration: clean line art, cel shading, flat vivid colour, expressive faces.", + "negative_prompt": null, + "derived_from": "edit-compose-four minus ref-shop-hesper and its Subject line; four->three wording" } } diff --git a/probe/issue-2/offline_fp8v.py b/probe/issue-2/offline_fp8v.py index 0459948a9..a056bb3ba 100644 --- a/probe/issue-2/offline_fp8v.py +++ b/probe/issue-2/offline_fp8v.py @@ -47,6 +47,12 @@ def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--out", required=True, type=pathlib.Path) parser.add_argument("--kv-dtype", default="fp8_v") + parser.add_argument( + "--quant", + default="none", + choices=["none", "fp8", "fp8-imgmlp", "fp8-dit", "fp8-encoder"], + help="weight quantization: none, fp8 (all), fp8 minus img_mlp, transformer-only, or text-encoder-only", + ) parser.add_argument("--fixture", default="edit-compose-four") parser.add_argument("--seeds", type=lambda s: tuple(int(x) for x in s.split(",")), default=(2301,)) parser.add_argument("--no-extras", action="store_true", help="run without the extras knob (control)") @@ -61,11 +67,25 @@ def main() -> int: enable_cpu_offload=True, init_timeout=1200, ) + if args.quant == "fp8": + omni_kwargs["quantization"] = "fp8" + elif args.quant == "fp8-imgmlp": + omni_kwargs["quantization_config"] = { + "method": "fp8", + "ignored_layers": ["img_mlp"], + } + elif args.quant == "fp8-dit": + omni_kwargs["quantization_config"] = {"transformer": {"method": "fp8"}} + elif args.quant == "fp8-encoder": + omni_kwargs["quantization_config"] = {"text_encoder": {"method": "fp8"}} if not args.no_extras: omni_kwargs["extras"] = {"prefix_kv_cache_dtype": args.kv_dtype} t0 = time.time() omni = Omni(**omni_kwargs) - print(f"load_s={time.time() - t0:.1f} extras={omni_kwargs.get('extras')}", flush=True) + print( + f"load_s={time.time() - t0:.1f} quant={args.quant} extras={omni_kwargs.get('extras')}", + flush=True, + ) model_class_name = get_model_class_name(omni) args.out.mkdir(parents=True, exist_ok=True) @@ -90,7 +110,7 @@ def main() -> int: ) torch.cuda.reset_peak_memory_stats() t1 = time.time() - rec = {"fixture_id": args.fixture, "seed": seed, "n_references": len(refs), "kv_dtype": args.kv_dtype} + rec = {"fixture_id": args.fixture, "seed": seed, "n_references": len(refs), "kv_dtype": args.kv_dtype, "quant": args.quant} try: outputs = omni.generate(prompt_dict, sampling_params_list=[params]) wall = time.time() - t1 diff --git a/probe/issue-2/probe.py b/probe/issue-2/probe.py index 1e974169d..95dcbb20d 100644 --- a/probe/issue-2/probe.py +++ b/probe/issue-2/probe.py @@ -33,6 +33,7 @@ CFG = 1.0 FUNCTIONAL_FIXTURES = ( "edit-expression-ysolde", # 1 reference "edit-compose-pair", # 2 references + "edit-compose-three", # 3 references (derived; see fixtures.json) "edit-compose-four", # 4 references ) MEASURED_FIXTURE = "edit-compose-four"