From 649558e1a1f4e94c5322388be19ba48b556135e4 Mon Sep 17 00:00:00 2001 From: Hongsheng Liu Date: Mon, 17 Aug 2026 16:21:47 +0800 Subject: [PATCH] [Docs] Show TP2 devices in DLO architecture Signed-off-by: Hongsheng Liu --- .../distributed_layerwise_offload.md | 12 +- .../dlo/dlo_no_allgather_shared_host.svg | 171 ++++++++++-------- 2 files changed, 102 insertions(+), 81 deletions(-) diff --git a/docs/design/feature/offloader/distributed_layerwise_offload.md b/docs/design/feature/offloader/distributed_layerwise_offload.md index 3d3f71dbf..32aaf93bf 100644 --- a/docs/design/feature/offloader/distributed_layerwise_offload.md +++ b/docs/design/feature/offloader/distributed_layerwise_offload.md @@ -394,11 +394,13 @@ ordinary runtime tensors. ![No-AllGather DLO shared host weights and independent runtime orchestration](../../figures/dlo/dlo_no_allgather_shared_host.svg) -The figure uses TP1 to make physical sharing explicit. With TP greater than -one, the cache keeps one immutable entry per TP coordinate and equivalent -workers in independent engines map the corresponding entry. An external -router or orchestrator chooses each engine's requests; DLO coordinates only -the worker-local mapping, hooks, transfer streams, events, and device slots. +The figure uses TP2 to make per-coordinate physical sharing explicit. Each +engine contains two rank-local workers and devices. The cache keeps one +immutable entry per TP coordinate, so equivalent TP0 workers across engines +share the TP0 entry and equivalent TP1 workers share the separate TP1 entry. +An external router or orchestrator chooses each engine's requests; DLO +coordinates only the worker-local mapping, hooks, transfer streams, events, +and device slots. For direct mmap, each process retains immutable safetensors views and uses two bounded pinned host staging slots. Processes on the same node that map the same diff --git a/docs/design/figures/dlo/dlo_no_allgather_shared_host.svg b/docs/design/figures/dlo/dlo_no_allgather_shared_host.svg index da4b9ca67..f7ec39116 100644 --- a/docs/design/figures/dlo/dlo_no_allgather_shared_host.svg +++ b/docs/design/figures/dlo/dlo_no_allgather_shared_host.svg @@ -1,6 +1,6 @@ No-AllGather DLO shared host weights and independent runtime orchestration - An external orchestrator routes requests to two independently scheduled diffusion engines on one node. Each engine has its own loader, mmap view, DLO runtime hooks, streams, events, and GPU device slots. Both engines map the same immutable runtime-cache weight pages stored at the bottom of the diagram, without a cross-engine DLO AllGather. TP1 is shown; tensor parallel deployments repeat one shared cache entry per TP coordinate. + An external orchestrator routes requests to two independently scheduled TP2 diffusion engines on one node. Each engine contains two rank-local workers and devices, with private loaders, mmap views, DLO runtime hooks, streams, events, and rotating GPU slots. Equivalent TP0 workers across engines map one immutable TP0 runtime-cache payload, while equivalent TP1 workers map a separate TP1 payload, without a cross-engine DLO AllGather. @@ -23,6 +23,7 @@ .mapping { fill: #fffbeb; stroke: #f59e0b; } .hook { fill: #eff6ff; stroke: #60a5fa; } .gpu { fill: #f0fdf4; stroke: #86efac; } + .device { fill: #ffffff; stroke: #86efac; } .slot-current { fill: #16a34a; } .slot-next { fill: #bbf7d0; stroke: #22c55e; } .no-collective { fill: #fef2f2; stroke: #fca5a5; } @@ -35,8 +36,9 @@ .box-text { font: 500 10.5px system-ui, -apple-system, Segoe UI, sans-serif; fill: #475569; } .payload-title { font: 700 14px system-ui, -apple-system, Segoe UI, sans-serif; fill: #ffffff; } .payload-text { font: 500 11px system-ui, -apple-system, Segoe UI, sans-serif; fill: #ffffff; } - .slot-text { font: 700 9px system-ui, -apple-system, Segoe UI, sans-serif; fill: #ffffff; } - .slot-next-text { font: 700 9px system-ui, -apple-system, Segoe UI, sans-serif; fill: #166534; } + .device-title { font: 700 9px system-ui, -apple-system, Segoe UI, sans-serif; fill: #166534; } + .slot-text { font: 700 8px system-ui, -apple-system, Segoe UI, sans-serif; fill: #ffffff; } + .slot-next-text { font: 700 8px system-ui, -apple-system, Segoe UI, sans-serif; fill: #166534; } .arrow-label { font: 600 10.5px system-ui, -apple-system, Segoe UI, sans-serif; fill: #475569; } .request-label { font: 600 10.5px system-ui, -apple-system, Segoe UI, sans-serif; fill: #6d28d9; } .mapping-label { font: 600 10.5px system-ui, -apple-system, Segoe UI, sans-serif; fill: #b45309; } @@ -51,8 +53,8 @@ - No-AllGather DLO: one shared host payload, independent runtimes - TP1 shown; TP>1 repeats this mapping once per TP coordinate, shared by equivalent workers across engines + No-AllGather DLO: TP-coordinate host sharing, independent engines + TP2 shown: each engine owns two devices; equivalent workers share one host payload per TP coordinate @@ -72,78 +74,92 @@ Independent engine A - own request schedule · own DLO backend state + TP2 · two rank-local workers · own request schedule and DLO state - - STARTUP - ordinary loader - final transforms - publish / join - runtime cache + + STARTUP ×2 + rank-local + loaders + final transforms + publish / join - - HOST VIEW - mmap of shared - weight pages - process-local view - optional registration + + HOST VIEWS ×2 + TP0 mmap + TP1 mmap + process-local + optional registration - - RUNTIME CONTROL - DLO backend - + block hook - select · prefetch - repoint · rotate + + RUNTIME ×2 + DLO backends + + block hooks + private streams + events + state - - GPU A - two device slots - - SLOT 0 · NOW - - SLOT 1 · NEXT + + TWO DEVICES + + TP0 · GPU A0 + + SLOT 0 NOW + + SLOT 1 NEXT + + TP1 · GPU A1 + + SLOT 0 NOW + + SLOT 1 NEXT - - - + + + Independent engine B - own request schedule · own DLO backend state + TP2 · two rank-local workers · own request schedule and DLO state - - STARTUP - ordinary loader - final transforms - publish / join - runtime cache + + STARTUP ×2 + rank-local + loaders + final transforms + publish / join - - HOST VIEW - mmap of shared - weight pages - process-local view - optional registration + + HOST VIEWS ×2 + TP0 mmap + TP1 mmap + process-local + optional registration - - RUNTIME CONTROL - DLO backend - + block hook - select · prefetch - repoint · rotate + + RUNTIME ×2 + DLO backends + + block hooks + private streams + events + state - - GPU B - two device slots - - SLOT 0 · NOW - - SLOT 1 · NEXT + + TWO DEVICES + + TP0 · GPU B0 + + SLOT 0 NOW + + SLOT 1 NEXT + + TP1 · GPU B1 + + SLOT 0 NOW + + SLOT 1 NEXT - - - + + + @@ -153,25 +169,28 @@ shared host pages do not couple request order, progress, or failure domains Ordinary TP / SP model collectives, when configured, remain inside each engine - - - same pages - - same pages + + + TP0 + TP1 mappings + + TP0 + TP1 mappings SHARING SCOPE - TP1: one full cache entry - TP>1: one entry per TP coordinate - Equivalent engines reuse that entry; - different TP ranks never alias layouts. + TP2: two coordinate entries + one physical copy per coordinate + A/TP0 + B/TP0 → TP0 entry + A/TP1 + B/TP1 → TP1 entry - NODE-LOCAL IMMUTABLE RUNTIME CACHE - - 1× physical runtime-weight payload - file-backed tensor pages shared by the OS page cache + NODE-LOCAL IMMUTABLE RUNTIME CACHE · ONE COPY PER TP COORDINATE + + TP0 physical payload + shared by A/TP0 + B/TP0 + + TP1 physical payload + shared by A/TP1 + B/TP1 Identity includes final layout + TP coordinate; engine and DP replica are excluded