[LingBot World] Eleven bit-identical removals of repeated work in the realtime path (#7648)

Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>

tests/diffusion/distributed/test_comm.py

@@ -461,3 +461,46 @@ def test_ring_p2p_parity(world_size: int, dtype: torch.dtype):
         device_kind="cuda",
         master_port=29501,
     )
+
+
+def _run_fused_qkv_matches_three_4d(
+    local_rank: int,
+    world_size: int,
+    device_kind: DeviceKind,
+    master_port: int,
+) -> None:
+    """LingBot's fused (B, S/N, 3, H, D) exchange must equal three per-tensor 4D exchanges."""
+    from vllm_omni.diffusion.distributed.comm import all_to_all_5D
+
+    device = _worker_device(local_rank, device_kind)
+    if device_kind == "cuda":
+        current_omni_platform.set_device(device)
+    _init_worker(local_rank, world_size, master_port, device_kind)
+    initialize_model_parallel(ulysses_degree=world_size)
+    sp_group = get_sp_group().ulysses_group
+    try:
+        batch, seq_per_rank, heads, head_size = 1, 6, 4 * world_size, 8
+        # Distinguishable values: rank, q/k/v index, position, head, feature.
+        base = torch.arange(seq_per_rank * heads * head_size, dtype=torch.float32, device=device).reshape(
+            batch, seq_per_rank, heads, head_size
+        )
+        tensors = [base + 1000.0 * local_rank + 100000.0 * which for which in range(3)]
+        separate = [SeqAllToAll4D.apply(sp_group, t, 2, 1, False) for t in tensors]
+        fused = all_to_all_5D(torch.stack(tensors, dim=2), scatter_idx=3, gather_idx=1, group=sp_group)
+        assert fused.shape == (batch, seq_per_rank * world_size, 3, heads // world_size, head_size)
+        for which, expected in enumerate(separate):
+            assert torch.equal(fused.unbind(2)[which], expected), f"q/k/v index {which} differs on rank {local_rank}"
+    finally:
+        destroy_distributed_env()
+
+
+@pytest.mark.core_model
+@pytest.mark.diffusion
+@pytest.mark.cpu
+@pytest.mark.parametrize("world_size", [2, 4])
+def test_fused_qkv_all_to_all_matches_three_4d_exchanges_cpu(world_size: int) -> None:
+    torch.multiprocessing.spawn(
+        _run_fused_qkv_matches_three_4d,
+        args=(world_size, "cpu", 29613),
+        nprocs=world_size,
+    )