[Misc] Migrate L1 tests to use pytest-mock (#1315)

Signed-off-by: Yuanheng Zhao <jonathan.zhaoyh@gmail.com>
Signed-off-by: yuanheng <jonathan.zhaoyh@gmail.com>
This commit is contained in:
Yuanheng Zhao authored and GitHub committed 2026-02-26 17:08:39 +08:00
1 parent 9913c00144
commit ed08a216c1
20 files changed
+977 -954

No files matched your search

@@ -1,19 +1,20 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import MagicMock
import pytest
from pytest_mock import MockerFixture
from vllm_omni.distributed.omni_connectors.adapter import try_recv_via_connector, try_send_via_connector
from vllm_omni.distributed.omni_connectors.connectors.shm_connector import SharedMemoryConnector
from vllm_omni.distributed.omni_connectors.utils.config import ConnectorSpec, OmniTransferConfig
from vllm_omni.distributed.omni_connectors.utils.initialization import get_connectors_config_for_stage
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@pytest.fixture
def mock_objects():
return {"connector": MagicMock(), "metrics": MagicMock(), "queue_fn": MagicMock()}
def mock_objects(mocker: MockerFixture):
return {"connector": mocker.MagicMock(), "metrics": mocker.MagicMock(), "queue_fn": mocker.MagicMock()}
def test_send_success(mock_objects):
@@ -138,7 +139,7 @@ def test_recv_no_connector():
assert inputs is None
def test_shm_connector_flow():
def test_shm_connector_flow(mocker: MockerFixture):
"""
Verify the full flow: Send -> Adapter -> Connector -> Adapter -> Recv.
Using real SharedMemoryConnector (inline mode for simplicity).
@@ -161,7 +162,7 @@ def test_shm_connector_flow():
def mock_submit(payload):
queue_capture.append(payload)
mock_metrics = MagicMock()
mock_metrics = mocker.MagicMock()
# 3. Send
success = try_send_via_connector(
@@ -217,7 +218,7 @@ def test_get_connectors_for_stage():
assert stage_2_config["from_stage_1"]["spec"]["name"] == "C2"
def test_recv_with_missing_metadata():
def test_recv_with_missing_metadata(mocker: MockerFixture):
"""Test recv when queue payload is malformed (missing metadata)."""
# Connector expects metadata but task doesn't have it
task = {
@@ -226,7 +227,7 @@ def test_recv_with_missing_metadata():
"from_stage": "0",
# Missing "connector_metadata"
}
mock_conn = MagicMock()
mock_conn = mocker.MagicMock()
# If get is called with None metadata, connector usually handles it or adapter handles exception
mock_conn.get.side_effect = Exception("Get failed")
@@ -1,14 +1,16 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import MagicMock
import pytest
from pytest_mock import MockerFixture
from vllm_omni.distributed.omni_connectors.connectors.shm_connector import SharedMemoryConnector
from vllm_omni.distributed.omni_connectors.factory import OmniConnectorFactory
from vllm_omni.distributed.omni_connectors.utils.config import ConnectorSpec
from vllm_omni.distributed.omni_connectors.utils.serialization import OmniSerializer
# pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def test_basic_serialization():
"""Test basic msgpack serialization."""
@@ -79,19 +81,19 @@ def test_put_get_inline(shm_connector):
assert size == ret_size
def test_put_get_shm(shm_connector, monkeypatch):
def test_put_get_shm(mocker: MockerFixture, shm_connector, monkeypatch: pytest.MonkeyPatch):
"""Test SHM transfer logic for large data (Mocked)."""
# Create data larger than 100 bytes
data = {"large": "x" * 200}
# Mock SHM return values
mock_handle = {"name": "test_shm", "size": 200}
mock_write = MagicMock(return_value=mock_handle)
mock_write = mocker.MagicMock(return_value=mock_handle)
monkeypatch.setattr("vllm_omni.distributed.omni_connectors.connectors.shm_connector.shm_write_bytes", mock_write)
# When reading, return the serialized bytes of the data
serialized_data = shm_connector.serialize_obj(data)
mock_read = MagicMock(return_value=serialized_data)
mock_read = mocker.MagicMock(return_value=serialized_data)
monkeypatch.setattr("vllm_omni.distributed.omni_connectors.connectors.shm_connector.shm_read_bytes", mock_read)
# Put
@@ -4,10 +4,10 @@
import threading
from collections import deque
from types import SimpleNamespace
from unittest.mock import MagicMock
import pytest
import torch
from pytest_mock import MockerFixture
from vllm.v1.request import RequestStatus
from vllm_omni.distributed.omni_connectors.transfer_adapter.base import OmniTransferAdapterBase
@@ -39,9 +39,9 @@ def _req(req_id: str, status: RequestStatus, external_req_id: str | None = None)
@pytest.fixture
def build_adapter(monkeypatch):
def build_adapter(monkeypatch, mocker: MockerFixture):
def _build(*, stage_id: int = 1, model_mode: str = "ar", max_num_seqs: int = 2):
connector = MagicMock()
connector = mocker.MagicMock()
connector.stage_id = stage_id
connector.get.return_value = None
connector.put.return_value = (True, 1, {})
@@ -7,7 +7,6 @@ These tests do NOT require Mooncake or RDMA environment.
"""
import threading
import unittest
import pytest
import torch
@@ -22,7 +21,7 @@ pytestmark = [pytest.mark.cpu, pytest.mark.parallel]
@pytest.mark.core_model
class TestBufferAllocator(unittest.TestCase):
class TestBufferAllocator:
"""Unit tests for BufferAllocator."""
def test_basic_alloc_free(self):
@@ -30,15 +29,15 @@ class TestBufferAllocator(unittest.TestCase):
allocator = BufferAllocator(total_size=4096, alignment=64)
offset1 = allocator.alloc(512)
self.assertEqual(offset1, 0)
assert offset1 == 0
offset2 = allocator.alloc(512)
self.assertGreater(offset2, 0)
assert offset2 > 0
# Free first block, should be reusable
allocator.free(offset1, 512)
offset3 = allocator.alloc(512)
self.assertEqual(offset3, 0)
assert offset3 == 0
def test_alignment(self):
"""Verify allocation respects alignment."""
@@ -47,22 +46,22 @@ class TestBufferAllocator(unittest.TestCase):
_offset1 = allocator.alloc(100)
offset2 = allocator.alloc(100)
self.assertEqual(offset2 % 128, 0)
self.assertEqual(offset2, 128)
assert offset2 % 128 == 0
assert offset2 == 128
def test_exhaustion_and_recovery(self):
"""Test that full allocation fails, then succeeds after free."""
allocator = BufferAllocator(total_size=1024, alignment=64)
offset = allocator.alloc(1024)
self.assertEqual(offset, 0)
assert offset == 0
with self.assertRaises(MemoryError):
with pytest.raises(MemoryError):
allocator.alloc(64)
allocator.free(offset, 1024)
offset2 = allocator.alloc(1024)
self.assertEqual(offset2, 0)
assert offset2 == 0
def test_thread_safety(self):
"""Verify allocator is thread-safe under concurrent access."""
@@ -84,10 +83,10 @@ class TestBufferAllocator(unittest.TestCase):
for t in threads:
t.join()
self.assertEqual(len(errors), 0, f"Thread safety errors: {errors}")
assert len(errors) == 0, f"Thread safety errors: {errors}"
class TestAllocatorInvariants(unittest.TestCase):
class TestAllocatorInvariants:
"""
Defensive invariant tests for BufferAllocator: double-free, partial
overlap corruption, and adjacent-block merging.
@@ -106,7 +105,7 @@ class TestAllocatorInvariants(unittest.TestCase):
allocator.free(offset, 256)
# Pool should still be consistent: allocate full size back
offset2 = allocator.alloc(4096)
self.assertEqual(offset2, 0)
assert offset2 == 0
@pytest.mark.slow
def test_double_free_after_merge_is_safe(self):
@@ -123,7 +122,7 @@ class TestAllocatorInvariants(unittest.TestCase):
allocator.free(a, 64) # should not raise
# Pool should still be fully usable
offset = allocator.alloc(4096)
self.assertEqual(offset, 0)
assert offset == 0
@pytest.mark.slow
def test_partial_overlap_raises_corruption(self):
@@ -133,7 +132,7 @@ class TestAllocatorInvariants(unittest.TestCase):
b = allocator.alloc(128)
allocator.free(a, 128) # [0, 128) is now free
# Try to free a region that starts inside [0,128) but extends beyond
with self.assertRaises(RuntimeError):
with pytest.raises(RuntimeError):
allocator.free(64, 128) # [64, 192) overlaps with free [0, 128)
# b is still allocated; freeing b should be fine
@@ -154,7 +153,7 @@ class TestAllocatorInvariants(unittest.TestCase):
# covering at least 192 bytes (3 * 64).
# Verify by allocating a contiguous block of 192 bytes.
offset = allocator.alloc(192)
self.assertEqual(offset, 0, "Adjacent blocks were not merged properly")
assert offset == 0, "Adjacent blocks were not merged properly"
@pytest.mark.slow
def test_fragmentation_and_defrag(self):
@@ -174,7 +173,7 @@ class TestAllocatorInvariants(unittest.TestCase):
allocator.free(d, 64) # free blocks: [64, 128) and [192, 256)
# Pool has two 64-byte holes; contiguous 128 is unavailable
with self.assertRaises(MemoryError):
with pytest.raises(MemoryError):
allocator.alloc(128)
allocator.free(a, 64)
@@ -182,14 +181,16 @@ class TestAllocatorInvariants(unittest.TestCase):
# After freeing everything, full pool should be available
offset = allocator.alloc(256)
self.assertEqual(offset, 0)
assert offset == 0
@pytest.mark.core_model
class TestManagedBuffer(unittest.TestCase):
class TestManagedBuffer:
"""Unit tests for ManagedBuffer."""
def setUp(self):
@pytest.fixture(autouse=True)
def _setup(self):
# automatically invoked for every test method in the class
self.allocator = BufferAllocator(total_size=4096, alignment=64)
self.pool = torch.zeros(4096, dtype=torch.uint8)
@@ -203,12 +204,12 @@ class TestManagedBuffer(unittest.TestCase):
self.pool[offset : offset + 64] = src.view(torch.uint8)
# Raw uint8 view
self.assertEqual(buf.tensor.shape[0], 64)
assert buf.tensor.shape[0] == 64
# Typed view
typed = buf.as_tensor(dtype=torch.float32, shape=(4, 4))
self.assertEqual(typed.shape, (4, 4))
self.assertTrue(torch.equal(typed.flatten(), src))
assert typed.shape == (4, 4)
assert torch.equal(typed.flatten(), src)
buf.release()
@@ -217,14 +218,10 @@ class TestManagedBuffer(unittest.TestCase):
offset = self.allocator.alloc(128)
with ManagedBuffer(self.allocator, offset, 128, self.pool) as buf:
self.assertFalse(buf._released)
assert not buf._released
self.assertTrue(buf._released)
assert buf._released
# Space should be reusable
new_offset = self.allocator.alloc(128)
self.assertEqual(new_offset, offset)
if __name__ == "__main__":
unittest.main()
assert new_offset == offset
@@ -13,7 +13,6 @@ import os
import subprocess
import threading
import time
import unittest
import pytest
import torch
@@ -195,43 +194,43 @@ def _md5(tensor: torch.Tensor) -> str:
# ---------------------------------------------------------------------------
@unittest.skipIf(TransferEngine is None, "Mooncake TransferEngine not available")
class TestBasicConnector(unittest.TestCase):
@pytest.mark.skipif(TransferEngine is None, reason="Mooncake TransferEngine not available")
class TestBasicConnector:
"""Verify connector initialization, put, cleanup, and health check."""
def test_initialization(self):
port = _free_port()
with MooncakeTransferEngineConnector(_connector_config(port, pool_size=1024 * 1024)) as c:
self.assertNotEqual(c.rpc_port, 0)
self.assertEqual(c.pool_size, 1024 * 1024)
self.assertTrue(c.pool.is_pinned())
assert c.rpc_port != 0
assert c.pool_size == 1024 * 1024
assert c.pool.is_pinned()
health = c.health()
self.assertEqual(health["status"], "healthy")
assert health["status"] == "healthy"
def test_put_tensor_bytes_object(self):
"""Put tensor / bytes / dict and verify metadata."""
port = _free_port()
with MooncakeTransferEngineConnector(_connector_config(port)) as c:
ok, sz, meta = c.put("s0", "s1", "t", torch.randn(100))
self.assertTrue(ok)
self.assertTrue(meta["is_fast_path"])
assert ok
assert meta["is_fast_path"]
ok, sz, meta = c.put("s0", "s1", "b", b"hello" * 100)
self.assertTrue(ok)
self.assertTrue(meta["is_fast_path"])
assert ok
assert meta["is_fast_path"]
ok, sz, meta = c.put("s0", "s1", "d", {"k": [1, 2, 3]})
self.assertTrue(ok)
self.assertFalse(meta["is_fast_path"])
assert ok
assert not meta["is_fast_path"]
def test_cleanup_releases_buffer(self):
port = _free_port()
with MooncakeTransferEngineConnector(_connector_config(port)) as c:
c.put("s0", "s1", "r1", torch.randn(100))
key = MooncakeTransferEngineConnector._make_key("r1", "s0", "s1")
self.assertIn(key, c._local_buffers)
assert key in c._local_buffers
c.cleanup("r1", from_stage="s0", to_stage="s1")
self.assertNotIn(key, c._local_buffers)
assert key not in c._local_buffers
def test_pool_exhaustion_and_recovery(self):
"""Fill pool, verify failure, free, verify recovery."""
@@ -247,7 +246,7 @@ class TestBasicConnector(unittest.TestCase):
for rid in ids:
c.cleanup(rid, from_stage="s", to_stage="s")
ok, _, _ = c.put("s", "s", "recovery", torch.randn(1000))
self.assertTrue(ok, "Pool recovery failed after cleanup")
assert ok, "Pool recovery failed after cleanup"
# ---------------------------------------------------------------------------
@@ -255,8 +254,8 @@ class TestBasicConnector(unittest.TestCase):
# ---------------------------------------------------------------------------
@unittest.skipIf(TransferEngine is None, "Mooncake TransferEngine not available")
class TestEndToEnd(unittest.TestCase):
@pytest.mark.skipif(TransferEngine is None, reason="Mooncake TransferEngine not available")
class TestEndToEnd:
"""E2E RDMA transfer: tensor, bytes, object, zero-copy, large payload, mixed types."""
def _pair(self, pool_size=16 * 1024 * 1024):
@@ -269,14 +268,14 @@ class TestEndToEnd(unittest.TestCase):
try:
orig = torch.randn(1024, 1024, dtype=torch.float32)
ok, sz, meta = p.put("s0", "s1", "t1", orig)
self.assertTrue(ok)
assert ok
time.sleep(0.5)
result = c.get("s0", "s1", "t1", meta)
self.assertIsNotNone(result)
assert result is not None
buf, _ = result
self.assertIsInstance(buf, ManagedBuffer)
assert isinstance(buf, ManagedBuffer)
recon = buf.as_tensor(dtype=orig.dtype, shape=orig.shape)
self.assertTrue(torch.equal(recon, orig))
assert torch.equal(recon, orig)
buf.release()
finally:
p.close()
@@ -287,15 +286,15 @@ class TestEndToEnd(unittest.TestCase):
try:
orig = b"Hello RDMA! " * 1000
ok, _, meta = p.put("s0", "s1", "b1", orig)
self.assertTrue(ok)
assert ok
time.sleep(0.5)
result = c.get("s0", "s1", "b1", meta)
self.assertIsNotNone(result)
assert result is not None
buf, _ = result
recv = buf.to_bytes() if hasattr(buf, "to_bytes") else buf
if hasattr(buf, "release"):
buf.release()
self.assertEqual(recv, orig)
assert recv == orig
finally:
p.close()
c.close()
@@ -305,13 +304,13 @@ class TestEndToEnd(unittest.TestCase):
try:
orig = {"msg": "hello", "nums": [1, 2, 3], "nested": {"k": "v"}}
ok, _, meta = p.put("s0", "s1", "o1", orig)
self.assertTrue(ok)
self.assertFalse(meta["is_fast_path"])
assert ok
assert not meta["is_fast_path"]
time.sleep(1.0)
result = c.get("s0", "s1", "o1", meta)
self.assertIsNotNone(result)
assert result is not None
recv, _ = result
self.assertEqual(recv, orig)
assert recv == orig
finally:
p.close()
c.close()
@@ -327,15 +326,15 @@ class TestEndToEnd(unittest.TestCase):
sbuf.as_tensor(dtype=torch.float32, shape=shape).copy_(ref)
ok, sz, meta = p.put("s0", "s1", "zc", sbuf)
self.assertTrue(ok)
self.assertTrue(meta.get("is_fast_path"))
assert ok
assert meta.get("is_fast_path")
time.sleep(0.5)
result = c.get("s0", "s1", "zc", meta)
self.assertIsNotNone(result)
assert result is not None
rbuf, _ = result
recon = rbuf.as_tensor(dtype=torch.float32, shape=shape)
self.assertTrue(torch.equal(recon, ref))
assert torch.equal(recon, ref)
rbuf.release()
finally:
p.close()
@@ -348,13 +347,13 @@ class TestEndToEnd(unittest.TestCase):
orig = torch.randn(5000, 5000, dtype=torch.float32)
md5_send = _md5(orig)
ok, _, meta = p.put("s0", "s1", "lg", orig)
self.assertTrue(ok)
assert ok
time.sleep(1.0)
result = c.get("s0", "s1", "lg", meta)
self.assertIsNotNone(result)
assert result is not None
buf, _ = result
recon = buf.as_tensor(dtype=orig.dtype, shape=orig.shape)
self.assertEqual(_md5(recon), md5_send)
assert _md5(recon) == md5_send
buf.release()
finally:
p.close()
@@ -374,23 +373,23 @@ class TestEndToEnd(unittest.TestCase):
time.sleep(0.5)
for name, data in cases:
ok, _, meta = p.put("s0", "s1", name, data)
self.assertTrue(ok, f"put failed: {name}")
assert ok, f"put failed: {name}"
time.sleep(0.3)
result = c.get("s0", "s1", name, meta)
self.assertIsNotNone(result, f"get failed: {name}")
assert result is not None, f"get failed: {name}"
recv, _ = result
if isinstance(data, torch.Tensor):
self.assertIsInstance(recv, ManagedBuffer)
assert isinstance(recv, ManagedBuffer)
recon = recv.as_tensor(dtype=data.dtype, shape=data.shape)
self.assertTrue(torch.equal(recon, data), f"mismatch: {name}")
assert torch.equal(recon, data), f"mismatch: {name}"
recv.release()
elif isinstance(data, bytes):
got = recv.to_bytes() if hasattr(recv, "to_bytes") else recv
if hasattr(recv, "release"):
recv.release()
self.assertEqual(got, data)
assert got == data
else:
self.assertEqual(recv, data)
assert recv == data
finally:
p.close()
c.close()
@@ -420,24 +419,24 @@ class TestEndToEnd(unittest.TestCase):
t.start()
for t in threads:
t.join()
self.assertEqual(len(errors), 0, f"errors: {errors}")
assert len(errors) == 0, f"errors: {errors}"
def test_auto_cleanup(self):
"""Producer buffer should be released after consumer get."""
p, c = self._pair()
try:
ok, _, meta = p.put("s0", "s1", "ac", torch.randn(100))
self.assertTrue(ok)
assert ok
key = MooncakeTransferEngineConnector._make_key("ac", "s0", "s1")
self.assertIn(key, p._local_buffers)
assert key in p._local_buffers
time.sleep(0.5)
result = c.get("s0", "s1", "ac", meta)
self.assertIsNotNone(result)
assert result is not None
buf, _ = result
if isinstance(buf, ManagedBuffer):
buf.release()
time.sleep(0.3)
self.assertNotIn(key, p._local_buffers)
assert key not in p._local_buffers
finally:
p.close()
c.close()
@@ -448,22 +447,22 @@ class TestEndToEnd(unittest.TestCase):
# ---------------------------------------------------------------------------
@unittest.skipIf(TransferEngine is None, "Mooncake TransferEngine not available")
class TestLifecycle(unittest.TestCase):
@pytest.mark.skipif(TransferEngine is None, reason="Mooncake TransferEngine not available")
class TestLifecycle:
"""Close, context manager, double-close safety."""
def test_close_releases_resources(self):
c = MooncakeTransferEngineConnector(_connector_config(_free_port(), pool_size=1024 * 1024))
c.put("s0", "s1", "x", torch.randn(100))
c.close()
self.assertTrue(c._stop_event.is_set())
self.assertEqual(len(c._local_buffers), 0)
assert c._stop_event.is_set()
assert len(c._local_buffers) == 0
def test_context_manager(self):
with MooncakeTransferEngineConnector(_connector_config(_free_port())) as c:
ok, _, _ = c.put("s0", "s1", "ctx", torch.randn(50))
self.assertTrue(ok)
self.assertTrue(c._stop_event.is_set())
assert ok
assert c._stop_event.is_set()
def test_double_close_safe(self):
c = MooncakeTransferEngineConnector(_connector_config(_free_port()))
@@ -477,9 +476,9 @@ class TestLifecycle(unittest.TestCase):
@pytest.mark.cuda
@unittest.skipIf(TransferEngine is None, "Mooncake TransferEngine not available")
@unittest.skipIf(not torch.cuda.is_available(), "CUDA not available")
class TestGPUPool(unittest.TestCase):
@pytest.mark.skipif(TransferEngine is None, reason="Mooncake TransferEngine not available")
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
class TestGPUPool:
"""GPU memory pool: initialization, put (CPU/GPU tensor), E2E transfer."""
@staticmethod
@@ -488,18 +487,18 @@ class TestGPUPool(unittest.TestCase):
def test_gpu_pool_init(self):
with MooncakeTransferEngineConnector(self._gpu_cfg(_free_port())) as c:
self.assertEqual(c.pool_device, "cuda:0")
self.assertTrue(c.pool.is_cuda)
assert c.pool_device == "cuda:0"
assert c.pool.is_cuda
def test_gpu_pool_put_cpu_and_gpu_tensor(self):
with MooncakeTransferEngineConnector(self._gpu_cfg(_free_port())) as c:
ok, _, meta = c.put("s0", "s1", "h2d", torch.randn(256, 256))
self.assertTrue(ok)
self.assertTrue(meta["is_fast_path"])
assert ok
assert meta["is_fast_path"]
ok, _, meta = c.put("s0", "s1", "d2d", torch.randn(256, 256, device="cuda:0"))
self.assertTrue(ok)
self.assertTrue(meta["is_fast_path"])
assert ok
assert meta["is_fast_path"]
def test_gpu_e2e_transfer(self):
p = MooncakeTransferEngineConnector(self._gpu_cfg(_free_port()))
@@ -507,14 +506,14 @@ class TestGPUPool(unittest.TestCase):
try:
orig = torch.randn(512, 512, dtype=torch.float32, device="cuda:0")
ok, _, meta = p.put("s0", "s1", "ge", orig)
self.assertTrue(ok)
assert ok
time.sleep(0.5)
result = c.get("s0", "s1", "ge", meta)
self.assertIsNotNone(result)
assert result is not None
buf, _ = result
recon = buf.as_tensor(dtype=orig.dtype, shape=orig.shape)
self.assertTrue(recon.is_cuda)
self.assertTrue(torch.equal(recon.cpu(), orig.cpu()))
assert recon.is_cuda
assert torch.equal(recon.cpu(), orig.cpu())
buf.release()
finally:
p.close()
@@ -527,8 +526,8 @@ class TestGPUPool(unittest.TestCase):
@pytest.mark.slow
@unittest.skipIf(TransferEngine is None, "Mooncake TransferEngine not available")
class TestStressCorrectness(unittest.TestCase):
@pytest.mark.skipif(TransferEngine is None, reason="Mooncake TransferEngine not available")
class TestStressCorrectness:
"""
Slow but high-value regression tests: concurrent put+get with data
integrity, large payload, edge cases, and sustained stress.
@@ -581,20 +580,20 @@ class TestStressCorrectness(unittest.TestCase):
for t in threads:
t.join()
self.assertEqual(len(errors), 0, f"Producer errors: {errors}")
self.assertEqual(len(meta_map), num_workers)
assert len(errors) == 0, f"Producer errors: {errors}"
assert len(meta_map) == num_workers
time.sleep(1.0)
# Consumer: get each tensor and verify integrity
for rid, meta in meta_map.items():
result = c.get("s0", "s1", rid, meta)
self.assertIsNotNone(result, f"get returned None for {rid}")
assert result is not None, f"get returned None for {rid}"
buf, _ = result
self.assertIsInstance(buf, ManagedBuffer, f"{rid} not ManagedBuffer")
assert isinstance(buf, ManagedBuffer), f"{rid} not ManagedBuffer"
recon = buf.as_tensor(dtype=torch.float32, shape=(512, 512))
recv_md5 = _md5(recon)
self.assertEqual(recv_md5, md5_sent[rid], f"MD5 mismatch for {rid}")
assert recv_md5 == md5_sent[rid], f"MD5 mismatch for {rid}"
buf.release()
finally:
p.close()
@@ -683,12 +682,8 @@ class TestStressCorrectness(unittest.TestCase):
for t in threads:
t.join()
ct.join(timeout=60)
self.assertEqual(len(errors), 0, f"errors: {errors}")
self.assertEqual(
consumed_count[0],
num_items,
f"Consumer only processed {consumed_count[0]}/{num_items} items",
)
assert len(errors) == 0, f"errors: {errors}"
assert consumed_count[0] == num_items, f"Consumer only processed {consumed_count[0]}/{num_items} items"
finally:
p.close()
c.close()
@@ -701,13 +696,13 @@ class TestStressCorrectness(unittest.TestCase):
try:
orig = torch.tensor([3.14], dtype=torch.float32)
ok, _, meta = p.put("s0", "s1", "tiny", orig)
self.assertTrue(ok)
assert ok
time.sleep(0.5)
result = c.get("s0", "s1", "tiny", meta)
self.assertIsNotNone(result)
assert result is not None
buf, _ = result
recon = buf.as_tensor(dtype=torch.float32, shape=(1,))
self.assertTrue(torch.equal(recon, orig))
assert torch.equal(recon, orig)
buf.release()
finally:
p.close()
@@ -718,7 +713,7 @@ class TestStressCorrectness(unittest.TestCase):
port = _free_port()
with MooncakeTransferEngineConnector(_connector_config(port, pool_size=8 * 1024 * 1024)) as c:
ok, sz, meta = c.put("s0", "s1", "empty_b", b"")
self.assertFalse(ok, "Empty bytes should be rejected by connector")
assert not ok, "Empty bytes should be rejected by connector"
# -- Large payload stress (500 MB) --
@@ -731,13 +726,13 @@ class TestStressCorrectness(unittest.TestCase):
orig = torch.randn(side, side, dtype=torch.float32)
md5_send = _md5(orig)
ok, _, meta = p.put("s0", "s1", "500mb", orig)
self.assertTrue(ok)
assert ok
time.sleep(2.0)
result = c.get("s0", "s1", "500mb", meta)
self.assertIsNotNone(result)
assert result is not None
buf, _ = result
recon = buf.as_tensor(dtype=orig.dtype, shape=orig.shape)
self.assertEqual(_md5(recon), md5_send)
assert _md5(recon) == md5_send
buf.release()
finally:
p.close()
@@ -752,12 +747,8 @@ class TestStressCorrectness(unittest.TestCase):
for i in range(50):
rid = f"cycle_{i}"
ok, _, _ = c.put("s0", "s1", rid, torch.randn(256, 256))
self.assertTrue(ok, f"Put failed at iteration {i}")
assert ok, f"Put failed at iteration {i}"
c.cleanup(rid, from_stage="s0", to_stage="s1")
# After all cycles, pool should be fully recovered
ok, _, _ = c.put("s0", "s1", "final", torch.randn(1024, 1024))
self.assertTrue(ok, "Pool not fully recovered after rapid cycles")
if __name__ == "__main__":
unittest.main()
assert ok, "Pool not fully recovered after rapid cycles"