[Attention] [P1] Keep H3 VSA policy in the model and share sparse primitives (#7535)
Signed-off-by: lishunyang12 <lishunyang12@users.noreply.github.com> Co-authored-by: lishunyang12 <lishunyang12@users.noreply.github.com>
This commit is contained in:
1 parent
99ff4f3070
commit
507cb1d834
19 files changed
+1117
-473
No files matched your search
@@ -23,6 +23,7 @@ from typing import Any, NamedTuple
|
||||
import numpy as np
|
||||
import psutil
|
||||
import yaml
|
||||
from filelock import FileLock, Timeout
|
||||
from vllm import TextPrompt
|
||||
from vllm.logger import init_logger
|
||||
|
||||
@@ -170,9 +171,54 @@ class OmniServer:
|
||||
self.use_omni = use_omni
|
||||
self.proc: subprocess.Popen | None = None
|
||||
self.host = "127.0.0.1"
|
||||
self._auto_port = port is None
|
||||
self._port_lock: FileLock | None = None
|
||||
self.port = get_open_port() if port is None else port
|
||||
|
||||
def _reserve_port(self) -> None:
|
||||
# bind(0)/close does not reserve a port across concurrent pytest workers.
|
||||
# Keep an advisory lock until teardown, including the long import phase
|
||||
# before the subprocess binds its HTTP socket. Never unlink lock files:
|
||||
# another worker may already have the same inode open.
|
||||
for attempt in range(128):
|
||||
if attempt:
|
||||
self.port = get_open_port(self.host)
|
||||
lock_path = Path(tempfile.gettempdir()) / f"vllm-omni-test-port-{os.getuid()}-{self.port}.lock"
|
||||
lock = FileLock(lock_path)
|
||||
try:
|
||||
lock.acquire(timeout=0)
|
||||
except Timeout as error:
|
||||
if not self._auto_port:
|
||||
raise RuntimeError(f"HTTP test port {self.port} is already reserved") from error
|
||||
continue
|
||||
try:
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe:
|
||||
probe.bind((self.host, self.port))
|
||||
except OSError as error:
|
||||
lock.release()
|
||||
if not self._auto_port or error.errno != errno.EADDRINUSE:
|
||||
raise
|
||||
continue
|
||||
self._port_lock = lock
|
||||
return
|
||||
raise RuntimeError("Could not reserve an HTTP test port after 128 attempts")
|
||||
|
||||
def _owns_listening_port(self) -> bool:
|
||||
assert self.proc is not None
|
||||
try:
|
||||
parent = psutil.Process(self.proc.pid)
|
||||
processes = [parent, *parent.children(recursive=True)]
|
||||
return any(
|
||||
conn.status == psutil.CONN_LISTEN and conn.laddr.port == self.port
|
||||
for process in processes
|
||||
for conn in process.net_connections(kind="tcp")
|
||||
)
|
||||
except psutil.NoSuchProcess:
|
||||
# A child can exit between enumeration and inspecting its sockets.
|
||||
return False
|
||||
|
||||
def _start_server(self) -> None:
|
||||
self._reserve_port()
|
||||
env = os.environ.copy()
|
||||
if self.env_dict is not None:
|
||||
env.update(self.env_dict)
|
||||
@@ -208,7 +254,10 @@ class OmniServer:
|
||||
raise RuntimeError(f"Server processes exited with code {ret} before becoming ready.")
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||
sock.settimeout(1)
|
||||
if sock.connect_ex((self.host, self.port)) == 0:
|
||||
if sock.connect_ex((self.host, self.port)) == 0 and self._owns_listening_port():
|
||||
ret = self.proc.poll()
|
||||
if ret is not None:
|
||||
raise RuntimeError(f"Server processes exited with code {ret} before becoming ready.")
|
||||
startup_s = time.perf_counter() - startup_t0
|
||||
if self.log_stats:
|
||||
print(
|
||||
@@ -368,9 +417,14 @@ class OmniServer:
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
if self.proc:
|
||||
self._kill_process_tree(self.proc.pid)
|
||||
cleanup_test_environment()
|
||||
try:
|
||||
if self.proc:
|
||||
self._kill_process_tree(self.proc.pid)
|
||||
finally:
|
||||
if self._port_lock is not None:
|
||||
self._port_lock.release()
|
||||
self._port_lock = None
|
||||
cleanup_test_environment()
|
||||
|
||||
|
||||
class OmniServerStageCli(OmniServer):
|
||||
|
||||
Reference in new issue
Block a user