Skip to content

vllm.models.deepseek_v41.amd.ops.engram_host_table

Engram host tables in HIP VMM host memory, shared across DP ranks by dmabuf FD.

The ROCm counterpart of nvidia/ops/engram_fabric_table.py, enabled by VLLM_ROCM_ENGRAM_HIP_VMM. HIP cannot export host allocations as FABRIC handles, so rank 0 sends POSIX FDs to its peers over a unix socket. Placing a table this way is faster than filling /dev/shm.

Classes:

  • ExportedHostTable –

    What a peer needs to map a table; the FDs travel over socket_path.

Functions:

  • host_memory_supported –

    Whether HIP host allocations export POSIX FDs that the CPU can map.

  • share_host_table –

    Map one table on every rank of a node-local group, or None on all of them.

ExportedHostTable

Bases: NamedTuple

What a peer needs to map a table; the FDs travel over socket_path.

Source code in vllm/models/deepseek_v41/amd/ops/engram_host_table.py
class ExportedHostTable(NamedTuple):
    """What a peer needs to map a table; the FDs travel over `socket_path`."""

    padded_bytes: int
    socket_path: str

_check_reserve(numa_node, created, padded_bytes)

HIP may place host memory on another node, so watch the whole host too.

Source code in vllm/models/deepseek_v41/amd/ops/engram_host_table.py
def _check_reserve(numa_node: int, created: int, padded_bytes: int) -> None:
    """HIP may place host memory on another node, so watch the whole host too."""
    node = get_memory_node_info(numa_node)
    host = psutil.virtual_memory()
    for name, available, total in (
        (f"NUMA node {numa_node}", node.available_memory, node.total_memory),
        ("Host memory", host.available, host.total),
    ):
        reserve = total * NUMA_RESERVE_FRACTION
        if available < reserve:
            raise MemoryError(
                f"{name} fell below its {reserve / 2**30:.1f} GiB reserve after "
                f"{created / 2**30:.1f} of {padded_bytes / 2**30:.1f} GiB"
            )

_create_table(num_bytes)

Allocate a table and export one FD per chunk.

Source code in vllm/models/deepseek_v41/amd/ops/engram_host_table.py
def _create_table(num_bytes: int) -> tuple[list, list[int], int]:
    """Allocate a table and export one FD per chunk."""
    numa_node = _gpu_numa_node()
    padded_bytes = round_up(num_bytes, _page_bytes())
    handles: list = []
    fds: list[int] = []
    try:
        for offset, length in _chunk_ranges(padded_bytes):
            handles.append(_create_chunk(length, numa_node))
            _check_reserve(numa_node, offset + length, padded_bytes)
        for handle in handles:
            fds.append(_export_fd(handle))
    except BaseException:
        _close(fds)
        _release(handles)
        raise
    return handles, fds, padded_bytes

_map_table(table, num_bytes, fds, handles)

Map a table on the GPU and the CPU, importing its FDs unless given handles.

Imported handles are appended to handles; the caller releases them and closes the FDs, since the mappings keep the allocations alive. HIP grants no CPU access to imported host allocations, so the CPU maps the dmabuf FDs directly. Returns CPU and GPU uint8 views; either keeps the mappings alive.

Source code in vllm/models/deepseek_v41/amd/ops/engram_host_table.py
def _map_table(
    table: ExportedHostTable, num_bytes: int, fds: list[int], handles: list
) -> tuple[torch.Tensor, torch.Tensor]:
    """Map a table on the GPU and the CPU, importing its FDs unless given handles.

    Imported handles are appended to `handles`; the caller releases them and
    closes the FDs, since the mappings keep the allocations alive. HIP grants
    no CPU access to imported host allocations, so the CPU maps the dmabuf FDs
    directly. Returns CPU and GPU uint8 views; either keeps the mappings alive.
    """
    device = torch.accelerator.current_device_index()
    size = table.padded_bytes
    ranges = _chunk_ranges(size)
    libc = ctypes.CDLL(None, use_errno=True)
    libc.mmap.restype = ctypes.c_void_p
    libc.mmap.argtypes = [
        ctypes.c_void_p,
        ctypes.c_size_t,
        ctypes.c_int,
        ctypes.c_int,
        ctypes.c_int,
        ctypes.c_long,
    ]
    host_address: int | None = None
    gpu_address: int | None = None
    mapped: list[tuple[int, int]] = []
    try:
        if not handles:
            for fd in fds:
                handle = ctypes.c_void_p()
                _call(
                    "hipMemImportFromShareableHandle",
                    ctypes.byref(handle),
                    ctypes.c_void_p(fd),
                    _POSIX_FD,
                )
                handles.append(handle)
        address = ctypes.c_void_p()
        _call(
            "hipMemAddressReserve", ctypes.byref(address), size, _page_bytes(), None, 0
        )
        gpu_address = address.value
        assert gpu_address is not None
        for (offset, length), handle in zip(ranges, handles):
            _call("hipMemMap", gpu_address + offset, length, 0, handle, 0)
            mapped.append((offset, length))
        access = _AccessDesc(_Location(_LOCATION_DEVICE, device), _PROT_READWRITE)
        _call("hipMemSetAccess", gpu_address, size, ctypes.byref(access), 1)

        host_address = libc.mmap(
            None, size, _PROT_NONE, mmap.MAP_PRIVATE | mmap.MAP_ANONYMOUS, -1, 0
        )
        if host_address == ctypes.c_void_p(-1).value:
            host_address = None
            raise OSError(ctypes.get_errno(), "mmap failed")
        for (offset, length), fd in zip(ranges, fds):
            chunk = libc.mmap(
                host_address + offset,
                length,
                mmap.PROT_READ | mmap.PROT_WRITE,
                mmap.MAP_SHARED | _MAP_FIXED,
                fd,
                0,
            )
            if chunk != host_address + offset:
                raise OSError(ctypes.get_errno(), "mmap of an Engram FD failed")
    except BaseException:
        if gpu_address is not None:
            _unmap(host_address, gpu_address, size, mapped)
        raise

    assert host_address is not None and gpu_address is not None
    host_buffer = (ctypes.c_uint8 * num_bytes).from_address(host_address)
    finalizer = weakref.finalize(
        host_buffer, _unmap, host_address, gpu_address, size, mapped
    )
    finalizer.atexit = False  # type: ignore[misc]
    # Torch retains this object, so the GPU view keeps the buffer alive too.
    # Name the device: an index-less "cuda" makes torch copy the table.
    gpu_view = torch.as_tensor(
        SimpleNamespace(
            __cuda_array_interface__={
                "shape": (num_bytes,),
                "typestr": "|u1",
                "data": (gpu_address, False),
                "version": 3,
            },
            owner=host_buffer,
        ),
        device=torch.device("cuda", device),
    )
    return torch.frombuffer(host_buffer, dtype=torch.uint8), gpu_view

_page_bytes() cached

The recommended host allocation granularity, which sets the GPU page size.

Source code in vllm/models/deepseek_v41/amd/ops/engram_host_table.py
@functools.cache
def _page_bytes() -> int:
    """The recommended host allocation granularity, which sets the GPU page size."""
    granularity = ctypes.c_size_t()
    prop = _host_allocation_prop(_gpu_numa_node())
    _call(
        "hipMemGetAllocationGranularity",
        ctypes.byref(granularity),
        ctypes.byref(prop),
        _GRANULARITY_RECOMMENDED,
    )
    return granularity.value

_try_call(name, *args)

Calls a HIP function and returns its error, clearing it: a failed hipMem* call otherwise fails the next torch launch on this thread.

Source code in vllm/models/deepseek_v41/amd/ops/engram_host_table.py
def _try_call(name: str, *args) -> int:
    """Calls a HIP function and returns its error, clearing it: a failed hipMem*
    call otherwise fails the next torch launch on this thread."""
    if error := getattr(_hip(), name)(*args):
        _hip().hipGetLastError()
    return error

host_memory_supported() cached

Whether HIP host allocations export POSIX FDs that the CPU can map.

Source code in vllm/models/deepseek_v41/amd/ops/engram_host_table.py
@functools.cache
def host_memory_supported() -> bool:
    """Whether HIP host allocations export POSIX FDs that the CPU can map."""
    try:
        page_bytes = _page_bytes()
        handle = _create_chunk(page_bytes, _gpu_numa_node())
        try:
            fd = _export_fd(handle)
            try:
                mmap.mmap(fd, page_bytes).close()
            finally:
                os.close(fd)
        finally:
            _try_call("hipMemRelease", handle)
    except Exception as exc:
        logger.info_once(
            "HIP VMM host memory is unavailable (%s); Engram DP replicas share "
            "tables through /dev/shm.",
            exc,
        )
        return False
    return True

share_host_table(num_bytes, group)

Map one table on every rank of a node-local group, or None on all of them.

Rank 0 creates the table and sends its FDs to the others over a unix socket in a private directory. Returns CPU and GPU uint8 views of the table.

Source code in vllm/models/deepseek_v41/amd/ops/engram_host_table.py
def share_host_table(
    num_bytes: int, group: GroupCoordinator
) -> tuple[torch.Tensor, torch.Tensor] | None:
    """Map one table on every rank of a node-local group, or None on all of them.

    Rank 0 creates the table and sends its FDs to the others over a unix socket
    in a private directory. Returns CPU and GPU uint8 views of the table.
    """
    handles: list = []
    fds: list[int] = []
    table = error = server = directory = None
    # Rank 0 decides, so every rank agrees whatever its own environment.
    if (
        group.rank_in_group == 0
        and envs.VLLM_ROCM_ENGRAM_HIP_VMM
        and host_memory_supported()
    ):
        try:
            handles, fds, padded_bytes = _create_table(num_bytes)
            # The /dev/shm fallback needs SHM_PATH shared too, so both reach
            # the same ranks.
            directory = tempfile.mkdtemp(prefix="vllm_engram_", dir=SHM_PATH)
            server = socket.socket(socket.AF_UNIX)
            server.settimeout(SOCKET_TIMEOUT_S)
            server.bind(path := os.path.join(directory, "fds"))
            server.listen(group.world_size)
            table = ExportedHostTable(padded_bytes, path)
        except Exception as exc:
            error = describe_error(exc)
    table, error = group.broadcast_object((table, error))
    views = None
    try:
        if table is not None:
            if server is not None:
                for _ in range(group.world_size - 1):
                    connection, _ = server.accept()
                    with connection:
                        for fd in fds:
                            socket.send_fds(connection, [b"\0"], [fd])
            else:
                count = len(_chunk_ranges(table.padded_bytes))
                fds = _receive_fds(table.socket_path, count)
            views = _map_table(table, num_bytes, fds, handles)
    except Exception as exc:
        error = describe_error(exc)
    finally:
        _close(fds)
        _release(handles)
        if server is not None:
            server.close()
        if directory is not None:
            shutil.rmtree(directory, ignore_errors=True)
    if table is not None and not (error := gather_errors(group, error)):
        return views
    if error and group.rank_in_group == 0:
        logger.warning(
            "Engram host table cannot use HIP VMM host memory (%s); falling back "
            "to /dev/shm.",
            error,
        )
    return None