openpilot v0.9.6 release

date: 2024-01-12T10:13:37
master commit: ba792d576a49a0899b88a753fa1c52956bedf9e6

cereal/visionipc/.gitignore

@@ -0,0 +1,2 @@
+visionipc_pyx.cpp
+*.so

cereal/visionipc/__init__.py

@@ -0,0 +1,6 @@
+from cereal.visionipc.visionipc_pyx import VisionBuf, VisionIpcClient, VisionIpcServer, VisionStreamType, get_endpoint_name
+assert VisionBuf
+assert VisionIpcClient
+assert VisionIpcServer
+assert VisionStreamType
+assert get_endpoint_name

cereal/visionipc/ipc.cc

@@ -0,0 +1,121 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <assert.h>
+#include <errno.h>
+
+#include <sys/mman.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+
+#ifdef __APPLE__
+#define getsocket() socket(AF_UNIX, SOCK_STREAM, 0)
+#else
+#define getsocket() socket(AF_UNIX, SOCK_SEQPACKET, 0)
+#endif
+
+#include "cereal/visionipc/ipc.h"
+
+int ipc_connect(const char* socket_path) {
+  int err;
+
+  int sock = getsocket();
+
+  if (sock < 0) return -1;
+  struct sockaddr_un addr = {
+    .sun_family = AF_UNIX,
+  };
+  snprintf(addr.sun_path, sizeof(addr.sun_path), "%s", socket_path);
+  err = connect(sock, (struct sockaddr*)&addr, sizeof(addr));
+  if (err != 0) {
+    close(sock);
+    return -1;
+  }
+
+  return sock;
+}
+
+int ipc_bind(const char* socket_path) {
+  int err;
+
+  unlink(socket_path);
+
+  int sock = getsocket();
+
+  struct sockaddr_un addr = {
+    .sun_family = AF_UNIX,
+  };
+  snprintf(addr.sun_path, sizeof(addr.sun_path), "%s", socket_path);
+  err = bind(sock, (struct sockaddr *)&addr, sizeof(addr));
+  assert(err == 0);
+
+  err = listen(sock, 3);
+  assert(err == 0);
+
+  return sock;
+}
+
+
+int ipc_sendrecv_with_fds(bool send, int fd, void *buf, size_t buf_size, int* fds, int num_fds,
+                          int *out_num_fds) {
+  char control_buf[CMSG_SPACE(sizeof(int) * num_fds)];
+  memset(control_buf, 0, CMSG_SPACE(sizeof(int) * num_fds));
+
+  struct iovec iov = {
+    .iov_base = buf,
+    .iov_len = buf_size,
+  };
+  struct msghdr msg = {
+    .msg_iov = &iov,
+    .msg_iovlen = 1,
+  };
+
+  if (num_fds > 0) {
+    assert(fds);
+
+    msg.msg_control = control_buf;
+    msg.msg_controllen = CMSG_SPACE(sizeof(int) * num_fds);
+  }
+
+  if (send) {
+    if (num_fds) {
+      struct cmsghdr *cmsg = CMSG_FIRSTHDR(&msg);
+      assert(cmsg);
+      cmsg->cmsg_level = SOL_SOCKET;
+      cmsg->cmsg_type = SCM_RIGHTS;
+      cmsg->cmsg_len = CMSG_LEN(sizeof(int) * num_fds);
+      memcpy(CMSG_DATA(cmsg), fds, sizeof(int) * num_fds);
+    }
+    return sendmsg(fd, &msg, 0);
+  } else {
+    int r = recvmsg(fd, &msg, 0);
+    if (r < 0) return r;
+
+    int recv_fds = 0;
+    if (msg.msg_controllen > 0) {
+      struct cmsghdr *cmsg = CMSG_FIRSTHDR(&msg);
+      assert(cmsg);
+      assert(cmsg->cmsg_level == SOL_SOCKET && cmsg->cmsg_type == SCM_RIGHTS);
+      recv_fds = (cmsg->cmsg_len - CMSG_LEN(0));
+      assert(recv_fds > 0 && (recv_fds % sizeof(int)) == 0);
+      recv_fds /= sizeof(int);
+
+      assert(fds && recv_fds <= num_fds);
+      memcpy(fds, CMSG_DATA(cmsg), sizeof(int) * recv_fds);
+    }
+
+    if (msg.msg_flags) {
+      for (int i=0; i<recv_fds; i++) {
+        close(fds[i]);
+      }
+      return -1;
+    }
+
+    if (fds) {
+      assert(out_num_fds);
+      *out_num_fds = recv_fds;
+    }
+    return r;
+  }
+}

cereal/visionipc/ipc.h

@@ -0,0 +1,7 @@
+#pragma once
+#include <cstddef>
+
+int ipc_connect(const char* socket_path);
+int ipc_bind(const char* socket_path);
+int ipc_sendrecv_with_fds(bool send, int fd, void *buf, size_t buf_size, int* fds, int num_fds,
+                          int *out_num_fds);

cereal/visionipc/test_runner.cc

@@ -0,0 +1,2 @@
+#define CATCH_CONFIG_MAIN
+#include "catch2/catch.hpp"

cereal/visionipc/visionbuf.cc

@@ -0,0 +1,35 @@
+#include "cereal/visionipc/visionbuf.h"
+
+#define ALIGN(x, align) (((x) + (align)-1) & ~((align)-1))
+
+void visionbuf_compute_aligned_width_and_height(int width, int height, int *aligned_w, int *aligned_h) {
+  *aligned_w = width;
+  *aligned_h = height;
+}
+
+void VisionBuf::init_rgb(size_t init_width, size_t init_height, size_t init_stride) {
+  this->rgb = true;
+  this->width = init_width;
+  this->height = init_height;
+  this->stride = init_stride;
+}
+
+void VisionBuf::init_yuv(size_t init_width, size_t init_height, size_t init_stride, size_t init_uv_offset){
+  this->rgb = false;
+  this->width = init_width;
+  this->height = init_height;
+  this->stride = init_stride;
+  this->uv_offset = init_uv_offset;
+
+  this->y = (uint8_t *)this->addr;
+  this->uv = this->y + this->uv_offset;
+}
+
+
+uint64_t VisionBuf::get_frame_id() {
+  return *frame_id;
+}
+
+void VisionBuf::set_frame_id(uint64_t id) {
+  *frame_id = id;
+}

cereal/visionipc/visionbuf.h

@@ -0,0 +1,66 @@
+#pragma once
+
+#include "cereal/visionipc/visionipc.h"
+
+#define CL_USE_DEPRECATED_OPENCL_1_2_APIS
+#ifdef __APPLE__
+#include <OpenCL/cl.h>
+#else
+#include <CL/cl.h>
+#endif
+
+#define VISIONBUF_SYNC_FROM_DEVICE 0
+#define VISIONBUF_SYNC_TO_DEVICE 1
+
+enum VisionStreamType {
+  VISION_STREAM_ROAD,
+  VISION_STREAM_DRIVER,
+  VISION_STREAM_WIDE_ROAD,
+
+  VISION_STREAM_MAP,
+  VISION_STREAM_MAX,
+};
+
+class VisionBuf {
+ public:
+  size_t len = 0;
+  size_t mmap_len = 0;
+  void * addr = nullptr;
+  uint64_t *frame_id;
+  int fd = 0;
+
+  bool rgb = false;
+  size_t width = 0;
+  size_t height = 0;
+  size_t stride = 0;
+  size_t uv_offset = 0;
+
+  // YUV
+  uint8_t * y = nullptr;
+  uint8_t * uv = nullptr;
+
+  // Visionipc
+  uint64_t server_id = 0;
+  size_t idx = 0;
+  VisionStreamType type;
+
+  // OpenCL
+  cl_mem buf_cl = nullptr;
+  cl_command_queue copy_q = nullptr;
+
+  // ion
+  int handle = 0;
+
+  void allocate(size_t len);
+  void import();
+  void init_cl(cl_device_id device_id, cl_context ctx);
+  void init_rgb(size_t width, size_t height, size_t stride);
+  void init_yuv(size_t width, size_t height, size_t stride, size_t uv_offset);
+  int sync(int dir);
+  int free();
+
+  void set_frame_id(uint64_t id);
+  uint64_t get_frame_id();
+};
+
+void visionbuf_compute_aligned_width_and_height(int width, int height, int *aligned_w, int *aligned_h);

cereal/visionipc/visionbuf_cl.cc

@@ -0,0 +1,94 @@
+#include "cereal/visionipc/visionbuf.h"
+
+#include <atomic>
+#include <stdio.h>
+#include <fcntl.h>
+#include <assert.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <sys/mman.h>
+#include <sys/types.h>
+
+std::atomic<int> offset = 0;
+
+static void *malloc_with_fd(size_t len, int *fd) {
+  char full_path[0x100];
+
+#ifdef __APPLE__
+  snprintf(full_path, sizeof(full_path)-1, "/tmp/visionbuf_%d_%d", getpid(), offset++);
+#else
+  snprintf(full_path, sizeof(full_path)-1, "/dev/shm/visionbuf_%d_%d", getpid(), offset++);
+#endif
+
+  *fd = open(full_path, O_RDWR | O_CREAT, 0664);
+  assert(*fd >= 0);
+
+  unlink(full_path);
+
+  ftruncate(*fd, len);
+  void *addr = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, *fd, 0);
+  assert(addr != MAP_FAILED);
+
+  return addr;
+}
+
+void VisionBuf::allocate(size_t length) {
+  this->len = length;
+  this->mmap_len = this->len + sizeof(uint64_t);
+  this->addr = malloc_with_fd(this->mmap_len, &this->fd);
+  this->frame_id = (uint64_t*)((uint8_t*)this->addr + this->len);
+}
+
+void VisionBuf::init_cl(cl_device_id device_id, cl_context ctx){
+  int err;
+
+  this->copy_q = clCreateCommandQueue(ctx, device_id, 0, &err);
+  assert(err == 0);
+
+  this->buf_cl = clCreateBuffer(ctx, CL_MEM_READ_WRITE | CL_MEM_USE_HOST_PTR, this->len, this->addr, &err);
+  assert(err == 0);
+}
+
+
+void VisionBuf::import(){
+  assert(this->fd >= 0);
+  this->addr = mmap(NULL, this->mmap_len, PROT_READ | PROT_WRITE, MAP_SHARED, this->fd, 0);
+  assert(this->addr != MAP_FAILED);
+
+  this->frame_id = (uint64_t*)((uint8_t*)this->addr + this->len);
+}
+
+
+int VisionBuf::sync(int dir) {
+  int err = 0;
+  if (!this->buf_cl) return 0;
+
+  if (dir == VISIONBUF_SYNC_FROM_DEVICE) {
+    err = clEnqueueReadBuffer(this->copy_q, this->buf_cl, CL_FALSE, 0, this->len, this->addr, 0, NULL, NULL);
+  } else {
+    err = clEnqueueWriteBuffer(this->copy_q, this->buf_cl, CL_FALSE, 0, this->len, this->addr, 0, NULL, NULL);
+  }
+
+  if (err == 0){
+    err = clFinish(this->copy_q);
+  }
+
+  return err;
+}
+
+int VisionBuf::free() {
+  int err = 0;
+  if (this->buf_cl){
+    err = clReleaseMemObject(this->buf_cl);
+    if (err != 0) return err;
+
+    err = clReleaseCommandQueue(this->copy_q);
+    if (err != 0) return err;
+  }
+
+  err = munmap(this->addr, this->len);
+  if (err != 0) return err;
+
+  err = close(this->fd);
+  return err;
+}

cereal/visionipc/visionbuf_ion.cc

@@ -0,0 +1,161 @@
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <assert.h>
+#include <errno.h>
+#include <sys/mman.h>
+#include <sys/ioctl.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <linux/ion.h>
+#include <CL/cl_ext.h>
+
+#include <msm_ion.h>
+
+#include "cereal/visionipc/visionbuf.h"
+
+// keep trying if x gets interrupted by a signal
+#define HANDLE_EINTR(x)                                       \
+  ({                                                          \
+    decltype(x) ret;                                          \
+    int try_cnt = 0;                                          \
+    do {                                                      \
+      ret = (x);                                              \
+    } while (ret == -1 && errno == EINTR && try_cnt++ < 100); \
+    ret;                                                      \
+  })
+
+// just hard-code these for convenience
+// size_t device_page_size = 0;
+// clGetDeviceInfo(device_id, CL_DEVICE_PAGE_SIZE_QCOM,
+//                 sizeof(device_page_size), &device_page_size,
+//                 NULL);
+
+// size_t padding_cl = 0;
+// clGetDeviceInfo(device_id, CL_DEVICE_EXT_MEM_PADDING_IN_BYTES_QCOM,
+//                 sizeof(padding_cl), &padding_cl,
+//                 NULL);
+#define DEVICE_PAGE_SIZE_CL 4096
+#define PADDING_CL 0
+
+struct IonFileHandle {
+  IonFileHandle() {
+    fd = open("/dev/ion", O_RDWR | O_NONBLOCK);
+    assert(fd >= 0);
+  }
+  ~IonFileHandle() {
+    close(fd);
+  }
+  int fd = -1;
+};
+
+int ion_fd() {
+  static IonFileHandle fh;
+  return fh.fd;
+}
+
+void VisionBuf::allocate(size_t length) {
+  struct ion_allocation_data ion_alloc = {0};
+  ion_alloc.len = length + PADDING_CL + sizeof(uint64_t);
+  ion_alloc.align = 4096;
+  ion_alloc.heap_id_mask = 1 << ION_IOMMU_HEAP_ID;
+  ion_alloc.flags = ION_FLAG_CACHED;
+
+  int err = HANDLE_EINTR(ioctl(ion_fd(), ION_IOC_ALLOC, &ion_alloc));
+  assert(err == 0);
+
+  struct ion_fd_data ion_fd_data = {0};
+  ion_fd_data.handle = ion_alloc.handle;
+  err = HANDLE_EINTR(ioctl(ion_fd(), ION_IOC_SHARE, &ion_fd_data));
+  assert(err == 0);
+
+  void *mmap_addr = mmap(NULL, ion_alloc.len,
+                         PROT_READ | PROT_WRITE,
+                         MAP_SHARED, ion_fd_data.fd, 0);
+  assert(mmap_addr != MAP_FAILED);
+
+  memset(mmap_addr, 0, ion_alloc.len);
+
+  this->len = length;
+  this->mmap_len = ion_alloc.len;
+  this->addr = mmap_addr;
+  this->handle = ion_alloc.handle;
+  this->fd = ion_fd_data.fd;
+  this->frame_id = (uint64_t*)((uint8_t*)this->addr + this->len + PADDING_CL);
+}
+
+void VisionBuf::import(){
+  int err;
+  assert(this->fd >= 0);
+
+  // Get handle
+  struct ion_fd_data fd_data = {0};
+  fd_data.fd = this->fd;
+  err = HANDLE_EINTR(ioctl(ion_fd(), ION_IOC_IMPORT, &fd_data));
+  assert(err == 0);
+
+  this->handle = fd_data.handle;
+  this->addr = mmap(NULL, this->mmap_len, PROT_READ | PROT_WRITE, MAP_SHARED, this->fd, 0);
+  assert(this->addr != MAP_FAILED);
+
+  this->frame_id = (uint64_t*)((uint8_t*)this->addr + this->len + PADDING_CL);
+}
+
+void VisionBuf::init_cl(cl_device_id device_id, cl_context ctx) {
+  int err;
+
+  assert(((uintptr_t)this->addr % DEVICE_PAGE_SIZE_CL) == 0);
+
+  cl_mem_ion_host_ptr ion_cl = {0};
+  ion_cl.ext_host_ptr.allocation_type = CL_MEM_ION_HOST_PTR_QCOM;
+  ion_cl.ext_host_ptr.host_cache_policy = CL_MEM_HOST_UNCACHED_QCOM;
+  ion_cl.ion_filedesc = this->fd;
+  ion_cl.ion_hostptr = this->addr;
+
+  this->buf_cl = clCreateBuffer(ctx,
+                              CL_MEM_USE_HOST_PTR | CL_MEM_EXT_HOST_PTR_QCOM,
+                              this->len, &ion_cl, &err);
+  assert(err == 0);
+}
+
+
+int VisionBuf::sync(int dir) {
+  struct ion_flush_data flush_data = {0};
+  flush_data.handle = this->handle;
+  flush_data.vaddr = this->addr;
+  flush_data.offset = 0;
+  flush_data.length = this->len;
+
+  // ION_IOC_INV_CACHES ~= DMA_FROM_DEVICE
+  // ION_IOC_CLEAN_CACHES ~= DMA_TO_DEVICE
+  // ION_IOC_CLEAN_INV_CACHES ~= DMA_BIDIRECTIONAL
+
+  struct ion_custom_data custom_data = {0};
+
+   assert(dir == VISIONBUF_SYNC_FROM_DEVICE || dir == VISIONBUF_SYNC_TO_DEVICE);
+   custom_data.cmd = (dir == VISIONBUF_SYNC_FROM_DEVICE) ?
+     ION_IOC_INV_CACHES : ION_IOC_CLEAN_CACHES;
+
+  custom_data.arg = (unsigned long)&flush_data;
+  return HANDLE_EINTR(ioctl(ion_fd(), ION_IOC_CUSTOM, &custom_data));
+}
+
+int VisionBuf::free() {
+  int err = 0;
+
+  if (this->buf_cl){
+    err = clReleaseMemObject(this->buf_cl);
+    if (err != 0) return err;
+  }
+
+  err = munmap(this->addr, this->mmap_len);
+  if (err != 0) return err;
+
+  err = close(this->fd);
+  if (err != 0) return err;
+
+  struct ion_handle_data handle_data = {.handle = this->handle};
+  return HANDLE_EINTR(ioctl(ion_fd(), ION_IOC_FREE, &handle_data));
+}

cereal/visionipc/visionipc.h

@@ -0,0 +1,19 @@
+#pragma once
+
+#include <cstdint>
+#include <cstddef>
+
+constexpr int VISIONIPC_MAX_FDS = 128;
+
+struct VisionIpcBufExtra {
+  uint32_t frame_id;
+  uint64_t timestamp_sof;
+  uint64_t timestamp_eof;
+  bool valid;
+};
+
+struct VisionIpcPacket {
+  uint64_t server_id;
+  size_t idx;
+  struct VisionIpcBufExtra extra;
+};

cereal/visionipc/visionipc.pxd

@@ -0,0 +1,60 @@
+# distutils: language = c++
+#cython: language_level=3
+
+from libcpp.string cimport string
+from libcpp.vector cimport vector
+from libcpp.set cimport set
+from libc.stdint cimport uint32_t, uint64_t
+from libcpp cimport bool, int
+
+cdef extern from "cereal/visionipc/visionbuf.h":
+  struct _cl_device_id
+  struct _cl_context
+  struct _cl_mem
+
+  ctypedef _cl_device_id * cl_device_id
+  ctypedef _cl_context * cl_context
+  ctypedef _cl_mem * cl_mem
+
+  cdef enum VisionStreamType:
+    pass
+
+  cdef cppclass VisionBuf:
+    void * addr
+    bool rgb
+    size_t len
+    size_t width
+    size_t height
+    size_t stride
+    size_t uv_offset
+    cl_mem buf_cl
+    void set_frame_id(uint64_t id)
+
+cdef extern from "cereal/visionipc/visionipc.h":
+  struct VisionIpcBufExtra:
+    uint32_t frame_id
+    uint64_t timestamp_sof
+    uint64_t timestamp_eof
+    bool valid
+
+cdef extern from "cereal/visionipc/visionipc_server.h":
+  string get_endpoint_name(string, VisionStreamType)
+
+  cdef cppclass VisionIpcServer:
+    VisionIpcServer(string, void*, void*)
+    void create_buffers(VisionStreamType, size_t, bool, size_t, size_t)
+    void create_buffers_with_sizes(VisionStreamType, size_t, bool, size_t, size_t, size_t, size_t, size_t)
+    VisionBuf * get_buffer(VisionStreamType)
+    void send(VisionBuf *, VisionIpcBufExtra *, bool)
+    void start_listener()
+
+cdef extern from "cereal/visionipc/visionipc_client.h":
+  cdef cppclass VisionIpcClient:
+    int num_buffers
+    VisionBuf buffers[1]
+    VisionIpcClient(string, VisionStreamType, bool, void*, void*)
+    VisionBuf * recv(VisionIpcBufExtra *, int)
+    bool connect(bool)
+    bool is_connected()
+    @staticmethod
+    set[VisionStreamType] getAvailableStreams(string, bool)

cereal/visionipc/visionipc_client.cc

@@ -0,0 +1,143 @@
+#include <chrono>
+#include <cassert>
+#include <iostream>
+#include <thread>
+
+#include <unistd.h>
+#include "cereal/visionipc/ipc.h"
+#include "cereal/visionipc/visionipc_client.h"
+#include "cereal/visionipc/visionipc_server.h"
+#include "cereal/logger/logger.h"
+
+static int connect_to_vipc_server(const std::string &name, bool blocking) {
+  const std::string ipc_path = get_ipc_path(name);
+  int socket_fd = ipc_connect(ipc_path.c_str());
+  while (socket_fd < 0 && blocking) {
+    std::cout << "VisionIpcClient connecting" << std::endl;
+    std::this_thread::sleep_for(std::chrono::milliseconds(100));
+    socket_fd = ipc_connect(ipc_path.c_str());
+  }
+  return socket_fd;
+}
+
+VisionIpcClient::VisionIpcClient(std::string name, VisionStreamType type, bool conflate, cl_device_id device_id, cl_context ctx) : name(name), type(type), device_id(device_id), ctx(ctx) {
+  msg_ctx = Context::create();
+  sock = SubSocket::create(msg_ctx, get_endpoint_name(name, type), "127.0.0.1", conflate, false);
+
+  poller = Poller::create();
+  poller->registerSocket(sock);
+}
+
+// Connect is not thread safe. Do not use the buffers while calling connect
+bool VisionIpcClient::connect(bool blocking){
+  connected = false;
+
+  // Cleanup old buffers on reconnect
+  for (size_t i = 0; i < num_buffers; i++){
+    if (buffers[i].free() != 0) {
+      LOGE("Failed to free buffer %zu", i);
+    }
+  }
+
+  num_buffers = 0;
+
+  int socket_fd = connect_to_vipc_server(name, blocking);
+  if (socket_fd < 0) {
+    return false;
+  }
+  // Send stream type to server to request FDs
+  int r = ipc_sendrecv_with_fds(true, socket_fd, &type, sizeof(type), nullptr, 0, nullptr);
+  assert(r == sizeof(type));
+
+  // Get FDs
+  int fds[VISIONIPC_MAX_FDS];
+  VisionBuf bufs[VISIONIPC_MAX_FDS];
+  r = ipc_sendrecv_with_fds(false, socket_fd, &bufs, sizeof(bufs), fds, VISIONIPC_MAX_FDS, &num_buffers);
+
+  assert(num_buffers >= 0);
+  assert(r == sizeof(VisionBuf) * num_buffers);
+
+  // Import buffers
+  for (size_t i = 0; i < num_buffers; i++){
+    buffers[i] = bufs[i];
+    buffers[i].fd = fds[i];
+    buffers[i].import();
+    if (buffers[i].rgb) {
+      buffers[i].init_rgb(buffers[i].width, buffers[i].height, buffers[i].stride);
+    } else {
+      buffers[i].init_yuv(buffers[i].width, buffers[i].height, buffers[i].stride, buffers[i].uv_offset);
+    }
+
+    if (device_id) buffers[i].init_cl(device_id, ctx);
+  }
+
+  close(socket_fd);
+  connected = true;
+  return true;
+}
+
+VisionBuf * VisionIpcClient::recv(VisionIpcBufExtra * extra, const int timeout_ms){
+  auto p = poller->poll(timeout_ms);
+
+  if (!p.size()){
+    return nullptr;
+  }
+
+  Message * r = sock->receive(true);
+  if (r == nullptr){
+    return nullptr;
+  }
+
+  // Get buffer
+  assert(r->getSize() == sizeof(VisionIpcPacket));
+  VisionIpcPacket *packet = (VisionIpcPacket*)r->getData();
+
+  assert(packet->idx < num_buffers);
+  VisionBuf * buf = &buffers[packet->idx];
+
+  if (buf->server_id != packet->server_id){
+    connected = false;
+    delete r;
+    return nullptr;
+  }
+
+  if (extra) {
+    *extra = packet->extra;
+  }
+
+  if (buf->sync(VISIONBUF_SYNC_TO_DEVICE) != 0) {
+    LOGE("Failed to sync buffer");
+  }
+
+  delete r;
+  return buf;
+}
+
+std::set<VisionStreamType> VisionIpcClient::getAvailableStreams(const std::string &name, bool blocking) {
+  int socket_fd = connect_to_vipc_server(name, blocking);
+  if (socket_fd < 0) {
+    return {};
+  }
+  // Send VISION_STREAM_MAX to server to request available streams
+  int request = VISION_STREAM_MAX;
+  int r = ipc_sendrecv_with_fds(true, socket_fd, &request, sizeof(request), nullptr, 0, nullptr);
+  assert(r == sizeof(request));
+
+  VisionStreamType available_streams[VISION_STREAM_MAX] = {};
+  r = ipc_sendrecv_with_fds(false, socket_fd, &available_streams, sizeof(available_streams), nullptr, 0, nullptr);
+  assert((r >= 0) && (r % sizeof(VisionStreamType) == 0));
+  close(socket_fd);
+  return std::set<VisionStreamType>(available_streams, available_streams + r / sizeof(VisionStreamType));
+}
+
+VisionIpcClient::~VisionIpcClient(){
+  for (size_t i = 0; i < num_buffers; i++){
+    if (buffers[i].free() != 0) {
+      LOGE("Failed to free buffer %zu", i);
+    }
+  }
+
+  delete sock;
+  delete poller;
+  delete msg_ctx;
+}

cereal/visionipc/visionipc_client.h

@@ -0,0 +1,30 @@
+#pragma once
+
+#include <set>
+#include <string>
+
+#include "cereal/messaging/messaging.h"
+#include "cereal/visionipc/visionbuf.h"
+
+class VisionIpcClient {
+private:
+  std::string name;
+  Context * msg_ctx;
+  SubSocket * sock;
+  Poller * poller;
+
+  cl_device_id device_id = nullptr;
+  cl_context ctx = nullptr;
+
+public:
+  bool connected = false;
+  VisionStreamType type;
+  int num_buffers = 0;
+  VisionBuf buffers[VISIONIPC_MAX_FDS];
+  VisionIpcClient(std::string name, VisionStreamType type, bool conflate, cl_device_id device_id=nullptr, cl_context ctx=nullptr);
+  ~VisionIpcClient();
+  VisionBuf * recv(VisionIpcBufExtra * extra=nullptr, const int timeout_ms=100);
+  bool connect(bool blocking=true);
+  bool is_connected() { return connected; }
+  static std::set<VisionStreamType> getAvailableStreams(const std::string &name, bool blocking = true);
+};

cereal/visionipc/visionipc_pyx.pxd

@@ -0,0 +1,15 @@
+# distutils: language = c++
+#cython: language_level=3
+
+from .visionipc cimport VisionBuf as cppVisionBuf
+from .visionipc cimport cl_device_id, cl_context
+
+cdef class CLContext:
+  cdef cl_device_id device_id
+  cdef cl_context context
+
+cdef class VisionBuf:
+  cdef cppVisionBuf * buf
+
+  @staticmethod
+  cdef create(cppVisionBuf*)

cereal/visionipc/visionipc_pyx.pyx

@@ -0,0 +1,168 @@
+# distutils: language = c++
+# cython: c_string_encoding=ascii, language_level=3
+
+import sys
+import numpy as np
+cimport numpy as cnp
+from cython.view cimport array
+from libc.string cimport memcpy
+from libc.stdint cimport uint32_t, uint64_t
+from libcpp cimport bool
+from libcpp.string cimport string
+
+from .visionipc cimport VisionIpcServer as cppVisionIpcServer
+from .visionipc cimport VisionIpcClient as cppVisionIpcClient
+from .visionipc cimport VisionBuf as cppVisionBuf
+from .visionipc cimport VisionIpcBufExtra
+from .visionipc cimport get_endpoint_name as cpp_get_endpoint_name
+
+
+def get_endpoint_name(string name, VisionStreamType stream):
+  return cpp_get_endpoint_name(name, stream).decode('utf-8')
+
+
+cpdef enum VisionStreamType:
+  VISION_STREAM_ROAD
+  VISION_STREAM_DRIVER
+  VISION_STREAM_WIDE_ROAD
+  VISION_STREAM_MAP
+
+
+cdef class VisionBuf:
+  @staticmethod
+  cdef create(cppVisionBuf * cbuf):
+    buf = VisionBuf()
+    buf.buf = cbuf
+    return buf
+
+  @property
+  def data(self):
+    return np.asarray(<cnp.uint8_t[:self.buf.len]> self.buf.addr)
+
+  @property
+  def width(self):
+    return self.buf.width
+
+  @property
+  def height(self):
+    return self.buf.height
+
+  @property
+  def stride(self):
+    return self.buf.stride
+
+  @property
+  def uv_offset(self):
+    return self.buf.uv_offset
+
+  @property
+  def rgb(self):
+    return self.buf.rgb
+
+
+cdef class VisionIpcServer:
+  cdef cppVisionIpcServer * server
+
+  def __init__(self, string name):
+    self.server = new cppVisionIpcServer(name, NULL, NULL)
+
+  def create_buffers(self, VisionStreamType tp, size_t num_buffers, bool rgb, size_t width, size_t height):
+    self.server.create_buffers(tp, num_buffers, rgb, width, height)
+
+  def create_buffers_with_sizes(self, VisionStreamType tp, size_t num_buffers, bool rgb, size_t width, size_t height, size_t size, size_t stride, size_t uv_offset):
+    self.server.create_buffers_with_sizes(tp, num_buffers, rgb, width, height, size, stride, uv_offset)
+
+  def send(self, VisionStreamType tp, const unsigned char[:] data, uint32_t frame_id=0, uint64_t timestamp_sof=0, uint64_t timestamp_eof=0):
+    cdef cppVisionBuf * buf = self.server.get_buffer(tp)
+
+    # Populate buffer
+    assert buf.len == len(data)
+    memcpy(buf.addr, &data[0], len(data))
+    buf.set_frame_id(frame_id)
+
+    cdef VisionIpcBufExtra extra
+    extra.frame_id = frame_id
+    extra.timestamp_sof = timestamp_sof
+    extra.timestamp_eof = timestamp_eof
+
+    self.server.send(buf, &extra, False)
+
+  def start_listener(self):
+    self.server.start_listener()
+
+  def __dealloc__(self):
+    del self.server
+
+
+cdef class VisionIpcClient:
+  cdef cppVisionIpcClient * client
+  cdef VisionIpcBufExtra extra
+
+  def __cinit__(self, string name, VisionStreamType stream, bool conflate, CLContext context = None):
+    if context:
+      self.client = new cppVisionIpcClient(name, stream, conflate, context.device_id, context.context)
+    else:
+      self.client = new cppVisionIpcClient(name, stream, conflate, NULL, NULL)
+
+  def __dealloc__(self):
+    del self.client
+
+  @property
+  def width(self):
+    return self.client.buffers[0].width if self.client.num_buffers else None
+
+  @property
+  def height(self):
+    return self.client.buffers[0].height if self.client.num_buffers else None
+
+  @property
+  def stride(self):
+    return self.client.buffers[0].stride if self.client.num_buffers else None
+
+  @property
+  def uv_offset(self):
+    return self.client.buffers[0].uv_offset if self.client.num_buffers else None
+
+  @property
+  def rgb(self):
+    return self.client.buffers[0].rgb if self.client.num_buffers else None
+
+  @property
+  def buffer_len(self):
+    return self.client.buffers[0].len if self.client.num_buffers else None
+
+  @property
+  def num_buffers(self):
+    return self.client.num_buffers
+
+  @property
+  def frame_id(self):
+    return self.extra.frame_id
+
+  @property
+  def timestamp_sof(self):
+    return self.extra.timestamp_sof
+
+  @property
+  def timestamp_eof(self):
+    return self.extra.timestamp_eof
+
+  @property
+  def valid(self):
+    return self.extra.valid
+
+  def recv(self, int timeout_ms=100):
+    buf = self.client.recv(&self.extra, timeout_ms)
+    if not buf:
+      return None
+    return VisionBuf.create(buf)
+
+  def connect(self, bool blocking):
+    return self.client.connect(blocking)
+
+  def is_connected(self):
+    return self.client.is_connected()
+
+  @staticmethod
+  def available_streams(string name, bool block):
+    return cppVisionIpcClient.getAvailableStreams(name, block)

cereal/visionipc/visionipc_server.cc

@@ -0,0 +1,214 @@
+#include <iostream>
+#include <chrono>
+#include <cassert>
+#include <random>
+#include <limits>
+
+#include <poll.h>
+#include <sys/socket.h>
+#include <unistd.h>
+
+#include "cereal/messaging/messaging.h"
+#include "cereal/visionipc/ipc.h"
+#include "cereal/visionipc/visionipc_server.h"
+#include "cereal/logger/logger.h"
+
+std::string get_endpoint_name(std::string name, VisionStreamType type){
+  if (messaging_use_zmq()){
+    assert(name == "camerad" || name == "navd");
+    return std::to_string(9000 + static_cast<int>(type));
+  } else {
+    return "visionipc_" + name + "_" + std::to_string(type);
+  }
+}
+
+std::string get_ipc_path(const std::string& name) {
+  std::string path = "/tmp/";
+  if (char* prefix = std::getenv("OPENPILOT_PREFIX")) {
+    path += std::string(prefix) + "_";
+  }
+  return path + "visionipc_" + name;
+}
+
+VisionIpcServer::VisionIpcServer(std::string name, cl_device_id device_id, cl_context ctx) : name(name), device_id(device_id), ctx(ctx) {
+  msg_ctx = Context::create();
+
+  std::random_device rd("/dev/urandom");
+  std::uniform_int_distribution<uint64_t> distribution(0, std::numeric_limits<uint64_t>::max());
+  server_id = distribution(rd);
+}
+
+void VisionIpcServer::create_buffers(VisionStreamType type, size_t num_buffers, bool rgb, size_t width, size_t height){
+  // TODO: assert that this type is not created yet
+  assert(num_buffers < VISIONIPC_MAX_FDS);
+  int aligned_w = 0, aligned_h = 0;
+
+  size_t size = 0;
+  size_t stride = 0;
+  size_t uv_offset = 0;
+
+  if (rgb) {
+    visionbuf_compute_aligned_width_and_height(width, height, &aligned_w, &aligned_h);
+    size = (size_t)aligned_w * (size_t)aligned_h * 3;
+    stride = aligned_w * 3;
+  } else {
+    size = width * height * 3 / 2;
+    stride = width;
+    uv_offset = width * height;
+  }
+
+  create_buffers_with_sizes(type, num_buffers, rgb, width, height, size, stride, uv_offset);
+}
+
+void VisionIpcServer::create_buffers_with_sizes(VisionStreamType type, size_t num_buffers, bool rgb, size_t width, size_t height, size_t size, size_t stride, size_t uv_offset) {
+  // Create map + alloc requested buffers
+  for (size_t i = 0; i < num_buffers; i++){
+    VisionBuf* buf = new VisionBuf();
+    buf->allocate(size);
+    buf->idx = i;
+    buf->type = type;
+
+    if (device_id) buf->init_cl(device_id, ctx);
+
+    rgb ? buf->init_rgb(width, height, stride) : buf->init_yuv(width, height, stride, uv_offset);
+
+    buffers[type].push_back(buf);
+  }
+
+  cur_idx[type] = 0;
+
+  // Create msgq publisher for each of the `name` + type combos
+  // TODO: compute port number directly if using zmq
+  sockets[type] = PubSocket::create(msg_ctx, get_endpoint_name(name, type), false);
+}
+
+
+void VisionIpcServer::start_listener(){
+  listener_thread = std::thread(&VisionIpcServer::listener, this);
+}
+
+
+void VisionIpcServer::listener(){
+  std::cout << "Starting listener for: " << name << std::endl;
+
+  const std::string ipc_path = get_ipc_path(name);
+  int sock = ipc_bind(ipc_path.c_str());
+  assert(sock >= 0);
+
+  while (!should_exit){
+    // Wait for incoming connection
+    struct pollfd polls[1] = {{0}};
+    polls[0].fd = sock;
+    polls[0].events = POLLIN;
+
+    int ret = poll(polls, 1, 100);
+    if (ret < 0) {
+      if (errno == EINTR || errno == EAGAIN) continue;
+      std::cout << "poll failed, stopping listener" << std::endl;
+      break;
+    }
+
+    if (should_exit) break;
+    if (!polls[0].revents) {
+      continue;
+    }
+
+    // Handle incoming request
+    int fd = accept(sock, NULL, NULL);
+    assert(fd >= 0);
+
+    VisionStreamType type = VisionStreamType::VISION_STREAM_MAX;
+    int r = ipc_sendrecv_with_fds(false, fd, &type, sizeof(type), nullptr, 0, nullptr);
+    assert(r == sizeof(type));
+
+    // send available stream types
+    if (type == VisionStreamType::VISION_STREAM_MAX) {
+      std::vector<VisionStreamType> available_stream_types;
+      for (auto& [stream_type, _] : buffers) {
+        available_stream_types.push_back(stream_type);
+      }
+      r = ipc_sendrecv_with_fds(true, fd, available_stream_types.data(), available_stream_types.size() * sizeof(VisionStreamType), nullptr, 0, nullptr);
+      assert(r == available_stream_types.size() * sizeof(VisionStreamType));
+      close(fd);
+      continue;
+    }
+
+    if (buffers.count(type) <= 0) {
+      std::cout << "got request for invalid buffer type: " << type << std::endl;
+      close(fd);
+      continue;
+    }
+
+    int fds[VISIONIPC_MAX_FDS];
+    int num_fds = buffers[type].size();
+    VisionBuf bufs[VISIONIPC_MAX_FDS];
+
+    for (int i = 0; i < num_fds; i++){
+      fds[i] = buffers[type][i]->fd;
+      bufs[i] = *buffers[type][i];
+
+      // Remove some private openCL/ion metadata
+      bufs[i].buf_cl = 0;
+      bufs[i].copy_q = 0;
+      bufs[i].handle = 0;
+
+      bufs[i].server_id = server_id;
+    }
+
+    r = ipc_sendrecv_with_fds(true, fd, &bufs, sizeof(VisionBuf) * num_fds, fds, num_fds, nullptr);
+
+    close(fd);
+  }
+
+  std::cout << "Stopping listener for: " << name << std::endl;
+  close(sock);
+  unlink(ipc_path.c_str());
+}
+
+
+
+VisionBuf * VisionIpcServer::get_buffer(VisionStreamType type){
+  // Do we want to keep track if the buffer has been sent out yet and warn user?
+  assert(buffers.count(type));
+  auto b = buffers[type];
+  return b[cur_idx[type]++ % b.size()];
+}
+
+void VisionIpcServer::send(VisionBuf * buf, VisionIpcBufExtra * extra, bool sync){
+  if (sync) {
+    if (buf->sync(VISIONBUF_SYNC_FROM_DEVICE) != 0) {
+      LOGE("Failed to sync buffer");
+    }
+  }
+  assert(buffers.count(buf->type));
+  assert(buf->idx < buffers[buf->type].size());
+
+  // Send over correct msgq socket
+  VisionIpcPacket packet = {0};
+  packet.server_id = server_id;
+  packet.idx = buf->idx;
+  packet.extra = *extra;
+
+  sockets[buf->type]->send((char*)&packet, sizeof(packet));
+}
+
+VisionIpcServer::~VisionIpcServer(){
+  should_exit = true;
+  listener_thread.join();
+
+  // VisionBuf cleanup
+  for (auto const& [type, buf] : buffers) {
+    for (VisionBuf* b : buf){
+      if (b->free() != 0) {
+        LOGE("Failed to free buffer");
+      }
+      delete b;
+    }
+  }
+
+  // Messaging cleanup
+  for (auto const& [type, sock] : sockets) {
+    delete sock;
+  }
+  delete msg_ctx;
+}

cereal/visionipc/visionipc_server.h

@@ -0,0 +1,42 @@
+#pragma once
+#include <vector>
+#include <string>
+#include <thread>
+#include <atomic>
+#include <map>
+
+#include "cereal/messaging/messaging.h"
+#include "cereal/visionipc/visionbuf.h"
+
+std::string get_endpoint_name(std::string name, VisionStreamType type);
+std::string get_ipc_path(const std::string &name);
+
+class VisionIpcServer {
+ private:
+  cl_device_id device_id = nullptr;
+  cl_context ctx = nullptr;
+  uint64_t server_id;
+
+  std::atomic<bool> should_exit = false;
+  std::string name;
+  std::thread listener_thread;
+
+  std::map<VisionStreamType, std::atomic<size_t> > cur_idx;
+  std::map<VisionStreamType, std::vector<VisionBuf*> > buffers;
+
+  Context * msg_ctx;
+  std::map<VisionStreamType, PubSocket*> sockets;
+
+  void listener(void);
+
+ public:
+  VisionIpcServer(std::string name, cl_device_id device_id=nullptr, cl_context ctx=nullptr);
+  ~VisionIpcServer();
+
+  VisionBuf * get_buffer(VisionStreamType type);
+
+  void create_buffers(VisionStreamType type, size_t num_buffers, bool rgb, size_t width, size_t height);
+  void create_buffers_with_sizes(VisionStreamType type, size_t num_buffers, bool rgb, size_t width, size_t height, size_t size, size_t stride, size_t uv_offset);
+  void send(VisionBuf * buf, VisionIpcBufExtra * extra, bool sync=true);
+  void start_listener();
+};

cereal/visionipc/visionipc_tests.cc

@@ -0,0 +1,148 @@
+#include <thread>
+#include <chrono>
+
+#include "catch2/catch.hpp"
+#include "cereal/visionipc/visionipc_server.h"
+#include "cereal/visionipc/visionipc_client.h"
+
+static void zmq_sleep(int milliseconds=1000){
+  if (messaging_use_zmq()){
+    std::this_thread::sleep_for(std::chrono::milliseconds(milliseconds));
+  }
+}
+
+TEST_CASE("Connecting"){
+  VisionIpcServer server("camerad");
+  server.create_buffers(VISION_STREAM_ROAD, 1, false, 100, 100);
+  server.start_listener();
+
+  VisionIpcClient client = VisionIpcClient("camerad", VISION_STREAM_ROAD, false);
+  REQUIRE(client.connect());
+
+  REQUIRE(client.connected);
+}
+
+TEST_CASE("getAvailableStreams"){
+  VisionIpcServer server("camerad");
+  server.create_buffers(VISION_STREAM_ROAD, 1, false, 100, 100);
+  server.create_buffers(VISION_STREAM_WIDE_ROAD, 1, false, 100, 100);
+  server.start_listener();
+  auto available_streams = VisionIpcClient::getAvailableStreams("camerad");
+  REQUIRE(available_streams.size() == 2);
+  REQUIRE(available_streams.count(VISION_STREAM_ROAD) == 1);
+  REQUIRE(available_streams.count(VISION_STREAM_WIDE_ROAD) == 1);
+}
+
+TEST_CASE("Check buffers"){
+  size_t width = 100, height = 200, num_buffers = 5;
+  VisionIpcServer server("camerad");
+  server.create_buffers(VISION_STREAM_ROAD, num_buffers, false, width, height);
+  server.start_listener();
+
+  VisionIpcClient client = VisionIpcClient("camerad", VISION_STREAM_ROAD, false);
+  REQUIRE(client.connect());
+
+  REQUIRE(client.buffers[0].width == width);
+  REQUIRE(client.buffers[0].height == height);
+  REQUIRE(client.buffers[0].len);
+  REQUIRE(client.num_buffers == num_buffers);
+}
+
+TEST_CASE("Check yuv/rgb"){
+  VisionIpcServer server("camerad");
+  server.create_buffers(VISION_STREAM_ROAD, 1, false, 100, 100);
+  server.create_buffers(VISION_STREAM_MAP, 1, true, 100, 100);
+  server.start_listener();
+
+  VisionIpcClient client_yuv = VisionIpcClient("camerad", VISION_STREAM_ROAD, false);
+  VisionIpcClient client_rgb = VisionIpcClient("camerad", VISION_STREAM_MAP, false);
+  client_yuv.connect();
+  client_rgb.connect();
+
+  REQUIRE(client_rgb.buffers[0].rgb == true);
+  REQUIRE(client_yuv.buffers[0].rgb == false);
+}
+
+TEST_CASE("Send single buffer"){
+  VisionIpcServer server("camerad");
+  server.create_buffers(VISION_STREAM_ROAD, 1, true, 100, 100);
+  server.start_listener();
+
+  VisionIpcClient client = VisionIpcClient("camerad", VISION_STREAM_ROAD, false);
+  REQUIRE(client.connect());
+  zmq_sleep();
+
+  VisionBuf * buf = server.get_buffer(VISION_STREAM_ROAD);
+  REQUIRE(buf != nullptr);
+
+  *((uint64_t*)buf->addr) = 1234;
+
+  VisionIpcBufExtra extra = {0};
+  extra.frame_id = 1337;
+  buf->set_frame_id(extra.frame_id);
+
+  server.send(buf, &extra);
+
+  VisionIpcBufExtra extra_recv = {0};
+  VisionBuf * recv_buf = client.recv(&extra_recv);
+  REQUIRE(recv_buf != nullptr);
+  REQUIRE(*(uint64_t*)recv_buf->addr == 1234);
+  REQUIRE(extra_recv.frame_id == extra.frame_id);
+  REQUIRE(recv_buf->get_frame_id() == extra.frame_id);
+}
+
+
+TEST_CASE("Test no conflate"){
+  VisionIpcServer server("camerad");
+  server.create_buffers(VISION_STREAM_ROAD, 1, true, 100, 100);
+  server.start_listener();
+
+  VisionIpcClient client = VisionIpcClient("camerad", VISION_STREAM_ROAD, false);
+  REQUIRE(client.connect());
+  zmq_sleep();
+
+  VisionBuf * buf = server.get_buffer(VISION_STREAM_ROAD);
+  REQUIRE(buf != nullptr);
+
+  VisionIpcBufExtra extra = {0};
+  extra.frame_id = 1;
+  server.send(buf, &extra);
+  extra.frame_id = 2;
+  server.send(buf, &extra);
+
+  VisionIpcBufExtra extra_recv = {0};
+  VisionBuf * recv_buf = client.recv(&extra_recv);
+  REQUIRE(recv_buf != nullptr);
+  REQUIRE(extra_recv.frame_id == 1);
+
+  recv_buf = client.recv(&extra_recv);
+  REQUIRE(recv_buf != nullptr);
+  REQUIRE(extra_recv.frame_id == 2);
+}
+
+TEST_CASE("Test conflate"){
+  VisionIpcServer server("camerad");
+  server.create_buffers(VISION_STREAM_ROAD, 1, true, 100, 100);
+  server.start_listener();
+
+  VisionIpcClient client = VisionIpcClient("camerad", VISION_STREAM_ROAD, true);
+  REQUIRE(client.connect());
+  zmq_sleep();
+
+  VisionBuf * buf = server.get_buffer(VISION_STREAM_ROAD);
+  REQUIRE(buf != nullptr);
+
+  VisionIpcBufExtra extra = {0};
+  extra.frame_id = 1;
+  server.send(buf, &extra);
+  extra.frame_id = 2;
+  server.send(buf, &extra);
+
+  VisionIpcBufExtra extra_recv = {0};
+  VisionBuf * recv_buf = client.recv(&extra_recv);
+  REQUIRE(recv_buf != nullptr);
+  REQUIRE(extra_recv.frame_id == 2);
+
+  recv_buf = client.recv(&extra_recv);
+  REQUIRE(recv_buf == nullptr);
+}