brianhansonxyz/clearpilot
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

openpilot v0.9.6 release

Browse Source 
date: 2024-02-21T23:02:42
master commit: 0b4d08fab8e35a264bc7383e878538f8083c33e5
This commit is contained in:
FrogAi
2024-02-27 16:34:45 -07:00
commit 2901597132
1940 changed files with 647891 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
common/.gitignore vendored Normal file
View File

@@ -0,0 +1 @@
*.cpp

39
common/SConscript Normal file
View File

@@ -0,0 +1,39 @@
Import('env', 'envCython', 'arch')
common_libs = [
'params.cc',
'swaglog.cc',
'util.cc',
'i2c.cc',
'watchdog.cc',
'ratekeeper.cc'
]
if arch != "Darwin":
common_libs.append('gpio.cc')
_common = env.Library('common', common_libs, LIBS="json11")
files = [
'clutil.cc',
]
_gpucommon = env.Library('gpucommon', files)
Export('_common', '_gpucommon')
if GetOption('extras'):
env.Program('tests/test_common',
['tests/test_runner.cc', 'tests/test_params.cc', 'tests/test_util.cc', 'tests/test_swaglog.cc'],
LIBS=[_common, 'json11', 'zmq', 'pthread'])
# Cython bindings
params_python = envCython.Program('params_pyx.so', 'params_pyx.pyx', LIBS=envCython['LIBS'] + [_common, 'zmq', 'json11'])
SConscript([
'transformations/SConscript',
])
Import('transformations_python')
common_python = [params_python, transformations_python]
Export('common_python')

0
common/__init__.py Normal file
View File

46
common/api/__init__.py Normal file
View File

@@ -0,0 +1,46 @@
import jwt
import os
import requests
from datetime import datetime, timedelta
from openpilot.system.hardware.hw import Paths
from openpilot.system.version import get_version
API_HOST = os.getenv('API_HOST', 'https://api.commadotai.com')
class Api():
def __init__(self, dongle_id):
self.dongle_id = dongle_id
with open(Paths.persist_root()+'/comma/id_rsa') as f:
self.private_key = f.read()
def get(self, *args, **kwargs):
return self.request('GET', *args, **kwargs)
def post(self, *args, **kwargs):
return self.request('POST', *args, **kwargs)
def request(self, method, endpoint, timeout=None, access_token=None, **params):
return api_get(endpoint, method=method, timeout=timeout, access_token=access_token, **params)
def get_token(self, expiry_hours=1):
now = datetime.utcnow()
payload = {
'identity': self.dongle_id,
'nbf': now,
'iat': now,
'exp': now + timedelta(hours=expiry_hours)
}
token = jwt.encode(payload, self.private_key, algorithm='RS256')
if isinstance(token, bytes):
token = token.decode('utf8')
return token
def api_get(endpoint, method='GET', timeout=None, access_token=None, **params):
headers = {}
if access_token is not None:
headers['Authorization'] = "JWT " + access_token
headers['User-Agent'] = "openpilot-" + get_version()
return requests.request(method, API_HOST + "/" + endpoint, timeout=timeout, headers=headers, params=params)

4
common/basedir.py Normal file
View File

@@ -0,0 +1,4 @@
import os
BASEDIR = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../"))

200
common/clutil.cc Normal file
View File

@@ -0,0 +1,200 @@
#include "common/clutil.h"
#include <cassert>
#include <iostream>
#include <memory>
#include "common/util.h"
#include "common/swaglog.h"
namespace { // helper functions
template <typename Func, typename Id, typename Name>
std::string get_info(Func get_info_func, Id id, Name param_name) {
size_t size = 0;
CL_CHECK(get_info_func(id, param_name, 0, NULL, &size));
std::string info(size, '\0');
CL_CHECK(get_info_func(id, param_name, size, info.data(), NULL));
return info;
}
inline std::string get_platform_info(cl_platform_id id, cl_platform_info name) { return get_info(&clGetPlatformInfo, id, name); }
inline std::string get_device_info(cl_device_id id, cl_device_info name) { return get_info(&clGetDeviceInfo, id, name); }
void cl_print_info(cl_platform_id platform, cl_device_id device) {
size_t work_group_size = 0;
cl_device_type device_type = 0;
clGetDeviceInfo(device, CL_DEVICE_MAX_WORK_GROUP_SIZE, sizeof(work_group_size), &work_group_size, NULL);
clGetDeviceInfo(device, CL_DEVICE_TYPE, sizeof(device_type), &device_type, NULL);
const char *type_str = "Other...";
switch (device_type) {
case CL_DEVICE_TYPE_CPU: type_str ="CL_DEVICE_TYPE_CPU"; break;
case CL_DEVICE_TYPE_GPU: type_str = "CL_DEVICE_TYPE_GPU"; break;
case CL_DEVICE_TYPE_ACCELERATOR: type_str = "CL_DEVICE_TYPE_ACCELERATOR"; break;
}
LOGD("vendor: %s", get_platform_info(platform, CL_PLATFORM_VENDOR).c_str());
LOGD("platform version: %s", get_platform_info(platform, CL_PLATFORM_VERSION).c_str());
LOGD("profile: %s", get_platform_info(platform, CL_PLATFORM_PROFILE).c_str());
LOGD("extensions: %s", get_platform_info(platform, CL_PLATFORM_EXTENSIONS).c_str());
LOGD("name: %s", get_device_info(device, CL_DEVICE_NAME).c_str());
LOGD("device version: %s", get_device_info(device, CL_DEVICE_VERSION).c_str());
LOGD("max work group size: %zu", work_group_size);
LOGD("type = %d, %s", (int)device_type, type_str);
}
void cl_print_build_errors(cl_program program, cl_device_id device) {
cl_build_status status;
clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_STATUS, sizeof(status), &status, NULL);
size_t log_size;
clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_LOG, 0, NULL, &log_size);
std::string log(log_size, '\0');
clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_LOG, log_size, &log[0], NULL);
LOGE("build failed; status=%d, log: %s", status, log.c_str());
}
} // namespace
cl_device_id cl_get_device_id(cl_device_type device_type) {
cl_uint num_platforms = 0;
CL_CHECK(clGetPlatformIDs(0, NULL, &num_platforms));
std::unique_ptr<cl_platform_id[]> platform_ids = std::make_unique<cl_platform_id[]>(num_platforms);
CL_CHECK(clGetPlatformIDs(num_platforms, &platform_ids[0], NULL));
for (size_t i = 0; i < num_platforms; ++i) {
LOGD("platform[%zu] CL_PLATFORM_NAME: %s", i, get_platform_info(platform_ids[i], CL_PLATFORM_NAME).c_str());
// Get first device
if (cl_device_id device_id = NULL; clGetDeviceIDs(platform_ids[i], device_type, 1, &device_id, NULL) == 0 && device_id) {
cl_print_info(platform_ids[i], device_id);
return device_id;
}
}
LOGE("No valid openCL platform found");
assert(0);
return nullptr;
}
cl_context cl_create_context(cl_device_id device_id) {
return CL_CHECK_ERR(clCreateContext(NULL, 1, &device_id, NULL, NULL, &err));
}
cl_program cl_program_from_file(cl_context ctx, cl_device_id device_id, const char* path, const char* args) {
return cl_program_from_source(ctx, device_id, util::read_file(path), args);
}
cl_program cl_program_from_source(cl_context ctx, cl_device_id device_id, const std::string& src, const char* args) {
const char *csrc = src.c_str();
cl_program prg = CL_CHECK_ERR(clCreateProgramWithSource(ctx, 1, &csrc, NULL, &err));
if (int err = clBuildProgram(prg, 1, &device_id, args, NULL, NULL); err != 0) {
cl_print_build_errors(prg, device_id);
assert(0);
}
return prg;
}
cl_program cl_program_from_binary(cl_context ctx, cl_device_id device_id, const uint8_t* binary, size_t length, const char* args) {
cl_program prg = CL_CHECK_ERR(clCreateProgramWithBinary(ctx, 1, &device_id, &length, &binary, NULL, &err));
if (int err = clBuildProgram(prg, 1, &device_id, args, NULL, NULL); err != 0) {
cl_print_build_errors(prg, device_id);
assert(0);
}
return prg;
}
// Given a cl code and return a string representation
#define CL_ERR_TO_STR(err) case err: return #err
const char* cl_get_error_string(int err) {
switch (err) {
CL_ERR_TO_STR(CL_SUCCESS);
CL_ERR_TO_STR(CL_DEVICE_NOT_FOUND);
CL_ERR_TO_STR(CL_DEVICE_NOT_AVAILABLE);
CL_ERR_TO_STR(CL_COMPILER_NOT_AVAILABLE);
CL_ERR_TO_STR(CL_MEM_OBJECT_ALLOCATION_FAILURE);
CL_ERR_TO_STR(CL_OUT_OF_RESOURCES);
CL_ERR_TO_STR(CL_OUT_OF_HOST_MEMORY);
CL_ERR_TO_STR(CL_PROFILING_INFO_NOT_AVAILABLE);
CL_ERR_TO_STR(CL_MEM_COPY_OVERLAP);
CL_ERR_TO_STR(CL_IMAGE_FORMAT_MISMATCH);
CL_ERR_TO_STR(CL_IMAGE_FORMAT_NOT_SUPPORTED);
CL_ERR_TO_STR(CL_MAP_FAILURE);
CL_ERR_TO_STR(CL_MISALIGNED_SUB_BUFFER_OFFSET);
CL_ERR_TO_STR(CL_EXEC_STATUS_ERROR_FOR_EVENTS_IN_WAIT_LIST);
CL_ERR_TO_STR(CL_COMPILE_PROGRAM_FAILURE);
CL_ERR_TO_STR(CL_LINKER_NOT_AVAILABLE);
CL_ERR_TO_STR(CL_LINK_PROGRAM_FAILURE);
CL_ERR_TO_STR(CL_DEVICE_PARTITION_FAILED);
CL_ERR_TO_STR(CL_KERNEL_ARG_INFO_NOT_AVAILABLE);
CL_ERR_TO_STR(CL_INVALID_VALUE);
CL_ERR_TO_STR(CL_INVALID_DEVICE_TYPE);
CL_ERR_TO_STR(CL_INVALID_PLATFORM);
CL_ERR_TO_STR(CL_INVALID_DEVICE);
CL_ERR_TO_STR(CL_INVALID_CONTEXT);
CL_ERR_TO_STR(CL_INVALID_QUEUE_PROPERTIES);
CL_ERR_TO_STR(CL_INVALID_COMMAND_QUEUE);
CL_ERR_TO_STR(CL_INVALID_HOST_PTR);
CL_ERR_TO_STR(CL_INVALID_MEM_OBJECT);
CL_ERR_TO_STR(CL_INVALID_IMAGE_FORMAT_DESCRIPTOR);
CL_ERR_TO_STR(CL_INVALID_IMAGE_SIZE);
CL_ERR_TO_STR(CL_INVALID_SAMPLER);
CL_ERR_TO_STR(CL_INVALID_BINARY);
CL_ERR_TO_STR(CL_INVALID_BUILD_OPTIONS);
CL_ERR_TO_STR(CL_INVALID_PROGRAM);
CL_ERR_TO_STR(CL_INVALID_PROGRAM_EXECUTABLE);
CL_ERR_TO_STR(CL_INVALID_KERNEL_NAME);
CL_ERR_TO_STR(CL_INVALID_KERNEL_DEFINITION);
CL_ERR_TO_STR(CL_INVALID_KERNEL);
CL_ERR_TO_STR(CL_INVALID_ARG_INDEX);
CL_ERR_TO_STR(CL_INVALID_ARG_VALUE);
CL_ERR_TO_STR(CL_INVALID_ARG_SIZE);
CL_ERR_TO_STR(CL_INVALID_KERNEL_ARGS);
CL_ERR_TO_STR(CL_INVALID_WORK_DIMENSION);
CL_ERR_TO_STR(CL_INVALID_WORK_GROUP_SIZE);
CL_ERR_TO_STR(CL_INVALID_WORK_ITEM_SIZE);
CL_ERR_TO_STR(CL_INVALID_GLOBAL_OFFSET);
CL_ERR_TO_STR(CL_INVALID_EVENT_WAIT_LIST);
CL_ERR_TO_STR(CL_INVALID_EVENT);
CL_ERR_TO_STR(CL_INVALID_OPERATION);
CL_ERR_TO_STR(CL_INVALID_GL_OBJECT);
CL_ERR_TO_STR(CL_INVALID_BUFFER_SIZE);
CL_ERR_TO_STR(CL_INVALID_MIP_LEVEL);
CL_ERR_TO_STR(CL_INVALID_GLOBAL_WORK_SIZE);
CL_ERR_TO_STR(CL_INVALID_PROPERTY);
CL_ERR_TO_STR(CL_INVALID_IMAGE_DESCRIPTOR);
CL_ERR_TO_STR(CL_INVALID_COMPILER_OPTIONS);
CL_ERR_TO_STR(CL_INVALID_LINKER_OPTIONS);
CL_ERR_TO_STR(CL_INVALID_DEVICE_PARTITION_COUNT);
case -69: return "CL_INVALID_PIPE_SIZE";
case -70: return "CL_INVALID_DEVICE_QUEUE";
case -71: return "CL_INVALID_SPEC_ID";
case -72: return "CL_MAX_SIZE_RESTRICTION_EXCEEDED";
case -1002: return "CL_INVALID_D3D10_DEVICE_KHR";
case -1003: return "CL_INVALID_D3D10_RESOURCE_KHR";
case -1004: return "CL_D3D10_RESOURCE_ALREADY_ACQUIRED_KHR";
case -1005: return "CL_D3D10_RESOURCE_NOT_ACQUIRED_KHR";
case -1006: return "CL_INVALID_D3D11_DEVICE_KHR";
case -1007: return "CL_INVALID_D3D11_RESOURCE_KHR";
case -1008: return "CL_D3D11_RESOURCE_ALREADY_ACQUIRED_KHR";
case -1009: return "CL_D3D11_RESOURCE_NOT_ACQUIRED_KHR";
case -1010: return "CL_INVALID_DX9_MEDIA_ADAPTER_KHR";
case -1011: return "CL_INVALID_DX9_MEDIA_SURFACE_KHR";
case -1012: return "CL_DX9_MEDIA_SURFACE_ALREADY_ACQUIRED_KHR";
case -1013: return "CL_DX9_MEDIA_SURFACE_NOT_ACQUIRED_KHR";
case -1093: return "CL_INVALID_EGL_OBJECT_KHR";
case -1092: return "CL_EGL_RESOURCE_NOT_ACQUIRED_KHR";
case -1001: return "CL_PLATFORM_NOT_FOUND_KHR";
case -1057: return "CL_DEVICE_PARTITION_FAILED_EXT";
case -1058: return "CL_INVALID_PARTITION_COUNT_EXT";
case -1059: return "CL_INVALID_PARTITION_NAME_EXT";
case -1094: return "CL_INVALID_ACCELERATOR_INTEL";
case -1095: return "CL_INVALID_ACCELERATOR_TYPE_INTEL";
case -1096: return "CL_INVALID_ACCELERATOR_DESCRIPTOR_INTEL";
case -1097: return "CL_ACCELERATOR_TYPE_NOT_SUPPORTED_INTEL";
case -1000: return "CL_INVALID_GL_SHAREGROUP_REFERENCE_KHR";
case -1098: return "CL_INVALID_VA_API_MEDIA_ADAPTER_INTEL";
case -1099: return "CL_INVALID_VA_API_MEDIA_SURFACE_INTEL";
case -1100: return "CL_VA_API_MEDIA_SURFACE_ALREADY_ACQUIRED_INTEL";
case -1101: return "CL_VA_API_MEDIA_SURFACE_NOT_ACQUIRED_INTEL";
default: return "CL_UNKNOWN_ERROR";
}
}

29
common/clutil.h Normal file
View File

@@ -0,0 +1,29 @@
#pragma once
#ifdef __APPLE__
#include <OpenCL/cl.h>
#else
#include <CL/cl.h>
#endif
#include <string>
#define CL_CHECK(_expr) \
do { \
assert(CL_SUCCESS == (_expr)); \
} while (0)
#define CL_CHECK_ERR(_expr) \
({ \
cl_int err = CL_INVALID_VALUE; \
__typeof__(_expr) _ret = _expr; \
assert(_ret&& err == CL_SUCCESS); \
_ret; \
})
cl_device_id cl_get_device_id(cl_device_type device_type);
cl_context cl_create_context(cl_device_id device_id);
cl_program cl_program_from_source(cl_context ctx, cl_device_id device_id, const std::string& src, const char* args = nullptr);
cl_program cl_program_from_binary(cl_context ctx, cl_device_id device_id, const uint8_t* binary, size_t length, const char* args = nullptr);
cl_program cl_program_from_file(cl_context ctx, cl_device_id device_id, const char* path, const char* args);
const char* cl_get_error_string(int err);

19
common/conversions.py Normal file
View File

@@ -0,0 +1,19 @@
import numpy as np
class Conversions:
# Speed
MPH_TO_KPH = 1.609344
KPH_TO_MPH = 1. / MPH_TO_KPH
MS_TO_KPH = 3.6
KPH_TO_MS = 1. / MS_TO_KPH
MS_TO_MPH = MS_TO_KPH * KPH_TO_MPH
MPH_TO_MS = MPH_TO_KPH * KPH_TO_MS
MS_TO_KNOTS = 1.9438
KNOTS_TO_MS = 1. / MS_TO_KNOTS
# Angle
DEG_TO_RAD = np.pi / 180.
RAD_TO_DEG = 1. / DEG_TO_RAD
# Mass
LB_TO_KG = 0.453592

9
common/dict_helpers.py Normal file
View File

@@ -0,0 +1,9 @@
# remove all keys that end in DEPRECATED
def strip_deprecated_keys(d):
for k in list(d.keys()):
if isinstance(k, str):
if k.endswith('DEPRECATED'):
d.pop(k)
elif isinstance(d[k], dict):
strip_deprecated_keys(d[k])
return d

8
common/ffi_wrapper.py Normal file
View File

@@ -0,0 +1,8 @@
import platform
def suffix():
if platform.system() == "Darwin":
return ".dylib"
else:
return ".so"

38
common/file_helpers.py Normal file
View File

@@ -0,0 +1,38 @@
import os
import tempfile
import contextlib
from typing import Optional
class CallbackReader:
"""Wraps a file, but overrides the read method to also
call a callback function with the number of bytes read so far."""
def __init__(self, f, callback, *args):
self.f = f
self.callback = callback
self.cb_args = args
self.total_read = 0
def __getattr__(self, attr):
return getattr(self.f, attr)
def read(self, *args, **kwargs):
chunk = self.f.read(*args, **kwargs)
self.total_read += len(chunk)
self.callback(*self.cb_args, self.total_read)
return chunk
@contextlib.contextmanager
def atomic_write_in_dir(path: str, mode: str = 'w', buffering: int = -1, encoding: Optional[str] = None, newline: Optional[str] = None,
overwrite: bool = False):
"""Write to a file atomically using a temporary file in the same directory as the destination file."""
dir_name = os.path.dirname(path)
if not overwrite and os.path.exists(path):
raise FileExistsError(f"File '{path}' already exists. To overwrite it, set 'overwrite' to True.")
with tempfile.NamedTemporaryFile(mode=mode, buffering=buffering, encoding=encoding, newline=newline, dir=dir_name, delete=False) as tmp_file:
yield tmp_file
tmp_file_name = tmp_file.name
os.replace(tmp_file_name, path)

18
common/filter_simple.py Normal file
View File

@@ -0,0 +1,18 @@
class FirstOrderFilter:
# first order filter
def __init__(self, x0, rc, dt, initialized=True):
self.x = x0
self.dt = dt
self.update_alpha(rc)
self.initialized = initialized
def update_alpha(self, rc):
self.alpha = self.dt / (rc + self.dt)
def update(self, x):
if self.initialized:
self.x = (1. - self.alpha) * self.x + self.alpha * x
else:
self.initialized = True
self.x = x
return self.x

84
common/gpio.cc Normal file
View File

@@ -0,0 +1,84 @@
#include "common/gpio.h"
#include <string>
#ifdef __APPLE__
int gpio_init(int pin_nr, bool output) {
return 0;
}
int gpio_set(int pin_nr, bool high) {
return 0;
}
int gpiochip_get_ro_value_fd(const char* consumer_label, int gpiochiop_id, int pin_nr) {
return 0;
}
#else
#include <fcntl.h>
#include <unistd.h>
#include <cstring>
#include <linux/gpio.h>
#include <sys/ioctl.h>
#include "common/util.h"
#include "common/swaglog.h"
int gpio_init(int pin_nr, bool output) {
char pin_dir_path[50];
int pin_dir_path_len = snprintf(pin_dir_path, sizeof(pin_dir_path),
"/sys/class/gpio/gpio%d/direction", pin_nr);
if (pin_dir_path_len <= 0) {
return -1;
}
const char *value = output ? "out" : "in";
return util::write_file(pin_dir_path, (void*)value, strlen(value));
}
int gpio_set(int pin_nr, bool high) {
char pin_val_path[50];
int pin_val_path_len = snprintf(pin_val_path, sizeof(pin_val_path),
"/sys/class/gpio/gpio%d/value", pin_nr);
if (pin_val_path_len <= 0) {
return -1;
}
return util::write_file(pin_val_path, (void*)(high ? "1" : "0"), 1);
}
int gpiochip_get_ro_value_fd(const char* consumer_label, int gpiochiop_id, int pin_nr) {
// Assumed that all interrupt pins are unexported and rights are given to
// read from gpiochip0.
std::string gpiochip_path = "/dev/gpiochip" + std::to_string(gpiochiop_id);
int fd = open(gpiochip_path.c_str(), O_RDONLY);
if (fd < 0) {
LOGE("Error opening gpiochip0 fd");
return -1;
}
// Setup event
struct gpioevent_request rq;
rq.lineoffset = pin_nr;
rq.handleflags = GPIOHANDLE_REQUEST_INPUT;
/* Requesting both edges as the data ready pulse from the lsm6ds sensor is
very short(75us) and is mostly detected as falling edge instead of rising.
So if it is detected as rising the following falling edge is skipped. */
rq.eventflags = GPIOEVENT_REQUEST_BOTH_EDGES;
strncpy(rq.consumer_label, consumer_label, std::size(rq.consumer_label) - 1);
int ret = util::safe_ioctl(fd, GPIO_GET_LINEEVENT_IOCTL, &rq);
if (ret == -1) {
LOGE("Unable to get line event from ioctl : %s", strerror(errno));
close(fd);
return -1;
}
close(fd);
return rq.fd;
}
#endif

33
common/gpio.h Normal file
View File

@@ -0,0 +1,33 @@
#pragma once
// Pin definitions
#ifdef QCOM2
#define GPIO_HUB_RST_N 30
#define GPIO_UBLOX_RST_N 32
#define GPIO_UBLOX_SAFEBOOT_N 33
#define GPIO_GNSS_PWR_EN 34 /* SCHEMATIC LABEL: GPIO_UBLOX_PWR_EN */
#define GPIO_STM_RST_N 124
#define GPIO_STM_BOOT0 134
#define GPIO_BMX_ACCEL_INT 21
#define GPIO_BMX_GYRO_INT 23
#define GPIO_BMX_MAGN_INT 87
#define GPIO_LSM_INT 84
#define GPIOCHIP_INT 0
#else
#define GPIO_HUB_RST_N 0
#define GPIO_UBLOX_RST_N 0
#define GPIO_UBLOX_SAFEBOOT_N 0
#define GPIO_GNSS_PWR_EN 0 /* SCHEMATIC LABEL: GPIO_UBLOX_PWR_EN */
#define GPIO_STM_RST_N 0
#define GPIO_STM_BOOT0 0
#define GPIO_BMX_ACCEL_INT 0
#define GPIO_BMX_GYRO_INT 0
#define GPIO_BMX_MAGN_INT 0
#define GPIO_LSM_INT 0
#define GPIOCHIP_INT 0
#endif
int gpio_init(int pin_nr, bool output);
int gpio_set(int pin_nr, bool high);
int gpiochip_get_ro_value_fd(const char* consumer_label, int gpiochiop_id, int pin_nr);

55
common/gpio.py Normal file
View File

@@ -0,0 +1,55 @@
import os
from functools import lru_cache
from typing import Optional, List
def gpio_init(pin: int, output: bool) -> None:
try:
with open(f"/sys/class/gpio/gpio{pin}/direction", 'wb') as f:
f.write(b"out" if output else b"in")
except Exception as e:
print(f"Failed to set gpio {pin} direction: {e}")
def gpio_set(pin: int, high: bool) -> None:
try:
with open(f"/sys/class/gpio/gpio{pin}/value", 'wb') as f:
f.write(b"1" if high else b"0")
except Exception as e:
print(f"Failed to set gpio {pin} value: {e}")
def gpio_read(pin: int) -> Optional[bool]:
val = None
try:
with open(f"/sys/class/gpio/gpio{pin}/value", 'rb') as f:
val = bool(int(f.read().strip()))
except Exception as e:
print(f"Failed to set gpio {pin} value: {e}")
return val
def gpio_export(pin: int) -> None:
if os.path.isdir(f"/sys/class/gpio/gpio{pin}"):
return
try:
with open("/sys/class/gpio/export", 'w') as f:
f.write(str(pin))
except Exception:
print(f"Failed to export gpio {pin}")
@lru_cache(maxsize=None)
def get_irq_action(irq: int) -> List[str]:
try:
with open(f"/sys/kernel/irq/{irq}/actions") as f:
actions = f.read().strip().split(',')
return actions
except FileNotFoundError:
return []
def get_irqs_for_action(action: str) -> List[str]:
ret = []
with open("/proc/interrupts") as f:
for l in f.readlines():
irq = l.split(':')[0].strip()
if irq.isdigit() and action in get_irq_action(irq):
ret.append(irq)
return ret

92
common/i2c.cc Normal file
View File

@@ -0,0 +1,92 @@
#include "common/i2c.h"
#include <fcntl.h>
#include <sys/ioctl.h>
#include <unistd.h>
#include <cassert>
#include <cstdio>
#include <stdexcept>
#include "common/swaglog.h"
#include "common/util.h"
#define UNUSED(x) (void)(x)
#ifdef QCOM2
// TODO: decide if we want to install libi2c-dev everywhere
extern "C" {
#include <linux/i2c-dev.h>
#include <i2c/smbus.h>
}
I2CBus::I2CBus(uint8_t bus_id) {
char bus_name[20];
snprintf(bus_name, 20, "/dev/i2c-%d", bus_id);
i2c_fd = HANDLE_EINTR(open(bus_name, O_RDWR));
if (i2c_fd < 0) {
throw std::runtime_error("Failed to open I2C bus");
}
}
I2CBus::~I2CBus() {
if (i2c_fd >= 0) {
close(i2c_fd);
}
}
int I2CBus::read_register(uint8_t device_address, uint register_address, uint8_t *buffer, uint8_t len) {
std::lock_guard lk(m);
int ret = 0;
ret = HANDLE_EINTR(ioctl(i2c_fd, I2C_SLAVE, device_address));
if (ret < 0) { goto fail; }
ret = i2c_smbus_read_i2c_block_data(i2c_fd, register_address, len, buffer);
if ((ret < 0) || (ret != len)) { goto fail; }
fail:
return ret;
}
int I2CBus::set_register(uint8_t device_address, uint register_address, uint8_t data) {
std::lock_guard lk(m);
int ret = 0;
ret = HANDLE_EINTR(ioctl(i2c_fd, I2C_SLAVE, device_address));
if (ret < 0) { goto fail; }
ret = i2c_smbus_write_byte_data(i2c_fd, register_address, data);
if (ret < 0) { goto fail; }
fail:
return ret;
}
#else
I2CBus::I2CBus(uint8_t bus_id) {
UNUSED(bus_id);
i2c_fd = -1;
}
I2CBus::~I2CBus() {}
int I2CBus::read_register(uint8_t device_address, uint register_address, uint8_t *buffer, uint8_t len) {
UNUSED(device_address);
UNUSED(register_address);
UNUSED(buffer);
UNUSED(len);
return -1;
}
int I2CBus::set_register(uint8_t device_address, uint register_address, uint8_t data) {
UNUSED(device_address);
UNUSED(register_address);
UNUSED(data);
return -1;
}
#endif

19
common/i2c.h Normal file
View File

@@ -0,0 +1,19 @@
#pragma once
#include <cstdint>
#include <mutex>
#include <sys/types.h>
class I2CBus {
private:
int i2c_fd;
std::mutex m;
public:
I2CBus(uint8_t bus_id);
~I2CBus();
int read_register(uint8_t device_address, uint register_address, uint8_t *buffer, uint8_t len);
int set_register(uint8_t device_address, uint register_address, uint8_t data);
};

247
common/logging_extra.py Normal file
View File

@@ -0,0 +1,247 @@
import io
import os
import sys
import copy
import json
import time
import uuid
import socket
import logging
import traceback
from threading import local
from collections import OrderedDict
from contextlib import contextmanager
LOG_TIMESTAMPS = "LOG_TIMESTAMPS" in os.environ
def json_handler(obj):
# if isinstance(obj, (datetime.date, datetime.time)):
# return obj.isoformat()
return repr(obj)
def json_robust_dumps(obj):
return json.dumps(obj, default=json_handler)
class NiceOrderedDict(OrderedDict):
def __str__(self):
return json_robust_dumps(self)
class SwagFormatter(logging.Formatter):
def __init__(self, swaglogger):
logging.Formatter.__init__(self, None, '%a %b %d %H:%M:%S %Z %Y')
self.swaglogger = swaglogger
self.host = socket.gethostname()
def format_dict(self, record):
record_dict = NiceOrderedDict()
if isinstance(record.msg, dict):
record_dict['msg'] = record.msg
else:
try:
record_dict['msg'] = record.getMessage()
except (ValueError, TypeError):
record_dict['msg'] = [record.msg]+record.args
record_dict['ctx'] = self.swaglogger.get_ctx()
if record.exc_info:
record_dict['exc_info'] = self.formatException(record.exc_info)
record_dict['level'] = record.levelname
record_dict['levelnum'] = record.levelno
record_dict['name'] = record.name
record_dict['filename'] = record.filename
record_dict['lineno'] = record.lineno
record_dict['pathname'] = record.pathname
record_dict['module'] = record.module
record_dict['funcName'] = record.funcName
record_dict['host'] = self.host
record_dict['process'] = record.process
record_dict['thread'] = record.thread
record_dict['threadName'] = record.threadName
record_dict['created'] = record.created
return record_dict
def format(self, record):
if self.swaglogger is None:
raise Exception("must set swaglogger before calling format()")
return json_robust_dumps(self.format_dict(record))
class SwagLogFileFormatter(SwagFormatter):
def fix_kv(self, k, v):
# append type to names to preserve legacy naming in logs
# avoids overlapping key namespaces with different types
# e.g. log.info() creates 'msg' -> 'msg$s'
# log.event() creates 'msg.health.logMonoTime' -> 'msg.health.logMonoTime$i'
# because overlapping namespace 'msg' caused problems
if isinstance(v, (str, bytes)):
k += "$s"
elif isinstance(v, float):
k += "$f"
elif isinstance(v, bool):
k += "$b"
elif isinstance(v, int):
k += "$i"
elif isinstance(v, dict):
nv = {}
for ik, iv in v.items():
ik, iv = self.fix_kv(ik, iv)
nv[ik] = iv
v = nv
elif isinstance(v, list):
k += "$a"
return k, v
def format(self, record):
if isinstance(record, str):
v = json.loads(record)
else:
v = self.format_dict(record)
mk, mv = self.fix_kv('msg', v['msg'])
del v['msg']
v[mk] = mv
v['id'] = uuid.uuid4().hex
return json_robust_dumps(v)
class SwagErrorFilter(logging.Filter):
def filter(self, record):
return record.levelno < logging.ERROR
def _tmpfunc():
return 0
def _srcfile():
return os.path.normcase(_tmpfunc.__code__.co_filename)
class SwagLogger(logging.Logger):
def __init__(self):
logging.Logger.__init__(self, "swaglog")
self.global_ctx = {}
self.log_local = local()
self.log_local.ctx = {}
def local_ctx(self):
try:
return self.log_local.ctx
except AttributeError:
self.log_local.ctx = {}
return self.log_local.ctx
def get_ctx(self):
return dict(self.local_ctx(), **self.global_ctx)
@contextmanager
def ctx(self, **kwargs):
old_ctx = self.local_ctx()
self.log_local.ctx = copy.copy(old_ctx) or {}
self.log_local.ctx.update(kwargs)
try:
yield
finally:
self.log_local.ctx = old_ctx
def bind(self, **kwargs):
self.local_ctx().update(kwargs)
def bind_global(self, **kwargs):
self.global_ctx.update(kwargs)
def event(self, event, *args, **kwargs):
evt = NiceOrderedDict()
evt['event'] = event
if args:
evt['args'] = args
evt.update(kwargs)
if 'error' in kwargs:
self.error(evt)
elif 'debug' in kwargs:
self.debug(evt)
else:
self.info(evt)
def timestamp(self, event_name):
if LOG_TIMESTAMPS:
t = time.monotonic()
tstp = NiceOrderedDict()
tstp['timestamp'] = NiceOrderedDict()
tstp['timestamp']["event"] = event_name
tstp['timestamp']["time"] = t*1e9
self.debug(tstp)
def findCaller(self, stack_info=False, stacklevel=1):
"""
Find the stack frame of the caller so that we can note the source
file name, line number and function name.
"""
f = sys._getframe(3)
#On some versions of IronPython, currentframe() returns None if
#IronPython isn't run with -X:Frames.
if f is not None:
f = f.f_back
orig_f = f
while f and stacklevel > 1:
f = f.f_back
stacklevel -= 1
if not f:
f = orig_f
rv = "(unknown file)", 0, "(unknown function)", None
while hasattr(f, "f_code"):
co = f.f_code
filename = os.path.normcase(co.co_filename)
# TODO: is this pylint exception correct?
if filename == _srcfile:
f = f.f_back
continue
sinfo = None
if stack_info:
sio = io.StringIO()
sio.write('Stack (most recent call last):\n')
traceback.print_stack(f, file=sio)
sinfo = sio.getvalue()
if sinfo[-1] == '\n':
sinfo = sinfo[:-1]
sio.close()
rv = (co.co_filename, f.f_lineno, co.co_name, sinfo)
break
return rv
if __name__ == "__main__":
log = SwagLogger()
stdout_handler = logging.StreamHandler(sys.stdout)
stdout_handler.setLevel(logging.INFO)
stdout_handler.addFilter(SwagErrorFilter())
log.addHandler(stdout_handler)
stderr_handler = logging.StreamHandler(sys.stderr)
stderr_handler.setLevel(logging.ERROR)
log.addHandler(stderr_handler)
log.info("asdasd %s", "a")
log.info({'wut': 1})
log.warning("warning")
log.error("error")
log.critical("critical")
log.event("test", x="y")
with log.ctx():
stdout_handler.setFormatter(SwagFormatter(log))
stderr_handler.setFormatter(SwagFormatter(log))
log.bind(user="some user")
log.info("in req")
print("")
log.warning("warning")
print("")
log.error("error")
print("")
log.critical("critical")
print("")
log.event("do_req", a=1, b="c")

85
common/mat.h Normal file
View File

@@ -0,0 +1,85 @@
#pragma once
typedef struct vec3 {
float v[3];
} vec3;
typedef struct vec4 {
float v[4];
} vec4;
typedef struct mat3 {
float v[3*3];
} mat3;
typedef struct mat4 {
float v[4*4];
} mat4;
static inline mat3 matmul3(const mat3 &a, const mat3 &b) {
mat3 ret = {{0.0}};
for (int r=0; r<3; r++) {
for (int c=0; c<3; c++) {
float v = 0.0;
for (int k=0; k<3; k++) {
v += a.v[r*3+k] * b.v[k*3+c];
}
ret.v[r*3+c] = v;
}
}
return ret;
}
static inline vec3 matvecmul3(const mat3 &a, const vec3 &b) {
vec3 ret = {{0.0}};
for (int r=0; r<3; r++) {
for (int c=0; c<3; c++) {
ret.v[r] += a.v[r*3+c] * b.v[c];
}
}
return ret;
}
static inline mat4 matmul(const mat4 &a, const mat4 &b) {
mat4 ret = {{0.0}};
for (int r=0; r<4; r++) {
for (int c=0; c<4; c++) {
float v = 0.0;
for (int k=0; k<4; k++) {
v += a.v[r*4+k] * b.v[k*4+c];
}
ret.v[r*4+c] = v;
}
}
return ret;
}
static inline vec4 matvecmul(const mat4 &a, const vec4 &b) {
vec4 ret = {{0.0}};
for (int r=0; r<4; r++) {
for (int c=0; c<4; c++) {
ret.v[r] += a.v[r*4+c] * b.v[c];
}
}
return ret;
}
// scales the input and output space of a transformation matrix
// that assumes pixel-center origin.
static inline mat3 transform_scale_buffer(const mat3 &in, float s) {
// in_pt = ( transform(out_pt/s + 0.5) - 0.5) * s
mat3 transform_out = {{
1.0f/s, 0.0f, 0.5f,
0.0f, 1.0f/s, 0.5f,
0.0f, 0.0f, 1.0f,
}};
mat3 transform_in = {{
s, 0.0f, -0.5f*s,
0.0f, s, -0.5f*s,
0.0f, 0.0f, 1.0f,
}};
return matmul3(transform_in, matmul3(in, transform_out));
}

19
common/numpy_fast.py Normal file
View File

@@ -0,0 +1,19 @@
def clip(x, lo, hi):
return max(lo, min(hi, x))
def interp(x, xp, fp):
N = len(xp)
def get_interp(xv):
hi = 0
while hi < N and xv > xp[hi]:
hi += 1
low = hi - 1
return fp[-1] if hi == N and xv > xp[low] else (
fp[0] if hi == 0 else
(xv - xp[low]) * (fp[hi] - fp[low]) / (xp[hi] - xp[low]) + fp[low])
return [get_interp(v) for v in x] if hasattr(x, '__iter__') else get_interp(x)
def mean(x):
return sum(x) / len(x)

353
common/params.cc Normal file
View File

@@ -0,0 +1,353 @@
#include "common/params.h"
#include <dirent.h>
#include <sys/file.h>
#include <algorithm>
#include <cassert>
#include <csignal>
#include <unordered_map>
#include "common/queue.h"
#include "common/swaglog.h"
#include "common/util.h"
#include "system/hardware/hw.h"
namespace {
volatile sig_atomic_t params_do_exit = 0;
void params_sig_handler(int signal) {
params_do_exit = 1;
}
int fsync_dir(const std::string &path) {
int result = -1;
int fd = HANDLE_EINTR(open(path.c_str(), O_RDONLY, 0755));
if (fd >= 0) {
result = fsync(fd);
close(fd);
}
return result;
}
bool create_params_path(const std::string &param_path, const std::string &key_path) {
// Make sure params path exists
if (!util::file_exists(param_path) && !util::create_directories(param_path, 0775)) {
return false;
}
// See if the symlink exists, otherwise create it
if (!util::file_exists(key_path)) {
// 1) Create temp folder
// 2) Symlink it to temp link
// 3) Move symlink to <params>/d
std::string tmp_path = param_path + "/.tmp_XXXXXX";
// this should be OK since mkdtemp just replaces characters in place
char *tmp_dir = mkdtemp((char *)tmp_path.c_str());
if (tmp_dir == NULL) {
return false;
}
std::string link_path = std::string(tmp_dir) + ".link";
if (symlink(tmp_dir, link_path.c_str()) != 0) {
return false;
}
// don't return false if it has been created by other
if (rename(link_path.c_str(), key_path.c_str()) != 0 && errno != EEXIST) {
return false;
}
}
return true;
}
std::string ensure_params_path(const std::string &prefix, const std::string &path = {}) {
std::string params_path = path.empty() ? Path::params() : path;
if (!create_params_path(params_path, params_path + prefix)) {
throw std::runtime_error(util::string_format(
"Failed to ensure params path, errno=%d, path=%s, param_prefix=%s",
errno, params_path.c_str(), prefix.c_str()));
}
return params_path;
}
class FileLock {
public:
FileLock(const std::string &fn) {
fd_ = HANDLE_EINTR(open(fn.c_str(), O_CREAT, 0775));
if (fd_ < 0 || HANDLE_EINTR(flock(fd_, LOCK_EX)) < 0) {
LOGE("Failed to lock file %s, errno=%d", fn.c_str(), errno);
}
}
~FileLock() { close(fd_); }
private:
int fd_ = -1;
};
std::unordered_map<std::string, uint32_t> keys = {
{"AccessToken", CLEAR_ON_MANAGER_START | DONT_LOG},
{"ApiCache_Device", PERSISTENT},
{"ApiCache_NavDestinations", PERSISTENT},
{"AssistNowToken", PERSISTENT},
{"AthenadPid", PERSISTENT},
{"AthenadUploadQueue", PERSISTENT},
{"AthenadRecentlyViewedRoutes", PERSISTENT},
{"BootCount", PERSISTENT},
{"CalibrationParams", PERSISTENT},
{"CameraDebugExpGain", CLEAR_ON_MANAGER_START},
{"CameraDebugExpTime", CLEAR_ON_MANAGER_START},
{"CarBatteryCapacity", PERSISTENT},
{"CarParams", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
{"CarParamsCache", CLEAR_ON_MANAGER_START},
{"CarParamsPersistent", PERSISTENT},
{"CarParamsPrevRoute", PERSISTENT},
{"CarVin", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
{"CompletedTrainingVersion", PERSISTENT},
{"ControlsReady", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
{"CurrentBootlog", PERSISTENT},
{"CurrentRoute", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
{"DisableLogging", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
{"DisablePowerDown", PERSISTENT},
{"DisableUpdates", PERSISTENT},
{"DisengageOnAccelerator", PERSISTENT},
{"DmModelInitialized", CLEAR_ON_ONROAD_TRANSITION},
{"DongleId", PERSISTENT},
{"DoReboot", CLEAR_ON_MANAGER_START},
{"DoShutdown", CLEAR_ON_MANAGER_START},
{"DoUninstall", CLEAR_ON_MANAGER_START},
{"ExperimentalLongitudinalEnabled", PERSISTENT | DEVELOPMENT_ONLY},
{"ExperimentalMode", PERSISTENT},
{"ExperimentalModeConfirmed", PERSISTENT},
{"FirmwareQueryDone", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
{"ForcePowerDown", PERSISTENT},
{"GitBranch", PERSISTENT},
{"GitCommit", PERSISTENT},
{"GitCommitDate", PERSISTENT},
{"GitDiff", PERSISTENT},
{"GithubSshKeys", PERSISTENT},
{"GithubUsername", PERSISTENT},
{"GitRemote", PERSISTENT},
{"GsmApn", PERSISTENT},
{"GsmMetered", PERSISTENT},
{"GsmRoaming", PERSISTENT},
{"HardwareSerial", PERSISTENT},
{"HasAcceptedTerms", PERSISTENT},
{"IMEI", PERSISTENT},
{"InstallDate", PERSISTENT},
{"IsDriverViewEnabled", CLEAR_ON_MANAGER_START},
{"IsEngaged", PERSISTENT},
{"IsLdwEnabled", PERSISTENT},
{"IsMetric", PERSISTENT},
{"IsOffroad", CLEAR_ON_MANAGER_START},
{"IsOnroad", PERSISTENT},
{"IsRhdDetected", PERSISTENT},
{"IsReleaseBranch", CLEAR_ON_MANAGER_START},
{"IsTakingSnapshot", CLEAR_ON_MANAGER_START},
{"IsTestedBranch", CLEAR_ON_MANAGER_START},
{"JoystickDebugMode", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
{"LanguageSetting", PERSISTENT},
{"LastAthenaPingTime", CLEAR_ON_MANAGER_START},
{"LastGPSPosition", PERSISTENT},
{"LastManagerExitReason", CLEAR_ON_MANAGER_START},
{"LastOffroadStatusPacket", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
{"LastPowerDropDetected", CLEAR_ON_MANAGER_START},
{"LastUpdateException", CLEAR_ON_MANAGER_START},
{"LastUpdateTime", PERSISTENT},
{"LiveParameters", PERSISTENT},
{"LiveTorqueParameters", PERSISTENT | DONT_LOG},
{"LongitudinalPersonality", PERSISTENT},
{"NavDestination", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
{"NavDestinationWaypoints", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
{"NavPastDestinations", PERSISTENT},
{"NavSettingLeftSide", PERSISTENT},
{"NavSettingTime24h", PERSISTENT},
{"NetworkMetered", PERSISTENT},
{"ObdMultiplexingChanged", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
{"ObdMultiplexingEnabled", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
{"Offroad_BadNvme", CLEAR_ON_MANAGER_START},
{"Offroad_CarUnrecognized", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
{"Offroad_ConnectivityNeeded", CLEAR_ON_MANAGER_START},
{"Offroad_ConnectivityNeededPrompt", CLEAR_ON_MANAGER_START},
{"Offroad_InvalidTime", CLEAR_ON_MANAGER_START},
{"Offroad_IsTakingSnapshot", CLEAR_ON_MANAGER_START},
{"Offroad_NeosUpdate", CLEAR_ON_MANAGER_START},
{"Offroad_NoFirmware", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
{"Offroad_Recalibration", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
{"Offroad_StorageMissing", CLEAR_ON_MANAGER_START},
{"Offroad_TemperatureTooHigh", CLEAR_ON_MANAGER_START},
{"Offroad_UnofficialHardware", CLEAR_ON_MANAGER_START},
{"Offroad_UpdateFailed", CLEAR_ON_MANAGER_START},
{"OpenpilotEnabledToggle", PERSISTENT},
{"PandaHeartbeatLost", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
{"PandaSomResetTriggered", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
{"PandaSignatures", CLEAR_ON_MANAGER_START},
{"PrimeType", PERSISTENT},
{"RecordFront", PERSISTENT},
{"RecordFrontLock", PERSISTENT}, // for the internal fleet
{"ReplayControlsState", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
{"SnoozeUpdate", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
{"SshEnabled", PERSISTENT},
{"TermsVersion", PERSISTENT},
{"Timezone", PERSISTENT},
{"TrainingVersion", PERSISTENT},
{"UbloxAvailable", PERSISTENT},
{"UpdateAvailable", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
{"UpdateFailedCount", CLEAR_ON_MANAGER_START},
{"UpdaterAvailableBranches", PERSISTENT},
{"UpdaterCurrentDescription", CLEAR_ON_MANAGER_START},
{"UpdaterCurrentReleaseNotes", CLEAR_ON_MANAGER_START},
{"UpdaterFetchAvailable", CLEAR_ON_MANAGER_START},
{"UpdaterNewDescription", CLEAR_ON_MANAGER_START},
{"UpdaterNewReleaseNotes", CLEAR_ON_MANAGER_START},
{"UpdaterState", CLEAR_ON_MANAGER_START},
{"UpdaterTargetBranch", CLEAR_ON_MANAGER_START},
{"UpdaterLastFetchTime", PERSISTENT},
{"Version", PERSISTENT},
{"VisionRadarToggle", PERSISTENT},
};
} // namespace
Params::Params(const std::string &path) {
params_prefix = "/" + util::getenv("OPENPILOT_PREFIX", "d");
params_path = ensure_params_path(params_prefix, path);
}
Params::~Params() {
if (future.valid()) {
future.wait();
}
assert(queue.empty());
}
std::vector<std::string> Params::allKeys() const {
std::vector<std::string> ret;
for (auto &p : keys) {
ret.push_back(p.first);
}
return ret;
}
bool Params::checkKey(const std::string &key) {
return keys.find(key) != keys.end();
}
ParamKeyType Params::getKeyType(const std::string &key) {
return static_cast<ParamKeyType>(keys[key]);
}
int Params::put(const char* key, const char* value, size_t value_size) {
// Information about safely and atomically writing a file: https://lwn.net/Articles/457667/
// 1) Create temp file
// 2) Write data to temp file
// 3) fsync() the temp file
// 4) rename the temp file to the real name
// 5) fsync() the containing directory
std::string tmp_path = params_path + "/.tmp_value_XXXXXX";
int tmp_fd = mkstemp((char*)tmp_path.c_str());
if (tmp_fd < 0) return -1;
int result = -1;
do {
// Write value to temp.
ssize_t bytes_written = HANDLE_EINTR(write(tmp_fd, value, value_size));
if (bytes_written < 0 || (size_t)bytes_written != value_size) {
result = -20;
break;
}
// fsync to force persist the changes.
if ((result = fsync(tmp_fd)) < 0) break;
FileLock file_lock(params_path + "/.lock");
// Move temp into place.
if ((result = rename(tmp_path.c_str(), getParamPath(key).c_str())) < 0) break;
// fsync parent directory
result = fsync_dir(getParamPath());
} while (false);
close(tmp_fd);
::unlink(tmp_path.c_str());
return result;
}
int Params::remove(const std::string &key) {
FileLock file_lock(params_path + "/.lock");
int result = unlink(getParamPath(key).c_str());
if (result != 0) {
return result;
}
return fsync_dir(getParamPath());
}
std::string Params::get(const std::string &key, bool block) {
if (!block) {
return util::read_file(getParamPath(key));
} else {
// blocking read until successful
params_do_exit = 0;
void (*prev_handler_sigint)(int) = std::signal(SIGINT, params_sig_handler);
void (*prev_handler_sigterm)(int) = std::signal(SIGTERM, params_sig_handler);
std::string value;
while (!params_do_exit) {
if (value = util::read_file(getParamPath(key)); !value.empty()) {
break;
}
util::sleep_for(100); // 0.1 s
}
std::signal(SIGINT, prev_handler_sigint);
std::signal(SIGTERM, prev_handler_sigterm);
return value;
}
}
std::map<std::string, std::string> Params::readAll() {
FileLock file_lock(params_path + "/.lock");
return util::read_files_in_dir(getParamPath());
}
void Params::clearAll(ParamKeyType key_type) {
FileLock file_lock(params_path + "/.lock");
// 1) delete params of key_type
// 2) delete files that are not defined in the keys.
if (DIR *d = opendir(getParamPath().c_str())) {
struct dirent *de = NULL;
while ((de = readdir(d))) {
if (de->d_type != DT_DIR) {
auto it = keys.find(de->d_name);
if (it == keys.end() || (it->second & key_type)) {
unlink(getParamPath(de->d_name).c_str());
}
}
}
closedir(d);
}
fsync_dir(getParamPath());
}
void Params::putNonBlocking(const std::string &key, const std::string &val) {
queue.push(std::make_pair(key, val));
// start thread on demand
if (!future.valid() || future.wait_for(std::chrono::milliseconds(0)) == std::future_status::ready) {
future = std::async(std::launch::async, &Params::asyncWriteThread, this);
}
}
void Params::asyncWriteThread() {
// TODO: write the latest one if a key has multiple values in the queue.
std::pair<std::string, std::string> p;
while (queue.try_pop(p, 0)) {
// Params::put is Thread-Safe
put(p.first, p.second);
}
}

70
common/params.h Normal file
View File

@@ -0,0 +1,70 @@
#pragma once
#include <future>
#include <map>
#include <string>
#include <tuple>
#include <utility>
#include <vector>
#include "common/queue.h"
enum ParamKeyType {
PERSISTENT = 0x02,
CLEAR_ON_MANAGER_START = 0x04,
CLEAR_ON_ONROAD_TRANSITION = 0x08,
CLEAR_ON_OFFROAD_TRANSITION = 0x10,
DONT_LOG = 0x20,
DEVELOPMENT_ONLY = 0x40,
ALL = 0xFFFFFFFF
};
class Params {
public:
explicit Params(const std::string &path = {});
~Params();
// Not copyable.
Params(const Params&) = delete;
Params& operator=(const Params&) = delete;
std::vector<std::string> allKeys() const;
bool checkKey(const std::string &key);
ParamKeyType getKeyType(const std::string &key);
inline std::string getParamPath(const std::string &key = {}) {
return params_path + params_prefix + (key.empty() ? "" : "/" + key);
}
// Delete a value
int remove(const std::string &key);
void clearAll(ParamKeyType type);
// helpers for reading values
std::string get(const std::string &key, bool block = false);
inline bool getBool(const std::string &key, bool block = false) {
return get(key, block) == "1";
}
std::map<std::string, std::string> readAll();
// helpers for writing values
int put(const char *key, const char *val, size_t value_size);
inline int put(const std::string &key, const std::string &val) {
return put(key.c_str(), val.data(), val.size());
}
inline int putBool(const std::string &key, bool val) {
return put(key.c_str(), val ? "1" : "0", 1);
}
void putNonBlocking(const std::string &key, const std::string &val);
inline void putBoolNonBlocking(const std::string &key, bool val) {
putNonBlocking(key, val ? "1" : "0");
}
private:
void asyncWriteThread();
std::string params_path;
std::string params_prefix;
// for nonblocking write
std::future<void> future;
SafeQueue<std::pair<std::string, std::string>> queue;
};

18
common/params.py Normal file
View File

@@ -0,0 +1,18 @@
from openpilot.common.params_pyx import Params, ParamKeyType, UnknownKeyName
assert Params
assert ParamKeyType
assert UnknownKeyName
if __name__ == "__main__":
import sys
params = Params()
key = sys.argv[1]
assert params.check_key(key), f"unknown param: {key}"
if len(sys.argv) == 3:
val = sys.argv[2]
print(f"SET: {key} = {val}")
params.put(key, val)
elif len(sys.argv) == 2:
print(f"GET: {key} = {params.get(key)}")

118
common/params_pyx.pyx Normal file
View File

@@ -0,0 +1,118 @@
# distutils: language = c++
# cython: language_level = 3
from libcpp cimport bool
from libcpp.string cimport string
from libcpp.vector cimport vector
cdef extern from "common/params.h":
cpdef enum ParamKeyType:
PERSISTENT
CLEAR_ON_MANAGER_START
CLEAR_ON_ONROAD_TRANSITION
CLEAR_ON_OFFROAD_TRANSITION
DEVELOPMENT_ONLY
ALL
cdef cppclass c_Params "Params":
c_Params(string) except + nogil
string get(string, bool) nogil
bool getBool(string, bool) nogil
int remove(string) nogil
int put(string, string) nogil
void putNonBlocking(string, string) nogil
void putBoolNonBlocking(string, bool) nogil
int putBool(string, bool) nogil
bool checkKey(string) nogil
string getParamPath(string) nogil
void clearAll(ParamKeyType)
vector[string] allKeys()
def ensure_bytes(v):
return v.encode() if isinstance(v, str) else v
class UnknownKeyName(Exception):
pass
cdef class Params:
cdef c_Params* p
def __cinit__(self, d=""):
cdef string path = <string>d.encode()
with nogil:
self.p = new c_Params(path)
def __dealloc__(self):
del self.p
def clear_all(self, tx_type=ParamKeyType.ALL):
self.p.clearAll(tx_type)
def check_key(self, key):
key = ensure_bytes(key)
if not self.p.checkKey(key):
raise UnknownKeyName(key)
return key
def get(self, key, bool block=False, encoding=None):
cdef string k = self.check_key(key)
cdef string val
with nogil:
val = self.p.get(k, block)
if val == b"":
if block:
# If we got no value while running in blocked mode
# it means we got an interrupt while waiting
raise KeyboardInterrupt
else:
return None
return val if encoding is None else val.decode(encoding)
def get_bool(self, key, bool block=False):
cdef string k = self.check_key(key)
cdef bool r
with nogil:
r = self.p.getBool(k, block)
return r
def put(self, key, dat):
"""
Warning: This function blocks until the param is written to disk!
In very rare cases this can take over a second, and your code will hang.
Use the put_nonblocking, put_bool_nonblocking in time sensitive code, but
in general try to avoid writing params as much as possible.
"""
cdef string k = self.check_key(key)
cdef string dat_bytes = ensure_bytes(dat)
with nogil:
self.p.put(k, dat_bytes)
def put_bool(self, key, bool val):
cdef string k = self.check_key(key)
with nogil:
self.p.putBool(k, val)
def put_nonblocking(self, key, dat):
cdef string k = self.check_key(key)
cdef string dat_bytes = ensure_bytes(dat)
with nogil:
self.p.putNonBlocking(k, dat_bytes)
def put_bool_nonblocking(self, key, bool val):
cdef string k = self.check_key(key)
with nogil:
self.p.putBoolNonBlocking(k, val)
def remove(self, key):
cdef string k = self.check_key(key)
with nogil:
self.p.remove(k)
def get_param_path(self, key=""):
cdef string key_bytes = ensure_bytes(key)
return self.p.getParamPath(key_bytes).decode("utf-8")
def all_keys(self):
return self.p.allKeys()

34
common/prefix.h Normal file
View File

@@ -0,0 +1,34 @@
#pragma once
#include <cassert>
#include <string>
#include "common/params.h"
#include "common/util.h"
class OpenpilotPrefix {
public:
OpenpilotPrefix(std::string prefix = {}) {
if (prefix.empty()) {
prefix = util::random_string(15);
}
msgq_path = "/dev/shm/" + prefix;
bool ret = util::create_directories(msgq_path, 0777);
assert(ret);
setenv("OPENPILOT_PREFIX", prefix.c_str(), 1);
}
~OpenpilotPrefix() {
auto param_path = Params().getParamPath();
if (util::file_exists(param_path)) {
std::string real_path = util::readlink(param_path);
system(util::string_format("rm %s -rf", real_path.c_str()).c_str());
unlink(param_path.c_str());
}
system(util::string_format("rm %s -rf", msgq_path.c_str()).c_str());
unsetenv("OPENPILOT_PREFIX");
}
private:
std::string msgq_path;
};

52
common/prefix.py Normal file
View File

@@ -0,0 +1,52 @@
import os
import shutil
import uuid
from typing import Optional
from openpilot.common.params import Params
from openpilot.system.hardware.hw import Paths
from openpilot.system.hardware.hw import DEFAULT_DOWNLOAD_CACHE_ROOT
class OpenpilotPrefix:
def __init__(self, prefix: Optional[str] = None, clean_dirs_on_exit: bool = True, shared_download_cache: bool = False):
self.prefix = prefix if prefix else str(uuid.uuid4().hex[0:15])
self.msgq_path = os.path.join('/dev/shm', self.prefix)
self.clean_dirs_on_exit = clean_dirs_on_exit
self.shared_download_cache = shared_download_cache
def __enter__(self):
self.original_prefix = os.environ.get('OPENPILOT_PREFIX', None)
os.environ['OPENPILOT_PREFIX'] = self.prefix
try:
os.mkdir(self.msgq_path)
except FileExistsError:
pass
os.makedirs(Paths.log_root(), exist_ok=True)
if self.shared_download_cache:
os.environ["COMMA_CACHE"] = DEFAULT_DOWNLOAD_CACHE_ROOT
return self
def __exit__(self, exc_type, exc_obj, exc_tb):
if self.clean_dirs_on_exit:
self.clean_dirs()
try:
del os.environ['OPENPILOT_PREFIX']
if self.original_prefix is not None:
os.environ['OPENPILOT_PREFIX'] = self.original_prefix
except KeyError:
pass
return False
def clean_dirs(self):
symlink_path = Params().get_param_path()
if os.path.exists(symlink_path):
shutil.rmtree(os.path.realpath(symlink_path), ignore_errors=True)
os.remove(symlink_path)
shutil.rmtree(self.msgq_path, ignore_errors=True)
shutil.rmtree(Paths.log_root(), ignore_errors=True)
if not os.environ.get("COMMA_CACHE", False):
shutil.rmtree(Paths.download_cache_root(), ignore_errors=True)
shutil.rmtree(Paths.comma_home(), ignore_errors=True)

52
common/queue.h Normal file
View File

@@ -0,0 +1,52 @@
#pragma once
#include <condition_variable>
#include <mutex>
#include <queue>
template <class T>
class SafeQueue {
public:
SafeQueue() = default;
void push(const T& v) {
{
std::unique_lock lk(m);
q.push(v);
}
cv.notify_one();
}
T pop() {
std::unique_lock lk(m);
cv.wait(lk, [this] { return !q.empty(); });
T v = q.front();
q.pop();
return v;
}
bool try_pop(T& v, int timeout_ms = 0) {
std::unique_lock lk(m);
if (!cv.wait_for(lk, std::chrono::milliseconds(timeout_ms), [this] { return !q.empty(); })) {
return false;
}
v = q.front();
q.pop();
return true;
}
bool empty() const {
std::scoped_lock lk(m);
return q.empty();
}
size_t size() const {
std::scoped_lock lk(m);
return q.size();
}
private:
mutable std::mutex m;
std::condition_variable cv;
std::queue<T> q;
};

40
common/ratekeeper.cc Normal file
View File

@@ -0,0 +1,40 @@
#include "common/ratekeeper.h"
#include <algorithm>
#include "common/swaglog.h"
#include "common/timing.h"
#include "common/util.h"
RateKeeper::RateKeeper(const std::string &name, float rate, float print_delay_threshold)
: name(name),
print_delay_threshold(std::max(0.f, print_delay_threshold)) {
interval = 1 / rate;
last_monitor_time = seconds_since_boot();
next_frame_time = last_monitor_time + interval;
}
bool RateKeeper::keepTime() {
bool lagged = monitorTime();
if (remaining_ > 0) {
util::sleep_for(remaining_ * 1000);
}
return lagged;
}
bool RateKeeper::monitorTime() {
++frame_;
last_monitor_time = seconds_since_boot();
remaining_ = next_frame_time - last_monitor_time;
bool lagged = remaining_ < 0;
if (lagged) {
if (print_delay_threshold > 0 && remaining_ < -print_delay_threshold) {
LOGW("%s lagging by %.2f ms", name.c_str(), -remaining_ * 1000);
}
next_frame_time = last_monitor_time + interval;
} else {
next_frame_time += interval;
}
return lagged;
}

23
common/ratekeeper.h Normal file
View File

@@ -0,0 +1,23 @@
#pragma once
#include <cstdint>
#include <string>
class RateKeeper {
public:
RateKeeper(const std::string &name, float rate, float print_delay_threshold = 0);
~RateKeeper() {}
bool keepTime();
bool monitorTime();
inline double frame() const { return frame_; }
inline double remaining() const { return remaining_; }
private:
double interval;
double next_frame_time;
double last_monitor_time;
double remaining_ = 0;
float print_delay_threshold = 0;
uint64_t frame_ = 0;
std::string name;
};

95
common/realtime.py Normal file
View File

@@ -0,0 +1,95 @@
"""Utilities for reading real time clocks and keeping soft real time constraints."""
import gc
import os
import time
from collections import deque
from typing import Optional, List, Union
from setproctitle import getproctitle
from openpilot.system.hardware import PC
# time step for each process
DT_CTRL = 0.01 # controlsd
DT_MDL = 0.05 # model
DT_TRML = 0.5 # thermald and manager
DT_DMON = 0.05 # driver monitoring
class Priority:
# CORE 2
# - modeld = 55
# - camerad = 54
CTRL_LOW = 51 # plannerd & radard
# CORE 3
# - boardd = 55
CTRL_HIGH = 53
def set_realtime_priority(level: int) -> None:
if not PC:
os.sched_setscheduler(0, os.SCHED_FIFO, os.sched_param(level))
def set_core_affinity(cores: List[int]) -> None:
if not PC:
os.sched_setaffinity(0, cores)
def config_realtime_process(cores: Union[int, List[int]], priority: int) -> None:
gc.disable()
set_realtime_priority(priority)
c = cores if isinstance(cores, list) else [cores, ]
set_core_affinity(c)
class Ratekeeper:
def __init__(self, rate: float, print_delay_threshold: Optional[float] = 0.0) -> None:
"""Rate in Hz for ratekeeping. print_delay_threshold must be nonnegative."""
self._interval = 1. / rate
self._next_frame_time = time.monotonic() + self._interval
self._print_delay_threshold = print_delay_threshold
self._frame = 0
self._remaining = 0.0
self._process_name = getproctitle()
self._dts = deque([self._interval], maxlen=100)
self._last_monitor_time = time.monotonic()
@property
def frame(self) -> int:
return self._frame
@property
def remaining(self) -> float:
return self._remaining
@property
def lagging(self) -> bool:
avg_dt = sum(self._dts) / len(self._dts)
expected_dt = self._interval * (1 / 0.9)
return avg_dt > expected_dt
# Maintain loop rate by calling this at the end of each loop
def keep_time(self) -> bool:
lagged = self.monitor_time()
if self._remaining > 0:
time.sleep(self._remaining)
return lagged
# Monitors the cumulative lag, but does not enforce a rate
def monitor_time(self) -> bool:
prev = self._last_monitor_time
self._last_monitor_time = time.monotonic()
self._dts.append(self._last_monitor_time - prev)
lagged = False
remaining = self._next_frame_time - time.monotonic()
self._next_frame_time += self._interval
if self._print_delay_threshold is not None and remaining < -self._print_delay_threshold:
print(f"{self._process_name} lagging by {-remaining * 1000:.2f} ms")
lagged = True
self._frame += 1
self._remaining = remaining
return lagged

30
common/retry.py Normal file
View File

@@ -0,0 +1,30 @@
import time
import functools
from openpilot.common.swaglog import cloudlog
def retry(attempts=3, delay=1.0, ignore_failure=False):
def decorator(func):
@functools.wraps(func)
def wrapper(*args, **kwargs):
for _ in range(attempts):
try:
return func(*args, **kwargs)
except Exception:
cloudlog.exception(f"{func.__name__} failed, trying again")
time.sleep(delay)
if ignore_failure:
cloudlog.error(f"{func.__name__} failed after retry")
else:
raise Exception(f"{func.__name__} failed after retry")
return wrapper
return decorator
if __name__ == "__main__":
@retry(attempts=10)
def abc():
raise ValueError("abc failed :(")
abc()

54
common/simple_kalman.py Normal file
View File

@@ -0,0 +1,54 @@
import numpy as np
def get_kalman_gain(dt, A, C, Q, R, iterations=100):
P = np.zeros_like(Q)
for _ in range(iterations):
P = A.dot(P).dot(A.T) + dt * Q
S = C.dot(P).dot(C.T) + R
K = P.dot(C.T).dot(np.linalg.inv(S))
P = (np.eye(len(P)) - K.dot(C)).dot(P)
return K
class KF1D:
# this EKF assumes constant covariance matrix, so calculations are much simpler
# the Kalman gain also needs to be precomputed using the control module
def __init__(self, x0, A, C, K):
self.x0_0 = x0[0][0]
self.x1_0 = x0[1][0]
self.A0_0 = A[0][0]
self.A0_1 = A[0][1]
self.A1_0 = A[1][0]
self.A1_1 = A[1][1]
self.C0_0 = C[0]
self.C0_1 = C[1]
self.K0_0 = K[0][0]
self.K1_0 = K[1][0]
self.A_K_0 = self.A0_0 - self.K0_0 * self.C0_0
self.A_K_1 = self.A0_1 - self.K0_0 * self.C0_1
self.A_K_2 = self.A1_0 - self.K1_0 * self.C0_0
self.A_K_3 = self.A1_1 - self.K1_0 * self.C0_1
# K matrix needs to be pre-computed as follow:
# import control
# (x, l, K) = control.dare(np.transpose(self.A), np.transpose(self.C), Q, R)
# self.K = np.transpose(K)
def update(self, meas):
#self.x = np.dot(self.A_K, self.x) + np.dot(self.K, meas)
x0_0 = self.A_K_0 * self.x0_0 + self.A_K_1 * self.x1_0 + self.K0_0 * meas
x1_0 = self.A_K_2 * self.x0_0 + self.A_K_3 * self.x1_0 + self.K1_0 * meas
self.x0_0 = x0_0
self.x1_0 = x1_0
return [self.x0_0, self.x1_0]
@property
def x(self):
return [[self.x0_0], [self.x1_0]]
def set_x(self, x):
self.x0_0 = x[0][0]
self.x1_0 = x[1][0]

52
common/spinner.py Normal file
View File

@@ -0,0 +1,52 @@
import os
import subprocess
from openpilot.common.basedir import BASEDIR
class Spinner():
def __init__(self):
try:
self.spinner_proc = subprocess.Popen(["./spinner"],
stdin=subprocess.PIPE,
cwd=os.path.join(BASEDIR, "selfdrive", "ui"),
close_fds=True)
except OSError:
self.spinner_proc = None
def __enter__(self):
return self
def update(self, spinner_text: str):
if self.spinner_proc is not None:
self.spinner_proc.stdin.write(spinner_text.encode('utf8') + b"\n")
try:
self.spinner_proc.stdin.flush()
except BrokenPipeError:
pass
def update_progress(self, cur: float, total: float):
self.update(str(round(100 * cur / total)))
def close(self):
if self.spinner_proc is not None:
self.spinner_proc.kill()
try:
self.spinner_proc.communicate(timeout=2.)
except subprocess.TimeoutExpired:
print("WARNING: failed to kill spinner")
self.spinner_proc = None
def __del__(self):
self.close()
def __exit__(self, exc_type, exc_value, traceback):
self.close()
if __name__ == "__main__":
import time
with Spinner() as s:
s.update("Spinner text")
time.sleep(5.0)
print("gone")
time.sleep(5.0)

73
common/stat_live.py Normal file
View File

@@ -0,0 +1,73 @@
import numpy as np
class RunningStat():
# tracks realtime mean and standard deviation without storing any data
def __init__(self, priors=None, max_trackable=-1):
self.max_trackable = max_trackable
if priors is not None:
# initialize from history
self.M = priors[0]
self.S = priors[1]
self.n = priors[2]
self.M_last = self.M
self.S_last = self.S
else:
self.reset()
def reset(self):
self.M = 0.
self.S = 0.
self.M_last = 0.
self.S_last = 0.
self.n = 0
def push_data(self, new_data):
# short term memory hack
if self.max_trackable < 0 or self.n < self.max_trackable:
self.n += 1
if self.n == 0:
self.M_last = new_data
self.M = self.M_last
self.S_last = 0.
else:
self.M = self.M_last + (new_data - self.M_last) / self.n
self.S = self.S_last + (new_data - self.M_last) * (new_data - self.M)
self.M_last = self.M
self.S_last = self.S
def mean(self):
return self.M
def variance(self):
if self.n >= 2:
return self.S / (self.n - 1.)
else:
return 0
def std(self):
return np.sqrt(self.variance())
def params_to_save(self):
return [self.M, self.S, self.n]
class RunningStatFilter():
def __init__(self, raw_priors=None, filtered_priors=None, max_trackable=-1):
self.raw_stat = RunningStat(raw_priors, -1)
self.filtered_stat = RunningStat(filtered_priors, max_trackable)
def reset(self):
self.raw_stat.reset()
self.filtered_stat.reset()
def push_and_update(self, new_data):
_std_last = self.raw_stat.std()
self.raw_stat.push_data(new_data)
_delta_std = self.raw_stat.std() - _std_last
if _delta_std <= 0:
self.filtered_stat.push_data(new_data)
else:
pass
# self.filtered_stat.push_data(self.filtered_stat.mean())
# class SequentialBayesian():

152
common/swaglog.cc Normal file
View File

@@ -0,0 +1,152 @@
#ifndef _GNU_SOURCE
#define _GNU_SOURCE
#endif
#include "common/swaglog.h"
#include <cassert>
#include <limits>
#include <mutex>
#include <string>
#include <zmq.h>
#include <stdarg.h>
#include "third_party/json11/json11.hpp"
#include "common/version.h"
#include "system/hardware/hw.h"
class SwaglogState {
public:
SwaglogState() {
zctx = zmq_ctx_new();
sock = zmq_socket(zctx, ZMQ_PUSH);
// Timeout on shutdown for messages to be received by the logging process
int timeout = 100;
zmq_setsockopt(sock, ZMQ_LINGER, &timeout, sizeof(timeout));
zmq_connect(sock, Path::swaglog_ipc().c_str());
print_level = CLOUDLOG_WARNING;
if (const char* print_lvl = getenv("LOGPRINT")) {
if (strcmp(print_lvl, "debug") == 0) {
print_level = CLOUDLOG_DEBUG;
} else if (strcmp(print_lvl, "info") == 0) {
print_level = CLOUDLOG_INFO;
} else if (strcmp(print_lvl, "warning") == 0) {
print_level = CLOUDLOG_WARNING;
}
}
ctx_j = json11::Json::object{};
if (char* dongle_id = getenv("DONGLE_ID")) {
ctx_j["dongle_id"] = dongle_id;
}
if (char* git_origin = getenv("GIT_ORIGIN")) {
ctx_j["origin"] = git_origin;
}
if (char* git_branch = getenv("GIT_BRANCH")) {
ctx_j["branch"] = git_branch;
}
if (char* git_commit = getenv("GIT_COMMIT")) {
ctx_j["commit"] = git_commit;
}
if (char* daemon_name = getenv("MANAGER_DAEMON")) {
ctx_j["daemon"] = daemon_name;
}
ctx_j["version"] = COMMA_VERSION;
ctx_j["dirty"] = !getenv("CLEAN");
ctx_j["device"] = Hardware::get_name();
}
~SwaglogState() {
zmq_close(sock);
zmq_ctx_destroy(zctx);
}
void log(int levelnum, const char* filename, int lineno, const char* func, const char* msg, const std::string& log_s) {
std::lock_guard lk(lock);
if (levelnum >= print_level) {
printf("%s: %s\n", filename, msg);
}
zmq_send(sock, log_s.data(), log_s.length(), ZMQ_NOBLOCK);
}
std::mutex lock;
void* zctx = nullptr;
void* sock = nullptr;
int print_level;
json11::Json::object ctx_j;
};
bool LOG_TIMESTAMPS = getenv("LOG_TIMESTAMPS");
uint32_t NO_FRAME_ID = std::numeric_limits<uint32_t>::max();
static void cloudlog_common(int levelnum, const char* filename, int lineno, const char* func,
char* msg_buf, const json11::Json::object &msg_j={}) {
static SwaglogState s;
json11::Json::object log_j = json11::Json::object {
{"ctx", s.ctx_j},
{"levelnum", levelnum},
{"filename", filename},
{"lineno", lineno},
{"funcname", func},
{"created", seconds_since_epoch()}
};
if (msg_j.empty()) {
log_j["msg"] = msg_buf;
} else {
log_j["msg"] = msg_j;
}
std::string log_s;
log_s += (char)levelnum;
((json11::Json)log_j).dump(log_s);
s.log(levelnum, filename, lineno, func, msg_buf, log_s);
free(msg_buf);
}
void cloudlog_e(int levelnum, const char* filename, int lineno, const char* func,
const char* fmt, ...) {
va_list args;
va_start(args, fmt);
char* msg_buf = nullptr;
int ret = vasprintf(&msg_buf, fmt, args);
va_end(args);
if (ret <= 0 || !msg_buf) return;
cloudlog_common(levelnum, filename, lineno, func, msg_buf);
}
void cloudlog_t_common(int levelnum, const char* filename, int lineno, const char* func,
uint32_t frame_id, const char* fmt, va_list args) {
if (!LOG_TIMESTAMPS) return;
char* msg_buf = nullptr;
int ret = vasprintf(&msg_buf, fmt, args);
if (ret <= 0 || !msg_buf) return;
json11::Json::object tspt_j = json11::Json::object{
{"event", msg_buf},
{"time", std::to_string(nanos_since_boot())}
};
if (frame_id < NO_FRAME_ID) {
tspt_j["frame_id"] = std::to_string(frame_id);
}
tspt_j = json11::Json::object{{"timestamp", tspt_j}};
cloudlog_common(levelnum, filename, lineno, func, msg_buf, tspt_j);
}
void cloudlog_te(int levelnum, const char* filename, int lineno, const char* func,
const char* fmt, ...) {
va_list args;
va_start(args, fmt);
cloudlog_t_common(levelnum, filename, lineno, func, NO_FRAME_ID, fmt, args);
va_end(args);
}
void cloudlog_te(int levelnum, const char* filename, int lineno, const char* func,
uint32_t frame_id, const char* fmt, ...) {
va_list args;
va_start(args, fmt);
cloudlog_t_common(levelnum, filename, lineno, func, frame_id, fmt, args);
va_end(args);
}

76
common/swaglog.h Normal file
View File

@@ -0,0 +1,76 @@
#pragma once
#include "common/timing.h"
#define CLOUDLOG_DEBUG 10
#define CLOUDLOG_INFO 20
#define CLOUDLOG_WARNING 30
#define CLOUDLOG_ERROR 40
#define CLOUDLOG_CRITICAL 50
#ifdef __GNUC__
#define SWAG_LOG_CHECK_FMT(a, b) __attribute__ ((format (printf, a, b)))
#else
#define SWAG_LOG_CHECK_FMT(a, b)
#endif
void cloudlog_e(int levelnum, const char* filename, int lineno, const char* func,
const char* fmt, ...) SWAG_LOG_CHECK_FMT(5, 6);
void cloudlog_te(int levelnum, const char* filename, int lineno, const char* func,
const char* fmt, ...) SWAG_LOG_CHECK_FMT(5, 6);
void cloudlog_te(int levelnum, const char* filename, int lineno, const char* func,
uint32_t frame_id, const char* fmt, ...) SWAG_LOG_CHECK_FMT(6, 7);
#define cloudlog(lvl, fmt, ...) cloudlog_e(lvl, __FILE__, __LINE__, \
__func__, \
fmt, ## __VA_ARGS__)
#define cloudlog_t(lvl, ...) cloudlog_te(lvl, __FILE__, __LINE__, \
__func__, \
__VA_ARGS__)
#define cloudlog_rl(burst, millis, lvl, fmt, ...) \
{ \
static uint64_t __begin = 0; \
static int __printed = 0; \
static int __missed = 0; \
\
int __burst = (burst); \
int __millis = (millis); \
uint64_t __ts = nanos_since_boot(); \
\
if (!__begin) { __begin = __ts; } \
\
if (__begin + __millis*1000000ULL < __ts) { \
if (__missed) { \
cloudlog(CLOUDLOG_WARNING, "cloudlog: %d messages suppressed", __missed); \
} \
__begin = 0; \
__printed = 0; \
__missed = 0; \
} \
\
if (__printed < __burst) { \
cloudlog(lvl, fmt, ## __VA_ARGS__); \
__printed++; \
} else { \
__missed++; \
} \
}
#define LOGT(...) cloudlog_t(CLOUDLOG_DEBUG, __VA_ARGS__)
#define LOGD(fmt, ...) cloudlog(CLOUDLOG_DEBUG, fmt, ## __VA_ARGS__)
#define LOG(fmt, ...) cloudlog(CLOUDLOG_INFO, fmt, ## __VA_ARGS__)
#define LOGW(fmt, ...) cloudlog(CLOUDLOG_WARNING, fmt, ## __VA_ARGS__)
#define LOGE(fmt, ...) cloudlog(CLOUDLOG_ERROR, fmt, ## __VA_ARGS__)
#define LOGD_100(fmt, ...) cloudlog_rl(2, 100, CLOUDLOG_DEBUG, fmt, ## __VA_ARGS__)
#define LOG_100(fmt, ...) cloudlog_rl(2, 100, CLOUDLOG_INFO, fmt, ## __VA_ARGS__)
#define LOGW_100(fmt, ...) cloudlog_rl(2, 100, CLOUDLOG_WARNING, fmt, ## __VA_ARGS__)
#define LOGE_100(fmt, ...) cloudlog_rl(2, 100, CLOUDLOG_ERROR, fmt, ## __VA_ARGS__)

135
common/swaglog.py Normal file
View File

@@ -0,0 +1,135 @@
import logging
import os
import time
import warnings
from pathlib import Path
from logging.handlers import BaseRotatingHandler
import zmq
from openpilot.common.logging_extra import SwagLogger, SwagFormatter, SwagLogFileFormatter
from openpilot.system.hardware.hw import Paths
def get_file_handler():
Path(Paths.swaglog_root()).mkdir(parents=True, exist_ok=True)
base_filename = os.path.join(Paths.swaglog_root(), "swaglog")
handler = SwaglogRotatingFileHandler(base_filename)
return handler
class SwaglogRotatingFileHandler(BaseRotatingHandler):
def __init__(self, base_filename, interval=60, max_bytes=1024*256, backup_count=2500, encoding=None):
super().__init__(base_filename, mode="a", encoding=encoding, delay=True)
self.base_filename = base_filename
self.interval = interval # seconds
self.max_bytes = max_bytes
self.backup_count = backup_count
self.log_files = self.get_existing_logfiles()
log_indexes = [f.split(".")[-1] for f in self.log_files]
self.last_file_idx = max([int(i) for i in log_indexes if i.isdigit()] or [-1])
self.last_rollover = None
self.doRollover()
def _open(self):
self.last_rollover = time.monotonic()
self.last_file_idx += 1
next_filename = f"{self.base_filename}.{self.last_file_idx:010}"
stream = open(next_filename, self.mode, encoding=self.encoding)
self.log_files.insert(0, next_filename)
return stream
def get_existing_logfiles(self):
log_files = list()
base_dir = os.path.dirname(self.base_filename)
for fn in os.listdir(base_dir):
fp = os.path.join(base_dir, fn)
if fp.startswith(self.base_filename) and os.path.isfile(fp):
log_files.append(fp)
return sorted(log_files)
def shouldRollover(self, record):
size_exceeded = self.max_bytes > 0 and self.stream.tell() >= self.max_bytes
time_exceeded = self.interval > 0 and self.last_rollover + self.interval <= time.monotonic()
return size_exceeded or time_exceeded
def doRollover(self):
if self.stream:
self.stream.close()
self.stream = self._open()
if self.backup_count > 0:
while len(self.log_files) > self.backup_count:
to_delete = self.log_files.pop()
if os.path.exists(to_delete): # just being safe, should always exist
os.remove(to_delete)
class UnixDomainSocketHandler(logging.Handler):
def __init__(self, formatter):
logging.Handler.__init__(self)
self.setFormatter(formatter)
self.pid = None
self.zctx = None
self.sock = None
def __del__(self):
self.close()
def close(self):
if self.sock is not None:
self.sock.close()
if self.zctx is not None:
self.zctx.term()
def connect(self):
self.zctx = zmq.Context()
self.sock = self.zctx.socket(zmq.PUSH)
self.sock.setsockopt(zmq.LINGER, 10)
self.sock.connect(Paths.swaglog_ipc())
self.pid = os.getpid()
def emit(self, record):
if os.getpid() != self.pid:
# TODO suppresses warning about forking proc with zmq socket, fix root cause
warnings.filterwarnings("ignore", category=ResourceWarning, message="unclosed.*<zmq.*>")
self.connect()
msg = self.format(record).rstrip('\n')
# print("SEND".format(repr(msg)))
try:
s = chr(record.levelno)+msg
self.sock.send(s.encode('utf8'), zmq.NOBLOCK)
except zmq.error.Again:
# drop :/
pass
def add_file_handler(log):
"""
Function to add the file log handler to swaglog.
This can be used to store logs when logmessaged is not running.
"""
handler = get_file_handler()
handler.setFormatter(SwagLogFileFormatter(log))
log.addHandler(handler)
cloudlog = log = SwagLogger()
log.setLevel(logging.DEBUG)
outhandler = logging.StreamHandler()
print_level = os.environ.get('LOGPRINT', 'warning')
if print_level == 'debug':
outhandler.setLevel(logging.DEBUG)
elif print_level == 'info':
outhandler.setLevel(logging.INFO)
elif print_level == 'warning':
outhandler.setLevel(logging.WARNING)
ipchandler = UnixDomainSocketHandler(SwagFormatter(log))
log.addHandler(outhandler)
# logs are sent through IPC before writing to disk to prevent disk I/O blocking
log.addHandler(ipchandler)

63
common/text_window.py Executable file
View File

@@ -0,0 +1,63 @@
#!/usr/bin/env python3
import os
import time
import subprocess
from openpilot.common.basedir import BASEDIR
class TextWindow:
def __init__(self, text):
try:
self.text_proc = subprocess.Popen(["./text", text],
stdin=subprocess.PIPE,
cwd=os.path.join(BASEDIR, "selfdrive", "ui"),
close_fds=True)
except OSError:
self.text_proc = None
def get_status(self):
if self.text_proc is not None:
self.text_proc.poll()
return self.text_proc.returncode
return None
def __enter__(self):
return self
def close(self):
if self.text_proc is not None:
self.text_proc.terminate()
self.text_proc = None
def wait_for_exit(self):
if self.text_proc is not None:
while True:
if self.get_status() == 1:
return
time.sleep(0.1)
def __del__(self):
self.close()
def __exit__(self, exc_type, exc_value, traceback):
self.close()
if __name__ == "__main__":
text = """Traceback (most recent call last):
File "./controlsd.py", line 608, in <module>
main()
File "./controlsd.py", line 604, in main
controlsd_thread(sm, pm, logcan)
File "./controlsd.py", line 455, in controlsd_thread
1/0
ZeroDivisionError: division by zero"""
print(text)
with TextWindow(text) as s:
for _ in range(100):
if s.get_status() == 1:
print("Got exit button")
break
time.sleep(0.1)
print("gone")

6
common/time.py Normal file
View File

@@ -0,0 +1,6 @@
import datetime
MIN_DATE = datetime.datetime(year=2024, month=1, day=28)
def system_time_valid():
return datetime.datetime.now() > MIN_DATE

27
common/timeout.py Normal file
View File

@@ -0,0 +1,27 @@
import signal
class TimeoutException(Exception):
pass
class Timeout:
"""
Timeout context manager.
For example this code will raise a TimeoutException:
with Timeout(seconds=5, error_msg="Sleep was too long"):
time.sleep(10)
"""
def __init__(self, seconds, error_msg=None):
if error_msg is None:
error_msg = f'Timed out after {seconds} seconds'
self.seconds = seconds
self.error_msg = error_msg
def handle_timeout(self, signume, frame):
raise TimeoutException(self.error_msg)
def __enter__(self):
signal.signal(signal.SIGALRM, self.handle_timeout)
signal.alarm(self.seconds)
def __exit__(self, exc_type, exc_val, exc_tb):
signal.alarm(0)

51
common/timing.h Normal file
View File

@@ -0,0 +1,51 @@
#pragma once
#include <cstdint>
#include <ctime>
#ifdef __APPLE__
#define CLOCK_BOOTTIME CLOCK_MONOTONIC
#endif
static inline uint64_t nanos_since_boot() {
struct timespec t;
clock_gettime(CLOCK_BOOTTIME, &t);
return t.tv_sec * 1000000000ULL + t.tv_nsec;
}
static inline double millis_since_boot() {
struct timespec t;
clock_gettime(CLOCK_BOOTTIME, &t);
return t.tv_sec * 1000.0 + t.tv_nsec * 1e-6;
}
static inline double seconds_since_boot() {
struct timespec t;
clock_gettime(CLOCK_BOOTTIME, &t);
return (double)t.tv_sec + t.tv_nsec * 1e-9;
}
static inline uint64_t nanos_since_epoch() {
struct timespec t;
clock_gettime(CLOCK_REALTIME, &t);
return t.tv_sec * 1000000000ULL + t.tv_nsec;
}
static inline double seconds_since_epoch() {
struct timespec t;
clock_gettime(CLOCK_REALTIME, &t);
return (double)t.tv_sec + t.tv_nsec * 1e-9;
}
// you probably should use nanos_since_boot instead
static inline uint64_t nanos_monotonic() {
struct timespec t;
clock_gettime(CLOCK_MONOTONIC, &t);
return t.tv_sec * 1000000000ULL + t.tv_nsec;
}
static inline uint64_t nanos_monotonic_raw() {
struct timespec t;
clock_gettime(CLOCK_MONOTONIC_RAW, &t);
return t.tv_sec * 1000000000ULL + t.tv_nsec;
}

5
common/transformations/SConscript Normal file
View File

@@ -0,0 +1,5 @@
Import('env', 'envCython')
transformations = env.Library('transformations', ['orientation.cc', 'coordinates.cc'])
transformations_python = envCython.Program('transformations.so', 'transformations.pyx')
Export('transformations', 'transformations_python')

0
common/transformations/__init__.py Normal file
View File

160
common/transformations/camera.py Normal file
View File

@@ -0,0 +1,160 @@
import numpy as np
import openpilot.common.transformations.orientation as orient
## -- hardcoded hardware params --
eon_f_focal_length = 910.0
eon_d_focal_length = 650.0
tici_f_focal_length = 2648.0
tici_e_focal_length = tici_d_focal_length = 567.0 # probably wrong? magnification is not consistent across frame
eon_f_frame_size = (1164, 874)
eon_d_frame_size = (816, 612)
tici_f_frame_size = tici_e_frame_size = tici_d_frame_size = (1928, 1208)
# aka 'K' aka camera_frame_from_view_frame
eon_fcam_intrinsics = np.array([
[eon_f_focal_length, 0.0, float(eon_f_frame_size[0])/2],
[0.0, eon_f_focal_length, float(eon_f_frame_size[1])/2],
[0.0, 0.0, 1.0]])
eon_intrinsics = eon_fcam_intrinsics # xx
eon_dcam_intrinsics = np.array([
[eon_d_focal_length, 0.0, float(eon_d_frame_size[0])/2],
[0.0, eon_d_focal_length, float(eon_d_frame_size[1])/2],
[0.0, 0.0, 1.0]])
tici_fcam_intrinsics = np.array([
[tici_f_focal_length, 0.0, float(tici_f_frame_size[0])/2],
[0.0, tici_f_focal_length, float(tici_f_frame_size[1])/2],
[0.0, 0.0, 1.0]])
tici_dcam_intrinsics = np.array([
[tici_d_focal_length, 0.0, float(tici_d_frame_size[0])/2],
[0.0, tici_d_focal_length, float(tici_d_frame_size[1])/2],
[0.0, 0.0, 1.0]])
tici_ecam_intrinsics = tici_dcam_intrinsics
# aka 'K_inv' aka view_frame_from_camera_frame
eon_fcam_intrinsics_inv = np.linalg.inv(eon_fcam_intrinsics)
eon_intrinsics_inv = eon_fcam_intrinsics_inv # xx
tici_fcam_intrinsics_inv = np.linalg.inv(tici_fcam_intrinsics)
tici_ecam_intrinsics_inv = np.linalg.inv(tici_ecam_intrinsics)
FULL_FRAME_SIZE = tici_f_frame_size
FOCAL = tici_f_focal_length
fcam_intrinsics = tici_fcam_intrinsics
W, H = FULL_FRAME_SIZE[0], FULL_FRAME_SIZE[1]
# device/mesh : x->forward, y-> right, z->down
# view : x->right, y->down, z->forward
device_frame_from_view_frame = np.array([
[ 0., 0., 1.],
[ 1., 0., 0.],
[ 0., 1., 0.]
])
view_frame_from_device_frame = device_frame_from_view_frame.T
# aka 'extrinsic_matrix'
# road : x->forward, y -> left, z->up
def get_view_frame_from_road_frame(roll, pitch, yaw, height):
device_from_road = orient.rot_from_euler([roll, pitch, yaw]).dot(np.diag([1, -1, -1]))
view_from_road = view_frame_from_device_frame.dot(device_from_road)
return np.hstack((view_from_road, [[0], [height], [0]]))
# aka 'extrinsic_matrix'
def get_view_frame_from_calib_frame(roll, pitch, yaw, height):
device_from_calib= orient.rot_from_euler([roll, pitch, yaw])
view_from_calib = view_frame_from_device_frame.dot(device_from_calib)
return np.hstack((view_from_calib, [[0], [height], [0]]))
def vp_from_ke(m):
"""
Computes the vanishing point from the product of the intrinsic and extrinsic
matrices C = KE.
The vanishing point is defined as lim x->infinity C (x, 0, 0, 1).T
"""
return (m[0, 0]/m[2, 0], m[1, 0]/m[2, 0])
def roll_from_ke(m):
# note: different from calibration.h/RollAnglefromKE: i think that one's just wrong
return np.arctan2(-(m[1, 0] - m[1, 1] * m[2, 0] / m[2, 1]),
-(m[0, 0] - m[0, 1] * m[2, 0] / m[2, 1]))
def normalize(img_pts, intrinsics=fcam_intrinsics):
# normalizes image coordinates
# accepts single pt or array of pts
intrinsics_inv = np.linalg.inv(intrinsics)
img_pts = np.array(img_pts)
input_shape = img_pts.shape
img_pts = np.atleast_2d(img_pts)
img_pts = np.hstack((img_pts, np.ones((img_pts.shape[0], 1))))
img_pts_normalized = img_pts.dot(intrinsics_inv.T)
img_pts_normalized[(img_pts < 0).any(axis=1)] = np.nan
return img_pts_normalized[:, :2].reshape(input_shape)
def denormalize(img_pts, intrinsics=fcam_intrinsics, width=np.inf, height=np.inf):
# denormalizes image coordinates
# accepts single pt or array of pts
img_pts = np.array(img_pts)
input_shape = img_pts.shape
img_pts = np.atleast_2d(img_pts)
img_pts = np.hstack((img_pts, np.ones((img_pts.shape[0], 1), dtype=img_pts.dtype)))
img_pts_denormalized = img_pts.dot(intrinsics.T)
if np.isfinite(width):
img_pts_denormalized[img_pts_denormalized[:, 0] > width] = np.nan
img_pts_denormalized[img_pts_denormalized[:, 0] < 0] = np.nan
if np.isfinite(height):
img_pts_denormalized[img_pts_denormalized[:, 1] > height] = np.nan
img_pts_denormalized[img_pts_denormalized[:, 1] < 0] = np.nan
return img_pts_denormalized[:, :2].reshape(input_shape)
def get_calib_from_vp(vp, intrinsics=fcam_intrinsics):
vp_norm = normalize(vp, intrinsics)
yaw_calib = np.arctan(vp_norm[0])
pitch_calib = -np.arctan(vp_norm[1]*np.cos(yaw_calib))
roll_calib = 0
return roll_calib, pitch_calib, yaw_calib
def device_from_ecef(pos_ecef, orientation_ecef, pt_ecef):
# device from ecef frame
# device frame is x -> forward, y-> right, z -> down
# accepts single pt or array of pts
input_shape = pt_ecef.shape
pt_ecef = np.atleast_2d(pt_ecef)
ecef_from_device_rot = orient.rotations_from_quats(orientation_ecef)
device_from_ecef_rot = ecef_from_device_rot.T
pt_ecef_rel = pt_ecef - pos_ecef
pt_device = np.einsum('jk,ik->ij', device_from_ecef_rot, pt_ecef_rel)
return pt_device.reshape(input_shape)
def img_from_device(pt_device):
# img coordinates from pts in device frame
# first transforms to view frame, then to img coords
# accepts single pt or array of pts
input_shape = pt_device.shape
pt_device = np.atleast_2d(pt_device)
pt_view = np.einsum('jk,ik->ij', view_frame_from_device_frame, pt_device)
# This function should never return negative depths
pt_view[pt_view[:, 2] < 0] = np.nan
pt_img = pt_view/pt_view[:, 2:3]
return pt_img.reshape(input_shape)[:, :2]

100
common/transformations/coordinates.cc Normal file
View File

@@ -0,0 +1,100 @@
#define _USE_MATH_DEFINES
#include "common/transformations/coordinates.hpp"
#include <iostream>
#include <cmath>
#include <eigen3/Eigen/Dense>
double a = 6378137; // lgtm [cpp/short-global-name]
double b = 6356752.3142; // lgtm [cpp/short-global-name]
double esq = 6.69437999014 * 0.001; // lgtm [cpp/short-global-name]
double e1sq = 6.73949674228 * 0.001;
static Geodetic to_degrees(Geodetic geodetic){
geodetic.lat = RAD2DEG(geodetic.lat);
geodetic.lon = RAD2DEG(geodetic.lon);
return geodetic;
}
static Geodetic to_radians(Geodetic geodetic){
geodetic.lat = DEG2RAD(geodetic.lat);
geodetic.lon = DEG2RAD(geodetic.lon);
return geodetic;
}
ECEF geodetic2ecef(Geodetic g){
g = to_radians(g);
double xi = sqrt(1.0 - esq * pow(sin(g.lat), 2));
double x = (a / xi + g.alt) * cos(g.lat) * cos(g.lon);
double y = (a / xi + g.alt) * cos(g.lat) * sin(g.lon);
double z = (a / xi * (1.0 - esq) + g.alt) * sin(g.lat);
return {x, y, z};
}
Geodetic ecef2geodetic(ECEF e){
// Convert from ECEF to geodetic using Ferrari's methods
// https://en.wikipedia.org/wiki/Geographic_coordinate_conversion#Ferrari.27s_solution
double x = e.x;
double y = e.y;
double z = e.z;
double r = sqrt(x * x + y * y);
double Esq = a * a - b * b;
double F = 54 * b * b * z * z;
double G = r * r + (1 - esq) * z * z - esq * Esq;
double C = (esq * esq * F * r * r) / (pow(G, 3));
double S = cbrt(1 + C + sqrt(C * C + 2 * C));
double P = F / (3 * pow((S + 1 / S + 1), 2) * G * G);
double Q = sqrt(1 + 2 * esq * esq * P);
double r_0 = -(P * esq * r) / (1 + Q) + sqrt(0.5 * a * a*(1 + 1.0 / Q) - P * (1 - esq) * z * z / (Q * (1 + Q)) - 0.5 * P * r * r);
double U = sqrt(pow((r - esq * r_0), 2) + z * z);
double V = sqrt(pow((r - esq * r_0), 2) + (1 - esq) * z * z);
double Z_0 = b * b * z / (a * V);
double h = U * (1 - b * b / (a * V));
double lat = atan((z + e1sq * Z_0) / r);
double lon = atan2(y, x);
return to_degrees({lat, lon, h});
}
LocalCoord::LocalCoord(Geodetic g, ECEF e){
init_ecef << e.x, e.y, e.z;
g = to_radians(g);
ned2ecef_matrix <<
-sin(g.lat)*cos(g.lon), -sin(g.lon), -cos(g.lat)*cos(g.lon),
-sin(g.lat)*sin(g.lon), cos(g.lon), -cos(g.lat)*sin(g.lon),
cos(g.lat), 0, -sin(g.lat);
ecef2ned_matrix = ned2ecef_matrix.transpose();
}
NED LocalCoord::ecef2ned(ECEF e) {
Eigen::Vector3d ecef;
ecef << e.x, e.y, e.z;
Eigen::Vector3d ned = (ecef2ned_matrix * (ecef - init_ecef));
return {ned[0], ned[1], ned[2]};
}
ECEF LocalCoord::ned2ecef(NED n) {
Eigen::Vector3d ned;
ned << n.n, n.e, n.d;
Eigen::Vector3d ecef = (ned2ecef_matrix * ned) + init_ecef;
return {ecef[0], ecef[1], ecef[2]};
}
NED LocalCoord::geodetic2ned(Geodetic g) {
ECEF e = ::geodetic2ecef(g);
return ecef2ned(e);
}
Geodetic LocalCoord::ned2geodetic(NED n){
ECEF e = ned2ecef(n);
return ::ecef2geodetic(e);
}

43
common/transformations/coordinates.hpp Normal file
View File

@@ -0,0 +1,43 @@
#pragma once
#include <eigen3/Eigen/Dense>
#define DEG2RAD(x) ((x) * M_PI / 180.0)
#define RAD2DEG(x) ((x) * 180.0 / M_PI)
struct ECEF {
double x, y, z;
Eigen::Vector3d to_vector(){
return Eigen::Vector3d(x, y, z);
}
};
struct NED {
double n, e, d;
Eigen::Vector3d to_vector(){
return Eigen::Vector3d(n, e, d);
}
};
struct Geodetic {
double lat, lon, alt;
bool radians=false;
};
ECEF geodetic2ecef(Geodetic g);
Geodetic ecef2geodetic(ECEF e);
class LocalCoord {
public:
Eigen::Matrix3d ned2ecef_matrix;
Eigen::Matrix3d ecef2ned_matrix;
Eigen::Vector3d init_ecef;
LocalCoord(Geodetic g, ECEF e);
LocalCoord(Geodetic g) : LocalCoord(g, ::geodetic2ecef(g)) {}
LocalCoord(ECEF e) : LocalCoord(::ecef2geodetic(e), e) {}
NED ecef2ned(ECEF e);
ECEF ned2ecef(NED n);
NED geodetic2ned(Geodetic g);
Geodetic ned2geodetic(NED n);
};

18
common/transformations/coordinates.py Normal file
View File

@@ -0,0 +1,18 @@
from openpilot.common.transformations.orientation import numpy_wrap
from openpilot.common.transformations.transformations import (ecef2geodetic_single,
geodetic2ecef_single)
from openpilot.common.transformations.transformations import LocalCoord as LocalCoord_single
class LocalCoord(LocalCoord_single):
ecef2ned = numpy_wrap(LocalCoord_single.ecef2ned_single, (3,), (3,))
ned2ecef = numpy_wrap(LocalCoord_single.ned2ecef_single, (3,), (3,))
geodetic2ned = numpy_wrap(LocalCoord_single.geodetic2ned_single, (3,), (3,))
ned2geodetic = numpy_wrap(LocalCoord_single.ned2geodetic_single, (3,), (3,))
geodetic2ecef = numpy_wrap(geodetic2ecef_single, (3,), (3,))
ecef2geodetic = numpy_wrap(ecef2geodetic_single, (3,), (3,))
geodetic_from_ecef = ecef2geodetic
ecef_from_geodetic = geodetic2ecef

78
common/transformations/model.py Normal file
View File

@@ -0,0 +1,78 @@
import numpy as np
from openpilot.common.transformations.orientation import rot_from_euler
from openpilot.common.transformations.camera import (
FULL_FRAME_SIZE, get_view_frame_from_calib_frame, view_frame_from_device_frame,
eon_fcam_intrinsics, tici_ecam_intrinsics, tici_fcam_intrinsics)
# segnet
SEGNET_SIZE = (512, 384)
def get_segnet_frame_from_camera_frame(segnet_size=SEGNET_SIZE, full_frame_size=FULL_FRAME_SIZE):
return np.array([[float(segnet_size[0]) / full_frame_size[0], 0.0],
[0.0, float(segnet_size[1]) / full_frame_size[1]]])
segnet_frame_from_camera_frame = get_segnet_frame_from_camera_frame() # xx
# MED model
MEDMODEL_INPUT_SIZE = (512, 256)
MEDMODEL_YUV_SIZE = (MEDMODEL_INPUT_SIZE[0], MEDMODEL_INPUT_SIZE[1] * 3 // 2)
MEDMODEL_CY = 47.6
medmodel_fl = 910.0
medmodel_intrinsics = np.array([
[medmodel_fl, 0.0, 0.5 * MEDMODEL_INPUT_SIZE[0]],
[0.0, medmodel_fl, MEDMODEL_CY],
[0.0, 0.0, 1.0]])
# BIG model
BIGMODEL_INPUT_SIZE = (1024, 512)
BIGMODEL_YUV_SIZE = (BIGMODEL_INPUT_SIZE[0], BIGMODEL_INPUT_SIZE[1] * 3 // 2)
bigmodel_fl = 910.0
bigmodel_intrinsics = np.array([
[bigmodel_fl, 0.0, 0.5 * BIGMODEL_INPUT_SIZE[0]],
[0.0, bigmodel_fl, 256 + MEDMODEL_CY],
[0.0, 0.0, 1.0]])
# SBIG model (big model with the size of small model)
SBIGMODEL_INPUT_SIZE = (512, 256)
SBIGMODEL_YUV_SIZE = (SBIGMODEL_INPUT_SIZE[0], SBIGMODEL_INPUT_SIZE[1] * 3 // 2)
sbigmodel_fl = 455.0
sbigmodel_intrinsics = np.array([
[sbigmodel_fl, 0.0, 0.5 * SBIGMODEL_INPUT_SIZE[0]],
[0.0, sbigmodel_fl, 0.5 * (256 + MEDMODEL_CY)],
[0.0, 0.0, 1.0]])
bigmodel_frame_from_calib_frame = np.dot(bigmodel_intrinsics,
get_view_frame_from_calib_frame(0, 0, 0, 0))
sbigmodel_frame_from_calib_frame = np.dot(sbigmodel_intrinsics,
get_view_frame_from_calib_frame(0, 0, 0, 0))
medmodel_frame_from_calib_frame = np.dot(medmodel_intrinsics,
get_view_frame_from_calib_frame(0, 0, 0, 0))
medmodel_frame_from_bigmodel_frame = np.dot(medmodel_intrinsics, np.linalg.inv(bigmodel_intrinsics))
calib_from_medmodel = np.linalg.inv(medmodel_frame_from_calib_frame[:, :3])
calib_from_sbigmodel = np.linalg.inv(sbigmodel_frame_from_calib_frame[:, :3])
# This function is verified to give similar results to xx.uncommon.utils.transform_img
def get_warp_matrix(device_from_calib_euler: np.ndarray, wide_camera: bool = False, bigmodel_frame: bool = False, tici: bool = True) -> np.ndarray:
if tici and wide_camera:
cam_intrinsics = tici_ecam_intrinsics
elif tici:
cam_intrinsics = tici_fcam_intrinsics
else:
cam_intrinsics = eon_fcam_intrinsics
calib_from_model = calib_from_sbigmodel if bigmodel_frame else calib_from_medmodel
device_from_calib = rot_from_euler(device_from_calib_euler)
camera_from_calib = cam_intrinsics @ view_frame_from_device_frame @ device_from_calib
warp_matrix: np.ndarray = camera_from_calib @ calib_from_model
return warp_matrix

144
common/transformations/orientation.cc Normal file
View File

@@ -0,0 +1,144 @@
#define _USE_MATH_DEFINES
#include <iostream>
#include <cmath>
#include <eigen3/Eigen/Dense>
#include "common/transformations/orientation.hpp"
#include "common/transformations/coordinates.hpp"
Eigen::Quaterniond ensure_unique(Eigen::Quaterniond quat){
if (quat.w() > 0){
return quat;
} else {
return Eigen::Quaterniond(-quat.w(), -quat.x(), -quat.y(), -quat.z());
}
}
Eigen::Quaterniond euler2quat(Eigen::Vector3d euler){
Eigen::Quaterniond q;
q = Eigen::AngleAxisd(euler(2), Eigen::Vector3d::UnitZ())
* Eigen::AngleAxisd(euler(1), Eigen::Vector3d::UnitY())
* Eigen::AngleAxisd(euler(0), Eigen::Vector3d::UnitX());
return ensure_unique(q);
}
Eigen::Vector3d quat2euler(Eigen::Quaterniond quat){
// TODO: switch to eigen implementation if the range of the Euler angles doesn't matter anymore
// Eigen::Vector3d euler = quat.toRotationMatrix().eulerAngles(2, 1, 0);
// return {euler(2), euler(1), euler(0)};
double gamma = atan2(2 * (quat.w() * quat.x() + quat.y() * quat.z()), 1 - 2 * (quat.x()*quat.x() + quat.y()*quat.y()));
double asin_arg_clipped = std::clamp(2 * (quat.w() * quat.y() - quat.z() * quat.x()), -1.0, 1.0);
double theta = asin(asin_arg_clipped);
double psi = atan2(2 * (quat.w() * quat.z() + quat.x() * quat.y()), 1 - 2 * (quat.y()*quat.y() + quat.z()*quat.z()));
return {gamma, theta, psi};
}
Eigen::Matrix3d quat2rot(Eigen::Quaterniond quat){
return quat.toRotationMatrix();
}
Eigen::Quaterniond rot2quat(const Eigen::Matrix3d &rot){
return ensure_unique(Eigen::Quaterniond(rot));
}
Eigen::Matrix3d euler2rot(Eigen::Vector3d euler){
return quat2rot(euler2quat(euler));
}
Eigen::Vector3d rot2euler(const Eigen::Matrix3d &rot){
return quat2euler(rot2quat(rot));
}
Eigen::Matrix3d rot_matrix(double roll, double pitch, double yaw){
return euler2rot({roll, pitch, yaw});
}
Eigen::Matrix3d rot(Eigen::Vector3d axis, double angle){
Eigen::Quaterniond q;
q = Eigen::AngleAxisd(angle, axis);
return q.toRotationMatrix();
}
Eigen::Vector3d ecef_euler_from_ned(ECEF ecef_init, Eigen::Vector3d ned_pose) {
/*
Using Rotations to Build Aerospace Coordinate Systems
Don Koks
https://apps.dtic.mil/dtic/tr/fulltext/u2/a484864.pdf
*/
LocalCoord converter = LocalCoord(ecef_init);
Eigen::Vector3d zero = ecef_init.to_vector();
Eigen::Vector3d x0 = converter.ned2ecef({1, 0, 0}).to_vector() - zero;
Eigen::Vector3d y0 = converter.ned2ecef({0, 1, 0}).to_vector() - zero;
Eigen::Vector3d z0 = converter.ned2ecef({0, 0, 1}).to_vector() - zero;
Eigen::Vector3d x1 = rot(z0, ned_pose(2)) * x0;
Eigen::Vector3d y1 = rot(z0, ned_pose(2)) * y0;
Eigen::Vector3d z1 = rot(z0, ned_pose(2)) * z0;
Eigen::Vector3d x2 = rot(y1, ned_pose(1)) * x1;
Eigen::Vector3d y2 = rot(y1, ned_pose(1)) * y1;
Eigen::Vector3d z2 = rot(y1, ned_pose(1)) * z1;
Eigen::Vector3d x3 = rot(x2, ned_pose(0)) * x2;
Eigen::Vector3d y3 = rot(x2, ned_pose(0)) * y2;
x0 = Eigen::Vector3d(1, 0, 0);
y0 = Eigen::Vector3d(0, 1, 0);
z0 = Eigen::Vector3d(0, 0, 1);
double psi = atan2(x3.dot(y0), x3.dot(x0));
double theta = atan2(-x3.dot(z0), sqrt(pow(x3.dot(x0), 2) + pow(x3.dot(y0), 2)));
y2 = rot(z0, psi) * y0;
z2 = rot(y2, theta) * z0;
double phi = atan2(y3.dot(z2), y3.dot(y2));
return {phi, theta, psi};
}
Eigen::Vector3d ned_euler_from_ecef(ECEF ecef_init, Eigen::Vector3d ecef_pose){
/*
Using Rotations to Build Aerospace Coordinate Systems
Don Koks
https://apps.dtic.mil/dtic/tr/fulltext/u2/a484864.pdf
*/
LocalCoord converter = LocalCoord(ecef_init);
Eigen::Vector3d x0 = Eigen::Vector3d(1, 0, 0);
Eigen::Vector3d y0 = Eigen::Vector3d(0, 1, 0);
Eigen::Vector3d z0 = Eigen::Vector3d(0, 0, 1);
Eigen::Vector3d x1 = rot(z0, ecef_pose(2)) * x0;
Eigen::Vector3d y1 = rot(z0, ecef_pose(2)) * y0;
Eigen::Vector3d z1 = rot(z0, ecef_pose(2)) * z0;
Eigen::Vector3d x2 = rot(y1, ecef_pose(1)) * x1;
Eigen::Vector3d y2 = rot(y1, ecef_pose(1)) * y1;
Eigen::Vector3d z2 = rot(y1, ecef_pose(1)) * z1;
Eigen::Vector3d x3 = rot(x2, ecef_pose(0)) * x2;
Eigen::Vector3d y3 = rot(x2, ecef_pose(0)) * y2;
Eigen::Vector3d zero = ecef_init.to_vector();
x0 = converter.ned2ecef({1, 0, 0}).to_vector() - zero;
y0 = converter.ned2ecef({0, 1, 0}).to_vector() - zero;
z0 = converter.ned2ecef({0, 0, 1}).to_vector() - zero;
double psi = atan2(x3.dot(y0), x3.dot(x0));
double theta = atan2(-x3.dot(z0), sqrt(pow(x3.dot(x0), 2) + pow(x3.dot(y0), 2)));
y2 = rot(z0, psi) * y0;
z2 = rot(y2, theta) * z0;
double phi = atan2(y3.dot(z2), y3.dot(y2));
return {phi, theta, psi};
}

17
common/transformations/orientation.hpp Normal file
View File

@@ -0,0 +1,17 @@
#pragma once
#include <eigen3/Eigen/Dense>
#include "common/transformations/coordinates.hpp"
Eigen::Quaterniond ensure_unique(Eigen::Quaterniond quat);
Eigen::Quaterniond euler2quat(Eigen::Vector3d euler);
Eigen::Vector3d quat2euler(Eigen::Quaterniond quat);
Eigen::Matrix3d quat2rot(Eigen::Quaterniond quat);
Eigen::Quaterniond rot2quat(const Eigen::Matrix3d &rot);
Eigen::Matrix3d euler2rot(Eigen::Vector3d euler);
Eigen::Vector3d rot2euler(const Eigen::Matrix3d &rot);
Eigen::Matrix3d rot_matrix(double roll, double pitch, double yaw);
Eigen::Matrix3d rot(Eigen::Vector3d axis, double angle);
Eigen::Vector3d ecef_euler_from_ned(ECEF ecef_init, Eigen::Vector3d ned_pose);
Eigen::Vector3d ned_euler_from_ecef(ECEF ecef_init, Eigen::Vector3d ecef_pose);

52
common/transformations/orientation.py Normal file
View File

@@ -0,0 +1,52 @@
import numpy as np
from typing import Callable
from openpilot.common.transformations.transformations import (ecef_euler_from_ned_single,
euler2quat_single,
euler2rot_single,
ned_euler_from_ecef_single,
quat2euler_single,
quat2rot_single,
rot2euler_single,
rot2quat_single)
def numpy_wrap(function, input_shape, output_shape) -> Callable[..., np.ndarray]:
"""Wrap a function to take either an input or list of inputs and return the correct shape"""
def f(*inps):
*args, inp = inps
inp = np.array(inp)
shape = inp.shape
if len(shape) == len(input_shape):
out_shape = output_shape
else:
out_shape = (shape[0],) + output_shape
# Add empty dimension if inputs is not a list
if len(shape) == len(input_shape):
inp.shape = (1, ) + inp.shape
result = np.asarray([function(*args, i) for i in inp])
result.shape = out_shape
return result
return f
euler2quat = numpy_wrap(euler2quat_single, (3,), (4,))
quat2euler = numpy_wrap(quat2euler_single, (4,), (3,))
quat2rot = numpy_wrap(quat2rot_single, (4,), (3, 3))
rot2quat = numpy_wrap(rot2quat_single, (3, 3), (4,))
euler2rot = numpy_wrap(euler2rot_single, (3,), (3, 3))
rot2euler = numpy_wrap(rot2euler_single, (3, 3), (3,))
ecef_euler_from_ned = numpy_wrap(ecef_euler_from_ned_single, (3,), (3,))
ned_euler_from_ecef = numpy_wrap(ned_euler_from_ecef_single, (3,), (3,))
quats_from_rotations = rot2quat
quat_from_rot = rot2quat
rotations_from_quats = quat2rot
rot_from_quat = quat2rot
euler_from_rot = rot2euler
euler_from_quat = quat2euler
rot_from_euler = euler2rot
quat_from_euler = euler2quat

72
common/transformations/transformations.pxd Normal file
View File

@@ -0,0 +1,72 @@
# cython: language_level=3
from libcpp cimport bool
cdef extern from "orientation.cc":
pass
cdef extern from "orientation.hpp":
cdef cppclass Quaternion "Eigen::Quaterniond":
Quaternion()
Quaternion(double, double, double, double)
double w()
double x()
double y()
double z()
cdef cppclass Vector3 "Eigen::Vector3d":
Vector3()
Vector3(double, double, double)
double operator()(int)
cdef cppclass Matrix3 "Eigen::Matrix3d":
Matrix3()
Matrix3(double*)
double operator()(int, int)
Quaternion euler2quat(Vector3)
Vector3 quat2euler(Quaternion)
Matrix3 quat2rot(Quaternion)
Quaternion rot2quat(Matrix3)
Vector3 rot2euler(Matrix3)
Matrix3 euler2rot(Vector3)
Matrix3 rot_matrix(double, double, double)
Vector3 ecef_euler_from_ned(ECEF, Vector3)
Vector3 ned_euler_from_ecef(ECEF, Vector3)
cdef extern from "coordinates.cc":
cdef struct ECEF:
double x
double y
double z
cdef struct NED:
double n
double e
double d
cdef struct Geodetic:
double lat
double lon
double alt
bool radians
ECEF geodetic2ecef(Geodetic)
Geodetic ecef2geodetic(ECEF)
cdef cppclass LocalCoord_c "LocalCoord":
Matrix3 ned2ecef_matrix
Matrix3 ecef2ned_matrix
LocalCoord_c(Geodetic, ECEF)
LocalCoord_c(Geodetic)
LocalCoord_c(ECEF)
NED ecef2ned(ECEF)
ECEF ned2ecef(NED)
NED geodetic2ned(Geodetic)
Geodetic ned2geodetic(NED)
cdef extern from "coordinates.hpp":
pass

173
common/transformations/transformations.pyx Normal file
View File

@@ -0,0 +1,173 @@
# distutils: language = c++
# cython: language_level = 3
from openpilot.common.transformations.transformations cimport Matrix3, Vector3, Quaternion
from openpilot.common.transformations.transformations cimport ECEF, NED, Geodetic
from openpilot.common.transformations.transformations cimport euler2quat as euler2quat_c
from openpilot.common.transformations.transformations cimport quat2euler as quat2euler_c
from openpilot.common.transformations.transformations cimport quat2rot as quat2rot_c
from openpilot.common.transformations.transformations cimport rot2quat as rot2quat_c
from openpilot.common.transformations.transformations cimport euler2rot as euler2rot_c
from openpilot.common.transformations.transformations cimport rot2euler as rot2euler_c
from openpilot.common.transformations.transformations cimport rot_matrix as rot_matrix_c
from openpilot.common.transformations.transformations cimport ecef_euler_from_ned as ecef_euler_from_ned_c
from openpilot.common.transformations.transformations cimport ned_euler_from_ecef as ned_euler_from_ecef_c
from openpilot.common.transformations.transformations cimport geodetic2ecef as geodetic2ecef_c
from openpilot.common.transformations.transformations cimport ecef2geodetic as ecef2geodetic_c
from openpilot.common.transformations.transformations cimport LocalCoord_c
import numpy as np
cimport numpy as np
cdef np.ndarray[double, ndim=2] matrix2numpy(Matrix3 m):
return np.array([
[m(0, 0), m(0, 1), m(0, 2)],
[m(1, 0), m(1, 1), m(1, 2)],
[m(2, 0), m(2, 1), m(2, 2)],
])
cdef Matrix3 numpy2matrix(np.ndarray[double, ndim=2, mode="fortran"] m):
assert m.shape[0] == 3
assert m.shape[1] == 3
return Matrix3(<double*>m.data)
cdef ECEF list2ecef(ecef):
cdef ECEF e
e.x = ecef[0]
e.y = ecef[1]
e.z = ecef[2]
return e
cdef NED list2ned(ned):
cdef NED n
n.n = ned[0]
n.e = ned[1]
n.d = ned[2]
return n
cdef Geodetic list2geodetic(geodetic):
cdef Geodetic g
g.lat = geodetic[0]
g.lon = geodetic[1]
g.alt = geodetic[2]
return g
def euler2quat_single(euler):
cdef Vector3 e = Vector3(euler[0], euler[1], euler[2])
cdef Quaternion q = euler2quat_c(e)
return [q.w(), q.x(), q.y(), q.z()]
def quat2euler_single(quat):
cdef Quaternion q = Quaternion(quat[0], quat[1], quat[2], quat[3])
cdef Vector3 e = quat2euler_c(q)
return [e(0), e(1), e(2)]
def quat2rot_single(quat):
cdef Quaternion q = Quaternion(quat[0], quat[1], quat[2], quat[3])
cdef Matrix3 r = quat2rot_c(q)
return matrix2numpy(r)
def rot2quat_single(rot):
cdef Matrix3 r = numpy2matrix(np.asfortranarray(rot, dtype=np.double))
cdef Quaternion q = rot2quat_c(r)
return [q.w(), q.x(), q.y(), q.z()]
def euler2rot_single(euler):
cdef Vector3 e = Vector3(euler[0], euler[1], euler[2])
cdef Matrix3 r = euler2rot_c(e)
return matrix2numpy(r)
def rot2euler_single(rot):
cdef Matrix3 r = numpy2matrix(np.asfortranarray(rot, dtype=np.double))
cdef Vector3 e = rot2euler_c(r)
return [e(0), e(1), e(2)]
def rot_matrix(roll, pitch, yaw):
return matrix2numpy(rot_matrix_c(roll, pitch, yaw))
def ecef_euler_from_ned_single(ecef_init, ned_pose):
cdef ECEF init = list2ecef(ecef_init)
cdef Vector3 pose = Vector3(ned_pose[0], ned_pose[1], ned_pose[2])
cdef Vector3 e = ecef_euler_from_ned_c(init, pose)
return [e(0), e(1), e(2)]
def ned_euler_from_ecef_single(ecef_init, ecef_pose):
cdef ECEF init = list2ecef(ecef_init)
cdef Vector3 pose = Vector3(ecef_pose[0], ecef_pose[1], ecef_pose[2])
cdef Vector3 e = ned_euler_from_ecef_c(init, pose)
return [e(0), e(1), e(2)]
def geodetic2ecef_single(geodetic):
cdef Geodetic g = list2geodetic(geodetic)
cdef ECEF e = geodetic2ecef_c(g)
return [e.x, e.y, e.z]
def ecef2geodetic_single(ecef):
cdef ECEF e = list2ecef(ecef)
cdef Geodetic g = ecef2geodetic_c(e)
return [g.lat, g.lon, g.alt]
cdef class LocalCoord:
cdef LocalCoord_c * lc
def __init__(self, geodetic=None, ecef=None):
assert (geodetic is not None) or (ecef is not None)
if geodetic is not None:
self.lc = new LocalCoord_c(list2geodetic(geodetic))
elif ecef is not None:
self.lc = new LocalCoord_c(list2ecef(ecef))
@property
def ned2ecef_matrix(self):
return matrix2numpy(self.lc.ned2ecef_matrix)
@property
def ecef2ned_matrix(self):
return matrix2numpy(self.lc.ecef2ned_matrix)
@property
def ned_from_ecef_matrix(self):
return self.ecef2ned_matrix
@property
def ecef_from_ned_matrix(self):
return self.ned2ecef_matrix
@classmethod
def from_geodetic(cls, geodetic):
return cls(geodetic=geodetic)
@classmethod
def from_ecef(cls, ecef):
return cls(ecef=ecef)
def ecef2ned_single(self, ecef):
assert self.lc
cdef ECEF e = list2ecef(ecef)
cdef NED n = self.lc.ecef2ned(e)
return [n.n, n.e, n.d]
def ned2ecef_single(self, ned):
assert self.lc
cdef NED n = list2ned(ned)
cdef ECEF e = self.lc.ned2ecef(n)
return [e.x, e.y, e.z]
def geodetic2ned_single(self, geodetic):
assert self.lc
cdef Geodetic g = list2geodetic(geodetic)
cdef NED n = self.lc.geodetic2ned(g)
return [n.n, n.e, n.d]
def ned2geodetic_single(self, ned):
assert self.lc
cdef NED n = list2ned(ned)
cdef Geodetic g = self.lc.ned2geodetic(n)
return [g.lat, g.lon, g.alt]
def __dealloc__(self):
del self.lc

295
common/util.cc Normal file
View File

@@ -0,0 +1,295 @@
#include "common/util.h"
#include <sys/ioctl.h>
#include <sys/stat.h>
#include <sys/resource.h>
#include <cassert>
#include <cerrno>
#include <cstring>
#include <dirent.h>
#include <fstream>
#include <iomanip>
#include <random>
#include <sstream>
#ifdef __linux__
#include <sys/prctl.h>
#include <sys/syscall.h>
#ifndef __USE_GNU
#define __USE_GNU
#endif
#include <sched.h>
#endif // __linux__
namespace util {
void set_thread_name(const char* name) {
#ifdef __linux__
// pthread_setname_np is dumb (fails instead of truncates)
prctl(PR_SET_NAME, (unsigned long)name, 0, 0, 0);
#endif
}
int set_realtime_priority(int level) {
#ifdef __linux__
long tid = syscall(SYS_gettid);
// should match python using chrt
struct sched_param sa;
memset(&sa, 0, sizeof(sa));
sa.sched_priority = level;
return sched_setscheduler(tid, SCHED_FIFO, &sa);
#else
return -1;
#endif
}
int set_core_affinity(std::vector<int> cores) {
#ifdef __linux__
long tid = syscall(SYS_gettid);
cpu_set_t cpu;
CPU_ZERO(&cpu);
for (const int n : cores) {
CPU_SET(n, &cpu);
}
return sched_setaffinity(tid, sizeof(cpu), &cpu);
#else
return -1;
#endif
}
int set_file_descriptor_limit(uint64_t limit_val) {
struct rlimit limit;
int status;
if ((status = getrlimit(RLIMIT_NOFILE, &limit)) < 0)
return status;
limit.rlim_cur = limit_val;
if ((status = setrlimit(RLIMIT_NOFILE, &limit)) < 0)
return status;
return 0;
}
std::string read_file(const std::string& fn) {
std::ifstream f(fn, std::ios::binary | std::ios::in);
if (f.is_open()) {
f.seekg(0, std::ios::end);
int size = f.tellg();
if (f.good() && size > 0) {
std::string result(size, '\0');
f.seekg(0, std::ios::beg);
f.read(result.data(), size);
// return either good() or has reached end-of-file (e.g. /sys/power/wakeup_count)
if (f.good() || f.eof()) {
result.resize(f.gcount());
return result;
}
}
// fallback for files created on read, e.g. procfs
std::stringstream buffer;
buffer << f.rdbuf();
return buffer.str();
}
return std::string();
}
std::map<std::string, std::string> read_files_in_dir(const std::string &path) {
std::map<std::string, std::string> ret;
DIR *d = opendir(path.c_str());
if (!d) return ret;
struct dirent *de = NULL;
while ((de = readdir(d))) {
if (de->d_type != DT_DIR) {
ret[de->d_name] = util::read_file(path + "/" + de->d_name);
}
}
closedir(d);
return ret;
}
int write_file(const char* path, const void* data, size_t size, int flags, mode_t mode) {
int fd = HANDLE_EINTR(open(path, flags, mode));
if (fd == -1) {
return -1;
}
ssize_t n = HANDLE_EINTR(write(fd, data, size));
close(fd);
return (n >= 0 && (size_t)n == size) ? 0 : -1;
}
FILE* safe_fopen(const char* filename, const char* mode) {
FILE* fp = NULL;
do {
fp = fopen(filename, mode);
} while ((nullptr == fp) && (errno == EINTR));
return fp;
}
size_t safe_fwrite(const void* ptr, size_t size, size_t count, FILE* stream) {
size_t written = 0;
do {
size_t ret = ::fwrite((void*)((char *)ptr + written * size), size, count - written, stream);
if (ret == 0 && errno != EINTR) break;
written += ret;
} while (written != count);
return written;
}
int safe_fflush(FILE *stream) {
int ret = EOF;
do {
ret = fflush(stream);
} while ((EOF == ret) && (errno == EINTR));
return ret;
}
int safe_ioctl(int fd, unsigned long request, void *argp) {
int ret;
do {
ret = ioctl(fd, request, argp);
} while ((ret == -1) && (errno == EINTR));
return ret;
}
std::string readlink(const std::string &path) {
char buff[4096];
ssize_t len = ::readlink(path.c_str(), buff, sizeof(buff)-1);
if (len != -1) {
buff[len] = '\0';
return std::string(buff);
}
return "";
}
bool file_exists(const std::string& fn) {
struct stat st = {};
return stat(fn.c_str(), &st) != -1;
}
static bool createDirectory(std::string dir, mode_t mode) {
auto verify_dir = [](const std::string& dir) -> bool {
struct stat st = {};
return (stat(dir.c_str(), &st) == 0 && (st.st_mode & S_IFMT) == S_IFDIR);
};
// remove trailing /'s
while (dir.size() > 1 && dir.back() == '/') {
dir.pop_back();
}
// try to mkdir this directory
if (mkdir(dir.c_str(), mode) == 0) return true;
if (errno == EEXIST) return verify_dir(dir);
if (errno != ENOENT) return false;
// mkdir failed because the parent dir doesn't exist, so try to create it
size_t slash = dir.rfind('/');
if ((slash == std::string::npos || slash < 1) ||
!createDirectory(dir.substr(0, slash), mode)) {
return false;
}
// try again
if (mkdir(dir.c_str(), mode) == 0) return true;
return errno == EEXIST && verify_dir(dir);
}
bool create_directories(const std::string& dir, mode_t mode) {
if (dir.empty()) return false;
return createDirectory(dir, mode);
}
std::string getenv(const char* key, std::string default_val) {
const char* val = ::getenv(key);
return val ? val : default_val;
}
int getenv(const char* key, int default_val) {
const char* val = ::getenv(key);
return val ? atoi(val) : default_val;
}
float getenv(const char* key, float default_val) {
const char* val = ::getenv(key);
return val ? atof(val) : default_val;
}
std::string hexdump(const uint8_t* in, const size_t size) {
std::stringstream ss;
ss << std::hex << std::setfill('0');
for (size_t i = 0; i < size; i++) {
ss << std::setw(2) << static_cast<unsigned int>(in[i]);
}
return ss.str();
}
int random_int(int min, int max) {
std::random_device dev;
std::mt19937 rng(dev());
std::uniform_int_distribution<std::mt19937::result_type> dist(min, max);
return dist(rng);
}
std::string random_string(std::string::size_type length) {
const std::string chrs = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
std::mt19937 rg{std::random_device{}()};
std::uniform_int_distribution<std::string::size_type> pick(0, chrs.length() - 1);
std::string s;
s.reserve(length);
while (length--) {
s += chrs[pick(rg)];
}
return s;
}
std::string dir_name(std::string const &path) {
size_t pos = path.find_last_of("/");
if (pos == std::string::npos) return "";
return path.substr(0, pos);
}
bool starts_with(const std::string &s1, const std::string &s2) {
return strncmp(s1.c_str(), s2.c_str(), s2.size()) == 0;
}
bool ends_with(const std::string &s1, const std::string &s2) {
return strcmp(s1.c_str() + (s1.size() - s2.size()), s2.c_str()) == 0;
}
std::string check_output(const std::string& command) {
char buffer[128];
std::string result;
std::unique_ptr<FILE, decltype(&pclose)> pipe(popen(command.c_str(), "r"), pclose);
if (!pipe) {
return "";
}
while (fgets(buffer, std::size(buffer), pipe.get()) != nullptr) {
result += std::string(buffer);
}
return result;
}
struct tm get_time() {
time_t rawtime;
time(&rawtime);
struct tm sys_time;
gmtime_r(&rawtime, &sys_time);
return sys_time;
}
bool time_valid(struct tm sys_time) {
int year = 1900 + sys_time.tm_year;
int month = 1 + sys_time.tm_mon;
return (year > 2023) || (year == 2023 && month >= 6);
}
} // namespace util

181
common/util.h Normal file
View File

@@ -0,0 +1,181 @@
#pragma once
#include <fcntl.h>
#include <sys/stat.h>
#include <unistd.h>
#include <algorithm>
#include <atomic>
#include <chrono>
#include <csignal>
#include <ctime>
#include <map>
#include <memory>
#include <mutex>
#include <string>
#include <thread>
#include <vector>
// keep trying if x gets interrupted by a signal
#define HANDLE_EINTR(x) \
({ \
decltype(x) ret_; \
int try_cnt = 0; \
do { \
ret_ = (x); \
} while (ret_ == -1 && errno == EINTR && try_cnt++ < 100); \
ret_; \
})
#ifndef sighandler_t
typedef void (*sighandler_t)(int sig);
#endif
const double MILE_TO_KM = 1.609344;
const double KM_TO_MILE = 1. / MILE_TO_KM;
const double MS_TO_KPH = 3.6;
const double MS_TO_MPH = MS_TO_KPH * KM_TO_MILE;
const double METER_TO_MILE = KM_TO_MILE / 1000.0;
const double METER_TO_FOOT = 3.28084;
namespace util {
void set_thread_name(const char* name);
int set_realtime_priority(int level);
int set_core_affinity(std::vector<int> cores);
int set_file_descriptor_limit(uint64_t limit);
// ***** Time helpers *****
struct tm get_time();
bool time_valid(struct tm sys_time);
// ***** math helpers *****
// map x from [a1, a2] to [b1, b2]
template <typename T>
T map_val(T x, T a1, T a2, T b1, T b2) {
x = std::clamp(x, a1, a2);
T ra = a2 - a1;
T rb = b2 - b1;
return (x - a1) * rb / ra + b1;
}
// ***** string helpers *****
template <typename... Args>
std::string string_format(const std::string& format, Args... args) {
size_t size = snprintf(nullptr, 0, format.c_str(), args...) + 1;
std::unique_ptr<char[]> buf(new char[size]);
snprintf(buf.get(), size, format.c_str(), args...);
return std::string(buf.get(), buf.get() + size - 1);
}
std::string getenv(const char* key, std::string default_val = "");
int getenv(const char* key, int default_val);
float getenv(const char* key, float default_val);
std::string hexdump(const uint8_t* in, const size_t size);
std::string dir_name(std::string const& path);
bool starts_with(const std::string &s1, const std::string &s2);
bool ends_with(const std::string &s1, const std::string &s2);
// ***** random helpers *****
int random_int(int min, int max);
std::string random_string(std::string::size_type length);
// **** file helpers *****
std::string read_file(const std::string& fn);
std::map<std::string, std::string> read_files_in_dir(const std::string& path);
int write_file(const char* path, const void* data, size_t size, int flags = O_WRONLY, mode_t mode = 0664);
FILE* safe_fopen(const char* filename, const char* mode);
size_t safe_fwrite(const void * ptr, size_t size, size_t count, FILE * stream);
int safe_fflush(FILE *stream);
int safe_ioctl(int fd, unsigned long request, void *argp);
std::string readlink(const std::string& path);
bool file_exists(const std::string& fn);
bool create_directories(const std::string &dir, mode_t mode);
std::string check_output(const std::string& command);
inline void sleep_for(const int milliseconds) {
if (milliseconds > 0) {
std::this_thread::sleep_for(std::chrono::milliseconds(milliseconds));
}
}
} // namespace util
class ExitHandler {
public:
ExitHandler() {
std::signal(SIGINT, (sighandler_t)set_do_exit);
std::signal(SIGTERM, (sighandler_t)set_do_exit);
#ifndef __APPLE__
std::signal(SIGPWR, (sighandler_t)set_do_exit);
#endif
}
inline static std::atomic<bool> power_failure = false;
inline static std::atomic<int> signal = 0;
inline operator bool() { return do_exit; }
inline ExitHandler& operator=(bool v) {
signal = 0;
do_exit = v;
return *this;
}
private:
static void set_do_exit(int sig) {
#ifndef __APPLE__
power_failure = (sig == SIGPWR);
#endif
signal = sig;
do_exit = true;
}
inline static std::atomic<bool> do_exit = false;
};
struct unique_fd {
unique_fd(int fd = -1) : fd_(fd) {}
unique_fd& operator=(unique_fd&& uf) {
fd_ = uf.fd_;
uf.fd_ = -1;
return *this;
}
~unique_fd() {
if (fd_ != -1) close(fd_);
}
operator int() const { return fd_; }
int fd_;
};
class FirstOrderFilter {
public:
FirstOrderFilter(float x0, float ts, float dt, bool initialized = true) {
k_ = (dt / ts) / (1.0 + dt / ts);
x_ = x0;
initialized_ = initialized;
}
inline float update(float x) {
if (initialized_) {
x_ = (1. - k_) * x_ + k_ * x;
} else {
initialized_ = true;
x_ = x;
}
return x_;
}
inline void reset(float x) { x_ = x; }
inline float x(){ return x_; }
private:
float x_, k_;
bool initialized_;
};
template<typename T>
void update_max_atomic(std::atomic<T>& max, T const& value) {
T prev = max;
while (prev < value && !max.compare_exchange_weak(prev, value)) {}
}

1
common/version.h Normal file
View File

@@ -0,0 +1 @@
#define COMMA_VERSION "0.9.6"

11
common/watchdog.cc Normal file
View File

@@ -0,0 +1,11 @@
#include <string>
#include "common/watchdog.h"
#include "common/util.h"
const std::string watchdog_fn_prefix = "/dev/shm/wd_"; // + <pid>
bool watchdog_kick(uint64_t ts) {
static std::string fn = watchdog_fn_prefix + std::to_string(getpid());
return util::write_file(fn.c_str(), &ts, sizeof(ts), O_WRONLY | O_CREAT) > 0;
}

5
common/watchdog.h Normal file
View File

@@ -0,0 +1,5 @@
#pragma once
#include <cstdint>
bool watchdog_kick(uint64_t ts);