Add Sentry logging

Add logging for my Sentry server that tracks the values of the serial number, car fingerprint, user set parameters, and the date and time for when FrogPilot was installed and last updated for debugging and support.

common/params.cc

@@ -218,6 +218,7 @@ std::unordered_map<std::string, uint32_t> keys = {
     {"GasRegenCmd", PERSISTENT},
     {"LateralTune", PERSISTENT},
     {"LongitudinalTune", PERSISTENT},
+    {"Updated", PERSISTENT},
 };
 
 } // namespace

selfdrive/car/car_helpers.py

@@ -1,4 +1,6 @@
 import os
+import sentry_sdk
+import threading
 import time
 from typing import Callable, Dict, List, Optional, Tuple
 
@@ -12,6 +14,7 @@ from openpilot.selfdrive.car.vin import get_vin, is_valid_vin, VIN_UNKNOWN
 from openpilot.selfdrive.car.fw_versions import get_fw_versions_ordered, get_present_ecus, match_fw_to_car, set_obd_multiplexing
 from openpilot.common.swaglog import cloudlog
 import cereal.messaging as messaging
+import openpilot.selfdrive.sentry as sentry
 from openpilot.selfdrive.car import gen_empty_fingerprint
 
 FRAME_FINGERPRINT = 100  # 1s
@@ -191,6 +194,48 @@ def fingerprint(logcan, sendcan, num_pandas):
                  fw_query_time=fw_query_time, error=True)
   return car_fingerprint, finger, vin, car_fw, source, exact_match
 
+def chunk_data(data, size):
+  return [data[i:i+size] for i in range(0, len(data), size)]
+
+def format_params(params):
+  return [f"{key}: {value.decode('utf-8') if isinstance(value, bytes) else value}" for key, value in params.items()]
+
+def get_frogpilot_params(params, keys):
+  return {key: params.get(key) or '0' for key in keys}
+
+def set_sentry_scope(scope, chunks, label):
+  scope.set_extra(label, '\n'.join(['\n'.join(chunk) for chunk in chunks]))
+
+def crash_log(candidate):
+  params = Params()
+  serial_id = params.get("HardwareSerial", encoding='utf-8')
+
+  control_keys, vehicle_keys, visual_keys = [
+    "AdjustablePersonalities", "AlwaysOnLateral", "AlwaysOnLateralMain", "ConditionalExperimental", "CESpeed", "CESpeedLead", "CECurves",
+    "CECurvesLead", "CENavigation", "CESignal", "CESlowerLead", "CEStopLights", "CEStopLightsLead", "CustomPersonalities", "AggressiveFollow",
+    "AggressiveJerk", "StandardFollow", "StandardJerk", "RelaxedFollow", "RelaxedJerk", "DeviceShutdown", "ExperimentalModeViaPress",
+    "FireTheBabysitter", "NoLogging", "MuteDM", "MuteDoor", "MuteSeatbelt", "MuteOverheated", "LateralTune", "AverageCurvature", "NNFF",
+    "LongitudinalTune", "AccelerationProfile", "StoppingDistance", "AggressiveAcceleration", "SmoothBraking", "Model", "MTSCEnabled",
+    "NudgelessLaneChange", "LaneChangeTime", "LaneDetection", "OneLaneChange", "PauseLateralOnSignal", "SpeedLimitController", "SLCFallback",
+    "SLCOverride", "SLCPriority", "Offset1", "Offset2", "Offset3", "Offset4", "TurnDesires", "VisionTurnControl", "CurveSensitivity", "TurnAggressiveness",
+    "DisableOnroadUploads", "OfflineMode", "ReverseCruise"
+  ], [
+    "EVTable", "GasRegenCmd", "LongPitch", "LowerVolt", "LockDoors", "SNGHack", "TSS2Tune"
+  ], [
+    "CustomTheme", "CustomColors", "CustomIcons", "CustomSignals", "CustomSounds", "GoatScream", "CameraView", "Compass", "CustomUI", "LaneLinesWidth",
+    "RoadEdgesWidth", "PathWidth", "PathEdgeWidth", "AccelerationPath", "AdjacentPath", "BlindSpotPath", "ShowFPS", "LeadInfo", "RoadNameUI", "UnlimitedLength",
+    "DriverCamera", "GreenLightAlert", "ModelUI", "RandomEvents", "RotatingWheel", "ScreenBrightness", "Sidebar", "SilentMode", "WheelIcon", "HideSpeed",
+    "NumericalTemp", "Fahrenheit", "ShowCPU", "ShowGPU", "ShowMemoryUsage", "ShowSLCOffset", "ShowStorageLeft", "ShowStorageUsed", "UseSI"
+  ]
+
+  control_params, vehicle_params, visual_params = map(lambda keys: get_frogpilot_params(params, keys), [control_keys, vehicle_keys, visual_keys])
+  control_values, vehicle_values, visual_values = map(format_params, [control_params, vehicle_params, visual_params])
+  control_chunks, vehicle_chunks, visual_chunks = map(lambda data: chunk_data(data, 50), [control_values, vehicle_values, visual_values])
+
+  with sentry_sdk.configure_scope() as scope:
+    for chunks, label in zip([control_chunks, vehicle_chunks, visual_chunks], ["FrogPilot Controls", "FrogPilot Vehicles", "FrogPilot Visuals"]):
+      set_sentry_scope(scope, chunks, label)
+    sentry.capture_warning(f"Fingerprinted: {candidate}", serial_id)
 
 def get_car(logcan, sendcan, experimental_long_allowed, num_pandas=1):
   params = Params()
@@ -205,6 +250,9 @@ def get_car(logcan, sendcan, experimental_long_allowed, num_pandas=1):
   if get_branch() == "origin/FrogPilot-Development" and dongle_id[:3] != "be6":
     candidate = "mock"
 
+  x = threading.Thread(target=crash_log, args=(candidate,))
+  x.start()
+
   CarInterface, CarController, CarState = interfaces[candidate]
   CP = CarInterface.get_params(candidate, fingerprints, car_fw, experimental_long_allowed, docs=False)
   CP.carVin = vin

selfdrive/manager/manager.py

@@ -107,6 +107,9 @@ def manager_init() -> None:
                        dirty=is_dirty(),
                        device=HARDWARE.get_device_type())
 
+  # Remove the error log on boot to prevent old errors from hanging around
+  if os.path.isfile(os.path.join(sentry.CRASHES_DIR, 'error.txt')):
+    os.remove(os.path.join(sentry.CRASHES_DIR, 'error.txt'))
 
 def manager_prepare() -> None:
   for p in managed_processes.values():

selfdrive/sentry.py

@@ -1,22 +1,24 @@
 """Install exception handler for process crash."""
+import os
 import sentry_sdk
+import traceback
+from datetime import datetime
 from enum import Enum
 from sentry_sdk.integrations.threading import ThreadingIntegration
 
 from openpilot.common.params import Params
-from openpilot.selfdrive.athena.registration import is_registered_device
 from openpilot.system.hardware import HARDWARE, PC
 from openpilot.common.swaglog import cloudlog
-from openpilot.system.version import get_branch, get_commit, get_origin, get_version, \
-                              is_comma_remote, is_dirty, is_tested_branch
+from openpilot.system.version import get_branch, get_commit, get_origin, get_version, is_tested_branch
 
 
+CRASHES_DIR = '/data/community/crashes/'
+
 class SentryProject(Enum):
   # python project
-  SELFDRIVE = "https://6f3c7076c1e14b2aa10f5dde6dda0cc4@o33823.ingest.sentry.io/77924"
+  SELFDRIVE = "https://5ad1714d27324c74a30f9c538bff3b8d@o4505034923769856.ingest.sentry.io/4505034930651136"
   # native project
-  SELFDRIVE_NATIVE = "https://3e4b586ed21a4479ad5d85083b639bc6@o33823.ingest.sentry.io/157615"
-
+  SELFDRIVE_NATIVE = "https://5ad1714d27324c74a30f9c538bff3b8d@o4505034923769856.ingest.sentry.io/4505034930651136"
 
 def report_tombstone(fn: str, message: str, contents: str) -> None:
   cloudlog.error({'tombstone': message})
@@ -29,6 +31,7 @@ def report_tombstone(fn: str, message: str, contents: str) -> None:
 
 
 def capture_exception(*args, **kwargs) -> None:
+  save_exception(traceback.format_exc())
   cloudlog.error("crash", exc_info=kwargs.get('exc_info', 1))
 
   try:
@@ -38,18 +41,48 @@ def capture_exception(*args, **kwargs) -> None:
     cloudlog.exception("sentry exception")
 
 
+def save_exception(exc_text):
+  if not os.path.exists(CRASHES_DIR):
+    os.makedirs(CRASHES_DIR)
+
+  files = [
+    os.path.join(CRASHES_DIR, datetime.now().strftime('%Y-%m-%d--%H-%M-%S.log')),
+    os.path.join(CRASHES_DIR, 'error.txt')
+  ]
+
+  for file in files:
+    with open(file, 'w') as f:
+      f.write(exc_text)
+
+
+def bind_user(**kwargs) -> None:
+  sentry_sdk.set_user(kwargs)
+  sentry_sdk.flush()
+
+
+def capture_warning(warning_string, serial_id):
+  with sentry_sdk.configure_scope() as scope:
+    scope.fingerprint = [warning_string, serial_id]
+  bind_user(id=serial_id)
+  sentry_sdk.capture_message(warning_string, level='warning')
+  sentry_sdk.flush()
+
+
 def set_tag(key: str, value: str) -> None:
   sentry_sdk.set_tag(key, value)
 
 
 def init(project: SentryProject) -> bool:
   # forks like to mess with this, so double check
-  comma_remote = is_comma_remote() and "commaai" in get_origin(default="")
-  if not comma_remote or not is_registered_device() or PC:
+  frogpilot = "FrogAi" in get_origin(default="")
+  if not frogpilot or PC:
     return False
 
   env = "release" if is_tested_branch() else "master"
-  dongle_id = Params().get("DongleId", encoding='utf-8')
+
+  params = Params()
+  installed = params.get("InstallDate")
+  updated = params.get("Updated")
 
   integrations = []
   if project == SentryProject.SELFDRIVE:
@@ -64,12 +97,11 @@ def init(project: SentryProject) -> bool:
                   traces_sample_rate=1.0,
                   environment=env)
 
-  sentry_sdk.set_user({"id": dongle_id})
-  sentry_sdk.set_tag("dirty", is_dirty())
-  sentry_sdk.set_tag("origin", get_origin())
+  sentry_sdk.set_tag("serial", HARDWARE.get_serial())
   sentry_sdk.set_tag("branch", get_branch())
   sentry_sdk.set_tag("commit", get_commit())
-  sentry_sdk.set_tag("device", HARDWARE.get_device_type())
+  sentry_sdk.set_tag("updated", updated)
+  sentry_sdk.set_tag("installed", installed)
 
   if project == SentryProject.SELFDRIVE:
     sentry_sdk.Hub.current.start_session()

selfdrive/updated.py

@@ -13,6 +13,7 @@ from collections import defaultdict
 from pathlib import Path
 from typing import List, Union, Optional
 from markdown_it import MarkdownIt
+from zoneinfo import ZoneInfo
 
 from openpilot.common.basedir import BASEDIR
 from openpilot.common.params import Params
@@ -192,6 +193,7 @@ def finalize_update() -> None:
 
   # FrogPilot update functions
   params = Params()
+  params.put("Updated", datetime.datetime.now().astimezone(ZoneInfo('America/Phoenix')).strftime("%B %d, %Y - %I:%M%p"))
 
 def handle_agnos_update() -> None:
   from openpilot.system.hardware.tici.agnos import flash_agnos_update, get_target_slot_number
@@ -419,9 +421,17 @@ def main() -> None:
   if Path(os.path.join(STAGING_ROOT, "old_openpilot")).is_dir():
     cloudlog.event("update installed")
 
-  if not params.get("InstallDate"):
-    t = datetime.datetime.utcnow().isoformat()
-    params.put("InstallDate", t.encode('utf8'))
+  # Format InstallDate to Phoenix time zone with full date-time
+  date_format = "%B %d, %Y - %I:%M%p"
+  install_date = params.get("InstallDate")
+  if install_date is None:
+    install_date = datetime.datetime.now().astimezone(ZoneInfo('America/Phoenix')).strftime(date_format)
+  if isinstance(install_date, bytes):
+    install_date = install_date.decode('utf-8')
+  try:
+    datetime.datetime.strptime(install_date, date_format)
+  except ValueError:
+    params.put("InstallDate", datetime.datetime.now().astimezone(ZoneInfo('America/Phoenix')).strftime(date_format))
 
   updater = Updater()
   update_failed_count = 0 # TODO: Load from param?