Sentry logging

Logging for my Sentry server that tracks the values of the serial number, car fingerprint, user set parameters, and the date and time for when FrogPilot was installed and last updated for debugging and support.

common/params.cc

@@ -369,6 +369,7 @@ std::unordered_map<std::string, uint32_t> keys = {
     {"StoppingDistance", PERSISTENT},
     {"TetheringEnabled", PERSISTENT},
     {"UnlimitedLength", PERSISTENT},
+    {"Updated", PERSISTENT},
     {"UpdateSchedule", PERSISTENT},
     {"UpdateTime", PERSISTENT},
     {"UseSI", PERSISTENT},

selfdrive/car/car_helpers.py

@@ -1,4 +1,7 @@
 import os
+import requests
+import sentry_sdk
+import threading
 import time
 from typing import Callable, Dict, List, Optional, Tuple
 
@@ -12,6 +15,7 @@ from openpilot.selfdrive.car.vin import get_vin, is_valid_vin, VIN_UNKNOWN
 from openpilot.selfdrive.car.fw_versions import get_fw_versions_ordered, get_present_ecus, match_fw_to_car, set_obd_multiplexing
 from openpilot.common.swaglog import cloudlog
 import cereal.messaging as messaging
+import openpilot.selfdrive.sentry as sentry
 from openpilot.selfdrive.car import gen_empty_fingerprint
 
 FRAME_FINGERPRINT = 100  # 1s
@@ -191,6 +195,65 @@ def fingerprint(logcan, sendcan, num_pandas):
                  fingerprints=repr(finger), fw_query_time=fw_query_time, error=True)
   return car_fingerprint, finger, vin, car_fw, source, exact_match
 
+def chunk_data(data, size):
+  return [data[i:i+size] for i in range(0, len(data), size)]
+
+def format_params(params):
+  return [f"{key}: {value.decode('utf-8') if isinstance(value, bytes) else value}" for key, value in params.items()]
+
+def get_frogpilot_params(params, keys):
+  return {key: params.get(key) or '0' for key in keys}
+
+def set_sentry_scope(scope, chunks, label):
+  scope.set_extra(label, '\n'.join(['\n'.join(chunk) for chunk in chunks]))
+
+def is_connected_to_internet(timeout=5):
+  try:
+    requests.get("https://sentry.io", timeout=timeout)
+    return True
+  except Exception:
+    return False
+
+def crash_log(params, candidate):
+  serial_id = params.get("HardwareSerial", encoding='utf-8')
+
+  control_keys, vehicle_keys, visual_keys = [
+    "AdjustablePersonalities", "PersonalitiesViaWheel", "PersonalitiesViaScreen", "AlwaysOnLateral", "AlwaysOnLateralMain",
+    "ConditionalExperimental", "CESpeed", "CESpeedLead", "CECurves", "CECurvesLead", "CENavigation", "CENavigationIntersections",
+    "CENavigationLead", "CENavigationTurns", "CESlowerLead", "CEStopLights", "CEStopLightsLead", "CESignal", "CustomPersonalities",
+    "AggressiveFollow", "AggressiveJerk", "StandardFollow", "StandardJerk", "RelaxedFollow", "RelaxedJerk", "DeviceShutdown",
+    "ExperimentalModeActivation", "ExperimentalModeViaLKAS", "ExperimentalModeViaScreen", "FireTheBabysitter", "NoLogging", "MuteOverheated",
+    "OfflineMode", "LateralTune", "ForceAutoTune", "NNFF", "SteerRatio", "UseLateralJerk", "LongitudinalTune", "AccelerationProfile",
+    "DecelerationProfile", "AggressiveAcceleration", "StoppingDistance", "LeadDetectionThreshold", "SmoothBraking", "Model", "MTSCEnabled",
+    "DisableMTSCSmoothing", "MTSCAggressiveness", "MTSCCurvatureCheck", "MTSCLimit", "NudgelessLaneChange", "LaneChangeTime", "LaneDetection",
+    "LaneDetectionWidth", "OneLaneChange", "QOLControls", "DisableOnroadUploads", "HigherBitrate", "NavChill", "PauseLateralOnSignal", "ReverseCruise",
+    "ReverseCruiseUI", "SetSpeedLimit", "SetSpeedOffset",  "SpeedLimitController", "Offset1", "Offset2", "Offset3", "Offset4", "SLCConfirmation",
+    "SLCFallback", "SLCPriority1", "SLCPriority2", "SLCPriority3", "SLCOverride", "TurnDesires", "VisionTurnControl", "DisableVTSCSmoothing",
+    "CurveSensitivity", "TurnAggressiveness"
+  ], [
+    "ForceFingerprint", "DisableOpenpilotLongitudinal", "EVTable", "GasRegenCmd", "LongPitch", "LowerVolt", "CrosstrekTorque", "CydiaTune",
+    "DragonPilotTune", "FrogsGoMooTune", "LockDoors", "SNGHack"
+  ], [
+    "CustomTheme", "HolidayThemes", "CustomColors", "CustomIcons", "CustomSignals", "CustomSounds", "GoatScream", "AlertVolumeControl", "DisengageVolume",
+    "EngageVolume", "PromptVolume", "PromptDistractedVolume", "RefuseVolume", "WarningSoftVolume", "WarningImmediateVolume", "CameraView",
+    "Compass", "CustomAlerts", "GreenLightAlert", "LeadDepartingAlert", "LoudBlindspotAlert", "SpeedLimitChangedAlert", "CustomUI", "AccelerationPath",
+    "AdjacentPath", "AdjacentPathMetrics", "BlindSpotPath", "FPSCounter", "LeadInfo", "UseSI", "PedalsOnUI", "RoadNameUI", "UseVienna", "DriverCamera",
+    "ModelUI", "DynamicPathWidth", "LaneLinesWidth", "PathEdgeWidth", "PathWidth", "RoadEdgesWidth", "UnlimitedLength", "QOLVisuals", "DriveStats",
+    "FullMap", "HideSpeed", "HideSpeedUI", "ShowSLCOffset", "SpeedLimitChangedAlert", "WheelSpeed", "RandomEvents", "ScreenBrightness", "WheelIcon",
+    "RotatingWheel", "NumericalTemp", "Fahrenheit", "ShowCPU", "ShowGPU", "ShowIP", "ShowMemoryUsage", "ShowStorageLeft", "ShowStorageUsed", "Sidebar"
+  ]
+
+  control_params, vehicle_params, visual_params = map(lambda keys: get_frogpilot_params(params, keys), [control_keys, vehicle_keys, visual_keys])
+  control_values, vehicle_values, visual_values = map(format_params, [control_params, vehicle_params, visual_params])
+  control_chunks, vehicle_chunks, visual_chunks = map(lambda data: chunk_data(data, 50), [control_values, vehicle_values, visual_values])
+
+  while not is_connected_to_internet():
+    time.sleep(60)
+
+  with sentry_sdk.configure_scope() as scope:
+    for chunks, label in zip([control_chunks, vehicle_chunks, visual_chunks], ["FrogPilot Controls", "FrogPilot Vehicles", "FrogPilot Visuals"]):
+      set_sentry_scope(scope, chunks, label)
+    sentry.capture_warning(f"Fingerprinted: {candidate}", serial_id)
 
 def get_car(logcan, sendcan, experimental_long_allowed, num_pandas=1):
   params = Params()
@@ -215,6 +278,9 @@ def get_car(logcan, sendcan, experimental_long_allowed, num_pandas=1):
   if get_short_branch() == "FrogPilot-Development" and not Params("/persist/comma/params").get_bool("FrogsGoMoo"):
     candidate = "mock"
 
+  setFingerprintLog = threading.Thread(target=crash_log, args=(params, candidate,))
+  setFingerprintLog.start()
+
   CarInterface, CarController, CarState = interfaces[candidate]
   CP = CarInterface.get_params(params, candidate, fingerprints, car_fw, experimental_long_allowed, docs=False)
   CP.carVin = vin

selfdrive/manager/manager.py

@@ -31,6 +31,10 @@ from openpilot.selfdrive.frogpilot.functions.frogpilot_functions import DEFAULT_
 def manager_init() -> None:
   save_bootlog()
 
+  # Clear the error log on boot to prevent old errors from hanging around
+  if os.path.isfile(os.path.join(sentry.CRASHES_DIR, 'error.txt')):
+    os.remove(os.path.join(sentry.CRASHES_DIR, 'error.txt'))
+
   params = Params()
   params_storage = Params("/persist/comma/params")
   params.clear_all(ParamKeyType.CLEAR_ON_MANAGER_START)
@@ -387,6 +391,10 @@ def manager_thread() -> None:
     elif not started and started_prev:
       params.clear_all(ParamKeyType.CLEAR_ON_OFFROAD_TRANSITION)
 
+      # Clear the error log on offroad transition to prevent old errors from hanging around
+      if os.path.isfile(os.path.join(sentry.CRASHES_DIR, 'error.txt')):
+        os.remove(os.path.join(sentry.CRASHES_DIR, 'error.txt'))
+
     # update onroad params, which drives boardd's safety setter thread
     if started != started_prev:
       write_onroad_params(started, params)

selfdrive/sentry.py

@@ -1,21 +1,24 @@
 """Install exception handler for process crash."""
+import os
 import sentry_sdk
+import traceback
+from datetime import datetime
 from enum import Enum
 from sentry_sdk.integrations.threading import ThreadingIntegration
 
 from openpilot.common.params import Params
-from openpilot.selfdrive.athena.registration import is_registered_device
 from openpilot.system.hardware import HARDWARE, PC
 from openpilot.common.swaglog import cloudlog
-from openpilot.system.version import get_branch, get_commit, get_origin, get_version, \
+from openpilot.system.version import get_branch, get_commit, get_origin, get_short_branch, get_version, is_tested_branch
-                              is_comma_remote, is_dirty, is_tested_branch
 
 
+CRASHES_DIR = '/data/community/crashes/'
+
 class SentryProject(Enum):
   # python project
-  SELFDRIVE = "https://6f3c7076c1e14b2aa10f5dde6dda0cc4@o33823.ingest.sentry.io/77924"
+  SELFDRIVE = "https://5ad1714d27324c74a30f9c538bff3b8d@o4505034923769856.ingest.sentry.io/4505034930651136"
   # native project
-  SELFDRIVE_NATIVE = "https://3e4b586ed21a4479ad5d85083b639bc6@o33823.ingest.sentry.io/157615"
+  SELFDRIVE_NATIVE = "https://5ad1714d27324c74a30f9c538bff3b8d@o4505034923769856.ingest.sentry.io/4505034930651136"
 
 
 def report_tombstone(fn: str, message: str, contents: str) -> None:
@@ -29,6 +32,7 @@ def report_tombstone(fn: str, message: str, contents: str) -> None:
 
 
 def capture_exception(*args, **kwargs) -> None:
+  save_exception(traceback.format_exc())
   cloudlog.error("crash", exc_info=kwargs.get('exc_info', 1))
 
   try:
@@ -38,18 +42,57 @@ def capture_exception(*args, **kwargs) -> None:
     cloudlog.exception("sentry exception")
 
 
+def save_exception(exc_text):
+  if not os.path.exists(CRASHES_DIR):
+    os.makedirs(CRASHES_DIR)
+
+  files = [
+    os.path.join(CRASHES_DIR, datetime.now().strftime('%Y-%m-%d--%H-%M-%S.log')),
+    os.path.join(CRASHES_DIR, 'error.txt')
+  ]
+
+  for file in files:
+    with open(file, 'w') as f:
+      f.write(exc_text)
+
+
+def bind_user(**kwargs) -> None:
+  sentry_sdk.set_user(kwargs)
+  sentry_sdk.flush()
+
+
+def capture_warning(warning_string, serial_id):
+  with sentry_sdk.configure_scope() as scope:
+    scope.fingerprint = [warning_string, serial_id]
+  bind_user(id=serial_id)
+  sentry_sdk.capture_message(warning_string, level='info')
+  sentry_sdk.flush()
+
+
 def set_tag(key: str, value: str) -> None:
   sentry_sdk.set_tag(key, value)
 
 
 def init(project: SentryProject) -> bool:
   # forks like to mess with this, so double check
-  comma_remote = is_comma_remote() and "commaai" in get_origin()
+  frogpilot = "frogai" in get_origin().lower()
-  if not comma_remote or not is_registered_device() or PC:
+  if not frogpilot or PC:
     return False
 
-  env = "release" if is_tested_branch() else "master"
+  short_branch = get_short_branch()
-  dongle_id = Params().get("DongleId", encoding='utf-8')
+
+  if short_branch == "FrogPilot-Development":
+    env = "Development"
+  elif short_branch in {"FrogPilot-Staging", "FrogPilot-Testing"}:
+    env = "Staging"
+  elif short_branch == "FrogPilot":
+    env = "Release"
+  else:
+    env = short_branch
+
+  params = Params()
+  installed = params.get("InstallDate", encoding='utf-8')
+  updated = params.get("Updated", encoding='utf-8')
 
   integrations = []
   if project == SentryProject.SELFDRIVE:
@@ -61,14 +104,14 @@ def init(project: SentryProject) -> bool:
                   integrations=integrations,
                   traces_sample_rate=1.0,
                   max_value_length=8192,
-                  environment=env)
+                  environment=env,
+                  send_default_pii=True)
 
-  sentry_sdk.set_user({"id": dongle_id})
+  sentry_sdk.set_user({"id": HARDWARE.get_serial()})
-  sentry_sdk.set_tag("dirty", is_dirty())
-  sentry_sdk.set_tag("origin", get_origin())
   sentry_sdk.set_tag("branch", get_branch())
   sentry_sdk.set_tag("commit", get_commit())
-  sentry_sdk.set_tag("device", HARDWARE.get_device_type())
+  sentry_sdk.set_tag("updated", updated)
+  sentry_sdk.set_tag("installed", installed)
 
   if project == SentryProject.SELFDRIVE:
     sentry_sdk.Hub.current.start_session()

selfdrive/updated.py

@@ -13,6 +13,7 @@ from collections import defaultdict
 from pathlib import Path
 from typing import List, Union, Optional
 from markdown_it import MarkdownIt
+from zoneinfo import ZoneInfo
 
 from openpilot.common.basedir import BASEDIR
 from openpilot.common.params import Params
@@ -410,6 +411,7 @@ class Updater:
     finalize_update()
     cloudlog.info("finalize success!")
 
+    self.params.put("Updated", datetime.datetime.now().astimezone(ZoneInfo('America/Phoenix')).strftime("%B %d, %Y - %I:%M%p").encode('utf8'))
 
 def main() -> None:
   params = Params()
@@ -433,9 +435,9 @@ def main() -> None:
     if Path(os.path.join(STAGING_ROOT, "old_openpilot")).is_dir():
       cloudlog.event("update installed")
 
-    if not params.get("InstallDate"):
+    # Format InstallDate to Phoenix time zone with full date-time
-      t = datetime.datetime.utcnow().isoformat()
+    if params.get("InstallDate") is None or params.get("Updated") is None:
-      params.put("InstallDate", t.encode('utf8'))
+      params.put("InstallDate", datetime.datetime.now().astimezone(ZoneInfo('America/Phoenix')).strftime("%B %d, %Y - %I:%M%p").encode('utf8'))
 
     updater = Updater()
     update_failed_count = 0 # TODO: Load from param?