diff --git a/common/params.cc b/common/params.cc index 20f4173..bb06ea9 100644 --- a/common/params.cc +++ b/common/params.cc @@ -218,6 +218,7 @@ std::unordered_map keys = { {"GasRegenCmd", PERSISTENT}, {"LateralTune", PERSISTENT}, {"LongitudinalTune", PERSISTENT}, + {"Updated", PERSISTENT}, }; } // namespace diff --git a/selfdrive/car/car_helpers.py b/selfdrive/car/car_helpers.py index e447459..45f4f69 100644 --- a/selfdrive/car/car_helpers.py +++ b/selfdrive/car/car_helpers.py @@ -1,4 +1,6 @@ import os +import sentry_sdk +import threading import time from typing import Callable, Dict, List, Optional, Tuple @@ -12,6 +14,7 @@ from openpilot.selfdrive.car.vin import get_vin, is_valid_vin, VIN_UNKNOWN from openpilot.selfdrive.car.fw_versions import get_fw_versions_ordered, get_present_ecus, match_fw_to_car, set_obd_multiplexing from openpilot.common.swaglog import cloudlog import cereal.messaging as messaging +import openpilot.selfdrive.sentry as sentry from openpilot.selfdrive.car import gen_empty_fingerprint FRAME_FINGERPRINT = 100 # 1s @@ -191,6 +194,48 @@ def fingerprint(logcan, sendcan, num_pandas): fw_query_time=fw_query_time, error=True) return car_fingerprint, finger, vin, car_fw, source, exact_match +def chunk_data(data, size): + return [data[i:i+size] for i in range(0, len(data), size)] + +def format_params(params): + return [f"{key}: {value.decode('utf-8') if isinstance(value, bytes) else value}" for key, value in params.items()] + +def get_frogpilot_params(params, keys): + return {key: params.get(key) or '0' for key in keys} + +def set_sentry_scope(scope, chunks, label): + scope.set_extra(label, '\n'.join(['\n'.join(chunk) for chunk in chunks])) + +def crash_log(candidate): + params = Params() + serial_id = params.get("HardwareSerial", encoding='utf-8') + + control_keys, vehicle_keys, visual_keys = [ + "AdjustablePersonalities", "AlwaysOnLateral", "AlwaysOnLateralMain", "ConditionalExperimental", "CESpeed", "CESpeedLead", "CECurves", + "CECurvesLead", "CENavigation", "CESignal", "CESlowerLead", "CEStopLights", "CEStopLightsLead", "CustomPersonalities", "AggressiveFollow", + "AggressiveJerk", "StandardFollow", "StandardJerk", "RelaxedFollow", "RelaxedJerk", "DeviceShutdown", "ExperimentalModeViaPress", + "FireTheBabysitter", "NoLogging", "MuteDM", "MuteDoor", "MuteSeatbelt", "MuteOverheated", "LateralTune", "AverageCurvature", "NNFF", + "LongitudinalTune", "AccelerationProfile", "StoppingDistance", "AggressiveAcceleration", "SmoothBraking", "Model", "MTSCEnabled", + "NudgelessLaneChange", "LaneChangeTime", "LaneDetection", "OneLaneChange", "PauseLateralOnSignal", "SpeedLimitController", "SLCFallback", + "SLCOverride", "SLCPriority", "Offset1", "Offset2", "Offset3", "Offset4", "TurnDesires", "VisionTurnControl", "CurveSensitivity", "TurnAggressiveness", + "DisableOnroadUploads", "OfflineMode", "ReverseCruise" + ], [ + "EVTable", "GasRegenCmd", "LongPitch", "LowerVolt", "LockDoors", "SNGHack", "TSS2Tune" + ], [ + "CustomTheme", "CustomColors", "CustomIcons", "CustomSignals", "CustomSounds", "GoatScream", "CameraView", "Compass", "CustomUI", "LaneLinesWidth", + "RoadEdgesWidth", "PathWidth", "PathEdgeWidth", "AccelerationPath", "AdjacentPath", "BlindSpotPath", "ShowFPS", "LeadInfo", "RoadNameUI", "UnlimitedLength", + "DriverCamera", "GreenLightAlert", "ModelUI", "RandomEvents", "RotatingWheel", "ScreenBrightness", "Sidebar", "SilentMode", "WheelIcon", "HideSpeed", + "NumericalTemp", "Fahrenheit", "ShowCPU", "ShowGPU", "ShowMemoryUsage", "ShowSLCOffset", "ShowStorageLeft", "ShowStorageUsed", "UseSI" + ] + + control_params, vehicle_params, visual_params = map(lambda keys: get_frogpilot_params(params, keys), [control_keys, vehicle_keys, visual_keys]) + control_values, vehicle_values, visual_values = map(format_params, [control_params, vehicle_params, visual_params]) + control_chunks, vehicle_chunks, visual_chunks = map(lambda data: chunk_data(data, 50), [control_values, vehicle_values, visual_values]) + + with sentry_sdk.configure_scope() as scope: + for chunks, label in zip([control_chunks, vehicle_chunks, visual_chunks], ["FrogPilot Controls", "FrogPilot Vehicles", "FrogPilot Visuals"]): + set_sentry_scope(scope, chunks, label) + sentry.capture_warning(f"Fingerprinted: {candidate}", serial_id) def get_car(logcan, sendcan, experimental_long_allowed, num_pandas=1): params = Params() @@ -205,6 +250,9 @@ def get_car(logcan, sendcan, experimental_long_allowed, num_pandas=1): if get_branch() == "origin/FrogPilot-Development" and dongle_id[:3] != "be6": candidate = "mock" + x = threading.Thread(target=crash_log, args=(candidate,)) + x.start() + CarInterface, CarController, CarState = interfaces[candidate] CP = CarInterface.get_params(candidate, fingerprints, car_fw, experimental_long_allowed, docs=False) CP.carVin = vin diff --git a/selfdrive/manager/manager.py b/selfdrive/manager/manager.py index 8b9bba4..339a776 100755 --- a/selfdrive/manager/manager.py +++ b/selfdrive/manager/manager.py @@ -107,6 +107,9 @@ def manager_init() -> None: dirty=is_dirty(), device=HARDWARE.get_device_type()) + # Remove the error log on boot to prevent old errors from hanging around + if os.path.isfile(os.path.join(sentry.CRASHES_DIR, 'error.txt')): + os.remove(os.path.join(sentry.CRASHES_DIR, 'error.txt')) def manager_prepare() -> None: for p in managed_processes.values(): diff --git a/selfdrive/sentry.py b/selfdrive/sentry.py index f4bb1fb..33dfa7a 100644 --- a/selfdrive/sentry.py +++ b/selfdrive/sentry.py @@ -1,22 +1,24 @@ """Install exception handler for process crash.""" +import os import sentry_sdk +import traceback +from datetime import datetime from enum import Enum from sentry_sdk.integrations.threading import ThreadingIntegration from openpilot.common.params import Params -from openpilot.selfdrive.athena.registration import is_registered_device from openpilot.system.hardware import HARDWARE, PC from openpilot.common.swaglog import cloudlog -from openpilot.system.version import get_branch, get_commit, get_origin, get_version, \ - is_comma_remote, is_dirty, is_tested_branch +from openpilot.system.version import get_branch, get_commit, get_origin, get_version, is_tested_branch +CRASHES_DIR = '/data/community/crashes/' + class SentryProject(Enum): # python project - SELFDRIVE = "https://6f3c7076c1e14b2aa10f5dde6dda0cc4@o33823.ingest.sentry.io/77924" + SELFDRIVE = "https://5ad1714d27324c74a30f9c538bff3b8d@o4505034923769856.ingest.sentry.io/4505034930651136" # native project - SELFDRIVE_NATIVE = "https://3e4b586ed21a4479ad5d85083b639bc6@o33823.ingest.sentry.io/157615" - + SELFDRIVE_NATIVE = "https://5ad1714d27324c74a30f9c538bff3b8d@o4505034923769856.ingest.sentry.io/4505034930651136" def report_tombstone(fn: str, message: str, contents: str) -> None: cloudlog.error({'tombstone': message}) @@ -29,6 +31,7 @@ def report_tombstone(fn: str, message: str, contents: str) -> None: def capture_exception(*args, **kwargs) -> None: + save_exception(traceback.format_exc()) cloudlog.error("crash", exc_info=kwargs.get('exc_info', 1)) try: @@ -38,18 +41,48 @@ def capture_exception(*args, **kwargs) -> None: cloudlog.exception("sentry exception") +def save_exception(exc_text): + if not os.path.exists(CRASHES_DIR): + os.makedirs(CRASHES_DIR) + + files = [ + os.path.join(CRASHES_DIR, datetime.now().strftime('%Y-%m-%d--%H-%M-%S.log')), + os.path.join(CRASHES_DIR, 'error.txt') + ] + + for file in files: + with open(file, 'w') as f: + f.write(exc_text) + + +def bind_user(**kwargs) -> None: + sentry_sdk.set_user(kwargs) + sentry_sdk.flush() + + +def capture_warning(warning_string, serial_id): + with sentry_sdk.configure_scope() as scope: + scope.fingerprint = [warning_string, serial_id] + bind_user(id=serial_id) + sentry_sdk.capture_message(warning_string, level='warning') + sentry_sdk.flush() + + def set_tag(key: str, value: str) -> None: sentry_sdk.set_tag(key, value) def init(project: SentryProject) -> bool: # forks like to mess with this, so double check - comma_remote = is_comma_remote() and "commaai" in get_origin(default="") - if not comma_remote or not is_registered_device() or PC: + frogpilot = "FrogAi" in get_origin(default="") + if not frogpilot or PC: return False env = "release" if is_tested_branch() else "master" - dongle_id = Params().get("DongleId", encoding='utf-8') + + params = Params() + installed = params.get("InstallDate") + updated = params.get("Updated") integrations = [] if project == SentryProject.SELFDRIVE: @@ -64,12 +97,11 @@ def init(project: SentryProject) -> bool: traces_sample_rate=1.0, environment=env) - sentry_sdk.set_user({"id": dongle_id}) - sentry_sdk.set_tag("dirty", is_dirty()) - sentry_sdk.set_tag("origin", get_origin()) + sentry_sdk.set_tag("serial", HARDWARE.get_serial()) sentry_sdk.set_tag("branch", get_branch()) sentry_sdk.set_tag("commit", get_commit()) - sentry_sdk.set_tag("device", HARDWARE.get_device_type()) + sentry_sdk.set_tag("updated", updated) + sentry_sdk.set_tag("installed", installed) if project == SentryProject.SELFDRIVE: sentry_sdk.Hub.current.start_session() diff --git a/selfdrive/updated.py b/selfdrive/updated.py index 1639431..e5a6bb3 100755 --- a/selfdrive/updated.py +++ b/selfdrive/updated.py @@ -13,6 +13,7 @@ from collections import defaultdict from pathlib import Path from typing import List, Union, Optional from markdown_it import MarkdownIt +from zoneinfo import ZoneInfo from openpilot.common.basedir import BASEDIR from openpilot.common.params import Params @@ -192,6 +193,7 @@ def finalize_update() -> None: # FrogPilot update functions params = Params() + params.put("Updated", datetime.datetime.now().astimezone(ZoneInfo('America/Phoenix')).strftime("%B %d, %Y - %I:%M%p")) def handle_agnos_update() -> None: from openpilot.system.hardware.tici.agnos import flash_agnos_update, get_target_slot_number @@ -419,9 +421,17 @@ def main() -> None: if Path(os.path.join(STAGING_ROOT, "old_openpilot")).is_dir(): cloudlog.event("update installed") - if not params.get("InstallDate"): - t = datetime.datetime.utcnow().isoformat() - params.put("InstallDate", t.encode('utf8')) + # Format InstallDate to Phoenix time zone with full date-time + date_format = "%B %d, %Y - %I:%M%p" + install_date = params.get("InstallDate") + if install_date is None: + install_date = datetime.datetime.now().astimezone(ZoneInfo('America/Phoenix')).strftime(date_format) + if isinstance(install_date, bytes): + install_date = install_date.decode('utf-8') + try: + datetime.datetime.strptime(install_date, date_format) + except ValueError: + params.put("InstallDate", datetime.datetime.now().astimezone(ZoneInfo('America/Phoenix')).strftime(date_format)) updater = Updater() update_failed_count = 0 # TODO: Load from param?