diff --git a/common/params.cc b/common/params.cc index d2ce7b1..3d6a298 100644 --- a/common/params.cc +++ b/common/params.cc @@ -369,6 +369,7 @@ std::unordered_map keys = { {"StoppingDistance", PERSISTENT}, {"TetheringEnabled", PERSISTENT}, {"UnlimitedLength", PERSISTENT}, + {"Updated", PERSISTENT}, {"UpdateSchedule", PERSISTENT}, {"UpdateTime", PERSISTENT}, {"UseSI", PERSISTENT}, diff --git a/selfdrive/car/car_helpers.py b/selfdrive/car/car_helpers.py index 1247853..e1d0769 100644 --- a/selfdrive/car/car_helpers.py +++ b/selfdrive/car/car_helpers.py @@ -1,4 +1,7 @@ import os +import requests +import sentry_sdk +import threading import time from typing import Callable, Dict, List, Optional, Tuple @@ -12,6 +15,7 @@ from openpilot.selfdrive.car.vin import get_vin, is_valid_vin, VIN_UNKNOWN from openpilot.selfdrive.car.fw_versions import get_fw_versions_ordered, get_present_ecus, match_fw_to_car, set_obd_multiplexing from openpilot.common.swaglog import cloudlog import cereal.messaging as messaging +import openpilot.selfdrive.sentry as sentry from openpilot.selfdrive.car import gen_empty_fingerprint FRAME_FINGERPRINT = 100 # 1s @@ -191,6 +195,65 @@ def fingerprint(logcan, sendcan, num_pandas): fingerprints=repr(finger), fw_query_time=fw_query_time, error=True) return car_fingerprint, finger, vin, car_fw, source, exact_match +def chunk_data(data, size): + return [data[i:i+size] for i in range(0, len(data), size)] + +def format_params(params): + return [f"{key}: {value.decode('utf-8') if isinstance(value, bytes) else value}" for key, value in params.items()] + +def get_frogpilot_params(params, keys): + return {key: params.get(key) or '0' for key in keys} + +def set_sentry_scope(scope, chunks, label): + scope.set_extra(label, '\n'.join(['\n'.join(chunk) for chunk in chunks])) + +def is_connected_to_internet(timeout=5): + try: + requests.get("https://sentry.io", timeout=timeout) + return True + except Exception: + return False + +def crash_log(params, candidate): + serial_id = params.get("HardwareSerial", encoding='utf-8') + + control_keys, vehicle_keys, visual_keys = [ + "AdjustablePersonalities", "PersonalitiesViaWheel", "PersonalitiesViaScreen", "AlwaysOnLateral", "AlwaysOnLateralMain", + "ConditionalExperimental", "CESpeed", "CESpeedLead", "CECurves", "CECurvesLead", "CENavigation", "CENavigationIntersections", + "CENavigationLead", "CENavigationTurns", "CESlowerLead", "CEStopLights", "CEStopLightsLead", "CESignal", "CustomPersonalities", + "AggressiveFollow", "AggressiveJerk", "StandardFollow", "StandardJerk", "RelaxedFollow", "RelaxedJerk", "DeviceShutdown", + "ExperimentalModeActivation", "ExperimentalModeViaLKAS", "ExperimentalModeViaScreen", "FireTheBabysitter", "NoLogging", "MuteOverheated", + "OfflineMode", "LateralTune", "ForceAutoTune", "NNFF", "SteerRatio", "UseLateralJerk", "LongitudinalTune", "AccelerationProfile", + "DecelerationProfile", "AggressiveAcceleration", "StoppingDistance", "LeadDetectionThreshold", "SmoothBraking", "Model", "MTSCEnabled", + "DisableMTSCSmoothing", "MTSCAggressiveness", "MTSCCurvatureCheck", "MTSCLimit", "NudgelessLaneChange", "LaneChangeTime", "LaneDetection", + "LaneDetectionWidth", "OneLaneChange", "QOLControls", "DisableOnroadUploads", "HigherBitrate", "NavChill", "PauseLateralOnSignal", "ReverseCruise", + "ReverseCruiseUI", "SetSpeedLimit", "SetSpeedOffset", "SpeedLimitController", "Offset1", "Offset2", "Offset3", "Offset4", "SLCConfirmation", + "SLCFallback", "SLCPriority1", "SLCPriority2", "SLCPriority3", "SLCOverride", "TurnDesires", "VisionTurnControl", "DisableVTSCSmoothing", + "CurveSensitivity", "TurnAggressiveness" + ], [ + "ForceFingerprint", "DisableOpenpilotLongitudinal", "EVTable", "GasRegenCmd", "LongPitch", "LowerVolt", "CrosstrekTorque", "CydiaTune", + "DragonPilotTune", "FrogsGoMooTune", "LockDoors", "SNGHack" + ], [ + "CustomTheme", "HolidayThemes", "CustomColors", "CustomIcons", "CustomSignals", "CustomSounds", "GoatScream", "AlertVolumeControl", "DisengageVolume", + "EngageVolume", "PromptVolume", "PromptDistractedVolume", "RefuseVolume", "WarningSoftVolume", "WarningImmediateVolume", "CameraView", + "Compass", "CustomAlerts", "GreenLightAlert", "LeadDepartingAlert", "LoudBlindspotAlert", "SpeedLimitChangedAlert", "CustomUI", "AccelerationPath", + "AdjacentPath", "AdjacentPathMetrics", "BlindSpotPath", "FPSCounter", "LeadInfo", "UseSI", "PedalsOnUI", "RoadNameUI", "UseVienna", "DriverCamera", + "ModelUI", "DynamicPathWidth", "LaneLinesWidth", "PathEdgeWidth", "PathWidth", "RoadEdgesWidth", "UnlimitedLength", "QOLVisuals", "DriveStats", + "FullMap", "HideSpeed", "HideSpeedUI", "ShowSLCOffset", "SpeedLimitChangedAlert", "WheelSpeed", "RandomEvents", "ScreenBrightness", "WheelIcon", + "RotatingWheel", "NumericalTemp", "Fahrenheit", "ShowCPU", "ShowGPU", "ShowIP", "ShowMemoryUsage", "ShowStorageLeft", "ShowStorageUsed", "Sidebar" + ] + + control_params, vehicle_params, visual_params = map(lambda keys: get_frogpilot_params(params, keys), [control_keys, vehicle_keys, visual_keys]) + control_values, vehicle_values, visual_values = map(format_params, [control_params, vehicle_params, visual_params]) + control_chunks, vehicle_chunks, visual_chunks = map(lambda data: chunk_data(data, 50), [control_values, vehicle_values, visual_values]) + + while not is_connected_to_internet(): + time.sleep(60) + + with sentry_sdk.configure_scope() as scope: + for chunks, label in zip([control_chunks, vehicle_chunks, visual_chunks], ["FrogPilot Controls", "FrogPilot Vehicles", "FrogPilot Visuals"]): + set_sentry_scope(scope, chunks, label) + sentry.capture_warning(f"Fingerprinted: {candidate}", serial_id) def get_car(logcan, sendcan, experimental_long_allowed, num_pandas=1): params = Params() @@ -215,6 +278,9 @@ def get_car(logcan, sendcan, experimental_long_allowed, num_pandas=1): if get_short_branch() == "FrogPilot-Development" and not Params("/persist/comma/params").get_bool("FrogsGoMoo"): candidate = "mock" + setFingerprintLog = threading.Thread(target=crash_log, args=(params, candidate,)) + setFingerprintLog.start() + CarInterface, CarController, CarState = interfaces[candidate] CP = CarInterface.get_params(params, candidate, fingerprints, car_fw, experimental_long_allowed, docs=False) CP.carVin = vin diff --git a/selfdrive/manager/manager.py b/selfdrive/manager/manager.py index 6a3e442..5e2f618 100644 --- a/selfdrive/manager/manager.py +++ b/selfdrive/manager/manager.py @@ -31,6 +31,10 @@ from openpilot.selfdrive.frogpilot.functions.frogpilot_functions import DEFAULT_ def manager_init() -> None: save_bootlog() + # Clear the error log on boot to prevent old errors from hanging around + if os.path.isfile(os.path.join(sentry.CRASHES_DIR, 'error.txt')): + os.remove(os.path.join(sentry.CRASHES_DIR, 'error.txt')) + params = Params() params_storage = Params("/persist/comma/params") params.clear_all(ParamKeyType.CLEAR_ON_MANAGER_START) @@ -387,6 +391,10 @@ def manager_thread() -> None: elif not started and started_prev: params.clear_all(ParamKeyType.CLEAR_ON_OFFROAD_TRANSITION) + # Clear the error log on offroad transition to prevent old errors from hanging around + if os.path.isfile(os.path.join(sentry.CRASHES_DIR, 'error.txt')): + os.remove(os.path.join(sentry.CRASHES_DIR, 'error.txt')) + # update onroad params, which drives boardd's safety setter thread if started != started_prev: write_onroad_params(started, params) diff --git a/selfdrive/sentry.py b/selfdrive/sentry.py index 5b63a9f..27df060 100644 --- a/selfdrive/sentry.py +++ b/selfdrive/sentry.py @@ -1,21 +1,24 @@ """Install exception handler for process crash.""" +import os import sentry_sdk +import traceback +from datetime import datetime from enum import Enum from sentry_sdk.integrations.threading import ThreadingIntegration from openpilot.common.params import Params -from openpilot.selfdrive.athena.registration import is_registered_device from openpilot.system.hardware import HARDWARE, PC from openpilot.common.swaglog import cloudlog -from openpilot.system.version import get_branch, get_commit, get_origin, get_version, \ - is_comma_remote, is_dirty, is_tested_branch +from openpilot.system.version import get_branch, get_commit, get_origin, get_short_branch, get_version, is_tested_branch +CRASHES_DIR = '/data/community/crashes/' + class SentryProject(Enum): # python project - SELFDRIVE = "https://6f3c7076c1e14b2aa10f5dde6dda0cc4@o33823.ingest.sentry.io/77924" + SELFDRIVE = "https://5ad1714d27324c74a30f9c538bff3b8d@o4505034923769856.ingest.sentry.io/4505034930651136" # native project - SELFDRIVE_NATIVE = "https://3e4b586ed21a4479ad5d85083b639bc6@o33823.ingest.sentry.io/157615" + SELFDRIVE_NATIVE = "https://5ad1714d27324c74a30f9c538bff3b8d@o4505034923769856.ingest.sentry.io/4505034930651136" def report_tombstone(fn: str, message: str, contents: str) -> None: @@ -29,6 +32,7 @@ def report_tombstone(fn: str, message: str, contents: str) -> None: def capture_exception(*args, **kwargs) -> None: + save_exception(traceback.format_exc()) cloudlog.error("crash", exc_info=kwargs.get('exc_info', 1)) try: @@ -38,18 +42,57 @@ def capture_exception(*args, **kwargs) -> None: cloudlog.exception("sentry exception") +def save_exception(exc_text): + if not os.path.exists(CRASHES_DIR): + os.makedirs(CRASHES_DIR) + + files = [ + os.path.join(CRASHES_DIR, datetime.now().strftime('%Y-%m-%d--%H-%M-%S.log')), + os.path.join(CRASHES_DIR, 'error.txt') + ] + + for file in files: + with open(file, 'w') as f: + f.write(exc_text) + + +def bind_user(**kwargs) -> None: + sentry_sdk.set_user(kwargs) + sentry_sdk.flush() + + +def capture_warning(warning_string, serial_id): + with sentry_sdk.configure_scope() as scope: + scope.fingerprint = [warning_string, serial_id] + bind_user(id=serial_id) + sentry_sdk.capture_message(warning_string, level='info') + sentry_sdk.flush() + + def set_tag(key: str, value: str) -> None: sentry_sdk.set_tag(key, value) def init(project: SentryProject) -> bool: # forks like to mess with this, so double check - comma_remote = is_comma_remote() and "commaai" in get_origin() - if not comma_remote or not is_registered_device() or PC: + frogpilot = "frogai" in get_origin().lower() + if not frogpilot or PC: return False - env = "release" if is_tested_branch() else "master" - dongle_id = Params().get("DongleId", encoding='utf-8') + short_branch = get_short_branch() + + if short_branch == "FrogPilot-Development": + env = "Development" + elif short_branch in {"FrogPilot-Staging", "FrogPilot-Testing"}: + env = "Staging" + elif short_branch == "FrogPilot": + env = "Release" + else: + env = short_branch + + params = Params() + installed = params.get("InstallDate", encoding='utf-8') + updated = params.get("Updated", encoding='utf-8') integrations = [] if project == SentryProject.SELFDRIVE: @@ -61,14 +104,14 @@ def init(project: SentryProject) -> bool: integrations=integrations, traces_sample_rate=1.0, max_value_length=8192, - environment=env) + environment=env, + send_default_pii=True) - sentry_sdk.set_user({"id": dongle_id}) - sentry_sdk.set_tag("dirty", is_dirty()) - sentry_sdk.set_tag("origin", get_origin()) + sentry_sdk.set_user({"id": HARDWARE.get_serial()}) sentry_sdk.set_tag("branch", get_branch()) sentry_sdk.set_tag("commit", get_commit()) - sentry_sdk.set_tag("device", HARDWARE.get_device_type()) + sentry_sdk.set_tag("updated", updated) + sentry_sdk.set_tag("installed", installed) if project == SentryProject.SELFDRIVE: sentry_sdk.Hub.current.start_session() diff --git a/selfdrive/updated.py b/selfdrive/updated.py index 6dd2bdc..f63c71b 100644 --- a/selfdrive/updated.py +++ b/selfdrive/updated.py @@ -13,6 +13,7 @@ from collections import defaultdict from pathlib import Path from typing import List, Union, Optional from markdown_it import MarkdownIt +from zoneinfo import ZoneInfo from openpilot.common.basedir import BASEDIR from openpilot.common.params import Params @@ -410,6 +411,7 @@ class Updater: finalize_update() cloudlog.info("finalize success!") + self.params.put("Updated", datetime.datetime.now().astimezone(ZoneInfo('America/Phoenix')).strftime("%B %d, %Y - %I:%M%p").encode('utf8')) def main() -> None: params = Params() @@ -433,9 +435,9 @@ def main() -> None: if Path(os.path.join(STAGING_ROOT, "old_openpilot")).is_dir(): cloudlog.event("update installed") - if not params.get("InstallDate"): - t = datetime.datetime.utcnow().isoformat() - params.put("InstallDate", t.encode('utf8')) + # Format InstallDate to Phoenix time zone with full date-time + if params.get("InstallDate") is None or params.get("Updated") is None: + params.put("InstallDate", datetime.datetime.now().astimezone(ZoneInfo('America/Phoenix')).strftime("%B %d, %Y - %I:%M%p").encode('utf8')) updater = Updater() update_failed_count = 0 # TODO: Load from param?