From 6427b1380e0a52dd9986185d9e1ec7260fddff92 Mon Sep 17 00:00:00 2001 From: FrogAi <91348155+FrogAi@users.noreply.github.com> Date: Tue, 27 Feb 2024 16:34:47 -0700 Subject: [PATCH] Sentry logging Logging for my Sentry server that tracks the values of the serial number, car fingerprint, user set parameters, and the date and time for when FrogPilot was installed and last updated for debugging and support. --- common/params.cc | 1 + selfdrive/car/car_helpers.py | 66 +++++++++++++++++++++++++++++++++ selfdrive/manager/manager.py | 8 ++++ selfdrive/sentry.py | 71 +++++++++++++++++++++++++++++------- selfdrive/updated.py | 8 ++-- 5 files changed, 137 insertions(+), 17 deletions(-) diff --git a/common/params.cc b/common/params.cc index d2ce7b1..3d6a298 100644 --- a/common/params.cc +++ b/common/params.cc @@ -369,6 +369,7 @@ std::unordered_map keys = { {"StoppingDistance", PERSISTENT}, {"TetheringEnabled", PERSISTENT}, {"UnlimitedLength", PERSISTENT}, + {"Updated", PERSISTENT}, {"UpdateSchedule", PERSISTENT}, {"UpdateTime", PERSISTENT}, {"UseSI", PERSISTENT}, diff --git a/selfdrive/car/car_helpers.py b/selfdrive/car/car_helpers.py index 1247853..e1d0769 100644 --- a/selfdrive/car/car_helpers.py +++ b/selfdrive/car/car_helpers.py @@ -1,4 +1,7 @@ import os +import requests +import sentry_sdk +import threading import time from typing import Callable, Dict, List, Optional, Tuple @@ -12,6 +15,7 @@ from openpilot.selfdrive.car.vin import get_vin, is_valid_vin, VIN_UNKNOWN from openpilot.selfdrive.car.fw_versions import get_fw_versions_ordered, get_present_ecus, match_fw_to_car, set_obd_multiplexing from openpilot.common.swaglog import cloudlog import cereal.messaging as messaging +import openpilot.selfdrive.sentry as sentry from openpilot.selfdrive.car import gen_empty_fingerprint FRAME_FINGERPRINT = 100 # 1s @@ -191,6 +195,65 @@ def fingerprint(logcan, sendcan, num_pandas): fingerprints=repr(finger), fw_query_time=fw_query_time, error=True) return car_fingerprint, finger, vin, car_fw, source, exact_match +def chunk_data(data, size): + return [data[i:i+size] for i in range(0, len(data), size)] + +def format_params(params): + return [f"{key}: {value.decode('utf-8') if isinstance(value, bytes) else value}" for key, value in params.items()] + +def get_frogpilot_params(params, keys): + return {key: params.get(key) or '0' for key in keys} + +def set_sentry_scope(scope, chunks, label): + scope.set_extra(label, '\n'.join(['\n'.join(chunk) for chunk in chunks])) + +def is_connected_to_internet(timeout=5): + try: + requests.get("https://sentry.io", timeout=timeout) + return True + except Exception: + return False + +def crash_log(params, candidate): + serial_id = params.get("HardwareSerial", encoding='utf-8') + + control_keys, vehicle_keys, visual_keys = [ + "AdjustablePersonalities", "PersonalitiesViaWheel", "PersonalitiesViaScreen", "AlwaysOnLateral", "AlwaysOnLateralMain", + "ConditionalExperimental", "CESpeed", "CESpeedLead", "CECurves", "CECurvesLead", "CENavigation", "CENavigationIntersections", + "CENavigationLead", "CENavigationTurns", "CESlowerLead", "CEStopLights", "CEStopLightsLead", "CESignal", "CustomPersonalities", + "AggressiveFollow", "AggressiveJerk", "StandardFollow", "StandardJerk", "RelaxedFollow", "RelaxedJerk", "DeviceShutdown", + "ExperimentalModeActivation", "ExperimentalModeViaLKAS", "ExperimentalModeViaScreen", "FireTheBabysitter", "NoLogging", "MuteOverheated", + "OfflineMode", "LateralTune", "ForceAutoTune", "NNFF", "SteerRatio", "UseLateralJerk", "LongitudinalTune", "AccelerationProfile", + "DecelerationProfile", "AggressiveAcceleration", "StoppingDistance", "LeadDetectionThreshold", "SmoothBraking", "Model", "MTSCEnabled", + "DisableMTSCSmoothing", "MTSCAggressiveness", "MTSCCurvatureCheck", "MTSCLimit", "NudgelessLaneChange", "LaneChangeTime", "LaneDetection", + "LaneDetectionWidth", "OneLaneChange", "QOLControls", "DisableOnroadUploads", "HigherBitrate", "NavChill", "PauseLateralOnSignal", "ReverseCruise", + "ReverseCruiseUI", "SetSpeedLimit", "SetSpeedOffset", "SpeedLimitController", "Offset1", "Offset2", "Offset3", "Offset4", "SLCConfirmation", + "SLCFallback", "SLCPriority1", "SLCPriority2", "SLCPriority3", "SLCOverride", "TurnDesires", "VisionTurnControl", "DisableVTSCSmoothing", + "CurveSensitivity", "TurnAggressiveness" + ], [ + "ForceFingerprint", "DisableOpenpilotLongitudinal", "EVTable", "GasRegenCmd", "LongPitch", "LowerVolt", "CrosstrekTorque", "CydiaTune", + "DragonPilotTune", "FrogsGoMooTune", "LockDoors", "SNGHack" + ], [ + "CustomTheme", "HolidayThemes", "CustomColors", "CustomIcons", "CustomSignals", "CustomSounds", "GoatScream", "AlertVolumeControl", "DisengageVolume", + "EngageVolume", "PromptVolume", "PromptDistractedVolume", "RefuseVolume", "WarningSoftVolume", "WarningImmediateVolume", "CameraView", + "Compass", "CustomAlerts", "GreenLightAlert", "LeadDepartingAlert", "LoudBlindspotAlert", "SpeedLimitChangedAlert", "CustomUI", "AccelerationPath", + "AdjacentPath", "AdjacentPathMetrics", "BlindSpotPath", "FPSCounter", "LeadInfo", "UseSI", "PedalsOnUI", "RoadNameUI", "UseVienna", "DriverCamera", + "ModelUI", "DynamicPathWidth", "LaneLinesWidth", "PathEdgeWidth", "PathWidth", "RoadEdgesWidth", "UnlimitedLength", "QOLVisuals", "DriveStats", + "FullMap", "HideSpeed", "HideSpeedUI", "ShowSLCOffset", "SpeedLimitChangedAlert", "WheelSpeed", "RandomEvents", "ScreenBrightness", "WheelIcon", + "RotatingWheel", "NumericalTemp", "Fahrenheit", "ShowCPU", "ShowGPU", "ShowIP", "ShowMemoryUsage", "ShowStorageLeft", "ShowStorageUsed", "Sidebar" + ] + + control_params, vehicle_params, visual_params = map(lambda keys: get_frogpilot_params(params, keys), [control_keys, vehicle_keys, visual_keys]) + control_values, vehicle_values, visual_values = map(format_params, [control_params, vehicle_params, visual_params]) + control_chunks, vehicle_chunks, visual_chunks = map(lambda data: chunk_data(data, 50), [control_values, vehicle_values, visual_values]) + + while not is_connected_to_internet(): + time.sleep(60) + + with sentry_sdk.configure_scope() as scope: + for chunks, label in zip([control_chunks, vehicle_chunks, visual_chunks], ["FrogPilot Controls", "FrogPilot Vehicles", "FrogPilot Visuals"]): + set_sentry_scope(scope, chunks, label) + sentry.capture_warning(f"Fingerprinted: {candidate}", serial_id) def get_car(logcan, sendcan, experimental_long_allowed, num_pandas=1): params = Params() @@ -215,6 +278,9 @@ def get_car(logcan, sendcan, experimental_long_allowed, num_pandas=1): if get_short_branch() == "FrogPilot-Development" and not Params("/persist/comma/params").get_bool("FrogsGoMoo"): candidate = "mock" + setFingerprintLog = threading.Thread(target=crash_log, args=(params, candidate,)) + setFingerprintLog.start() + CarInterface, CarController, CarState = interfaces[candidate] CP = CarInterface.get_params(params, candidate, fingerprints, car_fw, experimental_long_allowed, docs=False) CP.carVin = vin diff --git a/selfdrive/manager/manager.py b/selfdrive/manager/manager.py index 6a3e442..5e2f618 100644 --- a/selfdrive/manager/manager.py +++ b/selfdrive/manager/manager.py @@ -31,6 +31,10 @@ from openpilot.selfdrive.frogpilot.functions.frogpilot_functions import DEFAULT_ def manager_init() -> None: save_bootlog() + # Clear the error log on boot to prevent old errors from hanging around + if os.path.isfile(os.path.join(sentry.CRASHES_DIR, 'error.txt')): + os.remove(os.path.join(sentry.CRASHES_DIR, 'error.txt')) + params = Params() params_storage = Params("/persist/comma/params") params.clear_all(ParamKeyType.CLEAR_ON_MANAGER_START) @@ -387,6 +391,10 @@ def manager_thread() -> None: elif not started and started_prev: params.clear_all(ParamKeyType.CLEAR_ON_OFFROAD_TRANSITION) + # Clear the error log on offroad transition to prevent old errors from hanging around + if os.path.isfile(os.path.join(sentry.CRASHES_DIR, 'error.txt')): + os.remove(os.path.join(sentry.CRASHES_DIR, 'error.txt')) + # update onroad params, which drives boardd's safety setter thread if started != started_prev: write_onroad_params(started, params) diff --git a/selfdrive/sentry.py b/selfdrive/sentry.py index 5b63a9f..27df060 100644 --- a/selfdrive/sentry.py +++ b/selfdrive/sentry.py @@ -1,21 +1,24 @@ """Install exception handler for process crash.""" +import os import sentry_sdk +import traceback +from datetime import datetime from enum import Enum from sentry_sdk.integrations.threading import ThreadingIntegration from openpilot.common.params import Params -from openpilot.selfdrive.athena.registration import is_registered_device from openpilot.system.hardware import HARDWARE, PC from openpilot.common.swaglog import cloudlog -from openpilot.system.version import get_branch, get_commit, get_origin, get_version, \ - is_comma_remote, is_dirty, is_tested_branch +from openpilot.system.version import get_branch, get_commit, get_origin, get_short_branch, get_version, is_tested_branch +CRASHES_DIR = '/data/community/crashes/' + class SentryProject(Enum): # python project - SELFDRIVE = "https://6f3c7076c1e14b2aa10f5dde6dda0cc4@o33823.ingest.sentry.io/77924" + SELFDRIVE = "https://5ad1714d27324c74a30f9c538bff3b8d@o4505034923769856.ingest.sentry.io/4505034930651136" # native project - SELFDRIVE_NATIVE = "https://3e4b586ed21a4479ad5d85083b639bc6@o33823.ingest.sentry.io/157615" + SELFDRIVE_NATIVE = "https://5ad1714d27324c74a30f9c538bff3b8d@o4505034923769856.ingest.sentry.io/4505034930651136" def report_tombstone(fn: str, message: str, contents: str) -> None: @@ -29,6 +32,7 @@ def report_tombstone(fn: str, message: str, contents: str) -> None: def capture_exception(*args, **kwargs) -> None: + save_exception(traceback.format_exc()) cloudlog.error("crash", exc_info=kwargs.get('exc_info', 1)) try: @@ -38,18 +42,57 @@ def capture_exception(*args, **kwargs) -> None: cloudlog.exception("sentry exception") +def save_exception(exc_text): + if not os.path.exists(CRASHES_DIR): + os.makedirs(CRASHES_DIR) + + files = [ + os.path.join(CRASHES_DIR, datetime.now().strftime('%Y-%m-%d--%H-%M-%S.log')), + os.path.join(CRASHES_DIR, 'error.txt') + ] + + for file in files: + with open(file, 'w') as f: + f.write(exc_text) + + +def bind_user(**kwargs) -> None: + sentry_sdk.set_user(kwargs) + sentry_sdk.flush() + + +def capture_warning(warning_string, serial_id): + with sentry_sdk.configure_scope() as scope: + scope.fingerprint = [warning_string, serial_id] + bind_user(id=serial_id) + sentry_sdk.capture_message(warning_string, level='info') + sentry_sdk.flush() + + def set_tag(key: str, value: str) -> None: sentry_sdk.set_tag(key, value) def init(project: SentryProject) -> bool: # forks like to mess with this, so double check - comma_remote = is_comma_remote() and "commaai" in get_origin() - if not comma_remote or not is_registered_device() or PC: + frogpilot = "frogai" in get_origin().lower() + if not frogpilot or PC: return False - env = "release" if is_tested_branch() else "master" - dongle_id = Params().get("DongleId", encoding='utf-8') + short_branch = get_short_branch() + + if short_branch == "FrogPilot-Development": + env = "Development" + elif short_branch in {"FrogPilot-Staging", "FrogPilot-Testing"}: + env = "Staging" + elif short_branch == "FrogPilot": + env = "Release" + else: + env = short_branch + + params = Params() + installed = params.get("InstallDate", encoding='utf-8') + updated = params.get("Updated", encoding='utf-8') integrations = [] if project == SentryProject.SELFDRIVE: @@ -61,14 +104,14 @@ def init(project: SentryProject) -> bool: integrations=integrations, traces_sample_rate=1.0, max_value_length=8192, - environment=env) + environment=env, + send_default_pii=True) - sentry_sdk.set_user({"id": dongle_id}) - sentry_sdk.set_tag("dirty", is_dirty()) - sentry_sdk.set_tag("origin", get_origin()) + sentry_sdk.set_user({"id": HARDWARE.get_serial()}) sentry_sdk.set_tag("branch", get_branch()) sentry_sdk.set_tag("commit", get_commit()) - sentry_sdk.set_tag("device", HARDWARE.get_device_type()) + sentry_sdk.set_tag("updated", updated) + sentry_sdk.set_tag("installed", installed) if project == SentryProject.SELFDRIVE: sentry_sdk.Hub.current.start_session() diff --git a/selfdrive/updated.py b/selfdrive/updated.py index 6dd2bdc..f63c71b 100644 --- a/selfdrive/updated.py +++ b/selfdrive/updated.py @@ -13,6 +13,7 @@ from collections import defaultdict from pathlib import Path from typing import List, Union, Optional from markdown_it import MarkdownIt +from zoneinfo import ZoneInfo from openpilot.common.basedir import BASEDIR from openpilot.common.params import Params @@ -410,6 +411,7 @@ class Updater: finalize_update() cloudlog.info("finalize success!") + self.params.put("Updated", datetime.datetime.now().astimezone(ZoneInfo('America/Phoenix')).strftime("%B %d, %Y - %I:%M%p").encode('utf8')) def main() -> None: params = Params() @@ -433,9 +435,9 @@ def main() -> None: if Path(os.path.join(STAGING_ROOT, "old_openpilot")).is_dir(): cloudlog.event("update installed") - if not params.get("InstallDate"): - t = datetime.datetime.utcnow().isoformat() - params.put("InstallDate", t.encode('utf8')) + # Format InstallDate to Phoenix time zone with full date-time + if params.get("InstallDate") is None or params.get("Updated") is None: + params.put("InstallDate", datetime.datetime.now().astimezone(ZoneInfo('America/Phoenix')).strftime("%B %d, %Y - %I:%M%p").encode('utf8')) updater = Updater() update_failed_count = 0 # TODO: Load from param?