openpilot v0.9.6 release

date: 2024-02-21T23:02:42
master commit: 0b4d08fab8e35a264bc7383e878538f8083c33e5

tools/lib/tests/test_caching.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+from functools import partial
+import http.server
+import os
+import unittest
+
+from parameterized import parameterized
+from openpilot.selfdrive.athena.tests.helpers import with_http_server
+
+from openpilot.tools.lib.url_file import URLFile
+
+
+class CachingTestRequestHandler(http.server.BaseHTTPRequestHandler):
+  FILE_EXISTS = True
+
+  def do_GET(self):
+    if self.FILE_EXISTS:
+      self.send_response(200, b'1234')
+    else:
+      self.send_response(404)
+    self.end_headers()
+
+  def do_HEAD(self):
+    if self.FILE_EXISTS:
+      self.send_response(200)
+      self.send_header("Content-Length", "4")
+    else:
+      self.send_response(404)
+    self.end_headers()
+
+
+with_caching_server = partial(with_http_server, handler=CachingTestRequestHandler)
+
+
+class TestFileDownload(unittest.TestCase):
+
+  def compare_loads(self, url, start=0, length=None):
+    """Compares range between cached and non cached version"""
+    file_cached = URLFile(url, cache=True)
+    file_downloaded = URLFile(url, cache=False)
+
+    file_cached.seek(start)
+    file_downloaded.seek(start)
+
+    self.assertEqual(file_cached.get_length(), file_downloaded.get_length())
+    self.assertLessEqual(length + start if length is not None else 0, file_downloaded.get_length())
+
+    response_cached = file_cached.read(ll=length)
+    response_downloaded = file_downloaded.read(ll=length)
+
+    self.assertEqual(response_cached, response_downloaded)
+
+    # Now test with cache in place
+    file_cached = URLFile(url, cache=True)
+    file_cached.seek(start)
+    response_cached = file_cached.read(ll=length)
+
+    self.assertEqual(file_cached.get_length(), file_downloaded.get_length())
+    self.assertEqual(response_cached, response_downloaded)
+
+  def test_small_file(self):
+    # Make sure we don't force cache
+    os.environ["FILEREADER_CACHE"] = "0"
+    small_file_url = "https://raw.githubusercontent.com/commaai/openpilot/master/docs/SAFETY.md"
+    #  If you want large file to be larger than a chunk
+    #  large_file_url = "https://commadataci.blob.core.windows.net/openpilotci/0375fdf7b1ce594d/2019-06-13--08-32-25/3/fcamera.hevc"
+
+    #  Load full small file
+    self.compare_loads(small_file_url)
+
+    file_small = URLFile(small_file_url)
+    length = file_small.get_length()
+
+    self.compare_loads(small_file_url, length - 100, 100)
+    self.compare_loads(small_file_url, 50, 100)
+
+    #  Load small file 100 bytes at a time
+    for i in range(length // 100):
+      self.compare_loads(small_file_url, 100 * i, 100)
+
+  def test_large_file(self):
+    large_file_url = "https://commadataci.blob.core.windows.net/openpilotci/0375fdf7b1ce594d/2019-06-13--08-32-25/3/qlog.bz2"
+    #  Load the end 100 bytes of both files
+    file_large = URLFile(large_file_url)
+    length = file_large.get_length()
+
+    self.compare_loads(large_file_url, length - 100, 100)
+    self.compare_loads(large_file_url)
+
+  @parameterized.expand([(True, ), (False, )])
+  @with_caching_server
+  def test_recover_from_missing_file(self, cache_enabled, host):
+    os.environ["FILEREADER_CACHE"] = "1" if cache_enabled else "0"
+
+    file_url = f"{host}/test.png"
+
+    CachingTestRequestHandler.FILE_EXISTS = False
+    length = URLFile(file_url).get_length()
+    self.assertEqual(length, -1)
+
+    CachingTestRequestHandler.FILE_EXISTS = True
+    length = URLFile(file_url).get_length()
+    self.assertEqual(length, 4)
+
+
+
+if __name__ == "__main__":
+  unittest.main()

tools/lib/tests/test_comma_car_segments.py

@@ -0,0 +1,41 @@
+
+
+import unittest
+
+import requests
+from openpilot.tools.lib.comma_car_segments import get_comma_car_segments_database, get_url
+from openpilot.tools.lib.logreader import LogReader
+from openpilot.tools.lib.route import SegmentRange
+
+
+class TestCommaCarSegments(unittest.TestCase):
+  def test_database(self):
+    database = get_comma_car_segments_database()
+
+    platforms = database.keys()
+
+    assert len(platforms) > 100
+
+  def test_download_segment(self):
+    database = get_comma_car_segments_database()
+
+    fp = "SUBARU FORESTER 2019"
+
+    segment = database[fp][0]
+
+    sr = SegmentRange(segment)
+
+    url = get_url(sr.route_name, sr.slice)
+
+    resp = requests.get(url)
+    self.assertEqual(resp.status_code, 200)
+
+    lr = LogReader(url)
+
+    CP = lr.first("carParams")
+
+    self.assertEqual(CP.carFingerprint, fp)
+
+
+if __name__ == "__main__":
+  unittest.main()

tools/lib/tests/test_logreader.py

@@ -0,0 +1,221 @@
+#!/usr/bin/env python3
+import contextlib
+import io
+import shutil
+import tempfile
+import os
+import unittest
+import pytest
+import requests
+
+from parameterized import parameterized
+from unittest import mock
+
+from openpilot.tools.lib.logreader import LogIterable, LogReader, comma_api_source, parse_indirect, ReadMode, InternalUnavailableException
+from openpilot.tools.lib.route import SegmentRange
+from openpilot.tools.lib.url_file import URLFileException
+
+NUM_SEGS = 17  # number of segments in the test route
+ALL_SEGS = list(range(NUM_SEGS))
+TEST_ROUTE = "344c5c15b34f2d8a/2024-01-03--09-37-12"
+QLOG_FILE = "https://commadataci.blob.core.windows.net/openpilotci/0375fdf7b1ce594d/2019-06-13--08-32-25/3/qlog.bz2"
+
+
+def noop(segment: LogIterable):
+  return segment
+
+
+@contextlib.contextmanager
+def setup_source_scenario(is_internal=False):
+  with (
+    mock.patch("openpilot.tools.lib.logreader.internal_source") as internal_source_mock,
+    mock.patch("openpilot.tools.lib.logreader.openpilotci_source") as openpilotci_source_mock,
+    mock.patch("openpilot.tools.lib.logreader.comma_api_source") as comma_api_source_mock,
+  ):
+    if is_internal:
+      internal_source_mock.return_value = [QLOG_FILE]
+    else:
+      internal_source_mock.side_effect = InternalUnavailableException
+
+    openpilotci_source_mock.return_value = [None]
+    comma_api_source_mock.return_value = [QLOG_FILE]
+
+    yield
+
+
+class TestLogReader(unittest.TestCase):
+  @parameterized.expand([
+    (f"{TEST_ROUTE}", ALL_SEGS),
+    (f"{TEST_ROUTE.replace('/', '|')}", ALL_SEGS),
+    (f"{TEST_ROUTE}--0", [0]),
+    (f"{TEST_ROUTE}--5", [5]),
+    (f"{TEST_ROUTE}/0", [0]),
+    (f"{TEST_ROUTE}/5", [5]),
+    (f"{TEST_ROUTE}/0:10", ALL_SEGS[0:10]),
+    (f"{TEST_ROUTE}/0:0", []),
+    (f"{TEST_ROUTE}/4:6", ALL_SEGS[4:6]),
+    (f"{TEST_ROUTE}/0:-1", ALL_SEGS[0:-1]),
+    (f"{TEST_ROUTE}/:5", ALL_SEGS[:5]),
+    (f"{TEST_ROUTE}/2:", ALL_SEGS[2:]),
+    (f"{TEST_ROUTE}/2:-1", ALL_SEGS[2:-1]),
+    (f"{TEST_ROUTE}/-1", [ALL_SEGS[-1]]),
+    (f"{TEST_ROUTE}/-2", [ALL_SEGS[-2]]),
+    (f"{TEST_ROUTE}/-2:-1", ALL_SEGS[-2:-1]),
+    (f"{TEST_ROUTE}/-4:-2", ALL_SEGS[-4:-2]),
+    (f"{TEST_ROUTE}/:10:2", ALL_SEGS[:10:2]),
+    (f"{TEST_ROUTE}/5::2", ALL_SEGS[5::2]),
+    (f"https://useradmin.comma.ai/?onebox={TEST_ROUTE}", ALL_SEGS),
+    (f"https://useradmin.comma.ai/?onebox={TEST_ROUTE.replace('/', '|')}", ALL_SEGS),
+    (f"https://useradmin.comma.ai/?onebox={TEST_ROUTE.replace('/', '%7C')}", ALL_SEGS),
+    (f"https://cabana.comma.ai/?route={TEST_ROUTE}", ALL_SEGS),
+  ])
+  def test_indirect_parsing(self, identifier, expected):
+    parsed, _, _ = parse_indirect(identifier)
+    sr = SegmentRange(parsed)
+    self.assertListEqual(list(sr.seg_idxs), expected, identifier)
+
+  @parameterized.expand([
+    (f"{TEST_ROUTE}", f"{TEST_ROUTE}"),
+    (f"{TEST_ROUTE.replace('/', '|')}", f"{TEST_ROUTE}"),
+    (f"{TEST_ROUTE}--5", f"{TEST_ROUTE}/5"),
+    (f"{TEST_ROUTE}/0/q", f"{TEST_ROUTE}/0/q"),
+    (f"{TEST_ROUTE}/5:6/r", f"{TEST_ROUTE}/5:6/r"),
+    (f"{TEST_ROUTE}/5", f"{TEST_ROUTE}/5"),
+  ])
+  def test_canonical_name(self, identifier, expected):
+    sr = SegmentRange(identifier)
+    self.assertEqual(str(sr), expected)
+
+  @parameterized.expand([(True,), (False,)])
+  @mock.patch("openpilot.tools.lib.logreader.file_exists")
+  def test_direct_parsing(self, cache_enabled, file_exists_mock):
+    os.environ["FILEREADER_CACHE"] = "1" if cache_enabled else "0"
+    qlog = tempfile.NamedTemporaryFile(mode='wb', delete=False)
+
+    with requests.get(QLOG_FILE, stream=True) as r:
+      with qlog as f:
+        shutil.copyfileobj(r.raw, f)
+
+    for f in [QLOG_FILE, qlog.name]:
+      l = len(list(LogReader(f)))
+      self.assertGreater(l, 100)
+
+    with self.assertRaises(URLFileException) if not cache_enabled else self.assertRaises(AssertionError):
+      l = len(list(LogReader(QLOG_FILE.replace("/3/", "/200/"))))
+
+    # file_exists should not be called for direct files
+    self.assertEqual(file_exists_mock.call_count, 0)
+
+  @parameterized.expand([
+    (f"{TEST_ROUTE}///",),
+    (f"{TEST_ROUTE}---",),
+    (f"{TEST_ROUTE}/-4:--2",),
+    (f"{TEST_ROUTE}/-a",),
+    (f"{TEST_ROUTE}/j",),
+    (f"{TEST_ROUTE}/0:1:2:3",),
+    (f"{TEST_ROUTE}/:::3",),
+    (f"{TEST_ROUTE}3",),
+    (f"{TEST_ROUTE}-3",),
+    (f"{TEST_ROUTE}--3a",),
+  ])
+  def test_bad_ranges(self, segment_range):
+    with self.assertRaises(AssertionError):
+      _ = SegmentRange(segment_range).seg_idxs
+
+  @parameterized.expand([
+    (f"{TEST_ROUTE}/0", False),
+    (f"{TEST_ROUTE}/:2", False),
+    (f"{TEST_ROUTE}/0:", True),
+    (f"{TEST_ROUTE}/-1", True),
+    (f"{TEST_ROUTE}", True),
+  ])
+  def test_slicing_api_call(self, segment_range, api_call):
+    with mock.patch("openpilot.tools.lib.route.get_max_seg_number_cached") as max_seg_mock:
+      max_seg_mock.return_value = NUM_SEGS
+      _ = SegmentRange(segment_range).seg_idxs
+      self.assertEqual(api_call, max_seg_mock.called)
+
+  @pytest.mark.slow
+  def test_modes(self):
+    qlog_len = len(list(LogReader(f"{TEST_ROUTE}/0", ReadMode.QLOG)))
+    rlog_len = len(list(LogReader(f"{TEST_ROUTE}/0", ReadMode.RLOG)))
+
+    self.assertLess(qlog_len * 6, rlog_len)
+
+  @pytest.mark.slow
+  def test_modes_from_name(self):
+    qlog_len = len(list(LogReader(f"{TEST_ROUTE}/0/q")))
+    rlog_len = len(list(LogReader(f"{TEST_ROUTE}/0/r")))
+
+    self.assertLess(qlog_len * 6, rlog_len)
+
+  @pytest.mark.slow
+  def test_list(self):
+    qlog_len = len(list(LogReader(f"{TEST_ROUTE}/0/q")))
+    qlog_len_2 = len(list(LogReader([f"{TEST_ROUTE}/0/q", f"{TEST_ROUTE}/0/q"])))
+
+    self.assertEqual(qlog_len * 2, qlog_len_2)
+
+  @pytest.mark.slow
+  @mock.patch("openpilot.tools.lib.logreader._LogFileReader")
+  def test_multiple_iterations(self, init_mock):
+    lr = LogReader(f"{TEST_ROUTE}/0/q")
+    qlog_len1 = len(list(lr))
+    qlog_len2 = len(list(lr))
+
+    # ensure we don't create multiple instances of _LogFileReader, which means downloading the files twice
+    self.assertEqual(init_mock.call_count, 1)
+
+    self.assertEqual(qlog_len1, qlog_len2)
+
+  @pytest.mark.slow
+  def test_helpers(self):
+    lr = LogReader(f"{TEST_ROUTE}/0/q")
+    self.assertEqual(lr.first("carParams").carFingerprint, "SUBARU OUTBACK 6TH GEN")
+    self.assertTrue(0 < len(list(lr.filter("carParams"))) < len(list(lr)))
+
+  @parameterized.expand([(True,), (False,)])
+  @pytest.mark.slow
+  def test_run_across_segments(self, cache_enabled):
+    os.environ["FILEREADER_CACHE"] = "1" if cache_enabled else "0"
+    lr = LogReader(f"{TEST_ROUTE}/0:4")
+    self.assertEqual(len(lr.run_across_segments(4, noop)), len(list(lr)))
+
+  @pytest.mark.slow
+  def test_auto_mode(self):
+    lr = LogReader(f"{TEST_ROUTE}/0/q")
+    qlog_len = len(list(lr))
+    with mock.patch("openpilot.tools.lib.route.Route.log_paths") as log_paths_mock:
+      log_paths_mock.return_value = [None] * NUM_SEGS
+      # Should fall back to qlogs since rlogs are not available
+
+      with self.subTest("interactive_yes"):
+        with mock.patch("sys.stdin", new=io.StringIO("y\n")):
+          lr = LogReader(f"{TEST_ROUTE}/0", default_mode=ReadMode.AUTO_INTERACTIVE, default_source=comma_api_source)
+          log_len = len(list(lr))
+        self.assertEqual(qlog_len, log_len)
+
+      with self.subTest("interactive_no"):
+        with mock.patch("sys.stdin", new=io.StringIO("n\n")):
+          with self.assertRaises(AssertionError):
+            lr = LogReader(f"{TEST_ROUTE}/0", default_mode=ReadMode.AUTO_INTERACTIVE, default_source=comma_api_source)
+
+      with self.subTest("non_interactive"):
+        lr = LogReader(f"{TEST_ROUTE}/0", default_mode=ReadMode.AUTO, default_source=comma_api_source)
+        log_len = len(list(lr))
+        self.assertEqual(qlog_len, log_len)
+
+  @parameterized.expand([(True,), (False,)])
+  @pytest.mark.slow
+  def test_auto_source_scenarios(self, is_internal):
+    lr = LogReader(QLOG_FILE)
+    qlog_len = len(list(lr))
+
+    with setup_source_scenario(is_internal=is_internal):
+      lr = LogReader(f"{TEST_ROUTE}/0/q")
+      log_len = len(list(lr))
+      self.assertEqual(qlog_len, log_len)
+
+
+if __name__ == "__main__":
+  unittest.main()

tools/lib/tests/test_readers.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+import unittest
+import requests
+import tempfile
+
+from collections import defaultdict
+import numpy as np
+from openpilot.tools.lib.framereader import FrameReader
+from openpilot.tools.lib.logreader import LogReader
+
+
+class TestReaders(unittest.TestCase):
+  @unittest.skip("skip for bandwidth reasons")
+  def test_logreader(self):
+    def _check_data(lr):
+      hist = defaultdict(int)
+      for l in lr:
+        hist[l.which()] += 1
+
+      self.assertEqual(hist['carControl'], 6000)
+      self.assertEqual(hist['logMessage'], 6857)
+
+    with tempfile.NamedTemporaryFile(suffix=".bz2") as fp:
+      r = requests.get("https://github.com/commaai/comma2k19/blob/master/Example_1/b0c9d2329ad1606b%7C2018-08-02--08-34-47/40/raw_log.bz2?raw=true", timeout=10)
+      fp.write(r.content)
+      fp.flush()
+
+      lr_file = LogReader(fp.name)
+      _check_data(lr_file)
+
+    lr_url = LogReader("https://github.com/commaai/comma2k19/blob/master/Example_1/b0c9d2329ad1606b%7C2018-08-02--08-34-47/40/raw_log.bz2?raw=true")
+    _check_data(lr_url)
+
+  @unittest.skip("skip for bandwidth reasons")
+  def test_framereader(self):
+    def _check_data(f):
+      self.assertEqual(f.frame_count, 1200)
+      self.assertEqual(f.w, 1164)
+      self.assertEqual(f.h, 874)
+
+      frame_first_30 = f.get(0, 30)
+      self.assertEqual(len(frame_first_30), 30)
+
+      print(frame_first_30[15])
+
+      print("frame_0")
+      frame_0 = f.get(0, 1)
+      frame_15 = f.get(15, 1)
+
+      print(frame_15[0])
+
+      assert np.all(frame_first_30[0] == frame_0[0])
+      assert np.all(frame_first_30[15] == frame_15[0])
+
+    with tempfile.NamedTemporaryFile(suffix=".hevc") as fp:
+      r = requests.get("https://github.com/commaai/comma2k19/blob/master/Example_1/b0c9d2329ad1606b%7C2018-08-02--08-34-47/40/video.hevc?raw=true", timeout=10)
+      fp.write(r.content)
+      fp.flush()
+
+      fr_file = FrameReader(fp.name)
+      _check_data(fr_file)
+
+    fr_url = FrameReader("https://github.com/commaai/comma2k19/blob/master/Example_1/b0c9d2329ad1606b%7C2018-08-02--08-34-47/40/video.hevc?raw=true")
+    _check_data(fr_url)
+
+if __name__ == "__main__":
+  unittest.main()

tools/lib/tests/test_route_library.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+import unittest
+from collections import namedtuple
+
+from openpilot.tools.lib.route import SegmentName
+
+class TestRouteLibrary(unittest.TestCase):
+  def test_segment_name_formats(self):
+    Case = namedtuple('Case', ['input', 'expected_route', 'expected_segment_num', 'expected_data_dir'])
+
+    cases = [ Case("a2a0ccea32023010|2023-07-27--13-01-19", "a2a0ccea32023010|2023-07-27--13-01-19", -1, None),
+              Case("a2a0ccea32023010/2023-07-27--13-01-19--1", "a2a0ccea32023010|2023-07-27--13-01-19", 1, None),
+              Case("a2a0ccea32023010|2023-07-27--13-01-19/2", "a2a0ccea32023010|2023-07-27--13-01-19", 2, None),
+              Case("a2a0ccea32023010/2023-07-27--13-01-19/3", "a2a0ccea32023010|2023-07-27--13-01-19", 3, None),
+              Case("/data/media/0/realdata/a2a0ccea32023010|2023-07-27--13-01-19", "a2a0ccea32023010|2023-07-27--13-01-19", -1, "/data/media/0/realdata"),
+              Case("/data/media/0/realdata/a2a0ccea32023010|2023-07-27--13-01-19--1", "a2a0ccea32023010|2023-07-27--13-01-19", 1, "/data/media/0/realdata"),
+              Case("/data/media/0/realdata/a2a0ccea32023010|2023-07-27--13-01-19/2", "a2a0ccea32023010|2023-07-27--13-01-19", 2, "/data/media/0/realdata") ]
+
+    def _validate(case):
+      route_or_segment_name = case.input
+
+      s = SegmentName(route_or_segment_name, allow_route_name=True)
+
+      self.assertEqual(str(s.route_name), case.expected_route)
+      self.assertEqual(s.segment_num, case.expected_segment_num)
+      self.assertEqual(s.data_dir, case.expected_data_dir)
+
+    for case in cases:
+      _validate(case)
+
+if __name__ == "__main__":
+  unittest.main()