wip

tools/lib/tests/test_caching.py

@@ -1,11 +1,12 @@
 #!/usr/bin/env python3
-from functools import partial
+from functools import wraps
 import http.server
 import os
+import threading
+import time
 import unittest
 
 from parameterized import parameterized
-from openpilot.selfdrive.athena.tests.helpers import with_http_server
 
 from openpilot.tools.lib.url_file import URLFile
 
@@ -29,7 +30,27 @@ class CachingTestRequestHandler(http.server.BaseHTTPRequestHandler):
     self.end_headers()
 
 
-with_caching_server = partial(with_http_server, handler=CachingTestRequestHandler)
+class CachingTestServer(threading.Thread):
+  def run(self):
+    self.server = http.server.HTTPServer(("127.0.0.1", 0), CachingTestRequestHandler)
+    self.port = self.server.server_port
+    self.server.serve_forever()
+
+  def stop(self):
+    self.server.server_close()
+    self.server.shutdown()
+
+def with_caching_server(func):
+  @wraps(func)
+  def wrapper(*args, **kwargs):
+    server = CachingTestServer()
+    server.start()
+    time.sleep(0.25) # wait for server to get it's port
+    try:
+      func(*args, **kwargs, port=server.port)
+    finally:
+      server.stop()
+  return wrapper
 
 
 class TestFileDownload(unittest.TestCase):
@@ -89,10 +110,10 @@ class TestFileDownload(unittest.TestCase):
 
   @parameterized.expand([(True, ), (False, )])
   @with_caching_server
-  def test_recover_from_missing_file(self, cache_enabled, host):
+  def test_recover_from_missing_file(self, cache_enabled, port):
     os.environ["FILEREADER_CACHE"] = "1" if cache_enabled else "0"
 
-    file_url = f"{host}/test.png"
+    file_url = f"http://localhost:{port}/test.png"
 
     CachingTestRequestHandler.FILE_EXISTS = False
     length = URLFile(file_url).get_length()

tools/lib/tests/test_srreader.py

@@ -0,0 +1,88 @@
+import shutil
+import tempfile
+import numpy as np
+import unittest
+from parameterized import parameterized
+import requests
+
+from openpilot.tools.lib.route import SegmentRange
+from openpilot.tools.lib.srreader import ReadMode, SegmentRangeReader, parse_slice, parse_indirect
+
+NUM_SEGS = 17 # number of segments in the test route
+ALL_SEGS = list(np.arange(NUM_SEGS))
+TEST_ROUTE = "344c5c15b34f2d8a/2024-01-03--09-37-12"
+QLOG_FILE = "https://commadataci.blob.core.windows.net/openpilotci/0375fdf7b1ce594d/2019-06-13--08-32-25/3/qlog.bz2"
+
+class TestSegmentRangeReader(unittest.TestCase):
+  @parameterized.expand([
+    (f"{TEST_ROUTE}", ALL_SEGS),
+    (f"{TEST_ROUTE.replace('/', '|')}", ALL_SEGS),
+    (f"{TEST_ROUTE}--0", [0]),
+    (f"{TEST_ROUTE}--5", [5]),
+    (f"{TEST_ROUTE}/0", [0]),
+    (f"{TEST_ROUTE}/5", [5]),
+    (f"{TEST_ROUTE}/0:10", ALL_SEGS[0:10]),
+    (f"{TEST_ROUTE}/0:0", []),
+    (f"{TEST_ROUTE}/4:6", ALL_SEGS[4:6]),
+    (f"{TEST_ROUTE}/0:-1", ALL_SEGS[0:-1]),
+    (f"{TEST_ROUTE}/:5", ALL_SEGS[:5]),
+    (f"{TEST_ROUTE}/2:", ALL_SEGS[2:]),
+    (f"{TEST_ROUTE}/2:-1", ALL_SEGS[2:-1]),
+    (f"{TEST_ROUTE}/-1", [ALL_SEGS[-1]]),
+    (f"{TEST_ROUTE}/-2", [ALL_SEGS[-2]]),
+    (f"{TEST_ROUTE}/-2:-1", ALL_SEGS[-2:-1]),
+    (f"{TEST_ROUTE}/-4:-2", ALL_SEGS[-4:-2]),
+    (f"{TEST_ROUTE}/:10:2", ALL_SEGS[:10:2]),
+    (f"{TEST_ROUTE}/5::2", ALL_SEGS[5::2]),
+    (f"https://useradmin.comma.ai/?onebox={TEST_ROUTE}", ALL_SEGS),
+    (f"https://useradmin.comma.ai/?onebox={TEST_ROUTE.replace('/', '|')}", ALL_SEGS),
+    (f"https://useradmin.comma.ai/?onebox={TEST_ROUTE.replace('/', '%7C')}", ALL_SEGS),
+    (f"https://cabana.comma.ai/?route={TEST_ROUTE}", ALL_SEGS),
+    (f"cd:/{TEST_ROUTE}", ALL_SEGS),
+  ])
+  def test_indirect_parsing(self, identifier, expected):
+    parsed, _, _ = parse_indirect(identifier)
+    sr = SegmentRange(parsed)
+    segs = parse_slice(sr)
+    self.assertListEqual(list(segs), expected)
+
+  def test_direct_parsing(self):
+    qlog = tempfile.NamedTemporaryFile(mode='wb', delete=False)
+
+    with requests.get(QLOG_FILE, stream=True) as r:
+      with qlog as f:
+        shutil.copyfileobj(r.raw, f)
+
+    for f in [QLOG_FILE, qlog.name]:
+      l = len(list(SegmentRangeReader(f)))
+      self.assertGreater(l, 100)
+
+  @parameterized.expand([
+    (f"{TEST_ROUTE}///",),
+    (f"{TEST_ROUTE}---",),
+    (f"{TEST_ROUTE}/-4:--2",),
+    (f"{TEST_ROUTE}/-a",),
+    (f"{TEST_ROUTE}/j",),
+    (f"{TEST_ROUTE}/0:1:2:3",),
+    (f"{TEST_ROUTE}/:::3",),
+  ])
+  def test_bad_ranges(self, segment_range):
+    with self.assertRaises(AssertionError):
+      sr = SegmentRange(segment_range)
+      parse_slice(sr)
+
+  def test_modes(self):
+    qlog_len = len(list(SegmentRangeReader(f"{TEST_ROUTE}/0", ReadMode.QLOG)))
+    rlog_len = len(list(SegmentRangeReader(f"{TEST_ROUTE}/0", ReadMode.RLOG)))
+
+    self.assertLess(qlog_len * 6, rlog_len)
+
+  def test_modes_from_name(self):
+    qlog_len = len(list(SegmentRangeReader(f"{TEST_ROUTE}/0/q")))
+    rlog_len = len(list(SegmentRangeReader(f"{TEST_ROUTE}/0/r")))
+
+    self.assertLess(qlog_len * 6, rlog_len)
+
+
+if __name__ == "__main__":
+  unittest.main()