openpilot v0.9.6 release

date: 2024-01-12T10:13:37
master commit: ba792d576a49a0899b88a753fa1c52956bedf9e6

selfdrive/ui/tests/test_translations.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+import json
+import os
+import re
+import unittest
+import shutil
+import tempfile
+import xml.etree.ElementTree as ET
+import string
+import requests
+from parameterized import parameterized_class
+
+from openpilot.selfdrive.ui.update_translations import TRANSLATIONS_DIR, LANGUAGES_FILE, update_translations
+
+with open(LANGUAGES_FILE, "r") as f:
+  translation_files = json.load(f)
+
+UNFINISHED_TRANSLATION_TAG = "<translation type=\"unfinished\""  # non-empty translations can be marked unfinished
+LOCATION_TAG = "<location "
+FORMAT_ARG = re.compile("%[0-9]+")
+
+
+@parameterized_class(("name", "file"), translation_files.items())
+class TestTranslations(unittest.TestCase):
+  name: str
+  file: str
+
+  @staticmethod
+  def _read_translation_file(path, file):
+    tr_file = os.path.join(path, f"{file}.ts")
+    with open(tr_file, "r") as f:
+      return f.read()
+
+  def test_missing_translation_files(self):
+    self.assertTrue(os.path.exists(os.path.join(TRANSLATIONS_DIR, f"{self.file}.ts")),
+                    f"{self.name} has no XML translation file, run selfdrive/ui/update_translations.py")
+
+  def test_translations_updated(self):
+    with tempfile.TemporaryDirectory() as tmpdir:
+      shutil.copytree(TRANSLATIONS_DIR, tmpdir, dirs_exist_ok=True)
+      update_translations(translation_files=[self.file], translations_dir=tmpdir)
+
+      cur_translations = self._read_translation_file(TRANSLATIONS_DIR, self.file)
+      new_translations = self._read_translation_file(tmpdir, self.file)
+      self.assertEqual(cur_translations, new_translations,
+                       f"{self.file} ({self.name}) XML translation file out of date. Run selfdrive/ui/update_translations.py to update the translation files")
+
+  @unittest.skip("Only test unfinished translations before going to release")
+  def test_unfinished_translations(self):
+    cur_translations = self._read_translation_file(TRANSLATIONS_DIR, self.file)
+    self.assertTrue(UNFINISHED_TRANSLATION_TAG not in cur_translations,
+                    f"{self.file} ({self.name}) translation file has unfinished translations. Finish translations or mark them as completed in Qt Linguist")
+
+  def test_vanished_translations(self):
+    cur_translations = self._read_translation_file(TRANSLATIONS_DIR, self.file)
+    self.assertTrue("<translation type=\"vanished\">" not in cur_translations,
+                    f"{self.file} ({self.name}) translation file has obsolete translations. Run selfdrive/ui/update_translations.py --vanish to remove them")
+
+  def test_finished_translations(self):
+    """
+      Tests ran on each translation marked "finished"
+      Plural:
+      - that any numerus (plural) translations have all plural forms non-empty
+      - that the correct format specifier is used (%n)
+      Non-plural:
+      - that translation is not empty
+      - that translation format arguments are consistent
+    """
+    tr_xml = ET.parse(os.path.join(TRANSLATIONS_DIR, f"{self.file}.ts"))
+
+    for context in tr_xml.getroot():
+      for message in context.iterfind("message"):
+        translation = message.find("translation")
+        source_text = message.find("source").text
+
+        # Do not test unfinished translations
+        if translation.get("type") == "unfinished":
+          continue
+
+        if message.get("numerus") == "yes":
+          numerusform = [t.text for t in translation.findall("numerusform")]
+
+          for nf in numerusform:
+            self.assertIsNotNone(nf, f"Ensure all plural translation forms are completed: {source_text}")
+            self.assertIn("%n", nf, "Ensure numerus argument (%n) exists in translation.")
+            self.assertIsNone(FORMAT_ARG.search(nf), "Plural translations must use %n, not %1, %2, etc.: {}".format(numerusform))
+
+        else:
+          self.assertIsNotNone(translation.text, f"Ensure translation is completed: {source_text}")
+
+          source_args = FORMAT_ARG.findall(source_text)
+          translation_args = FORMAT_ARG.findall(translation.text)
+          self.assertEqual(sorted(source_args), sorted(translation_args),
+                           f"Ensure format arguments are consistent: `{source_text}` vs. `{translation.text}`")
+
+  def test_no_locations(self):
+    for line in self._read_translation_file(TRANSLATIONS_DIR, self.file).splitlines():
+      self.assertFalse(line.strip().startswith(LOCATION_TAG),
+                       f"Line contains location tag: {line.strip()}, remove all line numbers.")
+
+  def test_entities_error(self):
+    cur_translations = self._read_translation_file(TRANSLATIONS_DIR, self.file)
+    matches = re.findall(r'@(\w+);', cur_translations)
+    self.assertEqual(len(matches), 0, f"The string(s) {matches} were found with '@' instead of '&'")
+
+  def test_bad_language(self):
+    IGNORED_WORDS = {'pédale'}
+
+    match = re.search(r'_([a-zA-Z]{2,3})', self.file)
+    assert match, f"{self.name} - could not parse language"
+
+    response = requests.get(f"https://raw.githubusercontent.com/LDNOOBW/List-of-Dirty-Naughty-Obscene-and-Otherwise-Bad-Words/master/{match.group(1)}")
+    response.raise_for_status()
+
+    banned_words = {line.strip() for line in response.text.splitlines()}
+
+    for context in ET.parse(os.path.join(TRANSLATIONS_DIR, f"{self.file}.ts")).getroot():
+      for message in context.iterfind("message"):
+        translation = message.find("translation")
+        if translation.get("type") == "unfinished":
+          continue
+
+        translation_text = " ".join([t.text for t in translation.findall("numerusform")]) if message.get("numerus") == "yes" else translation.text
+
+        if not translation_text:
+          continue
+
+        words = set(translation_text.translate(str.maketrans('', '', string.punctuation + '%n')).lower().split())
+        bad_words_found = words & (banned_words - IGNORED_WORDS)
+        assert not bad_words_found, f"Bad language found in {self.name}: '{translation_text}'. Bad word(s): {', '.join(bad_words_found)}"
+
+
+if __name__ == "__main__":
+  unittest.main()