Add openpilot tests

2024-03-06 14:58:47 -07:00
parent 2901597132
commit b39097a12d
259 changed files with 31176 additions and 12 deletions
--- a/panda/tests/misra/.gitignore
+++ b/panda/tests/misra/.gitignore
@@ -0,0 +1,5 @@
+*.pdf
+*.txt
+.output.log
+new_table
+cppcheck/
--- a/panda/tests/misra/coverage_table
+++ b/panda/tests/misra/coverage_table
@@ -0,0 +1,156 @@
+1.1     
+1.2     X (Addon)
+1.3     X (Cppcheck)
+2.1     X (Cppcheck)
+2.2     X (Addon)
+2.3     X (Addon)
+2.4     X (Addon)
+2.5     X (Addon)
+2.6     X (Cppcheck)
+2.7     X (Addon)
+3.1     X (Addon)
+3.2     X (Addon)
+4.1     X (Addon)
+4.2     X (Addon)
+5.1     X (Addon)
+5.2     X (Addon)
+5.3     X (Cppcheck)
+5.4     X (Addon)
+5.5     X (Addon)
+5.6     X (Addon)
+5.7     X (Addon)
+5.8     X (Addon)
+5.9     X (Addon)
+6.1     X (Addon)
+6.2     X (Addon)
+7.1     X (Addon)
+7.2     X (Addon)
+7.3     X (Addon)
+7.4     X (Addon)
+8.1     X (Addon)
+8.2     X (Addon)
+8.3     X (Cppcheck)
+8.4     X (Addon)
+8.5     X (Addon)
+8.6     X (Addon)
+8.7     X (Addon)
+8.8     X (Addon)
+8.9     X (Addon)
+8.10    X (Addon)
+8.11    X (Addon)
+8.12    X (Addon)
+8.13    X (Cppcheck)
+8.14    X (Addon)
+9.1     X (Cppcheck)
+9.2     X (Addon)
+9.3     X (Addon)
+9.4     X (Addon)
+9.5     X (Addon)
+10.1    X (Addon)
+10.2    X (Addon)
+10.3    X (Addon)
+10.4    X (Addon)
+10.5    X (Addon)
+10.6    X (Addon)
+10.7    X (Addon)
+10.8    X (Addon)
+11.1    X (Addon)
+11.2    X (Addon)
+11.3    X (Addon)
+11.4    X (Addon)
+11.5    X (Addon)
+11.6    X (Addon)
+11.7    X (Addon)
+11.8    X (Addon)
+11.9    X (Addon)
+12.1    X (Addon)
+12.2    X (Addon)
+12.3    X (Addon)
+12.4    X (Addon)
+13.1    X (Addon)
+13.2    X (Cppcheck)
+13.3    X (Addon)
+13.4    X (Addon)
+13.5    X (Addon)
+13.6    X (Addon)
+14.1    X (Addon)
+14.2    X (Addon)
+14.3    X (Cppcheck)
+14.4    X (Addon)
+15.1    X (Addon)
+15.2    X (Addon)
+15.3    X (Addon)
+15.4    X (Addon)
+15.5    X (Addon)
+15.6    X (Addon)
+15.7    X (Addon)
+16.1    X (Addon)
+16.2    X (Addon)
+16.3    X (Addon)
+16.4    X (Addon)
+16.5    X (Addon)
+16.6    X (Addon)
+16.7    X (Addon)
+17.1    X (Addon)
+17.2    X (Addon)
+17.3    X (Addon)
+17.4    X (Cppcheck)
+17.5    X (Cppcheck)
+17.6    X (Addon)
+17.7    X (Addon)
+17.8    X (Addon)
+18.1    X (Cppcheck)
+18.2    X (Cppcheck)
+18.3    X (Cppcheck)
+18.4    X (Addon)
+18.5    X (Addon)
+18.6    X (Cppcheck)
+18.7    X (Addon)
+18.8    X (Addon)
+19.1    X (Cppcheck)
+19.2    X (Addon)
+20.1    X (Addon)
+20.2    X (Addon)
+20.3    X (Addon)
+20.4    X (Addon)
+20.5    X (Addon)
+20.6    X (Cppcheck)
+20.7    X (Addon)
+20.8    X (Addon)
+20.9    X (Addon)
+20.10   X (Addon)
+20.11   X (Addon)
+20.12   X (Addon)
+20.13   X (Addon)
+20.14   X (Addon)
+21.1    X (Addon)
+21.2    X (Addon)
+21.3    X (Addon)
+21.4    X (Addon)
+21.5    X (Addon)
+21.6    X (Addon)
+21.7    X (Addon)
+21.8    X (Addon)
+21.9    X (Addon)
+21.10   X (Addon)
+21.11   X (Addon)
+21.12   X (Addon)
+21.13   X (Cppcheck)
+21.14   X (Addon)
+21.15   X (Addon)
+21.16   X (Addon)
+21.17   X (Cppcheck)
+21.18   X (Cppcheck)
+21.19   X (Addon)
+21.20   X (Addon)
+21.21   X (Addon)
+22.1    X (Cppcheck)
+22.2    X (Cppcheck)
+22.3    X (Cppcheck)
+22.4    X (Cppcheck)
+22.5    X (Addon)
+22.6    X (Cppcheck)
+22.7    X (Addon)
+22.8    X (Addon)
+22.9    X (Addon)
+22.10   X (Addon)
--- a/panda/tests/misra/install.sh
+++ b/panda/tests/misra/install.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+set -e
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+: "${CPPCHECK_DIR:=$DIR/cppcheck/}"
+
+if [ ! -d "$CPPCHECK_DIR" ]; then
+  git clone https://github.com/danmar/cppcheck.git $CPPCHECK_DIR
+fi
+
+cd $CPPCHECK_DIR
+
+VERS="2.13.0"
+git fetch --all --tags
+git checkout $VERS
+git cherry-pick -n f6b538e855f0bacea33c4074664628024ef39dc6 b11b42087ff29569bc3740f5aa07eb6616ea4f63
+
+#make clean
+make MATCHCOMPILTER=yes CXXFLAGS="-O2" -j8
--- a/panda/tests/misra/test_misra.sh
+++ b/panda/tests/misra/test_misra.sh
@@ -0,0 +1,61 @@
+#!/bin/bash
+set -e
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+PANDA_DIR=$(realpath $DIR/../../)
+
+GREEN="\e[1;32m"
+NC='\033[0m'
+
+: "${CPPCHECK_DIR:=$DIR/cppcheck/}"
+
+# install cppcheck if missing
+if [ -z "${SKIP_BUILD}" ]; then
+  $DIR/install.sh
+fi
+
+# ensure checked in coverage table is up to date
+cd $DIR
+python $CPPCHECK_DIR/addons/misra.py -generate-table > new_table
+if ! cmp -s new_table coverage_table; then
+  echo "MISRA coverage table doesn't match. Update and commit:"
+  echo "mv new_table coverage_table && git add . && git commit -m 'update table'"
+  exit 1
+fi
+
+cd $PANDA_DIR
+if [ -z "${SKIP_BUILD}" ]; then
+  scons -j8
+fi
+
+cppcheck() {
+  # note that cppcheck build cache results in inconsistent results as of v2.13.0
+  OUTPUT=$DIR/.output.log
+  $CPPCHECK_DIR/cppcheck --force --inline-suppr -I $PANDA_DIR/board/ \
+          -I $gcc_inc "$(arm-none-eabi-gcc -print-file-name=include)" \
+          --suppressions-list=$DIR/suppressions.txt --suppress=*:*inc/* \
+          --suppress=*:*include/* --error-exitcode=2 --check-level=exhaustive \
+          --platform=arm32-wchar_t2 \
+          "$@" |& tee $OUTPUT
+
+  # cppcheck bug: some MISRA errors won't result in the error exit code,
+  # so check the output (https://trac.cppcheck.net/ticket/12440#no1)
+  if grep -e "misra violation" -e "error" -e "style: " $OUTPUT > /dev/null; then
+    exit 1
+  fi
+}
+
+PANDA_OPTS="--enable=all --disable=unusedFunction -DPANDA --addon=misra"
+
+printf "\n${GREEN}** PANDA F4 CODE **${NC}\n"
+cppcheck $PANDA_OPTS -DSTM32F4 -DUID_BASE $PANDA_DIR/board/main.c
+
+printf "\n${GREEN}** PANDA H7 CODE **${NC}\n"
+cppcheck $PANDA_OPTS -DSTM32H7 -DUID_BASE $PANDA_DIR/board/main.c
+
+# unused needs to run globally
+#printf "\n${GREEN}** UNUSED ALL CODE **${NC}\n"
+#cppcheck --enable=unusedFunction --quiet $PANDA_DIR/board/
+
+printf "\n${GREEN}Success!${NC} took $SECONDS seconds\n"
+
--- a/panda/tests/misra/test_mutation.py
+++ b/panda/tests/misra/test_mutation.py
@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+import os
+import glob
+import pytest
+import shutil
+import subprocess
+import tempfile
+import random
+
+HERE = os.path.abspath(os.path.dirname(__file__))
+ROOT = os.path.join(HERE, "../../")
+
+IGNORED_PATHS = (
+  'board/obj',
+  'board/jungle',
+  'board/stm32h7/inc',
+  'board/stm32fx/inc',
+  'board/fake_stm.h',
+
+  # bootstub only files
+  'board/flasher.h',
+  'board/bootstub.c',
+  'board/bootstub_declarations.h',
+  'board/stm32fx/llflash.h'
+)
+
+mutations = [
+  # default
+  (None, None, False),
+  # F4 only
+  ("board/stm32fx/llbxcan.h", "s/1U/1/g", True),
+  # H7 only
+  ("board/stm32h7/llfdcan.h", "s/return ret;/if (true) { return ret; } else { return false; }/g", True),
+  # general safety
+  ("board/safety/safety_toyota.h", "s/is_lkas_msg =.*;/is_lkas_msg = addr == 1 || addr == 2;/g", True),
+]
+
+patterns = [
+  # misra-c2012-13.3
+  "$a void test(int tmp) { int tmp2 = tmp++ + 2; if (tmp2) {;}}",
+  # misra-c2012-13.4
+  "$a int test(int x, int y) { return (x=2) && (y=2); }",
+  # misra-c2012-13.5
+  "$a void test(int tmp) { if (true && tmp++) {;} }",
+  # misra-c2012-13.6
+  "$a void test(int tmp) { if (sizeof(tmp++)) {;} }",
+  # misra-c2012-14.1
+  "$a void test(float len) { for (float j = 0; j < len; j++) {;} }",
+  # misra-c2012-14.4
+  "$a void test(int len) { if (len - 8) {;} }",
+  # misra-c2012-16.4
+  r"$a void test(int temp) {switch (temp) { case 1: ; }}\n",
+  # misra-c2012-17.8
+  "$a void test(int cnt) { for (cnt=0;;cnt++) {;} }",
+  # misra-c2012-20.4
+  r"$a #define auto 1\n",
+  # misra-c2012-20.5
+  r"$a #define TEST 1\n#undef TEST\n",
+]
+
+all_files = glob.glob('board/**', root_dir=ROOT, recursive=True)
+files = [f for f in all_files if f.endswith(('.c', '.h')) and not f.startswith(IGNORED_PATHS)]
+assert len(files) > 70, all(d in files for d in ('board/main.c', 'board/stm32fx/llbxcan.h', 'board/stm32h7/llfdcan.h', 'board/safety/safety_toyota.h'))
+
+for p in patterns:
+  mutations.append((random.choice(files), p, True))
+
+@pytest.mark.parametrize("fn, patch, should_fail", mutations)
+def test_misra_mutation(fn, patch, should_fail):
+  with tempfile.TemporaryDirectory() as tmp:
+    shutil.copytree(ROOT, tmp, dirs_exist_ok=True)
+
+    # apply patch
+    if fn is not None:
+      r = os.system(f"cd {tmp} && sed -i '{patch}' {fn}")
+      assert r == 0
+
+    # run test
+    r = subprocess.run("tests/misra/test_misra.sh", cwd=tmp, shell=True)
+    failed = r.returncode != 0
+    assert failed == should_fail
+
+if __name__ == "__main__":
+  pytest.main([__file__, "-n 8"])