diff --git a/.github/workflows/test_native.yml b/.github/workflows/test_native.yml index a0af13ff3..9ff192a16 100644 --- a/.github/workflows/test_native.yml +++ b/.github/workflows/test_native.yml @@ -161,6 +161,14 @@ jobs: name: platformio-test-report-${{ steps.version.outputs.long }} merge-multiple: true + - name: Drop no-status testsuites from the report + # PlatformIO emits a self-closing row for every test_* dir + # crossed with every hardware variant it cannot run on the native host (~4900 rows). + # They carry no pass/fail/skip status and bury the suites that actually ran. Strip + # them so the Test Report lists only suites with a real status. Only the copy the + # reporter renders is trimmed; the uploaded artifact keeps the full XML. + run: sed -i -E 's#]*tests="0"[^>]*/>##g' testreport.xml + - name: Test Report uses: dorny/test-reporter@v3.0.0 with: diff --git a/bin/run-tests.sh b/bin/run-tests.sh new file mode 100755 index 000000000..5965999cd --- /dev/null +++ b/bin/run-tests.sh @@ -0,0 +1,249 @@ +#!/usr/bin/env bash +# Run native PlatformIO unit tests and emit a single, unambiguous RED/AMBER/GREEN verdict. +# +# Why this exists: PlatformIO reports failures three different ways ([FAILED], :FAIL:, +# [ERRORED]) and an all-pass run prints "N succeeded" with NO "0 failed" clause — so naive +# greps produce false greens (see .notes/test-passfail-filter.md). This script encodes the +# correct logic once, and cross-checks the number of suites that actually ran against the +# canonical set in test/ so a suite silently going missing shows up as AMBER, not green. +# +# Usage: +# ./bin/run-tests.sh # run all suites, full RAG + count cross-check +# ./bin/run-tests.sh -f test_utf8 # run one suite (no count cross-check) +# ./bin/run-tests.sh -e native # override env (default: coverage) +# ./bin/run-tests.sh --quiet # only print the final RESULT line +# +# Exit codes: 0 = GREEN, 1 = RED, 2 = AMBER. +# +# The final line is machine-readable, e.g.: +# RESULT: GREEN 19/19 suites passed +# RESULT: AMBER 17/19 suites ran (missing: test_radio test_serial) — all that ran passed +# RESULT: RED test_traffic_management: 1 failed (or: build/crash error) +# RESULT: RED sanitizer fault — SUMMARY: AddressSanitizer: 1272 byte(s) leaked (tests may have +# all passed; the coverage build aborts at exit on an ASan/LSan fault — often shown only +# as [ERRORED]/SIGHUP. The script names it and points at running the binary bare.) + +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +ROOT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +cd "$ROOT_DIR" + +ENV="coverage" +FILTER="" +QUIET=false +PASSTHRU=() + +while [[ $# -gt 0 ]]; do + case "$1" in + -f) + FILTER="$2" + PASSTHRU+=("-f" "$2") + shift 2 + ;; + -e) + ENV="$2" + shift 2 + ;; + --quiet) + QUIET=true + shift + ;; + *) + PASSTHRU+=("$1") + shift + ;; + esac +done + +# Locate pio (PATH, then the standard PlatformIO venv). +PIO="$(command -v pio || command -v platformio || echo "$HOME/.platformio/penv/bin/pio")" +if [[ ! -x $PIO ]] && ! command -v "$PIO" >/dev/null 2>&1; then + echo "RESULT: RED pio not found (looked in PATH and ~/.platformio/penv/bin)" + exit 1 +fi + +LOG="$(mktemp -t meshtest.XXXXXX.log)" +MARKER="" +PROGRESS_PID="" +trap 'rm -f "$LOG" "${MARKER:-}"; [[ -n ${PROGRESS_PID:-} ]] && kill "$PROGRESS_PID" 2>/dev/null' EXIT + +# Canonical suite set = the directories in test/. This is the source of truth for +# "what should run"; a filtered run only expects its filtered suite. +mapfile -t ALL_SUITES < <(find test -maxdepth 1 -type d -name 'test_*' -printf '%f\n' | sort) +EXPECTED_COUNT=${#ALL_SUITES[@]} + +# Cached object-count for this env, written after each completed build (in the gitignored build +# dir). Used as the progress denominator: accurate for a full rebuild (every object recompiles), +# only a rough upper bound for an incremental run. +BASELINE_FILE=".pio/build/${ENV}/.runtests-objcount" + +# Progress trail file (gitignored build dir). ALWAYS written so a backgrounded/piped run can be +# checked mid-build with `tail -f` — that's the whole point: don't fly blind on a 20-min rebuild. +PROGRESS_FILE=".pio/build/${ENV}/.runtests-progress" + +# --- Progress heartbeat ------------------------------------------------------ +# Emit ONE status line every few seconds: build = objects (re)compiled this run / cached total + +# best-effort ETA; test = suites finished / expected. Appends to $PROGRESS_FILE always (tail it to +# check on a backgrounded run); also live-updates the tty when $5=1 (interactive --quiet). Never +# touches $LOG, which is parsed for the verdict, so piped/CI captures stay clean. +progress_monitor() { + local marker="$1" objtotal="$2" testtotal="$3" pfile="$4" totty="$5" start now el done ran eta line + start=$(date +%s) + while :; do + now=$(date +%s) + el=$((now - start)) + if grep -q 'Testing\.\.\.' "$LOG" 2>/dev/null; then + ran=$(grep -cE "${ENV}:test_[a-z0-9_]+ \[(PASSED|FAILED|ERRORED)\]" "$LOG" 2>/dev/null) + line=$(printf '[test] %s/%s suites done — %dm%02ds' "$ran" "$testtotal" $((el / 60)) $((el % 60))) + else + done=$(find ".pio/build/${ENV}" -name '*.o' -newer "$marker" 2>/dev/null | wc -l) + if ((objtotal > 0 && done > 0)); then + eta=$((objtotal > done ? (objtotal - done) * el / done : 0)) + line=$(printf '[build] %d/%d objs — %dm%02ds — ETA ~%dm%02ds' \ + "$done" "$objtotal" $((el / 60)) $((el % 60)) $((eta / 60)) $((eta % 60))) + else + # done==0 (incremental: nothing to rebuild yet) or no cached baseline — no ETA yet. + line=$(printf '[build] %d objs compiled — %dm%02ds' "$done" $((el / 60)) $((el % 60))) + fi + fi + printf '%s\n' "$line" >>"$pfile" 2>/dev/null # file trail (always) + [[ $totty == 1 ]] && printf '\r\033[K%s' "$line" >/dev/tty 2>/dev/null # live line (human) + sleep 4 + done +} + +# Launch the heartbeat for every run. It writes the progress file unconditionally; the live tty +# line only when interactive AND --quiet (where pio's own output is hidden — otherwise pio's +# streamed compile lines already show progress and a \r line would just fight them). +mkdir -p ".pio/build/${ENV}" 2>/dev/null || true +: >"$PROGRESS_FILE" 2>/dev/null || true +MARKER="$(mktemp -t meshtest-mark.XXXXXX)" +TOTTY=0 +{ $QUIET && [[ -t 1 ]]; } && TOTTY=1 +progress_monitor "$MARKER" "$(cat "$BASELINE_FILE" 2>/dev/null || echo 0)" \ + "$([[ -n $FILTER ]] && echo 1 || echo "$EXPECTED_COUNT")" "$PROGRESS_FILE" "$TOTTY" & +PROGRESS_PID=$! + +if ! $QUIET; then + echo "Running: $PIO test -e $ENV ${PASSTHRU[*]-} (expecting $EXPECTED_COUNT suites)" +fi +echo "progress: tail -f $PROGRESS_FILE" >&2 + +# Run pio, tee to log. PIPESTATUS[0] is pio's real exit (NOT tee's). +if $QUIET; then + "$PIO" test -e "$ENV" "${PASSTHRU[@]}" >"$LOG" 2>&1 +else + "$PIO" test -e "$ENV" "${PASSTHRU[@]}" 2>&1 | tee "$LOG" +fi +PIO_RC=${PIPESTATUS[0]} + +# Stop the heartbeat, clear its line, and cache this build's object total for next time. +if [[ -n $PROGRESS_PID ]]; then + kill "$PROGRESS_PID" 2>/dev/null + wait "$PROGRESS_PID" 2>/dev/null + PROGRESS_PID="" + # Clear the live line only if we were writing one — opening /dev/tty when there is none is + # itself a redirect-open error the trailing 2>/dev/null cannot suppress. + [[ $TOTTY == 1 ]] && printf '\r\033[K' >/dev/tty 2>/dev/null +fi +[[ -d ".pio/build/${ENV}" ]] && find ".pio/build/${ENV}" -name '*.o' 2>/dev/null | wc -l >"$BASELINE_FILE" 2>/dev/null || true + +# --- Outcome detection ------------------------------------------------------- +# The SAME outcome is spelled differently depending on which layer emitted the line — this is +# the trap that produces false greens (grepping ":PASS" misses pio's "[PASSED]", grepping +# "[FAILED]" misses Unity's ":FAIL:"). So every regex below matches BOTH spellings: +# pass: Unity per-assertion ":PASS" | pio per-suite "[PASSED]" | summary "N succeeded" +# fail: Unity per-assertion ":FAIL:" | pio per-suite "[FAILED]" | summary "M failed" +# error: pio build/crash "[ERRORED]" | Unity "M Failures" | compiler "error:" +# Match \b after :PASS/:FAIL so ":PASSED"/":FAILED" forms are also caught either way. +FAIL_RE=':FAIL\b|\[FAILED\]|\[ERRORED\]|[1-9][0-9]* failed|[0-9]+ Tests [1-9][0-9]* Failures|error:|undefined reference|Segmentation fault|terminate called|SIGHUP|SIGSEGV|SIGABRT' +# Positive proof tests actually ran & passed (absence != success). Accept any pass spelling: +# the per-test/per-suite tokens OR a success summary line. +PASS_RE=':PASS\b|\[PASSED\]|test cases: *[0-9]+ succeeded|[0-9]+ Tests 0 Failures' +# Sanitizer (ASan/LSan/UBSan/TSan) fault signatures. The coverage build is sanitizer-instrumented +# and aborts NON-ZERO at exit on a fault — most often a LeakSanitizer leak — AFTER every test has +# already printed [PASSED]. pio then reports [ERRORED]/SIGHUP with no :FAIL: anywhere, so it +# masquerades as a phantom "N-1 of N succeeded". See .notes/test-passfail-filter.md. +# Match only real FAULT lines, never the benign "AddressSanitizer: failed to intercept '...'" +# startup noise that prints on every sanitizer run (it'd mislabel a normal [FAILED] as a leak). +# Formats per LLVM/Google sanitizer docs: ASan/LSan emit "==PID==ERROR: : ...", UBSan emits +# "file:line:col: runtime error: ...", TSan emits "WARNING: ThreadSanitizer: ..."; all close with +# a "SUMMARY: : ..." line (LSan-under-ASan reports its SUMMARY as "AddressSanitizer"). +SAN_RE='(ERROR|WARNING): (Address|Leak|Thread|UndefinedBehavior)Sanitizer:|SUMMARY: (Address|Leak|Thread|UndefinedBehavior)Sanitizer:|Direct leak of|Indirect leak of|detected memory leaks|heap-use-after-free|heap-buffer-overflow|stack-buffer-overflow|attempting double-free|LeakSanitizer has encountered a fatal error|runtime error:' + +# Suites that produced a per-suite verdict. pio emits "coverage:test_x [PASSED|FAILED|ERRORED]"; +# a SKIPPED suite (hardware-only on native) is "accounted for" too, so it doesn't read as missing. +mapfile -t RAN_SUITES < <(grep -oE "${ENV}:test_[a-z0-9_]+ \[(PASSED|FAILED|ERRORED)\]" "$LOG" | + sed -E "s/^${ENV}:(test_[a-z0-9_]+) .*/\1/" | sort -u) +RAN_COUNT=${#RAN_SUITES[@]} +# Suites pio explicitly skipped (don't count these as "missing" in the canonical cross-check). +mapfile -t SKIPPED_SUITES < <(grep -oE "${ENV}:test_[a-z0-9_]+.*\bSKIPPED\b" "$LOG" | + grep -oE "test_[a-z0-9_]+" | sort -u) + +verdict_red() { + local detail bin + detail="$(grep -nE '\[FAILED\]|:FAIL:|\[ERRORED\]' "$LOG" | head -3 | sed 's/^/ /')" + echo "" + echo "RED — failures detected:" + [[ -n $detail ]] && echo "$detail" + grep -E 'test cases:' "$LOG" | tail -1 | sed 's/^/ /' + + # Path to the test binary for the "run it bare" hint. For native/coverage the test program is + # the env executable (e.g. .pio/build/coverage/meshtasticd), NOT a file named 'program'. + bin="$(find ".pio/build/${ENV}" -maxdepth 1 -type f -executable ! -name '*.so' 2>/dev/null | head -1)" + [[ -z $bin ]] && bin=".pio/build/${ENV}/ (build it first: $PIO test -e ${ENV} ${FILTER:+-f $FILTER} --without-testing)" + + # Sanitizer fault (ASan/LSan/UBSan/TSan): name the real cause instead of "build/crash error". + if grep -qE "$SAN_RE" "$LOG"; then + grep -nE "$SAN_RE" "$LOG" | head -4 | sed 's/^/ /' + echo " -> sanitizer fault: if every test above is PASS, this is an exit-time abort, not a failed assertion." + echo " -> read the full report by running the binary BARE (gdb hides it via ptrace): ./$bin 2>&1 | tail -40" + echo "RESULT: RED sanitizer fault — $(grep -ohE 'SUMMARY: [A-Za-z]+Sanitizer:.*' "$LOG" | tail -1 || echo 'see report above')" + exit 1 + fi + + # All tests passed but the process still aborted at EXIT (ERRORED/SIGHUP/SIGABRT) and the + # sanitizer report was swallowed by the runner (often surfaced only as SIGHUP). Almost always a + # sanitizer fault — point at how to surface it rather than calling it a generic crash. + if grep -qE "$PASS_RE" "$LOG" && grep -qE '\[ERRORED\]|SIGHUP|SIGABRT' "$LOG" && ! grep -qE ':FAIL\b|\[FAILED\]' "$LOG"; then + echo " -> all tests passed but the process aborted at EXIT — likely an ASan/LSan fault whose report" + echo " the runner swallowed (commonly shown as SIGHUP). Run the binary BARE to see it: ./$bin 2>&1 | tail -40" + echo "RESULT: RED exit-time abort (tests passed; likely sanitizer — see hint above)" + exit 1 + fi + + echo "RESULT: RED $(grep -oE '[0-9]+ failed' "$LOG" | tail -1 || echo 'build/crash error')" + exit 1 +} + +# RED: pio non-zero, any failure marker, or no positive summary at all (build died early). +if [[ $PIO_RC -ne 0 ]] || grep -qE "$FAIL_RE" "$LOG"; then + verdict_red +fi +if ! grep -qE "$PASS_RE" "$LOG"; then + echo "" + echo "RESULT: RED no success summary found (build error / no tests ran?) — see log" + exit 1 +fi + +# AMBER: everything that ran passed, but (full run only) a canonical suite neither ran NOR was +# explicitly skipped — i.e. it silently went missing. SKIPPED suites are accounted for. +ACCOUNTED_COUNT=$((RAN_COUNT + ${#SKIPPED_SUITES[@]})) +if [[ -z $FILTER && $ACCOUNTED_COUNT -lt $EXPECTED_COUNT ]]; then + missing=() + for s in "${ALL_SUITES[@]}"; do + printf '%s\n' "${RAN_SUITES[@]}" "${SKIPPED_SUITES[@]}" | grep -qx "$s" || missing+=("$s") + done + echo "" + echo "RESULT: AMBER ${RAN_COUNT}/${EXPECTED_COUNT} suites ran (missing: ${missing[*]}) — all that ran passed" + exit 2 +fi + +# GREEN. +if [[ -n $FILTER ]]; then + echo "RESULT: GREEN ${RAN_COUNT} suite(s) passed (filtered: $FILTER)" +else + echo "RESULT: GREEN ${RAN_COUNT}/${EXPECTED_COUNT} suites passed" +fi +exit 0