meshtastic_firmware/.github/workflows/test_native.yml

name: Run Tests on Native platform

on:
  workflow_call:
    inputs:
      suite_order_seed:
        description: >-
          Seed for shuffling the test-area order. Empty (the default) means: fixed declared order on
          pull_request, so a contributor's PR never turns red because of an order they did not
          choose; commit-SHA-derived elsewhere. Set a number to force that exact order anywhere -
          that is how you replay a shuffled failure.
        type: string
        required: false
        default: ""
  workflow_dispatch:

permissions: {}

env:
  # Only pushes to the default branch (develop) populate the cache; PR / merge_group runs
  # restore it but never save, so they stop filling up the repo's Actions cache storage.
  SAVE_CACHE: ${{ github.event_name == 'push' && github.ref_name == github.event.repository.default_branch }}
  LCOV_CAPTURE_FLAGS: --quiet --capture --include "${PWD}/src/*" --exclude '*/src/mesh/generated/*' --directory .pio/build/coverage/src --base-directory "${PWD}"

jobs:
  # Tripwire against the native suite set shrinking by accident. `platformio test` discovers and
  # runs whatever test_* directories exist, and bin/run-tests.sh derives its expected count from
  # the same walk - so a suite directory lost in a bad rebase or an overzealous cleanup just means
  # fewer suites run, and every remaining check stays green. Compare the test_* directory list
  # against the PR's merge base and fail when a suite vanished without the PR saying so: a removed
  # suite's name must appear in the PR title, the PR body, or a commit message in the PR's range.
  # A deliberate removal satisfies that by stating what it removes; an accidental loss cannot.
  # Only pull_request runs have a base to compare against (and PRs are where accidents arrive);
  # every other event skips. No job depends on this one: a skipped job would skip its dependents,
  # and the expensive jobs should not wait on a full-history clone.
  suite-shrinkage-check:
    name: Native Suite Shrinkage
    if: github.event_name == 'pull_request'
    runs-on: ubuntu-slim
    permissions:
      contents: read
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          persist-credentials: false
          # Full history: the merge base must be computed, not guessed from a possibly stale
          # event payload, and the acknowledgment scan reads every commit message in the range.
          fetch-depth: 0

      - name: Fail if a test_* suite vanished unacknowledged
        shell: bash
        # PR title/body are attacker-controlled text; they reach the script through env: only,
        # never spliced into the shell source (same rule as the suite-order seed below).
        env:
          BASE_REF: ${{ github.base_ref }}
          PR_TITLE: ${{ github.event.pull_request.title }}
          PR_BODY: ${{ github.event.pull_request.body }}
        run: |
          set -euo pipefail
          git fetch --quiet origin "$BASE_REF"
          base=$(git merge-base FETCH_HEAD HEAD)
          # Same canonical set every other consumer derives: directories named test_* directly
          # under test/, read from the git trees so the comparison is exact at both endpoints.
          list_suites() { git ls-tree -d --name-only "$1" test/ | sed 's#^test/##' | grep '^test_' | sort; }
          removed=$(comm -23 <(list_suites "$base") <(list_suites HEAD))
          if [[ -z $removed ]]; then
            echo "No suite removed: $(list_suites HEAD | wc -l) test_* directories, none lost since merge base ${base:0:8}."
            exit 0
          fi
          messages=$(git log --format=%B "$base..HEAD")
          fail=0
          while IFS= read -r suite; do
            if printf '%s\n%s\n%s\n' "$PR_TITLE" "$PR_BODY" "$messages" | grep -qF "$suite"; then
              echo "Removed suite $suite is named in the PR title/body or a commit message - acknowledged."
            else
              echo "::error title=Native suite vanished::test/$suite exists on the merge base but is gone from this PR, and nothing in the PR title, body, or commit messages mentions it. If the removal is deliberate, name $suite in the PR description or a commit message; if not, restore the directory - platformio test would silently run without it."
              fail=1
            fi
          done <<<"$removed"
          exit $fail

  simulator-tests:
    name: Native Simulator Tests
    runs-on: ubuntu-24.04-arm
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          submodules: recursive

      - name: Setup native build
        id: base
        uses: ./.github/actions/setup-native

      - name: Install simulator dependencies
        run: pip install -U dotmap

      - name: Restore PlatformIO cache
        id: pio-cache
        uses: actions/cache/restore@v6
        with:
          path: ~/.platformio/.cache
          key: pio-simulator-tests-${{ hashFiles('platformio.ini', 'variants/native/portduino.ini', 'variants/native/portduino/platformio.ini') }}
          restore-keys: |
            pio-simulator-tests-

      # We now run integration test before other build steps (to quickly see runtime failures)
      - name: Build for native/coverage
        run: platformio run -e coverage

      - name: Save PlatformIO cache
        if: env.SAVE_CACHE == 'true' && steps.pio-cache.outputs.cache-hit != 'true'
        uses: actions/cache/save@v6
        with:
          path: ~/.platformio/.cache
          key: pio-simulator-tests-${{ hashFiles('platformio.ini', 'variants/native/portduino.ini', 'variants/native/portduino/platformio.ini') }}

      - name: Capture initial coverage information
        shell: bash
        run: |
          sudo apt-get install -y lcov
          lcov ${{ env.LCOV_CAPTURE_FLAGS }} --initial --output-file coverage_base.info
          sed -i -e "s#${PWD}#.#" coverage_base.info # Make paths relative.

      - name: Config check tests
        # Drives the same binary against test/fixtures/portduino-config: asserts that
        # `--check` reports each planted fault, and that a normal run still refuses the
        # configs it should. Runs before the simulator test because it is seconds long
        # and a failure here explains a lot of downstream weirdness.
        timeout-minutes: 5
        run: ./bin/test-config-check.sh .pio/build/coverage/meshtasticd

      - name: Integration test
        # Cap the whole step: if the simulator ever fails to exit (e.g. the
        # exit_simulator admin path regresses again) the job must fail fast,
        # not run to GitHub's 6-hour limit.
        timeout-minutes: 5
        run: |
          .pio/build/coverage/meshtasticd -s &
          PID=$!
          trap 'kill "$PID" 2>/dev/null || true' EXIT
          timeout 20 bash -c "until ls -al /proc/$PID/fd | grep socket; do sleep 1; done"
          echo "Simulator started, launching python test..."
          python3 -c 'from meshtastic.test import testSimulator; testSimulator()'
          # The Python harness sends exit_simulator and exits; the simulator is
          # expected to terminate on its own. Give it a moment, then verify.
          # If it is still alive the exit handshake is broken - fail loudly and
          # do NOT fall through to `wait`, which would otherwise block until the
          # job's hard timeout.
          for i in $(seq 1 10); do
            kill -0 "$PID" 2>/dev/null || break
            sleep 1
          done
          if kill -0 "$PID" 2>/dev/null; then
            echo "::error title=Simulator did not exit::meshtasticd ignored exit_simulator and is still running after the integration test. The exit_simulator admin path is broken (see AdminModule::handleReceivedProtobuf, ARCH_PORTDUINO bypass). Killing it to avoid a 6-hour CI overrun."
            kill -9 "$PID" 2>/dev/null || true
            wait "$PID" 2>/dev/null || true
            exit 1
          fi
          wait "$PID" 2>/dev/null || true

      - name: Capture coverage information
        if: always() # run this step even if previous step failed
        run: |
          lcov ${{ env.LCOV_CAPTURE_FLAGS }} --test-name integration --output-file coverage_integration.info
          sed -i -e "s#${PWD}#.#" coverage_integration.info # Make paths relative.

      - name: Get release version string
        if: always() # run this step even if previous step failed
        run: echo "long=$(./bin/buildinfo.py long)" >> $GITHUB_OUTPUT
        id: version

      - name: Save coverage information
        uses: actions/upload-artifact@v7
        if: always() # run this step even if previous step failed
        with:
          name: lcov-coverage-info-native-simulator-test-${{ steps.version.outputs.long }}
          overwrite: true
          path: ./coverage_*.info

  platformio-tests:
    name: Native PlatformIO Tests
    runs-on: ubuntu-24.04-arm
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          submodules: recursive

      - name: Setup native build
        id: base
        uses: ./.github/actions/setup-native

      - name: Get release version string
        run: echo "long=$(./bin/buildinfo.py long)" >> $GITHUB_OUTPUT
        id: version

      # Disable (comment-out) BUILD_EPOCH. It causes a full rebuild between tests and resets the
      # coverage information each time.
      - name: Disable BUILD_EPOCH
        run: sed -i 's/-DBUILD_EPOCH=$UNIX_TIME/#-DBUILD_EPOCH=$UNIX_TIME/' platformio.ini

      - name: Restore PlatformIO cache
        id: pio-cache
        uses: actions/cache/restore@v6
        with:
          path: ~/.platformio/.cache
          key: pio-coverage-tests-${{ hashFiles('platformio.ini', 'variants/native/portduino.ini', 'variants/native/portduino/platformio.ini') }}
          restore-keys: |
            pio-coverage-tests-

      - name: Build test programs once
        # One shared build of src + every test program. This is the single source build; gcov then
        # accumulates coverage counts into this shared .pio/build/coverage/src as the chunks run.
        run: platformio test -e coverage --without-testing

      - name: Save PlatformIO cache
        if: env.SAVE_CACHE == 'true' && steps.pio-cache.outputs.cache-hit != 'true'
        uses: actions/cache/save@v6
        with:
          path: ~/.platformio/.cache
          key: pio-coverage-tests-${{ hashFiles('platformio.ini', 'variants/native/portduino.ini', 'variants/native/portduino/platformio.ini') }}

      - name: Run tests one area at a time
        shell: bash
        # Both values reach the script through env: rather than ${{ }} inside run:, so nothing from
        # the event payload is ever spliced into the shell text.
        env:
          SUITE_ORDER_SEED: ${{ inputs.suite_order_seed }}
          EVENT_NAME: ${{ github.event_name }}
        run: |
          set -uo pipefail
          # One runner, no matrix, no concurrency. Group the test_* suites by area and run each
          # area sequentially, reusing the single build above (--without-building). Each area gets
          # its own JUnit report and its own collapsible log, so a failure lands in a small named
          # section instead of being buried past the log limit. Sequential runs share one build
          # dir, so gcov coverage accumulates and the single capture in the next step has the union.

          # Ordered area rules "name:ERE"; first match wins. Anything unmatched falls to "misc", so
          # a newly added suite always runs even before it is placed. Add a suite to an area by
          # extending that area's regex; add a new area by inserting a rule line.
          area_rules=(
            "admin:^test_(admin|pki)_"
            "crypto:^test_(crypto|packet_signing)$"
            "routing:^test_(mesh|nexthop|traceroute|hop|traffic|nodedb|warm)_"
            "position:^test_position_"
            "fuzz:^test_fuzz_"
            "packets:^test_(packet|transmit|meshpacket)_"
            "io:^test_(serial|stream|xmodem|http|mqtt)"
          )

          mapfile -t suites < <(find test -maxdepth 1 -type d -name 'test_*' -printf '%f\n' | sort)
          declare -A group
          for s in "${suites[@]}"; do
            a="misc"
            for rule in "${area_rules[@]}"; do
              if [[ "$s" =~ ${rule#*:} ]]; then a="${rule%%:*}"; break; fi
            done
            group[$a]="${group[$a]:-} -f $s"
          done

          run_order=()
          for rule in "${area_rules[@]}"; do run_order+=("${rule%%:*}"); done
          run_order+=("misc")

          # Area order. The rule order above is an accident of how the areas were written, and
          # running it fixed forever means order dependence between areas is never observed - but
          # randomising it on a contributor's PR would turn their run red for an order they did not
          # choose, which is how a randomisation gets reverted instead of the coupling fixed.
          #
          # So: pull_request keeps the fixed declared order. Everywhere else (push, schedule) the
          # order is shuffled, seeded from the commit SHA - deterministic per commit, replayable,
          # attributable, and it never blocks someone else's PR. An explicit seed input overrides
          # both, which is how you replay a specific failing order anywhere.
          #
          # Intra-area order stays PlatformIO's: filters select suites, they do not order them
          # (list_test_names() walks test/ with os.walk()), so controlling it needs one invocation
          # per suite. bin/run-tests.sh --shuffle does exactly that locally.
          seed_input="${SUITE_ORDER_SEED:-}"
          if [ -n "$seed_input" ]; then
            seed="$seed_input"
            echo "area order: shuffled with explicitly supplied seed $seed"
          elif [ "${EVENT_NAME:-}" = "pull_request" ]; then
            seed=""
            echo "area order: fixed declared order (pull_request) - ${run_order[*]}"
            echo "  to exercise a different order, re-run this workflow with a suite_order_seed input"
          else
            seed=$((16#${GITHUB_SHA:0:8}))
            echo "area order: shuffled with seed $seed (from ${GITHUB_SHA:0:8})"
          fi

          if [ -n "$seed" ]; then
            # Same shuffle_suites() bin/run-tests.sh uses, so the replay hint below is true by
            # construction rather than by two copies happening to agree.
            source bin/lib/shuffle.sh
            mapfile -t run_order < <(shuffle_suites "$seed" "${run_order[@]}")
            echo "area order: ${run_order[*]}"
            echo "  replay locally: ./bin/run-tests.sh --shuffle --seed $seed"
          fi

          fail=0
          for a in "${run_order[@]}"; do
            [ -n "${group[$a]:-}" ] || continue
            echo "::group::area $a (${group[$a]# })"
            # Capture platformio's real exit status (not grep's) via a log file, then show the log
            # with the noisy per-variant SKIPPED rows filtered out.
            if ! platformio test -e coverage --without-building -v ${group[$a]# } \
                 --junit-output-path "testreport-$a.xml" > "area-$a.log" 2>&1; then
              fail=1
              echo "::error::area $a had test failures"
            fi
            grep -v "[[:space:]]SKIPPED$" "area-$a.log" || true
            echo "::endgroup::"
          done
          exit $fail

      - name: Merge per-area reports into testreport.xml
        # Preserve the single-file JUnit contract that downstream consumers rely on
        # (pr_tests.yml's summary and generate-reports' Test Report both read testreport.xml).
        # The per-area split is only for readable logs; the report stays consolidated.
        if: always() # run even when a chunk failed, so the report captures the failures
        shell: bash
        run: |
          python3 - <<'PY'
          import glob, xml.etree.ElementTree as ET
          out = ET.Element('testsuites')
          for f in sorted(glob.glob('testreport-*.xml')):
              try:
                  root = ET.parse(f).getroot()
              except ET.ParseError:
                  continue
              # PlatformIO writes a <testsuites> root; fold in a bare <testsuite> too, just in case.
              out.extend(root.findall('testsuite') if root.tag == 'testsuites' else [root])
          ET.ElementTree(out).write('testreport.xml', encoding='utf-8', xml_declaration=True)
          PY

      - name: Capture coverage information
        if: always() # run this step even if previous step failed
        run: |
          sudo apt-get install -y lcov
          lcov ${{ env.LCOV_CAPTURE_FLAGS }} --test-name tests --output-file coverage_tests.info
          sed -i -e "s#${PWD}#.#" coverage_tests.info # Make paths relative.

      - name: Event channel policy tests
        run: platformio test -e coverage-event-policy -v --junit-output-path event-policy-testreport.xml

      - name: Save test results
        if: always() # run this step even if previous step failed
        uses: actions/upload-artifact@v7
        with:
          name: platformio-test-report-${{ steps.version.outputs.long }}
          overwrite: true
          path: ./*testreport.xml

      - name: Save coverage information
        uses: actions/upload-artifact@v7
        if: always() # run this step even if previous step failed
        with:
          name: lcov-coverage-info-native-platformio-tests-${{ steps.version.outputs.long }}
          overwrite: true
          path: ./coverage_*.info

  generate-reports:
    name: Generate Test Reports
    runs-on: ubuntu-latest
    permissions: # Needed for dorny/test-reporter.
      contents: read
      actions: read
      checks: write
    needs:
      - simulator-tests
      - platformio-tests
    # Run this job even if the previous jobs failed, but skip if the workflow was cancelled.
    if: ${{ !cancelled() }}
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7

      - name: Get release version string
        run: echo "long=$(./bin/buildinfo.py long)" >> $GITHUB_OUTPUT
        id: version

      - name: Download test artifacts
        uses: actions/download-artifact@v8
        with:
          name: platformio-test-report-${{ steps.version.outputs.long }}
          merge-multiple: true

      - name: Drop no-status testsuites from the report
        # PlatformIO emits a self-closing <testsuite tests="0"/> row for every test_* dir
        # crossed with every hardware variant it cannot run on the native host (~4900 rows).
        # They carry no pass/fail/skip status and bury the suites that actually ran. Strip
        # them so the Test Report lists only suites with a real status. Only the copy the
        # reporter renders is trimmed; the uploaded artifact keeps the full XML.
        run: sed -i -E 's#<testsuite [^>]*tests="0"[^>]*/>##g' testreport.xml

      - name: Test Report
        uses: dorny/test-reporter@v3.0.0
        with:
          name: PlatformIO Tests
          path: testreport.xml
          reporter: java-junit

      - name: Download coverage artifacts
        uses: actions/download-artifact@v8
        with:
          pattern: lcov-coverage-info-native-*-${{ steps.version.outputs.long }}
          path: code-coverage-report
          merge-multiple: true

      - name: Generate Code Coverage Report
        # Merge every tracefile the jobs produced: coverage_base.info (zeroed baseline),
        # coverage_integration.info, and one coverage_tests_<chunk>.info per chunk. lcov
        # sums hit counts across them, so the merged report is the union of all chunks -
        # identical to running the whole suite in one job.
        run: |
          sudo apt-get install -y lcov
          args=()
          for f in code-coverage-report/coverage_*.info; do
            args+=(--add-tracefile "$f")
          done
          lcov --quiet "${args[@]}" --output-file code-coverage-report/coverage_src.info
          genhtml --quiet --legend --prefix "${PWD}" code-coverage-report/coverage_src.info --output-directory code-coverage-report

      - name: Save Code Coverage Report
        uses: actions/upload-artifact@v7
        with:
          name: code-coverage-report-${{ steps.version.outputs.long }}
          path: code-coverage-report