Merge pull request #11130 from meshtastic/ci-native-tests-by-area

Run native PlatformIO tests by area for readable failures

.github/workflows/test_native.yml

@@ -105,14 +105,90 @@ jobs:
       - name: Disable BUILD_EPOCH
         run: sed -i 's/-DBUILD_EPOCH=$UNIX_TIME/#-DBUILD_EPOCH=$UNIX_TIME/' platformio.ini
 
-      - name: PlatformIO Tests
+      - name: Build test programs once
+        # One shared build of src + every test program. This is the single source build; gcov then
+        # accumulates coverage counts into this shared .pio/build/coverage/src as the chunks run.
+        run: platformio test -e coverage --without-testing
+
+      - name: Run tests one area at a time
+        shell: bash
         run: |
-          set -o pipefail
+          set -uo pipefail
-          # Filter out SKIPPED summary rows for hardware variants that can't run on the
+          # One runner, no matrix, no concurrency. Group the test_* suites by area and run each
-          # native host. They flood the log and make it harder to spot real failures.
+          # area sequentially, reusing the single build above (--without-building). Each area gets
-          # The JUnit XML is written directly to testreport.xml before the pipe, so
+          # its own JUnit report and its own collapsible log, so a failure lands in a small named
-          # the test artifact is unaffected.
+          # section instead of being buried past the log limit. Sequential runs share one build
-          platformio test -e coverage -v --junit-output-path testreport.xml 2>&1 | grep -v "[[:space:]]SKIPPED$"
+          # dir, so gcov coverage accumulates and the single capture in the next step has the union.
+
+          # Ordered area rules "name:ERE"; first match wins. Anything unmatched falls to "misc", so
+          # a newly added suite always runs even before it is placed. Add a suite to an area by
+          # extending that area's regex; add a new area by inserting a rule line.
+          area_rules=(
+            "admin:^test_(admin|pki)_"
+            "crypto:^test_(crypto|packet_signing)$"
+            "routing:^test_(mesh|nexthop|traceroute|hop|traffic|nodedb|warm)_"
+            "position:^test_position_"
+            "fuzz:^test_fuzz_"
+            "packets:^test_(packet|transmit|meshpacket)_"
+            "io:^test_(serial|stream|xmodem|http|mqtt)"
+          )
+
+          mapfile -t suites < <(find test -maxdepth 1 -type d -name 'test_*' -printf '%f\n' | sort)
+          declare -A group
+          for s in "${suites[@]}"; do
+            a="misc"
+            for rule in "${area_rules[@]}"; do
+              if [[ "$s" =~ ${rule#*:} ]]; then a="${rule%%:*}"; break; fi
+            done
+            group[$a]="${group[$a]:-} -f $s"
+          done
+
+          run_order=()
+          for rule in "${area_rules[@]}"; do run_order+=("${rule%%:*}"); done
+          run_order+=("misc")
+
+          fail=0
+          for a in "${run_order[@]}"; do
+            [ -n "${group[$a]:-}" ] || continue
+            echo "::group::area $a (${group[$a]# })"
+            # Capture platformio's real exit status (not grep's) via a log file, then show the log
+            # with the noisy per-variant SKIPPED rows filtered out.
+            if ! platformio test -e coverage --without-building -v ${group[$a]# } \
+                 --junit-output-path "testreport-$a.xml" > "area-$a.log" 2>&1; then
+              fail=1
+              echo "::error::area $a had test failures"
+            fi
+            grep -v "[[:space:]]SKIPPED$" "area-$a.log" || true
+            echo "::endgroup::"
+          done
+          exit $fail
+
+      - name: Merge per-area reports into testreport.xml
+        # Preserve the single-file JUnit contract that downstream consumers rely on
+        # (pr_tests.yml's summary and generate-reports' Test Report both read testreport.xml).
+        # The per-area split is only for readable logs; the report stays consolidated.
+        if: always() # run even when a chunk failed, so the report captures the failures
+        shell: bash
+        run: |
+          python3 - <<'PY'
+          import glob, xml.etree.ElementTree as ET
+          out = ET.Element('testsuites')
+          for f in sorted(glob.glob('testreport-*.xml')):
+              try:
+                  root = ET.parse(f).getroot()
+              except ET.ParseError:
+                  continue
+              # PlatformIO writes a <testsuites> root; fold in a bare <testsuite> too, just in case.
+              out.extend(root.findall('testsuite') if root.tag == 'testsuites' else [root])
+          ET.ElementTree(out).write('testreport.xml', encoding='utf-8', xml_declaration=True)
+          PY
+
+      - name: Capture coverage information
+        if: always() # run this step even if previous step failed
+        run: |
+          sudo apt-get install -y lcov
+          lcov ${{ env.LCOV_CAPTURE_FLAGS }} --test-name tests --output-file coverage_tests.info
+          sed -i -e "s#${PWD}#.#" coverage_tests.info # Make paths relative.
 
       - name: Save test results
         if: always() # run this step even if previous step failed
@@ -122,13 +198,6 @@ jobs:
           overwrite: true
           path: ./testreport.xml
 
-      - name: Capture coverage information
-        if: always() # run this step even if previous step failed
-        run: |
-          sudo apt-get install -y lcov
-          lcov ${{ env.LCOV_CAPTURE_FLAGS }} --test-name tests --output-file coverage_tests.info
-          sed -i -e "s#${PWD}#.#" coverage_tests.info # Make paths relative.
-
       - name: Save coverage information
         uses: actions/upload-artifact@v7
         if: always() # run this step even if previous step failed
@@ -184,9 +253,17 @@ jobs:
           merge-multiple: true
 
       - name: Generate Code Coverage Report
+        # Merge every tracefile the jobs produced: coverage_base.info (zeroed baseline),
+        # coverage_integration.info, and one coverage_tests_<chunk>.info per chunk. lcov
+        # sums hit counts across them, so the merged report is the union of all chunks -
+        # identical to running the whole suite in one job.
         run: |
           sudo apt-get install -y lcov
-          lcov --quiet --add-tracefile code-coverage-report/coverage_base.info --add-tracefile code-coverage-report/coverage_integration.info --add-tracefile code-coverage-report/coverage_tests.info --output-file code-coverage-report/coverage_src.info
+          args=()
+          for f in code-coverage-report/coverage_*.info; do
+            args+=(--add-tracefile "$f")
+          done
+          lcov --quiet "${args[@]}" --output-file code-coverage-report/coverage_src.info
           genhtml --quiet --legend --prefix "${PWD}" code-coverage-report/coverage_src.info --output-directory code-coverage-report
 
       - name: Save Code Coverage Report